diff --git a/Cargo.lock b/Cargo.lock index 9a69b24d1a..53a0f0e82a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11943,8 +11943,6 @@ dependencies = [ "human-repr", "iggy_binary_protocol", "iggy_common", - "lending-iterator", - "moka", "nix", "opentelemetry", "opentelemetry-appender-tracing", @@ -11961,7 +11959,6 @@ dependencies = [ "smallvec", "tempfile", "thiserror 2.0.19", - "tokio", "tracing", "tracing-appender", "tracing-opentelemetry", diff --git a/Cargo.toml b/Cargo.toml index 7820ec9cbc..18a95963f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -227,7 +227,6 @@ metadata = { path = "core/metadata" } mimalloc = "0.1" mime_guess = "2.0" mockall = "0.15.0" -moka = { version = "0.12.15", features = ["future"] } mongodb = { version = "3.8.0", features = ["rustls-tls"] } nix = { version = "0.31.3", features = ["feature", "fs", "resource", "sched"] } nonzero_lit = "0.1.2" diff --git a/bdd/cpp/features/step_definitions/messaging_steps.cpp b/bdd/cpp/features/step_definitions/messaging_steps.cpp index df4afc7db6..6be384e28d 100644 --- a/bdd/cpp/features/step_definitions/messaging_steps.cpp +++ b/bdd/cpp/features/step_definitions/messaging_steps.cpp @@ -78,9 +78,9 @@ WHEN("^I create a topic with name \"([^\"]{1,255})\" in stream ([0-9]+) with ([0 context->client->create_topic(bdd::make_numeric_identifier(static_cast(stream_id)), topic_name, static_cast(partitions_count), - std::string(compression.compression_algorithm_value()), 0, + std::string(compression.compression_algorithm_value()), std::string(message_expiry.expiry_kind()), message_expiry.expiry_value(), - std::string(max_topic_size.max_topic_size())); + std::string(max_topic_size.max_topic_size()), {}); } THEN("^the topic should be created successfully$") { @@ -117,9 +117,9 @@ WHEN("^I send ([0-9]+) messages to stream ([0-9]+), topic ([0-9]+), partition ([ rust::Vec messages; for (int index = 0; index < message_count; ++index) { - iggy::ffi::IggyMessageToSend message = iggy::ffi::make_message( - bdd::to_payload(bdd::expected_payload(static_cast(index))), - rust::Vec()); + iggy::ffi::IggyMessageToSend message = + iggy::ffi::make_message(bdd::to_payload(bdd::expected_payload(static_cast(index))), + rust::Vec()); // Assign an explicit, 1-based id so the last-sent/last-polled comparison is meaningful. message.id_lo = static_cast(index + 1); messages.push_back(std::move(message)); diff --git a/bdd/go/tests/basic_messaging.go b/bdd/go/tests/basic_messaging.go index 960543c9fd..e5d7089f68 100644 --- a/bdd/go/tests/basic_messaging.go +++ b/bdd/go/tests/basic_messaging.go @@ -269,7 +269,6 @@ func (s basicMessagingSteps) whenCreateTopic(ctx context.Context, iggcon.CompressionAlgorithmNone, iggcon.IggyExpiryNeverExpire, 0, - nil, ) if err != nil { return fmt.Errorf("failed to create topic: %w", err) diff --git a/bdd/go/tests/tcp_test/stream_feature_get_by_id.go b/bdd/go/tests/tcp_test/stream_feature_get_by_id.go index 41e7e62e64..71d4cc497b 100644 --- a/bdd/go/tests/tcp_test/stream_feature_get_by_id.go +++ b/bdd/go/tests/tcp_test/stream_feature_get_by_id.go @@ -63,8 +63,7 @@ var _ = ginkgo.Describe("GET STREAM BY ID:", func() { 2, iggcon.CompressionAlgorithmNone, iggcon.Millisecond, - math.MaxUint64, - nil) + math.MaxUint64) itShouldNotReturnError(err) t2, err := client.CreateTopic( context.Background(), @@ -73,8 +72,7 @@ var _ = ginkgo.Describe("GET STREAM BY ID:", func() { 2, iggcon.CompressionAlgorithmNone, iggcon.Millisecond, - math.MaxUint64, - nil) + math.MaxUint64) itShouldNotReturnError(err) itShouldSuccessfullyCreateTopic(streamId, t1.Id, t1Name, client) itShouldSuccessfullyCreateTopic(streamId, t2.Id, t2Name, client) diff --git a/bdd/go/tests/tcp_test/topic_feature_create.go b/bdd/go/tests/tcp_test/topic_feature_create.go index d5777601ba..902abcd2b2 100644 --- a/bdd/go/tests/tcp_test/topic_feature_create.go +++ b/bdd/go/tests/tcp_test/topic_feature_create.go @@ -32,7 +32,6 @@ var _ = ginkgo.Describe("CREATE TOPIC:", func() { ginkgo.Context("and tries to create topic unique name and id", func() { client := createAuthorizedConnection() streamId, _ := successfullyCreateStream(prefix, client) - replicationFactor := uint8(1) name := createRandomString(32) defer deleteStreamAfterTests(streamId, client) streamIdentifier, _ := iggcon.NewIdentifier(streamId) @@ -43,8 +42,7 @@ var _ = ginkgo.Describe("CREATE TOPIC:", func() { 2, iggcon.CompressionAlgorithmNone, iggcon.Millisecond, - math.MaxUint64, - &replicationFactor) + math.MaxUint64) itShouldNotReturnError(err) }) @@ -52,7 +50,6 @@ var _ = ginkgo.Describe("CREATE TOPIC:", func() { ginkgo.Context("and tries to create topic for a non existing stream", func() { client := createAuthorizedConnection() streamId := createRandomUInt32() - replicationFactor := uint8(1) name := createRandomString(32) streamIdentifier, _ := iggcon.NewIdentifier(streamId) _, err := client.CreateTopic( @@ -62,8 +59,7 @@ var _ = ginkgo.Describe("CREATE TOPIC:", func() { 2, iggcon.CompressionAlgorithmNone, iggcon.Millisecond, - math.MaxUint64, - &replicationFactor) + math.MaxUint64) itShouldReturnSpecificError(err, ierror.ErrStreamIdNotFound) }) @@ -74,7 +70,6 @@ var _ = ginkgo.Describe("CREATE TOPIC:", func() { defer deleteStreamAfterTests(streamId, client) _, name := successfullyCreateTopic(streamId, client) - replicationFactor := uint8(1) streamIdentifier, _ := iggcon.NewIdentifier(streamId) _, err := client.CreateTopic( context.Background(), @@ -83,8 +78,7 @@ var _ = ginkgo.Describe("CREATE TOPIC:", func() { 2, iggcon.CompressionAlgorithmNone, iggcon.IggyExpiryServerDefault, - math.MaxUint64, - &replicationFactor) + math.MaxUint64) itShouldReturnSpecificError(err, ierror.ErrTopicNameAlreadyExists) }) @@ -94,7 +88,6 @@ var _ = ginkgo.Describe("CREATE TOPIC:", func() { defer deleteStreamAfterTests(streamId, createAuthorizedConnection()) streamIdentifier, _ := iggcon.NewIdentifier(streamId) - replicationFactor := uint8(1) _, err := client.CreateTopic( context.Background(), streamIdentifier, @@ -102,8 +95,7 @@ var _ = ginkgo.Describe("CREATE TOPIC:", func() { 2, iggcon.CompressionAlgorithmNone, iggcon.IggyExpiryServerDefault, - math.MaxUint64, - &replicationFactor) + math.MaxUint64) itShouldReturnSpecificError(err, ierror.ErrInvalidTopicName) }) @@ -112,7 +104,6 @@ var _ = ginkgo.Describe("CREATE TOPIC:", func() { ginkgo.When("User is not logged in", func() { ginkgo.Context("and tries to create topic", func() { client := createClient() - replicationFactor := uint8(1) streamIdentifier, _ := iggcon.NewIdentifier[uint32](10) _, err := client.CreateTopic( context.Background(), @@ -121,8 +112,7 @@ var _ = ginkgo.Describe("CREATE TOPIC:", func() { 2, iggcon.CompressionAlgorithmNone, iggcon.IggyExpiryServerDefault, - math.MaxUint64, - &replicationFactor) + math.MaxUint64) itShouldReturnUnauthenticatedError(err) }) diff --git a/bdd/go/tests/tcp_test/topic_feature_update.go b/bdd/go/tests/tcp_test/topic_feature_update.go index e8ce2e527e..2928830213 100644 --- a/bdd/go/tests/tcp_test/topic_feature_update.go +++ b/bdd/go/tests/tcp_test/topic_feature_update.go @@ -35,7 +35,6 @@ var _ = ginkgo.Describe("UPDATE TOPIC:", func() { defer deleteStreamAfterTests(streamId, client) topicId, _ := successfullyCreateTopic(streamId, client) newName := createRandomString(128) - replicationFactor := uint8(1) streamIdentifier, _ := iggcon.NewIdentifier(streamId) topicIdentifier, _ := iggcon.NewIdentifier(topicId) err := client.UpdateTopic( @@ -45,8 +44,7 @@ var _ = ginkgo.Describe("UPDATE TOPIC:", func() { newName, iggcon.CompressionAlgorithmNone, iggcon.Microsecond, - math.MaxUint64, - &replicationFactor) + math.MaxUint64) itShouldNotReturnError(err) itShouldSuccessfullyUpdateTopic(streamId, topicId, newName, client) }) @@ -57,7 +55,6 @@ var _ = ginkgo.Describe("UPDATE TOPIC:", func() { defer deleteStreamAfterTests(streamId, client) _, topic1Name := successfullyCreateTopic(streamId, client) topic2Id, _ := successfullyCreateTopic(streamId, client) - replicationFactor := uint8(1) streamIdentifier, _ := iggcon.NewIdentifier(streamId) topic2Identifier, _ := iggcon.NewIdentifier(topic2Id) err := client.UpdateTopic( @@ -67,15 +64,13 @@ var _ = ginkgo.Describe("UPDATE TOPIC:", func() { topic1Name, iggcon.CompressionAlgorithmNone, iggcon.IggyExpiryServerDefault, - math.MaxUint64, - &replicationFactor) + math.MaxUint64) itShouldReturnSpecificError(err, ierror.ErrTopicNameAlreadyExists) }) ginkgo.Context("and tries to update non-existing topic", func() { client := createAuthorizedConnection() - replicationFactor := uint8(1) err := client.UpdateTopic( context.Background(), randomU32Identifier(), @@ -83,8 +78,7 @@ var _ = ginkgo.Describe("UPDATE TOPIC:", func() { createRandomString(128), 1, 0, - math.MaxUint64, - &replicationFactor) + math.MaxUint64) itShouldReturnSpecificError(err, ierror.ErrStreamIdNotFound) }) @@ -93,7 +87,6 @@ var _ = ginkgo.Describe("UPDATE TOPIC:", func() { client := createAuthorizedConnection() streamId, _ := successfullyCreateStream(prefix, client) defer deleteStreamAfterTests(streamId, createAuthorizedConnection()) - replicationFactor := uint8(1) streamIdentifier, _ := iggcon.NewIdentifier(streamId) err := client.UpdateTopic( context.Background(), @@ -102,8 +95,7 @@ var _ = ginkgo.Describe("UPDATE TOPIC:", func() { createRandomString(128), 1, 0, - math.MaxUint64, - &replicationFactor) + math.MaxUint64) itShouldReturnSpecificError(err, ierror.ErrTopicIdNotFound) }) @@ -113,7 +105,6 @@ var _ = ginkgo.Describe("UPDATE TOPIC:", func() { streamId, _ := successfullyCreateStream(prefix, client) defer deleteStreamAfterTests(streamId, createAuthorizedConnection()) topicId, _ := successfullyCreateTopic(streamId, client) - replicationFactor := uint8(1) streamIdentifier, _ := iggcon.NewIdentifier(streamId) topicIdentifier, _ := iggcon.NewIdentifier(topicId) err := client.UpdateTopic( @@ -123,8 +114,7 @@ var _ = ginkgo.Describe("UPDATE TOPIC:", func() { createRandomString(256), 1, 0, - math.MaxUint64, - &replicationFactor) + math.MaxUint64) itShouldReturnSpecificError(err, ierror.ErrInvalidTopicName) }) diff --git a/bdd/go/tests/tcp_test/topic_steps.go b/bdd/go/tests/tcp_test/topic_steps.go index fae8d6a40c..a359f688d6 100644 --- a/bdd/go/tests/tcp_test/topic_steps.go +++ b/bdd/go/tests/tcp_test/topic_steps.go @@ -31,7 +31,6 @@ import ( //operations func successfullyCreateTopic(streamId uint32, client iggcon.Client) (uint32, string) { - replicationFactor := uint8(1) name := createRandomString(128) streamIdentifier, _ := iggcon.NewIdentifier(streamId) topic, err := client.CreateTopic( @@ -41,8 +40,7 @@ func successfullyCreateTopic(streamId uint32, client iggcon.Client) (uint32, str 2, 1, 0, - math.MaxUint64, - &replicationFactor) + math.MaxUint64) itShouldSuccessfullyCreateTopic(streamId, topic.Id, name, client) itShouldNotReturnError(err) diff --git a/bdd/java/src/test/java/org/apache/iggy/bdd/BasicMessagingSteps.java b/bdd/java/src/test/java/org/apache/iggy/bdd/BasicMessagingSteps.java index c14615b286..6b644b4410 100644 --- a/bdd/java/src/test/java/org/apache/iggy/bdd/BasicMessagingSteps.java +++ b/bdd/java/src/test/java/org/apache/iggy/bdd/BasicMessagingSteps.java @@ -108,7 +108,6 @@ public void createTopic(String topicName, int streamId, int partitions) { CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, - Optional.empty(), topicName); context.lastTopicId = topic.id(); diff --git a/bdd/rust/tests/steps/topics.rs b/bdd/rust/tests/steps/topics.rs index 4515bba9d2..2506908b82 100644 --- a/bdd/rust/tests/steps/topics.rs +++ b/bdd/rust/tests/steps/topics.rs @@ -17,7 +17,7 @@ use crate::common::global_context::GlobalContext; use cucumber::{then, when}; -use iggy::prelude::{CompressionAlgorithm, Identifier, IggyExpiry, MaxTopicSize, TopicClient}; +use iggy::prelude::{Identifier, IggyExpiry, TopicClient, TopicCreateOptions}; #[when(regex = r"^I create a topic with name (.+) in stream (\d+) with (\d+) partitions$")] pub async fn when_create_topic( @@ -31,11 +31,11 @@ pub async fn when_create_topic( .create_topic( &Identifier::numeric(stream_id).unwrap(), &topic_name, - partitions_count, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(partitions_count), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .expect("Should be able to create topic"); diff --git a/core/ai/mcp/src/service/mod.rs b/core/ai/mcp/src/service/mod.rs index 1653b02fba..7ba7b027dd 100644 --- a/core/ai/mcp/src/service/mod.rs +++ b/core/ai/mcp/src/service/mod.rs @@ -16,10 +16,12 @@ // under the License. use iggy::prelude::{ - ClusterClient, Consumer, ConsumerGroupClient, ConsumerOffsetClient, Identifier, IggyClient, - IggyError, IggyMessage, IggyTimestamp, MessageClient, PartitionClient, Partitioning, - PersonalAccessTokenClient, PollingKind, PollingStrategy, SegmentClient, StreamClient, - SystemClient, SystemSnapshotType, TopicClient, UserClient, UserStatus, + ClusterClient, CompressionAlgorithm, Consumer, ConsumerGroupClient, ConsumerOffsetClient, + Identifier, IggyClient, IggyError, IggyExpiry, IggyMessage, IggyTimestamp, MaxTopicSize, + MessageClient, PartitionClient, Partitioning, PersonalAccessTokenClient, PollingKind, + PollingStrategy, SegmentClient, StreamClient, StreamUpdateOptions, SystemClient, + SystemSnapshotType, TopicClient, TopicCreateOptions, TopicUpdateOptions, UserClient, + UserStatus, UserUpdateOptions, }; use requests::*; use rmcp::{ @@ -94,7 +96,11 @@ impl IggyService { Parameters(UpdateStream { stream_id, name }): Parameters, ) -> Result { self.permissions.ensure_update()?; - request(self.client.update_stream(&id(&stream_id)?, &name).await) + request( + self.client + .update_stream(&id(&stream_id)?, &name, &StreamUpdateOptions::default()) + .await, + ) } #[tool(description = "Delete stream")] @@ -148,9 +154,9 @@ impl IggyService { name, partitions_count, compression_algorithm, - replication_factor, message_expiry, max_size, + options, }): Parameters, ) -> Result { self.permissions.ensure_create()?; @@ -166,11 +172,21 @@ impl IggyService { .create_topic( &id(&stream_id)?, &name, - partitions_count, - compression_algorithm, - replication_factor, - message_expiry, - max_size, + &TopicCreateOptions { + partitions_count: Some(partitions_count), + compression_algorithm: (compression_algorithm + != CompressionAlgorithm::default()) + .then_some(compression_algorithm), + message_expiry: (message_expiry != IggyExpiry::ServerDefault) + .then_some(message_expiry), + max_topic_size: (max_size != MaxTopicSize::ServerDefault) + .then_some(max_size), + // Raw keys ride as strings and are parsed server-side by + // the same rules a config value goes through, so a key + // added after this build shipped is still reachable. + raw: options, + ..TopicCreateOptions::default() + }, ) .await, ) @@ -184,30 +200,23 @@ impl IggyService { topic_id, name, compression_algorithm, - replication_factor, message_expiry, max_size, + options, }): Parameters, ) -> Result { self.permissions.ensure_update()?; - let compression_algorithm = compression_algorithm - .and_then(|ca| ca.parse().ok()) - .unwrap_or_default(); - let message_expiry = message_expiry - .and_then(|me| me.parse().ok()) - .unwrap_or_default(); - let max_size = max_size.and_then(|ms| ms.parse().ok()).unwrap_or_default(); + // Absent means "leave alone" now, so an unparsable value must not + // silently become a reset to the server default. + let update_options = TopicUpdateOptions { + compression_algorithm: compression_algorithm.and_then(|ca| ca.parse().ok()), + message_expiry: message_expiry.and_then(|me| me.parse().ok()), + max_topic_size: max_size.and_then(|ms| ms.parse().ok()), + raw: options, + }; request( self.client - .update_topic( - &id(&stream_id)?, - &id(&topic_id)?, - &name, - compression_algorithm, - replication_factor, - message_expiry, - max_size, - ) + .update_topic(&id(&stream_id)?, &id(&topic_id)?, &name, &update_options) .await, ) } @@ -691,7 +700,12 @@ impl IggyService { }); request( self.client - .update_user(&id(&user_id)?, username.as_deref(), status) + .update_user( + &id(&user_id)?, + username.as_deref(), + status, + &UserUpdateOptions::default(), + ) .await, ) } diff --git a/core/ai/mcp/src/service/requests.rs b/core/ai/mcp/src/service/requests.rs index a0833eb1e9..0a498c65a7 100644 --- a/core/ai/mcp/src/service/requests.rs +++ b/core/ai/mcp/src/service/requests.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use iggy::prelude; use rmcp::schemars::{self, JsonSchema}; @@ -82,14 +82,18 @@ pub struct CreateTopic { #[schemars(description = "compression algorithm (optional, can be one of 'none', 'gzip')")] pub compression_algorithm: Option, - #[schemars(description = "replication factor (optional, must be greater than 0)")] - pub replication_factor: Option, - #[schemars(description = "message expiry (optional)")] pub message_expiry: Option, #[schemars(description = "maximum size (optional)")] pub max_size: Option, + + #[schemars( + description = "additional options as string key-values, e.g. {\"segment_size\": \"128 MiB\"}; \ + call describe_options for the keys the server accepts (optional)" + )] + #[serde(default)] + pub options: BTreeMap, } #[derive(Debug, Deserialize, JsonSchema)] @@ -106,14 +110,19 @@ pub struct UpdateTopic { #[schemars(description = "compression algorithm (optional, can be one of 'none', 'gzip')")] pub compression_algorithm: Option, - #[schemars(description = "replication factor (optional, must be greater than 0)")] - pub replication_factor: Option, - #[schemars(description = "message expiry (optional)")] pub message_expiry: Option, #[schemars(description = "maximum size (optional)")] pub max_size: Option, + + #[schemars( + description = "options to change, as string key-values; only compression_algorithm, \ + message_expiry and max_topic_size may be updated, and a key left out \ + keeps its current value (optional)" + )] + #[serde(default)] + pub options: BTreeMap, } #[derive(Debug, Deserialize, JsonSchema)] diff --git a/core/bench/src/actors/producer/client/high_level.rs b/core/bench/src/actors/producer/client/high_level.rs index 840b076f79..3db74f869e 100644 --- a/core/bench/src/actors/producer/client/high_level.rs +++ b/core/bench/src/actors/producer/client/high_level.rs @@ -105,7 +105,6 @@ impl BenchmarkInit for HighLevelProducerClient { .create_stream_if_not_exists() .create_topic_if_not_exists( self.config.partitions, - Some(1), IggyExpiry::NeverExpire, MaxTopicSize::ServerDefault, ) diff --git a/core/bench/src/args/common.rs b/core/bench/src/args/common.rs index 1a6a3a46b3..a7c5f9f7d6 100644 --- a/core/bench/src/args/common.rs +++ b/core/bench/src/args/common.rs @@ -100,6 +100,12 @@ pub struct IggyBenchArgs { /// consumers start with fresh data and accurate latency measurements. #[arg(long, default_value_t = false)] pub reuse_streams: bool, + + /// Fsync every write on the benchmark topic instead of leaving it to the + /// page cache. Set as a topic option at creation, so it has no effect with + /// `--reuse-streams` against an already-created topic. + #[arg(long, default_value_t = false)] + pub enforce_fsync: bool, } impl IggyBenchArgs { @@ -327,6 +333,10 @@ impl IggyBenchArgs { self.reuse_streams } + pub const fn enforce_fsync(&self) -> bool { + self.enforce_fsync + } + pub fn username(&self) -> &str { &self.username } diff --git a/core/bench/src/benchmarks/benchmark.rs b/core/bench/src/benchmarks/benchmark.rs index d1724e5a1f..3d61bc3319 100644 --- a/core/bench/src/benchmarks/benchmark.rs +++ b/core/bench/src/benchmarks/benchmark.rs @@ -130,21 +130,26 @@ pub trait Benchmarkable: Send { .max_topic_size() .map_or(MaxTopicSize::Unlimited, MaxTopicSize::Custom); let message_expiry = self.args().message_expiry(); + let enforce_fsync = self.args().enforce_fsync(); info!( - "Creating the test topic '{}' for stream '{}' with max topic size: {:?}, message expiry: {}", - topic_name, stream_name, max_topic_size, message_expiry + "Creating the test topic '{}' for stream '{}' with max topic size: {:?}, message expiry: {}, enforce fsync: {}", + topic_name, stream_name, max_topic_size, message_expiry, enforce_fsync ); client .create_topic( &stream_id, &topic_name, - partitions_count, - CompressionAlgorithm::default(), - None, - message_expiry, - max_topic_size, + &TopicCreateOptions { + partitions_count: Some(partitions_count), + message_expiry: (message_expiry != IggyExpiry::ServerDefault) + .then_some(message_expiry), + max_topic_size: (max_topic_size != MaxTopicSize::ServerDefault) + .then_some(max_topic_size), + enforce_fsync: enforce_fsync.then_some(true), + ..TopicCreateOptions::default() + }, ) .await?; } diff --git a/core/binary_protocol/src/codec.rs b/core/binary_protocol/src/codec.rs index 801ba787cf..4edb5304ba 100644 --- a/core/binary_protocol/src/codec.rs +++ b/core/binary_protocol/src/codec.rs @@ -251,16 +251,23 @@ pub fn read_bytes(buf: &[u8], offset: usize, len: usize) -> Result<&[u8], WireEr }) } -/// Cap a pre-allocation hint so a bogus wire count cannot cause OOM. -/// The actual count is validated by the decode loop - this only limits -/// the upfront allocation. -#[inline] +/// Capacity to preallocate for `count` elements that still have to be decoded +/// out of `remaining` bytes, each at least `min_element_size` bytes on the +/// wire. +/// +/// A count read off the wire is chosen by the peer, and `Vec::with_capacity` +/// reserves for it before a single element has been bounds-checked. A hostile +/// or corrupt frame claiming `u32::MAX` elements therefore aborts the process +/// on allocation failure instead of returning a decode error. The bytes left +/// to decode are the honest ceiling on how many elements can exist. +/// +/// A zero `min_element_size` is treated as one byte rather than passing the +/// wire count through: an element that occupies no bytes cannot exist, and +/// trusting the count is the exact hole this function was written to close. #[must_use] -pub fn capped_capacity(count: usize, remaining: usize, min_item_size: usize) -> usize { - if min_item_size == 0 { - return count; - } - count.min(remaining / min_item_size) +#[inline] +pub fn bounded_capacity(count: usize, remaining: usize, min_element_size: usize) -> usize { + count.min(remaining / min_element_size.max(1)) } #[cfg(test)] @@ -268,17 +275,17 @@ mod tests { use super::*; #[test] - fn capped_capacity_limits_allocation() { - assert_eq!(capped_capacity(1_000_000, 100, 10), 10); - assert_eq!(capped_capacity(5, 100, 10), 5); - assert_eq!(capped_capacity(10, 100, 10), 10); - assert_eq!(capped_capacity(11, 100, 10), 10); - assert_eq!(capped_capacity(0, 100, 10), 0); - assert_eq!(capped_capacity(100, 0, 10), 0); + fn bounded_capacity_limits_allocation() { + assert_eq!(bounded_capacity(1_000_000, 100, 10), 10); + assert_eq!(bounded_capacity(5, 100, 10), 5); + assert_eq!(bounded_capacity(10, 100, 10), 10); + assert_eq!(bounded_capacity(11, 100, 10), 10); + assert_eq!(bounded_capacity(0, 100, 10), 0); + assert_eq!(bounded_capacity(100, 0, 10), 0); } #[test] - fn capped_capacity_zero_item_size_returns_count() { - assert_eq!(capped_capacity(1_000_000, 100, 0), 1_000_000); + fn bounded_capacity_zero_element_size_still_bounds_by_the_buffer() { + assert_eq!(bounded_capacity(1_000_000, 100, 0), 100); } } diff --git a/core/binary_protocol/src/codes.rs b/core/binary_protocol/src/codes.rs index 3a538130ca..90420293de 100644 --- a/core/binary_protocol/src/codes.rs +++ b/core/binary_protocol/src/codes.rs @@ -25,6 +25,7 @@ pub const PING_CODE: u32 = 1; pub const GET_STATS_CODE: u32 = 10; pub const GET_SNAPSHOT_FILE_CODE: u32 = 11; pub const GET_CLUSTER_METADATA_CODE: u32 = 12; +pub const DESCRIBE_OPTIONS_CODE: u32 = 13; pub const GET_ME_CODE: u32 = 20; pub const GET_CLIENT_CODE: u32 = 21; pub const GET_CLIENTS_CODE: u32 = 22; diff --git a/core/binary_protocol/src/dispatch.rs b/core/binary_protocol/src/dispatch.rs index 92f0203030..576fadb0fd 100644 --- a/core/binary_protocol/src/dispatch.rs +++ b/core/binary_protocol/src/dispatch.rs @@ -193,6 +193,8 @@ pub const COMMAND_TABLE: &[CommandMeta] = &[ CommandMeta::non_replicated(SYNC_CONSUMER_GROUP_CODE, "consumer_group.sync"), // Login + Register (PAT - Personal Access Token variant) CommandMeta::non_replicated(LOGIN_REGISTER_WITH_PAT_CODE, "user.login_register_with_pat"), + // Options catalog discovery + CommandMeta::non_replicated(DESCRIBE_OPTIONS_CODE, "options.describe"), ]; /// Lookup command metadata by command code. @@ -256,6 +258,7 @@ pub const fn lookup_command(code: u32) -> Option<&'static CommandMeta> { LEAVE_CONSUMER_GROUP_CODE => 49, SYNC_CONSUMER_GROUP_CODE => 50, LOGIN_REGISTER_WITH_PAT_CODE => 51, + DESCRIBE_OPTIONS_CODE => 52, _ => return None, }; Some(&COMMAND_TABLE[idx]) @@ -367,6 +370,7 @@ mod tests { DELETE_CONSUMER_GROUP_CODE, JOIN_CONSUMER_GROUP_CODE, LEAVE_CONSUMER_GROUP_CODE, + DESCRIBE_OPTIONS_CODE, ]; for code in all_codes { assert!( diff --git a/core/binary_protocol/src/lib.rs b/core/binary_protocol/src/lib.rs index 0825d0f449..b308e5f73c 100644 --- a/core/binary_protocol/src/lib.rs +++ b/core/binary_protocol/src/lib.rs @@ -90,6 +90,7 @@ pub use message_view::{ pub use primitives::ack_level::AckLevel; pub use primitives::consumer::{KIND_CONSUMER_GROUP, WireConsumer}; pub use primitives::identifier::{MAX_WIRE_NAME_LENGTH, WireIdentifier, WireName}; +pub use primitives::options::{MAX_OPTIONS, MAX_OPTIONS_BYTES, WireOptions, validate_options}; pub use primitives::partition_assignment::CreatedPartitionAssignment; pub use primitives::partitioning::{MAX_MESSAGES_KEY_LENGTH, WirePartitioning}; pub use primitives::permissions::{ diff --git a/core/binary_protocol/src/primitives/mod.rs b/core/binary_protocol/src/primitives/mod.rs index f3c7ee4e32..c725289256 100644 --- a/core/binary_protocol/src/primitives/mod.rs +++ b/core/binary_protocol/src/primitives/mod.rs @@ -20,6 +20,7 @@ pub mod ack_level; pub mod consumer; pub mod identifier; +pub mod options; pub mod partition_assignment; pub mod partitioning; pub mod permissions; diff --git a/core/binary_protocol/src/primitives/options.rs b/core/binary_protocol/src/primitives/options.rs new file mode 100644 index 0000000000..e7139fd8ae --- /dev/null +++ b/core/binary_protocol/src/primitives/options.rs @@ -0,0 +1,481 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Key-value options block for `Create*` requests. +//! +//! Options reuse the user-headers TLV encoding and its structural validator +//! wholesale, so unknown value kind codes remain forwardable across +//! mixed-version clusters. On top of the structural walk this layer enforces +//! the stricter contract options need and messages do not: string keys only, +//! no duplicate keys, and hard bounds on entry count, key length, and total +//! block size, all rejected at the wire edge before a request reaches +//! admission. + +use std::borrow::Cow; + +use bytes::{BufMut, Bytes, BytesMut}; + +use crate::WireError; +use crate::codec::WireEncode; +use crate::primitives::user_headers::{ + WireHeaderKind, WireUserHeaderEntry, WireUserHeaderIterator, validate_user_headers, +}; + +/// Maximum number of key-value entries in one options block. +pub const MAX_OPTIONS: u32 = 1024; + +/// Maximum total byte length of an encoded options block. +/// +/// Mirrors `iggy_common::MAX_USER_HEADERS_SIZE`: options ride the user-headers +/// codec, so they inherit its budget rather than inventing a second one. The +/// constant is duplicated because `iggy_common` depends on this crate, not the +/// other way round; `options_block_budget_matches_user_headers` pins the two +/// together from the side that can see both. +/// +/// Sized so [`MAX_OPTIONS`] is actually reachable: the cheapest entry is 12 +/// bytes (kind + length + one byte, for key and value each), so 1024 entries +/// need at least 12 KiB. A smaller budget would silently cap the entry count +/// below `MAX_OPTIONS` and make that limit a lie. +pub const MAX_OPTIONS_BYTES: usize = 100 * 1000; + +/// Wire kind code for UTF-8 string fields, matching `HeaderKind::String` +/// in `iggy_common`. +const STRING_KIND: WireHeaderKind = WireHeaderKind(2); + +/// Validate the structural and semantic integrity of an options byte buffer. +/// +/// Runs [`validate_user_headers`] first, then enforces the options contract: +/// +/// - Total block size within [`MAX_OPTIONS_BYTES`] +/// - At most [`MAX_OPTIONS`] entries +/// - Every key is a UTF-8 string (length already bounded by the codec) +/// - No duplicate keys +/// +/// Value kind codes are deliberately NOT restricted to the currently defined +/// range, mirroring the user-headers forward-compatibility contract. +/// +/// Returns the number of key-value pairs on success. +/// +/// # Errors +/// +/// Returns `WireError::UnexpectedEof` if the buffer is truncated mid-entry, +/// or `WireError::Validation` if any constraint above is violated. +pub fn validate_options(buf: &[u8]) -> Result { + if buf.len() > MAX_OPTIONS_BYTES { + return Err(WireError::Validation(Cow::Owned(format!( + "options block is {} bytes, exceeds maximum {MAX_OPTIONS_BYTES}", + buf.len() + )))); + } + + let pairs = validate_user_headers(buf)?; + if pairs == 0 { + return Ok(0); + } + if pairs > MAX_OPTIONS { + return Err(WireError::Validation(Cow::Owned(format!( + "options block has {pairs} entries, exceeds maximum {MAX_OPTIONS}" + )))); + } + + let mut keys: Vec<&[u8]> = Vec::with_capacity(pairs as usize); + for entry in WireUserHeaderIterator::new(buf) { + if entry.key_kind != STRING_KIND { + return Err(WireError::Validation(Cow::Owned(format!( + "option key kind {} is not a string", + entry.key_kind.0 + )))); + } + if std::str::from_utf8(entry.key).is_err() { + return Err(WireError::Validation(Cow::Borrowed( + "option key is not valid UTF-8", + ))); + } + keys.push(entry.key); + } + + keys.sort_unstable(); + for pair in keys.windows(2) { + if pair[0] == pair[1] { + return Err(WireError::Validation(Cow::Owned(format!( + "duplicate option key: {}", + String::from_utf8_lossy(pair[0]) + )))); + } + } + + Ok(pairs) +} + +/// Encoded size of a `u32`-length-prefixed options block. +/// +/// Response elements (topic/stream/user entries in a list) cannot run their +/// options to end-of-payload the way requests do, so they carry the block +/// behind a length prefix. +#[must_use] +pub fn options_prefixed_size(options: &WireOptions) -> usize { + 4 + options.encoded_size() +} + +/// Encode a `u32`-length-prefixed options block. +pub fn encode_options_prefixed(options: &WireOptions, buf: &mut BytesMut) { + #[allow(clippy::cast_possible_truncation)] + buf.put_u32_le(options.encoded_size() as u32); + options.encode(buf); +} + +/// Decode a `u32`-length-prefixed options block at `pos`. +/// +/// Returns the block and the total bytes consumed (prefix included). +/// +/// # Errors +/// +/// Returns `WireError::UnexpectedEof` on truncation, or +/// `WireError::Validation` when the block violates [`validate_options`]. +pub fn decode_options_prefixed(buf: &[u8], pos: usize) -> Result<(WireOptions, usize), WireError> { + let length = crate::codec::read_u32_le(buf, pos)? as usize; + let start = pos + 4; + // `checked_add`: on a 32-bit target a wire-supplied length wraps, the + // truncation guard below then passes, and the slice panics instead. + let end = start + .checked_add(length) + .ok_or_else(|| WireError::UnexpectedEof { + offset: start, + need: length, + have: buf.len().saturating_sub(start), + })?; + if buf.len() < end { + return Err(WireError::UnexpectedEof { + offset: start, + need: length, + have: buf.len().saturating_sub(start), + }); + } + let options = WireOptions::from_slice(&buf[start..end])?; + Ok((options, 4 + length)) +} + +/// Pre-validated options as a contiguous TLV byte buffer. +/// +/// Construction validates both structural TLV integrity and the options +/// contract via [`validate_options`]. The inner bytes are immutable. +/// +/// Like `WireUserHeaders`, this type is opaque at the wire layer. Domain +/// interpretation of value kind codes happens in `iggy_common`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct WireOptions(Bytes); + +impl WireOptions { + /// Validate and wrap a borrowed byte slice (copies into new `Bytes`). + /// + /// # Errors + /// Returns `WireError` if the buffer violates [`validate_options`]. + pub fn from_slice(buf: &[u8]) -> Result { + validate_options(buf)?; + Ok(Self(Bytes::copy_from_slice(buf))) + } + + /// Validate and wrap an owned `Bytes` buffer (zero-copy). + /// + /// # Errors + /// Returns `WireError` if the buffer violates [`validate_options`]. + pub fn from_bytes(buf: Bytes) -> Result { + validate_options(&buf)?; + Ok(Self(buf)) + } + + /// Wrap pre-validated bytes without re-checking. + /// + /// # Safety contract (not `unsafe`, but caller must ensure): + /// The bytes must satisfy [`validate_options`]. Iterating invalid bytes + /// will panic. + pub const fn from_validated(buf: Bytes) -> Self { + Self(buf) + } + + /// Empty options (zero-length buffer). + #[must_use] + pub const fn empty() -> Self { + Self(Bytes::new()) + } + + #[must_use] + pub const fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// The raw validated bytes, suitable for wire transmission. + #[must_use] + pub fn as_bytes(&self) -> &[u8] { + &self.0 + } + + /// Consume into the inner `Bytes` handle (zero-copy). + #[must_use] + pub fn into_bytes(self) -> Bytes { + self.0 + } + + /// Zero-copy iteration over the pre-validated entries. + #[must_use] + pub fn iter(&self) -> WireUserHeaderIterator<'_> { + WireUserHeaderIterator::new(&self.0) + } +} + +impl<'a> IntoIterator for &'a WireOptions { + type Item = WireUserHeaderEntry<'a>; + type IntoIter = WireUserHeaderIterator<'a>; + + fn into_iter(self) -> Self::IntoIter { + self.iter() + } +} + +impl WireEncode for WireOptions { + fn encoded_size(&self) -> usize { + self.0.len() + } + + fn encode(&self, buf: &mut BytesMut) { + buf.put_slice(&self.0); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::primitives::user_headers::encode_user_headers; + + const STRING: u8 = 2; + const UINT64: u8 = 12; + + fn encode(entries: &[(u8, &[u8], u8, &[u8])]) -> BytesMut { + let mut buf = BytesMut::new(); + encode_user_headers(entries, &mut buf); + buf + } + + #[test] + fn empty_buffer_is_zero_options() { + assert_eq!(validate_options(&[]).unwrap(), 0); + assert!(WireOptions::empty().is_empty()); + } + + #[test] + fn valid_block_roundtrips() { + let value = 1_073_741_824u64.to_le_bytes(); + let buf = encode(&[ + (STRING, b"segment_size", UINT64, &value), + (STRING, b"message_expiry", STRING, b"7 days"), + ]); + assert_eq!(validate_options(&buf).unwrap(), 2); + + let options = WireOptions::from_slice(&buf).unwrap(); + let entries: Vec<_> = options.iter().collect(); + assert_eq!(entries.len(), 2); + assert_eq!(entries[0].key, b"segment_size"); + assert_eq!(entries[0].value, value); + assert_eq!(entries[1].key, b"message_expiry"); + assert_eq!(entries[1].value, b"7 days"); + } + + #[test] + fn unknown_value_kind_is_forwardable() { + let buf = encode(&[(STRING, b"future_option", 200, b"opaque")]); + assert_eq!(validate_options(&buf).unwrap(), 1); + } + + #[test] + fn duplicate_key_is_rejected() { + let buf = encode(&[ + (STRING, b"segment_size", STRING, b"1 GiB"), + (STRING, b"segment_size", STRING, b"2 GiB"), + ]); + assert!(matches!( + validate_options(&buf), + Err(WireError::Validation(_)) + )); + } + + #[test] + fn non_string_key_kind_is_rejected() { + let buf = encode(&[(UINT64, &1u64.to_le_bytes(), STRING, b"value")]); + assert!(matches!( + validate_options(&buf), + Err(WireError::Validation(_)) + )); + } + + #[test] + fn non_utf8_key_is_rejected() { + let buf = encode(&[(STRING, &[0xFF, 0xFE], STRING, b"value")]); + assert!(matches!( + validate_options(&buf), + Err(WireError::Validation(_)) + )); + } + + #[test] + fn key_length_is_bounded_by_the_user_headers_codec() { + // Options add no key-length rule of their own: every field is already + // bounded to 1..=255 by `validate_user_headers`, so the codec's limit + // is the option key's limit. + let over = [b'k'; 256]; + let buf = encode(&[(STRING, &over, STRING, b"value")]); + assert!(matches!( + validate_options(&buf), + Err(WireError::Validation(_)) + )); + + let at_limit = [b'k'; 255]; + let buf = encode(&[(STRING, &at_limit, STRING, b"value")]); + assert_eq!(validate_options(&buf).unwrap(), 1); + } + + #[test] + fn entry_count_over_limit_is_rejected() { + let keys: Vec = (0..=MAX_OPTIONS).map(|i| format!("key_{i}")).collect(); + let entries: Vec<(u8, &[u8], u8, &[u8])> = keys + .iter() + .map(|key| (STRING, key.as_bytes(), STRING, b"v".as_slice())) + .collect(); + let buf = encode(&entries); + assert!(matches!( + validate_options(&buf), + Err(WireError::Validation(_)) + )); + + let buf = encode(&entries[..MAX_OPTIONS as usize]); + assert_eq!(validate_options(&buf).unwrap(), MAX_OPTIONS); + } + + #[test] + fn block_over_byte_limit_is_rejected() { + // 200 maximum-size entries are ~104 KB, over the byte budget while + // still well under `MAX_OPTIONS`, so this proves the byte cap trips + // independently of the entry count. + let value = [b'v'; 255]; + let keys: Vec = (0..200).map(|i| format!("{i:0>255}")).collect(); + let entries: Vec<(u8, &[u8], u8, &[u8])> = keys + .iter() + .map(|key| (STRING, key.as_bytes(), STRING, value.as_slice())) + .collect(); + let buf = encode(&entries); + assert!(buf.len() > MAX_OPTIONS_BYTES); + assert!(matches!( + validate_options(&buf), + Err(WireError::Validation(_)) + )); + } + + #[test] + fn truncated_block_errors() { + let buf = encode(&[(STRING, b"segment_size", STRING, b"1 GiB")]); + for i in 1..buf.len() { + assert!( + validate_options(&buf[..i]).is_err(), + "expected error for truncation at byte {i}" + ); + } + } + + #[test] + fn wire_options_constructors_validate() { + let buf = encode(&[ + (STRING, b"enforce_fsync", STRING, b"true"), + (STRING, b"enforce_fsync", STRING, b"false"), + ]); + assert!(WireOptions::from_slice(&buf).is_err()); + assert!(WireOptions::from_bytes(buf.freeze()).is_err()); + } + + #[test] + fn encoded_size_matches_bytes() { + let buf = encode(&[(STRING, b"key", STRING, b"value")]); + let options = WireOptions::from_slice(&buf).unwrap(); + assert_eq!(options.encoded_size(), buf.len()); + assert_eq!(options.as_bytes(), &buf[..]); + } + + #[test] + fn prefixed_roundtrip() { + let buf = encode(&[(STRING, b"key", STRING, b"value")]); + let options = WireOptions::from_slice(&buf).unwrap(); + let mut encoded = BytesMut::new(); + encode_options_prefixed(&options, &mut encoded); + assert_eq!(encoded.len(), options_prefixed_size(&options)); + let (decoded, consumed) = decode_options_prefixed(&encoded, 0).unwrap(); + assert_eq!(consumed, encoded.len()); + assert_eq!(decoded, options); + } + + /// The cross-SDK golden vector for an options block. + /// + /// Every SDK writes this block from its own encoder, so "it round-trips + /// through my own decoder" proves nothing about interoperability. The bytes + /// here are the contract: `foreign/{node,go,java}` pin the identical vector + /// in their own unit tests, and a change to the TLV layout has to break all + /// of them together instead of leaving one SDK talking to itself. + /// + /// `enforce_fsync=true` (a `Bool`) and `segment_size=1 GiB` (a `Uint64`) + /// cover both a one-byte and an eight-byte value, in the sorted key order + /// every encoder has to produce. + const GOLDEN_OPTIONS_BLOCK: &[u8] = &[ + 2, 13, 0, 0, 0, // key kind String, length 13 + b'e', b'n', b'f', b'o', b'r', b'c', b'e', b'_', b'f', b's', b'y', b'n', b'c', 3, 1, 0, 0, + 0, 1, // value kind Bool, length 1, true + 2, 12, 0, 0, 0, // key kind String, length 12 + b's', b'e', b'g', b'm', b'e', b'n', b't', b'_', b's', b'i', b'z', b'e', 12, 8, 0, 0, + 0, // value kind Uint64, length 8 + 0, 0, 0, 64, 0, 0, 0, 0, // 1 GiB little-endian + ]; + + #[test] + fn golden_options_block_is_byte_stable() { + let encoded = encode(&[ + (STRING, b"enforce_fsync", 3, &[1]), + ( + STRING, + b"segment_size", + UINT64, + &1_073_741_824u64.to_le_bytes(), + ), + ]); + + assert_eq!( + &encoded[..], + GOLDEN_OPTIONS_BLOCK, + "the options TLV layout changed; update every SDK's copy of this vector" + ); + assert_eq!(validate_options(GOLDEN_OPTIONS_BLOCK).unwrap(), 2); + } + + #[test] + fn prefixed_truncation_errors() { + let buf = encode(&[(STRING, b"key", STRING, b"value")]); + let options = WireOptions::from_slice(&buf).unwrap(); + let mut encoded = BytesMut::new(); + encode_options_prefixed(&options, &mut encoded); + for i in 0..encoded.len() { + assert!( + decode_options_prefixed(&encoded[..i], 0).is_err(), + "expected error for truncation at byte {i}" + ); + } + } +} diff --git a/core/binary_protocol/src/primitives/partition_assignment.rs b/core/binary_protocol/src/primitives/partition_assignment.rs index 2c36732bd1..27e8f9e729 100644 --- a/core/binary_protocol/src/primitives/partition_assignment.rs +++ b/core/binary_protocol/src/primitives/partition_assignment.rs @@ -25,9 +25,14 @@ pub struct CreatedPartitionAssignment { pub consensus_group_id: u64, } +impl CreatedPartitionAssignment { + /// Fixed wire size: a `u32` partition id and a `u64` consensus group id. + pub const ENCODED_SIZE: usize = 12; +} + impl WireEncode for CreatedPartitionAssignment { fn encoded_size(&self) -> usize { - 12 + Self::ENCODED_SIZE } fn encode(&self, buf: &mut BytesMut) { @@ -38,10 +43,10 @@ impl WireEncode for CreatedPartitionAssignment { impl WireDecode for CreatedPartitionAssignment { fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> { - if buf.len() < 12 { + if buf.len() < Self::ENCODED_SIZE { return Err(WireError::UnexpectedEof { offset: 0, - need: 12, + need: Self::ENCODED_SIZE, have: buf.len(), }); } diff --git a/core/binary_protocol/src/requests/streams/create_stream.rs b/core/binary_protocol/src/requests/streams/create_stream.rs index 93af616817..ebcb915255 100644 --- a/core/binary_protocol/src/requests/streams/create_stream.rs +++ b/core/binary_protocol/src/requests/streams/create_stream.rs @@ -18,39 +18,56 @@ use crate::WireError; use crate::codec::{WireDecode, WireEncode}; use crate::primitives::identifier::WireName; +use crate::primitives::options::WireOptions; use bytes::BytesMut; -/// `CreateStream` request. Wire format: `[name_len:1][name:N]` +/// `CreateStream` request. +/// +/// Wire format: `[name_len:1][name:N][options TLV to end]` +/// +/// The options block runs to the end of the payload; its validator requires +/// exact consumption, so no length prefix is needed. #[derive(Debug, Clone, PartialEq, Eq)] pub struct CreateStreamRequest { pub name: WireName, + pub options: WireOptions, } impl WireEncode for CreateStreamRequest { fn encoded_size(&self) -> usize { - self.name.encoded_size() + self.name.encoded_size() + self.options.encoded_size() } fn encode(&self, buf: &mut BytesMut) { self.name.encode(buf); + self.options.encode(buf); } } impl WireDecode for CreateStreamRequest { fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> { let (name, consumed) = WireName::decode(buf)?; - Ok((Self { name }, consumed)) + let options = WireOptions::from_slice(&buf[consumed..])?; + Ok((Self { name, options }, buf.len())) } } #[cfg(test)] mod tests { use super::*; + use crate::primitives::user_headers::encode_user_headers; + + fn sample_options() -> WireOptions { + let mut buf = BytesMut::new(); + encode_user_headers(&[(2, b"future_key", 2, b"value")], &mut buf); + WireOptions::from_bytes(buf.freeze()).unwrap() + } #[test] fn roundtrip() { let req = CreateStreamRequest { name: WireName::new("test-stream").unwrap(), + options: WireOptions::empty(), }; let bytes = req.to_bytes(); assert_eq!(bytes.len(), 1 + 11); @@ -59,6 +76,18 @@ mod tests { assert_eq!(decoded, req); } + #[test] + fn roundtrip_with_options() { + let req = CreateStreamRequest { + name: WireName::new("test-stream").unwrap(), + options: sample_options(), + }; + let bytes = req.to_bytes(); + let (decoded, consumed) = CreateStreamRequest::decode(&bytes).unwrap(); + assert_eq!(consumed, bytes.len()); + assert_eq!(decoded, req); + } + #[test] fn empty_name_rejected() { let buf = [0u8]; @@ -66,23 +95,28 @@ mod tests { } #[test] - fn truncated_returns_error() { + fn truncated_options_return_error() { let req = CreateStreamRequest { name: WireName::new("test").unwrap(), + options: sample_options(), }; let bytes = req.to_bytes(); - for i in 0..bytes.len() { + let name_end = 1 + 4; + for i in name_end + 1..bytes.len() { assert!( CreateStreamRequest::decode(&bytes[..i]).is_err(), "expected error for truncation at byte {i}" ); } + let (decoded, _) = CreateStreamRequest::decode(&bytes[..name_end]).unwrap(); + assert!(decoded.options.is_empty()); } #[test] fn wire_compat_byte_layout() { let req = CreateStreamRequest { name: WireName::new("test").unwrap(), + options: WireOptions::empty(), }; let bytes = req.to_bytes(); assert_eq!(&bytes[..], &[4, b't', b'e', b's', b't']); diff --git a/core/binary_protocol/src/requests/streams/update_stream.rs b/core/binary_protocol/src/requests/streams/update_stream.rs index 5fa962981e..d36fadd161 100644 --- a/core/binary_protocol/src/requests/streams/update_stream.rs +++ b/core/binary_protocol/src/requests/streams/update_stream.rs @@ -19,23 +19,33 @@ use crate::WireError; use crate::WireIdentifier; use crate::codec::{WireDecode, WireEncode}; use crate::primitives::identifier::WireName; +use crate::primitives::options::WireOptions; use bytes::BytesMut; -/// `UpdateStream` request. Wire format: `[identifier][name_len:1][name:N]` +/// `UpdateStream` request. +/// +/// Wire format: `[identifier][name_len:1][name:N][options TLV to end]` +/// +/// The options block mirrors `CreateStream`'s, so a stream setting added to +/// the catalog is updatable without another layout change. Keys absent from +/// the block are left alone rather than reset: an update patches the stored +/// map, so a client built before a key existed cannot erase it. #[derive(Debug, Clone, PartialEq, Eq)] pub struct UpdateStreamRequest { pub stream_id: WireIdentifier, pub name: WireName, + pub options: WireOptions, } impl WireEncode for UpdateStreamRequest { fn encoded_size(&self) -> usize { - self.stream_id.encoded_size() + self.name.encoded_size() + self.stream_id.encoded_size() + self.name.encoded_size() + self.options.encoded_size() } fn encode(&self, buf: &mut BytesMut) { self.stream_id.encode(buf); self.name.encode(buf); + self.options.encode(buf); } } @@ -44,19 +54,35 @@ impl WireDecode for UpdateStreamRequest { let (stream_id, mut pos) = WireIdentifier::decode(buf)?; let (name, consumed) = WireName::decode(&buf[pos..])?; pos += consumed; - Ok((Self { stream_id, name }, pos)) + let options = WireOptions::from_slice(&buf[pos..])?; + Ok(( + Self { + stream_id, + name, + options, + }, + buf.len(), + )) } } #[cfg(test)] mod tests { use super::*; + use crate::primitives::user_headers::encode_user_headers; + + fn sample_options() -> WireOptions { + let mut buf = BytesMut::new(); + encode_user_headers(&[(2, b"future_key", 2, b"value")], &mut buf); + WireOptions::from_bytes(buf.freeze()).unwrap() + } #[test] fn roundtrip() { let req = UpdateStreamRequest { stream_id: WireIdentifier::named("old-name").unwrap(), name: WireName::new("new-name").unwrap(), + options: sample_options(), }; let bytes = req.to_bytes(); let (decoded, consumed) = UpdateStreamRequest::decode(&bytes).unwrap(); @@ -65,10 +91,11 @@ mod tests { } #[test] - fn truncated_returns_error() { + fn truncated_fixed_fields_return_error() { let req = UpdateStreamRequest { stream_id: WireIdentifier::numeric(1), name: WireName::new("test").unwrap(), + options: WireOptions::empty(), }; let bytes = req.to_bytes(); for i in 0..bytes.len() { @@ -78,4 +105,23 @@ mod tests { ); } } + + #[test] + fn truncated_options_return_error() { + // One pair, so any strict-interior truncation is invalid. Cutting at + // the block boundary is a legitimate update carrying no options. + let req = UpdateStreamRequest { + stream_id: WireIdentifier::numeric(1), + name: WireName::new("test").unwrap(), + options: sample_options(), + }; + let bytes = req.to_bytes(); + let fixed_end = bytes.len() - req.options.encoded_size(); + for i in fixed_end + 1..bytes.len() { + assert!( + UpdateStreamRequest::decode(&bytes[..i]).is_err(), + "expected error for truncation at byte {i}" + ); + } + } } diff --git a/core/binary_protocol/src/requests/system/describe_options.rs b/core/binary_protocol/src/requests/system/describe_options.rs new file mode 100644 index 0000000000..a01b7147f7 --- /dev/null +++ b/core/binary_protocol/src/requests/system/describe_options.rs @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::WireError; +use crate::codec::{WireDecode, WireEncode, read_u8}; +use bytes::{BufMut, BytesMut}; +use std::borrow::Cow; + +/// Resource whose option catalog is being described. +pub const OPTIONS_SCOPE_TOPIC: u8 = 1; +pub const OPTIONS_SCOPE_STREAM: u8 = 2; +pub const OPTIONS_SCOPE_USER: u8 = 3; + +/// `DescribeOptions` request. Wire format: `[scope:u8]` +/// +/// Because unknown option keys are rejected at create, a client cannot probe +/// support by trying; this read-only command serves the catalog instead. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DescribeOptionsRequest { + pub scope: u8, +} + +impl WireEncode for DescribeOptionsRequest { + fn encoded_size(&self) -> usize { + 1 + } + + fn encode(&self, buf: &mut BytesMut) { + buf.put_u8(self.scope); + } +} + +impl WireDecode for DescribeOptionsRequest { + fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> { + let scope = read_u8(buf, 0)?; + if !(OPTIONS_SCOPE_TOPIC..=OPTIONS_SCOPE_USER).contains(&scope) { + return Err(WireError::Validation(Cow::Owned(format!( + "unknown options scope: {scope}" + )))); + } + Ok((Self { scope }, 1)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn roundtrip() { + for scope in [ + OPTIONS_SCOPE_TOPIC, + OPTIONS_SCOPE_STREAM, + OPTIONS_SCOPE_USER, + ] { + let req = DescribeOptionsRequest { scope }; + let bytes = req.to_bytes(); + let (decoded, consumed) = DescribeOptionsRequest::decode(&bytes).unwrap(); + assert_eq!(consumed, bytes.len()); + assert_eq!(decoded, req); + } + } + + #[test] + fn unknown_scope_rejected() { + assert!(DescribeOptionsRequest::decode(&[0]).is_err()); + assert!(DescribeOptionsRequest::decode(&[4]).is_err()); + } + + #[test] + fn empty_buffer_rejected() { + assert!(DescribeOptionsRequest::decode(&[]).is_err()); + } +} diff --git a/core/binary_protocol/src/requests/system/mod.rs b/core/binary_protocol/src/requests/system/mod.rs index b490d39c96..c39bb2fb0e 100644 --- a/core/binary_protocol/src/requests/system/mod.rs +++ b/core/binary_protocol/src/requests/system/mod.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +pub mod describe_options; pub mod get_client; pub mod get_clients; pub mod get_cluster_metadata; @@ -23,6 +24,9 @@ pub mod get_snapshot; pub mod get_stats; pub mod ping; +pub use describe_options::{ + DescribeOptionsRequest, OPTIONS_SCOPE_STREAM, OPTIONS_SCOPE_TOPIC, OPTIONS_SCOPE_USER, +}; pub use get_client::GetClientRequest; pub use get_clients::GetClientsRequest; pub use get_cluster_metadata::GetClusterMetadataRequest; diff --git a/core/binary_protocol/src/requests/topics/create_topic.rs b/core/binary_protocol/src/requests/topics/create_topic.rs index 67cc5de17b..dadbc30117 100644 --- a/core/binary_protocol/src/requests/topics/create_topic.rs +++ b/core/binary_protocol/src/requests/topics/create_topic.rs @@ -17,41 +17,44 @@ use crate::WireError; use crate::WireIdentifier; -use crate::codec::{WireDecode, WireEncode, read_u8, read_u32_le, read_u64_le}; +use crate::codec::{WireDecode, WireEncode, read_u32_le}; use crate::primitives::identifier::WireName; +use crate::primitives::options::WireOptions; use bytes::{BufMut, BytesMut}; /// `CreateTopic` request. /// /// Wire format: -/// `[stream_id:WireIdentifier][partitions_count:u32_le][compression_algorithm:u8] -/// [message_expiry:u64_le][max_topic_size:u64_le][replication_factor:u8][name_len:u8][name:N]` +/// `[stream_id:WireIdentifier][partitions_count:u32_le][name_len:u8][name:N][options TLV to end]` +/// +/// The fixed fields are the shape of the operation itself: which stream, how +/// many partitions to allocate, and what to call the topic. `partitions_count` +/// is an argument, not a setting -- admission consumes it to compute the +/// partition assignments and it is deliberately never persisted as an option +/// (`CreatePartitions` would make a stored count stale). +/// +/// Every actual topic SETTING -- expiry, size caps, segment sizing, durability +/// and any future knob -- rides the options block, so adding a knob never +/// changes this layout. The block runs to the end of the payload; its +/// validator requires exact consumption, so no length prefix is needed. #[derive(Debug, Clone, PartialEq, Eq)] pub struct CreateTopicRequest { pub stream_id: WireIdentifier, pub partitions_count: u32, - pub compression_algorithm: u8, - pub message_expiry: u64, - pub max_topic_size: u64, - pub replication_factor: u8, pub name: WireName, + pub options: WireOptions, } -const FIXED_FIELDS_SIZE: usize = 4 + 1 + 8 + 8 + 1; // 22 bytes - impl WireEncode for CreateTopicRequest { fn encoded_size(&self) -> usize { - self.stream_id.encoded_size() + FIXED_FIELDS_SIZE + self.name.encoded_size() + self.stream_id.encoded_size() + 4 + self.name.encoded_size() + self.options.encoded_size() } fn encode(&self, buf: &mut BytesMut) { self.stream_id.encode(buf); buf.put_u32_le(self.partitions_count); - buf.put_u8(self.compression_algorithm); - buf.put_u64_le(self.message_expiry); - buf.put_u64_le(self.max_topic_size); - buf.put_u8(self.replication_factor); self.name.encode(buf); + self.options.encode(buf); } } @@ -60,27 +63,17 @@ impl WireDecode for CreateTopicRequest { let (stream_id, mut pos) = WireIdentifier::decode(buf)?; let partitions_count = read_u32_le(buf, pos)?; pos += 4; - let compression_algorithm = read_u8(buf, pos)?; - pos += 1; - let message_expiry = read_u64_le(buf, pos)?; - pos += 8; - let max_topic_size = read_u64_le(buf, pos)?; - pos += 8; - let replication_factor = read_u8(buf, pos)?; - pos += 1; let (name, consumed) = WireName::decode(&buf[pos..])?; pos += consumed; + let options = WireOptions::from_slice(&buf[pos..])?; Ok(( Self { stream_id, partitions_count, - compression_algorithm, - message_expiry, - max_topic_size, - replication_factor, name, + options, }, - pos, + buf.len(), )) } } @@ -88,16 +81,26 @@ impl WireDecode for CreateTopicRequest { #[cfg(test)] mod tests { use super::*; + use crate::primitives::user_headers::encode_user_headers; + + fn sample_options() -> WireOptions { + let mut buf = BytesMut::new(); + encode_user_headers( + &[ + (2, b"message_expiry", 2, b"7 days"), + (2, b"max_topic_size", 12, &1024u64.to_le_bytes()), + ], + &mut buf, + ); + WireOptions::from_bytes(buf.freeze()).unwrap() + } fn sample_request() -> CreateTopicRequest { CreateTopicRequest { stream_id: WireIdentifier::numeric(1), partitions_count: 3, - compression_algorithm: 1, - message_expiry: 3600, - max_topic_size: 1_000_000, - replication_factor: 1, name: WireName::new("orders").unwrap(), + options: sample_options(), } } @@ -111,15 +114,12 @@ mod tests { } #[test] - fn roundtrip_named_stream() { + fn roundtrip_without_options() { let req = CreateTopicRequest { stream_id: WireIdentifier::named("my-stream").unwrap(), - partitions_count: 10, - compression_algorithm: 2, - message_expiry: 0, - max_topic_size: u64::MAX, - replication_factor: 3, + partitions_count: 1, name: WireName::new("events").unwrap(), + options: WireOptions::empty(), }; let bytes = req.to_bytes(); let (decoded, consumed) = CreateTopicRequest::decode(&bytes).unwrap(); @@ -128,8 +128,38 @@ mod tests { } #[test] - fn truncated_returns_error() { - let req = sample_request(); + fn truncated_options_return_error() { + // A single key-value pair: any strict-interior truncation of the + // block is structurally invalid (a multi-pair block truncates + // cleanly at pair boundaries). + let mut buf = BytesMut::new(); + encode_user_headers(&[(2, b"message_expiry", 2, b"7 days")], &mut buf); + let req = CreateTopicRequest { + stream_id: WireIdentifier::numeric(1), + partitions_count: 2, + name: WireName::new("orders").unwrap(), + options: WireOptions::from_bytes(buf.freeze()).unwrap(), + }; + let bytes = req.to_bytes(); + let fixed_end = bytes.len() - req.options.encoded_size(); + for i in fixed_end + 1..bytes.len() { + assert!( + CreateTopicRequest::decode(&bytes[..i]).is_err(), + "expected error for truncation at byte {i}" + ); + } + let (decoded, _) = CreateTopicRequest::decode(&bytes[..fixed_end]).unwrap(); + assert!(decoded.options.is_empty()); + } + + #[test] + fn truncated_fixed_fields_return_error() { + let req = CreateTopicRequest { + stream_id: WireIdentifier::numeric(1), + partitions_count: 1, + name: WireName::new("orders").unwrap(), + options: WireOptions::empty(), + }; let bytes = req.to_bytes(); for i in 0..bytes.len() { assert!( diff --git a/core/binary_protocol/src/requests/topics/create_topic_with_assignments.rs b/core/binary_protocol/src/requests/topics/create_topic_with_assignments.rs index b5199abb33..b6665b6188 100644 --- a/core/binary_protocol/src/requests/topics/create_topic_with_assignments.rs +++ b/core/binary_protocol/src/requests/topics/create_topic_with_assignments.rs @@ -17,6 +17,7 @@ use crate::WireError; use crate::codec::{WireDecode, WireEncode, read_u32_le}; +use crate::primitives::options::{WireOptions, decode_options_prefixed}; use crate::primitives::partition_assignment::CreatedPartitionAssignment; use crate::requests::topics::CreateTopicRequest; use bytes::{BufMut, BytesMut}; @@ -25,15 +26,25 @@ fn usize_to_u32(value: usize, context: &str) -> u32 { u32::try_from(value).unwrap_or_else(|_| panic!("{context} exceeds u32")) } +/// Internal create-topic form the admitting primary replicates. +/// +/// `request` carries the client's explicit options verbatim; `derived_options` +/// carries the values the primary resolved from server defaults for keys the +/// client did not send. Splitting the two preserves per-key provenance across +/// replication, so `GetTopic` can distinguish client-pinned settings from +/// defaults filled at creation. #[derive(Debug, Clone, PartialEq, Eq)] pub struct CreateTopicWithAssignmentsRequest { pub request: CreateTopicRequest, + pub derived_options: WireOptions, pub partitions: Vec, } impl WireEncode for CreateTopicWithAssignmentsRequest { fn encoded_size(&self) -> usize { 4 + self.request.encoded_size() + + 4 + + self.derived_options.encoded_size() + 4 + self .partitions @@ -48,6 +59,11 @@ impl WireEncode for CreateTopicWithAssignmentsRequest { "create topic request size", )); self.request.encode(buf); + buf.put_u32_le(usize_to_u32( + self.derived_options.encoded_size(), + "create topic derived options size", + )); + self.derived_options.encode(buf); buf.put_u32_le(usize_to_u32( self.partitions.len(), "create topic partition count", @@ -61,8 +77,19 @@ impl WireEncode for CreateTopicWithAssignmentsRequest { impl WireDecode for CreateTopicWithAssignmentsRequest { fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> { let request_size = read_u32_le(buf, 0)? as usize; - let request_start = 4; - let request_end = request_start + request_size; + let request_start: usize = 4; + // `checked_add`: on a 32-bit target a wire-supplied length wraps, the + // truncation guard below then passes, and the slice panics instead. + // This decode runs on journal replay, so that panic would take a + // replica down rather than fail one client request. + let request_end = + request_start + .checked_add(request_size) + .ok_or_else(|| WireError::UnexpectedEof { + offset: request_start, + need: request_size, + have: buf.len().saturating_sub(request_start), + })?; if buf.len() < request_end { return Err(WireError::UnexpectedEof { offset: request_start, @@ -72,9 +99,17 @@ impl WireDecode for CreateTopicWithAssignmentsRequest { } let request = CreateTopicRequest::decode_from(&buf[request_start..request_end])?; - let partitions_count = read_u32_le(buf, request_end)? as usize; - let mut offset = request_end + 4; - let mut partitions = Vec::with_capacity(partitions_count); + + let (derived_options, derived_consumed) = decode_options_prefixed(buf, request_end)?; + let derived_end = request_end + derived_consumed; + + let partitions_count = read_u32_le(buf, derived_end)? as usize; + let mut offset = derived_end + 4; + let mut partitions = Vec::with_capacity(crate::codec::bounded_capacity( + partitions_count, + buf.len().saturating_sub(offset), + CreatedPartitionAssignment::ENCODED_SIZE, + )); for _ in 0..partitions_count { let (partition, consumed) = CreatedPartitionAssignment::decode(&buf[offset..])?; offset += consumed; @@ -84,6 +119,7 @@ impl WireDecode for CreateTopicWithAssignmentsRequest { Ok(( Self { request, + derived_options, partitions, }, offset, @@ -97,8 +133,17 @@ mod tests { use crate::WireIdentifier; use crate::codec::{WireDecode, WireEncode}; use crate::primitives::identifier::WireName; + use crate::primitives::options::WireOptions; use crate::primitives::partition_assignment::CreatedPartitionAssignment; + use crate::primitives::user_headers::encode_user_headers; use crate::requests::topics::CreateTopicRequest; + use bytes::BytesMut; + + fn options_block(entries: &[(u8, &[u8], u8, &[u8])]) -> WireOptions { + let mut buf = BytesMut::new(); + encode_user_headers(entries, &mut buf); + WireOptions::from_bytes(buf.freeze()).unwrap() + } #[test] fn roundtrip() { @@ -106,12 +151,10 @@ mod tests { request: CreateTopicRequest { stream_id: WireIdentifier::numeric(1), partitions_count: 2, - compression_algorithm: 1, - message_expiry: 3600, - max_topic_size: 1024, - replication_factor: 1, name: WireName::new("events").unwrap(), + options: options_block(&[(2, b"message_expiry", 2, b"7 days")]), }, + derived_options: options_block(&[(2, b"max_topic_size", 12, &1024u64.to_le_bytes())]), partitions: vec![ CreatedPartitionAssignment { partition_id: 0, @@ -128,4 +171,22 @@ mod tests { assert_eq!(consumed, bytes.len()); assert_eq!(decoded, request); } + + #[test] + fn roundtrip_with_empty_blocks() { + let request = CreateTopicWithAssignmentsRequest { + request: CreateTopicRequest { + stream_id: WireIdentifier::numeric(7), + partitions_count: 0, + name: WireName::new("bare").unwrap(), + options: WireOptions::empty(), + }, + derived_options: WireOptions::empty(), + partitions: vec![], + }; + let bytes = request.to_bytes(); + let (decoded, consumed) = CreateTopicWithAssignmentsRequest::decode(&bytes).unwrap(); + assert_eq!(consumed, bytes.len()); + assert_eq!(decoded, request); + } } diff --git a/core/binary_protocol/src/requests/topics/update_topic.rs b/core/binary_protocol/src/requests/topics/update_topic.rs index f7151242b8..902971b375 100644 --- a/core/binary_protocol/src/requests/topics/update_topic.rs +++ b/core/binary_protocol/src/requests/topics/update_topic.rs @@ -17,44 +17,48 @@ use crate::WireError; use crate::WireIdentifier; -use crate::codec::{WireDecode, WireEncode, read_u8, read_u64_le}; +use crate::codec::{WireDecode, WireEncode}; use crate::primitives::identifier::WireName; -use bytes::{BufMut, BytesMut}; +use crate::primitives::options::WireOptions; +use bytes::BytesMut; /// `UpdateTopic` request. /// /// Wire format: -/// `[stream_id:WireIdentifier][topic_id:WireIdentifier][compression_algorithm:u8] -/// [message_expiry:u64_le][max_topic_size:u64_le][replication_factor:u8][name_len:u8][name:N]` +/// `[stream_id:WireIdentifier][topic_id:WireIdentifier][name_len:u8][name:N] +/// [options TLV to end]` +/// +/// Identity and the new name are the only fixed fields; every SETTING rides the +/// options block, which mirrors `CreateTopic`'s and carries the same catalog. +/// A knob added there is updatable here without another layout change, and no +/// setting has two homes to disagree between. +/// +/// Keys absent from the block are LEFT ALONE rather than reset to their +/// defaults. A client built before a key existed cannot send it, so treating +/// the block as the topic's complete option set would let an old client wipe a +/// newer knob just by updating the name -- the same forward-compatibility +/// argument that makes unknown keys survive a round trip. #[derive(Debug, Clone, PartialEq, Eq)] pub struct UpdateTopicRequest { pub stream_id: WireIdentifier, pub topic_id: WireIdentifier, - pub compression_algorithm: u8, - pub message_expiry: u64, - pub max_topic_size: u64, - pub replication_factor: u8, pub name: WireName, + pub options: WireOptions, } -const FIXED_FIELDS_SIZE: usize = 1 + 8 + 8 + 1; // 18 bytes - impl WireEncode for UpdateTopicRequest { fn encoded_size(&self) -> usize { self.stream_id.encoded_size() + self.topic_id.encoded_size() - + FIXED_FIELDS_SIZE + self.name.encoded_size() + + self.options.encoded_size() } fn encode(&self, buf: &mut BytesMut) { self.stream_id.encode(buf); self.topic_id.encode(buf); - buf.put_u8(self.compression_algorithm); - buf.put_u64_le(self.message_expiry); - buf.put_u64_le(self.max_topic_size); - buf.put_u8(self.replication_factor); self.name.encode(buf); + self.options.encode(buf); } } @@ -63,27 +67,17 @@ impl WireDecode for UpdateTopicRequest { let (stream_id, mut pos) = WireIdentifier::decode(buf)?; let (topic_id, consumed) = WireIdentifier::decode(&buf[pos..])?; pos += consumed; - let compression_algorithm = read_u8(buf, pos)?; - pos += 1; - let message_expiry = read_u64_le(buf, pos)?; - pos += 8; - let max_topic_size = read_u64_le(buf, pos)?; - pos += 8; - let replication_factor = read_u8(buf, pos)?; - pos += 1; let (name, name_consumed) = WireName::decode(&buf[pos..])?; pos += name_consumed; + let options = WireOptions::from_slice(&buf[pos..])?; Ok(( Self { stream_id, topic_id, - compression_algorithm, - message_expiry, - max_topic_size, - replication_factor, name, + options, }, - pos, + buf.len(), )) } } @@ -91,16 +85,20 @@ impl WireDecode for UpdateTopicRequest { #[cfg(test)] mod tests { use super::*; + use crate::primitives::user_headers::encode_user_headers; + + fn sample_options() -> WireOptions { + let mut buf = BytesMut::new(); + encode_user_headers(&[(2, b"future_option", 9, &[3u8])], &mut buf); + WireOptions::from_bytes(buf.freeze()).unwrap() + } fn sample_request() -> UpdateTopicRequest { UpdateTopicRequest { stream_id: WireIdentifier::numeric(1), topic_id: WireIdentifier::numeric(2), - compression_algorithm: 1, - message_expiry: 7200, - max_topic_size: 500_000, - replication_factor: 2, name: WireName::new("updated-topic").unwrap(), + options: sample_options(), } } @@ -118,11 +116,8 @@ mod tests { let req = UpdateTopicRequest { stream_id: WireIdentifier::named("stream-a").unwrap(), topic_id: WireIdentifier::named("topic-b").unwrap(), - compression_algorithm: 0, - message_expiry: 0, - max_topic_size: u64::MAX, - replication_factor: 1, name: WireName::new("new-name").unwrap(), + options: WireOptions::empty(), }; let bytes = req.to_bytes(); let (decoded, consumed) = UpdateTopicRequest::decode(&bytes).unwrap(); @@ -131,9 +126,28 @@ mod tests { } #[test] - fn truncated_returns_error() { + fn truncated_options_return_error() { + // One key-value pair, so any strict-interior truncation of the block + // is structurally invalid. Cutting at the block boundary is NOT an + // error: that is a legitimate update carrying no options. let req = sample_request(); let bytes = req.to_bytes(); + let fixed_end = bytes.len() - req.options.encoded_size(); + for i in fixed_end + 1..bytes.len() { + assert!( + UpdateTopicRequest::decode(&bytes[..i]).is_err(), + "expected error for truncation at byte {i}" + ); + } + } + + #[test] + fn truncated_fixed_fields_return_error() { + let req = UpdateTopicRequest { + options: WireOptions::empty(), + ..sample_request() + }; + let bytes = req.to_bytes(); for i in 0..bytes.len() { assert!( UpdateTopicRequest::decode(&bytes[..i]).is_err(), diff --git a/core/binary_protocol/src/requests/users/create_user.rs b/core/binary_protocol/src/requests/users/create_user.rs index 2cb6634ed8..d58ee49852 100644 --- a/core/binary_protocol/src/requests/users/create_user.rs +++ b/core/binary_protocol/src/requests/users/create_user.rs @@ -18,6 +18,7 @@ use crate::WireError; use crate::codec::{WireDecode, WireEncode, read_str, read_u8, read_u32_le}; use crate::primitives::identifier::WireName; +use crate::primitives::options::WireOptions; use crate::primitives::permissions::WirePermissions; use bytes::{BufMut, BytesMut}; use std::borrow::Cow; @@ -26,13 +27,18 @@ use std::borrow::Cow; /// /// Wire format: /// `[username_len:u8][username:N][password_len:u8][password:N][status:u8] -/// [has_permissions:u8][permissions_len:u32_le?][permissions:M?]` +/// [has_permissions:u8][permissions_len:u32_le?][permissions:M?] +/// [options TLV to end]` +/// +/// The options block runs to the end of the payload; its validator requires +/// exact consumption, so no length prefix is needed. #[derive(Debug, Clone, PartialEq, Eq)] pub struct CreateUserRequest { pub username: WireName, pub password: String, pub status: u8, pub permissions: Option, + pub options: WireOptions, } impl WireEncode for CreateUserRequest { @@ -46,6 +52,7 @@ impl WireEncode for CreateUserRequest { .permissions .as_ref() .map_or(0, |p| 4 + p.encoded_size()) + + self.options.encoded_size() } fn encode(&self, buf: &mut BytesMut) { @@ -67,6 +74,7 @@ impl WireEncode for CreateUserRequest { } else { buf.put_u8(0); } + self.options.encode(buf); } } @@ -100,14 +108,17 @@ impl WireDecode for CreateUserRequest { None }; + let options = WireOptions::from_slice(&buf[pos..])?; + Ok(( Self { username, password, status, permissions, + options, }, - pos, + buf.len(), )) } } @@ -135,6 +146,13 @@ mod tests { } } + fn sample_options() -> WireOptions { + use crate::primitives::user_headers::encode_user_headers; + let mut buf = BytesMut::new(); + encode_user_headers(&[(2, b"future_key", 2, b"value")], &mut buf); + WireOptions::from_bytes(buf.freeze()).unwrap() + } + #[test] fn roundtrip_without_permissions() { let req = CreateUserRequest { @@ -142,6 +160,7 @@ mod tests { password: "secret123".to_string(), status: 1, permissions: None, + options: WireOptions::empty(), }; let bytes = req.to_bytes(); let (decoded, consumed) = CreateUserRequest::decode(&bytes).unwrap(); @@ -156,6 +175,22 @@ mod tests { password: "p@ssw0rd".to_string(), status: 2, permissions: Some(sample_permissions()), + options: WireOptions::empty(), + }; + let bytes = req.to_bytes(); + let (decoded, consumed) = CreateUserRequest::decode(&bytes).unwrap(); + assert_eq!(consumed, bytes.len()); + assert_eq!(decoded, req); + } + + #[test] + fn roundtrip_with_permissions_and_options() { + let req = CreateUserRequest { + username: WireName::new("admin").unwrap(), + password: "p@ssw0rd".to_string(), + status: 2, + permissions: Some(sample_permissions()), + options: sample_options(), }; let bytes = req.to_bytes(); let (decoded, consumed) = CreateUserRequest::decode(&bytes).unwrap(); @@ -170,6 +205,7 @@ mod tests { password: "pw".to_string(), status: 0, permissions: Some(sample_permissions()), + options: sample_options(), }; assert_eq!(req.encoded_size(), req.to_bytes().len()); } @@ -181,6 +217,7 @@ mod tests { password: "pass".to_string(), status: 1, permissions: None, + options: WireOptions::empty(), }; let bytes = req.to_bytes(); for i in 0..bytes.len() { @@ -191,6 +228,27 @@ mod tests { } } + #[test] + fn truncated_options_return_error() { + let req = CreateUserRequest { + username: WireName::new("user").unwrap(), + password: "pass".to_string(), + status: 1, + permissions: None, + options: sample_options(), + }; + let bytes = req.to_bytes(); + let fixed_end = bytes.len() - req.options.encoded_size(); + for i in fixed_end + 1..bytes.len() { + assert!( + CreateUserRequest::decode(&bytes[..i]).is_err(), + "expected error for truncation at byte {i}" + ); + } + let (decoded, _) = CreateUserRequest::decode(&bytes[..fixed_end]).unwrap(); + assert!(decoded.options.is_empty()); + } + #[test] fn none_permissions_wire_layout() { let req = CreateUserRequest { @@ -198,10 +256,11 @@ mod tests { password: "p".to_string(), status: 0, permissions: None, + options: WireOptions::empty(), }; let bytes = req.to_bytes(); // username: [1, b'u'] + password: [1, b'p'] + status: [0] - // + has_perm: [0] + // + has_perm: [0]; empty options add zero bytes let expected: &[u8] = &[1, b'u', 1, b'p', 0, 0]; assert_eq!(&bytes[..], expected); } diff --git a/core/binary_protocol/src/requests/users/update_user.rs b/core/binary_protocol/src/requests/users/update_user.rs index 4e2d70aa17..125e1df02d 100644 --- a/core/binary_protocol/src/requests/users/update_user.rs +++ b/core/binary_protocol/src/requests/users/update_user.rs @@ -19,18 +19,25 @@ use crate::WireError; use crate::WireIdentifier; use crate::codec::{WireDecode, WireEncode, read_u8}; use crate::primitives::identifier::WireName; +use crate::primitives::options::WireOptions; use bytes::{BufMut, BytesMut}; /// `UpdateUser` request. /// /// Wire format: /// `[user_id:WireIdentifier][has_username:u8][username_len:u8?][username:N?] -/// [has_status:u8][status:u8?]` +/// [has_status:u8][status:u8?][options TLV to end]` +/// +/// The options block mirrors `CreateUser`'s, so a user setting added to the +/// catalog is updatable without another layout change. Keys absent from the +/// block are left alone rather than reset: an update patches the stored map, +/// so a client built before a key existed cannot erase it. #[derive(Debug, Clone, PartialEq, Eq)] pub struct UpdateUserRequest { pub user_id: WireIdentifier, pub username: Option, pub status: Option, + pub options: WireOptions, } impl WireEncode for UpdateUserRequest { @@ -40,6 +47,7 @@ impl WireEncode for UpdateUserRequest { + self.username.as_ref().map_or(0, WireEncode::encoded_size) + 1 // has_status + self.status.map_or(0, |_| 1) + + self.options.encoded_size() } fn encode(&self, buf: &mut BytesMut) { @@ -62,6 +70,7 @@ impl WireEncode for UpdateUserRequest { buf.put_u8(0); } } + self.options.encode(buf); } } @@ -89,13 +98,15 @@ impl WireDecode for UpdateUserRequest { None }; + let options = WireOptions::from_slice(&buf[pos..])?; Ok(( Self { user_id, username, status, + options, }, - pos, + buf.len(), )) } } @@ -103,6 +114,13 @@ impl WireDecode for UpdateUserRequest { #[cfg(test)] mod tests { use super::*; + use crate::primitives::user_headers::encode_user_headers; + + fn sample_options() -> WireOptions { + let mut buf = BytesMut::new(); + encode_user_headers(&[(2, b"future_key", 2, b"value")], &mut buf); + WireOptions::from_bytes(buf.freeze()).unwrap() + } #[test] fn roundtrip_both_present() { @@ -110,6 +128,7 @@ mod tests { user_id: WireIdentifier::numeric(1), username: Some(WireName::new("new-name").unwrap()), status: Some(2), + options: WireOptions::empty(), }; let bytes = req.to_bytes(); let (decoded, consumed) = UpdateUserRequest::decode(&bytes).unwrap(); @@ -123,6 +142,7 @@ mod tests { user_id: WireIdentifier::numeric(5), username: None, status: None, + options: WireOptions::empty(), }; let bytes = req.to_bytes(); let (decoded, consumed) = UpdateUserRequest::decode(&bytes).unwrap(); @@ -136,6 +156,7 @@ mod tests { user_id: WireIdentifier::named("admin").unwrap(), username: Some(WireName::new("super-admin").unwrap()), status: None, + options: WireOptions::empty(), }; let bytes = req.to_bytes(); let (decoded, consumed) = UpdateUserRequest::decode(&bytes).unwrap(); @@ -149,6 +170,7 @@ mod tests { user_id: WireIdentifier::numeric(10), username: None, status: Some(0), + options: WireOptions::empty(), }; let bytes = req.to_bytes(); let (decoded, consumed) = UpdateUserRequest::decode(&bytes).unwrap(); @@ -162,16 +184,38 @@ mod tests { user_id: WireIdentifier::numeric(1), username: Some(WireName::new("test").unwrap()), status: Some(1), + options: WireOptions::empty(), }; assert_eq!(req.encoded_size(), req.to_bytes().len()); } + #[test] + fn truncated_options_return_error() { + // One pair, so any strict-interior truncation is invalid. Cutting at + // the block boundary is a legitimate update carrying no options. + let req = UpdateUserRequest { + user_id: WireIdentifier::numeric(1), + username: Some(WireName::new("name").unwrap()), + status: Some(1), + options: sample_options(), + }; + let bytes = req.to_bytes(); + let fixed_end = bytes.len() - req.options.encoded_size(); + for i in fixed_end + 1..bytes.len() { + assert!( + UpdateUserRequest::decode(&bytes[..i]).is_err(), + "expected error for truncation at byte {i}" + ); + } + } + #[test] fn truncated_returns_error() { let req = UpdateUserRequest { user_id: WireIdentifier::numeric(1), username: Some(WireName::new("name").unwrap()), status: Some(1), + options: WireOptions::empty(), }; let bytes = req.to_bytes(); for i in 0..bytes.len() { diff --git a/core/binary_protocol/src/responses/clients/get_client.rs b/core/binary_protocol/src/responses/clients/get_client.rs index 3fa4ffedc0..aa721c0c34 100644 --- a/core/binary_protocol/src/responses/clients/get_client.rs +++ b/core/binary_protocol/src/responses/clients/get_client.rs @@ -57,7 +57,7 @@ impl WireDecode for ClientDetailsResponse { let (client, mut pos) = ClientResponse::decode(buf)?; let count = client.consumer_groups_count as usize; let remaining = buf.len().saturating_sub(pos); - let mut consumer_groups = Vec::with_capacity(crate::codec::capped_capacity( + let mut consumer_groups = Vec::with_capacity(crate::codec::bounded_capacity( count, remaining, ConsumerGroupInfoResponse::SIZE, diff --git a/core/binary_protocol/src/responses/consumer_groups/get_consumer_group.rs b/core/binary_protocol/src/responses/consumer_groups/get_consumer_group.rs index 551cf0da16..cc4f175f33 100644 --- a/core/binary_protocol/src/responses/consumer_groups/get_consumer_group.rs +++ b/core/binary_protocol/src/responses/consumer_groups/get_consumer_group.rs @@ -52,7 +52,7 @@ impl WireDecode for ConsumerGroupMemberResponse { let id = read_u32_le(buf, 0)?; let partitions_count = read_u32_le(buf, 4)?; let remaining = buf.len().saturating_sub(8); - let mut partitions = Vec::with_capacity(crate::codec::capped_capacity( + let mut partitions = Vec::with_capacity(crate::codec::bounded_capacity( partitions_count as usize, remaining, 4, diff --git a/core/binary_protocol/src/responses/consumer_groups/sync_consumer_group.rs b/core/binary_protocol/src/responses/consumer_groups/sync_consumer_group.rs index 9f9c3283a5..3fe5d65513 100644 --- a/core/binary_protocol/src/responses/consumer_groups/sync_consumer_group.rs +++ b/core/binary_protocol/src/responses/consumer_groups/sync_consumer_group.rs @@ -16,7 +16,7 @@ // under the License. use crate::WireError; -use crate::codec::{WireDecode, WireEncode, capped_capacity, read_u32_le, read_u64_le}; +use crate::codec::{WireDecode, WireEncode, bounded_capacity, read_u32_le, read_u64_le}; use bytes::{BufMut, BytesMut}; /// `SyncConsumerGroup` response. @@ -53,7 +53,7 @@ impl WireDecode for SyncConsumerGroupResponse { let partitions_count = read_u32_le(buf, 8)?; let remaining = buf.len().saturating_sub(12); let mut partitions = - Vec::with_capacity(capped_capacity(partitions_count as usize, remaining, 4)); + Vec::with_capacity(bounded_capacity(partitions_count as usize, remaining, 4)); let mut offset = 12; for _ in 0..partitions_count { partitions.push(read_u32_le(buf, offset)?); diff --git a/core/binary_protocol/src/responses/messages/send_messages.rs b/core/binary_protocol/src/responses/messages/send_messages.rs index 22a7a4624f..32785b7be3 100644 --- a/core/binary_protocol/src/responses/messages/send_messages.rs +++ b/core/binary_protocol/src/responses/messages/send_messages.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use crate::codec::{WireDecode, WireEncode, capped_capacity, read_u32_le, read_u64_le}; +use crate::codec::{WireDecode, WireEncode, bounded_capacity, read_u32_le, read_u64_le}; use crate::error::WireError; use bytes::{BufMut, BytesMut}; use std::borrow::Cow; @@ -122,7 +122,7 @@ impl WireDecode for SendMessagesResponse { fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> { let confirmations_count = read_u32_le(buf, 0)?; let remaining = buf.len().saturating_sub(4); - let mut confirmations = Vec::with_capacity(capped_capacity( + let mut confirmations = Vec::with_capacity(bounded_capacity( confirmations_count as usize, remaining, CONFIRMATION_SIZE, diff --git a/core/binary_protocol/src/responses/streams/get_stream.rs b/core/binary_protocol/src/responses/streams/get_stream.rs index 25aae51776..8966de31d9 100644 --- a/core/binary_protocol/src/responses/streams/get_stream.rs +++ b/core/binary_protocol/src/responses/streams/get_stream.rs @@ -18,18 +18,26 @@ use crate::WireError; use crate::codec::{WireDecode, WireEncode, read_u8, read_u32_le, read_u64_le}; use crate::primitives::identifier::WireName; +use crate::primitives::options::{ + WireOptions, decode_options_prefixed, encode_options_prefixed, options_prefixed_size, +}; use crate::responses::streams::StreamResponse; use bytes::{BufMut, BytesMut}; -use std::borrow::Cow; /// Topic header within a `GetStream` response. /// -/// Wire format (51 + `name_len` bytes): +/// Wire format (50 + `name_len` + options bytes): /// ```text /// [id:4][created_at:8][partitions_count:4][message_expiry:8] -/// [compression_algorithm:1][max_topic_size:8][replication_factor:1] +/// [compression_algorithm:1][max_topic_size:8] /// [size_bytes:8][messages_count:8][name_len:1][name:N] +/// [explicit_options_len:4][explicit options TLV] +/// [derived_options_len:4][derived options TLV] /// ``` +/// +/// `options` are the keys the client pinned at create; `derived_options` are +/// the defaults the admitting primary resolved. A round-tripping client +/// re-sends only `options`; the effective configuration is the union. #[derive(Debug, Clone, PartialEq, Eq)] pub struct TopicHeader { pub id: u32, @@ -38,19 +46,23 @@ pub struct TopicHeader { pub message_expiry: u64, pub compression_algorithm: u8, pub max_topic_size: u64, - pub replication_factor: u8, pub size_bytes: u64, pub messages_count: u64, pub name: WireName, + pub options: WireOptions, + pub derived_options: WireOptions, } impl TopicHeader { - const FIXED_SIZE: usize = 4 + 8 + 4 + 8 + 1 + 8 + 1 + 8 + 8 + 1; // 51 + pub(crate) const FIXED_SIZE: usize = 4 + 8 + 4 + 8 + 1 + 8 + 8 + 8 + 1; // 50 } impl WireEncode for TopicHeader { fn encoded_size(&self) -> usize { - Self::FIXED_SIZE + self.name.len() + Self::FIXED_SIZE + + self.name.len() + + options_prefixed_size(&self.options) + + options_prefixed_size(&self.derived_options) } fn encode(&self, buf: &mut BytesMut) { @@ -60,10 +72,11 @@ impl WireEncode for TopicHeader { buf.put_u64_le(self.message_expiry); buf.put_u8(self.compression_algorithm); buf.put_u64_le(self.max_topic_size); - buf.put_u8(self.replication_factor); buf.put_u64_le(self.size_bytes); buf.put_u64_le(self.messages_count); self.name.encode(buf); + encode_options_prefixed(&self.options, buf); + encode_options_prefixed(&self.derived_options, buf); } } @@ -75,11 +88,14 @@ impl WireDecode for TopicHeader { let message_expiry = read_u64_le(buf, 16)?; let compression_algorithm = read_u8(buf, 24)?; let max_topic_size = read_u64_le(buf, 25)?; - let replication_factor = read_u8(buf, 33)?; - let size_bytes = read_u64_le(buf, 34)?; - let messages_count = read_u64_le(buf, 42)?; - let (name, name_consumed) = WireName::decode(&buf[50..])?; - let consumed = 50 + name_consumed; + let size_bytes = read_u64_le(buf, 33)?; + let messages_count = read_u64_le(buf, 41)?; + let (name, name_consumed) = WireName::decode(&buf[49..])?; + let mut consumed = 49 + name_consumed; + let (options, options_consumed) = decode_options_prefixed(buf, consumed)?; + consumed += options_consumed; + let (derived_options, derived_consumed) = decode_options_prefixed(buf, consumed)?; + consumed += derived_consumed; Ok(( Self { @@ -89,10 +105,11 @@ impl WireDecode for TopicHeader { message_expiry, compression_algorithm, max_topic_size, - replication_factor, size_bytes, messages_count, name, + options, + derived_options, }, consumed, )) @@ -132,22 +149,25 @@ impl WireEncode for GetStreamResponse { } } +/// Smallest a topic header can encode as: the fixed ids, timestamps, sizes and +/// counts, plus a name length and two length-prefixed option blocks. +const MIN_TOPIC_HEADER_SIZE: usize = 45; + impl WireDecode for GetStreamResponse { fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> { let (stream, mut pos) = StreamResponse::decode(buf)?; - let mut topics = Vec::new(); - while pos < buf.len() { + // Count-driven: a topic element carries variable-length options + // blocks, so "consume until the buffer ends" no longer delimits it. + let mut topics = Vec::with_capacity(crate::codec::bounded_capacity( + stream.topics_count as usize, + buf.len().saturating_sub(pos), + MIN_TOPIC_HEADER_SIZE, + )); + for _ in 0..stream.topics_count { let (topic, consumed) = TopicHeader::decode(&buf[pos..])?; pos += consumed; topics.push(topic); } - if topics.len() != stream.topics_count as usize { - return Err(WireError::Validation(Cow::Owned(format!( - "stream.topics_count={} but decoded {} topics", - stream.topics_count, - topics.len() - )))); - } Ok((Self { stream, topics }, pos)) } } @@ -164,6 +184,7 @@ mod tests { size_bytes: 2048, messages_count: 200, name: WireName::new("my-stream").unwrap(), + options: WireOptions::empty(), } } @@ -175,10 +196,11 @@ mod tests { message_expiry: 0, compression_algorithm: 1, max_topic_size: 0, - replication_factor: 1, size_bytes: 1024, messages_count: 100, name: WireName::new(name).unwrap(), + options: WireOptions::empty(), + derived_options: WireOptions::empty(), } } @@ -213,7 +235,7 @@ mod tests { fn topic_header_roundtrip() { let topic = sample_topic(5, "events"); let bytes = topic.to_bytes(); - assert_eq!(bytes.len(), TopicHeader::FIXED_SIZE + 6); + assert_eq!(bytes.len(), TopicHeader::FIXED_SIZE + 6 + 4 + 4); let (decoded, consumed) = TopicHeader::decode(&bytes).unwrap(); assert_eq!(consumed, bytes.len()); assert_eq!(decoded, topic); diff --git a/core/binary_protocol/src/responses/streams/get_streams.rs b/core/binary_protocol/src/responses/streams/get_streams.rs index 3221c45e44..11e30ab554 100644 --- a/core/binary_protocol/src/responses/streams/get_streams.rs +++ b/core/binary_protocol/src/responses/streams/get_streams.rs @@ -61,7 +61,7 @@ impl WireDecode for GetStreamsResponse { #[cfg(test)] mod tests { use super::*; - use crate::WireName; + use crate::{WireName, WireOptions}; #[test] fn roundtrip_empty() { @@ -84,6 +84,7 @@ mod tests { size_bytes: 512, messages_count: 50, name: WireName::new("s1").unwrap(), + options: WireOptions::empty(), }, StreamResponse { id: 2, @@ -92,6 +93,7 @@ mod tests { size_bytes: 0, messages_count: 0, name: WireName::new("stream-two").unwrap(), + options: WireOptions::empty(), }, ], }; diff --git a/core/binary_protocol/src/responses/streams/stream_response.rs b/core/binary_protocol/src/responses/streams/stream_response.rs index aeeef4f58c..8beb76f6df 100644 --- a/core/binary_protocol/src/responses/streams/stream_response.rs +++ b/core/binary_protocol/src/responses/streams/stream_response.rs @@ -18,14 +18,21 @@ use crate::WireError; use crate::codec::{WireDecode, WireEncode, read_u32_le, read_u64_le}; use crate::primitives::identifier::WireName; +use crate::primitives::options::{ + WireOptions, decode_options_prefixed, encode_options_prefixed, options_prefixed_size, +}; use bytes::{BufMut, BytesMut}; /// Stream header on the wire. Used in both single-stream and multi-stream responses. /// -/// Wire format (33 + `name_len` bytes): +/// Wire format (33 + `name_len` + options bytes): /// ```text /// [id:4][created_at:8][topics_count:4][size_bytes:8][messages_count:8][name_len:1][name:N] +/// [options_len:4][options TLV] /// ``` +/// +/// Streams have no server-derived options (no catalog keys resolve against +/// config yet), so a single client-explicit block suffices. #[derive(Debug, Clone, PartialEq, Eq)] pub struct StreamResponse { pub id: u32, @@ -34,6 +41,7 @@ pub struct StreamResponse { pub size_bytes: u64, pub messages_count: u64, pub name: WireName, + pub options: WireOptions, } impl StreamResponse { @@ -42,7 +50,7 @@ impl StreamResponse { impl WireEncode for StreamResponse { fn encoded_size(&self) -> usize { - Self::FIXED_SIZE + self.name.len() + Self::FIXED_SIZE + self.name.len() + options_prefixed_size(&self.options) } fn encode(&self, buf: &mut BytesMut) { @@ -52,6 +60,7 @@ impl WireEncode for StreamResponse { buf.put_u64_le(self.size_bytes); buf.put_u64_le(self.messages_count); self.name.encode(buf); + encode_options_prefixed(&self.options, buf); } } @@ -63,7 +72,9 @@ impl WireDecode for StreamResponse { let size_bytes = read_u64_le(buf, 16)?; let messages_count = read_u64_le(buf, 24)?; let (name, name_consumed) = WireName::decode(&buf[32..])?; - let consumed = 32 + name_consumed; + let mut consumed = 32 + name_consumed; + let (options, options_consumed) = decode_options_prefixed(buf, consumed)?; + consumed += options_consumed; Ok(( Self { @@ -73,6 +84,7 @@ impl WireDecode for StreamResponse { size_bytes, messages_count, name, + options, }, consumed, )) @@ -91,6 +103,7 @@ mod tests { size_bytes: 1024, messages_count: 100, name: WireName::new("test-stream").unwrap(), + options: WireOptions::empty(), } } @@ -98,7 +111,7 @@ mod tests { fn roundtrip() { let resp = sample(); let bytes = resp.to_bytes(); - assert_eq!(bytes.len(), StreamResponse::FIXED_SIZE + 11); + assert_eq!(bytes.len(), StreamResponse::FIXED_SIZE + 11 + 4); let (decoded, consumed) = StreamResponse::decode(&bytes).unwrap(); assert_eq!(consumed, bytes.len()); assert_eq!(decoded, resp); diff --git a/core/binary_protocol/src/responses/system/describe_options.rs b/core/binary_protocol/src/responses/system/describe_options.rs new file mode 100644 index 0000000000..154196ef7c --- /dev/null +++ b/core/binary_protocol/src/responses/system/describe_options.rs @@ -0,0 +1,220 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::WireError; +use crate::codec::{WireDecode, WireEncode, read_u8, read_u32_le}; +use crate::primitives::identifier::WireName; +use bytes::{BufMut, Bytes, BytesMut}; + +/// One catalog entry in a `DescribeOptions` response. +/// +/// Wire format: +/// ```text +/// [key_len:u8][key:N][kind:u8] +/// [default_len:u32][default bytes][description_len:u32][description] +/// ``` +/// +/// `kind` is this key's canonical header kind code: what the server encodes its +/// default under, and what a value set at create is stored as, since create +/// admission re-encodes the block from its own parse (a `String` value parsed by +/// the same rules as a config file value is accepted and canonicalized). An +/// update stores the client's bytes verbatim and is the exception. `default` is +/// that default in `kind`'s encoding, empty when the key has none, and is a +/// build constant rather than a per-node value. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OptionDescriptor { + pub key: WireName, + pub kind: u8, + pub default_value: Bytes, + pub description: String, +} + +impl WireEncode for OptionDescriptor { + fn encoded_size(&self) -> usize { + self.key.encoded_size() + 1 + 4 + self.default_value.len() + 4 + self.description.len() + } + + fn encode(&self, buf: &mut BytesMut) { + self.key.encode(buf); + buf.put_u8(self.kind); + #[allow(clippy::cast_possible_truncation)] + buf.put_u32_le(self.default_value.len() as u32); + buf.put_slice(&self.default_value); + #[allow(clippy::cast_possible_truncation)] + buf.put_u32_le(self.description.len() as u32); + buf.put_slice(self.description.as_bytes()); + } +} + +impl WireDecode for OptionDescriptor { + fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> { + let (key, mut pos) = WireName::decode(buf)?; + let kind = read_u8(buf, pos)?; + pos += 1; + let default_len = read_u32_le(buf, pos)? as usize; + pos += 4; + // `checked_add`: on a 32-bit target a wire-supplied length wraps, the + // truncation guard below then passes, and the slice panics instead. + let default_end = pos + .checked_add(default_len) + .ok_or_else(|| WireError::UnexpectedEof { + offset: pos, + need: default_len, + have: buf.len().saturating_sub(pos), + })?; + if buf.len() < default_end { + return Err(WireError::UnexpectedEof { + offset: pos, + need: default_len, + have: buf.len().saturating_sub(pos), + }); + } + let default_value = Bytes::copy_from_slice(&buf[pos..default_end]); + pos = default_end; + let description_len = read_u32_le(buf, pos)? as usize; + pos += 4; + let description_end = + pos.checked_add(description_len) + .ok_or_else(|| WireError::UnexpectedEof { + offset: pos, + need: description_len, + have: buf.len().saturating_sub(pos), + })?; + if buf.len() < description_end { + return Err(WireError::UnexpectedEof { + offset: pos, + need: description_len, + have: buf.len().saturating_sub(pos), + }); + } + let description = String::from_utf8_lossy(&buf[pos..description_end]).into_owned(); + pos = description_end; + + Ok(( + Self { + key, + kind, + default_value, + description, + }, + pos, + )) + } +} + +/// `DescribeOptions` response: count-prefixed catalog entries. +/// +/// Wire format: `[count:u32][OptionDescriptor]*` +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DescribeOptionsResponse { + pub entries: Vec, +} + +impl WireEncode for DescribeOptionsResponse { + fn encoded_size(&self) -> usize { + 4 + self + .entries + .iter() + .map(WireEncode::encoded_size) + .sum::() + } + + fn encode(&self, buf: &mut BytesMut) { + #[allow(clippy::cast_possible_truncation)] + buf.put_u32_le(self.entries.len() as u32); + for entry in &self.entries { + entry.encode(buf); + } + } +} + +/// Smallest a descriptor can encode as: a one-character `WireName` (a length +/// byte plus at least one byte of name), a kind byte, and empty +/// length-prefixed default and description. +const MIN_DESCRIPTOR_SIZE: usize = 2 + 1 + 4 + 4; + +impl WireDecode for DescribeOptionsResponse { + fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> { + let count = read_u32_le(buf, 0)? as usize; + let mut pos = 4; + let mut entries = Vec::with_capacity(crate::codec::bounded_capacity( + count, + buf.len().saturating_sub(pos), + MIN_DESCRIPTOR_SIZE, + )); + for _ in 0..count { + let (entry, consumed) = OptionDescriptor::decode(&buf[pos..])?; + pos += consumed; + entries.push(entry); + } + Ok((Self { entries }, pos)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample() -> DescribeOptionsResponse { + DescribeOptionsResponse { + entries: vec![ + OptionDescriptor { + key: WireName::new("partitions_count").unwrap(), + kind: 11, + default_value: Bytes::copy_from_slice(&1u32.to_le_bytes()), + description: "Number of partitions to create".to_string(), + }, + OptionDescriptor { + key: WireName::new("max_topic_size").unwrap(), + kind: 12, + default_value: Bytes::new(), + description: String::new(), + }, + ], + } + } + + #[test] + fn roundtrip() { + let resp = sample(); + let bytes = resp.to_bytes(); + let (decoded, consumed) = DescribeOptionsResponse::decode(&bytes).unwrap(); + assert_eq!(consumed, bytes.len()); + assert_eq!(decoded, resp); + } + + #[test] + fn roundtrip_empty() { + let resp = DescribeOptionsResponse { entries: vec![] }; + let bytes = resp.to_bytes(); + assert_eq!(bytes.len(), 4); + let (decoded, consumed) = DescribeOptionsResponse::decode(&bytes).unwrap(); + assert_eq!(consumed, bytes.len()); + assert_eq!(decoded, resp); + } + + #[test] + fn truncated_returns_error() { + let bytes = sample().to_bytes(); + for i in 5..bytes.len() { + assert!( + DescribeOptionsResponse::decode(&bytes[..i]).is_err(), + "expected error for truncation at byte {i}" + ); + } + } +} diff --git a/core/binary_protocol/src/responses/system/get_cluster_metadata.rs b/core/binary_protocol/src/responses/system/get_cluster_metadata.rs index 0b343e6444..22748b4e75 100644 --- a/core/binary_protocol/src/responses/system/get_cluster_metadata.rs +++ b/core/binary_protocol/src/responses/system/get_cluster_metadata.rs @@ -138,7 +138,7 @@ impl WireDecode for ClusterMetadataResponse { pos += 4; let remaining = buf.len().saturating_sub(pos); - let mut nodes = Vec::with_capacity(crate::codec::capped_capacity( + let mut nodes = Vec::with_capacity(crate::codec::bounded_capacity( nodes_count, remaining, NODE_FIXED_SIZE, diff --git a/core/binary_protocol/src/responses/system/get_stats.rs b/core/binary_protocol/src/responses/system/get_stats.rs index 3e84bd4394..7921876ebc 100644 --- a/core/binary_protocol/src/responses/system/get_stats.rs +++ b/core/binary_protocol/src/responses/system/get_stats.rs @@ -207,7 +207,7 @@ impl WireDecode for StatsResponse { pos += 4; let remaining = buf.len().saturating_sub(pos); - let mut cache_metrics = Vec::with_capacity(crate::codec::capped_capacity( + let mut cache_metrics = Vec::with_capacity(crate::codec::bounded_capacity( cache_count, remaining, CacheMetricEntry::SIZE, diff --git a/core/binary_protocol/src/responses/system/mod.rs b/core/binary_protocol/src/responses/system/mod.rs index 8b3cafc704..113eee3914 100644 --- a/core/binary_protocol/src/responses/system/mod.rs +++ b/core/binary_protocol/src/responses/system/mod.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +pub mod describe_options; pub mod get_cluster_metadata; pub mod get_me; pub mod get_snapshot; @@ -22,6 +23,7 @@ pub mod get_stats; mod ping; pub use super::EmptyResponse; +pub use describe_options::{DescribeOptionsResponse, OptionDescriptor}; pub use get_cluster_metadata::{ClusterMetadataResponse, ClusterNodeResponse}; pub use get_me::GetMeResponse; pub use get_snapshot::GetSnapshotResponse; diff --git a/core/binary_protocol/src/responses/topics/get_topic.rs b/core/binary_protocol/src/responses/topics/get_topic.rs index b90e521566..793800a3c0 100644 --- a/core/binary_protocol/src/responses/topics/get_topic.rs +++ b/core/binary_protocol/src/responses/topics/get_topic.rs @@ -19,7 +19,6 @@ use crate::WireError; use crate::codec::{WireDecode, WireEncode, read_u32_le, read_u64_le}; use crate::responses::streams::get_stream::TopicHeader; use bytes::{BufMut, BytesMut}; -use std::borrow::Cow; /// Partition details within a `GetTopic` response. /// @@ -114,19 +113,19 @@ impl WireEncode for GetTopicResponse { impl WireDecode for GetTopicResponse { fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> { let (topic, mut pos) = TopicHeader::decode(buf)?; - let mut partitions = Vec::new(); - while pos < buf.len() { + // Count-driven so the element stays delimited even when embedded in a + // larger payload; the header's variable-length options blocks removed + // the old "everything after the header is partitions" property. + let mut partitions = Vec::with_capacity(crate::codec::bounded_capacity( + topic.partitions_count as usize, + buf.len().saturating_sub(pos), + PartitionResponse::FIXED_SIZE, + )); + for _ in 0..topic.partitions_count { let (partition, consumed) = PartitionResponse::decode(&buf[pos..])?; pos += consumed; partitions.push(partition); } - if partitions.len() != topic.partitions_count as usize { - return Err(WireError::Validation(Cow::Owned(format!( - "topic.partitions_count={} but decoded {} partitions", - topic.partitions_count, - partitions.len() - )))); - } Ok((Self { topic, partitions }, pos)) } } @@ -134,7 +133,7 @@ impl WireDecode for GetTopicResponse { #[cfg(test)] mod tests { use super::*; - use crate::WireName; + use crate::{WireName, WireOptions}; fn sample_topic(partitions_count: u32) -> TopicHeader { TopicHeader { @@ -144,10 +143,11 @@ mod tests { message_expiry: 0, compression_algorithm: 1, max_topic_size: 0, - replication_factor: 1, size_bytes: 2048, messages_count: 200, name: WireName::new("my-topic").unwrap(), + options: WireOptions::empty(), + derived_options: WireOptions::empty(), } } diff --git a/core/binary_protocol/src/responses/topics/get_topics.rs b/core/binary_protocol/src/responses/topics/get_topics.rs index b36d4ee12e..6852866e32 100644 --- a/core/binary_protocol/src/responses/topics/get_topics.rs +++ b/core/binary_protocol/src/responses/topics/get_topics.rs @@ -16,18 +16,23 @@ // under the License. use crate::WireError; -use crate::codec::{WireDecode, WireEncode}; +use crate::codec::{WireDecode, WireEncode, read_u32_le}; use crate::responses::streams::get_stream::TopicHeader; -use bytes::BytesMut; +use bytes::{BufMut, BytesMut}; -/// `GetTopics` response: sequential topic headers. +/// `GetTopics` response: count-prefixed topic headers. /// /// Wire format: /// ```text -/// [TopicHeader]* +/// [topics_count:u32_le][TopicHeader]* /// ``` /// -/// Empty payload means zero topics. +/// The count prefix is what lets a decoder pre-size its `Vec`: an element +/// carries two variable-length option blocks, so the payload length alone says +/// nothing useful about how many elements are in it. Detecting a short read +/// does not need it - the element decoder errors on one, which is how +/// `GetStreams` and `GetUsers` still run their loops to end-of-payload. This +/// response is the odd one out of the three, and every SDK special-cases it. #[derive(Debug, Clone, PartialEq, Eq)] pub struct GetTopicsResponse { pub topics: Vec, @@ -35,21 +40,37 @@ pub struct GetTopicsResponse { impl WireEncode for GetTopicsResponse { fn encoded_size(&self) -> usize { - self.topics.iter().map(WireEncode::encoded_size).sum() + 4 + self + .topics + .iter() + .map(WireEncode::encoded_size) + .sum::() } fn encode(&self, buf: &mut BytesMut) { + #[allow(clippy::cast_possible_truncation)] + buf.put_u32_le(self.topics.len() as u32); for topic in &self.topics { topic.encode(buf); } } } +/// Smallest a topic header can encode as: the fixed ids, timestamps, sizes, +/// counts and name length, plus the shortest name `WireName` accepts (one byte, +/// since it rejects an empty one) and two empty length-prefixed option blocks. +const MIN_TOPIC_HEADER_SIZE: usize = TopicHeader::FIXED_SIZE + 1 + 4 + 4; + impl WireDecode for GetTopicsResponse { fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> { - let mut topics = Vec::new(); - let mut pos = 0; - while pos < buf.len() { + let topics_count = read_u32_le(buf, 0)? as usize; + let mut pos = 4; + let mut topics = Vec::with_capacity(crate::codec::bounded_capacity( + topics_count, + buf.len().saturating_sub(pos), + MIN_TOPIC_HEADER_SIZE, + )); + for _ in 0..topics_count { let (topic, consumed) = TopicHeader::decode(&buf[pos..])?; pos += consumed; topics.push(topic); @@ -61,7 +82,7 @@ impl WireDecode for GetTopicsResponse { #[cfg(test)] mod tests { use super::*; - use crate::WireName; + use crate::{WireName, WireOptions}; fn sample_topic(id: u32, name: &str) -> TopicHeader { TopicHeader { @@ -71,10 +92,11 @@ mod tests { message_expiry: 0, compression_algorithm: 1, max_topic_size: 0, - replication_factor: 1, size_bytes: 1024, messages_count: 100, name: WireName::new(name).unwrap(), + options: WireOptions::empty(), + derived_options: WireOptions::empty(), } } @@ -82,9 +104,9 @@ mod tests { fn roundtrip_empty() { let resp = GetTopicsResponse { topics: vec![] }; let bytes = resp.to_bytes(); - assert!(bytes.is_empty()); + assert_eq!(bytes.len(), 4); let (decoded, consumed) = GetTopicsResponse::decode(&bytes).unwrap(); - assert_eq!(consumed, 0); + assert_eq!(consumed, bytes.len()); assert_eq!(decoded, resp); } diff --git a/core/binary_protocol/src/responses/users/get_user.rs b/core/binary_protocol/src/responses/users/get_user.rs index cae86cf659..6392e37cef 100644 --- a/core/binary_protocol/src/responses/users/get_user.rs +++ b/core/binary_protocol/src/responses/users/get_user.rs @@ -89,10 +89,10 @@ impl WireDecode for UserDetailsResponse { #[cfg(test)] mod tests { use super::*; - use crate::WireName; use crate::primitives::permissions::{ WireGlobalPermissions, WireStreamPermissions, WireTopicPermissions, }; + use crate::{WireName, WireOptions}; fn sample_user() -> UserResponse { UserResponse { @@ -100,6 +100,7 @@ mod tests { created_at: 1_710_000_000_000, status: 1, username: WireName::new("admin").unwrap(), + options: WireOptions::empty(), } } diff --git a/core/binary_protocol/src/responses/users/get_users.rs b/core/binary_protocol/src/responses/users/get_users.rs index 5e522aea1e..87811f6f47 100644 --- a/core/binary_protocol/src/responses/users/get_users.rs +++ b/core/binary_protocol/src/responses/users/get_users.rs @@ -59,7 +59,7 @@ impl WireDecode for GetUsersResponse { #[cfg(test)] mod tests { use super::*; - use crate::WireName; + use crate::{WireName, WireOptions}; #[test] fn roundtrip_empty() { @@ -80,12 +80,14 @@ mod tests { created_at: 100, status: 1, username: WireName::new("admin").unwrap(), + options: WireOptions::empty(), }, UserResponse { id: 2, created_at: 200, status: 2, username: WireName::new("alice").unwrap(), + options: WireOptions::empty(), }, ], }; diff --git a/core/binary_protocol/src/responses/users/user_response.rs b/core/binary_protocol/src/responses/users/user_response.rs index 6eb03f0b09..ef495a9eb1 100644 --- a/core/binary_protocol/src/responses/users/user_response.rs +++ b/core/binary_protocol/src/responses/users/user_response.rs @@ -18,20 +18,28 @@ use crate::WireError; use crate::codec::{WireDecode, WireEncode, read_u8, read_u32_le, read_u64_le}; use crate::primitives::identifier::WireName; +use crate::primitives::options::{ + WireOptions, decode_options_prefixed, encode_options_prefixed, options_prefixed_size, +}; use bytes::{BufMut, BytesMut}; /// User header on the wire. Used in both single-user and multi-user responses. /// -/// Wire format (13 + `username_len` bytes): +/// Wire format (13 + `username_len` + options bytes): /// ```text /// [id:4][created_at:8][status:1][username_len:1][username:N] +/// [options_len:4][options TLV] /// ``` +/// +/// Users have no server-derived options (no catalog keys resolve against +/// config yet), so a single client-explicit block suffices. #[derive(Debug, Clone, PartialEq, Eq)] pub struct UserResponse { pub id: u32, pub created_at: u64, pub status: u8, pub username: WireName, + pub options: WireOptions, } impl UserResponse { @@ -40,7 +48,7 @@ impl UserResponse { impl WireEncode for UserResponse { fn encoded_size(&self) -> usize { - Self::FIXED_SIZE + self.username.encoded_size() + Self::FIXED_SIZE + self.username.encoded_size() + options_prefixed_size(&self.options) } fn encode(&self, buf: &mut BytesMut) { @@ -48,6 +56,7 @@ impl WireEncode for UserResponse { buf.put_u64_le(self.created_at); buf.put_u8(self.status); self.username.encode(buf); + encode_options_prefixed(&self.options, buf); } } @@ -57,7 +66,9 @@ impl WireDecode for UserResponse { let created_at = read_u64_le(buf, 4)?; let status = read_u8(buf, 12)?; let (username, name_consumed) = WireName::decode(&buf[13..])?; - let consumed = 13 + name_consumed; + let mut consumed = 13 + name_consumed; + let (options, options_consumed) = decode_options_prefixed(buf, consumed)?; + consumed += options_consumed; Ok(( Self { @@ -65,6 +76,7 @@ impl WireDecode for UserResponse { created_at, status, username, + options, }, consumed, )) @@ -81,6 +93,7 @@ mod tests { created_at: 1_710_000_000_000, status: 1, username: WireName::new("admin").unwrap(), + options: WireOptions::empty(), } } @@ -88,7 +101,7 @@ mod tests { fn roundtrip() { let resp = sample(); let bytes = resp.to_bytes(); - assert_eq!(bytes.len(), UserResponse::FIXED_SIZE + 1 + 5); + assert_eq!(bytes.len(), UserResponse::FIXED_SIZE + 1 + 5 + 4); let (decoded, consumed) = UserResponse::decode(&bytes).unwrap(); assert_eq!(consumed, bytes.len()); assert_eq!(decoded, resp); diff --git a/core/cli/src/args/common.rs b/core/cli/src/args/common.rs index bb186bf115..2accf01179 100644 --- a/core/cli/src/args/common.rs +++ b/core/cli/src/args/common.rs @@ -122,3 +122,33 @@ impl From for GetStatsOutput { } } } + +/// Longest an option key or value may be, matching the header-field codec the +/// options block rides. Enforced here so `--set` reports the offending pair +/// with clap's own error rather than failing at encode time. +const MAX_OPTION_FIELD_LEN: usize = 255; + +/// Parse a repeatable `--set KEY=VALUE` pair. +/// +/// Shared by every resource's option args: divergent copies disagreed on +/// whether an empty value was legal, which decided whether a pair reached the +/// encoder at all. +pub(crate) fn parse_key_value(raw: &str) -> Result<(String, String), String> { + let (key, value) = raw + .split_once('=') + .ok_or_else(|| format!("expected KEY=VALUE, got: {raw}"))?; + for (label, field) in [("key", key), ("value", value)] { + if field.is_empty() { + return Err(format!( + "expected non-empty {label} in KEY=VALUE, got: {raw}" + )); + } + if field.len() > MAX_OPTION_FIELD_LEN { + return Err(format!( + "option {label} is {} bytes, maximum {MAX_OPTION_FIELD_LEN}", + field.len() + )); + } + } + Ok((key.to_string(), value.to_string())) +} diff --git a/core/cli/src/args/mod.rs b/core/cli/src/args/mod.rs index a7159addb7..42d517810d 100644 --- a/core/cli/src/args/mod.rs +++ b/core/cli/src/args/mod.rs @@ -37,7 +37,7 @@ use crate::args::{ partition::PartitionAction, personal_access_token::PersonalAccessTokenAction, stream::StreamAction, - system::{PingArgs, StatsArgs}, + system::{OptionsArgs, PingArgs, StatsArgs}, topic::TopicAction, }; @@ -162,6 +162,13 @@ pub(crate) enum Command { /// server address and protocol type. #[clap(verbatim_doc_comment)] Me, + /// list the options a resource's create command accepts + /// + /// Prints the server's option catalog for one scope (topic, stream, user): + /// every key, the kind its default is reported in, that default, and the + /// bounds each value is checked against. These are the keys `--set` takes. + #[clap(verbatim_doc_comment)] + Options(OptionsArgs), /// get iggy server statistics /// /// Collect basic Iggy server statistics like number of streams, topics, partitions, etc. diff --git a/core/cli/src/args/system.rs b/core/cli/src/args/system.rs index 39f58fc37e..a7195a2f17 100644 --- a/core/cli/src/args/system.rs +++ b/core/cli/src/args/system.rs @@ -17,7 +17,7 @@ use crate::args::common::ListModeExt; use clap::Args; -use iggy::prelude::{SnapshotCompression, SystemSnapshotType}; +use iggy::prelude::{OptionsScope, SnapshotCompression, SystemSnapshotType}; use iggy_cli::commands::utils::login_session_expiry::LoginSessionExpiry; #[derive(Debug, Clone, Args)] @@ -38,6 +38,13 @@ pub(crate) struct LoginArgs { pub(crate) expiry: Option>, } +#[derive(Debug, Clone, Args)] +pub(crate) struct OptionsArgs { + /// Resource whose option catalog to print (topic, stream, user) + #[arg(value_parser = clap::value_parser!(OptionsScope))] + pub(crate) scope: OptionsScope, +} + #[derive(Debug, Clone, Args)] pub(crate) struct StatsArgs { /// List mode (table, list, JSON, TOML) diff --git a/core/cli/src/args/topic.rs b/core/cli/src/args/topic.rs index 694d763e09..875b3705f8 100644 --- a/core/cli/src/args/topic.rs +++ b/core/cli/src/args/topic.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use crate::args::common::ListMode; +use crate::args::common::{ListMode, parse_key_value}; use clap::{Args, Subcommand}; use iggy::prelude::{CompressionAlgorithm, Identifier, IggyExpiry, MaxTopicSize}; @@ -117,16 +117,21 @@ pub(crate) struct TopicCreateArgs { /// Can't be lower than segment size in the config. #[arg(short, long, default_value = "server_default", verbatim_doc_comment)] pub(crate) max_topic_size: MaxTopicSize, - /// Replication factor for the topic - #[arg(short, long, default_value = "1")] - pub(crate) replication_factor: u8, /// Message expiry time in human-readable format like "unlimited" or "15days 2min 2s" /// /// "server_default" or skipping parameter makes CLI to use server default (from current server config) expiry time #[arg(default_value = "server_default", value_parser = clap::value_parser!(IggyExpiry), verbatim_doc_comment)] pub(crate) message_expiry: Vec, + /// Additional topic option as key=value, repeatable + /// + /// Values are sent as strings and parsed server-side through each option's + /// own FromStr (e.g. --set segment_size=128MiB). The server rejects keys it + /// does not support; run "iggy options topic" to list the ones it accepts. + #[arg(long = "set", value_name = "KEY=VALUE", value_parser = parse_key_value, verbatim_doc_comment)] + pub(crate) set: Vec<(String, String)>, } +/// Parse one `--set key=value` occurrence. #[derive(Debug, Clone, Args)] pub(crate) struct TopicDeleteArgs { /// Stream ID to delete topic @@ -164,9 +169,6 @@ pub(crate) struct TopicUpdateArgs { /// Can't be lower than segment size in the config. #[arg(short, long, default_value = "server_default", verbatim_doc_comment)] pub(crate) max_topic_size: MaxTopicSize, - #[arg(short, long, default_value = "1")] - /// New replication factor for the topic - pub(crate) replication_factor: u8, /// New message expiry time in human-readable format like "unlimited" or "15days 2min 2s" /// /// "server_default" or skipping parameter makes CLI to use server default (from current server config) expiry time diff --git a/core/cli/src/commands/binary_streams/update_stream.rs b/core/cli/src/commands/binary_streams/update_stream.rs index 0f612d5b17..209c7af59f 100644 --- a/core/cli/src/commands/binary_streams/update_stream.rs +++ b/core/cli/src/commands/binary_streams/update_stream.rs @@ -20,17 +20,25 @@ use anyhow::Context; use async_trait::async_trait; use iggy_common::Client; use iggy_common::Identifier; +use iggy_common::StreamUpdateOptions; use iggy_common::update_stream::UpdateStream; +use std::collections::BTreeMap; use tracing::{Level, event}; pub struct UpdateStreamCmd { update_stream: UpdateStream, + options: StreamUpdateOptions, } impl UpdateStreamCmd { pub fn new(stream_id: Identifier, name: String) -> Self { UpdateStreamCmd { - update_stream: UpdateStream { stream_id, name }, + update_stream: UpdateStream { + stream_id, + name, + options: BTreeMap::new(), + }, + options: StreamUpdateOptions::default(), } } } @@ -46,7 +54,11 @@ impl CliCommand for UpdateStreamCmd { async fn execute_cmd(&mut self, client: &dyn Client) -> anyhow::Result<(), anyhow::Error> { client - .update_stream(&self.update_stream.stream_id, &self.update_stream.name) + .update_stream( + &self.update_stream.stream_id, + &self.update_stream.name, + &self.options, + ) .await .with_context(|| { format!( diff --git a/core/cli/src/commands/binary_system/mod.rs b/core/cli/src/commands/binary_system/mod.rs index 979ebeabe7..2bbc45a3b9 100644 --- a/core/cli/src/commands/binary_system/mod.rs +++ b/core/cli/src/commands/binary_system/mod.rs @@ -18,6 +18,7 @@ pub mod login; pub mod logout; pub mod me; +pub mod options; pub mod ping; pub mod session; pub mod session_status; diff --git a/core/cli/src/commands/binary_system/options.rs b/core/cli/src/commands/binary_system/options.rs new file mode 100644 index 0000000000..5fc7f6beb8 --- /dev/null +++ b/core/cli/src/commands/binary_system/options.rs @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::commands::cli_command::{CliCommand, PRINT_TARGET}; +use anyhow::Context; +use async_trait::async_trait; +use comfy_table::Table; +use iggy_common::{Client, HeaderValue, OptionSpec, OptionsScope}; +use tracing::{Level, event}; + +/// `iggy options `: print the server's option catalog. +/// +/// This is the discovery surface for `--set`. A key outside the catalog is +/// refused at create, and the binary transports carry only the error code back, +/// so without this command a CLI user has no way to see which keys their server +/// knows or what each one is bounded by. +pub struct DescribeOptionsCmd { + scope: OptionsScope, +} + +impl DescribeOptionsCmd { + pub fn new(scope: OptionsScope) -> Self { + Self { scope } + } +} + +#[async_trait] +impl CliCommand for DescribeOptionsCmd { + fn explain(&self) -> String { + format!("describe {} options", self.scope) + } + + async fn execute_cmd(&mut self, client: &dyn Client) -> anyhow::Result<(), anyhow::Error> { + let specs = client + .describe_options(self.scope) + .await + .with_context(|| "Problem sending describe_options command".to_owned())?; + + if specs.is_empty() { + event!(target: PRINT_TARGET, Level::INFO, + "{} resources accept no options yet", self.scope); + return Ok(()); + } + + let mut table = Table::new(); + table.set_header(vec!["Key", "Kind", "Default", "Description"]); + for spec in &specs { + table.add_row(vec![ + spec.key.as_str(), + spec.kind.to_string().as_str(), + render_default(spec).as_str(), + spec.description.as_str(), + ]); + } + event!(target: PRINT_TARGET, Level::INFO, "{table}"); + + Ok(()) + } +} + +/// The catalog's default in the string form `--set` takes, so the printed value +/// can be handed straight back to a create. +fn render_default(spec: &OptionSpec) -> String { + if spec.default_value.is_empty() { + return String::new(); + } + match HeaderValue::from_raw(spec.kind, &spec.default_value) { + Ok(value) => value.to_string_value(), + // A kind this build cannot name still has a readable byte count. + Err(_) => format!("<{} bytes>", spec.default_value.len()), + } +} diff --git a/core/cli/src/commands/binary_system/session.rs b/core/cli/src/commands/binary_system/session.rs index 5133de4d97..737241a522 100644 --- a/core/cli/src/commands/binary_system/session.rs +++ b/core/cli/src/commands/binary_system/session.rs @@ -27,7 +27,7 @@ use windows_native_keyring_store::store::Store; #[cfg(secret_service_keyring)] use zbus_secret_service_keyring_store::store::Store; -use crate::commands::cli_command::PRINT_TARGET; +use crate::commands::cli_command::DIAGNOSTIC_TARGET; const SESSION_TOKEN_NAME: &str = "iggy-cli-session"; const SESSION_KEYRING_SERVICE_NAME: &str = "iggy-cli-session"; @@ -79,7 +79,7 @@ impl ServerSession { pub fn is_active(&self) -> bool { if let Err(e) = ensure_default_store() { - warn!(target: PRINT_TARGET, "keyring backend unavailable, treating session as inactive: {e}"); + warn!(target: DIAGNOSTIC_TARGET, "keyring backend unavailable, treating session as inactive: {e}"); return false; } if let Ok(entry) = Entry::new(&self.get_service_name(), &self.get_token_name()) { @@ -98,7 +98,7 @@ impl ServerSession { pub fn get_token(&self) -> Option { if let Err(e) = ensure_default_store() { - warn!(target: PRINT_TARGET, "keyring backend unavailable, cannot read session token: {e}"); + warn!(target: DIAGNOSTIC_TARGET, "keyring backend unavailable, cannot read session token: {e}"); return None; } if let Ok(entry) = Entry::new(&self.get_service_name(), &self.get_token_name()) diff --git a/core/cli/src/commands/binary_topics/create_topic.rs b/core/cli/src/commands/binary_topics/create_topic.rs index a6e0d662af..ec9209ddb8 100644 --- a/core/cli/src/commands/binary_topics/create_topic.rs +++ b/core/cli/src/commands/binary_topics/create_topic.rs @@ -21,14 +21,15 @@ use async_trait::async_trait; use core::fmt; use iggy_common::Client; use iggy_common::create_topic::CreateTopic; -use iggy_common::{CompressionAlgorithm, Identifier, IggyExpiry, MaxTopicSize}; +use iggy_common::{CompressionAlgorithm, Identifier, IggyExpiry, MaxTopicSize, TopicCreateOptions}; +use std::collections::BTreeMap; use tracing::{Level, event}; pub struct CreateTopicCmd { create_topic: CreateTopic, message_expiry: IggyExpiry, max_topic_size: MaxTopicSize, - replication_factor: u8, + raw_options: BTreeMap, } impl CreateTopicCmd { @@ -40,7 +41,7 @@ impl CreateTopicCmd { name: String, message_expiry: IggyExpiry, max_topic_size: MaxTopicSize, - replication_factor: u8, + raw_options: BTreeMap, ) -> Self { Self { create_topic: CreateTopic { @@ -50,11 +51,11 @@ impl CreateTopicCmd { name, message_expiry, max_topic_size, - replication_factor: Some(replication_factor), + options: raw_options.clone(), }, message_expiry, max_topic_size, - replication_factor, + raw_options, } } } @@ -70,11 +71,19 @@ impl CliCommand for CreateTopicCmd { .create_topic( &self.create_topic.stream_id, &self.create_topic.name, - self.create_topic.partitions_count, - self.create_topic.compression_algorithm, - self.create_topic.replication_factor, - self.create_topic.message_expiry, - self.create_topic.max_topic_size, + &TopicCreateOptions { + partitions_count: Some(self.create_topic.partitions_count), + compression_algorithm: (self.create_topic.compression_algorithm + != CompressionAlgorithm::default()) + .then_some(self.create_topic.compression_algorithm), + message_expiry: (self.create_topic.message_expiry != IggyExpiry::ServerDefault) + .then_some(self.create_topic.message_expiry), + max_topic_size: (self.create_topic.max_topic_size + != MaxTopicSize::ServerDefault) + .then_some(self.create_topic.max_topic_size), + raw: self.raw_options.clone(), + ..TopicCreateOptions::default() + }, ) .await .with_context(|| { @@ -87,13 +96,12 @@ impl CliCommand for CreateTopicCmd { })?; event!(target: PRINT_TARGET, Level::INFO, - "Topic with name: {}, partitions count: {}, compression algorithm: {}, message expiry: {}, max topic size: {}, replication factor: {} created in stream with ID: {}", + "Topic with name: {}, partitions count: {}, compression algorithm: {}, message expiry: {}, max topic size: {} created in stream with ID: {}", self.create_topic.name, self.create_topic.partitions_count, self.create_topic.compression_algorithm, self.message_expiry, self.max_topic_size, - self.replication_factor, self.create_topic.stream_id, ); @@ -107,13 +115,13 @@ impl fmt::Display for CreateTopicCmd { let compression_algorithm = &self.create_topic.compression_algorithm; let message_expiry = &self.message_expiry; let max_topic_size = &self.max_topic_size; - let replication_factor = self.replication_factor; let stream_id = &self.create_topic.stream_id; write!( f, - "create topic with name: {topic_name}, message expiry: {message_expiry}, compression algorithm: {compression_algorithm}, \ - max topic size: {max_topic_size}, replication factor: {replication_factor} in stream with ID: {stream_id}", + "create topic with name: {topic_name}, message expiry: {message_expiry}, \ + compression algorithm: {compression_algorithm}, max topic size: {max_topic_size} \ + in stream with ID: {stream_id}", ) } } diff --git a/core/cli/src/commands/binary_topics/update_topic.rs b/core/cli/src/commands/binary_topics/update_topic.rs index 2615a975f1..04f3b827a8 100644 --- a/core/cli/src/commands/binary_topics/update_topic.rs +++ b/core/cli/src/commands/binary_topics/update_topic.rs @@ -21,14 +21,16 @@ use async_trait::async_trait; use core::fmt; use iggy_common::Client; use iggy_common::update_topic::UpdateTopic; -use iggy_common::{CompressionAlgorithm, Identifier, IggyExpiry, MaxTopicSize}; +use iggy_common::{CompressionAlgorithm, Identifier, IggyExpiry, MaxTopicSize, TopicUpdateOptions}; +use std::collections::BTreeMap; use tracing::{Level, event}; pub struct UpdateTopicCmd { update_topic: UpdateTopic, + compression_algorithm: CompressionAlgorithm, message_expiry: IggyExpiry, max_topic_size: MaxTopicSize, - replication_factor: u8, + options: TopicUpdateOptions, } impl UpdateTopicCmd { @@ -39,21 +41,26 @@ impl UpdateTopicCmd { name: String, message_expiry: IggyExpiry, max_topic_size: MaxTopicSize, - replication_factor: u8, ) -> Self { Self { update_topic: UpdateTopic { stream_id, topic_id, name, - compression_algorithm, - message_expiry, - max_topic_size, - replication_factor: Some(replication_factor), + compression_algorithm: Some(compression_algorithm), + message_expiry: Some(message_expiry), + max_topic_size: Some(max_topic_size), + options: BTreeMap::new(), }, + compression_algorithm, message_expiry, max_topic_size, - replication_factor, + options: TopicUpdateOptions { + compression_algorithm: Some(compression_algorithm), + message_expiry: Some(message_expiry), + max_topic_size: Some(max_topic_size), + ..TopicUpdateOptions::default() + }, } } } @@ -66,7 +73,7 @@ impl CliCommand for UpdateTopicCmd { async fn execute_cmd(&mut self, client: &dyn Client) -> anyhow::Result<(), anyhow::Error> { client - .update_topic(&self.update_topic.stream_id, &self.update_topic.topic_id, &self.update_topic.name, self.update_topic.compression_algorithm, self.replication_factor.into(), self.message_expiry, self.max_topic_size) + .update_topic(&self.update_topic.stream_id, &self.update_topic.topic_id, &self.update_topic.name, &self.options) .await .with_context(|| { format!( @@ -79,13 +86,12 @@ impl CliCommand for UpdateTopicCmd { })?; event!(target: PRINT_TARGET, Level::INFO, - "Topic with ID: {} updated name: {}, updated message expiry: {}, updated compression algorithm: {}, updated max topic size: {}, updated replication factor: {} in stream with ID: {}", + "Topic with ID: {} updated name: {}, updated message expiry: {}, updated compression algorithm: {}, updated max topic size: {} in stream with ID: {}", self.update_topic.topic_id, self.update_topic.name, self.message_expiry, - self.update_topic.compression_algorithm, + self.compression_algorithm, self.max_topic_size, - self.replication_factor, self.update_topic.stream_id, ); @@ -97,17 +103,16 @@ impl fmt::Display for UpdateTopicCmd { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let topic_id = &self.update_topic.topic_id; let topic_name = &self.update_topic.name; - let compression_algorithm = &self.update_topic.compression_algorithm; + let compression_algorithm = &self.compression_algorithm; let message_expiry = &self.message_expiry; let max_topic_size = &self.max_topic_size; - let replication_factor = self.replication_factor; let stream_id = &self.update_topic.stream_id; write!( f, "update topic with ID: {topic_id}, name: {topic_name}, message expiry: \ - {message_expiry}, compression algorithm: {compression_algorithm}, max topic size: {max_topic_size}, replication \ - factor: {replication_factor}, in stream with ID: {stream_id}", + {message_expiry}, compression algorithm: {compression_algorithm}, max topic size: \ + {max_topic_size}, in stream with ID: {stream_id}", ) } } diff --git a/core/cli/src/commands/binary_users/update_user.rs b/core/cli/src/commands/binary_users/update_user.rs index e397e61c81..7366c6c6f5 100644 --- a/core/cli/src/commands/binary_users/update_user.rs +++ b/core/cli/src/commands/binary_users/update_user.rs @@ -21,7 +21,9 @@ use async_trait::async_trait; use iggy_common::Client; use iggy_common::Identifier; use iggy_common::UserStatus; +use iggy_common::UserUpdateOptions; use iggy_common::update_user::UpdateUser; +use std::collections::BTreeMap; use tracing::{Level, event}; #[derive(Debug, Clone)] @@ -48,6 +50,7 @@ impl UpdateUserCmd { user_id, username, status, + options: BTreeMap::new(), }, } } @@ -76,6 +79,9 @@ impl CliCommand for UpdateUserCmd { &self.update_user.user_id, self.update_user.username.as_deref(), self.update_user.status, + // `iggy user name` / `iggy user status` are single-purpose + // commands; users have no option keys to set yet. + &UserUpdateOptions::default(), ) .await .with_context(|| { diff --git a/core/cli/src/commands/cli_command.rs b/core/cli/src/commands/cli_command.rs index 9ecdf2d775..eb2520a02b 100644 --- a/core/cli/src/commands/cli_command.rs +++ b/core/cli/src/commands/cli_command.rs @@ -21,6 +21,12 @@ use iggy_common::Client; pub static PRINT_TARGET: &str = "iggy::cli::output"; +/// Diagnostics that must not land on [`PRINT_TARGET`]: that channel is the +/// command's result, and callers pipe it. Routed to stderr instead, so a +/// degraded-but-working run (no keyring, for one) stays visible without +/// corrupting parseable output. +pub static DIAGNOSTIC_TARGET: &str = "iggy::cli::diagnostic"; + #[async_trait] pub trait CliCommand { fn explain(&self) -> String; diff --git a/core/cli/src/logging.rs b/core/cli/src/logging.rs index a2363511da..9dec122464 100644 --- a/core/cli/src/logging.rs +++ b/core/cli/src/logging.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use iggy_cli::commands::cli_command::PRINT_TARGET; +use iggy_cli::commands::cli_command::{DIAGNOSTIC_TARGET, PRINT_TARGET}; use std::path::PathBuf; use tracing_appender::non_blocking::WorkerGuard; use tracing_subscriber::{ @@ -59,6 +59,21 @@ impl Logging { layers.push(stdout_layer.with_filter(stdout_filter).boxed()); + // Diagnostics go to stderr even under `--quiet`: quiet suppresses the + // command's result, not a warning that the run was degraded. Scoped to + // DIAGNOSTIC_TARGET rather than "every non-print event" so dependency + // logs stay suppressed as before. + let stderr_filter = + filter::filter_fn(|metadata| metadata.target().contains(DIAGNOSTIC_TARGET)); + let stderr_layer = fmt::Layer::default() + .without_time() + .with_target(false) + .with_writer(std::io::stderr) + .with_filter(LevelFilter::WARN) + .boxed(); + + layers.push(stderr_layer.with_filter(stderr_filter).boxed()); + if let Some(file_path) = debug { let _ = std::fs::remove_file(file_path); // Remove file if it exists let file_appender = tracing_appender::rolling::never("", file_path); diff --git a/core/cli/src/main.rs b/core/cli/src/main.rs index 3187811d63..93a73618f3 100644 --- a/core/cli/src/main.rs +++ b/core/cli/src/main.rs @@ -46,6 +46,8 @@ use iggy_cli::commands::binary_context::show_context::ShowContextCmd; use iggy_cli::commands::binary_context::use_context::UseContextCmd; use iggy_cli::commands::binary_segments::delete_segments::DeleteSegmentsCmd; use iggy_cli::commands::binary_system::snapshot::GetSnapshotCmd; +#[cfg(feature = "login-session")] +use iggy_cli::commands::cli_command::DIAGNOSTIC_TARGET; use iggy_cli::commands::cli_command::{CliCommand, PRINT_TARGET}; use iggy_cli::commands::{ binary_client::{get_client::GetClientCmd, get_clients::GetClientsCmd}, @@ -75,7 +77,7 @@ use iggy_cli::commands::{ create_stream::CreateStreamCmd, delete_stream::DeleteStreamCmd, get_stream::GetStreamCmd, get_streams::GetStreamsCmd, purge_stream::PurgeStreamCmd, update_stream::UpdateStreamCmd, }, - binary_system::{me::GetMeCmd, ping::PingCmd, stats::GetStatsCmd}, + binary_system::{me::GetMeCmd, options::DescribeOptionsCmd, ping::PingCmd, stats::GetStatsCmd}, binary_topics::{ create_topic::CreateTopicCmd, delete_topic::DeleteTopicCmd, get_topic::GetTopicCmd, get_topics::GetTopicsCmd, purge_topic::PurgeTopicCmd, update_topic::UpdateTopicCmd, @@ -131,7 +133,7 @@ fn get_command( args.name.clone(), args.message_expiry.clone().into(), args.max_topic_size, - args.replication_factor, + args.set.iter().cloned().collect(), )), TopicAction::Delete(args) => Box::new(DeleteTopicCmd::new( args.stream_id.clone(), @@ -144,7 +146,6 @@ fn get_command( args.name.clone(), args.message_expiry.clone().into(), args.max_topic_size, - args.replication_factor, )), TopicAction::Get(args) => Box::new(GetTopicCmd::new( args.stream_id.clone(), @@ -181,6 +182,7 @@ fn get_command( }, Command::Ping(args) => Box::new(PingCmd::new(args.count)), Command::Me => Box::new(GetMeCmd::new()), + Command::Options(args) => Box::new(DescribeOptionsCmd::new(args.scope)), Command::Stats(args) => Box::new(GetStatsCmd::new(cli_options.quiet, args.output.into())), Command::Snapshot(args) => Box::new(GetSnapshotCmd::new( args.compression, @@ -382,7 +384,7 @@ async fn main() -> Result<(), IggyCmdError> { // token-name lookup) sees the backend regardless of code-path ordering. #[cfg(feature = "login-session")] if let Err(e) = ensure_default_store() { - tracing::warn!(target: PRINT_TARGET, "keyring backend unavailable: {e}"); + tracing::warn!(target: DIAGNOSTIC_TARGET, "keyring backend unavailable: {e}"); } let command = args.command.clone().unwrap(); diff --git a/core/common/src/error/iggy_error.rs b/core/common/src/error/iggy_error.rs index fead3058d6..d80006ade6 100644 --- a/core/common/src/error/iggy_error.rs +++ b/core/common/src/error/iggy_error.rs @@ -400,6 +400,16 @@ pub enum IggyError { InvalidBatchChecksum(u64, u64, u64) = 4039, #[error("Invalid header kind code: {0}")] InvalidHeaderKind(u8) = 4040, + /// The key is only populated where the error text itself travels, which + /// today means HTTP. A binary transport sends the code alone, so a client + /// rebuilding this variant from it renders an empty key; `DescribeOptions` + /// is what tells that client which keys exist. + #[error("Unsupported option key: {0}")] + UnsupportedOptionKey(String) = 4041, + #[error("Invalid option value for key: {0}")] + InvalidOptionValue(String) = 4042, + #[error("Options block exceeds its limits: {0}")] + OptionsBlockTooLarge(String) = 4043, #[error("Cannot sed messages due to client disconnection")] CannotSendMessagesDueToClientDisconnection = 4050, #[error("Background send error")] diff --git a/core/common/src/http/streams/update_stream.rs b/core/common/src/http/streams/update_stream.rs index dc64962b38..1c92741ace 100644 --- a/core/common/src/http/streams/update_stream.rs +++ b/core/common/src/http/streams/update_stream.rs @@ -20,6 +20,7 @@ use crate::Identifier; use crate::Validatable; use crate::error::IggyError; use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; /// `UpdateStream` command is used to update an existing stream. /// It has additional payload: @@ -32,6 +33,10 @@ pub struct UpdateStream { pub stream_id: Identifier, /// Unique stream name (string), max length is 255 characters. pub name: String, + /// Additional stream options as string key-values. Restricted to the keys + /// an update may change; anything else is rejected. + #[serde(default)] + pub options: BTreeMap, } impl Default for UpdateStream { @@ -39,6 +44,7 @@ impl Default for UpdateStream { UpdateStream { stream_id: Identifier::default(), name: "stream".to_string(), + options: BTreeMap::new(), } } } diff --git a/core/common/src/http/topics/create_topic.rs b/core/common/src/http/topics/create_topic.rs index e476f123a0..99a18f706a 100644 --- a/core/common/src/http/topics/create_topic.rs +++ b/core/common/src/http/topics/create_topic.rs @@ -23,6 +23,7 @@ use crate::error::IggyError; use crate::utils::expiry::IggyExpiry; use crate::utils::topic_size::MaxTopicSize; use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; /// `CreateTopic` command is used to create a new topic in a stream. /// It has additional payload: @@ -31,7 +32,6 @@ use serde::{Deserialize, Serialize}; /// - `message_expiry` - message expiry, if `NeverExpire` then messages will never expire. /// - `max_topic_size` - maximum size of the topic, if `Unlimited` then topic size is unlimited. /// Can't be lower than segment size in the config. -/// - `replication_factor` - replication factor for the topic. /// - `name` - unique topic name, max length is 255 characters. #[derive(Debug, Serialize, Deserialize, PartialEq, Clone)] pub struct CreateTopic { @@ -47,10 +47,16 @@ pub struct CreateTopic { /// Max topic size, if `Unlimited` then topic size is unlimited. /// Can't be lower than segment size in the config. pub max_topic_size: MaxTopicSize, - /// Replication factor for the topic. - pub replication_factor: Option, /// Unique topic name, max length is 255 characters. pub name: String, + /// Additional topic options as string key-values, parsed by the server + /// with the same rules as its config file. Unknown keys are rejected. + /// + /// A response renders each option as `{"value": "", "explicit": + /// bool}`, in this same string form, so the values a `GET` reports can be + /// sent straight back here. + #[serde(default)] + pub options: BTreeMap, } impl Default for CreateTopic { @@ -61,8 +67,8 @@ impl Default for CreateTopic { compression_algorithm: CompressionAlgorithm::None, message_expiry: IggyExpiry::NeverExpire, max_topic_size: MaxTopicSize::ServerDefault, - replication_factor: None, name: "topic".to_string(), + options: BTreeMap::new(), } } } @@ -77,12 +83,6 @@ impl Validatable for CreateTopic { return Err(IggyError::TooManyPartitions); } - if let Some(replication_factor) = self.replication_factor - && replication_factor == 0 - { - return Err(IggyError::InvalidReplicationFactor); - } - Ok(()) } } diff --git a/core/common/src/http/topics/update_topic.rs b/core/common/src/http/topics/update_topic.rs index ac2303f9ae..7208c88b9d 100644 --- a/core/common/src/http/topics/update_topic.rs +++ b/core/common/src/http/topics/update_topic.rs @@ -23,16 +23,19 @@ use crate::error::IggyError; use crate::utils::expiry::IggyExpiry; use crate::utils::topic_size::MaxTopicSize; use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; /// `UpdateTopic` command is used to update a topic in a stream. /// It has additional payload: /// - `stream_id` - unique stream ID (numeric or name). /// - `topic_id` - unique topic ID (numeric or name). -/// - `message_expiry` - message expiry, if `NeverExpire` then messages will never expire. -/// - `max_topic_size` - maximum size of the topic in bytes, if `Unlimited` then topic size is unlimited. -/// Can't be lower than segment size in the config. -/// - `replication_factor` - replication factor for the topic. /// - `name` - unique topic name, max length is 255 characters. +/// - `compression_algorithm`, `message_expiry`, `max_topic_size` - omit a field +/// to leave the topic's current value alone. Named here for REST ergonomics; +/// the server folds them into the same option keys the binary protocol uses, +/// so there is still one source per setting. +/// - `options` - additional option keys as strings; only keys the update path +/// accepts are allowed. #[derive(Debug, Serialize, Deserialize, PartialEq, Clone)] pub struct UpdateTopic { /// Unique stream ID (numeric or name). @@ -41,17 +44,21 @@ pub struct UpdateTopic { /// Unique topic ID (numeric or name). #[serde(skip)] pub topic_id: Identifier, - /// Compression algorithm for the topic. - pub compression_algorithm: CompressionAlgorithm, - /// Message expiry, if `NeverExpire` then messages will never expire. - pub message_expiry: IggyExpiry, - /// Max topic size, if `Unlimited` then topic size is unlimited. - /// Can't be lower than segment size in the config. - pub max_topic_size: MaxTopicSize, - /// Replication factor for the topic. - pub replication_factor: Option, + /// Compression algorithm; omit to leave the current one alone. + #[serde(default)] + pub compression_algorithm: Option, + /// Message expiry; omit to leave the current one alone. + #[serde(default)] + pub message_expiry: Option, + /// Max topic size; omit to leave the current one alone. + #[serde(default)] + pub max_topic_size: Option, /// Unique topic name, max length is 255 characters. pub name: String, + /// Additional topic options as string key-values. Restricted to the keys + /// an update may change; anything else is rejected. + #[serde(default)] + pub options: BTreeMap, } impl Default for UpdateTopic { @@ -59,11 +66,11 @@ impl Default for UpdateTopic { UpdateTopic { stream_id: Identifier::default(), topic_id: Identifier::default(), - compression_algorithm: Default::default(), - message_expiry: IggyExpiry::NeverExpire, - max_topic_size: MaxTopicSize::ServerDefault, - replication_factor: None, + compression_algorithm: None, + message_expiry: None, + max_topic_size: None, name: "topic".to_string(), + options: BTreeMap::new(), } } } @@ -74,12 +81,6 @@ impl Validatable for UpdateTopic { return Err(IggyError::InvalidTopicName); } - if let Some(replication_factor) = self.replication_factor - && replication_factor == 0 - { - return Err(IggyError::InvalidReplicationFactor); - } - Ok(()) } } diff --git a/core/common/src/http/users/update_user.rs b/core/common/src/http/users/update_user.rs index 0bce1d423c..f29711cb36 100644 --- a/core/common/src/http/users/update_user.rs +++ b/core/common/src/http/users/update_user.rs @@ -21,6 +21,7 @@ use crate::UserStatus; use crate::Validatable; use crate::error::IggyError; use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; /// `UpdateUser` command is used to update a user's username and status. /// It has additional payload: @@ -33,6 +34,10 @@ pub struct UpdateUser { pub user_id: Identifier, pub username: Option, pub status: Option, + /// Additional user options as string key-values. Restricted to the keys + /// an update may change; anything else is rejected. + #[serde(default)] + pub options: BTreeMap, } impl Validatable for UpdateUser { diff --git a/core/common/src/lib.rs b/core/common/src/lib.rs index b3bfc1c8fc..777b5ef276 100644 --- a/core/common/src/lib.rs +++ b/core/common/src/lib.rs @@ -109,6 +109,7 @@ pub use types::either::Either; pub use types::http::HttpMethod; pub use types::identifier::*; pub use types::message::*; +pub use types::options::*; pub use types::partition::*; pub use types::permissions::permissions_global::*; pub use types::permissions::personal_access_token::*; diff --git a/core/common/src/traits/binary_impls/streams.rs b/core/common/src/traits/binary_impls/streams.rs index c783397870..81375d97d9 100644 --- a/core/common/src/traits/binary_impls/streams.rs +++ b/core/common/src/traits/binary_impls/streams.rs @@ -17,8 +17,9 @@ use crate::traits::binary_auth::fail_if_not_authenticated; use crate::wire_conversions::{identifier_to_wire, streams_from_wire}; -use crate::{BinaryClient, Identifier, IggyError, Stream, StreamClient, StreamDetails}; -use iggy_binary_protocol::WireName; +use crate::{ + BinaryClient, Identifier, IggyError, Stream, StreamClient, StreamDetails, StreamUpdateOptions, +}; use iggy_binary_protocol::codec::WireEncode; use iggy_binary_protocol::codes::{ CREATE_STREAM_CODE, DELETE_STREAM_CODE, GET_STREAM_CODE, GET_STREAMS_CODE, PURGE_STREAM_CODE, @@ -30,6 +31,7 @@ use iggy_binary_protocol::requests::streams::{ }; use iggy_binary_protocol::responses::streams::get_stream::GetStreamResponse; use iggy_binary_protocol::responses::streams::get_streams::GetStreamsResponse; +use iggy_binary_protocol::{WireName, WireOptions}; #[async_trait::async_trait] impl StreamClient for B { @@ -58,7 +60,7 @@ impl StreamClient for B { return Ok(Vec::new()); } let wire_resp = super::decode_response::(&response)?; - Ok(streams_from_wire(wire_resp)) + Ok(streams_from_wire(wire_resp)?) } async fn create_stream(&self, name: &str) -> Result { @@ -67,14 +69,23 @@ impl StreamClient for B { let response = self .send_raw_with_response( CREATE_STREAM_CODE, - CreateStreamRequest { name: wire_name }.to_bytes(), + CreateStreamRequest { + name: wire_name, + options: WireOptions::empty(), + } + .to_bytes(), ) .await?; let wire_resp = super::decode_response::(&response)?; Ok(StreamDetails::try_from(wire_resp)?) } - async fn update_stream(&self, stream_id: &Identifier, name: &str) -> Result<(), IggyError> { + async fn update_stream( + &self, + stream_id: &Identifier, + name: &str, + options: &StreamUpdateOptions, + ) -> Result<(), IggyError> { fail_if_not_authenticated(self).await?; let wire_id = identifier_to_wire(stream_id)?; let wire_name = WireName::new(name).map_err(|_| IggyError::InvalidFormat)?; @@ -83,6 +94,7 @@ impl StreamClient for B { UpdateStreamRequest { stream_id: wire_id, name: wire_name, + options: options.to_wire()?, } .to_bytes(), ) diff --git a/core/common/src/traits/binary_impls/system.rs b/core/common/src/traits/binary_impls/system.rs index 09f82a997d..b9a526c09f 100644 --- a/core/common/src/traits/binary_impls/system.rs +++ b/core/common/src/traits/binary_impls/system.rs @@ -18,20 +18,21 @@ use crate::traits::binary_auth::fail_if_not_authenticated; use crate::wire_conversions::clients_from_wire; use crate::{ - BinaryClient, ClientInfo, ClientInfoDetails, IggyDuration, IggyError, Snapshot, - SnapshotCompression, Stats, SystemClient, SystemSnapshotType, + BinaryClient, ClientInfo, ClientInfoDetails, IggyDuration, IggyError, OptionSpec, OptionsScope, + Snapshot, SnapshotCompression, Stats, SystemClient, SystemSnapshotType, }; use iggy_binary_protocol::codec::WireEncode; use iggy_binary_protocol::codes::{ - GET_CLIENT_CODE, GET_CLIENTS_CODE, GET_ME_CODE, GET_SNAPSHOT_FILE_CODE, GET_STATS_CODE, - PING_CODE, + DESCRIBE_OPTIONS_CODE, GET_CLIENT_CODE, GET_CLIENTS_CODE, GET_ME_CODE, GET_SNAPSHOT_FILE_CODE, + GET_STATS_CODE, PING_CODE, }; use iggy_binary_protocol::requests::system::{ - GetClientRequest, GetClientsRequest, GetMeRequest, GetSnapshotRequest, GetStatsRequest, - PingRequest, + DescribeOptionsRequest, GetClientRequest, GetClientsRequest, GetMeRequest, GetSnapshotRequest, + GetStatsRequest, PingRequest, }; use iggy_binary_protocol::responses::clients::get_client::ClientDetailsResponse; use iggy_binary_protocol::responses::clients::get_clients::GetClientsResponse; +use iggy_binary_protocol::responses::system::DescribeOptionsResponse; use iggy_binary_protocol::responses::system::get_stats::StatsResponse; #[async_trait::async_trait] @@ -44,6 +45,21 @@ impl SystemClient for B { Ok(Stats::from(wire_resp)) } + async fn describe_options(&self, scope: OptionsScope) -> Result, IggyError> { + fail_if_not_authenticated(self).await?; + let response = self + .send_raw_with_response( + DESCRIBE_OPTIONS_CODE, + DescribeOptionsRequest { + scope: scope.as_code(), + } + .to_bytes(), + ) + .await?; + let wire_resp = super::decode_response::(&response)?; + crate::wire_conversions::option_specs_from_wire(wire_resp) + } + async fn get_me(&self) -> Result { fail_if_not_authenticated(self).await?; let response = self diff --git a/core/common/src/traits/binary_impls/topics.rs b/core/common/src/traits/binary_impls/topics.rs index b0bfd1a5d8..976c8b9d1c 100644 --- a/core/common/src/traits/binary_impls/topics.rs +++ b/core/common/src/traits/binary_impls/topics.rs @@ -18,8 +18,8 @@ use crate::traits::binary_auth::fail_if_not_authenticated; use crate::wire_conversions::{identifier_to_wire, topics_from_wire}; use crate::{ - BinaryClient, CompressionAlgorithm, Identifier, IggyError, IggyExpiry, MaxTopicSize, Topic, - TopicClient, TopicDetails, + BinaryClient, DEFAULT_PARTITIONS_COUNT, Identifier, IggyError, Topic, TopicClient, + TopicCreateOptions, TopicDetails, TopicUpdateOptions, }; use iggy_binary_protocol::WireName; use iggy_binary_protocol::codec::WireEncode; @@ -84,11 +84,7 @@ impl TopicClient for B { &self, stream_id: &Identifier, name: &str, - partitions_count: u32, - compression_algorithm: CompressionAlgorithm, - replication_factor: Option, - message_expiry: IggyExpiry, - max_topic_size: MaxTopicSize, + options: &TopicCreateOptions, ) -> Result { fail_if_not_authenticated(self).await?; let wire_stream_id = identifier_to_wire(stream_id)?; @@ -98,12 +94,9 @@ impl TopicClient for B { CREATE_TOPIC_CODE, CreateTopicRequest { stream_id: wire_stream_id, - partitions_count, - compression_algorithm: compression_algorithm.as_code(), - message_expiry: u64::from(message_expiry), - max_topic_size: u64::from(max_topic_size), - replication_factor: replication_factor.unwrap_or(0), + partitions_count: options.partitions_count.unwrap_or(DEFAULT_PARTITIONS_COUNT), name: wire_name, + options: options.to_wire()?, } .to_bytes(), ) @@ -117,10 +110,7 @@ impl TopicClient for B { stream_id: &Identifier, topic_id: &Identifier, name: &str, - compression_algorithm: CompressionAlgorithm, - replication_factor: Option, - message_expiry: IggyExpiry, - max_topic_size: MaxTopicSize, + options: &TopicUpdateOptions, ) -> Result<(), IggyError> { fail_if_not_authenticated(self).await?; let wire_stream_id = identifier_to_wire(stream_id)?; @@ -131,11 +121,8 @@ impl TopicClient for B { UpdateTopicRequest { stream_id: wire_stream_id, topic_id: wire_topic_id, - compression_algorithm: compression_algorithm.as_code(), - message_expiry: u64::from(message_expiry), - max_topic_size: u64::from(max_topic_size), - replication_factor: replication_factor.unwrap_or(0), name: wire_name, + options: options.to_wire()?, } .to_bytes(), ) diff --git a/core/common/src/traits/binary_impls/users.rs b/core/common/src/traits/binary_impls/users.rs index a375803668..eb785109a6 100644 --- a/core/common/src/traits/binary_impls/users.rs +++ b/core/common/src/traits/binary_impls/users.rs @@ -19,9 +19,8 @@ use crate::traits::binary_auth::fail_if_not_authenticated; use crate::wire_conversions::{identifier_to_wire, permissions_to_wire, users_from_wire}; use crate::{ BinaryClient, ClientState, DiagnosticEvent, Identifier, IdentityInfo, IggyError, Permissions, - UserClient, UserInfo, UserInfoDetails, UserStatus, + UserClient, UserInfo, UserInfoDetails, UserStatus, UserUpdateOptions, }; -use iggy_binary_protocol::WireName; use iggy_binary_protocol::codec::WireEncode; use iggy_binary_protocol::codes::LOGIN_REGISTER_CODE; use iggy_binary_protocol::codes::{ @@ -35,6 +34,7 @@ use iggy_binary_protocol::requests::users::{ }; use iggy_binary_protocol::responses::users::LoginRegisterResponse; use iggy_binary_protocol::responses::users::{GetUsersResponse, UserDetailsResponse}; +use iggy_binary_protocol::{WireName, WireOptions}; use secrecy::SecretString; #[async_trait::async_trait] @@ -87,6 +87,7 @@ impl UserClient for B { password: password.to_string(), status: status.as_code(), permissions: wire_perms, + options: WireOptions::empty(), } .to_bytes(), ) @@ -111,6 +112,7 @@ impl UserClient for B { user_id: &Identifier, username: Option<&str>, status: Option, + options: &UserUpdateOptions, ) -> Result<(), IggyError> { fail_if_not_authenticated(self).await?; let wire_id = identifier_to_wire(user_id)?; @@ -124,6 +126,7 @@ impl UserClient for B { user_id: wire_id, username: wire_username, status: status.map(|s| s.as_code()), + options: options.to_wire()?, } .to_bytes(), ) diff --git a/core/common/src/traits/stream_client.rs b/core/common/src/traits/stream_client.rs index 41aa0aee42..e5fa1ea462 100644 --- a/core/common/src/traits/stream_client.rs +++ b/core/common/src/traits/stream_client.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use crate::{Identifier, IggyError, Stream, StreamDetails}; +use crate::{Identifier, IggyError, Stream, StreamDetails, StreamUpdateOptions}; use async_trait::async_trait; /// This trait defines the methods to interact with the stream module. @@ -36,7 +36,12 @@ pub trait StreamClient { /// Update a stream by unique ID or name. /// /// Authentication is required, and the permission to manage the streams. - async fn update_stream(&self, stream_id: &Identifier, name: &str) -> Result<(), IggyError>; + async fn update_stream( + &self, + stream_id: &Identifier, + name: &str, + options: &StreamUpdateOptions, + ) -> Result<(), IggyError>; /// Delete a stream by unique ID or name. /// /// Authentication is required, and the permission to manage the streams. diff --git a/core/common/src/traits/system_client.rs b/core/common/src/traits/system_client.rs index 4ccee4de80..fe85ef9072 100644 --- a/core/common/src/traits/system_client.rs +++ b/core/common/src/traits/system_client.rs @@ -16,8 +16,8 @@ // under the License. use crate::{ - ClientInfo, ClientInfoDetails, IggyDuration, IggyError, Snapshot, SnapshotCompression, Stats, - SystemSnapshotType, + ClientInfo, ClientInfoDetails, IggyDuration, IggyError, OptionSpec, OptionsScope, Snapshot, + SnapshotCompression, Stats, SystemSnapshotType, }; use async_trait::async_trait; @@ -40,6 +40,13 @@ pub trait SystemClient { /// /// Authentication is required, and the permission to read the server info. async fn get_clients(&self) -> Result, IggyError>; + /// Describe the option catalog for a resource scope: the keys its create + /// command accepts, their canonical kinds, and this server's current + /// defaults. Unknown keys are rejected at create, so this is the way to + /// discover support rather than probing. + /// + /// Authentication is required. + async fn describe_options(&self, scope: OptionsScope) -> Result, IggyError>; /// Ping the server to check if it's alive. async fn ping(&self) -> Result<(), IggyError>; async fn heartbeat_interval(&self) -> IggyDuration; diff --git a/core/common/src/traits/topic_client.rs b/core/common/src/traits/topic_client.rs index 26b31be96a..930fa1ead8 100644 --- a/core/common/src/traits/topic_client.rs +++ b/core/common/src/traits/topic_client.rs @@ -15,9 +15,7 @@ // specific language governing permissions and limitations // under the License. -use crate::{ - CompressionAlgorithm, Identifier, IggyError, IggyExpiry, MaxTopicSize, Topic, TopicDetails, -}; +use crate::{Identifier, IggyError, Topic, TopicCreateOptions, TopicDetails, TopicUpdateOptions}; use async_trait::async_trait; /// This trait defines the methods to interact with the topic module. @@ -38,17 +36,15 @@ pub trait TopicClient { async fn get_topics(&self, stream_id: &Identifier) -> Result, IggyError>; /// Create a new topic. /// - /// Authentication is required, and the permission to manage the topics. - #[allow(clippy::too_many_arguments)] + /// Every knob rides `options`; an absent key resolves against the + /// server's defaults at admission, so a future key costs no signature + /// change here. Authentication is required, and the permission to manage + /// the topics. async fn create_topic( &self, stream_id: &Identifier, name: &str, - partitions_count: u32, - compression_algorithm: CompressionAlgorithm, - replication_factor: Option, - message_expiry: IggyExpiry, - max_topic_size: MaxTopicSize, + options: &TopicCreateOptions, ) -> Result; /// Update a topic by unique ID or name. /// @@ -58,10 +54,7 @@ pub trait TopicClient { stream_id: &Identifier, topic_id: &Identifier, name: &str, - compression_algorithm: CompressionAlgorithm, - replication_factor: Option, - message_expiry: IggyExpiry, - max_topic_size: MaxTopicSize, + options: &TopicUpdateOptions, ) -> Result<(), IggyError>; /// Delete a topic by unique ID or name. /// diff --git a/core/common/src/traits/user_client.rs b/core/common/src/traits/user_client.rs index 43fdddbb84..97766964c3 100644 --- a/core/common/src/traits/user_client.rs +++ b/core/common/src/traits/user_client.rs @@ -17,6 +17,7 @@ use crate::{ Identifier, IdentityInfo, IggyError, Permissions, UserInfo, UserInfoDetails, UserStatus, + UserUpdateOptions, }; use async_trait::async_trait; @@ -53,6 +54,7 @@ pub trait UserClient { user_id: &Identifier, username: Option<&str>, status: Option, + options: &UserUpdateOptions, ) -> Result<(), IggyError>; /// Update the permissions of a user by unique ID or username. /// diff --git a/core/common/src/types/message/user_headers.rs b/core/common/src/types/message/user_headers.rs index 54e2522f4d..07223ce80b 100644 --- a/core/common/src/types/message/user_headers.rs +++ b/core/common/src/types/message/user_headers.rs @@ -616,6 +616,28 @@ impl HeaderField { _marker: PhantomData, } } + + /// Wrap raw wire bytes as a field of `kind`. + /// + /// The option catalog hands out each key's default as a kind plus bytes, so + /// anything rendering a catalog entry needs a way back to a typed field. + /// + /// # Errors + /// + /// `IggyError::InvalidHeaderValue` when a fixed-size kind's payload is the + /// wrong length, or the payload exceeds the field length limit. + pub fn from_raw(kind: HeaderKind, value: &[u8]) -> Result { + // 255 is the field length the TLV codec's own `TryFrom` impls enforce. + if value.is_empty() || value.len() > 255 { + return Err(IggyError::InvalidHeaderValue); + } + if let Some(expected) = kind.expected_size() + && value.len() != expected + { + return Err(IggyError::InvalidHeaderValue); + } + Ok(Self::new_unchecked(kind, value)) + } } impl Ord for HeaderField { diff --git a/core/common/src/types/mod.rs b/core/common/src/types/mod.rs index f1a2f37afe..feb5b4b566 100644 --- a/core/common/src/types/mod.rs +++ b/core/common/src/types/mod.rs @@ -27,6 +27,7 @@ pub(crate) mod either; pub(crate) mod http; pub(crate) mod identifier; pub(crate) mod message; +pub(crate) mod options; pub(crate) mod partition; pub(crate) mod permissions; pub(crate) mod personal_access_tokens; diff --git a/core/common/src/types/options/mod.rs b/core/common/src/types/options/mod.rs new file mode 100644 index 0000000000..a7ff727f2e --- /dev/null +++ b/core/common/src/types/options/mod.rs @@ -0,0 +1,1531 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Key-value options attached to streams, topics, and users at creation. +//! +//! Options reuse the typed [`HeaderKey`] / [`HeaderValue`] machinery from +//! message user headers. The map MUST stay a `BTreeMap`: metadata snapshots +//! require deterministic ordering across replicas, and a hash map would make +//! snapshot bytes diverge per replica. +//! +//! # SDK parity +//! +//! Every SDK can set any option key, read both blocks back, and ask the server +//! which keys it accepts. What differs is only how each language spells the +//! argument, since each reuses the type it already had for message user headers: +//! +//! | SDK | Arbitrary keys on create/update | Reads the blocks | `describe_options` | +//! |--------|--------------------------------------|------------------|--------------------| +//! | Rust | `TopicCreateOptions::raw` | yes | yes | +//! | Node | `options?: OptionEntry[]` | yes | yes | +//! | Python | `options=` string dict | yes | yes | +//! | Go | trailing `options ...HeaderEntry` | yes | yes | +//! | Java | `Map` overload | yes | yes | +//! | C# | optional `IReadOnlyDictionary` | yes | yes | +//! | C++ | trailing `Vec` | yes | yes | +//! +//! One transport caveat everywhere: REST renders option values as readable +//! strings rather than the typed TLV, so a value read back over HTTP is +//! String-kinded while the same value over a binary transport carries the kind +//! the server stored. REST also reports both provenances in one map with a +//! per-entry flag, which each SDK splits into the two maps its binary path +//! produces. +//! +//! Beyond the eight typed keys, Go, Java, C# and C++ ship typed constructors or +//! builders for the five keys that have no named parameter of their own, so no +//! caller has to spell an option key as a bare string. +//! +//! The block's byte layout is pinned by a golden vector that Rust, Node, Go and +//! Java each assert independently, so a new encoder has a fixture to match +//! rather than a description to interpret. + +use std::collections::BTreeMap; +use std::str::FromStr; + +use iggy_binary_protocol::{WireOptions, WireUserHeaderEntry}; +use serde::{Deserialize, Serialize}; + +use crate::types::compression::compression_algorithm::CompressionAlgorithm; +use crate::types::message::{HeaderKey, HeaderKind, HeaderValue}; +use crate::{IggyByteSize, IggyError, IggyExpiry, MaxTopicSize}; + +/// A single resolved option entry. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct OptionValue { + /// The effective value, resolved by the admitting primary at creation. + pub value: HeaderValue, + /// Whether the client explicitly sent this key. Derived entries + /// (`explicit == false`) were filled from server defaults at admission + /// and would have resolved differently under other server configs. + pub explicit: bool, +} + +/// Which provenance classes an encoded options block carries. +/// +/// Responses split a resource's options into two blocks so `GetTopic` can say +/// which values the client chose and which admission filled in. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OptionsProvenance { + /// Every entry, whatever its provenance. + All, + /// Only the keys the client sent. + Explicit, + /// Only the keys admission resolved from server defaults. + Derived, +} + +impl OptionValue { + /// Whether this entry belongs in a block encoded for `provenance`. + #[must_use] + pub const fn matches(&self, provenance: OptionsProvenance) -> bool { + match provenance { + OptionsProvenance::All => true, + OptionsProvenance::Explicit => self.explicit, + OptionsProvenance::Derived => !self.explicit, + } + } + + #[must_use] + pub fn explicit(value: HeaderValue) -> Self { + Self { + value, + explicit: true, + } + } + + #[must_use] + pub fn derived(value: HeaderValue) -> Self { + Self { + value, + explicit: false, + } + } +} + +/// Options attached to a stream, topic, or user, keyed by option name. +pub type ResourceOptions = BTreeMap; + +/// JSON form for [`ResourceOptions`]. +/// +/// [`HeaderKey`] serializes as a struct (`kind` + base64 `value`), and JSON +/// object keys must be strings, so the derived impl fails outright with +/// "key must be a string" the moment a map is non-empty. Every HTTP response +/// carrying options goes through here instead, rendering the key as its plain +/// text. Binary transports are unaffected: they never touch serde. +pub mod resource_options_json { + use super::{HeaderKey, HeaderValue, OptionValue, ResourceOptions}; + use serde::de::Error as _; + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + use std::collections::BTreeMap; + use std::str::FromStr; + + /// One entry's JSON form: the value in the same string form a config file + /// or a `POST` body would carry it in, plus its provenance. + /// + /// The stored value is a typed [`HeaderValue`], and serializing that gives + /// `{"kind":"uint64","value":""}` - which no operator can read and + /// no client can feed back, since the create body takes options as a plain + /// string map. Rendering the string form makes the "re-send only the + /// explicit options" round trip actually expressible over HTTP. + #[derive(Serialize, Deserialize)] + struct OptionValueJson { + value: String, + explicit: bool, + } + + /// # Errors + /// + /// Propagates the serializer's own errors. + pub fn serialize( + options: &ResourceOptions, + serializer: S, + ) -> Result { + let readable: BTreeMap = options + .iter() + .map(|(key, option)| { + ( + String::from_utf8_lossy(key.as_bytes()).into_owned(), + OptionValueJson { + value: option.value.to_string_value(), + explicit: option.explicit, + }, + ) + }) + .collect(); + readable.serialize(serializer) + } + + /// # Errors + /// + /// Returns a deserializer error when a key is not a valid option name or a + /// value does not fit a header value. + pub fn deserialize<'de, D: Deserializer<'de>>( + deserializer: D, + ) -> Result { + let readable = BTreeMap::::deserialize(deserializer)?; + readable + .into_iter() + .map(|(key, option)| { + let key = HeaderKey::from_str(&key).map_err(D::Error::custom)?; + let value = HeaderValue::from_str(&option.value).map_err(D::Error::custom)?; + Ok(( + key, + OptionValue { + value, + explicit: option.explicit, + }, + )) + }) + .collect() + } +} + +/// Topic option catalog: the keys `CreateTopic` accepts. +pub mod topic_option_keys { + /// Compression algorithm name (`none`, `gzip`). + pub const COMPRESSION_ALGORITHM: &str = "compression_algorithm"; + /// Message expiry: `Uint64` micros or a humantime string (`7 days`). + pub const MESSAGE_EXPIRY: &str = "message_expiry"; + /// Topic size cap: `Uint64` bytes or a byte-size string (`1 GiB`). + pub const MAX_TOPIC_SIZE: &str = "max_topic_size"; + /// Segment size: `Uint64` bytes or a byte-size string (`128 MiB`). + /// Bounded: a 512-byte multiple within + /// [`super::MIN_TOPIC_SEGMENT_SIZE`]..=the server's segment ceiling. + pub const SEGMENT_SIZE: &str = "segment_size"; + /// Whether writes to this topic's partitions fsync: `Bool`, or the + /// strings `true` / `false`. + pub const ENFORCE_FSYNC: &str = "enforce_fsync"; + /// Flush the journal once it holds this many messages: `Uint32`. + /// Must be non-zero. + pub const MESSAGES_REQUIRED_TO_SAVE: &str = "messages_required_to_save"; + /// Flush the journal once it holds this many bytes: `Uint64` or a + /// byte-size string. Paired with [`Self::MESSAGES_REQUIRED_TO_SAVE`]; + /// whichever threshold trips first flushes. + pub const SIZE_OF_MESSAGES_REQUIRED_TO_SAVE: &str = "size_of_messages_required_to_save"; + /// Reserve the segment's bytes up front on a filesystem that supports it: + /// `Bool`, or the strings `true` / `false`. Pairs with + /// [`Self::SEGMENT_SIZE`] -- preallocation reserves exactly that much, so + /// the two only make sense decided together. + pub const PREALLOCATE_SEGMENTS: &str = "preallocate_segments"; +} + +/// Values an absent topic option resolves to at admission. +/// +/// These are the knobs' single source of truth: they used to live in +/// `config.toml` (`[system.topic]`, `[system.partition]`, `[system.segment]`), +/// which meant every one of them had two homes and an operator could not tell +/// which won. A topic carries whatever it was created with; anything the +/// client did not send resolves to the constant here and is persisted as a +/// derived entry, so the effective value is always visible on `GetTopic`. +/// +/// Each value matches what the shipped `config.toml` carried, so removing the +/// keys changed no behavior for a topic created without options. +pub const DEFAULT_PARTITIONS_COUNT: u32 = 1; +/// `MaxTopicSize::Unlimited` (was `[system.topic] max_size = "unlimited"`). +pub const DEFAULT_MAX_TOPIC_SIZE: u64 = u64::MAX; +/// `IggyExpiry::NeverExpire` (was `[system.topic] message_expiry = "none"`). +pub const DEFAULT_MESSAGE_EXPIRY: u64 = u64::MAX; +/// 1 GiB (was `[system.segment] size = "1 GiB"`). +pub const DEFAULT_SEGMENT_SIZE: u64 = 1024 * 1024 * 1024; +/// Was `[system.partition] enforce_fsync = false`. +pub const DEFAULT_ENFORCE_FSYNC: bool = false; +/// Was `[system.partition] messages_required_to_save = 1024`. +pub const DEFAULT_MESSAGES_REQUIRED_TO_SAVE: u32 = 1024; +/// Opt-in, unlike the `[system.segment] preallocate = true` this replaced. +/// +/// That default was never actually in force: the reservation ran through +/// `compio::spawn_blocking`, which panics the shard because shard executors +/// disable the blocking pool, so any deployment that worked at all had +/// preallocation off. With the call fixed to run inline it reserves real +/// extents, and `FALLOC_FL_KEEP_SIZE` against the 1 GiB default segment size +/// means 1 GiB of disk per partition the moment it is created -- a full test +/// sweep reserved 393 GB before this was flipped. A topic that wants the +/// latency benefit asks for it with `preallocate_segments`. +pub const DEFAULT_PREALLOCATE_SEGMENTS: bool = false; +/// 1 MiB (was `[system.partition] size_of_messages_required_to_save`). +pub const DEFAULT_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE: u64 = 1024 * 1024; + +/// Every runtime knob at its default, for a partition built with no resolved +/// topic options (simulator, unit tests). +impl Default for TopicRuntimeDefaults { + fn default() -> Self { + Self { + segment_size: IggyByteSize::from(DEFAULT_SEGMENT_SIZE), + enforce_fsync: DEFAULT_ENFORCE_FSYNC, + messages_required_to_save: DEFAULT_MESSAGES_REQUIRED_TO_SAVE, + size_of_messages_required_to_save: IggyByteSize::from( + DEFAULT_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE, + ), + preallocate_segments: DEFAULT_PREALLOCATE_SEGMENTS, + } + } +} + +/// Ceiling for `size_of_messages_required_to_save`. +/// +/// A threshold above the largest a segment may be can never trip, so every +/// committed message would sit in the journal until its segment rotates, and a +/// crash does not preserve the journal. These two thresholds used to be +/// operator-only config; anyone holding `create_topic` can set them now, so the +/// ceilings are enforced at admission. Mirrors +/// `configs::validators::SEGMENT_MAX_SIZE_BYTES`, which lives in the crate that +/// depends on this one. +pub const MAX_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE: u64 = 1024 * 1024 * 1024; + +/// Ceiling for `messages_required_to_save`, derived from the byte ceiling: a +/// segment that large cannot hold more messages than this, because a message +/// costs at least its header on disk. +pub const MAX_MESSAGES_REQUIRED_TO_SAVE: u32 = 16 * 1024 * 1024; + +/// Smallest per-topic segment size. Segments far below the shipped default +/// explode the per-partition segment count, which state transfer bounds via +/// its manifest entry cap; a partition that crosses it becomes unservable +/// for transfer, so the floor is enforced at admission. +pub const MIN_TOPIC_SEGMENT_SIZE: u64 = 1024 * 1024; + +/// Largest per-topic `segment_size` a node admits. +/// +/// The ceiling used to be computed per node as the smaller of the global +/// segment maximum and `transfer_artifact_bytes_max - max_message_size` (a +/// received segment artifact can be one whole batch past the cap, and an +/// artifact ceiling below that livelocks a partition's rejoin). Config +/// validation now refuses boot unless that subtraction is at least the segment +/// maximum, so the minimum is always the maximum and the per-node expression was +/// dead. Mirrors `configs::validators::SEGMENT_MAX_SIZE_BYTES`, which lives in +/// the crate that depends on this one and pins the two together in a test. +pub const MAX_TOPIC_SEGMENT_SIZE: u64 = 1024 * 1024 * 1024; + +/// Ceiling on what one `preallocate_segments` topic may reserve when created: +/// `segment_size * partitions_count`. +/// +/// The reservation runs inline on the shard thread - once per partition as +/// segments rotate, and again for every owned partition at boot - and +/// `FALLOC_FL_KEEP_SIZE` makes it real disk rather than a sparse hole. Without a +/// cap, one create at the 1 GiB default segment size and the maximum partition +/// count reserves a terabyte before a single message arrives. +pub const MAX_PREALLOCATED_TOPIC_BYTES: u64 = 64 * 1024 * 1024 * 1024; + +/// Validate what a preallocating topic would reserve at admission. +/// +/// `segment_size_bytes` is the topic's resolved segment size. Callers skip this +/// for a topic whose effective `preallocate_segments` is false: nothing is +/// reserved up front there, so the product does not bound anything. +/// +/// # Errors +/// +/// Returns `IggyError::InvalidOptionValue("preallocate_segments")` when the +/// reservation would exceed [`MAX_PREALLOCATED_TOPIC_BYTES`]. +pub fn validate_preallocated_topic_bytes( + segment_size_bytes: u64, + partitions_count: u32, +) -> Result<(), IggyError> { + let reserved = segment_size_bytes.saturating_mul(u64::from(partitions_count)); + if reserved > MAX_PREALLOCATED_TOPIC_BYTES { + return Err(IggyError::InvalidOptionValue( + topic_option_keys::PREALLOCATE_SEGMENTS.to_string(), + )); + } + Ok(()) +} + +/// Validate an explicit per-topic `segment_size` against its bounds. +/// +/// `ceiling` is node-derived: the smaller of the global segment maximum and +/// the state-transfer artifact budget minus one bus frame (a segment may +/// close one whole batch past its cap; an artifact ceiling below that +/// refuses a legal segment and livelocks the partition's rejoin). +/// +/// # Errors +/// +/// Returns `IggyError::InvalidOptionValue("segment_size")` when the value is +/// below [`MIN_TOPIC_SEGMENT_SIZE`], above `ceiling`, or not a 512-byte +/// multiple. +pub fn validate_topic_segment_size(size_bytes: u64, ceiling: u64) -> Result<(), IggyError> { + if size_bytes < MIN_TOPIC_SEGMENT_SIZE + || size_bytes > ceiling + || !size_bytes.is_multiple_of(512) + { + return Err(IggyError::InvalidOptionValue( + topic_option_keys::SEGMENT_SIZE.to_string(), + )); + } + Ok(()) +} + +/// Resource whose option catalog `DescribeOptions` serves. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum OptionsScope { + Topic, + Stream, + User, +} + +impl OptionsScope { + #[must_use] + pub fn as_code(&self) -> u8 { + match self { + Self::Topic => 1, + Self::Stream => 2, + Self::User => 3, + } + } + + /// # Errors + /// + /// Returns `IggyError::InvalidCommand` for an unknown scope code. + pub fn from_code(code: u8) -> Result { + match code { + 1 => Ok(Self::Topic), + 2 => Ok(Self::Stream), + 3 => Ok(Self::User), + _ => Err(IggyError::InvalidCommand), + } + } +} + +impl std::fmt::Display for OptionsScope { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Topic => write!(f, "topic"), + Self::Stream => write!(f, "stream"), + Self::User => write!(f, "user"), + } + } +} + +impl FromStr for OptionsScope { + type Err = IggyError; + + fn from_str(scope: &str) -> Result { + match scope { + "topic" => Ok(Self::Topic), + "stream" => Ok(Self::Stream), + "user" => Ok(Self::User), + _ => Err(IggyError::InvalidCommand), + } + } +} + +/// One entry of a resource's option catalog, as served by `DescribeOptions`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct OptionSpec { + /// Option key accepted by the create command. + pub key: String, + /// Canonical kind for this key: what the server encodes its default under, + /// and what a value set at CREATE is stored as whatever kind the client sent + /// it as. Create admission re-encodes the block from its own parse, so a + /// `--set segment_size=128MiB` string lands as a `Uint64`. + /// + /// An UPDATE is the exception: it stores the client's bytes verbatim, so a + /// key set that way keeps the kind it arrived in. + pub kind: HeaderKind, + /// The key's default in `kind`'s encoding; empty when the key has no + /// default. A build constant rather than a per-node value: these defaults + /// stopped being config-derived when the `[system.*]` keys became options. + #[serde(default)] + pub default_value: Vec, + /// Human-readable description. + #[serde(default)] + pub description: String, +} + +/// Every key `CreateTopic` currently accepts. Unknown keys are rejected at +/// the edge, never skipped: a silently ignored knob would hand the client +/// server defaults without it ever learning. +pub const TOPIC_OPTION_KEYS: &[&str] = &[ + topic_option_keys::COMPRESSION_ALGORITHM, + topic_option_keys::MESSAGE_EXPIRY, + topic_option_keys::MAX_TOPIC_SIZE, + topic_option_keys::SEGMENT_SIZE, + topic_option_keys::ENFORCE_FSYNC, + topic_option_keys::MESSAGES_REQUIRED_TO_SAVE, + topic_option_keys::SIZE_OF_MESSAGES_REQUIRED_TO_SAVE, + topic_option_keys::PREALLOCATE_SEGMENTS, +]; + +/// The subset of [`TOPIC_OPTION_KEYS`] an `UpdateTopic` options block may +/// carry. +/// +/// These three used to be fixed fields of the update command, which meant one +/// setting had two homes and an update always rewrote all three whether the +/// caller meant to or not. As options they are patched: a key the client did +/// not send keeps its current value. +/// +/// The partition runtime knobs (`segment_size`, `enforce_fsync`, both flush +/// thresholds, `preallocate_segments`) stay out, and not only because nothing +/// re-pushes them to a live partition. They describe how a partition's storage +/// was laid down: changing `segment_size` mid-segment leaves one segment sized +/// by the old cap and the next by the new one, and `preallocate_segments` can +/// only act on a file not yet opened. A topic gets them at creation and keeps +/// them, so its segments stay uniform. +pub const UPDATABLE_TOPIC_OPTION_KEYS: &[&str] = &[ + topic_option_keys::COMPRESSION_ALGORITHM, + topic_option_keys::MESSAGE_EXPIRY, + topic_option_keys::MAX_TOPIC_SIZE, +]; + +/// Keys an `UpdateStream` options block may carry. Empty because streams have +/// no catalog keys yet: the block exists so the first one costs a catalog +/// entry instead of another wire change, and until then every key is rejected +/// by name rather than stored and ignored. +pub const UPDATABLE_STREAM_OPTION_KEYS: &[&str] = &[]; + +/// Keys an `UpdateUser` options block may carry. Empty for the same reason as +/// [`UPDATABLE_STREAM_OPTION_KEYS`]. +pub const UPDATABLE_USER_OPTION_KEYS: &[&str] = &[]; + +/// Longest key echoed back in an option error. A rejected key is attacker-sized +/// by definition, and over HTTP the error text rides the response body back to +/// the client. +/// +/// Binary transports carry only the error code, so a TCP, QUIC or WebSocket +/// client rebuilds [`IggyError::UnsupportedOptionKey`] with an empty string and +/// never sees which key was refused. `DescribeOptions` is the discovery path +/// there - see the note on that variant. +const ERROR_KEY_PREVIEW_LEN: usize = 64; + +fn key_preview(key: &str) -> String { + key.chars().take(ERROR_KEY_PREVIEW_LEN).collect() +} + +/// Build an options map from string key-values, all marked explicit. +/// +/// Shared by the update-options types: their keys are all client-sent, so none +/// of the derived-provenance handling that create needs applies. Callers that +/// also have typed fields insert those afterwards, so a typed value wins on +/// collision and keeps its canonical kind. +/// +/// # Errors +/// +/// `UnsupportedOptionKey` when a key is empty or over 255 bytes, +/// `InvalidOptionValue` when a value is. Both bounds come from the +/// header-field codec these entries ride. +fn raw_options_map(raw: &BTreeMap) -> Result { + let mut options = ResourceOptions::new(); + for (key, value) in raw { + let header_key = HeaderKey::from_str(key) + .map_err(|_| IggyError::UnsupportedOptionKey(key_preview(key)))?; + let header_value = HeaderValue::try_from(value.as_str()) + .map_err(|_| IggyError::InvalidOptionValue(key_preview(key)))?; + options.insert(header_key, OptionValue::explicit(header_value)); + } + Ok(options) +} + +/// Encode string key-values into an options block, all marked explicit. +/// +/// # Errors +/// +/// See [`raw_options_map`]. +fn raw_options_to_wire(raw: &BTreeMap) -> Result { + crate::wire_conversions::resource_options_to_wire( + &raw_options_map(raw)?, + OptionsProvenance::All, + ) +} + +/// The options an `UpdateStream` may carry. +/// +/// Absent keys leave the stream's existing values alone -- an update patches +/// the option map, it does not replace it. See [`TopicUpdateOptions`] for why. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct StreamUpdateOptions { + /// Keys sent as `String` values, checked against + /// [`UPDATABLE_STREAM_OPTION_KEYS`] server-side. That list is empty, so + /// every key is currently refused by name; the field exists so the first + /// updatable stream key costs a catalog entry, not a wire change. + pub raw: BTreeMap, +} + +impl StreamUpdateOptions { + /// Encode the present keys into an options block. + /// + /// # Errors + /// + /// See [`raw_options_to_wire`]. + pub fn to_wire(&self) -> Result { + raw_options_to_wire(&self.raw) + } +} + +/// The options an `UpdateUser` may carry. Same patch semantics as +/// [`StreamUpdateOptions`]. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct UserUpdateOptions { + /// Keys sent as `String` values, checked against + /// [`UPDATABLE_USER_OPTION_KEYS`] server-side. That list is empty, so every + /// key is currently refused by name; the field exists so the first + /// updatable user key costs a catalog entry, not a wire change. + pub raw: BTreeMap, +} + +impl UserUpdateOptions { + /// Encode the present keys into an options block. + /// + /// # Errors + /// + /// See [`raw_options_to_wire`]. + pub fn to_wire(&self) -> Result { + raw_options_to_wire(&self.raw) + } +} + +/// This node's configured fallbacks for the runtime knobs, used to fill the +/// derived-options block for keys a client did not send. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct TopicRuntimeDefaults { + pub segment_size: IggyByteSize, + pub enforce_fsync: bool, + pub messages_required_to_save: u32, + pub size_of_messages_required_to_save: IggyByteSize, + pub preallocate_segments: bool, +} + +/// A topic's resolved runtime knobs, as carried from the metadata plane to +/// each of its partitions. `None` means "keep the shard-wide configured +/// value": topics created without an options block (simulator, unit tests) +/// have no resolved values to carry. +#[derive(Debug, Clone, Copy, Default, PartialEq)] +pub struct TopicRuntimeOptions { + pub segment_size: Option, + pub enforce_fsync: Option, + pub messages_required_to_save: Option, + pub size_of_messages_required_to_save: Option, + pub preallocate_segments: Option, +} + +impl TopicRuntimeOptions { + /// Derive the runtime knobs from a topic's persisted options map. + /// + /// Degrades per key, not per map. An entry this build cannot interpret + /// leaves its own knob unset and every other knob intact, so one key a + /// newer node wrote cannot silently drop a topic's `enforce_fsync` or + /// reset its segment size along with it. + #[must_use] + pub fn from_resource_options(options: &ResourceOptions) -> Self { + let parsed = TopicCreateOptions::from_resource_options(options); + Self { + segment_size: parsed.segment_size, + enforce_fsync: parsed.enforce_fsync, + messages_required_to_save: parsed.messages_required_to_save, + size_of_messages_required_to_save: parsed.size_of_messages_required_to_save, + preallocate_segments: parsed.preallocate_segments, + } + } +} + +/// The options an `UpdateTopic` may carry, as a type that cannot express a +/// key the update path rejects (see [`UPDATABLE_TOPIC_OPTION_KEYS`]). +/// +/// Separate from [`TopicCreateOptions`] on purpose: reusing that struct would +/// let a caller set `segment_size` on an update, encode it, and only learn at +/// the server that it was refused. Absent keys leave the topic's existing +/// value alone -- an update patches the option map, it does not replace it. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct TopicUpdateOptions { + /// `None` leaves the topic's current algorithm alone. + pub compression_algorithm: Option, + /// `None` leaves the topic's current expiry alone. + pub message_expiry: Option, + /// `None` leaves the topic's current cap alone. + pub max_topic_size: Option, + /// Keys sent as `String` values, checked against + /// [`UPDATABLE_TOPIC_OPTION_KEYS`] server-side. Lets a client reach an + /// updatable key added to the catalog after this build shipped. + pub raw: BTreeMap, +} + +impl TopicUpdateOptions { + /// Encode the present keys into an options block, in canonical kinds. + /// + /// A typed field is inserted after the raw entries, so it wins on collision + /// and keeps its canonical kind. + /// + /// # Errors + /// + /// See [`raw_options_map`]. + pub fn to_wire(&self) -> Result { + let mut options = raw_options_map(&self.raw)?; + if let Some(compression_algorithm) = self.compression_algorithm { + options.insert( + HeaderKey::from_str(topic_option_keys::COMPRESSION_ALGORITHM) + .expect("catalog key is a valid header key"), + OptionValue::explicit( + HeaderValue::try_from(compression_algorithm.to_string().as_str()) + .expect("compression name fits a header value"), + ), + ); + } + if let Some(message_expiry) = self.message_expiry { + options.insert( + HeaderKey::from_str(topic_option_keys::MESSAGE_EXPIRY) + .expect("catalog key is a valid header key"), + OptionValue::explicit(HeaderValue::from(u64::from(message_expiry))), + ); + } + if let Some(max_topic_size) = self.max_topic_size { + options.insert( + HeaderKey::from_str(topic_option_keys::MAX_TOPIC_SIZE) + .expect("catalog key is a valid header key"), + OptionValue::explicit(HeaderValue::from(u64::from(max_topic_size))), + ); + } + crate::wire_conversions::resource_options_to_wire(&options, OptionsProvenance::All) + } +} + +/// Typed view of the known topic option keys, parsed from a wire block. +/// +/// `None` means the key was absent, which always means "resolve from server +/// defaults at admission". Values that parse to their type's `ServerDefault` +/// sentinel are normalized to `None` for the same reason. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct TopicCreateOptions { + /// Partitions to allocate. NOT an option key: it fills the `CreateTopic` + /// command's own fixed field, because it is an argument to the operation + /// rather than a property of the topic (admission consumes it to compute + /// assignments, and a stored count would go stale on the first + /// `CreatePartitions`). Carried here so callers pass one bundle. + /// `None` means [`DEFAULT_PARTITIONS_COUNT`]. + pub partitions_count: Option, + pub compression_algorithm: Option, + pub message_expiry: Option, + pub max_topic_size: Option, + /// Per-topic segment size; `None` resolves against `[system.segment] + /// size` at admission. `0` is normalized to `None`. + pub segment_size: Option, + /// Per-topic fsync enforcement; `None` resolves against + /// `[system.partition] enforce_fsync`. + pub enforce_fsync: Option, + /// Per-topic message-count flush threshold; `None` resolves against + /// `[system.partition] messages_required_to_save`. `0` is rejected. + pub messages_required_to_save: Option, + /// Per-topic byte flush threshold; `None` resolves against + /// [`DEFAULT_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE`]. + pub size_of_messages_required_to_save: Option, + /// Whether this topic's segments reserve their bytes on open; `None` + /// resolves against [`DEFAULT_PREALLOCATE_SEGMENTS`]. + pub preallocate_segments: Option, + /// String-valued keys with no typed field above, parsed server-side + /// through each key's `FromStr`. Lets a client reach a key added to the + /// server catalog after this build shipped. Outbound only: a typed field + /// wins on collision in [`Self::to_wire`], and [`Self::parse`] never + /// populates it. + pub raw: BTreeMap, +} + +impl TopicCreateOptions { + /// Parse a wire options block against the topic catalog. + /// + /// # Errors + /// + /// `UnsupportedOptionKey` for a key outside [`TOPIC_OPTION_KEYS`]; + /// `InvalidOptionValue` when a value has the wrong kind or fails its + /// type's `FromStr`. + pub fn parse(options: &WireOptions) -> Result { + let mut parsed = Self::default(); + for entry in options { + // Wire validation already enforced UTF-8 string keys. + let key = String::from_utf8_lossy(entry.key); + parsed.absorb_strict(&entry, &key)?; + } + Ok(parsed) + } + + /// Parse a block that is already COMMITTED, skipping what this build + /// cannot interpret. + /// + /// Admission gated the block against the catalog on the primary, once. + /// Re-running that gate at apply would make the verdict depend on the + /// build running it, so a replica that predates a key would reject an + /// operation its peers accepted and diverge from the group permanently. + /// The catalog belongs at the edge; apply only reads what it knows. + #[must_use] + pub fn parse_committed(options: &WireOptions) -> Self { + let mut parsed = Self::default(); + for entry in options { + let key = String::from_utf8_lossy(entry.key); + // Discarding the error is the skip: see `absorb_strict`. + let _ = parsed.absorb_strict(&entry, &key); + } + parsed + } + + /// Fold one catalog entry into `self`, refusing an entry this build cannot + /// interpret: a key outside [`TOPIC_OPTION_KEYS`], or a catalog key whose + /// value kind or payload does not parse. Shared by [`Self::parse`] (wire + /// block) and [`Self::from_resource_options`] (persisted map) so both read + /// the identical key set, kinds and value bounds. + /// + /// Every arm assigns only after its own parse succeeds, which is what makes + /// discarding the error a safe skip: a refused entry leaves `self` exactly + /// as it found it. + fn absorb_strict( + &mut self, + entry: &WireUserHeaderEntry<'_>, + key: &str, + ) -> Result<(), IggyError> { + let parsed = self; + { + match key { + topic_option_keys::COMPRESSION_ALGORITHM => { + let value = parse_compression(entry, key)?; + parsed.compression_algorithm = Some(value); + } + topic_option_keys::MESSAGE_EXPIRY => { + let expiry = IggyExpiry::from(parse_u64_or(entry, key, IggyExpiry::from_str)?); + parsed.message_expiry = (expiry != IggyExpiry::ServerDefault).then_some(expiry); + } + topic_option_keys::MAX_TOPIC_SIZE => { + let size = + MaxTopicSize::from(parse_u64_or(entry, key, MaxTopicSize::from_str)?); + parsed.max_topic_size = (size != MaxTopicSize::ServerDefault).then_some(size); + } + topic_option_keys::SEGMENT_SIZE => { + let size = parse_byte_size(entry, key)?; + parsed.segment_size = (size != 0).then_some(IggyByteSize::from(size)); + } + topic_option_keys::ENFORCE_FSYNC => { + parsed.enforce_fsync = Some(parse_bool(entry, key)?); + } + topic_option_keys::MESSAGES_REQUIRED_TO_SAVE => { + let messages = parse_u32(entry, key)?; + if messages == 0 || messages > MAX_MESSAGES_REQUIRED_TO_SAVE { + return Err(IggyError::InvalidOptionValue(key.to_string())); + } + parsed.messages_required_to_save = Some(messages); + } + topic_option_keys::SIZE_OF_MESSAGES_REQUIRED_TO_SAVE => { + let size = parse_byte_size(entry, key)?; + if size > MAX_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE { + return Err(IggyError::InvalidOptionValue(key.to_string())); + } + parsed.size_of_messages_required_to_save = + (size != 0).then_some(IggyByteSize::from(size)); + } + topic_option_keys::PREALLOCATE_SEGMENTS => { + parsed.preallocate_segments = Some(parse_bool(entry, key)?); + } + _ => return Err(IggyError::UnsupportedOptionKey(key.to_string())), + } + } + Ok(()) + } + + /// Field-wise overlay: every key this block sets wins, the rest fall back + /// to `defaults`. + /// + /// Apply resolves the client block over the derived one this way. `raw` is + /// dropped: by the time a block reaches apply, every key it named has + /// already been parsed into a typed field or refused by admission. + #[must_use] + pub fn resolved_over(&self, defaults: &Self) -> Self { + Self { + partitions_count: self.partitions_count.or(defaults.partitions_count), + compression_algorithm: self + .compression_algorithm + .or(defaults.compression_algorithm), + message_expiry: self.message_expiry.or(defaults.message_expiry), + max_topic_size: self.max_topic_size.or(defaults.max_topic_size), + segment_size: self.segment_size.or(defaults.segment_size), + enforce_fsync: self.enforce_fsync.or(defaults.enforce_fsync), + messages_required_to_save: self + .messages_required_to_save + .or(defaults.messages_required_to_save), + size_of_messages_required_to_save: self + .size_of_messages_required_to_save + .or(defaults.size_of_messages_required_to_save), + preallocate_segments: self.preallocate_segments.or(defaults.preallocate_segments), + raw: BTreeMap::new(), + } + } + + /// Encode the present keys into a client options block, in canonical + /// kinds. The client-side counterpart of [`Self::parse`]. + /// + /// # Errors + /// + /// See [`Self::to_option_map`]. + pub fn to_wire(&self) -> Result { + crate::wire_conversions::resource_options_to_wire( + &self.to_option_map()?, + OptionsProvenance::All, + ) + } + + /// The present keys as a canonically-kinded option map. + /// + /// A typed field is inserted after the raw entries, so it wins on + /// collision and keeps its canonical kind. + /// + /// # Errors + /// + /// See [`raw_options_map`]. + pub fn to_option_map(&self) -> Result { + let mut options = raw_options_map(&self.raw)?; + if let Some(compression_algorithm) = self.compression_algorithm { + options.insert( + HeaderKey::from_str(topic_option_keys::COMPRESSION_ALGORITHM) + .expect("catalog key is a valid header key"), + OptionValue::explicit( + HeaderValue::try_from(compression_algorithm.to_string().as_str()) + .expect("compression name fits a header value"), + ), + ); + } + if let Some(message_expiry) = self.message_expiry { + options.insert( + HeaderKey::from_str(topic_option_keys::MESSAGE_EXPIRY) + .expect("catalog key is a valid header key"), + OptionValue::explicit(HeaderValue::from(u64::from(message_expiry))), + ); + } + if let Some(max_topic_size) = self.max_topic_size { + options.insert( + HeaderKey::from_str(topic_option_keys::MAX_TOPIC_SIZE) + .expect("catalog key is a valid header key"), + OptionValue::explicit(HeaderValue::from(u64::from(max_topic_size))), + ); + } + if let Some(segment_size) = self.segment_size { + options.insert( + HeaderKey::from_str(topic_option_keys::SEGMENT_SIZE) + .expect("catalog key is a valid header key"), + OptionValue::explicit(HeaderValue::from(segment_size.as_bytes_u64())), + ); + } + if let Some(enforce_fsync) = self.enforce_fsync { + options.insert( + HeaderKey::from_str(topic_option_keys::ENFORCE_FSYNC) + .expect("catalog key is a valid header key"), + OptionValue::explicit(HeaderValue::from(enforce_fsync)), + ); + } + if let Some(messages_required_to_save) = self.messages_required_to_save { + options.insert( + HeaderKey::from_str(topic_option_keys::MESSAGES_REQUIRED_TO_SAVE) + .expect("catalog key is a valid header key"), + OptionValue::explicit(HeaderValue::from(messages_required_to_save)), + ); + } + if let Some(size_of_messages) = self.size_of_messages_required_to_save { + options.insert( + HeaderKey::from_str(topic_option_keys::SIZE_OF_MESSAGES_REQUIRED_TO_SAVE) + .expect("catalog key is a valid header key"), + OptionValue::explicit(HeaderValue::from(size_of_messages.as_bytes_u64())), + ); + } + if let Some(preallocate_segments) = self.preallocate_segments { + options.insert( + HeaderKey::from_str(topic_option_keys::PREALLOCATE_SEGMENTS) + .expect("catalog key is a valid header key"), + OptionValue::explicit(HeaderValue::from(preallocate_segments)), + ); + } + Ok(options) + } + + /// Render the present keys as string key-values, for a transport that + /// carries options as a JSON object instead of a TLV block. + /// + /// `compression_algorithm`, `message_expiry` and `max_topic_size` are left + /// out: the JSON body has dedicated fields for those three, and sending + /// both would be the two-sources-for-one-setting the update path refuses. + /// The rest ride as strings and are parsed server-side by the same + /// `FromStr` rules a config file value goes through, so a typed field + /// survives the round trip rather than being silently dropped. + #[must_use] + pub fn to_string_options(&self) -> BTreeMap { + // Typed fields are inserted over the raw entries, matching the + // collision rule `to_wire` applies. + let mut options = self.raw.clone(); + if let Some(segment_size) = self.segment_size { + options.insert( + topic_option_keys::SEGMENT_SIZE.to_owned(), + segment_size.as_bytes_u64().to_string(), + ); + } + if let Some(enforce_fsync) = self.enforce_fsync { + options.insert( + topic_option_keys::ENFORCE_FSYNC.to_owned(), + enforce_fsync.to_string(), + ); + } + if let Some(messages_required_to_save) = self.messages_required_to_save { + options.insert( + topic_option_keys::MESSAGES_REQUIRED_TO_SAVE.to_owned(), + messages_required_to_save.to_string(), + ); + } + if let Some(size_of_messages) = self.size_of_messages_required_to_save { + options.insert( + topic_option_keys::SIZE_OF_MESSAGES_REQUIRED_TO_SAVE.to_owned(), + size_of_messages.as_bytes_u64().to_string(), + ); + } + if let Some(preallocate_segments) = self.preallocate_segments { + options.insert( + topic_option_keys::PREALLOCATE_SEGMENTS.to_owned(), + preallocate_segments.to_string(), + ); + } + options + } + + /// Encode the resolved values for every key the client did NOT send into + /// a derived-options wire block, in canonical kinds. `partitions_count` + /// is never included: it is consumed at admission. + /// + /// # Errors + /// + /// See [`crate::wire_conversions::resource_options_to_wire`]. + pub fn derived_block( + &self, + compression_algorithm: CompressionAlgorithm, + message_expiry: IggyExpiry, + max_topic_size: MaxTopicSize, + runtime_defaults: TopicRuntimeDefaults, + ) -> Result { + let mut derived = ResourceOptions::new(); + if self.compression_algorithm.is_none() { + derived.insert( + HeaderKey::from_str(topic_option_keys::COMPRESSION_ALGORITHM) + .expect("catalog key is a valid header key"), + OptionValue::derived( + HeaderValue::try_from(compression_algorithm.to_string().as_str()) + .expect("compression name fits a header value"), + ), + ); + } + if self.message_expiry.is_none() { + derived.insert( + HeaderKey::from_str(topic_option_keys::MESSAGE_EXPIRY) + .expect("catalog key is a valid header key"), + OptionValue::derived(HeaderValue::from(u64::from(message_expiry))), + ); + } + if self.max_topic_size.is_none() { + derived.insert( + HeaderKey::from_str(topic_option_keys::MAX_TOPIC_SIZE) + .expect("catalog key is a valid header key"), + OptionValue::derived(HeaderValue::from(u64::from(max_topic_size))), + ); + } + if self.segment_size.is_none() { + derived.insert( + HeaderKey::from_str(topic_option_keys::SEGMENT_SIZE) + .expect("catalog key is a valid header key"), + OptionValue::derived(HeaderValue::from( + runtime_defaults.segment_size.as_bytes_u64(), + )), + ); + } + if self.enforce_fsync.is_none() { + derived.insert( + HeaderKey::from_str(topic_option_keys::ENFORCE_FSYNC) + .expect("catalog key is a valid header key"), + OptionValue::derived(HeaderValue::from(runtime_defaults.enforce_fsync)), + ); + } + if self.messages_required_to_save.is_none() { + derived.insert( + HeaderKey::from_str(topic_option_keys::MESSAGES_REQUIRED_TO_SAVE) + .expect("catalog key is a valid header key"), + OptionValue::derived(HeaderValue::from( + runtime_defaults.messages_required_to_save, + )), + ); + } + if self.size_of_messages_required_to_save.is_none() { + derived.insert( + HeaderKey::from_str(topic_option_keys::SIZE_OF_MESSAGES_REQUIRED_TO_SAVE) + .expect("catalog key is a valid header key"), + OptionValue::derived(HeaderValue::from( + runtime_defaults + .size_of_messages_required_to_save + .as_bytes_u64(), + )), + ); + } + if self.preallocate_segments.is_none() { + derived.insert( + HeaderKey::from_str(topic_option_keys::PREALLOCATE_SEGMENTS) + .expect("catalog key is a valid header key"), + OptionValue::derived(HeaderValue::from(runtime_defaults.preallocate_segments)), + ); + } + crate::wire_conversions::resource_options_to_wire(&derived, OptionsProvenance::All) + } + + /// Parse a topic's PERSISTED options map (explicit plus admission-derived + /// entries) back into typed values. + /// + /// The persisted map is the single source of truth for a topic's knobs: + /// nothing on the STM `Topic` duplicates it per key, so a new key costs + /// one catalog entry rather than one field on every stored type. + /// + /// Infallible by design. A persisted map can hold keys a newer build + /// wrote, and refusing the whole map for one of them is what would make + /// this replica read a topic differently from the node that stored it. + /// Unreadable entries are skipped; their knobs stay unset and resolve + /// against shard-wide config exactly as an absent key does. + #[must_use] + pub fn from_resource_options(options: &ResourceOptions) -> Self { + let mut parsed = Self::default(); + for (key, option) in options { + let key = String::from_utf8_lossy(key.as_bytes()); + let entry = WireUserHeaderEntry { + key_kind: iggy_binary_protocol::WireHeaderKind(HeaderKind::String.as_code()), + key: key.as_bytes(), + value_kind: iggy_binary_protocol::WireHeaderKind(option.value.kind().as_code()), + value: option.value.as_bytes(), + }; + // Discarding the error is the skip: see `absorb_strict`. + let _ = parsed.absorb_strict(&entry, &key); + } + parsed + } +} + +fn parse_u32(entry: &WireUserHeaderEntry<'_>, key: &str) -> Result { + if entry.value_kind.0 == HeaderKind::Uint32.as_code() { + let bytes: [u8; 4] = entry + .value + .try_into() + .map_err(|_| IggyError::InvalidOptionValue(key.to_string()))?; + return Ok(u32::from_le_bytes(bytes)); + } + if entry.value_kind.0 == HeaderKind::String.as_code() { + return std::str::from_utf8(entry.value) + .ok() + .and_then(|text| text.parse().ok()) + .ok_or_else(|| IggyError::InvalidOptionValue(key.to_string())); + } + Err(IggyError::InvalidOptionValue(key.to_string())) +} + +/// `Uint64` verbatim, or a `String` routed through `from_str` and converted +/// back to the type's `u64` sentinel encoding. +fn parse_u64_or( + entry: &WireUserHeaderEntry<'_>, + key: &str, + from_str: impl Fn(&str) -> Result, +) -> Result +where + u64: From, +{ + if entry.value_kind.0 == HeaderKind::Uint64.as_code() { + let bytes: [u8; 8] = entry + .value + .try_into() + .map_err(|_| IggyError::InvalidOptionValue(key.to_string()))?; + return Ok(u64::from_le_bytes(bytes)); + } + if entry.value_kind.0 == HeaderKind::String.as_code() { + return std::str::from_utf8(entry.value) + .ok() + .and_then(|text| from_str(text).ok()) + .map(u64::from) + .ok_or_else(|| IggyError::InvalidOptionValue(key.to_string())); + } + Err(IggyError::InvalidOptionValue(key.to_string())) +} + +/// `Uint64` verbatim, or a byte-size `String` (`128MiB`). +fn parse_byte_size(entry: &WireUserHeaderEntry<'_>, key: &str) -> Result { + parse_u64_or(entry, key, |text| { + IggyByteSize::from_str(text).map(|size| size.as_bytes_u64()) + }) +} + +fn parse_bool(entry: &WireUserHeaderEntry<'_>, key: &str) -> Result { + if entry.value_kind.0 == HeaderKind::Bool.as_code() { + // Exactly one byte of 0 or 1. Anything looser admits a value that + // `HeaderValue::as_bool` later refuses to read, so the stored map would + // hold an entry its own public accessor rejects; a multi-byte payload + // would pass this gate and only fail at apply, turning a client + // mistake into a committed state-machine rejection. + return match entry.value { + [0] => Ok(false), + [1] => Ok(true), + _ => Err(IggyError::InvalidOptionValue(key.to_string())), + }; + } + if entry.value_kind.0 == HeaderKind::String.as_code() { + return std::str::from_utf8(entry.value) + .ok() + .and_then(|text| text.parse().ok()) + .ok_or_else(|| IggyError::InvalidOptionValue(key.to_string())); + } + Err(IggyError::InvalidOptionValue(key.to_string())) +} + +fn parse_compression( + entry: &WireUserHeaderEntry<'_>, + key: &str, +) -> Result { + if entry.value_kind.0 == HeaderKind::Uint8.as_code() { + // Exactly one byte, for the same reason as `parse_bool`. + let [code] = entry.value else { + return Err(IggyError::InvalidOptionValue(key.to_string())); + }; + return CompressionAlgorithm::from_code(*code) + .map_err(|_| IggyError::InvalidOptionValue(key.to_string())); + } + if entry.value_kind.0 == HeaderKind::String.as_code() { + return std::str::from_utf8(entry.value) + .ok() + .and_then(|text| CompressionAlgorithm::from_str(text).ok()) + .ok_or_else(|| IggyError::InvalidOptionValue(key.to_string())); + } + Err(IggyError::InvalidOptionValue(key.to_string())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn to_wire_parse_roundtrip_preserves_typed_fields() { + let options = TopicCreateOptions { + compression_algorithm: Some(CompressionAlgorithm::Gzip), + message_expiry: Some(IggyExpiry::from(5_000_000u64)), + max_topic_size: Some(MaxTopicSize::from(2_000_000_000u64)), + segment_size: Some(IggyByteSize::from(134_217_728u64)), + enforce_fsync: Some(true), + messages_required_to_save: Some(500), + size_of_messages_required_to_save: Some(IggyByteSize::from(2_097_152u64)), + preallocate_segments: Some(false), + partitions_count: None, + raw: BTreeMap::new(), + }; + let parsed = TopicCreateOptions::parse(&options.to_wire().unwrap()).unwrap(); + assert_eq!(parsed, options); + } + + #[test] + fn partitions_count_never_enters_the_options_block() { + // It is a fixed field of the CreateTopic command, so encoding it as a + // TLV entry would both duplicate it and make it look like a stored + // topic setting. + let options = TopicCreateOptions { + partitions_count: Some(7), + ..TopicCreateOptions::default() + }; + assert!(options.to_wire().unwrap().is_empty()); + // ...and the key is rejected if a client hand-rolls it into the block. + let raw = TopicCreateOptions { + raw: BTreeMap::from([("partitions_count".to_string(), "7".to_string())]), + ..TopicCreateOptions::default() + }; + assert_eq!( + TopicCreateOptions::parse(&raw.to_wire().unwrap()), + Err(IggyError::UnsupportedOptionKey( + "partitions_count".to_string() + )) + ); + } + + #[test] + fn resource_options_render_as_readable_json_and_round_trip() { + // `HeaderKey` serializes as a struct, and JSON object keys must be + // strings, so the derived impl fails with "key must be a string" for + // any non-empty map -- which 500'd every HTTP response carrying + // options. Keys must render as their plain text, and values in the + // string form the create body takes them in: a base64 payload behind a + // kind tag is neither readable nor re-sendable. + #[derive(Debug, serde::Serialize, serde::Deserialize, PartialEq)] + struct Holder { + #[serde(default, with = "super::resource_options_json")] + options: ResourceOptions, + } + + let holder = Holder { + options: ResourceOptions::from([( + HeaderKey::from_str(topic_option_keys::SEGMENT_SIZE).unwrap(), + OptionValue::derived(HeaderValue::from(1024u64 * 1024)), + )]), + }; + let json = serde_json::to_string(&holder).expect("options must serialize as JSON"); + assert!( + json.contains("\"segment_size\""), + "the key must render as plain text, got: {json}" + ); + assert!( + json.contains("\"value\":\"1048576\""), + "the value must render as its string form, got: {json}" + ); + + let decoded: Holder = serde_json::from_str(&json).expect("options must round-trip"); + let (key, option) = decoded.options.iter().next().unwrap(); + assert_eq!(key.as_bytes(), topic_option_keys::SEGMENT_SIZE.as_bytes()); + assert!(!option.explicit, "provenance survives the round trip"); + // Re-read as a string-kinded value, which is exactly what the create + // body's string map produces, so the server parses it the same way. + assert_eq!(option.value.kind(), HeaderKind::String); + assert_eq!(option.value.as_bytes(), b"1048576"); + assert!( + TopicCreateOptions::parse( + &crate::wire_conversions::resource_options_to_wire( + &decoded.options, + OptionsProvenance::All + ) + .unwrap() + ) + .is_ok(), + "the rendered value must parse back through the catalog" + ); + } + + #[test] + fn zero_valued_sentinels_normalize_to_unspecified() { + // 0 is the ServerDefault sentinel for both expiry and size on the + // wire, so a client sending it means "resolve from the default" + // rather than "expire immediately" / "no space". This used to be a + // boot-time config rejection; with the keys per-topic it is a + // normalization at parse. + let options = TopicCreateOptions { + message_expiry: Some(IggyExpiry::from(0u64)), + max_topic_size: Some(MaxTopicSize::from(0u64)), + segment_size: Some(IggyByteSize::from(0u64)), + ..TopicCreateOptions::default() + }; + let parsed = TopicCreateOptions::parse(&options.to_wire().unwrap()).unwrap(); + assert_eq!(parsed.message_expiry, None); + assert_eq!(parsed.max_topic_size, None); + assert_eq!(parsed.segment_size, None); + } + + #[test] + fn sentinel_zeros_do_not_survive_re_encoding() { + // A client may send 0 to mean "resolve the default". Parsing normalizes + // it to absent, so admission puts the resolved value in the derived + // block -- but apply merges with explicit winning. If the literal 0 + // were still in the explicit block it would land back on top as the + // stored effective value, and a restart would re-parse it to absent and + // fall back to the node default rather than the value resolved at + // creation. Re-encoding from the parse is what drops it. + let sent = TopicCreateOptions { + message_expiry: Some(IggyExpiry::from(0u64)), + max_topic_size: Some(MaxTopicSize::from(0u64)), + segment_size: Some(IggyByteSize::from(0u64)), + size_of_messages_required_to_save: Some(IggyByteSize::from(0u64)), + enforce_fsync: Some(true), + ..TopicCreateOptions::default() + }; + let parsed = TopicCreateOptions::parse(&sent.to_wire().unwrap()).unwrap(); + let re_encoded = parsed.to_wire().unwrap(); + let stored = + crate::wire_conversions::resource_options_from_wire(&re_encoded, true).unwrap(); + + for key in [ + topic_option_keys::MESSAGE_EXPIRY, + topic_option_keys::MAX_TOPIC_SIZE, + topic_option_keys::SEGMENT_SIZE, + topic_option_keys::SIZE_OF_MESSAGES_REQUIRED_TO_SAVE, + ] { + assert!( + !stored.contains_key(&HeaderKey::from_str(key).unwrap()), + "{key} sentinel must not be persisted as an explicit value" + ); + } + // A non-sentinel key alongside them still rides through untouched. + assert!( + stored.contains_key(&HeaderKey::from_str(topic_option_keys::ENFORCE_FSYNC).unwrap()) + ); + } + + #[test] + fn runtime_options_derive_from_the_persisted_map() { + let options = TopicCreateOptions { + segment_size: Some(IggyByteSize::from(2_097_152u64)), + enforce_fsync: Some(true), + messages_required_to_save: Some(9), + ..TopicCreateOptions::default() + }; + // What admission persists: the client block merged with derived + // defaults, keyed by HeaderKey. The channel reads exactly this. + let persisted = + crate::wire_conversions::resource_options_from_wire(&options.to_wire().unwrap(), true) + .unwrap(); + let runtime = TopicRuntimeOptions::from_resource_options(&persisted); + assert_eq!(runtime.segment_size, Some(IggyByteSize::from(2_097_152u64))); + assert_eq!(runtime.enforce_fsync, Some(true)); + assert_eq!(runtime.messages_required_to_save, Some(9)); + assert_eq!(runtime.size_of_messages_required_to_save, None); + } + + #[test] + fn typed_field_wins_over_raw_entry_for_the_same_key() { + let options = TopicCreateOptions { + enforce_fsync: Some(true), + raw: BTreeMap::from([("enforce_fsync".to_string(), "false".to_string())]), + ..TopicCreateOptions::default() + }; + let parsed = TopicCreateOptions::parse(&options.to_wire().unwrap()).unwrap(); + assert_eq!(parsed.enforce_fsync, Some(true)); + } + + #[test] + fn raw_entries_ride_as_strings_and_unknown_keys_reject() { + // `prepare_queue_depth` stays server-side on purpose: it is a + // node-resource bound (the view-change wire caps it), so a + // client-chosen value is an amplifier rather than a topic property. + let options = TopicCreateOptions { + raw: BTreeMap::from([("prepare_queue_depth".to_string(), "64".to_string())]), + ..TopicCreateOptions::default() + }; + assert_eq!( + TopicCreateOptions::parse(&options.to_wire().unwrap()), + Err(IggyError::UnsupportedOptionKey( + "prepare_queue_depth".to_string() + )) + ); + + // Raw entries for catalog keys parse via the config-file rules. + let options = TopicCreateOptions { + raw: BTreeMap::from([ + ("segment_size".to_string(), "128MiB".to_string()), + ("enforce_fsync".to_string(), "true".to_string()), + ( + "size_of_messages_required_to_save".to_string(), + "4KiB".to_string(), + ), + ]), + ..TopicCreateOptions::default() + }; + let parsed = TopicCreateOptions::parse(&options.to_wire().unwrap()).unwrap(); + assert_eq!( + parsed.segment_size, + Some(IggyByteSize::from(134_217_728u64)) + ); + assert_eq!(parsed.enforce_fsync, Some(true)); + assert_eq!( + parsed.size_of_messages_required_to_save, + Some(IggyByteSize::from(4096u64)) + ); + } + + #[test] + fn string_values_parse_via_config_rules() { + let options = TopicCreateOptions { + raw: BTreeMap::from([ + ("message_expiry".to_string(), "5s".to_string()), + ("max_topic_size".to_string(), "2GB".to_string()), + ("compression_algorithm".to_string(), "gzip".to_string()), + ]), + ..TopicCreateOptions::default() + }; + let parsed = TopicCreateOptions::parse(&options.to_wire().unwrap()).unwrap(); + assert_eq!(parsed.message_expiry, Some(IggyExpiry::from(5_000_000u64))); + assert_eq!( + parsed.compression_algorithm, + Some(CompressionAlgorithm::Gzip) + ); + assert!(parsed.max_topic_size.is_some()); + } + + #[test] + fn options_block_budget_matches_user_headers() { + // `MAX_OPTIONS_BYTES` duplicates `MAX_USER_HEADERS_SIZE` because + // `iggy_binary_protocol` cannot import this crate. This is the only + // place both are visible, so it is where they get tied together. + assert_eq!( + iggy_binary_protocol::MAX_OPTIONS_BYTES, + crate::MAX_USER_HEADERS_SIZE as usize, + "options inherit the user-headers byte budget; update both" + ); + } + + #[test] + fn max_options_is_reachable_within_the_byte_budget() { + // The cheapest entry is 12 bytes: kind + u32 length + one byte, for + // key and value each. If the budget ever drops below this product the + // entry cap becomes unreachable and `MAX_OPTIONS` stops being the + // limit that actually binds. + const MIN_ENTRY_BYTES: usize = 2 * (1 + 4 + 1); + assert!( + iggy_binary_protocol::MAX_OPTIONS as usize * MIN_ENTRY_BYTES + <= iggy_binary_protocol::MAX_OPTIONS_BYTES, + "MAX_OPTIONS entries must fit in MAX_OPTIONS_BYTES" + ); + } + + #[test] + fn flush_threshold_ceilings_are_derived_from_the_segment_maximum() { + // A threshold no segment can reach never trips, so committed messages + // sit in the journal - which a crash does not preserve - until the + // segment rotates. The message ceiling follows from the byte one: a + // message costs at least its header on disk. + assert_eq!( + MAX_MESSAGES_REQUIRED_TO_SAVE as u64, + MAX_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE + / crate::IGGY_MESSAGE_HEADER_SIZE + .try_into() + .unwrap_or(u64::MAX), + "message ceiling must stay derived from the byte ceiling" + ); + } + + #[test] + fn parse_rejects_flush_thresholds_above_their_ceilings() { + let over_messages = TopicCreateOptions { + raw: BTreeMap::from([( + topic_option_keys::MESSAGES_REQUIRED_TO_SAVE.to_string(), + (u64::from(MAX_MESSAGES_REQUIRED_TO_SAVE) + 1).to_string(), + )]), + ..TopicCreateOptions::default() + }; + assert!( + TopicCreateOptions::parse(&over_messages.to_wire().unwrap()).is_err(), + "a message threshold above the ceiling must be refused" + ); + + let over_bytes = TopicCreateOptions { + raw: BTreeMap::from([( + topic_option_keys::SIZE_OF_MESSAGES_REQUIRED_TO_SAVE.to_string(), + (MAX_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE + 1).to_string(), + )]), + ..TopicCreateOptions::default() + }; + assert!( + TopicCreateOptions::parse(&over_bytes.to_wire().unwrap()).is_err(), + "a byte threshold above the ceiling must be refused" + ); + + let at_ceiling = TopicCreateOptions { + messages_required_to_save: Some(MAX_MESSAGES_REQUIRED_TO_SAVE), + size_of_messages_required_to_save: Some(IggyByteSize::from( + MAX_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE, + )), + ..TopicCreateOptions::default() + }; + assert!( + TopicCreateOptions::parse(&at_ceiling.to_wire().unwrap()).is_ok(), + "the ceiling itself must remain settable" + ); + } + + #[test] + fn preallocation_cap_bounds_segment_size_times_partitions() { + assert!(validate_preallocated_topic_bytes(DEFAULT_SEGMENT_SIZE, 64).is_ok()); + assert!(validate_preallocated_topic_bytes(DEFAULT_SEGMENT_SIZE, 65).is_err()); + // The product saturates rather than wrapping into a passing value. + assert!(validate_preallocated_topic_bytes(u64::MAX, u32::MAX).is_err()); + } +} diff --git a/core/common/src/types/stream/mod.rs b/core/common/src/types/stream/mod.rs index 5a3c9ce0a6..9a48f1aae2 100644 --- a/core/common/src/types/stream/mod.rs +++ b/core/common/src/types/stream/mod.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use crate::{IggyByteSize, IggyTimestamp, Topic}; +use crate::{IggyByteSize, IggyTimestamp, ResourceOptions, Topic}; use serde::{Deserialize, Serialize}; /// `Stream` represents the highest level of logical separation of data. @@ -40,6 +40,9 @@ pub struct Stream { pub messages_count: u64, /// The total number of topics in the stream. pub topics_count: u32, + /// Creation options, all client-explicit (streams have no derived keys). + #[serde(default, with = "crate::resource_options_json")] + pub options: ResourceOptions, } /// `StreamDetails` represents the detailed information about the stream. @@ -67,4 +70,7 @@ pub struct StreamDetails { pub topics_count: u32, /// The collection of topics in the stream. pub topics: Vec, + /// Creation options, all client-explicit (streams have no derived keys). + #[serde(default, with = "crate::resource_options_json")] + pub options: ResourceOptions, } diff --git a/core/common/src/types/topic/mod.rs b/core/common/src/types/topic/mod.rs index 1135f305c9..9379c84a92 100644 --- a/core/common/src/types/topic/mod.rs +++ b/core/common/src/types/topic/mod.rs @@ -17,6 +17,7 @@ use crate::CompressionAlgorithm; use crate::Partition; +use crate::ResourceOptions; use crate::utils::byte_size::IggyByteSize; use crate::utils::expiry::IggyExpiry; use crate::utils::timestamp::IggyTimestamp; @@ -31,7 +32,6 @@ use serde::{Deserialize, Serialize}; /// - `size`: the total size of the topic in bytes. /// - `message_expiry`: the expiry of the messages in the topic. /// - `max_topic_size`: the maximum size of the topic. -/// - `replication_factor`: replication factor for the topic. /// - `messages_count`: the total number of messages in the topic. /// - `partitions_count`: the total number of partitions in the topic. #[derive(Debug, Serialize, Deserialize)] @@ -51,12 +51,14 @@ pub struct Topic { /// The optional maximum size of the topic. /// Can't be lower than segment size in the config. pub max_topic_size: MaxTopicSize, - /// Replication factor for the topic. - pub replication_factor: u8, /// The total number of messages in the topic. pub messages_count: u64, /// The total number of partitions in the topic. pub partitions_count: u32, + /// Creation options: client-explicit keys plus admission-derived + /// defaults, distinguished by each entry's `explicit` flag. + #[serde(default, with = "crate::resource_options_json")] + pub options: ResourceOptions, } /// `TopicDetails` represents the detailed information about the topic. @@ -67,7 +69,6 @@ pub struct Topic { /// - `size`: the total size of the topic. /// - `message_expiry`: the expiry of the messages in the topic. /// - `max_topic_size`: the maximum size of the topic. -/// - `replication_factor`: replication factor for the topic. /// - `messages_count`: the total number of messages in the topic. /// - `partitions_count`: the total number of partitions in the topic. /// - `partitions`: the collection of partitions in the topic. @@ -88,12 +89,14 @@ pub struct TopicDetails { /// The optional maximum size of the topic. /// Can't be lower than segment size in the config. pub max_topic_size: MaxTopicSize, - /// Replication factor for the topic. - pub replication_factor: u8, /// The total number of messages in the topic. pub messages_count: u64, /// The total number of partitions in the topic. pub partitions_count: u32, /// The collection of partitions in the topic. pub partitions: Vec, + /// Creation options: client-explicit keys plus admission-derived + /// defaults, distinguished by each entry's `explicit` flag. + #[serde(default, with = "crate::resource_options_json")] + pub options: ResourceOptions, } diff --git a/core/common/src/types/user/user_info.rs b/core/common/src/types/user/user_info.rs index 4ec944ca7d..1c6ddec855 100644 --- a/core/common/src/types/user/user_info.rs +++ b/core/common/src/types/user/user_info.rs @@ -16,6 +16,7 @@ // under the License. use crate::Permissions; +use crate::ResourceOptions; use crate::types::user::user_status::UserStatus; use crate::utils::timestamp::IggyTimestamp; use serde::{Deserialize, Serialize}; @@ -43,6 +44,9 @@ pub struct UserInfo { pub status: UserStatus, /// The username of the user. pub username: String, + /// Creation options, all client-explicit (users have no derived keys). + #[serde(default, with = "crate::resource_options_json")] + pub options: ResourceOptions, } /// `UserInfoDetails` represents the detailed information about the user. @@ -64,4 +68,7 @@ pub struct UserInfoDetails { pub username: String, /// The optional permissions of the user. pub permissions: Option, + /// Creation options, all client-explicit (users have no derived keys). + #[serde(default, with = "crate::resource_options_json")] + pub options: ResourceOptions, } diff --git a/core/common/src/wire_conversions.rs b/core/common/src/wire_conversions.rs index 44975f159e..d87e392a02 100644 --- a/core/common/src/wire_conversions.rs +++ b/core/common/src/wire_conversions.rs @@ -26,10 +26,13 @@ use crate::{ ClusterNodeRole, ClusterNodeStatus, CompressionAlgorithm, Consumer, ConsumerGroup, ConsumerGroupDetails, ConsumerGroupInfo, ConsumerGroupMember, ConsumerOffsetInfo, GlobalPermissions, HeaderKey, HeaderKind, HeaderValue, IdKind, IdentityInfo, IggyByteSize, - IggyError, IggyExpiry, MaxTopicSize, Partition, Permissions, PersonalAccessTokenInfo, - RawPersonalAccessToken, Stats, Stream, StreamDetails, StreamPermissions, Topic, TopicDetails, - TopicPermissions, TransportEndpoints, UserInfo, UserInfoDetails, UserStatus, + IggyError, IggyExpiry, MaxTopicSize, OptionSpec, OptionValue, OptionsProvenance, Partition, + Permissions, PersonalAccessTokenInfo, RawPersonalAccessToken, ResourceOptions, Stats, Stream, + StreamDetails, StreamPermissions, Topic, TopicDetails, TopicPermissions, TransportEndpoints, + UserInfo, UserInfoDetails, UserStatus, }; +use bytes::{BufMut, BytesMut}; +use iggy_binary_protocol::primitives::options::{MAX_OPTIONS, MAX_OPTIONS_BYTES, WireOptions}; use iggy_binary_protocol::primitives::permissions::{ WireGlobalPermissions, WirePermissions, WireStreamPermissions, WireTopicPermissions, }; @@ -70,16 +73,19 @@ const WIRE_NO_USER_ID: u32 = u32::MAX; // Streams // --------------------------------------------------------------------------- -impl From for Stream { - fn from(w: StreamResponse) -> Self { - Self { +impl TryFrom for Stream { + type Error = IggyError; + + fn try_from(w: StreamResponse) -> Result { + Ok(Self { id: w.id, created_at: w.created_at.into(), name: w.name.to_string(), size: IggyByteSize::from(w.size_bytes), messages_count: w.messages_count, topics_count: w.topics_count, - } + options: resource_options_from_wire(&w.options, true)?, + }) } } @@ -101,14 +107,19 @@ impl TryFrom for StreamDetails { messages_count: w.stream.messages_count, topics_count: w.stream.topics_count, topics, + options: resource_options_from_wire(&w.stream.options, true)?, }) } } -pub fn streams_from_wire(w: GetStreamsResponse) -> Vec { - let mut streams: Vec = w.streams.into_iter().map(Stream::from).collect(); +pub fn streams_from_wire(w: GetStreamsResponse) -> Result, IggyError> { + let mut streams: Vec = w + .streams + .into_iter() + .map(Stream::try_from) + .collect::>()?; streams.sort_by_key(|s| s.id); - streams + Ok(streams) } // --------------------------------------------------------------------------- @@ -124,6 +135,7 @@ impl TryFrom for Topic { v => v.into(), }; let max_topic_size: MaxTopicSize = w.max_topic_size.into(); + let options = resource_options_from_wire_split(&w.options, &w.derived_options)?; Ok(Self { id: w.id, created_at: w.created_at.into(), @@ -134,7 +146,7 @@ impl TryFrom for Topic { message_expiry, compression_algorithm: CompressionAlgorithm::from_code(w.compression_algorithm)?, max_topic_size, - replication_factor: w.replication_factor, + options, }) } } @@ -169,9 +181,9 @@ impl TryFrom for TopicDetails { message_expiry: topic.message_expiry, compression_algorithm: topic.compression_algorithm, max_topic_size: topic.max_topic_size, - replication_factor: topic.replication_factor, partitions_count: topic.partitions_count, partitions, + options: topic.options, }) } } @@ -199,6 +211,7 @@ impl TryFrom for UserInfo { created_at: w.created_at.into(), status: UserStatus::from_code(w.status)?, username: w.username.to_string(), + options: resource_options_from_wire(&w.options, true)?, }) } } @@ -215,6 +228,7 @@ impl TryFrom for UserInfoDetails { status: user.status, username: user.username, permissions, + options: user.options, }) } } @@ -771,20 +785,46 @@ pub fn user_headers_from_wire( /// slices each TLV field without bounds checks, relying on that validation. pub(crate) fn user_headers_from_validated_slice( buf: &[u8], +) -> Result, IggyError> { + headers_from_validated_slice(buf, UnknownKinds::Reject) +} + +/// What a decode does with an entry whose kind code has no domain meaning. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum UnknownKinds { + /// Fail the whole decode. A message header the reader cannot interpret is + /// the caller's problem, not something to hide from it. + Reject, + /// Drop the entry and keep the rest. + Skip, +} + +fn headers_from_validated_slice( + buf: &[u8], + unknown: UnknownKinds, ) -> Result, IggyError> { if buf.is_empty() { return Ok(BTreeMap::new()); } let mut headers = BTreeMap::new(); for entry in WireUserHeaderIterator::new(buf) { - let key_kind = HeaderKind::from_code(entry.key_kind.0)?; + let (key_kind, value_kind) = match ( + HeaderKind::from_code(entry.key_kind.0), + HeaderKind::from_code(entry.value_kind.0), + ) { + (Ok(key_kind), Ok(value_kind)) => (key_kind, value_kind), + (Err(error), _) | (Ok(_), Err(error)) => { + if unknown == UnknownKinds::Skip { + continue; + } + return Err(error); + } + }; if let Some(expected) = key_kind.expected_size() && entry.key.len() != expected { return Err(IggyError::InvalidHeaderKey); } - - let value_kind = HeaderKind::from_code(entry.value_kind.0)?; if let Some(expected) = value_kind.expected_size() && entry.value.len() != expected { @@ -799,6 +839,159 @@ pub(crate) fn user_headers_from_validated_slice( Ok(headers) } +// -- Options conversions -- + +/// Decode a `DescribeOptions` response into domain option specs. +/// +/// # Errors +/// +/// Returns `IggyError::InvalidHeaderKind` when an entry carries an unknown +/// canonical kind code. +pub fn option_specs_from_wire( + wire: iggy_binary_protocol::responses::system::DescribeOptionsResponse, +) -> Result, IggyError> { + wire.entries + .into_iter() + .map(|entry| { + Ok(OptionSpec { + key: entry.key.to_string(), + kind: HeaderKind::from_code(entry.kind)?, + default_value: entry.default_value.to_vec(), + description: entry.description, + }) + }) + .collect() +} + +/// Decode a [`WireOptions`] block into domain resource options. +/// +/// Every decoded entry is marked with the given `explicit` flag; the wire +/// block carries no per-entry provenance. Admission calls this once for the +/// client block (`explicit == true`) and fills defaults separately. +/// +/// Entries whose kind code has no domain meaning are skipped, not rejected. +/// The wire layer forwards unknown kinds verbatim so a mixed-version cluster +/// can round-trip them, and this decode sits on the apply path, on the SDK's +/// list-decode and on the server's own reads: refusing the entry would mean an +/// old replica rejecting a commit a new one accepted, one unreadable entry +/// failing a whole `get_topics()`, and a topic that cannot be read back. +/// +/// # Errors +/// +/// Returns `IggyError::InvalidHeaderKey` / `InvalidHeaderValue` when a known +/// fixed-size kind carries a mismatched payload. +pub fn resource_options_from_wire( + wire: &iggy_binary_protocol::WireOptions, + explicit: bool, +) -> Result { + let headers = headers_from_validated_slice(wire.as_bytes(), UnknownKinds::Skip)?; + Ok(headers + .into_iter() + .map(|(key, value)| (key, OptionValue { value, explicit })) + .collect()) +} + +/// Merge response-side `(explicit, derived)` wire blocks back into domain +/// resource options, restoring per-key provenance. Explicit wins on a key +/// collision (which a well-formed response never produces). +/// +/// # Errors +/// +/// Same contract as [`resource_options_from_wire`]. +pub fn resource_options_from_wire_split( + explicit: &iggy_binary_protocol::WireOptions, + derived: &iggy_binary_protocol::WireOptions, +) -> Result { + let mut options = resource_options_from_wire(derived, false)?; + options.extend(resource_options_from_wire(explicit, true)?); + Ok(options) +} + +/// Split domain resource options into `(explicit, derived)` wire blocks, +/// the response-side layout that preserves per-key provenance. +/// +/// # Errors +/// +/// Same contract as [`resource_options_to_wire`]. +pub fn resource_options_to_wire_split( + options: &ResourceOptions, +) -> Result<(WireOptions, WireOptions), IggyError> { + Ok(( + resource_options_to_wire(options, OptionsProvenance::Explicit)?, + resource_options_to_wire(options, OptionsProvenance::Derived)?, + )) +} + +/// Fixed per-entry cost of the TLV encoding: a kind byte and a `u32` length +/// for the key and for the value. +const OPTION_ENTRY_OVERHEAD: usize = 2 * (1 + 4); + +/// Encode domain resource options into a [`WireOptions`] block, keeping only +/// the entries matching `provenance`. Encoding just the explicit ones is what +/// lets a client round-trip a create without pinning server defaults. +/// +/// # Errors +/// +/// Returns `IggyError::OptionsBlockTooLarge` when the selected entries exceed +/// `MAX_OPTIONS` or `MAX_OPTIONS_BYTES`. [`WireOptions::from_validated`] skips +/// revalidation, so the two caps have to hold here: a block written past them +/// is one the receiving peer's `decode_options_prefixed` refuses, which would +/// make the resource permanently unreadable rather than merely oversized. +pub fn resource_options_to_wire( + options: &ResourceOptions, + provenance: OptionsProvenance, +) -> Result { + let entries: Vec<(&HeaderKey, &OptionValue)> = options + .iter() + .filter(|(_, option)| option.matches(provenance)) + .collect(); + if entries.is_empty() { + return Ok(WireOptions::empty()); + } + if entries.len() > MAX_OPTIONS as usize { + return Err(IggyError::OptionsBlockTooLarge(format!( + "{} entries, maximum {MAX_OPTIONS}", + entries.len() + ))); + } + let size: usize = entries + .iter() + .map(|(key, option)| { + OPTION_ENTRY_OVERHEAD + key.as_bytes().len() + option.value.as_bytes().len() + }) + .sum(); + if size > MAX_OPTIONS_BYTES { + return Err(IggyError::OptionsBlockTooLarge(format!( + "{size} bytes, maximum {MAX_OPTIONS_BYTES}" + ))); + } + let mut buf = BytesMut::with_capacity(size); + for (key, option) in entries { + // A non-string key would encode a block the receiving peer refuses, + // making the resource permanently unreadable. It holds because every + // producer went through `validate_options`, not because `HeaderKey` + // guarantees it, so the invariant is asserted where it is relied on. + debug_assert_eq!( + key.kind(), + HeaderKind::String, + "option key must be a string" + ); + buf.put_u8(key.kind().as_code()); + #[allow(clippy::cast_possible_truncation)] + buf.put_u32_le(key.as_bytes().len() as u32); + buf.put_slice(key.as_bytes()); + buf.put_u8(option.value.kind().as_code()); + #[allow(clippy::cast_possible_truncation)] + buf.put_u32_le(option.value.as_bytes().len() as u32); + buf.put_slice(option.value.as_bytes()); + } + // Structural validity holds by construction: entries come from valid + // `HeaderKey`s walked in `BTreeMap` order, so keys are sorted and unique, + // and the loop above asserts they are string-kinded. The caps cover what + // construction cannot. + Ok(WireOptions::from_validated(buf.freeze())) +} + #[cfg(test)] mod tests { use super::*; @@ -868,4 +1061,35 @@ mod tests { assert!(user_headers_from_validated_slice(&buf).is_err()); assert!(user_headers_from_wire(&WireUserHeaders::from_slice(&buf).unwrap()).is_err()); } + + fn put_option_entry(buf: &mut Vec, key: &str, value_kind: u8, value: &[u8]) { + buf.push(HeaderKind::String.as_code()); + buf.extend_from_slice(&u32::try_from(key.len()).unwrap().to_le_bytes()); + buf.extend_from_slice(key.as_bytes()); + buf.push(value_kind); + buf.extend_from_slice(&u32::try_from(value.len()).unwrap().to_le_bytes()); + buf.extend_from_slice(value); + } + + #[test] + fn given_option_with_unknown_value_kind_when_decoded_should_skip_only_that_entry() { + // A newer peer's option value kind. The wire layer forwards it, so the + // domain decode has to drop the entry rather than fail the block: this + // decode runs on the apply path, where an error means one replica + // rejecting a commit another accepted. + let mut buf = Vec::new(); + put_option_entry(&mut buf, "from_the_future", 200, b"opaque"); + put_option_entry( + &mut buf, + "segment_size", + HeaderKind::String.as_code(), + b"1MB", + ); + let wire = WireOptions::from_slice(&buf).expect("unknown value kinds stay wire-valid"); + + let options = resource_options_from_wire(&wire, true).unwrap(); + + assert_eq!(options.len(), 1); + assert!(options.contains_key(&HeaderKey::from_str("segment_size").unwrap())); + } } diff --git a/core/configs/src/common/defaults.rs b/core/configs/src/common/defaults.rs index 2cc6d22007..e733c5b7cb 100644 --- a/core/configs/src/common/defaults.rs +++ b/core/configs/src/common/defaults.rs @@ -23,8 +23,8 @@ use super::server::{ }; use super::system::{ BackupConfig, CompatibilityConfig, CompressionConfig, EncryptionConfig, LoggingConfig, - MessageDeduplicationConfig, PartitionConfig, RecoveryConfig, RuntimeConfig, SegmentConfig, - StateConfig, StreamConfig, SystemConfig, TopicConfig, + PartitionConfig, RecoveryConfig, RuntimeConfig, SegmentConfig, StateConfig, StreamConfig, + SystemConfig, TopicConfig, }; use configs::ConfigEnvMappings; @@ -197,7 +197,6 @@ impl Default for SystemConfig { segment: SegmentConfig::default(), state: StateConfig::default(), compression: CompressionConfig::default(), - message_deduplication: MessageDeduplicationConfig::default(), recovery: RecoveryConfig::default(), memory_pool: MemoryPoolConfig::default(), sharding: S::default(), @@ -322,8 +321,6 @@ impl Default for TopicConfig { fn default() -> TopicConfig { TopicConfig { path: SERVER_CONFIG.system.topic.path.parse().unwrap(), - max_size: SERVER_CONFIG.system.topic.max_size.parse().unwrap(), - message_expiry: SERVER_CONFIG.system.topic.message_expiry.parse().unwrap(), } } } @@ -332,15 +329,6 @@ impl Default for PartitionConfig { fn default() -> PartitionConfig { PartitionConfig { path: SERVER_CONFIG.system.partition.path.parse().unwrap(), - size_of_messages_required_to_save: SERVER_CONFIG - .system - .partition - .size_of_messages_required_to_save - .parse() - .unwrap(), - messages_required_to_save: SERVER_CONFIG.system.partition.messages_required_to_save - as u32, - enforce_fsync: SERVER_CONFIG.system.partition.enforce_fsync, validate_checksum: SERVER_CONFIG.system.partition.validate_checksum, } } @@ -349,8 +337,6 @@ impl Default for PartitionConfig { impl Default for SegmentConfig { fn default() -> SegmentConfig { SegmentConfig { - size: SERVER_CONFIG.system.segment.size.parse().unwrap(), - preallocate: SERVER_CONFIG.system.segment.preallocate, cache_indexes: SERVER_CONFIG.system.segment.cache_indexes.parse().unwrap(), archive_expired: SERVER_CONFIG.system.segment.archive_expired, } @@ -368,21 +354,6 @@ impl Default for StateConfig { } } -impl Default for MessageDeduplicationConfig { - fn default() -> MessageDeduplicationConfig { - MessageDeduplicationConfig { - enabled: SERVER_CONFIG.system.message_deduplication.enabled, - max_entries: SERVER_CONFIG.system.message_deduplication.max_entries as u64, - expiry: SERVER_CONFIG - .system - .message_deduplication - .expiry - .parse() - .unwrap(), - } - } -} - impl Default for RecoveryConfig { fn default() -> RecoveryConfig { RecoveryConfig { diff --git a/core/configs/src/common/displays.rs b/core/configs/src/common/displays.rs index 3d0b432017..234cc04125 100644 --- a/core/configs/src/common/displays.rs +++ b/core/configs/src/common/displays.rs @@ -19,7 +19,6 @@ use super::server::{ ConsumerGroupConfig, DataMaintenanceConfig, HeartbeatConfig, MessagesMaintenanceConfig, TelemetryConfig, TelemetryLogsConfig, TelemetryTracesConfig, }; -use super::system::MessageDeduplicationConfig; use super::{ http::{HttpConfig, HttpCorsConfig, HttpJwtConfig, HttpMetricsConfig, HttpTlsConfig}, server::MessageSaverConfig, @@ -164,11 +163,7 @@ impl Display for StreamConfig { impl Display for TopicConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ path: {}, max_size: {}, message_expiry: {} }}", - self.path, self.max_size, self.message_expiry - ) + write!(f, "{{ path: {} }}", self.path) } } @@ -176,22 +171,8 @@ impl Display for PartitionConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( f, - "{{ path: {}, messages_required_to_save: {}, size_of_messages_required_to_save: {}, enforce_fsync: {}, validate_checksum: {} }}", - self.path, - self.messages_required_to_save, - self.size_of_messages_required_to_save, - self.enforce_fsync, - self.validate_checksum - ) - } -} - -impl Display for MessageDeduplicationConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ enabled: {}, max_entries: {:?}, expiry: {:?} }}", - self.enabled, self.max_entries, self.expiry + "{{ path: {}, validate_checksum: {} }}", + self.path, self.validate_checksum ) } } @@ -200,8 +181,8 @@ impl Display for SegmentConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( f, - "{{ size_bytes: {}, preallocate: {}, cache_indexes: {}, archive_expired: {} }}", - self.size, self.preallocate, self.cache_indexes, self.archive_expired, + "{{ cache_indexes: {}, archive_expired: {} }}", + self.cache_indexes, self.archive_expired, ) } } diff --git a/core/configs/src/common/system.rs b/core/configs/src/common/system.rs index d72913fc5c..a9172b0fb5 100644 --- a/core/configs/src/common/system.rs +++ b/core/configs/src/common/system.rs @@ -19,9 +19,6 @@ use super::cache_indexes::CacheIndexesConfig; use super::server::MemoryPoolConfig; use configs::{ConfigEnv, ConfigEnvMappings}; use iggy_common::IggyByteSize; -use iggy_common::IggyError; -use iggy_common::IggyExpiry; -use iggy_common::MaxTopicSize; use iggy_common::{CompressionAlgorithm, IggyDuration}; use serde::{Deserialize, Serialize}; use serde_with::DisplayFromStr; @@ -48,7 +45,6 @@ pub struct SystemConfig { pub segment: SegmentConfig, pub encryption: EncryptionConfig, pub compression: CompressionConfig, - pub message_deduplication: MessageDeduplicationConfig, pub recovery: RecoveryConfig, pub memory_pool: MemoryPoolConfig, pub sharding: S, @@ -133,50 +129,33 @@ pub struct StreamConfig { pub path: String, } -#[serde_as] +/// Only the on-disk layout: a topic's size cap and message expiry are its own +/// creation options now (`max_topic_size`, `message_expiry`), defaulting to +/// `iggy_common::DEFAULT_MAX_TOPIC_SIZE` / `DEFAULT_MESSAGE_EXPIRY`. #[derive(Debug, Deserialize, Serialize, ConfigEnv)] pub struct TopicConfig { pub path: String, - #[config_env(leaf)] - #[serde_as(as = "DisplayFromStr")] - pub max_size: MaxTopicSize, - #[config_env(leaf)] - #[serde_as(as = "DisplayFromStr")] - pub message_expiry: IggyExpiry, } +/// `enforce_fsync`, `messages_required_to_save` and +/// `size_of_messages_required_to_save` are per-topic creation options now, +/// defaulting to the `iggy_common::DEFAULT_*` constants. #[derive(Debug, Deserialize, Serialize, ConfigEnv)] pub struct PartitionConfig { pub path: String, - pub messages_required_to_save: u32, - #[config_env(leaf)] - pub size_of_messages_required_to_save: IggyByteSize, - pub enforce_fsync: bool, pub validate_checksum: bool, } -#[serde_as] -#[derive(Debug, Deserialize, Serialize, ConfigEnv)] -pub struct MessageDeduplicationConfig { - pub enabled: bool, - pub max_entries: u64, - #[config_env(leaf)] - #[serde_as(as = "DisplayFromStr")] - pub expiry: IggyDuration, -} - #[derive(Debug, Deserialize, Serialize, ConfigEnv)] pub struct RecoveryConfig { pub recreate_missing_state: bool, } -#[serde_as] +/// `size` and `preallocate` are per-topic creation options now +/// (`segment_size`, `preallocate_segments`), defaulting to +/// `iggy_common::DEFAULT_SEGMENT_SIZE` / `DEFAULT_PREALLOCATE_SEGMENTS`. #[derive(Debug, Deserialize, Serialize, ConfigEnv)] pub struct SegmentConfig { - #[config_env(leaf)] - pub size: IggyByteSize, - #[serde(default)] - pub preallocate: bool, #[config_env(leaf)] pub cache_indexes: CacheIndexesConfig, pub archive_expired: bool, @@ -328,32 +307,6 @@ impl SystemConfig { let path = self.get_segment_path(stream_id, topic_id, partition_id, start_offset); format!("{path}.{INDEX_EXTENSION}") } - - pub fn resolve_max_topic_size( - &self, - max_topic_size: MaxTopicSize, - ) -> Result { - match max_topic_size { - MaxTopicSize::ServerDefault => Ok(self.topic.max_size), - _ => { - if max_topic_size.as_bytes_u64() < self.segment.size.as_bytes_u64() { - Err(IggyError::InvalidTopicSize( - max_topic_size, - self.segment.size, - )) - } else { - Ok(max_topic_size) - } - } - } - } - - pub fn resolve_message_expiry(&self, message_expiry: IggyExpiry) -> IggyExpiry { - match message_expiry { - IggyExpiry::ServerDefault => self.topic.message_expiry, - _ => message_expiry, - } - } } impl SystemPaths for SystemConfig { diff --git a/core/configs/src/common/validators.rs b/core/configs/src/common/validators.rs index 9c7d54c43c..34bfb82a4d 100644 --- a/core/configs/src/common/validators.rs +++ b/core/configs/src/common/validators.rs @@ -74,34 +74,17 @@ impl Validatable for TelemetryConfig { impl Validatable for PartitionConfig { fn validate(&self) -> Result<(), ConfigurationError> { - if self.messages_required_to_save == 0 { - eprintln!("Configured system.partition.messages_required_to_save cannot be 0"); - return Err(ConfigurationError::InvalidConfigurationValue); - } - + // The flush thresholds this used to check are per-topic creation + // options now; their bounds are enforced at admission. Ok(()) } } impl Validatable for SegmentConfig { fn validate(&self) -> Result<(), ConfigurationError> { - if self.size > SEGMENT_MAX_SIZE_BYTES { - eprintln!( - "Configured system.segment.size {} B is greater than maximum {} B", - self.size.as_bytes_u64(), - SEGMENT_MAX_SIZE_BYTES - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - if !self.size.as_bytes_u64().is_multiple_of(512) { - eprintln!( - "Configured system.segment.size {} B is not a multiple of 512 B", - self.size.as_bytes_u64() - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - + // Segment size is a per-topic creation option now; its ceiling, floor + // and 512 B-multiple rule are enforced by + // `iggy_common::validate_topic_segment_size` at admission. Ok(()) } } diff --git a/core/configs/src/configs_impl/file_provider.rs b/core/configs/src/configs_impl/file_provider.rs index a404208b91..b6b8d67cfd 100644 --- a/core/configs/src/configs_impl/file_provider.rs +++ b/core/configs/src/configs_impl/file_provider.rs @@ -28,12 +28,44 @@ use tracing::{error, info, warn}; const DISPLAY_CONFIG_ENV: &str = "IGGY_DISPLAY_CONFIG"; +/// A config key that no longer exists, and what took over from it. +/// +/// Nothing else catches such a key. Its struct field is gone and no config +/// table sets `deny_unknown_fields`, so figment drops an unrecognized key +/// without a word from either source: a server still carrying +/// `enforce_fsync = true` would boot reporting success and run without fsync. +/// Every relocated key used to change behavior, so the boot is refused rather +/// than warned about. +#[derive(Debug, Clone, Copy)] +pub struct RelocatedKey { + /// Dotted config path, for example `system.segment.size`. A deleted table + /// matches everything nested under it as well. + pub path: &'static str, + /// The per-topic option key that replaced it, or `None` when the feature + /// it configured was removed outright. + pub replacement: Option<&'static str>, +} + +impl RelocatedKey { + /// Sentence telling the operator where the setting went. + fn guidance(&self) -> String { + match self.replacement { + Some(option) => { + format!("it is now the per-topic '{option}' option, set on CreateTopic") + } + None => "the feature it configured was removed".to_string(), + } + } +} + /// File-based configuration provider that combines file, default, and environment configurations. pub struct FileConfigProvider

{ file_path: String, default_config: Option>, env_provider: P, display_config: bool, + env_prefix: &'static str, + relocated_keys: &'static [RelocatedKey], } impl FileConfigProvider

{ @@ -55,14 +87,75 @@ impl FileConfigProvider

{ env_provider, default_config, display_config, + env_prefix: "", + relocated_keys: &[], } } + + /// Refuse to load when any of `keys` is still set, in the config file or in + /// the environment. + /// + /// `env_prefix` is the prefix this config's env provider reads, used to + /// derive the variable name for each path. The table is per-config on + /// purpose: a server key left over in a shared container environment must + /// not stop the connectors runtime or the MCP server from booting. + pub fn with_relocated_keys( + mut self, + env_prefix: &'static str, + keys: &'static [RelocatedKey], + ) -> Self { + self.env_prefix = env_prefix; + self.relocated_keys = keys; + self + } + + fn reject_relocated_keys(&self) -> Result<(), ConfigurationError> { + if self.relocated_keys.is_empty() { + return Ok(()); + } + let file = file_exists(&self.file_path).then(|| Figment::from(Toml::file(&self.file_path))); + let env_names = env::vars_os().filter_map(|(name, _)| name.into_string().ok()); + let mut found = false; + + for key in self.relocated_keys { + if file + .as_ref() + .is_some_and(|file| file.find_value(key.path).is_ok()) + { + found = true; + error!( + "Config key '{}' no longer exists; {}. Remove the key to boot.", + key.path, + key.guidance() + ); + } + } + for (name, key) in relocated_env_vars(env_names, self.env_prefix, self.relocated_keys) { + found = true; + error!( + "Environment variable '{name}' sets config key '{}', which no longer exists; {}. \ + Unset it to boot.", + key.path, + key.guidance() + ); + } + + if found { + return Err(ConfigurationError::InvalidConfigurationValue); + } + Ok(()) + } } impl ConfigProvider for FileConfigProvider

{ async fn load_config(&self) -> Result { info!("Loading config from path: '{}'...", self.file_path); + // Both sources are checked before either is merged: the env provider + // below is just as silent about a key no field reads, and the + // pure-env container never touches the file branch at all. + self.reject_relocated_keys()?; + // Start with the default configuration if provided let mut config_builder = Figment::new(); let has_default = self.default_config.is_some(); @@ -113,6 +206,42 @@ impl ConfigProvider for FileConfigProvider

{ } } +/// Pair every name in `names` that addresses a relocated key with that key. +/// +/// A key's variable name is its path uppercased with dots turned into +/// underscores, matching what `#[derive(ConfigEnv)]` generates. A deleted table +/// also matches its children, so `system.message_deduplication` catches +/// `IGGY_SYSTEM_MESSAGE_DEDUPLICATION_ENABLED`. +fn relocated_env_vars<'keys>( + names: impl Iterator, + prefix: &str, + keys: &'keys [RelocatedKey], +) -> Vec<(String, &'keys RelocatedKey)> { + let derived: Vec<(String, &RelocatedKey)> = keys + .iter() + .map(|key| { + ( + format!("{prefix}{}", key.path.replace('.', "_").to_uppercase()), + key, + ) + }) + .collect(); + + let mut found = Vec::new(); + for name in names { + for (env_name, key) in &derived { + let nested = name + .strip_prefix(env_name.as_str()) + .is_some_and(|rest| rest.starts_with('_')); + if &name == env_name || nested { + found.push((name, *key)); + break; + } + } + } + found +} + fn file_exists>(path: P) -> bool { let path = path.as_ref(); @@ -138,3 +267,89 @@ fn file_exists>(path: P) -> bool { }; } } + +#[cfg(test)] +mod tests { + use super::*; + + const KEYS: &[RelocatedKey] = &[ + RelocatedKey { + path: "system.partition.enforce_fsync", + replacement: Some("enforce_fsync"), + }, + RelocatedKey { + path: "system.message_deduplication", + replacement: None, + }, + ]; + + fn names(names: &[&str]) -> Vec { + names.iter().map(|name| (*name).to_string()).collect() + } + + #[test] + fn given_env_var_for_relocated_leaf_when_matching_then_should_report_it() { + let found = relocated_env_vars( + names(&["IGGY_SYSTEM_PARTITION_ENFORCE_FSYNC"]).into_iter(), + "IGGY_", + KEYS, + ); + + assert_eq!(found.len(), 1); + assert_eq!(found[0].0, "IGGY_SYSTEM_PARTITION_ENFORCE_FSYNC"); + assert_eq!(found[0].1.replacement, Some("enforce_fsync")); + } + + #[test] + fn given_env_var_under_removed_table_when_matching_then_should_report_it() { + let found = relocated_env_vars( + names(&["IGGY_SYSTEM_MESSAGE_DEDUPLICATION_ENABLED"]).into_iter(), + "IGGY_", + KEYS, + ); + + assert_eq!(found.len(), 1); + assert_eq!(found[0].1.path, "system.message_deduplication"); + assert_eq!(found[0].1.replacement, None); + } + + #[test] + fn given_live_env_vars_when_matching_then_should_report_none() { + let found = relocated_env_vars( + names(&[ + "IGGY_SYSTEM_PARTITION_PATH", + "IGGY_SYSTEM_PARTITION_ENFORCE_FSYNCHRONIZATION", + "IGGY_TCP_ADDRESS", + "RUST_LOG", + ]) + .into_iter(), + "IGGY_", + KEYS, + ); + + assert!(found.is_empty(), "unexpected matches: {found:?}"); + } + + #[test] + fn given_another_configs_prefix_when_matching_then_should_report_none() { + let found = relocated_env_vars( + names(&["IGGY_SYSTEM_PARTITION_ENFORCE_FSYNC"]).into_iter(), + "IGGY_CONNECTORS_", + KEYS, + ); + + assert!(found.is_empty(), "unexpected matches: {found:?}"); + } + + #[test] + fn given_no_relocated_keys_when_rejecting_then_should_accept() { + let provider = FileConfigProvider::new( + "nonexistent-config.toml".to_string(), + Toml::string(""), + false, + None, + ); + + assert!(provider.reject_relocated_keys().is_ok()); + } +} diff --git a/core/configs/src/configs_impl/mod.rs b/core/configs/src/configs_impl/mod.rs index 6b16acc0bc..905cb46d71 100644 --- a/core/configs/src/configs_impl/mod.rs +++ b/core/configs/src/configs_impl/mod.rs @@ -33,7 +33,7 @@ mod typed_env_provider; pub use env_mapping::{ConfigEnvMappings, EnvVarMapping}; pub use error::ConfigurationError; -pub use file_provider::FileConfigProvider; +pub use file_provider::{FileConfigProvider, RelocatedKey}; pub use parsing::parse_env_value_to_json; pub use traits::{ConfigProvider, ConfigurationType}; pub use typed_env_provider::TypedEnvProvider; diff --git a/core/configs/src/lib.rs b/core/configs/src/lib.rs index e9339b2aa6..3b2c595419 100644 --- a/core/configs/src/lib.rs +++ b/core/configs/src/lib.rs @@ -24,7 +24,7 @@ pub use common::{COMPONENT, cache_indexes, defaults, displays, http, system, val pub use configs_derive::ConfigEnv; pub use configs_impl::{ ConfigEnvMappings, ConfigProvider, ConfigurationError, ConfigurationType, EnvVarMapping, - FileConfigProvider, TypedEnvProvider, parse_env_value_to_json, + FileConfigProvider, RelocatedKey, TypedEnvProvider, parse_env_value_to_json, }; pub use server_config::{ cluster, message_bus, metadata, partition, quic, server, sharding, tcp, websocket, diff --git a/core/configs/src/server_config/server.rs b/core/configs/src/server_config/server.rs index 4dbd7e81de..5f0522c1a2 100644 --- a/core/configs/src/server_config/server.rs +++ b/core/configs/src/server_config/server.rs @@ -26,7 +26,10 @@ use super::websocket::WebSocketConfig; use crate::ConfigurationError; use crate::common::http::HttpConfig; use crate::common::system::SystemConfig; -use configs::{ConfigEnv, ConfigEnvMappings, ConfigProvider, FileConfigProvider, TypedEnvProvider}; +use configs::{ + ConfigEnv, ConfigEnvMappings, ConfigProvider, FileConfigProvider, RelocatedKey, + TypedEnvProvider, +}; use err_trail::ErrContext; use figment::providers::{Format, Toml}; use figment::value::Dict; @@ -46,6 +49,48 @@ pub use crate::common::server::{ const DEFAULT_CONFIG_PATH: &str = "core/server/config.toml"; +/// Server config keys that became per-topic options, or went away with the +/// feature they configured. +/// +/// The provider refuses to boot while any of them is still set, in the config +/// file or in the environment. See [`RelocatedKey`] for why a warning is not +/// enough. The partition knobs matter most: they are create-only options now, +/// so a topic that boots without one can never be given it afterwards. +const RELOCATED_CONFIG_KEYS: &[RelocatedKey] = &[ + RelocatedKey { + path: "system.topic.max_size", + replacement: Some("max_topic_size"), + }, + RelocatedKey { + path: "system.topic.message_expiry", + replacement: Some("message_expiry"), + }, + RelocatedKey { + path: "system.partition.enforce_fsync", + replacement: Some("enforce_fsync"), + }, + RelocatedKey { + path: "system.partition.messages_required_to_save", + replacement: Some("messages_required_to_save"), + }, + RelocatedKey { + path: "system.partition.size_of_messages_required_to_save", + replacement: Some("size_of_messages_required_to_save"), + }, + RelocatedKey { + path: "system.segment.size", + replacement: Some("segment_size"), + }, + RelocatedKey { + path: "system.segment.preallocate", + replacement: Some("preallocate_segments"), + }, + RelocatedKey { + path: "system.message_deduplication", + replacement: None, + }, +]; + /// [`SystemConfig`] bound to this crate's own /// [`super::sharding::ShardingConfig`]. `core/server` names this alias /// wherever it refers to the system config. @@ -139,6 +184,7 @@ impl ServerConfig { true, Some(default_config), ) + .with_relocated_keys(ServerConfig::ENV_PREFIX, RELOCATED_CONFIG_KEYS) } /// All recognised env var names for [`ServerConfig`]. diff --git a/core/configs/src/server_config/validators.rs b/core/configs/src/server_config/validators.rs index 6784ef294e..60541d4495 100644 --- a/core/configs/src/server_config/validators.rs +++ b/core/configs/src/server_config/validators.rs @@ -26,8 +26,9 @@ use super::COMPONENT; use super::cluster::STATE_CHUNK_HEADER_LEN; use super::server::{ExtraConfig, NamespaceConfig, ServerConfig}; use crate::ConfigurationError; +use crate::common::validators::SEGMENT_MAX_SIZE_BYTES; use err_trail::ErrContext; -use iggy_common::{IggyExpiry, MaxTopicSize, Validatable}; +use iggy_common::{IggyExpiry, Validatable}; use server_common::sharding::IggyNamespace; use tracing::warn; @@ -102,31 +103,6 @@ impl Validatable for ServerConfig { format!("{COMPONENT} (error: {e}) - failed to validate message saver config") })?; - let topic_size = match self.system.topic.max_size { - MaxTopicSize::Custom(size) => Ok(size.as_bytes_u64()), - MaxTopicSize::Unlimited => Ok(u64::MAX), - MaxTopicSize::ServerDefault => { - eprintln!("system.topic.max_size cannot be ServerDefault in the server config"); - Err(ConfigurationError::InvalidConfigurationValue) - } - }?; - - if let IggyExpiry::ServerDefault = self.system.topic.message_expiry { - eprintln!("system.topic.message_expiry cannot be ServerDefault in the server config"); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - // A zero duration encodes to wire value 0, the same value the wire uses - // for ServerDefault, so it would silently collide with that sentinel. - if let IggyExpiry::ExpireDuration(duration) = self.system.topic.message_expiry - && duration.as_micros() == 0 - { - eprintln!( - "system.topic.message_expiry is a zero duration, which collides with the server-default sentinel on the wire; use \"none\" to never expire or a positive duration" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if self.http.enabled && let IggyExpiry::ServerDefault = self.http.jwt.access_token_expiry { @@ -171,15 +147,6 @@ impl Validatable for ServerConfig { } } - if topic_size < self.system.segment.size.as_bytes_u64() { - eprintln!( - "system.topic.max_size ({} B) must be >= system.segment.size ({} B)", - topic_size, - self.system.segment.size.as_bytes_u64() - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - // A received segment artifact can be one whole batch larger than the // segment cap (rotation checks the cap AFTER appending), and the real // batch bound is the BUS frame cap -- the server never enforces @@ -188,21 +155,18 @@ impl Validatable for ServerConfig { // partition livelocks re-requesting the same segment from every peer at // the backoff ceiling. Caught here so it is a boot error rather than one // partition that silently never rejoins. - let artifact_floor = self - .system - .segment - .size - .as_bytes_u64() - .saturating_add(self.message_bus.max_message_size.as_bytes_u64()); + // Segment size is per topic now, so the floor is the LARGEST segment + // any topic may legally be created with, not a configured value. + let artifact_floor = + SEGMENT_MAX_SIZE_BYTES.saturating_add(self.message_bus.max_message_size.as_bytes_u64()); if self.partition.transfer_artifact_bytes_max.as_bytes_u64() < artifact_floor { eprintln!( - "{COMPONENT} partition.transfer_artifact_bytes_max ({} B) must be at least \ - system.segment.size ({} B) + message_bus.max_message_size ({} B) = \ - {artifact_floor} B: a segment may close one whole batch past its cap, and an \ - artifact ceiling below that refuses a legal segment and livelocks the \ - partition's rejoin", + "{COMPONENT} partition.transfer_artifact_bytes_max ({} B) must be at least the \ + largest legal segment ({SEGMENT_MAX_SIZE_BYTES} B) + \ + message_bus.max_message_size ({} B) = {artifact_floor} B: a segment may close \ + one whole batch past its cap, and an artifact ceiling below that refuses a \ + legal segment and livelocks the partition's rejoin", self.partition.transfer_artifact_bytes_max.as_bytes_u64(), - self.system.segment.size.as_bytes_u64(), self.message_bus.max_message_size.as_bytes_u64(), ); return Err(ConfigurationError::InvalidConfigurationValue); @@ -363,10 +327,6 @@ impl Validatable for ServerConfig { /// pristine config.toml boots without noise. The guard test below pins the /// compared knobs against drift. fn reject_unsupported_and_warn_inert(config: &ServerConfig) -> Result<(), ConfigurationError> { - if config.system.message_deduplication.enabled { - eprintln!("system.message_deduplication.enabled is not supported"); - return Err(ConfigurationError::InvalidConfigurationValue); - } if config.system.segment.archive_expired { eprintln!("system.segment.archive_expired is not supported"); return Err(ConfigurationError::InvalidConfigurationValue); @@ -474,6 +434,33 @@ mod tests { .expect("config deserializes") } + /// The per-topic `segment_size` ceiling lives in `iggy_common`, which cannot + /// import this crate. Admission reads it from there; boot validation reads + /// the constant here. This is the only place both are visible. + #[test] + fn given_segment_maximum_when_compared_to_the_option_ceiling_should_match() { + assert_eq!( + SEGMENT_MAX_SIZE_BYTES, + iggy_common::MAX_TOPIC_SEGMENT_SIZE, + "the option ceiling and the segment maximum must move together" + ); + } + + /// Admission may cap a topic's `segment_size` at + /// [`iggy_common::MAX_TOPIC_SEGMENT_SIZE`] flat only while boot refuses any + /// config whose artifact budget cannot carry a segment that large plus one + /// bus frame. Without that refusal the ceiling would have to be per node. + #[test] + fn given_artifact_budget_below_the_segment_ceiling_when_validating_should_reject() { + let config = config_with_override( + "[partition]\ntransfer_artifact_bytes_max = \"1 GiB\"\n[message_bus]\nmax_message_size = \"1 MiB\"\n", + ); + assert!( + config.validate().is_err(), + "an artifact budget under segment maximum + one frame must refuse boot" + ); + } + #[test] fn given_shipped_default_config_when_validating_should_pass() { let config: ServerConfig = Figment::new() @@ -483,12 +470,6 @@ mod tests { config.validate().expect("pristine config must validate"); } - #[test] - fn given_message_deduplication_enabled_when_validating_should_reject() { - let config = config_with_override("[system.message_deduplication]\nenabled = true\n"); - assert!(config.validate().is_err()); - } - #[test] fn given_web_ui_enabled_when_validating_should_pass() { let config = config_with_override("[http]\nweb_ui = true\n"); @@ -509,12 +490,6 @@ mod tests { assert!(config.validate().is_err()); } - #[test] - fn given_zero_message_expiry_when_validating_should_reject() { - let config = config_with_override("[system.topic]\nmessage_expiry = \"0s\"\n"); - assert!(config.validate().is_err()); - } - #[test] fn given_peer_queue_capacity_not_above_repair_chunk_max_when_validating_should_reject() { // The default repair_chunk_max (128) must stay strictly below diff --git a/core/harness_derive/src/attrs.rs b/core/harness_derive/src/attrs.rs index ee4ff391e2..76fee823c5 100644 --- a/core/harness_derive/src/attrs.rs +++ b/core/harness_derive/src/attrs.rs @@ -371,7 +371,7 @@ fn parse_transport_ident(ident: Ident) -> syn::Result { } } -/// Parses a dot-notation config key like `segment.size` or `partition.messages_required_to_save`. +/// Parses a dot-notation config key like `segment.cache_indexes` or `metadata.journal_slots`. fn parse_config_key(input: ParseStream) -> syn::Result<(String, Span)> { let first: Ident = input.parse()?; let span = first.span(); @@ -629,9 +629,9 @@ mod tests { #[test] fn parse_server_static() { - let attrs: IggyTestAttrs = syn::parse_quote!(server(segment.size = "1MiB")); - let segment_size = attrs.server.find_override("segment.size").unwrap(); - assert!(matches!(&segment_size.value, ConfigValue::Static(s) if s == "1MiB")); + let attrs: IggyTestAttrs = syn::parse_quote!(server(segment.cache_indexes = "all")); + let cache_indexes = attrs.server.find_override("segment.cache_indexes").unwrap(); + assert!(matches!(&cache_indexes.value, ConfigValue::Static(s) if s == "all")); } #[test] @@ -642,9 +642,10 @@ mod tests { #[test] fn parse_server_matrix() { - let attrs: IggyTestAttrs = syn::parse_quote!(server(segment.size = ["512B", "1MiB"])); - let segment_size = attrs.server.find_override("segment.size").unwrap(); - assert!(matches!(&segment_size.value, ConfigValue::Matrix(v) if v.len() == 2)); + let attrs: IggyTestAttrs = + syn::parse_quote!(server(segment.cache_indexes = ["none", "all"])); + let cache_indexes = attrs.server.find_override("segment.cache_indexes").unwrap(); + assert!(matches!(&cache_indexes.value, ConfigValue::Matrix(v) if v.len() == 2)); } #[test] @@ -652,17 +653,20 @@ mod tests { let attrs: IggyTestAttrs = syn::parse_quote!( test_client_transport = [Tcp, Http], server( - segment.size = ["512B", "1MiB"], - segment.cache_indexes = "none", + segment.cache_indexes = ["none", "all"], + metadata.journal_slots = 1024, tcp.socket.nodelay = true ) ); assert_eq!(attrs.transports.len(), 2); - let segment_size = attrs.server.find_override("segment.size").unwrap(); let cache_indexes = attrs.server.find_override("segment.cache_indexes").unwrap(); + let journal_slots = attrs + .server + .find_override("metadata.journal_slots") + .unwrap(); let tcp_nodelay = attrs.server.find_override("tcp.socket.nodelay").unwrap(); - assert!(matches!(&segment_size.value, ConfigValue::Matrix(v) if v.len() == 2)); - assert!(matches!(&cache_indexes.value, ConfigValue::Static(s) if s == "none")); + assert!(matches!(&cache_indexes.value, ConfigValue::Matrix(v) if v.len() == 2)); + assert!(matches!(&journal_slots.value, ConfigValue::Static(s) if s == "1024")); assert!(matches!(&tcp_nodelay.value, ConfigValue::Static(s) if s == "true")); } @@ -718,11 +722,11 @@ mod tests { #[test] fn parse_mcp_combined() { let attrs: IggyTestAttrs = - syn::parse_quote!(seed = my_seed, server(mcp, segment.size = "1MiB")); + syn::parse_quote!(seed = my_seed, server(mcp, segment.cache_indexes = "all")); assert!(attrs.server.mcp.is_some()); assert!(attrs.seed_fn.is_some()); - let segment_size = attrs.server.find_override("segment.size").unwrap(); - assert!(matches!(&segment_size.value, ConfigValue::Static(s) if s == "1MiB")); + let cache_indexes = attrs.server.find_override("segment.cache_indexes").unwrap(); + assert!(matches!(&cache_indexes.value, ConfigValue::Static(s) if s == "all")); } #[test] @@ -750,13 +754,13 @@ mod tests { #[test] fn parse_dot_notation_deep() { let attrs: IggyTestAttrs = syn::parse_quote!(server( - partition.messages_required_to_save = [32, 64], + metadata.journal_slots = [512, 1024], system.encryption.enabled = true )); assert_eq!(attrs.server.config_overrides.len(), 2); let msgs = attrs .server - .find_override("partition.messages_required_to_save") + .find_override("metadata.journal_slots") .unwrap(); assert!(matches!(&msgs.value, ConfigValue::Matrix(v) if v.len() == 2)); } @@ -808,11 +812,11 @@ mod tests { let attrs: IggyTestAttrs = syn::parse_quote!( cluster_nodes = [3, 5], test_client_transport = [Tcp, Http], - server(segment.size = ["512B", "1MiB"]) + server(segment.cache_indexes = ["none", "all"]) ); assert!(matches!(&attrs.cluster_nodes, ClusterNodesValue::Matrix(v) if v == &[3, 5])); assert_eq!(attrs.transports.len(), 2); - let segment_size = attrs.server.find_override("segment.size").unwrap(); - assert!(matches!(&segment_size.value, ConfigValue::Matrix(v) if v.len() == 2)); + let cache_indexes = attrs.server.find_override("segment.cache_indexes").unwrap(); + assert!(matches!(&cache_indexes.value, ConfigValue::Matrix(v) if v.len() == 2)); } } diff --git a/core/harness_derive/src/codegen.rs b/core/harness_derive/src/codegen.rs index 76e2b664ad..d6adb360a4 100644 --- a/core/harness_derive/src/codegen.rs +++ b/core/harness_derive/src/codegen.rs @@ -753,12 +753,8 @@ mod tests { transport: Transport::Http, transport_explicit: true, config_values: vec![ - ("segment.size".to_string(), "1MiB".to_string()), ("segment.cache_indexes".to_string(), "all".to_string()), - ( - "partition.messages_required_to_save".to_string(), - "64".to_string(), - ), + ("metadata.journal_slots".to_string(), "1024".to_string()), ], tls: None, websocket_tls: None, @@ -766,7 +762,7 @@ mod tests { }; assert_eq!( v.suffix(), - "http_segment_size_1mib_segment_cache_indexes_all_partition_messages_required_to_save_64" + "http_segment_cache_indexes_all_metadata_journal_slots_1024" ); } @@ -828,14 +824,14 @@ mod tests { transport_explicit: true, server: crate::attrs::ServerAttrs { config_overrides: vec![ - ConfigOverride { - path: "segment.size".to_string(), - value: ConfigValue::Matrix(vec!["512B".to_string(), "1MiB".to_string()]), - }, ConfigOverride { path: "segment.cache_indexes".to_string(), value: ConfigValue::Matrix(vec!["none".to_string(), "all".to_string()]), }, + ConfigOverride { + path: "partition.validate_checksum".to_string(), + value: ConfigValue::Matrix(vec!["true".to_string(), "false".to_string()]), + }, ], ..Default::default() }, @@ -844,7 +840,7 @@ mod tests { jwks_server: None, }; let variants = generate_variants(&attrs); - // 2 transports * 2 segment sizes * 2 cache modes = 8 variants + // 2 transports * 2 cache modes * 2 checksum modes = 8 variants assert_eq!(variants.len(), 8); } @@ -857,18 +853,18 @@ mod tests { #[test] fn cartesian_product_single() { let overrides = vec![ConfigOverride { - path: "segment.size".to_string(), - value: ConfigValue::Matrix(vec!["512B".to_string(), "1MiB".to_string()]), + path: "segment.cache_indexes".to_string(), + value: ConfigValue::Matrix(vec!["none".to_string(), "all".to_string()]), }]; let result = cartesian_product(&overrides); assert_eq!(result.len(), 2); assert_eq!( result[0], - vec![("segment.size".to_string(), "512B".to_string())] + vec![("segment.cache_indexes".to_string(), "none".to_string())] ); assert_eq!( result[1], - vec![("segment.size".to_string(), "1MiB".to_string())] + vec![("segment.cache_indexes".to_string(), "all".to_string())] ); } diff --git a/core/harness_derive/src/lib.rs b/core/harness_derive/src/lib.rs index fb527932c5..757e02127c 100644 --- a/core/harness_derive/src/lib.rs +++ b/core/harness_derive/src/lib.rs @@ -40,11 +40,11 @@ //! Test with server config matrix: //! ```ignore //! #[iggy_harness(server( -//! segment_size = ["512B", "1MiB"], -//! cache_indexes = ["none", "all"], +//! segment.cache_indexes = ["none", "all"], +//! partition.validate_checksum = [true, false], //! ))] //! async fn test_caching(client: &IggyClient) { -//! // 2 segment sizes × 2 cache modes = 4 tests +//! // 2 cache modes × 2 checksum modes = 4 tests //! } //! ``` diff --git a/core/integration/src/harness/config/mod.rs b/core/integration/src/harness/config/mod.rs index fa3aab6d63..81f4a79a93 100644 --- a/core/integration/src/harness/config/mod.rs +++ b/core/integration/src/harness/config/mod.rs @@ -28,5 +28,5 @@ pub use common::{EncryptionConfig, IpAddrKind, TlsConfig}; pub use connectors_runtime::ConnectorsRuntimeConfig; pub use jwks::JwksConfig; pub use mcp::McpConfig; -pub use resolve::resolve_config_paths; +pub use resolve::{resolve_config_paths, validate_env_var_names}; pub use server::TestServerConfig; diff --git a/core/integration/src/harness/config/resolve.rs b/core/integration/src/harness/config/resolve.rs index e5f36f60a4..5ca0ccbc1e 100644 --- a/core/integration/src/harness/config/resolve.rs +++ b/core/integration/src/harness/config/resolve.rs @@ -21,9 +21,20 @@ use configs::server::ServerConfig; use configs::{ConfigEnvMappings, EnvVarMapping}; use std::collections::HashMap; +/// `IGGY_`-prefixed variables the server reads outside the config struct, so +/// `ServerConfig::all_env_var_names` cannot know them. `IGGY_CONFIG_PATH` +/// selects the config file itself and the root credentials are consumed by +/// `args.rs` before the config loads; `IGGY_TEST_VERBOSE` is harness-only. +pub const NON_CONFIG_ENV_VARS: [&str; 4] = [ + "IGGY_CONFIG_PATH", + "IGGY_ROOT_USERNAME", + "IGGY_ROOT_PASSWORD", + "IGGY_TEST_VERBOSE", +]; + /// Resolve config paths to environment variable names. /// -/// Takes a map of dot-notation config paths (e.g., "segment.size") and their values, +/// Takes a map of dot-notation config paths (e.g., "partition.validate_checksum") and their values, /// validates them against the `ServerConfig` mappings, and returns the /// corresponding environment variable names with values. /// @@ -115,6 +126,69 @@ fn find_mapping(path: &str) -> Option<&'static EnvVarMapping> { ServerConfig::find_by_config_path(path) } +/// Reject `IGGY_*` names that no config leaf reads. +/// +/// `extra_envs` is a raw env map with no schema behind it, so a name that was +/// valid before a config key moved or was deleted keeps being set and simply +/// stops doing anything. That failure mode is silent and expensive: the server +/// boots on defaults, the test appears to configure something it does not, and +/// the resulting behavior change surfaces somewhere unrelated. Attribute +/// overrides never had this problem (`resolve_config_paths` validates them); +/// this closes the same gap for the direct path. +/// +/// Names outside the `IGGY_` prefix are left alone: those address the process +/// environment (`RUST_LOG`, test scaffolding), not the config schema. +/// [`NON_CONFIG_ENV_VARS`] carries the `IGGY_`-prefixed names the server reads +/// outside the config struct. +/// +/// # Errors +/// +/// Returns a message naming every unknown variable, with near-miss +/// suggestions drawn from the live mapping table. +pub fn validate_env_var_names(envs: &HashMap) -> Result<(), String> { + let known: Vec<&'static str> = ServerConfig::all_env_var_names(); + let mut unknown: Vec<&String> = envs + .keys() + .filter(|name| { + name.starts_with("IGGY_") + && !known.contains(&name.as_str()) + && !NON_CONFIG_ENV_VARS.contains(&name.as_str()) + }) + .collect(); + if unknown.is_empty() { + return Ok(()); + } + unknown.sort(); + let mut report = String::new(); + for name in unknown { + report.push_str(&format!(" unknown config env var: {name}\n")); + let lowered = name.trim_start_matches("IGGY_").to_ascii_lowercase(); + let mut near: Vec<&&str> = known + .iter() + .filter(|candidate| { + let candidate = candidate.trim_start_matches("IGGY_").to_ascii_lowercase(); + candidate.ends_with(&lowered) + || lowered.ends_with(&candidate) + || levenshtein(&candidate, &lowered) <= 4 + }) + .collect(); + near.sort(); + near.truncate(5); + if !near.is_empty() { + report.push_str(" did you mean: "); + report.push_str( + &near + .iter() + .map(std::string::ToString::to_string) + .collect::>() + .join(", "), + ); + report.push('\n'); + } + } + Err(report) +} + fn levenshtein(a: &str, b: &str) -> usize { let a_len = a.len(); let b_len = b.len(); @@ -193,27 +267,69 @@ mod tests { #[test] fn resolve_valid_path() { let mut overrides = HashMap::new(); - overrides.insert("system.segment.size".to_string(), "1MiB".to_string()); + overrides.insert( + "system.partition.validate_checksum".to_string(), + "false".to_string(), + ); let result = resolve_config_paths(&overrides); assert!(result.is_ok()); let env_vars = result.unwrap(); - assert!(env_vars.contains_key("IGGY_SYSTEM_SEGMENT_SIZE")); + assert!(env_vars.contains_key("IGGY_SYSTEM_PARTITION_VALIDATE_CHECKSUM")); assert_eq!( - env_vars.get("IGGY_SYSTEM_SEGMENT_SIZE"), - Some(&"1MiB".to_string()) + env_vars.get("IGGY_SYSTEM_PARTITION_VALIDATE_CHECKSUM"), + Some(&"false".to_string()) ); } #[test] fn resolve_with_implicit_system_prefix() { let mut overrides = HashMap::new(); - overrides.insert("segment.size".to_string(), "1MiB".to_string()); + overrides.insert( + "partition.validate_checksum".to_string(), + "false".to_string(), + ); let result = resolve_config_paths(&overrides); assert!(result.is_ok()); let env_vars = result.unwrap(); - assert!(env_vars.contains_key("IGGY_SYSTEM_SEGMENT_SIZE")); + assert!(env_vars.contains_key("IGGY_SYSTEM_PARTITION_VALIDATE_CHECKSUM")); + } + + #[test] + fn validate_env_var_names_accepts_live_names_and_passes_through_non_iggy() { + let envs = HashMap::from([ + ( + "IGGY_SYSTEM_PARTITION_VALIDATE_CHECKSUM".to_string(), + "false".to_string(), + ), + ("RUST_LOG".to_string(), "debug".to_string()), + ]); + assert!(validate_env_var_names(&envs).is_ok()); + } + + #[test] + fn validate_env_var_names_accepts_the_non_config_variables() { + let envs: HashMap = NON_CONFIG_ENV_VARS + .iter() + .map(|name| ((*name).to_string(), "value".to_string())) + .collect(); + assert!( + validate_env_var_names(&envs).is_ok(), + "variables the server reads outside the config struct must pass" + ); + } + + #[test] + fn validate_env_var_names_rejects_a_name_no_config_leaf_reads() { + // The exact shape that went silent when these keys moved to per-topic + // options: a name that was valid before and now does nothing. + let envs = HashMap::from([("IGGY_SYSTEM_SEGMENT_SIZE".to_string(), "1MiB".to_string())]); + let error = validate_env_var_names(&envs).expect_err("deleted key must be rejected"); + assert!( + error.contains("IGGY_SYSTEM_SEGMENT_SIZE"), + "the report must name the offending variable, got: {error}" + ); } #[test] diff --git a/core/integration/src/harness/handle/server.rs b/core/integration/src/harness/handle/server.rs index 5965bf10b4..bb6a6486d2 100644 --- a/core/integration/src/harness/handle/server.rs +++ b/core/integration/src/harness/handle/server.rs @@ -350,7 +350,14 @@ impl ServerHandle { self.set_tls_envs("WEBSOCKET", &tls); } - // Extra envs from config (includes resolved config paths from macro) + // Extra envs from config (includes resolved config paths from macro). + // Validated first: a name no config leaf reads is a silent no-op, and + // a test that believes it configured the server but did not is worse + // than one that fails to start. + if let Err(report) = crate::harness::config::validate_env_var_names(&self.config.extra_envs) + { + panic!("invalid extra_envs for the test server:\n{report}"); + } for (k, v) in &self.config.extra_envs { self.envs.insert(k.clone(), v.clone()); } diff --git a/core/integration/src/harness/orchestrator/builder.rs b/core/integration/src/harness/orchestrator/builder.rs index 4eb8b3d78f..23de6d23a8 100644 --- a/core/integration/src/harness/orchestrator/builder.rs +++ b/core/integration/src/harness/orchestrator/builder.rs @@ -386,8 +386,8 @@ mod tests { .server( TestServerConfig::builder() .extra_envs(HashMap::from([( - "IGGY_SYSTEM_SEGMENT_SIZE".to_string(), - "1MiB".to_string(), + "IGGY_SYSTEM_PARTITION_VALIDATE_CHECKSUM".to_string(), + "false".to_string(), )])) .build(), ) @@ -446,7 +446,10 @@ mod tests { .quic_enabled(false) .websocket_enabled(false) .extra_envs(HashMap::from([ - ("IGGY_SYSTEM_SEGMENT_SIZE".to_string(), "2MiB".to_string()), + ( + "IGGY_SYSTEM_PARTITION_VALIDATE_CHECKSUM".to_string(), + "false".to_string(), + ), ("TEST".to_string(), "value".to_string()), ])) .build(), diff --git a/core/integration/src/harness/seeds.rs b/core/integration/src/harness/seeds.rs index 60b6ae00b1..5485dbc6d9 100644 --- a/core/integration/src/harness/seeds.rs +++ b/core/integration/src/harness/seeds.rs @@ -15,10 +15,10 @@ // specific language governing permissions and limitations // under the License. -use iggy::prelude::{IggyClient, StreamClient, TopicClient}; +use iggy::prelude::{IggyClient, StreamClient, TopicClient, TopicCreateOptions}; use iggy_common::{ - CompressionAlgorithm, Consumer, Identifier, IggyExpiry, IggyMessage, MaxTopicSize, - Partitioning, PersonalAccessTokenExpiry, UserStatus, + Consumer, Identifier, IggyExpiry, IggyMessage, MaxTopicSize, Partitioning, + PersonalAccessTokenExpiry, UserStatus, }; use iggy_common::{ ConsumerGroupClient, ConsumerOffsetClient, MessageClient, PersonalAccessTokenClient, UserClient, @@ -55,11 +55,12 @@ pub async fn stream_with_topic(client: &IggyClient) -> Result<(), SeedError> { .create_topic( &"test_stream".try_into()?, "test_topic", - 1, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::Unlimited, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + max_topic_size: Some(MaxTopicSize::Unlimited), + ..TopicCreateOptions::default() + }, ) .await?; Ok(()) @@ -75,11 +76,10 @@ pub async fn connector_stream(client: &IggyClient) -> Result<(), SeedError> { .create_topic( &stream_id, names::TOPIC, - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::ServerDefault, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + ..TopicCreateOptions::default() + }, ) .await?; @@ -98,11 +98,10 @@ pub async fn connector_multi_topic_stream(client: &IggyClient) -> Result<(), See .create_topic( &stream_id, names::TOPIC, - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::ServerDefault, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + ..TopicCreateOptions::default() + }, ) .await?; @@ -110,11 +109,10 @@ pub async fn connector_multi_topic_stream(client: &IggyClient) -> Result<(), See .create_topic( &stream_id, names::TOPIC_2, - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::ServerDefault, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + ..TopicCreateOptions::default() + }, ) .await?; @@ -132,11 +130,10 @@ pub async fn mcp_standard(client: &IggyClient) -> Result<(), SeedError> { .create_topic( &stream_id, names::TOPIC, - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::ServerDefault, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + ..TopicCreateOptions::default() + }, ) .await?; diff --git a/core/integration/tests/cli/consumer_group/test_consumer_group_create_command.rs b/core/integration/tests/cli/consumer_group/test_consumer_group_create_command.rs index 2c9a62c111..f1400d2ccd 100644 --- a/core/integration/tests/cli/consumer_group/test_consumer_group_create_command.rs +++ b/core/integration/tests/cli/consumer_group/test_consumer_group_create_command.rs @@ -21,7 +21,7 @@ use crate::cli::common::{ }; use assert_cmd::assert::Assert; use async_trait::async_trait; -use iggy::prelude::{Client, IggyExpiry, MaxTopicSize}; +use iggy::prelude::{Client, IggyExpiry, TopicCreateOptions}; use predicates::str::diff; use serial_test::parallel; @@ -94,11 +94,11 @@ impl IggyCmdTestCase for TestConsumerGroupCreateCmd { .create_topic( &self.stream_id.try_into().unwrap(), &self.topic_name, - 1, - Default::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await; assert!(topic.is_ok()); diff --git a/core/integration/tests/cli/consumer_group/test_consumer_group_delete_command.rs b/core/integration/tests/cli/consumer_group/test_consumer_group_delete_command.rs index 8012d74354..21fbdb1d13 100644 --- a/core/integration/tests/cli/consumer_group/test_consumer_group_delete_command.rs +++ b/core/integration/tests/cli/consumer_group/test_consumer_group_delete_command.rs @@ -23,7 +23,7 @@ use assert_cmd::assert::Assert; use async_trait::async_trait; use iggy::prelude::Client; use iggy::prelude::IggyExpiry; -use iggy::prelude::MaxTopicSize; +use iggy::prelude::TopicCreateOptions; use predicates::str::diff; use serial_test::parallel; @@ -97,11 +97,11 @@ impl IggyCmdTestCase for TestConsumerGroupDeleteCmd { .create_topic( &self.stream_id.try_into().unwrap(), &self.topic_name, - 1, - Default::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await; assert!(topic.is_ok()); diff --git a/core/integration/tests/cli/consumer_group/test_consumer_group_get_command.rs b/core/integration/tests/cli/consumer_group/test_consumer_group_get_command.rs index 57bb5aec39..6ceb3c94c3 100644 --- a/core/integration/tests/cli/consumer_group/test_consumer_group_get_command.rs +++ b/core/integration/tests/cli/consumer_group/test_consumer_group_get_command.rs @@ -23,7 +23,7 @@ use assert_cmd::assert::Assert; use async_trait::async_trait; use iggy::prelude::Client; use iggy::prelude::IggyExpiry; -use iggy::prelude::MaxTopicSize; +use iggy::prelude::TopicCreateOptions; use predicates::str::{contains, starts_with}; use serial_test::parallel; @@ -97,11 +97,11 @@ impl IggyCmdTestCase for TestConsumerGroupGetCmd { .create_topic( &self.stream_id.try_into().unwrap(), &self.topic_name, - 1, - Default::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await; assert!(topic.is_ok()); diff --git a/core/integration/tests/cli/consumer_group/test_consumer_group_list_command.rs b/core/integration/tests/cli/consumer_group/test_consumer_group_list_command.rs index bd4641089f..8cf6e2f3bf 100644 --- a/core/integration/tests/cli/consumer_group/test_consumer_group_list_command.rs +++ b/core/integration/tests/cli/consumer_group/test_consumer_group_list_command.rs @@ -23,7 +23,7 @@ use assert_cmd::assert::Assert; use async_trait::async_trait; use iggy::prelude::Client; use iggy::prelude::IggyExpiry; -use iggy::prelude::MaxTopicSize; +use iggy::prelude::TopicCreateOptions; use predicates::str::{contains, starts_with}; use serial_test::parallel; @@ -94,11 +94,11 @@ impl IggyCmdTestCase for TestConsumerGroupListCmd { .create_topic( &stream.id.try_into().unwrap(), &self.topic_name, - 1, - Default::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .expect("Failed to create topic"); diff --git a/core/integration/tests/cli/consumer_offset/test_consumer_offset_get_command.rs b/core/integration/tests/cli/consumer_offset/test_consumer_offset_get_command.rs index 1aa27db003..41288c58b7 100644 --- a/core/integration/tests/cli/consumer_offset/test_consumer_offset_get_command.rs +++ b/core/integration/tests/cli/consumer_offset/test_consumer_offset_get_command.rs @@ -104,11 +104,11 @@ impl IggyCmdTestCase for TestConsumerOffsetGetCmd { .create_topic( &stream.id.try_into().unwrap(), &self.topic_name, - 1, - Default::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .expect("Failed to create topic"); diff --git a/core/integration/tests/cli/consumer_offset/test_consumer_offset_set_command.rs b/core/integration/tests/cli/consumer_offset/test_consumer_offset_set_command.rs index 6bb2508dd1..1de4938089 100644 --- a/core/integration/tests/cli/consumer_offset/test_consumer_offset_set_command.rs +++ b/core/integration/tests/cli/consumer_offset/test_consumer_offset_set_command.rs @@ -105,11 +105,11 @@ impl IggyCmdTestCase for TestConsumerOffsetSetCmd { .create_topic( &self.stream_id.try_into().unwrap(), &self.topic_name, - 1, - Default::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await; assert!(topic.is_ok()); diff --git a/core/integration/tests/cli/general/test_help_command.rs b/core/integration/tests/cli/general/test_help_command.rs index 5fbac9129b..499073b546 100644 --- a/core/integration/tests/cli/general/test_help_command.rs +++ b/core/integration/tests/cli/general/test_help_command.rs @@ -38,6 +38,7 @@ Commands: segment segments operations [alias: seg] ping ping iggy server me get current client info + options list the options a resource's create command accepts stats get iggy server statistics snapshot collect iggy server troubleshooting data pat personal access token operations diff --git a/core/integration/tests/cli/general/test_overview_command.rs b/core/integration/tests/cli/general/test_overview_command.rs index 6b528208ad..f8f447e04f 100644 --- a/core/integration/tests/cli/general/test_overview_command.rs +++ b/core/integration/tests/cli/general/test_overview_command.rs @@ -49,6 +49,7 @@ Commands: segment segments operations [alias: seg] ping ping iggy server me get current client info + options list the options a resource's create command accepts stats get iggy server statistics snapshot collect iggy server troubleshooting data pat personal access token operations diff --git a/core/integration/tests/cli/message/test_message_flush_command.rs b/core/integration/tests/cli/message/test_message_flush_command.rs index 6b1dfde463..c07b18968b 100644 --- a/core/integration/tests/cli/message/test_message_flush_command.rs +++ b/core/integration/tests/cli/message/test_message_flush_command.rs @@ -28,7 +28,7 @@ use async_trait::async_trait; use iggy::prelude::Client; use iggy::prelude::Identifier; use iggy::prelude::IggyExpiry; -use iggy::prelude::MaxTopicSize; +use iggy::prelude::TopicCreateOptions; use predicates::str::contains; use serial_test::parallel; use std::str::FromStr; @@ -105,11 +105,11 @@ impl IggyCmdTestCase for TestMessageFetchCmd { .create_topic( &stream.id.try_into().unwrap(), &self.topic_name, - self.partitions_count, - Default::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(self.partitions_count), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .expect("Failed to create topic"); diff --git a/core/integration/tests/cli/message/test_message_poll_command.rs b/core/integration/tests/cli/message/test_message_poll_command.rs index ccb4959fb3..cc91fec1be 100644 --- a/core/integration/tests/cli/message/test_message_poll_command.rs +++ b/core/integration/tests/cli/message/test_message_poll_command.rs @@ -124,11 +124,11 @@ impl IggyCmdTestCase for TestMessagePollCmd { .create_topic( &stream.id.try_into().unwrap(), &self.topic_name, - self.partitions_count, - Default::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(self.partitions_count), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await; assert!(topic.is_ok()); diff --git a/core/integration/tests/cli/message/test_message_poll_to_file_command.rs b/core/integration/tests/cli/message/test_message_poll_to_file_command.rs index 7632cb3e51..a433191af5 100644 --- a/core/integration/tests/cli/message/test_message_poll_to_file_command.rs +++ b/core/integration/tests/cli/message/test_message_poll_to_file_command.rs @@ -117,11 +117,11 @@ impl IggyCmdTestCase for TestMessagePollToFileCmd<'_> { .create_topic( &stream.id.try_into().unwrap(), &self.topic_name, - 1, - Default::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await; assert!(topic.is_ok()); diff --git a/core/integration/tests/cli/message/test_message_send_command.rs b/core/integration/tests/cli/message/test_message_send_command.rs index 65fd96eae8..a24a6c526e 100644 --- a/core/integration/tests/cli/message/test_message_send_command.rs +++ b/core/integration/tests/cli/message/test_message_send_command.rs @@ -162,11 +162,11 @@ impl IggyCmdTestCase for TestMessageSendCmd { .create_topic( &stream.id.try_into().unwrap(), &self.topic_name, - self.partitions_count, - Default::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(self.partitions_count), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .expect("Failed to create topic"); diff --git a/core/integration/tests/cli/message/test_message_send_from_file_command.rs b/core/integration/tests/cli/message/test_message_send_from_file_command.rs index d7b3b5ca12..cc67a0d797 100644 --- a/core/integration/tests/cli/message/test_message_send_from_file_command.rs +++ b/core/integration/tests/cli/message/test_message_send_from_file_command.rs @@ -105,11 +105,11 @@ impl IggyCmdTestCase for TestMessageSendFromFileCmd<'_> { .create_topic( &stream.id.try_into().unwrap(), &self.topic_name, - 1, - Default::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .expect("Failed to create topic"); diff --git a/core/integration/tests/cli/partition/test_partition_create_command.rs b/core/integration/tests/cli/partition/test_partition_create_command.rs index 90299bb9c8..ff243f0556 100644 --- a/core/integration/tests/cli/partition/test_partition_create_command.rs +++ b/core/integration/tests/cli/partition/test_partition_create_command.rs @@ -23,7 +23,7 @@ use async_trait::async_trait; use iggy::prelude::Client; use iggy::prelude::CompressionAlgorithm; use iggy::prelude::IggyExpiry; -use iggy::prelude::MaxTopicSize; +use iggy::prelude::TopicCreateOptions; use predicates::str::diff; use serial_test::parallel; @@ -91,11 +91,14 @@ impl IggyCmdTestCase for TestPartitionCreateCmd { .create_topic( &self.actual_stream_id.unwrap().try_into().unwrap(), &self.topic_name, - self.partitions_count, - self.compression_algorithm, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(self.partitions_count), + compression_algorithm: (self.compression_algorithm + != CompressionAlgorithm::default()) + .then_some(self.compression_algorithm), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await; assert!(topic.is_ok()); diff --git a/core/integration/tests/cli/partition/test_partition_delete_command.rs b/core/integration/tests/cli/partition/test_partition_delete_command.rs index 0025666ee0..d74424b026 100644 --- a/core/integration/tests/cli/partition/test_partition_delete_command.rs +++ b/core/integration/tests/cli/partition/test_partition_delete_command.rs @@ -22,7 +22,7 @@ use assert_cmd::assert::Assert; use async_trait::async_trait; use iggy::prelude::Client; use iggy::prelude::IggyExpiry; -use iggy::prelude::MaxTopicSize; +use iggy::prelude::TopicCreateOptions; use predicates::str::diff; use serial_test::parallel; @@ -79,11 +79,11 @@ impl IggyCmdTestCase for TestPartitionDeleteCmd { .create_topic( &self.actual_stream_id.unwrap().try_into().unwrap(), &self.topic_name, - self.partitions_count, - Default::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(self.partitions_count), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await; assert!(topic.is_ok()); diff --git a/core/integration/tests/cli/stream/test_stream_purge_command.rs b/core/integration/tests/cli/stream/test_stream_purge_command.rs index e0e7bace5c..5a3339a14b 100644 --- a/core/integration/tests/cli/stream/test_stream_purge_command.rs +++ b/core/integration/tests/cli/stream/test_stream_purge_command.rs @@ -61,11 +61,11 @@ impl IggyCmdTestCase for TestStreamPurgeCmd { .create_topic( &self.stream_name.clone().try_into().unwrap(), &self.topic_name, - 10, - Default::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(10), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await; assert!(topic.is_ok()); diff --git a/core/integration/tests/cli/system/test_stats_command.rs b/core/integration/tests/cli/system/test_stats_command.rs index 1510448acf..49575babca 100644 --- a/core/integration/tests/cli/system/test_stats_command.rs +++ b/core/integration/tests/cli/system/test_stats_command.rs @@ -24,8 +24,8 @@ use iggy::prelude::Client; use iggy::prelude::Identifier; use iggy::prelude::IggyExpiry; use iggy::prelude::IggyMessage; -use iggy::prelude::MaxTopicSize; use iggy::prelude::Partitioning; +use iggy::prelude::TopicCreateOptions; use iggy_cli::commands::binary_system::stats::GetStatsOutput; use iggy_common::Stats; use predicates::str::{contains, starts_with}; @@ -74,11 +74,11 @@ impl IggyCmdTestCase for TestStatsCmd { .create_topic( &stream_id, "topic", - 5, - Default::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(5), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await; assert!(topic.is_ok()); @@ -184,11 +184,11 @@ impl IggyCmdTestCase for TestStatsCmdWithMessages { .create_topic( &self.stream_id.try_into().unwrap(), "topic", - 1, - Default::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await; assert!(topic.is_ok()); diff --git a/core/integration/tests/cli/topic/test_topic_create_command.rs b/core/integration/tests/cli/topic/test_topic_create_command.rs index b2d8437a17..af8dbb02a8 100644 --- a/core/integration/tests/cli/topic/test_topic_create_command.rs +++ b/core/integration/tests/cli/topic/test_topic_create_command.rs @@ -40,7 +40,6 @@ struct TestTopicCreateCmd { compression_algorithm: CompressionAlgorithm, message_expiry: Option>, max_topic_size: MaxTopicSize, - replication_factor: u8, using_identifier: TestStreamId, } @@ -54,7 +53,6 @@ impl TestTopicCreateCmd { compression_algorithm: CompressionAlgorithm, message_expiry: Option>, max_topic_size: MaxTopicSize, - replication_factor: u8, using_identifier: TestStreamId, ) -> Self { Self { @@ -65,7 +63,6 @@ impl TestTopicCreateCmd { compression_algorithm, message_expiry, max_topic_size, - replication_factor, using_identifier, } } @@ -120,13 +117,12 @@ impl IggyCmdTestCase for TestTopicCreateCmd { let max_topic_size = self.max_topic_size.to_string(); - let replication_factor = self.replication_factor; - let message = format!( - "Executing create topic with name: {topic_name}, message expiry: {message_expiry}, compression algorithm: {compression_algorithm}, \ - max topic size: {max_topic_size}, replication factor: {replication_factor} in stream with ID: {stream_id}\n\ + "Executing create topic with name: {topic_name}, message expiry: {message_expiry}, \ + compression algorithm: {compression_algorithm}, max topic size: {max_topic_size} \ + in stream with ID: {stream_id}\n\ Topic with name: {topic_name}, partitions count: {partitions_count}, compression algorithm: {compression_algorithm}, message expiry: {message_expiry}, \ - max topic size: {max_topic_size}, replication factor: {replication_factor} created in stream with ID: {stream_id}\n", + max topic size: {max_topic_size} created in stream with ID: {stream_id}\n", ); command_state.success().stdout(diff(message)); @@ -189,7 +185,6 @@ pub async fn should_be_successful() { Default::default(), None, MaxTopicSize::ServerDefault, - 1, TestStreamId::Named, )) .await; @@ -202,7 +197,6 @@ pub async fn should_be_successful() { Default::default(), None, MaxTopicSize::ServerDefault, - 1, TestStreamId::Named, )) .await; @@ -215,7 +209,6 @@ pub async fn should_be_successful() { Default::default(), Some(vec![String::from("3days"), String::from("5s")]), MaxTopicSize::Unlimited, - 1, TestStreamId::Named, )) .await; @@ -233,7 +226,6 @@ pub async fn should_be_successful() { String::from("1s"), ]), MaxTopicSize::Custom(IggyByteSize::from_str("2GiB").unwrap()), - 1, TestStreamId::Named, )) .await; @@ -295,10 +287,12 @@ Options: {CLAP_INDENT} [default: server_default] - -r, --replication-factor - Replication factor for the topic + --set + Additional topic option as key=value, repeatable {CLAP_INDENT} - [default: 1] + Values are sent as strings and parsed server-side through each option's + own FromStr (e.g. --set segment_size=128MiB). The server rejects keys it + does not support; run "iggy options topic" to list the ones it accepts. -h, --help Print help (see a summary with '-h') @@ -328,10 +322,10 @@ Arguments: [MESSAGE_EXPIRY]... Message expiry time in human-readable format like "unlimited" or "15days 2min 2s" [default: server_default] Options: - -t, --topic-id Topic ID to create - -m, --max-topic-size Max topic size in human-readable format like "unlimited" or "15GB" [default: server_default] - -r, --replication-factor Replication factor for the topic [default: 1] - -h, --help Print help (see more with '--help') + -t, --topic-id Topic ID to create + -m, --max-topic-size Max topic size in human-readable format like "unlimited" or "15GB" [default: server_default] + --set Additional topic option as key=value, repeatable + -h, --help Print help (see more with '--help') "#, ), )) diff --git a/core/integration/tests/cli/topic/test_topic_delete_command.rs b/core/integration/tests/cli/topic/test_topic_delete_command.rs index 9400f2ee02..24b572c4de 100644 --- a/core/integration/tests/cli/topic/test_topic_delete_command.rs +++ b/core/integration/tests/cli/topic/test_topic_delete_command.rs @@ -23,7 +23,7 @@ use assert_cmd::assert::Assert; use async_trait::async_trait; use iggy::prelude::Client; use iggy::prelude::IggyExpiry; -use iggy::prelude::MaxTopicSize; +use iggy::prelude::TopicCreateOptions; use predicates::str::diff; use serial_test::parallel; @@ -80,11 +80,11 @@ impl IggyCmdTestCase for TestTopicDeleteCmd { .create_topic( &self.stream_name.clone().try_into().unwrap(), &self.topic_name, - 1, - Default::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await; assert!(topic.is_ok()); diff --git a/core/integration/tests/cli/topic/test_topic_get_command.rs b/core/integration/tests/cli/topic/test_topic_get_command.rs index 46435dde3c..5715e92118 100644 --- a/core/integration/tests/cli/topic/test_topic_get_command.rs +++ b/core/integration/tests/cli/topic/test_topic_get_command.rs @@ -23,7 +23,7 @@ use assert_cmd::assert::Assert; use async_trait::async_trait; use iggy::prelude::Client; use iggy::prelude::IggyExpiry; -use iggy::prelude::MaxTopicSize; +use iggy::prelude::TopicCreateOptions; use predicates::str::{contains, starts_with}; use serial_test::parallel; @@ -80,11 +80,11 @@ impl IggyCmdTestCase for TestTopicGetCmd { .create_topic( &self.stream_name.clone().try_into().unwrap(), &self.topic_name, - 1, - Default::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await; assert!(topic.is_ok()); diff --git a/core/integration/tests/cli/topic/test_topic_list_command.rs b/core/integration/tests/cli/topic/test_topic_list_command.rs index a0162ae8cc..08a277e43b 100644 --- a/core/integration/tests/cli/topic/test_topic_list_command.rs +++ b/core/integration/tests/cli/topic/test_topic_list_command.rs @@ -23,7 +23,7 @@ use assert_cmd::assert::Assert; use async_trait::async_trait; use iggy::prelude::Client; use iggy::prelude::IggyExpiry; -use iggy::prelude::MaxTopicSize; +use iggy::prelude::TopicCreateOptions; use predicates::str::{contains, starts_with}; use serial_test::parallel; @@ -74,11 +74,11 @@ impl IggyCmdTestCase for TestTopicListCmd { .create_topic( &self.stream_name.clone().try_into().unwrap(), &self.topic_name, - 1, - Default::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await; assert!(topic.is_ok()); diff --git a/core/integration/tests/cli/topic/test_topic_purge_command.rs b/core/integration/tests/cli/topic/test_topic_purge_command.rs index 4d1323c6f6..e028f0dadd 100644 --- a/core/integration/tests/cli/topic/test_topic_purge_command.rs +++ b/core/integration/tests/cli/topic/test_topic_purge_command.rs @@ -79,11 +79,11 @@ impl IggyCmdTestCase for TestTopicPurgeCmd { .create_topic( &self.stream_name.clone().try_into().unwrap(), &self.topic_name, - 10, - Default::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(10), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await; assert!(topic.is_ok()); diff --git a/core/integration/tests/cli/topic/test_topic_update_command.rs b/core/integration/tests/cli/topic/test_topic_update_command.rs index b259f06b89..9cad22689f 100644 --- a/core/integration/tests/cli/topic/test_topic_update_command.rs +++ b/core/integration/tests/cli/topic/test_topic_update_command.rs @@ -27,6 +27,7 @@ use iggy::prelude::CompressionAlgorithm; use iggy::prelude::IggyByteSize; use iggy::prelude::IggyExpiry; use iggy::prelude::MaxTopicSize; +use iggy::prelude::TopicCreateOptions; use predicates::str::diff; use serial_test::parallel; use std::str::FromStr; @@ -40,12 +41,10 @@ struct TestTopicUpdateCmd { compression_algorithm: CompressionAlgorithm, message_expiry: Option>, max_topic_size: MaxTopicSize, - replication_factor: u8, topic_new_name: String, topic_new_compression_algorithm: CompressionAlgorithm, topic_new_message_expiry: Option>, topic_new_max_size: MaxTopicSize, - topic_new_replication_factor: u8, using_stream_id: TestStreamId, using_topic_id: TestTopicId, } @@ -60,12 +59,10 @@ impl TestTopicUpdateCmd { compression_algorithm: CompressionAlgorithm, message_expiry: Option>, max_topic_size: MaxTopicSize, - replication_factor: u8, topic_new_name: String, topic_new_compression_algorithm: CompressionAlgorithm, topic_new_message_expiry: Option>, topic_new_max_size: MaxTopicSize, - topic_new_replication_factor: u8, using_stream_id: TestStreamId, using_topic_id: TestTopicId, ) -> Self { @@ -77,12 +74,10 @@ impl TestTopicUpdateCmd { compression_algorithm, message_expiry, max_topic_size, - replication_factor, topic_new_name, topic_new_compression_algorithm, topic_new_message_expiry, topic_new_max_size, - topic_new_replication_factor, using_stream_id, using_topic_id, } @@ -103,13 +98,6 @@ impl TestTopicUpdateCmd { command.push(self.topic_new_compression_algorithm.to_string()); command.push(format!("--max-topic-size={}", self.topic_new_max_size)); - if self.topic_new_replication_factor != 1 { - command.push(format!( - "--replication-factor={}", - self.topic_new_replication_factor - )); - } - if let Some(message_expiry) = &self.topic_new_message_expiry { command.extend(message_expiry.clone()); } @@ -138,11 +126,17 @@ impl IggyCmdTestCase for TestTopicUpdateCmd { .create_topic( &self.stream_name.clone().try_into().unwrap(), &self.topic_name, - 1, - self.compression_algorithm, - Some(self.replication_factor), - message_expiry, - self.max_topic_size, + &TopicCreateOptions { + partitions_count: Some(1), + compression_algorithm: (self.compression_algorithm + != CompressionAlgorithm::default()) + .then_some(self.compression_algorithm), + message_expiry: (message_expiry != IggyExpiry::ServerDefault) + .then_some(message_expiry), + max_topic_size: (self.max_topic_size != MaxTopicSize::ServerDefault) + .then_some(self.max_topic_size), + ..TopicCreateOptions::default() + }, ) .await; assert!(topic.is_ok()); @@ -175,17 +169,16 @@ impl IggyCmdTestCase for TestTopicUpdateCmd { }) .to_string(); - let replication_factor = self.topic_new_replication_factor; let new_topic_name = &self.topic_new_name; let new_max_topic_size = self.topic_new_max_size.to_string(); let expected_message = format!( "Executing update topic with ID: {topic_id}, name: {new_topic_name}, \ - message expiry: {message_expiry}, compression algorithm: {compression_algorithm}, max topic size: {new_max_topic_size}, \ - replication factor: {replication_factor}, in stream with ID: {stream_id}\n\ + message expiry: {message_expiry}, compression algorithm: {compression_algorithm}, max topic size: \ + {new_max_topic_size}, in stream with ID: {stream_id}\n\ Topic with ID: {topic_id} updated name: {new_topic_name}, updated message expiry: {message_expiry}, \ - updated compression algorithm: {compression_algorithm}, updated max topic size: {new_max_topic_size}, \ - updated replication factor: {replication_factor} in stream with ID: {stream_id}\n" + updated compression algorithm: {compression_algorithm}, \ + updated max topic size: {new_max_topic_size} in stream with ID: {stream_id}\n" ); command_state.success().stdout(diff(expected_message)); @@ -246,12 +239,10 @@ pub async fn should_be_successful() { Default::default(), None, MaxTopicSize::ServerDefault, - 1, String::from("new_name"), CompressionAlgorithm::Gzip, None, MaxTopicSize::Custom(IggyByteSize::from_str("2GiB").unwrap()), - 1, TestStreamId::Named, TestTopicId::Named, )) @@ -265,12 +256,10 @@ pub async fn should_be_successful() { Default::default(), None, MaxTopicSize::ServerDefault, - 1, String::from("testing"), CompressionAlgorithm::Gzip, None, MaxTopicSize::Unlimited, - 1, TestStreamId::Named, TestTopicId::Named, )) @@ -338,11 +327,6 @@ Options: {CLAP_INDENT} [default: server_default] - -r, --replication-factor - New replication factor for the topic -{CLAP_INDENT} - [default: 1] - -h, --help Print help (see a summary with '-h') "#, @@ -372,9 +356,8 @@ Arguments: [MESSAGE_EXPIRY]... New message expiry time in human-readable format like "unlimited" or "15days 2min 2s" [default: server_default] Options: - -m, --max-topic-size New max topic size in human-readable format like "unlimited" or "15GB" [default: server_default] - -r, --replication-factor New replication factor for the topic [default: 1] - -h, --help Print help (see more with '--help') + -m, --max-topic-size New max topic size in human-readable format like "unlimited" or "15GB" [default: server_default] + -h, --help Print help (see more with '--help') "#, ), )) diff --git a/core/integration/tests/cluster/client_table_restart.rs b/core/integration/tests/cluster/client_table_restart.rs index b6fd026d14..2d93d106bd 100644 --- a/core/integration/tests/cluster/client_table_restart.rs +++ b/core/integration/tests/cluster/client_table_restart.rs @@ -81,7 +81,9 @@ use iggy_binary_protocol::consensus::{ use iggy_binary_protocol::requests::streams::CreateStreamRequest; use iggy_binary_protocol::requests::users::LoginRegisterRequest; use iggy_binary_protocol::responses::users::LoginRegisterResponse; -use iggy_binary_protocol::{ClientVersionInfo, HEADER_SIZE, IGGY_PROTOCOL_VERSION, WireName}; +use iggy_binary_protocol::{ + ClientVersionInfo, HEADER_SIZE, IGGY_PROTOCOL_VERSION, WireName, WireOptions, +}; use integration::harness::TestHarness; use integration::iggy_harness; use secrecy::SecretString; @@ -258,6 +260,7 @@ pub(super) fn tcp_addrs(harness: &TestHarness) -> Vec { pub(super) fn create_stream_payload(name: &str) -> Bytes { CreateStreamRequest { name: WireName::new(name).unwrap(), + options: WireOptions::empty(), } .to_bytes() } diff --git a/core/integration/tests/cluster/multi_shard_partition_convergence.rs b/core/integration/tests/cluster/multi_shard_partition_convergence.rs index 5379074b94..88a782efa7 100644 --- a/core/integration/tests/cluster/multi_shard_partition_convergence.rs +++ b/core/integration/tests/cluster/multi_shard_partition_convergence.rs @@ -67,11 +67,11 @@ async fn create_topic(client: &IggyClient, stream: &Identifier, name: &str) { .create_topic( stream, name, - 1, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .unwrap_or_else(|error| panic!("create_topic {name}: {error}")); diff --git a/core/integration/tests/cluster/partition_state_transfer.rs b/core/integration/tests/cluster/partition_state_transfer.rs index fbc5c93919..daef6ed976 100644 --- a/core/integration/tests/cluster/partition_state_transfer.rs +++ b/core/integration/tests/cluster/partition_state_transfer.rs @@ -51,6 +51,10 @@ const STORED_CONSUMER_OFFSET: u64 = 17; /// 16 MiB pull completed in ~120ms on a release build, inside one marker /// poll). Also enough commits (> ring capacity 64) that the fresh /// rejoiner's floor refuses. +/// Segment cap for the multi-artifact spec: small enough that the 64 MiB bulky +/// seed spans several sealed segments, so the install runs from a manifest with +/// one spill per artifact rather than a single segment. +const SEGMENT_SIZE_MULTI_ARTIFACT: u64 = 8 * 1024 * 1024; const BULKY_MESSAGES_COUNT: u32 = 256; const BULKY_PAYLOAD_LEN: usize = 256 * 1024; /// Poll for the kill gate: the serving marker appears at descriptor-serve @@ -85,8 +89,7 @@ const MARKER_POLL: Duration = Duration::from_millis(200); cluster_nodes = 3, server( system.sharding.cpu_allocation = "0..1", - partition.evicted_ring_capacity = "64", - system.partition.messages_required_to_save = "1" + partition.evicted_ring_capacity = "64" ) )] async fn given_evicted_ring_when_fresh_node_joins_late_should_state_transfer_partition( @@ -190,8 +193,7 @@ async fn given_evicted_ring_when_fresh_node_joins_late_should_state_transfer_par cluster_nodes = 3, server( system.sharding.cpu_allocation = "0..1", - partition.evicted_ring_capacity = "64", - system.partition.messages_required_to_save = "1" + partition.evicted_ring_capacity = "64" ) )] async fn given_evicted_ring_when_node_restarts_with_data_should_state_transfer_partition( @@ -204,7 +206,7 @@ async fn given_evicted_ring_when_node_restarts_with_data_should_state_transfer_p .root_client_for_node(0) .await .expect("connect a root client to the node"); - seed_topic(&client).await; + seed_topic(&client, None).await; produce(&client, 40).await; sleep(Duration::from_secs(1)).await; harness.stop_node(2).expect("stop node 2"); @@ -242,12 +244,10 @@ async fn given_evicted_ring_when_node_restarts_with_data_should_state_transfer_p cluster_nodes = 3, server( system.sharding.cpu_allocation = "0..1", - partition.evicted_ring_capacity = "64", - system.partition.messages_required_to_save = "1", - system.segment.size = "8MiB" + partition.evicted_ring_capacity = "64" ) )] -// The 8 MiB segment cap makes the 64 MiB bulky seed span several sealed +// The topic's 8 MiB segment cap makes the 64 MiB bulky seed span several sealed // segments, so unlike the other specs (single-segment under the default 1 GiB // cap) this one installs from a MULTI-ARTIFACT manifest, one spill per artifact. // The installed segment count at the end is what asserts that. The re-armed @@ -261,7 +261,7 @@ async fn given_transfer_peer_dies_when_stalled_should_leave_dead_peer_and_recove .root_client_for_node(0) .await .expect("connect a root client to the node"); - seed_topic(&client).await; + seed_topic(&client, Some(SEGMENT_SIZE_MULTI_ARTIFACT)).await; // Bulky payloads so the pull spans many 256 KiB chunks: the kill below // must land while the transfer is provably in flight, and a small // partition finishes inside the marker-poll latency, leaving the @@ -357,7 +357,12 @@ async fn connect_any(harness: &TestHarness, nodes: &[usize]) -> Option) { client .create_stream(STREAM_NAME) .await @@ -366,11 +371,13 @@ async fn seed_topic(client: &IggyClient) { .create_topic( &Identifier::named(STREAM_NAME).expect("stream identifier"), TOPIC_NAME, - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + messages_required_to_save: Some(1), + segment_size: segment_size.map(IggyByteSize::from), + ..TopicCreateOptions::default() + }, ) .await .expect("create topic with one partition"); @@ -413,7 +420,7 @@ async fn produce_bulky(client: &IggyClient, count: u32, payload_len: usize) { } async fn seed_partition(client: &IggyClient) { - seed_topic(client).await; + seed_topic(client, None).await; produce(client, MESSAGES_COUNT).await; assert_eq!( poll_count(client, MESSAGES_COUNT).await, diff --git a/core/integration/tests/cluster/register_forwarding.rs b/core/integration/tests/cluster/register_forwarding.rs index 4811956f32..f85d74db5e 100644 --- a/core/integration/tests/cluster/register_forwarding.rs +++ b/core/integration/tests/cluster/register_forwarding.rs @@ -341,11 +341,11 @@ async fn given_a_backup_when_auto_login_dials_it_should_settle_on_the_leader( .create_topic( &stream_id, TOPIC_NAME, - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .expect("create topic after a backup-dialed login"); diff --git a/core/integration/tests/config_provider/mod.rs b/core/integration/tests/config_provider/mod.rs index 32d0d9aec6..a3bfcbe839 100644 --- a/core/integration/tests/config_provider/mod.rs +++ b/core/integration/tests/config_provider/mod.rs @@ -32,7 +32,7 @@ async fn validate_config_env_override() { let expected_http = true; let expected_tcp = true; let expected_message_saver = true; - let expected_message_expiry = "1s"; + let expected_validate_checksum = false; unsafe { env::set_var("IGGY_HTTP_ENABLED", expected_http.to_string()); @@ -41,7 +41,10 @@ async fn validate_config_env_override() { "IGGY_MESSAGE_SAVER_ENABLED", expected_message_saver.to_string(), ); - env::set_var("IGGY_SYSTEM_TOPIC_MESSAGE_EXPIRY", expected_message_expiry); + env::set_var( + "IGGY_SYSTEM_PARTITION_VALIDATE_CHECKSUM", + expected_validate_checksum.to_string(), + ); } let config_path = get_root_path().join("../server/config.toml"); @@ -56,15 +59,15 @@ async fn validate_config_env_override() { assert_eq!(config.tcp.enabled, expected_tcp); assert_eq!(config.message_saver.enabled, expected_message_saver); assert_eq!( - config.system.topic.message_expiry.to_string(), - expected_message_expiry + config.system.partition.validate_checksum, + expected_validate_checksum ); unsafe { env::remove_var("IGGY_HTTP_ENABLED"); env::remove_var("IGGY_TCP_ENABLED"); env::remove_var("IGGY_MESSAGE_SAVER_ENABLED"); - env::remove_var("IGGY_SYSTEM_TOPIC_MESSAGE_EXPIRY"); + env::remove_var("IGGY_SYSTEM_PARTITION_VALIDATE_CHECKSUM"); } } diff --git a/core/integration/tests/connectors/fixtures/doris/container.rs b/core/integration/tests/connectors/fixtures/doris/container.rs index f435084ec4..ae972af7b2 100644 --- a/core/integration/tests/connectors/fixtures/doris/container.rs +++ b/core/integration/tests/connectors/fixtures/doris/container.rs @@ -194,6 +194,88 @@ fn check_vm_max_map_count() -> Result<(), TestBinaryError> { Ok(()) } +/// Attempts to create-or-attach the shared container before giving up. Only +/// name conflicts are retried, and each one means another process already +/// created it, so the bound is about how many tests can be racing at once +/// rather than about how long Doris takes to boot. +const CONTAINER_START_ATTEMPTS: u32 = 30; +const CONTAINER_START_RETRY_DELAY: Duration = Duration::from_secs(1); + +/// Creates the shared Doris container, or attaches to it if another test won +/// the race. +/// +/// `ReuseDirective::Always` resolves reuse by inspecting the daemon and then +/// creating when nothing matches, with no lock spanning the two steps. On a +/// cold daemon with several doris tests in flight, every one of them inspects +/// before any of them creates, so one wins and the rest get +/// `409 Conflict: name already in use`. Retrying is what turns those losers +/// into attachers: by the next attempt the winner's container exists, so the +/// inspect half succeeds. Any other failure is returned as-is. +async fn start_shared_container( + entrypoint_cmd: &str, +) -> Result, TestBinaryError> { + let mut conflict = String::new(); + for attempt in 1..=CONTAINER_START_ATTEMPTS { + // FE HTTP and FE MySQL get ephemeral host ports (the connector and + // tests connect via the resolved mapping). BE HTTP must be pinned + // 1:1 — the FE always returns Location: http://127.0.0.1:8040/... + // for the Stream Load redirect, and that's only reachable from the + // host if container:8040 is bound to host:8040. + // + // `with_container_name` + `with_reuse(Always)` is what makes the + // container survive across nextest's per-test processes: the first + // test creates `iggy-test-doris`, every later test (in any process) + // attaches to it. The 1:1 BE port is therefore held continuously by + // one container, never racing with itself across container restarts. + // + // Rebuilt per attempt because `start` consumes the request. + let result = GenericImage::new(DORIS_IMAGE, DORIS_TAG) + // GenericImage's own with_entrypoint/with_wait_for must come before + // any ImageExt method, which turns GenericImage into ContainerRequest. + .with_entrypoint("bash") + .with_wait_for(WaitFor::http( + HttpWaitStrategy::new(FE_HEALTH_ENDPOINT) + .with_port(FE_HTTP_PORT.tcp()) + .with_expected_status_code(200u16), + )) + .with_env_var("SKIP_CHECK_ULIMIT", "true") + .with_cmd(["-c", entrypoint_cmd]) + .with_mapped_port(0, FE_HTTP_PORT.tcp()) + .with_mapped_port(0, FE_MYSQL_PORT.tcp()) + .with_mapped_port(BE_HTTP_PORT, BE_HTTP_PORT.tcp()) + .with_container_name(DORIS_CONTAINER_NAME) + .with_reuse(ReuseDirective::Always) + .start() + .await; + + match result { + Ok(container) => return Ok(container), + Err(error) => { + let message = error.to_string(); + if !message.contains("is already in use") { + return Err(TestBinaryError::FixtureSetup { + fixture_type: "DorisContainer".to_string(), + message: format!("Failed to start container: {message}"), + }); + } + info!( + "Doris container name taken by another test (attempt {attempt}), retrying to attach" + ); + conflict = message; + sleep(CONTAINER_START_RETRY_DELAY).await; + } + } + } + + Err(TestBinaryError::FixtureSetup { + fixture_type: "DorisContainer".to_string(), + message: format!( + "Failed to attach to container '{DORIS_CONTAINER_NAME}' after \ + {CONTAINER_START_ATTEMPTS} attempts: {conflict}" + ), + }) +} + pub struct DorisContainer { // Held only so testcontainers' Drop runs on test exit. ReuseDirective::Always // makes that Drop a no-op (the container is left running for the next test @@ -222,39 +304,7 @@ impl DorisContainer { exec bash /usr/local/bin/entry_point.sh" ); - // FE HTTP and FE MySQL get ephemeral host ports (the connector and - // tests connect via the resolved mapping). BE HTTP must be pinned - // 1:1 — the FE always returns Location: http://127.0.0.1:8040/... - // for the Stream Load redirect, and that's only reachable from the - // host if container:8040 is bound to host:8040. - // - // `with_container_name` + `with_reuse(Always)` is what makes the - // container survive across nextest's per-test processes: the first - // test creates `iggy-test-doris`, every later test (in any process) - // attaches to it. The 1:1 BE port is therefore held continuously by - // one container, never racing with itself across container restarts. - let container = GenericImage::new(DORIS_IMAGE, DORIS_TAG) - // GenericImage's own with_entrypoint/with_wait_for must come before - // any ImageExt method, which turns GenericImage into ContainerRequest. - .with_entrypoint("bash") - .with_wait_for(WaitFor::http( - HttpWaitStrategy::new(FE_HEALTH_ENDPOINT) - .with_port(FE_HTTP_PORT.tcp()) - .with_expected_status_code(200u16), - )) - .with_env_var("SKIP_CHECK_ULIMIT", "true") - .with_cmd(["-c", entrypoint_cmd.as_str()]) - .with_mapped_port(0, FE_HTTP_PORT.tcp()) - .with_mapped_port(0, FE_MYSQL_PORT.tcp()) - .with_mapped_port(BE_HTTP_PORT, BE_HTTP_PORT.tcp()) - .with_container_name(DORIS_CONTAINER_NAME) - .with_reuse(ReuseDirective::Always) - .start() - .await - .map_err(|e| TestBinaryError::FixtureSetup { - fixture_type: "DorisContainer".to_string(), - message: format!("Failed to start container: {e}"), - })?; + let container = start_shared_container(&entrypoint_cmd).await?; let ports = container .ports() diff --git a/core/integration/tests/connectors/fixtures/elasticsearch/container.rs b/core/integration/tests/connectors/fixtures/elasticsearch/container.rs index 5749b18096..1fded111f3 100644 --- a/core/integration/tests/connectors/fixtures/elasticsearch/container.rs +++ b/core/integration/tests/connectors/fixtures/elasticsearch/container.rs @@ -20,12 +20,14 @@ use reqwest_middleware::ClientWithMiddleware as HttpClient; use reqwest_retry::RetryTransientMiddleware; use reqwest_retry::policies::ExponentialBackoff; use serde::Deserialize; +use std::time::Duration; use testcontainers_modules::testcontainers::core::wait::HttpWaitStrategy; use testcontainers_modules::testcontainers::core::{IntoContainerPort, WaitFor}; use testcontainers_modules::testcontainers::runners::AsyncRunner; use testcontainers_modules::testcontainers::{ ContainerAsync, GenericImage, ImageExt, ReuseDirective, }; +use tokio::time::sleep; use tracing::info; const ELASTICSEARCH_IMAGE: &str = "docker.io/library/elasticsearch"; @@ -99,16 +101,30 @@ pub struct ElasticsearchContainer { pub base_url: String, } -impl ElasticsearchContainer { - pub async fn start() -> Result { - let container = GenericImage::new(ELASTICSEARCH_IMAGE, ELASTICSEARCH_TAG) +/// See [`start_shared_container`]: same create-or-attach race as the Doris +/// fixture, same bound. +const CONTAINER_START_ATTEMPTS: u32 = 30; +const CONTAINER_START_RETRY_DELAY: Duration = Duration::from_secs(1); + +/// Creates the shared Elasticsearch container, or attaches to it if another +/// test won the race. +/// +/// `ReuseDirective::Always` inspects the daemon and then creates when nothing +/// matches, with no lock across the two steps, so a cold daemon with several +/// elasticsearch tests in flight leaves one winner and the rest holding +/// `409 Conflict: name already in use`. Retrying converts those into attaches. +async fn start_shared_container() -> Result, TestBinaryError> { + let mut conflict = String::new(); + for attempt in 1..=CONTAINER_START_ATTEMPTS { + // Rebuilt per attempt because `start` consumes the request. + let result = GenericImage::new(ELASTICSEARCH_IMAGE, ELASTICSEARCH_TAG) .with_exposed_port(ELASTICSEARCH_PORT.tcp()) .with_wait_for(WaitFor::http( HttpWaitStrategy::new(ELASTICSEARCH_HEALTH_ENDPOINT) .with_port(ELASTICSEARCH_PORT.tcp()) .with_expected_status_code(200u16), )) - .with_startup_timeout(std::time::Duration::from_secs(120)) + .with_startup_timeout(Duration::from_secs(120)) .with_env_var("discovery.type", "single-node") .with_env_var("xpack.security.enabled", "false") .with_env_var("ES_JAVA_OPTS", "-Xms512m -Xmx512m") @@ -116,11 +132,39 @@ impl ElasticsearchContainer { .with_container_name(ELASTICSEARCH_CONTAINER_NAME) .with_reuse(ReuseDirective::Always) .start() - .await - .map_err(|e| TestBinaryError::FixtureSetup { - fixture_type: "ElasticsearchContainer".to_string(), - message: format!("Failed to start container: {e}"), - })?; + .await; + + match result { + Ok(container) => return Ok(container), + Err(error) => { + let message = error.to_string(); + if !message.contains("is already in use") { + return Err(TestBinaryError::FixtureSetup { + fixture_type: "ElasticsearchContainer".to_string(), + message: format!("Failed to start container: {message}"), + }); + } + info!( + "Elasticsearch container name taken by another test (attempt {attempt}), retrying to attach" + ); + conflict = message; + sleep(CONTAINER_START_RETRY_DELAY).await; + } + } + } + + Err(TestBinaryError::FixtureSetup { + fixture_type: "ElasticsearchContainer".to_string(), + message: format!( + "Failed to attach to container '{ELASTICSEARCH_CONTAINER_NAME}' after \ + {CONTAINER_START_ATTEMPTS} attempts: {conflict}" + ), + }) +} + +impl ElasticsearchContainer { + pub async fn start() -> Result { + let container = start_shared_container().await?; info!("Started Elasticsearch container"); diff --git a/core/integration/tests/connectors/fixtures/s3/fixture.rs b/core/integration/tests/connectors/fixtures/s3/fixture.rs index adeb03405d..3fdbf17818 100644 --- a/core/integration/tests/connectors/fixtures/s3/fixture.rs +++ b/core/integration/tests/connectors/fixtures/s3/fixture.rs @@ -21,6 +21,7 @@ use integration::harness::{TestBinaryError, TestFixture}; use s3::creds::Credentials; use s3::{Bucket, Region}; use std::collections::HashMap; +use std::time::Duration; use testcontainers_modules::testcontainers::core::wait::HttpWaitStrategy; use testcontainers_modules::testcontainers::core::{IntoContainerPort, WaitFor}; use testcontainers_modules::testcontainers::runners::AsyncRunner; @@ -36,6 +37,9 @@ const MINIO_CONSOLE_PORT: u16 = 9001; const MINIO_ACCESS_KEY: &str = "admin"; const MINIO_SECRET_KEY: &str = "password"; const MINIO_BUCKET: &str = "iggy-s3-test"; +/// Bounds the wait for MinIO's S3 API to come up behind its health endpoint. +const BUCKET_CREATE_ATTEMPTS: u32 = 30; +const BUCKET_CREATE_RETRY_DELAY: Duration = Duration::from_secs(1); const ENV_SINK_PATH: &str = "IGGY_CONNECTORS_SINK_S3_PATH"; const ENV_SINK_STREAMS_0_STREAM: &str = "IGGY_CONNECTORS_SINK_S3_STREAMS_0_STREAM"; @@ -204,22 +208,44 @@ impl TestFixture for S3SinkFixture { message: format!("Failed to create credentials: {e}"), })?; - let config = s3::BucketConfiguration::default(); - let response = Bucket::create_with_path_style( - MINIO_BUCKET, - region.clone(), - credentials.clone(), - config, - ) - .await - .map_err(|e| TestBinaryError::FixtureSetup { - fixture_type: "S3SinkFixture".to_string(), - message: format!("Failed to create bucket: {e}"), - })?; - info!( - "S3 bucket '{}' ready (status: {})", - MINIO_BUCKET, response.response_code - ); + // MinIO answers on its health endpoint before it can serve the S3 API, + // so bucket creation can come back 503 while it finishes starting. The + // call itself is `Ok` in that case -- the status lives in the response + // -- so taking it as success left the bucket absent, and the first + // `list_objects` then parsed an S3 error document as a listing and + // failed with the unhelpful `missing field 'Name'`. Retry until the + // status is a real one: 2xx created, 409 already owned by us. + let mut last_status = 0; + let mut created = false; + for _ in 0..BUCKET_CREATE_ATTEMPTS { + let response = Bucket::create_with_path_style( + MINIO_BUCKET, + region.clone(), + credentials.clone(), + s3::BucketConfiguration::default(), + ) + .await + .map_err(|e| TestBinaryError::FixtureSetup { + fixture_type: "S3SinkFixture".to_string(), + message: format!("Failed to create bucket: {e}"), + })?; + last_status = response.response_code; + if (200..300).contains(&last_status) || last_status == 409 { + created = true; + break; + } + tokio::time::sleep(BUCKET_CREATE_RETRY_DELAY).await; + } + if !created { + return Err(TestBinaryError::FixtureSetup { + fixture_type: "S3SinkFixture".to_string(), + message: format!( + "Bucket '{MINIO_BUCKET}' not creatable after \ + {BUCKET_CREATE_ATTEMPTS} attempts (last status: {last_status})" + ), + }); + } + info!("S3 bucket '{MINIO_BUCKET}' ready (status: {last_status})"); let mut bucket = Bucket::new(MINIO_BUCKET, region, credentials).map_err(|e| { TestBinaryError::FixtureSetup { diff --git a/core/integration/tests/data_integrity/verify_after_server_restart.rs b/core/integration/tests/data_integrity/verify_after_server_restart.rs index 1bed4408e8..1ac9b91fa4 100644 --- a/core/integration/tests/data_integrity/verify_after_server_restart.rs +++ b/core/integration/tests/data_integrity/verify_after_server_restart.rs @@ -40,31 +40,19 @@ fn build_server_config(cache_setting: &str) -> TestServerConfig { "IGGY_SYSTEM_SEGMENT_CACHE_INDEXES".to_string(), cache_setting.to_string(), ); - // The server flushes on the journal thresholds (no flush primitive), so - // force every committed batch straight to disk: the restart asserts - // below need everything durable, and the explicit flush calls are - // cfg'd out under vsr (`flush_unsaved_buffer` answers - // FeatureUnavailable there and is slated for removal). Legacy keeps its - // shipped buffered defaults; the flush loops below are its barrier. - extra_envs.insert( - "IGGY_SYSTEM_PARTITION_MESSAGES_REQUIRED_TO_SAVE".to_string(), - "1".to_string(), - ); - extra_envs.insert( - "IGGY_SYSTEM_PARTITION_ENFORCE_FSYNC".to_string(), - "true".to_string(), - ); TestServerConfig::builder().extra_envs(extra_envs).build() } // TODO(numminex) - Move the message generation method from benchmark run to a special method. // -// The durability barrier is the eager-flush envs in `build_server_config` -// (`flush_unsaved_buffer` answers FeatureUnavailable on VSR, so there is no -// explicit flush loop), and `iggy-bench` must be freshly built: the harness -// spawns the prebuilt binary, and a stale one never completes a frame -// against the server, tripping the bench timeout in -// `run_bench_and_wait_for_finish`. +// The durability barrier is the graceful restart itself: shutdown force-flushes +// the committed journal. The eager-flush knobs that used to stand in for it are +// topic creation options now, and the topics here are created by `iggy-bench`, +// which exposes no flag for them. +// +// `iggy-bench` must be freshly built: the harness spawns the prebuilt binary, +// and a stale one never completes a frame against the server, tripping the +// bench timeout in `run_bench_and_wait_for_finish`. #[test_matrix( [cache_all(), cache_open_segment(), cache_none()] )] @@ -331,11 +319,12 @@ async fn should_handle_resource_deletion_and_restart() { .create_topic( &stream_ident, &format!("topic-{}", topic_idx), - 3, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::Unlimited, + &TopicCreateOptions { + partitions_count: Some(3), + message_expiry: Some(IggyExpiry::NeverExpire), + max_topic_size: Some(MaxTopicSize::Unlimited), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); @@ -411,11 +400,12 @@ async fn should_handle_resource_deletion_and_restart() { .create_topic( &stream_0_ident, "topic-reused", - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::Unlimited, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + max_topic_size: Some(MaxTopicSize::Unlimited), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); diff --git a/core/integration/tests/data_integrity/verify_auto_commit_offset_replicates.rs b/core/integration/tests/data_integrity/verify_auto_commit_offset_replicates.rs index cb953b1b8c..a2ec465bda 100644 --- a/core/integration/tests/data_integrity/verify_auto_commit_offset_replicates.rs +++ b/core/integration/tests/data_integrity/verify_auto_commit_offset_replicates.rs @@ -65,11 +65,11 @@ async fn run(harness: &TestHarness) { .create_topic( &stream, TOPIC_NAME, - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); diff --git a/core/integration/tests/data_integrity/verify_cluster_replica_data_identical.rs b/core/integration/tests/data_integrity/verify_cluster_replica_data_identical.rs index e242d81b32..1efe145939 100644 --- a/core/integration/tests/data_integrity/verify_cluster_replica_data_identical.rs +++ b/core/integration/tests/data_integrity/verify_cluster_replica_data_identical.rs @@ -76,12 +76,9 @@ const CONVERGENCE_STABLE_POLLS: u32 = 3; // `messages_required_to_save = 1` forces every committed batch to persist to its // segment immediately on every node, so each replica materialises the segment // files while running (the VSR server serves no flush_unsaved_buffer, and -// shutdown-flush would couple the test to drain behaviour). The harness applies -// the `server(...)` config to every cluster node. -#[iggy_harness( - cluster_nodes = 3, - server(partition.messages_required_to_save = 1) -)] +// shutdown-flush would couple the test to drain behaviour). It is a topic +// creation option, so it travels with the topic to every replica. +#[iggy_harness(cluster_nodes = 3)] async fn should_persist_byte_identical_data_across_cluster_replicas(harness: &mut TestHarness) { let client = harness.tcp_root_client().await.unwrap(); client.create_stream(STREAM_NAME).await.unwrap(); @@ -89,11 +86,12 @@ async fn should_persist_byte_identical_data_across_cluster_replicas(harness: &mu .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + messages_required_to_save: Some(1), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); diff --git a/core/integration/tests/data_integrity/verify_consumer_group_partition_assignment.rs b/core/integration/tests/data_integrity/verify_consumer_group_partition_assignment.rs index 8fac77b964..f4f935a827 100644 --- a/core/integration/tests/data_integrity/verify_consumer_group_partition_assignment.rs +++ b/core/integration/tests/data_integrity/verify_consumer_group_partition_assignment.rs @@ -47,6 +47,10 @@ const STREAM_NAME: &str = "cg-partition-test-stream"; const TOPIC_NAME: &str = "cg-partition-test-topic"; const CONSUMER_GROUP_NAME: &str = "cg-partition-test-group"; const PARTITIONS_COUNT: u32 = 3; +/// Bounds [`await_members_count`]. Generous because the slowest path it covers +/// is heartbeat eviction (2s interval x 1.2 threshold) on a loaded machine. +const MEMBERS_CONVERGENCE_TIMEOUT: Duration = Duration::from_secs(15); +const MEMBERS_RETRY_INTERVAL: Duration = Duration::from_millis(100); /// Slices the slab-reuse wait so the surviving consumer can prove liveness /// inside the server's staleness window. Product of the two is the 3s the /// spec waits for the freed slab to become reusable. @@ -102,11 +106,11 @@ async fn should_not_duplicate_partition_assignments_after_stale_client_cleanup( .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - PARTITIONS_COUNT, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(PARTITIONS_COUNT), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); @@ -187,16 +191,9 @@ async fn should_not_duplicate_partition_assignments_after_stale_client_cleanup( // Server heartbeat interval = 2s, threshold = 2s * 1.2 = 2.4s. // Stale clients' heartbeat interval is 1h so they won't ping. // But they DID send one initial ping on connect, so we wait for that to expire. - // Give it 5s to be safe. - sleep(Duration::from_secs(5)).await; - - // 8. Verify ghosts have been evicted - let cg = get_consumer_group(&root_client).await; - assert_eq!( - cg.members_count, 0, - "Expected 0 members after heartbeat eviction of stale clients, got {}. Members: {:?}", - cg.members_count, cg.members - ); + // + // 8. Verify ghosts have been evicted. + await_members_count(&root_client, 0).await; // 9. Now create 3 new clients and join same CG (simulating app restart after kill -9). let client1 = create_tcp_client(&server_addr).await; @@ -758,11 +755,11 @@ async fn should_handle_partition_delete_while_multiple_consumers_polling(harness .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - 6, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(6), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); @@ -895,11 +892,11 @@ async fn should_reach_even_distribution_after_multiple_joins(harness: &TestHarne .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - 6, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(6), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); @@ -1299,11 +1296,11 @@ async fn should_handle_delete_partitions_with_uncommitted_work(harness: &TestHar .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - 6, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(6), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); @@ -1643,11 +1640,11 @@ async fn should_rebalance_after_deleting_partitions(harness: &TestHarness) { .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - 6, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(6), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); @@ -1929,11 +1926,11 @@ async fn should_not_duplicate_after_reconnect_without_heartbeat(harness: &TestHa .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - PARTITIONS_COUNT, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(PARTITIONS_COUNT), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); @@ -2071,11 +2068,11 @@ async fn should_not_duplicate_partition_assignments_after_client_reconnect(harne .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - PARTITIONS_COUNT, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(PARTITIONS_COUNT), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); @@ -2151,10 +2148,8 @@ async fn should_not_duplicate_partition_assignments_after_client_reconnect(harne drop(client1); drop(client2); drop(client3); - sleep(Duration::from_millis(500)).await; - let cg = get_consumer_group(&root_client).await; - assert_eq!(cg.members_count, 0); + await_members_count(&root_client, 0).await; // 6. Restart: 3 new clients join same CG let new_client1 = harness.new_client().await.unwrap(); @@ -2212,6 +2207,31 @@ async fn should_not_duplicate_partition_assignments_after_client_reconnect(harne .unwrap(); } +/// Poll the group until it reports `expected` members, then return it. +/// +/// Member removal is server-side work that a client cannot observe completing: +/// dropping a client sends a FIN, and eviction of a client that never sends one +/// waits on the heartbeat verifier. Sleeping a fixed span and asserting assumes +/// a bound on that work, which does not hold when the machine is running the +/// rest of the suite -- the assert then reads one leftover member and fails. +async fn await_members_count(client: &IggyClient, expected: u32) -> ConsumerGroupDetails { + let deadline = tokio::time::Instant::now() + MEMBERS_CONVERGENCE_TIMEOUT; + loop { + let group = get_consumer_group(client).await; + if group.members_count == expected { + return group; + } + assert!( + tokio::time::Instant::now() < deadline, + "expected {expected} members within {MEMBERS_CONVERGENCE_TIMEOUT:?}, \ + last saw {}. Members: {:?}", + group.members_count, + group.members + ); + sleep(MEMBERS_RETRY_INTERVAL).await; + } +} + async fn get_consumer_group(client: &IggyClient) -> ConsumerGroupDetails { client .get_consumer_group( @@ -2250,11 +2270,11 @@ async fn setup_stream_topic_cg_with_partitions(client: &IggyClient, partitions: .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - partitions, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(partitions), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); diff --git a/core/integration/tests/sdk/consumer_group.rs b/core/integration/tests/sdk/consumer_group.rs index 5ea300c6b9..c6aec31049 100644 --- a/core/integration/tests/sdk/consumer_group.rs +++ b/core/integration/tests/sdk/consumer_group.rs @@ -53,11 +53,11 @@ async fn consumer_group_retries_rejoin_after_failure(harness: &TestHarness) { .create_topic( &stream_id, TOPIC_NAME, - 1, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); diff --git a/core/integration/tests/sdk/consumer_group_membership.rs b/core/integration/tests/sdk/consumer_group_membership.rs index ab07bb3f1f..05342d3b1d 100644 --- a/core/integration/tests/sdk/consumer_group_membership.rs +++ b/core/integration/tests/sdk/consumer_group_membership.rs @@ -67,11 +67,11 @@ async fn given_group_member_holds_no_partitions_when_group_deleted_should_surfac .create_topic( &stream_id, TOPIC_NAME, - 1, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); @@ -189,11 +189,11 @@ async fn given_join_and_leave_failures_when_sent_over_the_wire_should_return_leg .create_topic( &stream_id, TOPIC_NAME, - 1, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); diff --git a/core/integration/tests/sdk/mod.rs b/core/integration/tests/sdk/mod.rs index cdf8dfb4e9..3b934e8c95 100644 --- a/core/integration/tests/sdk/mod.rs +++ b/core/integration/tests/sdk/mod.rs @@ -20,6 +20,7 @@ mod consumer_group_membership; mod consumer_offset; mod hello_world; mod http_refresh; +mod options; mod producer; mod protocol_version; mod raw; diff --git a/core/integration/tests/sdk/options.rs b/core/integration/tests/sdk/options.rs new file mode 100644 index 0000000000..16b46fad53 --- /dev/null +++ b/core/integration/tests/sdk/options.rs @@ -0,0 +1,521 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use iggy::prelude::*; +use iggy_common::{OptionsScope, TOPIC_OPTION_KEYS, topic_option_keys}; +use integration::iggy_harness; +use std::collections::BTreeMap; +use std::str::FromStr; + +#[iggy_harness] +async fn given_topic_scope_when_describing_options_should_list_the_catalog(harness: &TestHarness) { + let client = harness.root_client().await.unwrap(); + let specs = client.describe_options(OptionsScope::Topic).await.unwrap(); + let mut served: Vec<&str> = specs.iter().map(|spec| spec.key.as_str()).collect(); + served.sort_unstable(); + let mut accepted: Vec<&str> = TOPIC_OPTION_KEYS.to_vec(); + accepted.sort_unstable(); + // Exact equality, not per-key contains: the catalog is the only way a + // client discovers a key, and create rejects anything outside it, so a key + // accepted but unlisted is undiscoverable and one listed but unaccepted is + // a lie. + assert_eq!(served, accepted, "catalog must match the accepted key set"); + // `partitions_count` is deliberately absent: it is a fixed field of the + // CreateTopic command, not a topic setting. + assert!(!served.contains(&"partitions_count")); +} + +#[iggy_harness] +async fn given_runtime_knobs_when_creating_topic_should_persist_them_per_topic( + harness: &TestHarness, +) { + let client = harness.root_client().await.unwrap(); + client.create_stream("knob-stream").await.unwrap(); + let stream = Identifier::named("knob-stream").unwrap(); + + // Zero is rejected: a flush threshold of zero never trips. + let zero_threshold = client + .create_topic( + &stream, + "knob-zero", + &TopicCreateOptions { + partitions_count: Some(1), + messages_required_to_save: Some(0), + ..TopicCreateOptions::default() + }, + ) + .await; + assert!(zero_threshold.is_err(), "zero flush threshold must reject"); + + client + .create_topic( + &stream, + "knob-topic", + &TopicCreateOptions { + partitions_count: Some(1), + enforce_fsync: Some(true), + messages_required_to_save: Some(7), + size_of_messages_required_to_save: Some(IggyByteSize::from(4096u64)), + segment_size: Some(IggyByteSize::from(1024 * 1024u64)), + ..TopicCreateOptions::default() + }, + ) + .await + .unwrap(); + + let details = client + .get_topic(&stream, &Identifier::named("knob-topic").unwrap()) + .await + .unwrap() + .expect("topic exists"); + for (key, expected_explicit) in [ + (topic_option_keys::ENFORCE_FSYNC, true), + (topic_option_keys::MESSAGES_REQUIRED_TO_SAVE, true), + (topic_option_keys::SIZE_OF_MESSAGES_REQUIRED_TO_SAVE, true), + // Never sent, so admission derived it from the built-in default. + (topic_option_keys::PREALLOCATE_SEGMENTS, false), + ] { + let option = details + .options + .get(&HeaderKey::from_str(key).unwrap()) + .unwrap_or_else(|| panic!("{key} echoes back")); + assert_eq!(option.explicit, expected_explicit, "{key} provenance"); + } + + // Messages still flow with the per-topic thresholds installed. + let mut messages = vec![ + IggyMessage::builder() + .payload("knob".into()) + .build() + .unwrap(), + ]; + let topic = Identifier::named("knob-topic").unwrap(); + client + .send_messages( + &stream, + &topic, + &Partitioning::partition_id(0), + &mut messages, + ) + .await + .unwrap(); + let polled = client + .poll_messages( + &stream, + &topic, + Some(0), + &Consumer::new(Identifier::numeric(1).unwrap()), + &PollingStrategy::offset(0), + 10, + false, + ) + .await + .unwrap(); + assert_eq!(polled.messages.len(), 1); +} + +#[iggy_harness] +async fn given_stream_scope_when_describing_options_should_be_empty(harness: &TestHarness) { + let client = harness.root_client().await.unwrap(); + let specs = client.describe_options(OptionsScope::Stream).await.unwrap(); + assert!(specs.is_empty(), "streams have no catalog keys yet"); +} + +#[iggy_harness] +async fn given_explicit_segment_size_when_creating_topic_should_roll_at_it(harness: &TestHarness) { + let client = harness.root_client().await.unwrap(); + client.create_stream("seg-stream").await.unwrap(); + let stream = Identifier::named("seg-stream").unwrap(); + + // Below the floor: denied with the key name before consensus. + let too_small = client + .create_topic( + &stream, + "seg-too-small", + &TopicCreateOptions { + partitions_count: Some(1), + segment_size: Some(IggyByteSize::from(1024u64)), + ..TopicCreateOptions::default() + }, + ) + .await; + assert!(too_small.is_err(), "1 KiB segment must be rejected"); + + client + .create_topic( + &stream, + "seg-topic", + &TopicCreateOptions { + partitions_count: Some(1), + segment_size: Some(IggyByteSize::from(1024 * 1024u64)), + ..TopicCreateOptions::default() + }, + ) + .await + .unwrap(); + + let topic = Identifier::named("seg-topic").unwrap(); + let details = client + .get_topic(&stream, &topic) + .await + .unwrap() + .expect("topic exists"); + let segment_key = HeaderKey::from_str(topic_option_keys::SEGMENT_SIZE).unwrap(); + let segment = details + .options + .get(&segment_key) + .expect("explicit segment size echoes back"); + assert!(segment.explicit); + assert_eq!( + u64::from_le_bytes(segment.value.as_bytes().try_into().unwrap()), + 1024 * 1024 + ); + + // Push ~3 MiB through a 1 MiB segment: the partition must roll. + let payload = vec![b'x'; 512 * 1024]; + for _ in 0..6 { + let mut messages = vec![ + IggyMessage::builder() + .payload(payload.clone().into()) + .build() + .unwrap(), + ]; + client + .send_messages( + &stream, + &topic, + &Partitioning::partition_id(0), + &mut messages, + ) + .await + .unwrap(); + } + let details = client + .get_topic(&stream, &topic) + .await + .unwrap() + .expect("topic exists"); + assert!( + details.partitions[0].segments_count > 1, + "3 MiB through a 1 MiB segment must roll at least once, got {} segments", + details.partitions[0].segments_count + ); +} + +#[iggy_harness] +async fn given_explicit_expiry_when_creating_topic_should_echo_provenance(harness: &TestHarness) { + let client = harness.root_client().await.unwrap(); + client.create_stream("options-stream").await.unwrap(); + client + .create_topic( + &Identifier::named("options-stream").unwrap(), + "options-topic", + &TopicCreateOptions { + partitions_count: Some(2), + message_expiry: Some(IggyExpiry::ExpireDuration("5m".parse().unwrap())), + segment_size: Some(IggyByteSize::from(1024 * 1024u64)), + ..TopicCreateOptions::default() + }, + ) + .await + .unwrap(); + + let details = client + .get_topic( + &Identifier::named("options-stream").unwrap(), + &Identifier::named("options-topic").unwrap(), + ) + .await + .unwrap() + .expect("topic exists"); + assert_eq!(details.partitions_count, 2); + + let expiry_key = HeaderKey::from_str(topic_option_keys::MESSAGE_EXPIRY).unwrap(); + let expiry = details + .options + .get(&expiry_key) + .expect("explicit expiry echoes back"); + assert!(expiry.explicit, "client-sent key keeps its provenance"); + + let size_key = HeaderKey::from_str(topic_option_keys::MAX_TOPIC_SIZE).unwrap(); + let size = details + .options + .get(&size_key) + .expect("derived size echoes back"); + assert!(!size.explicit, "default-filled key is marked derived"); + + // partitions_count never reaches the options map: it is a command field. + let partitions_key = HeaderKey::from_str("partitions_count").unwrap(); + assert!(!details.options.contains_key(&partitions_key)); +} + +#[iggy_harness] +async fn given_update_options_when_updating_topic_should_patch_not_replace(harness: &TestHarness) { + let client = harness.root_client().await.unwrap(); + client.create_stream("update-stream").await.unwrap(); + let stream = Identifier::named("update-stream").unwrap(); + let topic = Identifier::named("update-topic").unwrap(); + + client + .create_topic( + &stream, + "update-topic", + &TopicCreateOptions { + partitions_count: Some(1), + segment_size: Some(IggyByteSize::from(1024 * 1024u64)), + enforce_fsync: Some(true), + ..TopicCreateOptions::default() + }, + ) + .await + .unwrap(); + + // A create-time knob is refused by name: nothing re-pushes it to the + // partitions, so storing it would read as applied and not be. + let create_only = client + .update_topic( + &stream, + &topic, + "update-topic", + &TopicUpdateOptions { + raw: BTreeMap::from([( + topic_option_keys::SEGMENT_SIZE.to_string(), + "2MiB".to_string(), + )]), + ..TopicUpdateOptions::default() + }, + ) + .await; + assert!(create_only.is_err(), "segment_size must be create-only"); + + client + .update_topic( + &stream, + &topic, + "update-topic", + &TopicUpdateOptions::default(), + ) + .await + .unwrap(); + + let details = client + .get_topic(&stream, &topic) + .await + .unwrap() + .expect("topic exists"); + // The keys the update did not mention keep the values create resolved. + for (key, expected_explicit) in [ + (topic_option_keys::SEGMENT_SIZE, true), + (topic_option_keys::ENFORCE_FSYNC, true), + (topic_option_keys::PREALLOCATE_SEGMENTS, false), + ] { + let option = details + .options + .get(&HeaderKey::from_str(key).unwrap()) + .unwrap_or_else(|| panic!("{key} survives the update")); + assert_eq!(option.explicit, expected_explicit, "{key} provenance"); + } + let segment_key = HeaderKey::from_str(topic_option_keys::SEGMENT_SIZE).unwrap(); + let segment = details.options.get(&segment_key).unwrap(); + assert_eq!( + u64::from_le_bytes(segment.value.as_bytes().try_into().unwrap()), + 1024 * 1024, + "an update must not reset a key it never sent" + ); +} + +#[iggy_harness] +async fn given_unknown_key_when_updating_stream_or_user_should_reject(harness: &TestHarness) { + let client = harness.root_client().await.unwrap(); + client.create_stream("opt-stream").await.unwrap(); + + // Streams and users have no catalog keys yet, so the update blocks exist + // as the extension point and reject every key by name rather than storing + // one nothing reads. + let stream_rejected = client + .update_stream( + &Identifier::named("opt-stream").unwrap(), + "opt-stream", + &StreamUpdateOptions { + raw: BTreeMap::from([("future_key".to_string(), "1".to_string())]), + }, + ) + .await; + assert!(stream_rejected.is_err(), "unknown stream key must reject"); + + // An empty block still succeeds: that is the plain rename path. + client + .update_stream( + &Identifier::named("opt-stream").unwrap(), + "opt-stream-renamed", + &StreamUpdateOptions::default(), + ) + .await + .unwrap(); + + client + .create_user("opt-user", "secret123", UserStatus::Active, None) + .await + .unwrap(); + let user = Identifier::named("opt-user").unwrap(); + let user_rejected = client + .update_user( + &user, + Some("opt-user"), + None, + &UserUpdateOptions { + raw: BTreeMap::from([("future_key".to_string(), "1".to_string())]), + }, + ) + .await; + assert!(user_rejected.is_err(), "unknown user key must reject"); + + client + .update_user( + &user, + Some("opt-user-renamed"), + None, + &UserUpdateOptions::default(), + ) + .await + .unwrap(); +} + +#[iggy_harness] +async fn given_sentinel_zeros_when_creating_topic_should_report_resolved_defaults( + harness: &TestHarness, +) { + let client = harness.root_client().await.unwrap(); + client.create_stream("sentinel-stream").await.unwrap(); + let stream = Identifier::named("sentinel-stream").unwrap(); + + // 0 means "resolve the server default", not "expire immediately" / "no + // space". Admission normalizes it away, so the stored map must report the + // resolved value as derived -- never a literal 0 marked explicit, which is + // what a client would then read back as the effective configuration. + client + .create_topic( + &stream, + "sentinel-topic", + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::from(0u64)), + max_topic_size: Some(MaxTopicSize::from(0u64)), + segment_size: Some(IggyByteSize::from(0u64)), + ..TopicCreateOptions::default() + }, + ) + .await + .unwrap(); + + let details = client + .get_topic(&stream, &Identifier::named("sentinel-topic").unwrap()) + .await + .unwrap() + .expect("topic exists"); + + for key in [ + topic_option_keys::MESSAGE_EXPIRY, + topic_option_keys::MAX_TOPIC_SIZE, + topic_option_keys::SEGMENT_SIZE, + ] { + let option = details + .options + .get(&HeaderKey::from_str(key).unwrap()) + .unwrap_or_else(|| panic!("{key} resolves to a default rather than vanishing")); + assert!( + !option.explicit, + "{key} sentinel must resolve to a derived default, not persist as explicit" + ); + assert_ne!( + option.value.as_bytes(), + 0u64.to_le_bytes(), + "{key} must report the resolved value, not the sentinel" + ); + } +} + +#[iggy_harness] +async fn given_rename_only_when_updating_topic_should_leave_settings_alone(harness: &TestHarness) { + let client = harness.root_client().await.unwrap(); + client.create_stream("patch-stream").await.unwrap(); + let stream = Identifier::named("patch-stream").unwrap(); + + client + .create_topic( + &stream, + "patch-topic", + &TopicCreateOptions { + partitions_count: Some(1), + compression_algorithm: Some(CompressionAlgorithm::Gzip), + message_expiry: Some(IggyExpiry::from(5_000_000u64)), + ..TopicCreateOptions::default() + }, + ) + .await + .unwrap(); + let topic = Identifier::named("patch-topic").unwrap(); + + // Settings live only in the options block, so an update that carries none + // of them is a pure rename. While they were fixed fields of the command + // this was impossible: every update rewrote all three whether or not the + // caller meant to. + client + .update_topic( + &stream, + &topic, + "patch-renamed", + &TopicUpdateOptions::default(), + ) + .await + .unwrap(); + + let details = client + .get_topic(&stream, &Identifier::named("patch-renamed").unwrap()) + .await + .unwrap() + .expect("topic exists"); + assert_eq!(details.name, "patch-renamed"); + assert_eq!(details.compression_algorithm, CompressionAlgorithm::Gzip); + assert_eq!(details.message_expiry, IggyExpiry::from(5_000_000u64)); + + // Sending one key changes that key and leaves the other alone. + client + .update_topic( + &stream, + &Identifier::named("patch-renamed").unwrap(), + "patch-renamed", + &TopicUpdateOptions { + message_expiry: Some(IggyExpiry::from(9_000_000u64)), + ..TopicUpdateOptions::default() + }, + ) + .await + .unwrap(); + + let details = client + .get_topic(&stream, &Identifier::named("patch-renamed").unwrap()) + .await + .unwrap() + .expect("topic exists"); + assert_eq!(details.message_expiry, IggyExpiry::from(9_000_000u64)); + assert_eq!( + details.compression_algorithm, + CompressionAlgorithm::Gzip, + "a key the update did not carry keeps its value" + ); +} diff --git a/core/integration/tests/sdk/producer/mod.rs b/core/integration/tests/sdk/producer/mod.rs index 3bc95aaa8c..b959eb004e 100644 --- a/core/integration/tests/sdk/producer/mod.rs +++ b/core/integration/tests/sdk/producer/mod.rs @@ -39,11 +39,11 @@ async fn init_system(client: &IggyClient) { .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - PARTITIONS_COUNT, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(PARTITIONS_COUNT), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); diff --git a/core/integration/tests/sdk/send_confirmation.rs b/core/integration/tests/sdk/send_confirmation.rs index 61d1c0e7e6..647c3e16e3 100644 --- a/core/integration/tests/sdk/send_confirmation.rs +++ b/core/integration/tests/sdk/send_confirmation.rs @@ -57,11 +57,11 @@ async fn create_stream_and_topic(client: &IggyClient, partitions_count: u32) -> .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - partitions_count, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(partitions_count), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .expect("create_topic"); diff --git a/core/integration/tests/server/flush_vsr.rs b/core/integration/tests/server/flush_vsr.rs index 56ff451c32..a40e0b5e11 100644 --- a/core/integration/tests/server/flush_vsr.rs +++ b/core/integration/tests/server/flush_vsr.rs @@ -47,11 +47,11 @@ async fn given_valid_partition_when_flushing_should_reject_feature_unavailable( .create_topic( &stream_id, "flush-topic", - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .expect("create topic"); diff --git a/core/integration/tests/server/general.rs b/core/integration/tests/server/general.rs index d21d118ab4..460c0efdb3 100644 --- a/core/integration/tests/server/general.rs +++ b/core/integration/tests/server/general.rs @@ -25,7 +25,6 @@ use integration::iggy_harness; #[iggy_harness( test_client_transport = [Tcp, Http, Quic, WebSocket], server( - segment.size = "1MiB", tcp.socket.override_defaults = true, tcp.socket.nodelay = true, quic.max_idle_timeout = "500s", diff --git a/core/integration/tests/server/http_rbac.rs b/core/integration/tests/server/http_rbac.rs index 6a113d03c9..6b949905ee 100644 --- a/core/integration/tests/server/http_rbac.rs +++ b/core/integration/tests/server/http_rbac.rs @@ -111,8 +111,8 @@ impl ClientExt for HttpClient { compression_algorithm: CompressionAlgorithm::None, message_expiry: IggyExpiry::NeverExpire, max_topic_size: MaxTopicSize::ServerDefault, - replication_factor: None, name: topic.to_string(), + options: Default::default(), }; let response = self .post_json( diff --git a/core/integration/tests/server/http_vsr.rs b/core/integration/tests/server/http_vsr.rs index ca4acf4c37..2ae84bac2f 100644 --- a/core/integration/tests/server/http_vsr.rs +++ b/core/integration/tests/server/http_vsr.rs @@ -80,6 +80,14 @@ trait HttpSessionExt { offset: u64, count: u32, ) -> PolledMessages; + async fn try_poll( + &self, + stream: &str, + topic: &str, + partition_id: u32, + offset: u64, + count: u32, + ) -> Option; async fn create_user(&self, username: &str, password: &str); } @@ -107,8 +115,8 @@ impl HttpSessionExt for HttpClient { compression_algorithm: CompressionAlgorithm::None, message_expiry: IggyExpiry::NeverExpire, max_topic_size: MaxTopicSize::ServerDefault, - replication_factor: None, name: topic.to_string(), + options: Default::default(), }; let response = self .client @@ -118,10 +126,17 @@ impl HttpSessionExt for HttpClient { .send() .await .expect("create topic request"); + // Body included on failure: a 500 from a serialization fault names + // itself only there, and status alone sent this down a long detour. + let status = response.status(); + let body = if status.is_success() { + String::new() + } else { + response.text().await.unwrap_or_default() + }; assert!( - response.status().is_success(), - "create topic failed: {}", - response.status() + status.is_success(), + "create topic failed: {status} body={body}" ); } @@ -168,6 +183,26 @@ impl HttpSessionExt for HttpClient { offset: u64, count: u32, ) -> PolledMessages { + self.try_poll(stream, topic, partition_id, offset, count) + .await + .expect("poll must answer 200") + } + + /// Poll without demanding that the topic already be visible. + /// + /// `None` means 404: a create commits on the metadata primary and then + /// propagates, so a poll that lands on a shard which has not caught up yet + /// answers 404 rather than an empty batch. Callers already looping for + /// eventual visibility want to keep waiting on that, so only a + /// non-OK-non-404 status is a hard failure. + async fn try_poll( + &self, + stream: &str, + topic: &str, + partition_id: u32, + offset: u64, + count: u32, + ) -> Option { let path = format!( "/streams/{stream}/topics/{topic}/messages\ ?consumer_id={CONSUMER_ID}&partition_id={partition_id}\ @@ -180,8 +215,11 @@ impl HttpSessionExt for HttpClient { .send() .await .expect("poll request"); + if response.status() == StatusCode::NOT_FOUND { + return None; + } assert_eq!(response.status(), StatusCode::OK, "poll must answer 200"); - response.json().await.expect("decode PolledMessages") + Some(response.json().await.expect("decode PolledMessages")) } /// Create an ungranted user (no permissions -> the permissioner holds no @@ -671,13 +709,16 @@ async fn given_ack_none_when_producing_should_return_202_and_commit(harness: &Te assert_eq!(durability(&response), DURABILITY_NONE); // The commit still happens, just asynchronously: poll bounded until the - // message becomes visible. + // message becomes visible. A 404 counts as not-yet-visible for the same + // reason an empty batch does -- nothing here waited for the create to + // reach the shard that serves the poll. let deadline = Instant::now() + ASYNC_COMMIT_TIMEOUT; loop { - let polled = http - .poll("http-ack-none", "fire", PARTITION_ID, 0, 10) - .await; - if !polled.messages.is_empty() { + if let Some(polled) = http + .try_poll("http-ack-none", "fire", PARTITION_ID, 0, 10) + .await + && !polled.messages.is_empty() + { assert_eq!(polled.messages.len(), 1, "exactly one message was produced"); assert_eq!( polled.messages[0].payload, diff --git a/core/integration/tests/server/message_cleanup.rs b/core/integration/tests/server/message_cleanup.rs index eedca5d335..fad8ab490b 100644 --- a/core/integration/tests/server/message_cleanup.rs +++ b/core/integration/tests/server/message_cleanup.rs @@ -82,8 +82,9 @@ async fn run_cleanup_scenario(scenario: CleanupScenarioFn) { let mut harness = TestHarness::builder() .server( TestServerConfig::builder() + // Segment size, flush threshold and fsync are topic creation + // options now; `message_cleanup_scenario` sets them per topic. .extra_envs(HashMap::from([ - ("IGGY_SYSTEM_SEGMENT_SIZE".to_string(), "10KiB".to_string()), ( "IGGY_DATA_MAINTENANCE_MESSAGES_CLEANER_ENABLED".to_string(), "true".to_string(), @@ -92,14 +93,6 @@ async fn run_cleanup_scenario(scenario: CleanupScenarioFn) { "IGGY_DATA_MAINTENANCE_MESSAGES_INTERVAL".to_string(), "100ms".to_string(), ), - ( - "IGGY_SYSTEM_PARTITION_MESSAGES_REQUIRED_TO_SAVE".to_string(), - "1".to_string(), - ), - ( - "IGGY_SYSTEM_PARTITION_ENFORCE_FSYNC".to_string(), - "true".to_string(), - ), ])) .build(), ) diff --git a/core/integration/tests/server/message_retrieval.rs b/core/integration/tests/server/message_retrieval.rs index ba4f48c2c7..2546dad970 100644 --- a/core/integration/tests/server/message_retrieval.rs +++ b/core/integration/tests/server/message_retrieval.rs @@ -16,21 +16,27 @@ // under the License. use crate::server::scenarios::{offset_scenario, timestamp_scenario}; +use iggy::prelude::*; use integration::harness::{TestHarness, TestServerConfig}; use serial_test::parallel; use std::collections::HashMap; use test_case::test_matrix; -fn segment_size_512b() -> &'static str { - "512B" +// A topic's segment size must be a 512-byte multiple in +// [`iggy_common::MIN_TOPIC_SEGMENT_SIZE`]..=1 GiB, so the old 512 B and 1 KiB +// cells (which made every batch its own segment) collapse onto the 1 MiB floor. +// The spread that remains still separates "rolls on almost every large batch" +// from "rolls a handful of times over the whole run". +fn segment_size_1mib() -> u64 { + 1024 * 1024 } -fn segment_size_1kb() -> &'static str { - "1KiB" +fn segment_size_2mib() -> u64 { + 2 * 1024 * 1024 } -fn segment_size_10mb() -> &'static str { - "10MiB" +fn segment_size_10mib() -> u64 { + 10 * 1024 * 1024 } fn cache_none() -> &'static str { @@ -45,40 +51,42 @@ fn cache_open_segment() -> &'static str { "open_segment" } -fn msgs_req_32() -> &'static str { - "32" +fn msgs_req_32() -> u32 { + 32 } -fn msgs_req_64() -> &'static str { - "64" +fn msgs_req_64() -> u32 { + 64 } -fn msgs_req_1024() -> &'static str { - "1024" +fn msgs_req_1024() -> u32 { + 1024 } -fn msgs_req_9984() -> &'static str { - "9984" +fn msgs_req_9984() -> u32 { + 9984 } -fn build_server_config( - segment_size: &str, - cache_indexes: &str, - messages_required_to_save: &str, -) -> TestServerConfig { +/// The two axes that used to be `[system.segment] size` and +/// `[system.partition] messages_required_to_save` are topic creation options +/// now, so they travel with the topic the scenario creates rather than with +/// the server. +fn topic_options(segment_size: u64, messages_required_to_save: u32) -> TopicCreateOptions { + TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + segment_size: Some(IggyByteSize::from(segment_size)), + messages_required_to_save: Some(messages_required_to_save), + ..TopicCreateOptions::default() + } +} + +fn build_server_config(cache_indexes: &str) -> TestServerConfig { let mut extra_envs = HashMap::new(); - extra_envs.insert( - "IGGY_SYSTEM_SEGMENT_SIZE".to_string(), - segment_size.to_string(), - ); extra_envs.insert( "IGGY_SYSTEM_SEGMENT_CACHE_INDEXES".to_string(), cache_indexes.to_string(), ); - extra_envs.insert( - "IGGY_SYSTEM_PARTITION_MESSAGES_REQUIRED_TO_SAVE".to_string(), - messages_required_to_save.to_string(), - ); extra_envs.insert( "IGGY_TCP_SOCKET_OVERRIDE_DEFAULTS".to_string(), "true".to_string(), @@ -92,56 +100,56 @@ fn build_server_config( /// the wide permutation set cheap under a parallel nextest run, where an /// oversubscribed multi-node cluster stalls past the liveness window and /// elects a new primary mid-scenario, killing the client session. -fn build_harness( - segment_size: &str, - cache_indexes: &str, - messages_required_to_save: &str, -) -> TestHarness { +fn build_harness(cache_indexes: &str) -> TestHarness { TestHarness::builder() - .server(build_server_config( - segment_size, - cache_indexes, - messages_required_to_save, - )) + .server(build_server_config(cache_indexes)) .cluster_nodes(1) .build() .unwrap() } #[test_matrix( - [segment_size_512b(), segment_size_1kb(), segment_size_10mb()], + [segment_size_1mib(), segment_size_2mib(), segment_size_10mib()], [cache_none(), cache_all(), cache_open_segment()], [msgs_req_32(), msgs_req_64(), msgs_req_1024(), msgs_req_9984()] )] #[tokio::test] #[parallel] async fn get_by_offset_scenario( - segment_size: &str, + segment_size: u64, cache_indexes: &str, - messages_required_to_save: &str, + messages_required_to_save: u32, ) { - let mut harness = build_harness(segment_size, cache_indexes, messages_required_to_save); + let mut harness = build_harness(cache_indexes); harness.start().await.unwrap(); - offset_scenario::run(&harness).await; + offset_scenario::run( + &harness, + &topic_options(segment_size, messages_required_to_save), + ) + .await; } #[test_matrix( - [segment_size_512b(), segment_size_1kb(), segment_size_10mb()], + [segment_size_1mib(), segment_size_2mib(), segment_size_10mib()], [cache_none(), cache_all(), cache_open_segment()], [msgs_req_32(), msgs_req_64(), msgs_req_1024(), msgs_req_9984()] )] #[tokio::test] #[parallel] async fn get_by_timestamp_scenario( - segment_size: &str, + segment_size: u64, cache_indexes: &str, - messages_required_to_save: &str, + messages_required_to_save: u32, ) { - let mut harness = build_harness(segment_size, cache_indexes, messages_required_to_save); + let mut harness = build_harness(cache_indexes); harness.start().await.unwrap(); - timestamp_scenario::run(&harness).await; + timestamp_scenario::run( + &harness, + &topic_options(segment_size, messages_required_to_save), + ) + .await; } diff --git a/core/integration/tests/server/partition_view_durability_vsr.rs b/core/integration/tests/server/partition_view_durability_vsr.rs index 0ea467e5ae..9d266e447f 100644 --- a/core/integration/tests/server/partition_view_durability_vsr.rs +++ b/core/integration/tests/server/partition_view_durability_vsr.rs @@ -77,11 +77,11 @@ async fn given_advanced_partition_view_when_survivor_restarts_should_recover_vie .create_topic( &stream_id, TOPIC_NAME, - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .expect("create topic with one partition"); diff --git a/core/integration/tests/server/poll_semantics_vsr.rs b/core/integration/tests/server/poll_semantics_vsr.rs index 9d211e9a6a..a2eb5e7247 100644 --- a/core/integration/tests/server/poll_semantics_vsr.rs +++ b/core/integration/tests/server/poll_semantics_vsr.rs @@ -45,11 +45,11 @@ async fn given_missing_partition_when_polling_should_reject_partition_not_found( .create_topic( &stream_id, "poll-topic", - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .expect("create topic"); @@ -173,11 +173,11 @@ async fn given_missing_partition_when_getting_consumer_offset_should_reject_part .create_topic( &stream_id, "offset-topic", - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .expect("create topic"); @@ -222,11 +222,11 @@ async fn given_message_at_polled_timestamp_when_polling_should_include_it(harnes .create_topic( &stream_id, "ts-topic", - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .expect("create topic"); diff --git a/core/integration/tests/server/purge_delete.rs b/core/integration/tests/server/purge_delete.rs index c597b9ed52..0d503ce096 100644 --- a/core/integration/tests/server/purge_delete.rs +++ b/core/integration/tests/server/purge_delete.rs @@ -24,10 +24,7 @@ use test_case::test_matrix; #[iggy_harness( cluster_nodes = 1, server( - segment.size = "5KiB", segment.cache_indexes = ["all", "none", "open_segment"], - partition.messages_required_to_save = "1", - partition.enforce_fsync = "true", ) )] #[test_matrix([restart_off(), restart_on()])] @@ -41,10 +38,7 @@ async fn should_delete_segments_and_validate_filesystem( #[iggy_harness( cluster_nodes = 1, server( - segment.size = "5KiB", segment.cache_indexes = ["all", "none", "open_segment"], - partition.messages_required_to_save = "1", - partition.enforce_fsync = "true", ) )] #[test_matrix([restart_off(), restart_on()])] @@ -53,10 +47,7 @@ async fn should_delete_segments_without_consumers(harness: &mut TestHarness, res } #[iggy_harness(server( - segment.size = "5KiB", segment.cache_indexes = ["all", "none", "open_segment"], - partition.messages_required_to_save = "1", - partition.enforce_fsync = "true", ))] async fn should_delete_segments_with_consumer_group_barrier(harness: &TestHarness) { let client = harness.tcp_root_client().await.unwrap(); @@ -68,10 +59,7 @@ async fn should_delete_segments_with_consumer_group_barrier(harness: &TestHarnes #[iggy_harness( cluster_nodes = 1, server( - segment.size = "5KiB", segment.cache_indexes = ["all", "none", "open_segment"], - partition.messages_required_to_save = "1", - partition.enforce_fsync = "true", ) )] #[test_matrix([restart_off(), restart_on()])] @@ -85,13 +73,10 @@ async fn should_block_deletion_until_all_consumers_pass_segment( #[iggy_harness( cluster_nodes = 1, server( - segment.size = "5KiB", segment.cache_indexes = ["all", "none", "open_segment"], - partition.messages_required_to_save = "1", - partition.enforce_fsync = "true", ) )] -// The scenario asserts the exact [0, 7, 14, 21] layout only on the legacy path; +// The scenario asserts the exact [0, 5, 10, 15, 20, 25] layout only on the legacy path; // under vsr it verifies the framing-agnostic purge outcome (offsets cleared, // files deleted, partition reset to a single segment at offset 0). #[test_matrix([restart_off(), restart_on()])] diff --git a/core/integration/tests/server/purge_vsr.rs b/core/integration/tests/server/purge_vsr.rs index 6a5eac2088..152bd63bf3 100644 --- a/core/integration/tests/server/purge_vsr.rs +++ b/core/integration/tests/server/purge_vsr.rs @@ -30,23 +30,16 @@ use integration::iggy_harness; // route polls to whichever node leads after the restart. Default fsync // config on purpose: the restart is graceful, so segment bytes survive // without fsync, and `purge.gen` is unconditionally synced by the purge. -#[iggy_harness( - cluster_nodes = 1, - server(partition.messages_required_to_save = "1") -)] +// The flush thresholds each scenario needs are topic creation options, set +// inside the scenario itself. +#[iggy_harness(cluster_nodes = 1)] async fn given_post_purge_messages_when_server_restarts_should_retain_them( harness: &mut TestHarness, ) { purge_delete_scenario::run_purge_survives_restart(harness).await; } -// The huge threshold keeps every batch journal-resident: nothing is ever -// flushed before the purge, so the purged bytes exist ONLY as consensus -// history the shutdown flush re-walks. -#[iggy_harness( - cluster_nodes = 1, - server(partition.messages_required_to_save = "10000") -)] +#[iggy_harness(cluster_nodes = 1)] async fn given_journal_resident_messages_when_purged_should_not_resurface( harness: &mut TestHarness, ) { @@ -75,11 +68,11 @@ async fn given_purged_topic_when_getting_topic_immediately_should_report_zero_st .create_topic( &stream_id, TOPIC, - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .expect("create topic"); diff --git a/core/integration/tests/server/scenarios/authentication_scenario.rs b/core/integration/tests/server/scenarios/authentication_scenario.rs index c5b3ddf3ff..2736ea56ae 100644 --- a/core/integration/tests/server/scenarios/authentication_scenario.rs +++ b/core/integration/tests/server/scenarios/authentication_scenario.rs @@ -166,6 +166,10 @@ async fn test_all_commands_require_auth(client: &IggyClient) { GET_CLIENTS_CODE => client.get_clients().await.map(|_| ()), GET_CLUSTER_METADATA_CODE => client.get_cluster_metadata().await.map(|_| ()), + DESCRIBE_OPTIONS_CODE => client + .describe_options(OptionsScope::Topic) + .await + .map(|_| ()), // Users GET_USER_CODE => client.get_user(&ctx.user_id).await.map(|_| ()), @@ -175,7 +179,11 @@ async fn test_all_commands_require_auth(client: &IggyClient) { .await .map(|_| ()), DELETE_USER_CODE => client.delete_user(&ctx.user_id).await, - UPDATE_USER_CODE => client.update_user(&ctx.user_id, Some("x"), None).await, + UPDATE_USER_CODE => { + client + .update_user(&ctx.user_id, Some("x"), None, &UserUpdateOptions::default()) + .await + } UPDATE_PERMISSIONS_CODE => client.update_permissions(&ctx.user_id, None).await, CHANGE_PASSWORD_CODE => client.change_password(&ctx.user_id, "old", "new").await, @@ -194,7 +202,11 @@ async fn test_all_commands_require_auth(client: &IggyClient) { GET_STREAMS_CODE => client.get_streams().await.map(|_| ()), CREATE_STREAM_CODE => client.create_stream("x").await.map(|_| ()), DELETE_STREAM_CODE => client.delete_stream(&ctx.stream_id).await, - UPDATE_STREAM_CODE => client.update_stream(&ctx.stream_id, "x").await, + UPDATE_STREAM_CODE => { + client + .update_stream(&ctx.stream_id, "x", &StreamUpdateOptions::default()) + .await + } PURGE_STREAM_CODE => client.purge_stream(&ctx.stream_id).await, // Topics @@ -207,11 +219,11 @@ async fn test_all_commands_require_auth(client: &IggyClient) { .create_topic( &ctx.stream_id, "x", - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .map(|_| ()), @@ -222,10 +234,7 @@ async fn test_all_commands_require_auth(client: &IggyClient) { &ctx.stream_id, &ctx.topic_id, "x", - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicUpdateOptions::default(), ) .await } @@ -400,11 +409,11 @@ async fn setup_test_resources(client: &IggyClient) { .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .expect("create topic"); diff --git a/core/integration/tests/server/scenarios/concurrent_produce_consume_scenario.rs b/core/integration/tests/server/scenarios/concurrent_produce_consume_scenario.rs index 97715afd19..bbf8521cf7 100644 --- a/core/integration/tests/server/scenarios/concurrent_produce_consume_scenario.rs +++ b/core/integration/tests/server/scenarios/concurrent_produce_consume_scenario.rs @@ -49,12 +49,7 @@ const MAX_TEST_DURATION: Duration = Duration::from_secs(60); /// `messages_required_to_save = "1"` forces every batch to trigger an inline /// journal commit. The next send arrives before async persist completes, creating /// State C (journal non-empty + in-flight non-empty simultaneously). -#[iggy_harness(server( - partition.messages_required_to_save = "1", - partition.enforce_fsync = false, - message_saver.enabled = true, - message_saver.interval = "100ms" -))] +#[iggy_harness(server(message_saver.enabled = true, message_saver.interval = "100ms"))] async fn concurrent_produce_consume_no_offset_skip(harness: &TestHarness) { let stream_id = Identifier::named(STREAM_NAME).unwrap(); let topic_id = Identifier::named(TOPIC_NAME).unwrap(); @@ -65,11 +60,16 @@ async fn concurrent_produce_consume_no_offset_skip(harness: &TestHarness) { .create_topic( &stream_id, TOPIC_NAME, - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + // A flush per message forces an inline journal commit on every + // batch; the next send then arrives while the previous is still + // persisting, which is State C in the header above. + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + messages_required_to_save: Some(1), + enforce_fsync: Some(false), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); diff --git a/core/integration/tests/server/scenarios/concurrent_scenario.rs b/core/integration/tests/server/scenarios/concurrent_scenario.rs index 0c606749a3..5103fcabc3 100644 --- a/core/integration/tests/server/scenarios/concurrent_scenario.rs +++ b/core/integration/tests/server/scenarios/concurrent_scenario.rs @@ -85,11 +85,11 @@ pub async fn run( .create_topic( &stream_id, TEST_TOPIC_NAME, - PARTITIONS_COUNT, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(PARTITIONS_COUNT), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); @@ -274,11 +274,11 @@ async fn execute_topics_hot( .create_topic( &stream_id, &topic_name, - PARTITIONS_COUNT, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(PARTITIONS_COUNT), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .map(|_| ()) @@ -314,11 +314,11 @@ async fn execute_topics_cold( .create_topic( &stream_id, DUPLICATE_TOPIC, - PARTITIONS_COUNT, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(PARTITIONS_COUNT), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .map(|_| ()) diff --git a/core/integration/tests/server/scenarios/consumer_group_auto_commit_reconnection_scenario.rs b/core/integration/tests/server/scenarios/consumer_group_auto_commit_reconnection_scenario.rs index 6f18eadbc4..13880f83b3 100644 --- a/core/integration/tests/server/scenarios/consumer_group_auto_commit_reconnection_scenario.rs +++ b/core/integration/tests/server/scenarios/consumer_group_auto_commit_reconnection_scenario.rs @@ -44,11 +44,11 @@ async fn init_system(client: &IggyClient) { .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - 1, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); diff --git a/core/integration/tests/server/scenarios/consumer_group_duplicate_name_create_scenario.rs b/core/integration/tests/server/scenarios/consumer_group_duplicate_name_create_scenario.rs index 71c310d5bd..679557407f 100644 --- a/core/integration/tests/server/scenarios/consumer_group_duplicate_name_create_scenario.rs +++ b/core/integration/tests/server/scenarios/consumer_group_duplicate_name_create_scenario.rs @@ -25,11 +25,9 @@ use crate::server::scenarios::{ CONSUMER_GROUP_NAME, PARTITIONS_COUNT, STREAM_NAME, TOPIC_NAME, USERNAME_1, cleanup, create_client, join_consumer_group, }; -use iggy::prelude::CompressionAlgorithm; use iggy::prelude::Identifier; use iggy::prelude::IggyExpiry; -use iggy::prelude::MaxTopicSize; -use iggy::prelude::{ConsumerGroupClient, StreamClient, TopicClient}; +use iggy::prelude::{ConsumerGroupClient, StreamClient, TopicClient, TopicCreateOptions}; use integration::harness::{ TestHarness, assert_clean_system, create_user, delete_user, login_user, }; @@ -46,11 +44,11 @@ pub async fn run(harness: &TestHarness) { .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - PARTITIONS_COUNT, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(PARTITIONS_COUNT), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); diff --git a/core/integration/tests/server/scenarios/consumer_group_join_scenario.rs b/core/integration/tests/server/scenarios/consumer_group_join_scenario.rs index 50b25feb48..00845a5934 100644 --- a/core/integration/tests/server/scenarios/consumer_group_join_scenario.rs +++ b/core/integration/tests/server/scenarios/consumer_group_join_scenario.rs @@ -21,12 +21,12 @@ use crate::server::scenarios::{ }; use iggy::clients::client::IggyClient; use iggy::prelude::ClientInfoDetails; -use iggy::prelude::CompressionAlgorithm; use iggy::prelude::ConsumerGroupDetails; use iggy::prelude::Identifier; use iggy::prelude::IggyExpiry; -use iggy::prelude::MaxTopicSize; -use iggy::prelude::{ConsumerGroupClient, StreamClient, SystemClient, TopicClient}; +use iggy::prelude::{ + ConsumerGroupClient, StreamClient, SystemClient, TopicClient, TopicCreateOptions, +}; use integration::harness::{TestHarness, assert_clean_system, create_user, login_user}; pub async fn run(harness: &TestHarness) { @@ -49,11 +49,11 @@ pub async fn run(harness: &TestHarness) { .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - PARTITIONS_COUNT, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(PARTITIONS_COUNT), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); diff --git a/core/integration/tests/server/scenarios/consumer_group_new_messages_after_restart_scenario.rs b/core/integration/tests/server/scenarios/consumer_group_new_messages_after_restart_scenario.rs index 7d1323f8f7..ad4c2012f2 100644 --- a/core/integration/tests/server/scenarios/consumer_group_new_messages_after_restart_scenario.rs +++ b/core/integration/tests/server/scenarios/consumer_group_new_messages_after_restart_scenario.rs @@ -41,11 +41,11 @@ async fn init_system(client: &IggyClient) { .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - 1, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); diff --git a/core/integration/tests/server/scenarios/consumer_group_offset_cleanup_scenario.rs b/core/integration/tests/server/scenarios/consumer_group_offset_cleanup_scenario.rs index ab3d08e0c7..a3881fb092 100644 --- a/core/integration/tests/server/scenarios/consumer_group_offset_cleanup_scenario.rs +++ b/core/integration/tests/server/scenarios/consumer_group_offset_cleanup_scenario.rs @@ -49,11 +49,11 @@ async fn init_system(client: &IggyClient) { .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - 1, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); diff --git a/core/integration/tests/server/scenarios/consumer_group_with_multiple_clients_polling_messages_scenario.rs b/core/integration/tests/server/scenarios/consumer_group_with_multiple_clients_polling_messages_scenario.rs index 04a66718b3..2f321f0345 100644 --- a/core/integration/tests/server/scenarios/consumer_group_with_multiple_clients_polling_messages_scenario.rs +++ b/core/integration/tests/server/scenarios/consumer_group_with_multiple_clients_polling_messages_scenario.rs @@ -55,11 +55,11 @@ async fn init_system( .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - PARTITIONS_COUNT, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(PARTITIONS_COUNT), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); diff --git a/core/integration/tests/server/scenarios/consumer_group_with_single_client_polling_messages_scenario.rs b/core/integration/tests/server/scenarios/consumer_group_with_single_client_polling_messages_scenario.rs index 4382e0f36e..8e94831403 100644 --- a/core/integration/tests/server/scenarios/consumer_group_with_single_client_polling_messages_scenario.rs +++ b/core/integration/tests/server/scenarios/consumer_group_with_single_client_polling_messages_scenario.rs @@ -50,11 +50,11 @@ async fn init_system(client: &IggyClient) { .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - PARTITIONS_COUNT, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(PARTITIONS_COUNT), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); diff --git a/core/integration/tests/server/scenarios/consumer_timestamp_polling_scenario.rs b/core/integration/tests/server/scenarios/consumer_timestamp_polling_scenario.rs index eb9f256da8..e8a9a7a7a6 100644 --- a/core/integration/tests/server/scenarios/consumer_timestamp_polling_scenario.rs +++ b/core/integration/tests/server/scenarios/consumer_timestamp_polling_scenario.rs @@ -135,11 +135,11 @@ async fn init_system(client: &IggyClient) { .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - 1, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); diff --git a/core/integration/tests/server/scenarios/encryption_scenario.rs b/core/integration/tests/server/scenarios/encryption_scenario.rs index 4316a1fd90..f62425f4c4 100644 --- a/core/integration/tests/server/scenarios/encryption_scenario.rs +++ b/core/integration/tests/server/scenarios/encryption_scenario.rs @@ -53,11 +53,11 @@ async fn should_fill_data_with_headers_and_verify_after_restart_using_api(encryp .create_topic( &Identifier::named(stream_name).unwrap(), topic_name, - partition_count, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(partition_count), + message_expiry: Some(IggyExpiry::NeverExpire), + ..eager_flush_options() + }, ) .await .unwrap(); @@ -346,11 +346,11 @@ async fn should_encrypt_and_decrypt_headers_with_client_side_encryption( .create_topic( &Identifier::named(&stream_name).unwrap(), topic_name, - 1, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..eager_flush_options() + }, ) .await .unwrap(); @@ -477,20 +477,20 @@ fn encryption_disabled() -> bool { false } +/// The server flushes on the journal thresholds (there is no flush primitive), +/// so force every committed batch straight to disk: the assertions below read +/// the segment files directly. Both knobs are topic creation options now. +fn eager_flush_options() -> TopicCreateOptions { + TopicCreateOptions { + enforce_fsync: Some(true), + messages_required_to_save: Some(1), + ..TopicCreateOptions::default() + } +} + fn build_server_config(encryption: bool) -> TestServerConfig { let mut extra_envs = HashMap::new(); - // The server flushes on the journal thresholds (no flush primitive), so - // force every committed batch straight to disk for the on-disk asserts. - extra_envs.insert( - "IGGY_SYSTEM_PARTITION_MESSAGES_REQUIRED_TO_SAVE".to_string(), - "1".to_string(), - ); - extra_envs.insert( - "IGGY_SYSTEM_PARTITION_ENFORCE_FSYNC".to_string(), - "true".to_string(), - ); - if encryption { extra_envs.insert( "IGGY_SYSTEM_ENCRYPTION_ENABLED".to_string(), diff --git a/core/integration/tests/server/scenarios/invalid_consumer_offset_scenario.rs b/core/integration/tests/server/scenarios/invalid_consumer_offset_scenario.rs index 2548ff507e..92e82fc9ef 100644 --- a/core/integration/tests/server/scenarios/invalid_consumer_offset_scenario.rs +++ b/core/integration/tests/server/scenarios/invalid_consumer_offset_scenario.rs @@ -91,11 +91,12 @@ async fn initialize(client: &IggyClient, stream: &Identifier, topic: &Identifier .create_topic( stream, TOPIC_NAME, - PARTITIONS_COUNT, - Default::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::Unlimited, + &TopicCreateOptions { + partitions_count: Some(PARTITIONS_COUNT), + message_expiry: Some(IggyExpiry::NeverExpire), + max_topic_size: Some(MaxTopicSize::Unlimited), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); diff --git a/core/integration/tests/server/scenarios/log_rotation_scenario.rs b/core/integration/tests/server/scenarios/log_rotation_scenario.rs index dbb180701d..b2c3f3c27e 100644 --- a/core/integration/tests/server/scenarios/log_rotation_scenario.rs +++ b/core/integration/tests/server/scenarios/log_rotation_scenario.rs @@ -17,9 +17,7 @@ use crate::server::scenarios::{PARTITIONS_COUNT, STREAM_NAME, TOPIC_NAME}; use iggy::prelude::*; -use iggy_common::{ - CompressionAlgorithm, Identifier, IggyByteSize, IggyDuration, IggyExpiry, MaxTopicSize, -}; +use iggy_common::{Identifier, IggyByteSize, IggyDuration, IggyExpiry, MaxTopicSize}; use integration::harness::{TestHarness, TestServerConfig}; use serial_test::parallel; use std::collections::HashMap; @@ -196,11 +194,12 @@ async fn generate_enough_logs(client: &IggyClient) -> Result<(), String> { .create_topic( &stream_identifier, &topic_name, - PARTITIONS_COUNT, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::Unlimited, + &TopicCreateOptions { + partitions_count: Some(PARTITIONS_COUNT), + message_expiry: Some(IggyExpiry::NeverExpire), + max_topic_size: Some(MaxTopicSize::Unlimited), + ..TopicCreateOptions::default() + }, ) .await .map_err(|e| format!("Failed to create topic {topic_name}: {e}"))?; diff --git a/core/integration/tests/server/scenarios/message_cleanup_scenario.rs b/core/integration/tests/server/scenarios/message_cleanup_scenario.rs index 51e67d6a46..112f484ccd 100644 --- a/core/integration/tests/server/scenarios/message_cleanup_scenario.rs +++ b/core/integration/tests/server/scenarios/message_cleanup_scenario.rs @@ -17,9 +17,13 @@ //! Tests for message retention policies (time-based and size-based). //! -//! Configuration: 10KiB segment size, 100ms cleaner interval, instant flush. -//! Message size: 64B header + 936B payload = 1KB per message. -//! Therefore a segment holds 9 messages and every ~10 messages rotates one. +//! Configuration: 1 MiB segments, 100ms cleaner interval, instant flush. The +//! segment size is a topic creation option now and 1 MiB is the smallest a +//! topic may declare, so the payload carries what a 10 KiB segment used to: +//! one 10-message batch is a hair over 1 MiB on disk, which makes roughly one +//! segment per batch. Every retention assertion below is a lower bound on the +//! segment count, so the exact rotation point does not matter -- only that the +//! volume sent clears the cap several times over. use bytes::Bytes; use iggy::prelude::*; @@ -33,8 +37,13 @@ const TOPIC_NAME: &str = "test_expiry_topic"; const PARTITION_ID: u32 = 0; const LOG_EXTENSION: &str = "log"; -/// Payload size chosen so that header (64B) + payload = 1KB per message. -const PAYLOAD_SIZE: usize = 936; +/// Smallest segment a topic may declare (`iggy_common::MIN_TOPIC_SEGMENT_SIZE`). +const SEGMENT_SIZE: u64 = 1024 * 1024; + +/// Payload size chosen so a 10-message batch lands just past [`SEGMENT_SIZE`] +/// on disk: a 256-byte command header per send plus 48 bytes per message, so +/// 256 + 10 * (48 + 105000) = 1050736 >= 1 MiB. +const PAYLOAD_SIZE: usize = 105_000; /// Buffer time for cleaner to run after expiry conditions are met. const CLEANER_BUFFER: Duration = Duration::from_millis(300); @@ -43,6 +52,20 @@ fn make_payload(fill: char) -> Bytes { Bytes::from(fill.to_string().repeat(PAYLOAD_SIZE)) } +/// Knobs every topic here shares. The 1 MiB segment gives the retention +/// policies sealed segments to reclaim (an active segment is never deleted), +/// and the flush per message puts a send on disk before the segment count is +/// read -- both were server config before they became topic options. +fn cleanup_topic_options() -> TopicCreateOptions { + TopicCreateOptions { + partitions_count: Some(1), + segment_size: Some(IggyByteSize::from(SEGMENT_SIZE)), + enforce_fsync: Some(true), + messages_required_to_save: Some(1), + ..TopicCreateOptions::default() + } +} + /// Tests time-based retention: segments are cleaned up after expiry. pub async fn run_expiry_after_rotation(client: &IggyClient, data_path: &Path) { let stream = client.create_stream(STREAM_NAME).await.unwrap(); @@ -59,11 +82,10 @@ pub async fn run_expiry_after_rotation(client: &IggyClient, data_path: &Path) { .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::ExpireDuration(IggyDuration::from(expiry)), - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + message_expiry: Some(IggyExpiry::ExpireDuration(IggyDuration::from(expiry))), + ..cleanup_topic_options() + }, ) .await .unwrap(); @@ -76,11 +98,11 @@ pub async fn run_expiry_after_rotation(client: &IggyClient, data_path: &Path) { .display() .to_string(); - // Send 110 messages (1KB each) in batches, spanning several 10KiB segments. + // Send 40 messages in batches, spanning several 1 MiB segments. // Batched rather than one request per message: the burst must fit inside // `expiry` with room to spare, and each request costs a round-trip. let payload = make_payload('A'); - let total_messages: usize = 110; + let total_messages: usize = 40; let batch_size = 10; for chunk_start in (0..total_messages).step_by(batch_size) { @@ -181,11 +203,10 @@ pub async fn run_active_segment_protection(client: &IggyClient, data_path: &Path .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::ExpireDuration(IggyDuration::from(expiry)), - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + message_expiry: Some(IggyExpiry::ExpireDuration(IggyDuration::from(expiry))), + ..cleanup_topic_options() + }, ) .await .unwrap(); @@ -240,17 +261,19 @@ pub async fn run_size_based_retention(client: &IggyClient, data_path: &Path) { let stream = client.create_stream(STREAM_NAME).await.unwrap(); let stream_id = stream.id; - // 150KB max, cleanup at 90% = 135KB. With 100KB segments, exceeding 135KB triggers cleanup. - let max_size_bytes = 150 * 1024; + // 4 MiB max, cleanup at 90% = 3.6 MiB. The cap has to sit above the 1 MiB + // segment size, or the topic would hit it before a single segment sealed and + // the cleaner would have nothing it is allowed to reclaim. + let max_size_bytes = 4 * 1024 * 1024; let topic = client .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::Custom(IggyByteSize::from(max_size_bytes)), + &TopicCreateOptions { + message_expiry: Some(IggyExpiry::NeverExpire), + max_topic_size: Some(MaxTopicSize::Custom(IggyByteSize::from(max_size_bytes))), + ..cleanup_topic_options() + }, ) .await .unwrap(); @@ -263,9 +286,9 @@ pub async fn run_size_based_retention(client: &IggyClient, data_path: &Path) { .display() .to_string(); - // Send 160 messages (160KB) to exceed 90% threshold (135KB) + // Send 80 messages (~8 MiB) to clear the 3.6 MiB threshold several times over let payload = make_payload('B'); - let total_messages = 160; + let total_messages = 80; for i in 0..total_messages { let message = IggyMessage::builder() @@ -340,11 +363,12 @@ pub async fn run_combined_retention(client: &IggyClient, data_path: &Path) { .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::ExpireDuration(IggyDuration::from(expiry)), - MaxTopicSize::Custom(IggyByteSize::from(500 * 1024)), // 500KB (won't trigger) + &TopicCreateOptions { + message_expiry: Some(IggyExpiry::ExpireDuration(IggyDuration::from(expiry))), + // 500 MiB (won't trigger) + max_topic_size: Some(MaxTopicSize::Custom(IggyByteSize::from(500 * 1024 * 1024))), + ..cleanup_topic_options() + }, ) .await .unwrap(); @@ -357,12 +381,12 @@ pub async fn run_combined_retention(client: &IggyClient, data_path: &Path) { .display() .to_string(); - // Send 110 messages to create 2 segments (under size threshold, but will - // expire). Batched so the burst finishes well inside `expiry`: a per-message - // request pays a consensus round-trip plus an fsync, and a loop that - // outlives the window has its head reclaimed before the count below. + // Send 40 messages to create several segments (under size threshold, but + // will expire). Batched so the burst finishes well inside `expiry`: a + // per-message request pays a consensus round-trip plus an fsync, and a loop + // that outlives the window has its head reclaimed before the count below. let payload = make_payload('C'); - let total_messages: usize = 110; + let total_messages: usize = 40; let batch_size = 10; for chunk_start in (0..total_messages).step_by(batch_size) { let mut messages: Vec = (chunk_start @@ -421,18 +445,18 @@ pub async fn run_expiry_with_multiple_partitions(client: &IggyClient, data_path: .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - PARTITIONS_COUNT, - CompressionAlgorithm::None, - None, - IggyExpiry::ExpireDuration(IggyDuration::from(expiry)), - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(PARTITIONS_COUNT), + message_expiry: Some(IggyExpiry::ExpireDuration(IggyDuration::from(expiry))), + ..cleanup_topic_options() + }, ) .await .unwrap(); let topic_id = topic.id; let payload = make_payload('D'); - let messages_per_partition: usize = 110; + let messages_per_partition: usize = 40; let batch_size = 10; // Send messages to all partitions. Batched: a per-message request costs a @@ -534,17 +558,19 @@ pub async fn run_fair_size_based_cleanup_multipartition(client: &IggyClient, dat let stream = client.create_stream(STREAM_NAME).await.unwrap(); let stream_id = stream.id; - // 200KB max, cleanup at 90% = 180KB - let max_size_bytes = 200 * 1024; + // 6 MiB max, cleanup at 90% = 5.4 MiB. Above `PARTITIONS_COUNT` * the 1 MiB + // segment size, so every partition can seal a segment before the cap trips. + let max_size_bytes = 6 * 1024 * 1024; let topic = client .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - PARTITIONS_COUNT, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::Custom(IggyByteSize::from(max_size_bytes)), + &TopicCreateOptions { + partitions_count: Some(PARTITIONS_COUNT), + message_expiry: Some(IggyExpiry::NeverExpire), + max_topic_size: Some(MaxTopicSize::Custom(IggyByteSize::from(max_size_bytes))), + ..cleanup_topic_options() + }, ) .await .unwrap(); @@ -552,9 +578,9 @@ pub async fn run_fair_size_based_cleanup_multipartition(client: &IggyClient, dat let payload = make_payload('E'); - // Send 70 messages per partition = 210KB total, exceeds 180KB threshold + // Send 30 messages per partition = ~9.5 MiB total, exceeds 5.4 MiB threshold for partition_id in 0..PARTITIONS_COUNT { - for i in 0..70 { + for i in 0..30 { let msg_id = partition_id as u128 * 1000 + i as u128; let message = IggyMessage::builder() .id(msg_id) @@ -606,7 +632,7 @@ pub async fn run_fair_size_based_cleanup_multipartition(client: &IggyClient, dat /// `delete_expired_segments_for_partition` does not. /// /// Scenario: -/// 1. Send 300 messages (3 segments at 100KB each) +/// 1. Send 100 messages (several 1 MiB segments) /// 2. Consumer reads only 50 messages (stored offset ~49, within segment 0) /// 3. Wait for all segments to expire (4s expiry) /// 4. Verify consumer can still poll Next() and get contiguous offsets @@ -630,11 +656,10 @@ pub async fn run_expiry_respects_consumer_offset(client: &IggyClient, data_path: .create_topic( &Identifier::named(TEST_STREAM).unwrap(), TEST_TOPIC, - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::ExpireDuration(IggyDuration::from(expiry)), - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + message_expiry: Some(IggyExpiry::ExpireDuration(IggyDuration::from(expiry))), + ..cleanup_topic_options() + }, ) .await .unwrap(); @@ -647,11 +672,11 @@ pub async fn run_expiry_respects_consumer_offset(client: &IggyClient, data_path: .display() .to_string(); - // Send 300 messages (1KB each) -> 3 sealed segments + active. Batched: - // one request per message costs a consensus round-trip plus an fsync each, + // Send 100 messages -> several sealed segments + active. Batched: one + // request per message costs a consensus round-trip plus an fsync each, // which on a 3-node debug cluster runs the burst well past `expiry`. let payload = make_payload('B'); - let total_messages = 300u32; + let total_messages = 100u32; let batch_size = 10u32; for chunk_start in (0..total_messages).step_by(batch_size as usize) { let mut messages: Vec = (chunk_start @@ -716,8 +741,8 @@ pub async fn run_expiry_respects_consumer_offset(client: &IggyClient, data_path: tokio::time::sleep(expiry + CLEANER_BUFFER + CLEANER_BUFFER).await; // Now poll Next() - consumer should continue from offset 50 without gaps. - // BUG: on unfixed code, the cleaner deleted the segment containing offsets - // 50-99 (expired, no consumer barrier check), so Next() jumps to offset 100+. + // BUG: on unfixed code, the cleaner deleted the segment holding offset 50 + // (expired, no consumer barrier check), so Next() jumps past it. let polled = client .poll_messages( &Identifier::named(TEST_STREAM).unwrap(), diff --git a/core/integration/tests/server/scenarios/message_headers_scenario.rs b/core/integration/tests/server/scenarios/message_headers_scenario.rs index 70265c5075..968ec1e5a3 100644 --- a/core/integration/tests/server/scenarios/message_headers_scenario.rs +++ b/core/integration/tests/server/scenarios/message_headers_scenario.rs @@ -114,11 +114,11 @@ async fn init_system(client: &IggyClient) { .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - PARTITIONS_COUNT, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(PARTITIONS_COUNT), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); diff --git a/core/integration/tests/server/scenarios/message_size_scenario.rs b/core/integration/tests/server/scenarios/message_size_scenario.rs index 0c5d77a091..087787d1e5 100644 --- a/core/integration/tests/server/scenarios/message_size_scenario.rs +++ b/core/integration/tests/server/scenarios/message_size_scenario.rs @@ -112,11 +112,11 @@ async fn init_system(client: &IggyClient) { .create_topic( &STREAM_NAME.try_into().unwrap(), TOPIC_NAME, - PARTITIONS_COUNT, - Default::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(PARTITIONS_COUNT), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); diff --git a/core/integration/tests/server/scenarios/offset_scenario.rs b/core/integration/tests/server/scenarios/offset_scenario.rs index e440ef28a5..7fdc31194a 100644 --- a/core/integration/tests/server/scenarios/offset_scenario.rs +++ b/core/integration/tests/server/scenarios/offset_scenario.rs @@ -52,7 +52,7 @@ fn all_message_sizes() -> Vec { vec![50, 1000, 20000] } -pub async fn run(harness: &TestHarness) { +pub async fn run(harness: &TestHarness, topic_options: &TopicCreateOptions) { let client = harness .root_client() .await @@ -60,7 +60,14 @@ pub async fn run(harness: &TestHarness) { for msg_size in all_message_sizes() { for (pattern_name, batch_pattern) in all_batch_patterns() { - run_offset_test(&client, msg_size, &batch_pattern, pattern_name).await; + run_offset_test( + &client, + msg_size, + &batch_pattern, + pattern_name, + topic_options, + ) + .await; } } } @@ -70,11 +77,12 @@ async fn run_offset_test( message_size: u64, batch_lengths: &[u32], pattern_name: &str, + topic_options: &TopicCreateOptions, ) { let stream_name = format!("test-stream-{}-{}", message_size, pattern_name); let topic_name = format!("test-topic-{}-{}", message_size, pattern_name); - init_system(client, &stream_name, &topic_name).await; + init_system(client, &stream_name, &topic_name, topic_options).await; let total_messages_count: u32 = batch_lengths.iter().sum(); @@ -113,17 +121,18 @@ async fn run_offset_test( cleanup(client, &stream_name).await; } -async fn init_system(client: &IggyClient, stream_name: &str, topic_name: &str) { +async fn init_system( + client: &IggyClient, + stream_name: &str, + topic_name: &str, + topic_options: &TopicCreateOptions, +) { client.create_stream(stream_name).await.unwrap(); client .create_topic( &Identifier::named(stream_name).unwrap(), topic_name, - 1, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + topic_options, ) .await .unwrap(); diff --git a/core/integration/tests/server/scenarios/permissions_scenario.rs b/core/integration/tests/server/scenarios/permissions_scenario.rs index cdbb514b07..6c27ff21c0 100644 --- a/core/integration/tests/server/scenarios/permissions_scenario.rs +++ b/core/integration/tests/server/scenarios/permissions_scenario.rs @@ -99,11 +99,11 @@ async fn setup_test_resources(root_client: &IggyClient) { .create_topic( &stream_id, topic_name, - PARTITIONS, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(PARTITIONS), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .expect("create topic"); @@ -186,11 +186,11 @@ async fn test_no_permissions(harness: &TestHarness, root_client: &IggyClient) { .create_topic( &stream_id, "x", - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await, "create_topic", @@ -400,6 +400,7 @@ async fn test_user_permissions(harness: &TestHarness, root_client: &IggyClient) &Identifier::named("temp-user").unwrap(), Some("temp-user-updated"), None, + &UserUpdateOptions::default(), ) .await .expect("manage_users: update_user should work"); @@ -450,7 +451,9 @@ async fn test_stream_permissions(harness: &TestHarness, root_client: &IggyClient "read_streams: create_stream", ); assert_unauthorized( - client.update_stream(&stream_id, "new-name").await, + client + .update_stream(&stream_id, "new-name", &StreamUpdateOptions::default()) + .await, "read_streams: update_stream", ); assert_unauthorized( @@ -488,7 +491,11 @@ async fn test_stream_permissions(harness: &TestHarness, root_client: &IggyClient .expect("manage_streams: create_stream should work"); client - .update_stream(&Identifier::named("temp-stream").unwrap(), "temp-stream-v2") + .update_stream( + &Identifier::named("temp-stream").unwrap(), + "temp-stream-v2", + &StreamUpdateOptions::default(), + ) .await .expect("manage_streams: update_stream should work"); @@ -556,11 +563,11 @@ async fn test_topic_permissions(harness: &TestHarness, root_client: &IggyClient) .create_topic( &stream_id, "x", - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await, "read_topics: create_topic", @@ -571,10 +578,7 @@ async fn test_topic_permissions(harness: &TestHarness, root_client: &IggyClient) &stream_id, &topic_id, "new-name", - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicUpdateOptions::default(), ) .await, "read_topics: update_topic", @@ -604,11 +608,11 @@ async fn test_topic_permissions(harness: &TestHarness, root_client: &IggyClient) .create_topic( &stream_id, "temp-topic", - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .expect("manage_topics: create_topic should work"); @@ -619,10 +623,7 @@ async fn test_topic_permissions(harness: &TestHarness, root_client: &IggyClient) &stream_id, &Identifier::named("temp-topic").unwrap(), "temp-topic-v2", - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicUpdateOptions::default(), ) .await .expect("manage_topics: update_topic should work"); @@ -1129,11 +1130,11 @@ async fn test_global_permission_inheritance(harness: &TestHarness, root_client: .create_topic( &stream_id, "x", - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await, "read_streams does NOT imply manage_topics", @@ -1175,11 +1176,11 @@ async fn test_global_permission_inheritance(harness: &TestHarness, root_client: .create_topic( &stream_id, "temp-inherit-topic", - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .expect("manage_streams → manage_topics: create_topic"); @@ -1495,11 +1496,11 @@ async fn test_stream_permission_inheritance(harness: &TestHarness, root_client: .create_topic( &stream_id, "x", - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await, "stream.read_stream does NOT imply manage_topics", @@ -1540,11 +1541,11 @@ async fn test_stream_permission_inheritance(harness: &TestHarness, root_client: .create_topic( &stream_id, "temp-manage-stream-topic", - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .expect("stream.manage_stream → manage_topics: create_topic"); @@ -1553,10 +1554,7 @@ async fn test_stream_permission_inheritance(harness: &TestHarness, root_client: &stream_id, &Identifier::named("temp-manage-stream-topic").unwrap(), "temp-manage-stream-topic-v2", - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicUpdateOptions::default(), ) .await .expect("stream.manage_stream → manage_topics: update_topic"); @@ -1640,11 +1638,11 @@ async fn test_stream_permission_inheritance(harness: &TestHarness, root_client: .create_topic( &stream_id, "temp-manage-topic", - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .expect("stream.manage_topics: create_topic should work"); diff --git a/core/integration/tests/server/scenarios/purge_delete_scenario.rs b/core/integration/tests/server/scenarios/purge_delete_scenario.rs index 30cda49ac4..79f1d29a57 100644 --- a/core/integration/tests/server/scenarios/purge_delete_scenario.rs +++ b/core/integration/tests/server/scenarios/purge_delete_scenario.rs @@ -31,12 +31,22 @@ const PARTITION_ID: u32 = 0; const LOG_EXTENSION: &str = "log"; const INDEX_EXTENSION: &str = "index"; +/// Smallest segment a topic may declare (`iggy_common::MIN_TOPIC_SEGMENT_SIZE`). +/// The layout below is built around it: a segment size is a per-topic creation +/// option now, and sub-MiB values are refused at admission, so the message +/// volume carries what a 5 KiB segment used to. +const SEGMENT_SIZE: u64 = 1024 * 1024; + /// The server persists the actual `SendMessages2` batch framing: a 256-byte /// command header per append (each send below is a single-message batch) plus /// a 48-byte per-message header, and a 24-byte sparse index entry per flush /// (one per message with messages_required_to_save = 1). See /// `server_common::send_messages2` and `stream_size_validation_scenario`. -const PAYLOAD_SIZE: usize = 936; +/// +/// Sized so five messages seal a [`SEGMENT_SIZE`] segment and four do not: +/// 4 * 220304 = 881216 < 1 MiB <= 5 * 220304 = 1101520. Must stay a multiple +/// of 4, since `send_messages` fills the payload with a 4-byte pattern. +const PAYLOAD_SIZE: usize = 220_000; const NG_BATCH_HEADER_SIZE: u64 = 256; const NG_MESSAGE_HEADER_SIZE: u64 = 48; const MESSAGE_ON_DISK_SIZE: u64 = @@ -44,11 +54,51 @@ const MESSAGE_ON_DISK_SIZE: u64 = const INDEX_SIZE_PER_MSG: u64 = 24; const TOTAL_MESSAGES: u32 = 25; -/// 5 sealed segments (5 msgs each at 1240B on disk; the post-append size -/// check seals at 6200B >= 5KiB) + 1 empty active segment at offset 25. +/// 5 sealed segments (5 msgs each at 220304B on disk; the post-append size +/// check seals at 1101520B >= 1MiB) + 1 empty active segment at offset 25. const EXPECTED_SEGMENT_OFFSETS: &[u64] = &[0, 5, 10, 15, 20, 25]; const MSGS_PER_SEALED_SEGMENT: u64 = 5; +/// Topic knobs the on-disk layout assertions depend on: segments that roll +/// every five messages, and a flush per message so an append is in the +/// segment (and its 24-byte index entry) before the next assertion reads it. +fn layout_topic_options() -> TopicCreateOptions { + TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + segment_size: Some(IggyByteSize::from(SEGMENT_SIZE)), + enforce_fsync: Some(true), + messages_required_to_save: Some(1), + ..TopicCreateOptions::default() + } +} + +/// Topic knobs for the purge-durability scenario: a flush per message so both +/// the pre- and post-purge appends reach a segment, and the default segment +/// size so the handful of messages never rotates. +fn flushing_topic_options() -> TopicCreateOptions { + TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + messages_required_to_save: Some(1), + ..TopicCreateOptions::default() + } +} + +/// Topic knobs that keep every append journal-resident: both flush thresholds +/// sit far past what the scenario sends, so nothing ever reaches a segment. +/// The byte threshold has to move too -- it defaults to 1 MiB and would flush +/// on its own long before the message count threshold trips. +fn journal_resident_topic_options() -> TopicCreateOptions { + TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + messages_required_to_save: Some(10_000), + size_of_messages_required_to_save: Some(IggyByteSize::from(1024 * 1024 * 1024u64)), + ..TopicCreateOptions::default() + } +} + /// Single consumer barrier: oldest-first deletion, barrier advancement, and edge cases. /// /// Covers: barrier blocks deletion, advancing barrier releases segments, delete(0) no-op, @@ -65,11 +115,7 @@ pub async fn run(harness: &mut TestHarness, restart_server: bool) { .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &layout_topic_options(), ) .await .unwrap(); @@ -96,14 +142,14 @@ pub async fn run(harness: &mut TestHarness, restart_server: bool) { // --- Consumer offset barrier --- // - // stored_offset = 7 (start of segment 1). Segment 0 end_offset = 6 <= 7 → deletable. - // Segment 1 end_offset = 13 > 7 → protected by barrier. + // stored_offset = 5 (start of segment 1). Segment 0 end_offset = 4 <= 5 → deletable. + // Segment 1 end_offset = 9 > 5 → protected by barrier. let consumer = Consumer { kind: ConsumerKind::Consumer, id: Identifier::numeric(1).unwrap(), }; - let stored_offset = EXPECTED_SEGMENT_OFFSETS[1]; // 7 - let seg1_end_offset = EXPECTED_SEGMENT_OFFSETS[2] - 1; // 13 + let stored_offset = EXPECTED_SEGMENT_OFFSETS[1]; // 5 + let seg1_end_offset = EXPECTED_SEGMENT_OFFSETS[2] - 1; // 9 client .store_consumer_offset( &consumer, @@ -120,7 +166,7 @@ pub async fn run(harness: &mut TestHarness, restart_server: bool) { .delete_segments(&stream_ident, &topic_ident, PARTITION_ID, 1) .await .unwrap(); - maybe_restart(harness, restart_server).await; + maybe_restart(harness, &client, restart_server).await; await_segment_layout(&partition_path, &EXPECTED_SEGMENT_OFFSETS[1..]).await; assert_segment_file_sizes(&partition_path, &EXPECTED_SEGMENT_OFFSETS[1..]); @@ -133,8 +179,8 @@ pub async fn run(harness: &mut TestHarness, restart_server: bool) { ) .await; - // After deleting segment 0 (7 messages removed): current_offset must still - // reflect the true partition max (24), not messages_count - 1 (17). + // After deleting segment 0 (5 messages removed): current_offset must still + // reflect the true partition max (24), not messages_count - 1 (19). { let max_offset = (TOTAL_MESSAGES - 1) as u64; // Short poll, not a one-shot read: the restart cells reconnect, and a @@ -176,7 +222,7 @@ pub async fn run(harness: &mut TestHarness, restart_server: bool) { .delete_segments(&stream_ident, &topic_ident, PARTITION_ID, 1) .await .unwrap(); - maybe_restart(harness, restart_server).await; + maybe_restart(harness, &client, restart_server).await; assert_layout_stable(&partition_path, &EXPECTED_SEGMENT_OFFSETS[1..]).await; assert_segment_file_sizes(&partition_path, &EXPECTED_SEGMENT_OFFSETS[1..]); @@ -197,7 +243,7 @@ pub async fn run(harness: &mut TestHarness, restart_server: bool) { .delete_segments(&stream_ident, &topic_ident, PARTITION_ID, 1) .await .unwrap(); - maybe_restart(harness, restart_server).await; + maybe_restart(harness, &client, restart_server).await; await_segment_layout(&partition_path, &EXPECTED_SEGMENT_OFFSETS[2..]).await; assert_segment_file_sizes(&partition_path, &EXPECTED_SEGMENT_OFFSETS[2..]); @@ -206,12 +252,12 @@ pub async fn run(harness: &mut TestHarness, restart_server: bool) { &stream_ident, &topic_ident, (2 * MSGS_PER_SEALED_SEGMENT..TOTAL_MESSAGES as u64).collect::>(), - "Messages 14..25 survive", + "Messages 10..25 survive", ) .await; - // After deleting segments 0 and 1 (14 messages removed): current_offset - // must still be 24, not messages_count - 1 (10). + // After deleting segments 0 and 1 (10 messages removed): current_offset + // must still be 24, not messages_count - 1 (14). { let max_offset = (TOTAL_MESSAGES - 1) as u64; let offset_info = client @@ -245,7 +291,7 @@ pub async fn run(harness: &mut TestHarness, restart_server: bool) { .unwrap(); assert_eq!( polled_next.messages[0].header.offset, EXPECTED_SEGMENT_OFFSETS[2], - "Next poll resumes at offset 14 (first message after stored_offset 13)" + "Next poll resumes at offset 10 (first message after stored_offset 9)" ); // --- delete(0) is a no-op --- @@ -272,7 +318,7 @@ pub async fn run(harness: &mut TestHarness, restart_server: bool) { .delete_segments(&stream_ident, &topic_ident, PARTITION_ID, u32::MAX) .await .unwrap(); - maybe_restart(harness, restart_server).await; + maybe_restart(harness, &client, restart_server).await; let active_segment_offset = *EXPECTED_SEGMENT_OFFSETS.last().unwrap(); await_segment_layout( @@ -353,11 +399,7 @@ pub async fn run_no_consumers(harness: &mut TestHarness, restart_server: bool) { .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &layout_topic_options(), ) .await .unwrap(); @@ -389,7 +431,7 @@ pub async fn run_no_consumers(harness: &mut TestHarness, restart_server: bool) { .delete_segments(&stream_ident, &topic_ident, PARTITION_ID, 1) .await .unwrap(); - maybe_restart(harness, restart_server).await; + maybe_restart(harness, &client, restart_server).await; let first_surviving = layout[i + 1]; // Deletion is asynchronous (metadata commit -> reconciler), so @@ -448,11 +490,7 @@ pub async fn run_consumer_group_barrier(client: &IggyClient, data_path: &Path) { .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &layout_topic_options(), ) .await .unwrap(); @@ -593,11 +631,7 @@ pub async fn run_multi_consumer_barrier(harness: &mut TestHarness, restart_serve .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &layout_topic_options(), ) .await .unwrap(); @@ -692,7 +726,7 @@ pub async fn run_multi_consumer_barrier(harness: &mut TestHarness, restart_serve .delete_segments(&stream_ident, &topic_ident, PARTITION_ID, u32::MAX) .await .unwrap(); - maybe_restart(harness, restart_server).await; + maybe_restart(harness, &client, restart_server).await; assert_layout_stable(&partition_path, EXPECTED_SEGMENT_OFFSETS).await; // Phase 2: slow→seg0_end, barrier=seg0_end → seg0 released @@ -719,7 +753,7 @@ pub async fn run_multi_consumer_barrier(harness: &mut TestHarness, restart_serve .delete_segments(&stream_ident, &topic_ident, PARTITION_ID, u32::MAX) .await .unwrap(); - maybe_restart(harness, restart_server).await; + maybe_restart(harness, &client, restart_server).await; await_segment_layout(&partition_path, &EXPECTED_SEGMENT_OFFSETS[1..]).await; // Phase 3: slow→mid-seg1, barrier below seg1_end → seg1 protected @@ -746,7 +780,7 @@ pub async fn run_multi_consumer_barrier(harness: &mut TestHarness, restart_serve .delete_segments(&stream_ident, &topic_ident, PARTITION_ID, u32::MAX) .await .unwrap(); - maybe_restart(harness, restart_server).await; + maybe_restart(harness, &client, restart_server).await; assert_layout_stable(&partition_path, &EXPECTED_SEGMENT_OFFSETS[1..]).await; // Phase 4: slow→seg1_end, barrier=seg1_end → seg1 released @@ -773,7 +807,7 @@ pub async fn run_multi_consumer_barrier(harness: &mut TestHarness, restart_serve .delete_segments(&stream_ident, &topic_ident, PARTITION_ID, u32::MAX) .await .unwrap(); - maybe_restart(harness, restart_server).await; + maybe_restart(harness, &client, restart_server).await; await_segment_layout(&partition_path, &EXPECTED_SEGMENT_OFFSETS[2..]).await; // Phase 5: slow→last sealed end → every sealed segment released @@ -800,7 +834,7 @@ pub async fn run_multi_consumer_barrier(harness: &mut TestHarness, restart_serve .delete_segments(&stream_ident, &topic_ident, PARTITION_ID, u32::MAX) .await .unwrap(); - maybe_restart(harness, restart_server).await; + maybe_restart(harness, &client, restart_server).await; await_segment_layout(&partition_path, &active_only).await; assert_no_orphaned_segment_files(&partition_path, 1).await; @@ -831,11 +865,7 @@ pub async fn run_purge_topic(harness: &mut TestHarness, restart_server: bool) { .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &layout_topic_options(), ) .await .unwrap(); @@ -954,7 +984,7 @@ pub async fn run_purge_topic(harness: &mut TestHarness, restart_server: bool) { // instant even in the restart cells. Only the kill-lands-mid-purge case // earns a tolerance. let drained_before_restart = is_dir_empty(&consumers_dir) && is_dir_empty(&groups_dir); - maybe_restart(harness, restart_server).await; + maybe_restart(harness, &client, restart_server).await; // Purge is asynchronous (metadata commit -> reconciler -> pump). The // pump's purge resets the partition to a single segment at offset 0 and @@ -1084,15 +1114,7 @@ pub async fn run_purge_survives_restart(harness: &mut TestHarness) { client.create_stream(STREAM_NAME).await.unwrap(); let stream_ident = Identifier::named(STREAM_NAME).unwrap(); client - .create_topic( - &stream_ident, - TOPIC_NAME, - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, - ) + .create_topic(&stream_ident, TOPIC_NAME, &flushing_topic_options()) .await .unwrap(); let topic_ident = Identifier::named(TOPIC_NAME).unwrap(); @@ -1115,7 +1137,7 @@ pub async fn run_purge_survives_restart(harness: &mut TestHarness) { send_messages(&client, &stream_ident, &topic_ident, 3).await; poll_exactly(&client, &stream_ident, &topic_ident, 3).await; - maybe_restart(harness, true).await; + maybe_restart(harness, &client, true).await; // Ride out the boot reconcile pass: an un-hydrated applied generation // would re-purge asynchronously, so an immediate poll could still see @@ -1142,15 +1164,7 @@ pub async fn run_resident_purge_no_resurface(harness: &mut TestHarness) { client.create_stream(STREAM_NAME).await.unwrap(); let stream_ident = Identifier::named(STREAM_NAME).unwrap(); client - .create_topic( - &stream_ident, - TOPIC_NAME, - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, - ) + .create_topic(&stream_ident, TOPIC_NAME, &journal_resident_topic_options()) .await .unwrap(); let topic_ident = Identifier::named(TOPIC_NAME).unwrap(); @@ -1174,7 +1188,7 @@ pub async fn run_resident_purge_no_resurface(harness: &mut TestHarness) { // Graceful restart: shutdown force-flushes the committed journal, whose // front still holds the five fenced pre-purge batches. - maybe_restart(harness, true).await; + maybe_restart(harness, &client, true).await; let polled = poll_exactly(&client, &stream_ident, &topic_ident, 3).await; let offsets: Vec = polled.messages.iter().map(|m| m.header.offset).collect(); @@ -1280,11 +1294,46 @@ async fn assert_layout_stable(partition_path: &str, expected: &[u64]) { ); } -async fn maybe_restart(harness: &mut TestHarness, restart_server: bool) { - if restart_server { - harness.restart_server().await.unwrap(); +/// Bounce the server and hand back a client that is connected to the new +/// process. +/// +/// `TestHarness::restart_server` cycles the clients it owns (disconnect, then +/// connect once the process is back); this scenario builds its own, so it has to +/// be cycled explicitly. The disconnect is the part that matters: the client +/// cannot notice the socket died on its own, so it still reports +/// `Authenticated`, `connect` short-circuits as a no-op, and the first real call +/// fails. `TcpClient::send_raw` deliberately does not retry that -- it drops the +/// connection and returns `Disconnected` for the caller to handle, because a +/// late reply would desync framing. +/// +/// Reconnecting is retried rather than attempted once: `ServerHandle::start` +/// only spawns the process, so the listener is not up yet, and a boot replaying +/// this scenario's WAL takes longer than any fixed sleep worth hard-coding. +async fn maybe_restart(harness: &mut TestHarness, client: &IggyClient, restart_server: bool) { + if !restart_server { + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + return; + } + + let _ = client.disconnect().await; + harness.restart_server().await.unwrap(); + + let deadline = tokio::time::Instant::now() + POLL_CONVERGENCE_TIMEOUT; + loop { + // `connect` re-authenticates from the embedded credentials, so a + // successful ping means the shards are serving, not merely listening. + if client.connect().await.is_ok() && client.ping().await.is_ok() { + return; + } + assert!( + tokio::time::Instant::now() < deadline, + "server did not serve again within {POLL_CONVERGENCE_TIMEOUT:?} of restart" + ); + // Back to Disconnected, else the next `connect` no-ops on a half-open + // connection and the ping keeps failing until the deadline. + let _ = client.disconnect().await; + tokio::time::sleep(POLL_RETRY_INTERVAL).await; } - tokio::time::sleep(std::time::Duration::from_millis(100)).await; } /// Build a root client with SDK-level auto-reconnect and auto-sign-in. diff --git a/core/integration/tests/server/scenarios/read_during_persistence_scenario.rs b/core/integration/tests/server/scenarios/read_during_persistence_scenario.rs index 608d8749f2..c5b726bfcb 100644 --- a/core/integration/tests/server/scenarios/read_during_persistence_scenario.rs +++ b/core/integration/tests/server/scenarios/read_during_persistence_scenario.rs @@ -35,9 +35,6 @@ const TOPIC_NAME: &str = "eventual-consistency-topic"; /// 3. Background saver starts async disk write /// 4. Poll arrives - sees empty journal, data not yet on disk #[iggy_harness(server( - partition.messages_required_to_save = "100000", - partition.size_of_messages_required_to_save = "1GB", - partition.enforce_fsync = false, message_saver.interval = "100ms", message_saver.enabled = true ))] @@ -51,11 +48,19 @@ async fn should_read_messages_during_background_saver_flush( .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - 1, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + // Both inline thresholds sit far past anything the loop below + // sends, so nothing is ever flushed on the send path and only the + // background saver moves data to disk -- which is the race under + // test. Both have to move: the byte threshold defaults to 1 MiB and + // would flush on its own long before the message count trips. + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + messages_required_to_save: Some(100_000), + size_of_messages_required_to_save: Some(IggyByteSize::from(1024 * 1024 * 1024u64)), + enforce_fsync: Some(false), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); diff --git a/core/integration/tests/server/scenarios/reconnect_after_restart_scenario.rs b/core/integration/tests/server/scenarios/reconnect_after_restart_scenario.rs index 32b4d0cd02..0ce68b8507 100644 --- a/core/integration/tests/server/scenarios/reconnect_after_restart_scenario.rs +++ b/core/integration/tests/server/scenarios/reconnect_after_restart_scenario.rs @@ -25,6 +25,19 @@ use tokio::time::{Duration, sleep, timeout}; const STREAM_NAME: &str = "test-reconnect-stream"; const TOPIC_NAME: &str = "test-reconnect-topic"; +/// The restart specs below need every committed batch on disk before the +/// server goes down: there is no flush primitive, so the topic carries the +/// eager-flush thresholds that used to be `[system.partition]` config. +fn eager_flush_options() -> TopicCreateOptions { + TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + enforce_fsync: Some(true), + messages_required_to_save: Some(1), + ..TopicCreateOptions::default() + } +} + pub async fn run_producer(harness: &mut TestHarness) { let client = create_client(harness); Client::connect(&client).await.expect("Failed to connect"); @@ -33,12 +46,7 @@ pub async fn run_producer(harness: &mut TestHarness) { .producer(STREAM_NAME, TOPIC_NAME) .expect("Failed to create producer builder") .create_stream_if_not_exists() - .create_topic_if_not_exists( - 1, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, - ) + .create_topic_if_not_exists(1, IggyExpiry::NeverExpire, MaxTopicSize::ServerDefault) .send_retries(Some(10), Some(IggyDuration::from_str("2s").unwrap())) .build(); @@ -113,11 +121,11 @@ pub async fn run_consumer(harness: &mut TestHarness) { .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - 1, - Default::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .expect("Failed to create topic"); @@ -267,11 +275,7 @@ pub async fn run_single_message_offset_zero_restart(harness: &mut TestHarness) { .create_topic( &Identifier::named(STREAM).unwrap(), TOPIC, - 1, - Default::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &eager_flush_options(), ) .await .unwrap(); @@ -388,11 +392,7 @@ pub async fn run_consumer_offset_ahead_after_crash(harness: &mut TestHarness) { .create_topic( &Identifier::named(STREAM).unwrap(), TOPIC, - 1, - Default::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &eager_flush_options(), ) .await .unwrap(); @@ -512,8 +512,8 @@ pub async fn run_consumer_offset_ahead_after_crash(harness: &mut TestHarness) { /// settled primary survives to answer the rejoin probes. The replicas must /// give up probing (`ViewChangeReason::ViewProbeUnanswered`), elect among /// their recovered logs, and serve the durable data across the outage. The -/// caller's server config must flush eagerly (`messages_required_to_save=1` -/// + fsync): with every journal dying at once, only flushed bytes survive. +/// topic flushes eagerly (see [`eager_flush_options`]): with every journal +/// dying at once, only flushed bytes survive. pub async fn run_full_cluster_restart(harness: &mut TestHarness) { let setup_client = harness .root_client() @@ -528,11 +528,7 @@ pub async fn run_full_cluster_restart(harness: &mut TestHarness) { .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - 1, - Default::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &eager_flush_options(), ) .await .expect("Failed to create topic"); @@ -643,11 +639,7 @@ pub async fn run_ring_overflow_rejoin(harness: &mut TestHarness) { .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - 1, - Default::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &eager_flush_options(), ) .await .expect("Failed to create topic"); diff --git a/core/integration/tests/server/scenarios/restart_offset_skip_scenario.rs b/core/integration/tests/server/scenarios/restart_offset_skip_scenario.rs index 03bf5774f2..9516fbb6ef 100644 --- a/core/integration/tests/server/scenarios/restart_offset_skip_scenario.rs +++ b/core/integration/tests/server/scenarios/restart_offset_skip_scenario.rs @@ -62,11 +62,16 @@ pub async fn run(harness: &mut TestHarness) { .create_topic( &stream_id, TOPIC_NAME, - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + // High flush threshold so post-restart messages accumulate in the + // journal (which is what exposed the base_offset=0 bug); the + // message_saver flushes the pre-restart data before the restart. + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + messages_required_to_save: Some(10_000), + enforce_fsync: Some(false), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); diff --git a/core/integration/tests/server/scenarios/segment_rotation_race_scenario.rs b/core/integration/tests/server/scenarios/segment_rotation_race_scenario.rs index ecf152f9d6..ae90c90a07 100644 --- a/core/integration/tests/server/scenarios/segment_rotation_race_scenario.rs +++ b/core/integration/tests/server/scenarios/segment_rotation_race_scenario.rs @@ -22,7 +22,9 @@ //! 4. Task A calls active_indexes().unwrap() - panics because indexes are None //! //! This test uses: -//! - Very small segment size (512B) to trigger frequent rotations +//! - The smallest segment size a topic may declare (1 MiB), with a payload +//! sized so the burst still rolls a few hundred segments (see +//! `PAYLOAD_FILLER_LEN`) //! - 8 concurrent producers (2 per protocol: TCP, HTTP, QUIC, WebSocket) //! - All producers write to the same partition for maximum lock contention //! - Short message_saver interval to add more concurrent persist operations @@ -42,6 +44,13 @@ const PRODUCERS_PER_PROTOCOL: usize = 2; const PARTITION_ID: u32 = 0; const TEST_DURATION_SECS: u64 = 10; const MESSAGES_PER_BATCH: usize = 5; +const SEGMENT_SIZE: u64 = 1024 * 1024; +/// Filler appended to each payload. A topic segment cannot go below 1 MiB, so +/// rotation frequency has to come from message volume instead. Measured over +/// the 10-second burst: no filler writes ~28 MiB and rolls ~27 segments, while +/// this rolls ~240 at the same server memory footprint. The old 512 B cap +/// rolled one per batch (~50k), which no legal segment size can reproduce. +const PAYLOAD_FILLER_LEN: usize = 2 * 1024; const MAX_ALLOWED_MEMORY_BYTES: u64 = 200 * 1024 * 1024; /// Runs the segment rotation race condition test with multiple protocols. @@ -176,11 +185,13 @@ async fn init_system(client: &IggyClient, total_producers: usize) { .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + segment_size: Some(IggyByteSize::from(SEGMENT_SIZE)), + messages_required_to_save: Some(32), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); @@ -205,7 +216,13 @@ async fn run_producer( let mut messages = Vec::with_capacity(MESSAGES_PER_BATCH); for i in 0..MESSAGES_PER_BATCH { - let payload = format!("p{}:b{}:m{}", producer_id, batch_num, i); + let payload = format!( + "p{}:b{}:m{}{}", + producer_id, + batch_num, + i, + "x".repeat(PAYLOAD_FILLER_LEN) + ); let message = IggyMessage::builder() .payload(payload.into_bytes().into()) .build() diff --git a/core/integration/tests/server/scenarios/single_message_per_batch_scenario.rs b/core/integration/tests/server/scenarios/single_message_per_batch_scenario.rs index d541824e35..ba22668677 100644 --- a/core/integration/tests/server/scenarios/single_message_per_batch_scenario.rs +++ b/core/integration/tests/server/scenarios/single_message_per_batch_scenario.rs @@ -50,11 +50,15 @@ pub async fn run(harness: &TestHarness, duration_secs: u64) { .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - 1, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + // Flush threshold far past what this test sends, so the + // messages accumulate in the journal instead of reaching a segment + // on the send path -- the delayed persistence under test. + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + messages_required_to_save: Some(10_000), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); diff --git a/core/integration/tests/server/scenarios/stale_client_consumer_group_scenario.rs b/core/integration/tests/server/scenarios/stale_client_consumer_group_scenario.rs index 81523b104b..6c12759bbf 100644 --- a/core/integration/tests/server/scenarios/stale_client_consumer_group_scenario.rs +++ b/core/integration/tests/server/scenarios/stale_client_consumer_group_scenario.rs @@ -25,6 +25,12 @@ use std::sync::Arc; use std::time::Duration; use tokio::time::sleep; +/// Bounds the wait for the heartbeat verifier to evict a client that stopped +/// pinging. Generous relative to the 2.4s threshold it covers, because the +/// eviction pass competes with the rest of the suite. +const STALE_EVICTION_TIMEOUT: Duration = Duration::from_secs(20); +const STALE_EVICTION_RETRY_INTERVAL: Duration = Duration::from_millis(200); + const STREAM_NAME: &str = "stale-test-stream"; const TOPIC_NAME: &str = "stale-test-topic"; const CONSUMER_GROUP_NAME: &str = "stale-test-cg"; @@ -77,11 +83,11 @@ async fn setup_resources(client: &IggyClient) { .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - PARTITIONS_COUNT, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(PARTITIONS_COUNT), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); @@ -162,10 +168,34 @@ async fn should_handle_stale_client_with_manual_reconnection( } assert_eq!(messages_polled, 5); - // Wait for heartbeat timeout (2s * 1.2 = 2.4s threshold) - sleep(Duration::from_secs(4)).await; + // Wait for the heartbeat verifier (2s interval, 2.4s threshold) to evict + // the stale client, observed through `setup_client` rather than by polling + // `stale_client` itself: a poll counts as activity and would keep resetting + // the staleness the test is waiting for. That is also why the assertion + // below gets only a couple of attempts -- each one refreshes liveness, so + // retrying harder makes eviction less likely, not more. + let deadline = tokio::time::Instant::now() + STALE_EVICTION_TIMEOUT; + loop { + let group = setup_client + .get_consumer_group( + &Identifier::named(STREAM_NAME).unwrap(), + &Identifier::named(TOPIC_NAME).unwrap(), + &Identifier::named(CONSUMER_GROUP_NAME).unwrap(), + ) + .await + .unwrap() + .expect("consumer group exists"); + if group.members_count == 0 { + break; + } + assert!( + tokio::time::Instant::now() < deadline, + "stale client was not evicted within {STALE_EVICTION_TIMEOUT:?}, group still reports {} member(s)", + group.members_count + ); + sleep(STALE_EVICTION_RETRY_INTERVAL).await; + } - // Should get error after stale detection let mut got_error = false; for _ in 0..3 { if stale_client @@ -186,7 +216,7 @@ async fn should_handle_stale_client_with_manual_reconnection( } sleep(Duration::from_millis(100)).await; } - assert!(got_error, "Expected error after heartbeat timeout"); + assert!(got_error, "Expected error after heartbeat eviction"); // Reconnect with new client drop(stale_client); diff --git a/core/integration/tests/server/scenarios/stream_size_validation_scenario.rs b/core/integration/tests/server/scenarios/stream_size_validation_scenario.rs index fb16f39918..a7b76d4ddc 100644 --- a/core/integration/tests/server/scenarios/stream_size_validation_scenario.rs +++ b/core/integration/tests/server/scenarios/stream_size_validation_scenario.rs @@ -170,11 +170,11 @@ async fn create_topic_assert_empty(client: &IggyClient, stream_name: &str, topic .create_topic( &Identifier::from_str(stream_name).unwrap(), topic_name, - PARTITIONS_COUNT, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(PARTITIONS_COUNT), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); diff --git a/core/integration/tests/server/scenarios/system_scenario.rs b/core/integration/tests/server/scenarios/system_scenario.rs index 5e1b81ba4d..ed33018480 100644 --- a/core/integration/tests/server/scenarios/system_scenario.rs +++ b/core/integration/tests/server/scenarios/system_scenario.rs @@ -123,11 +123,11 @@ pub async fn run(harness: &TestHarness) { .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - PARTITIONS_COUNT, - Default::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(PARTITIONS_COUNT), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); @@ -150,7 +150,6 @@ pub async fn run(harness: &TestHarness) { assert_eq!(topic.messages_count, 0); assert_eq!(topic.message_expiry, IggyExpiry::NeverExpire); assert_eq!(topic.max_topic_size, MaxTopicSize::Unlimited); - assert_eq!(topic.replication_factor, 1); // 11. Get topic details by ID. The owning shards materialize fresh // partitions asynchronously after the commit, but the reply reports the @@ -213,11 +212,11 @@ pub async fn run(harness: &TestHarness) { .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - PARTITIONS_COUNT, - Default::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(PARTITIONS_COUNT), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await; assert!(create_topic_result.is_err()); @@ -596,17 +595,18 @@ pub async fn run(harness: &TestHarness) { let updated_message_expiry = 1000; let message_expiry_duration = updated_message_expiry.into(); let updated_max_topic_size = MaxTopicSize::Custom(IggyByteSize::from_str("2 GB").unwrap()); - let updated_replication_factor = 5; client .update_topic( &Identifier::named(STREAM_NAME).unwrap(), &Identifier::named(TOPIC_NAME).unwrap(), &updated_topic_name, - CompressionAlgorithm::Gzip, - Some(updated_replication_factor), - IggyExpiry::ExpireDuration(message_expiry_duration), - updated_max_topic_size, + &TopicUpdateOptions { + compression_algorithm: Some(CompressionAlgorithm::Gzip), + message_expiry: Some(IggyExpiry::ExpireDuration(message_expiry_duration)), + max_topic_size: Some(updated_max_topic_size), + ..TopicUpdateOptions::default() + }, ) .await .unwrap(); @@ -630,7 +630,31 @@ pub async fn run(harness: &TestHarness) { CompressionAlgorithm::Gzip ); assert_eq!(updated_topic.max_topic_size, updated_max_topic_size); - assert_eq!(updated_topic.replication_factor, updated_replication_factor); + // The three settings the update carries as fixed fields are mirrored into + // the stored map, so `options` cannot report a value the typed field has + // already moved past. Compared through the rendered value rather than the + // raw bytes: this scenario runs on every transport, and HTTP renders option + // values as readable strings where the binary transports carry the kind the + // server stored. + let expiry_key = HeaderKey::from_str(topic_option_keys::MESSAGE_EXPIRY).unwrap(); + let expiry = updated_topic + .options + .get(&expiry_key) + .expect("message expiry echoes back as an option"); + assert!(expiry.explicit, "an updated key is explicit"); + assert_eq!( + expiry.value.to_string_value(), + u64::from(updated_topic.message_expiry).to_string() + ); + let max_size_key = HeaderKey::from_str(topic_option_keys::MAX_TOPIC_SIZE).unwrap(); + let max_size = updated_topic + .options + .get(&max_size_key) + .expect("max topic size echoes back as an option"); + assert_eq!( + max_size.value.to_string_value(), + u64::from(updated_topic.max_topic_size).to_string() + ); // 39. Purge the existing topic and ensure it has no messages client @@ -674,6 +698,7 @@ pub async fn run(harness: &TestHarness) { .update_stream( &Identifier::named(STREAM_NAME).unwrap(), &updated_stream_name, + &StreamUpdateOptions::default(), ) .await .unwrap(); @@ -761,11 +786,11 @@ pub async fn run(harness: &TestHarness) { .create_topic( &Identifier::named(&stream_name).unwrap(), &topic_name, - PARTITIONS_COUNT, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(PARTITIONS_COUNT), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); diff --git a/core/integration/tests/server/scenarios/tcp_tls_scenario.rs b/core/integration/tests/server/scenarios/tcp_tls_scenario.rs index 642eba587a..7a21f03e4c 100644 --- a/core/integration/tests/server/scenarios/tcp_tls_scenario.rs +++ b/core/integration/tests/server/scenarios/tcp_tls_scenario.rs @@ -35,11 +35,11 @@ pub async fn run(client: &IggyClient) { .create_topic( &Identifier::named(stream_name).unwrap(), topic_name, - 1, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); diff --git a/core/integration/tests/server/scenarios/timestamp_scenario.rs b/core/integration/tests/server/scenarios/timestamp_scenario.rs index 343a4151a6..6a592bb8e1 100644 --- a/core/integration/tests/server/scenarios/timestamp_scenario.rs +++ b/core/integration/tests/server/scenarios/timestamp_scenario.rs @@ -53,7 +53,7 @@ fn all_message_sizes() -> Vec { vec![50, 1000, 20000] } -pub async fn run(harness: &TestHarness) { +pub async fn run(harness: &TestHarness, topic_options: &TopicCreateOptions) { let client = harness .root_client() .await @@ -61,7 +61,14 @@ pub async fn run(harness: &TestHarness) { for msg_size in all_message_sizes() { for (pattern_name, batch_pattern) in all_batch_patterns() { - run_timestamp_test(&client, msg_size, &batch_pattern, pattern_name).await; + run_timestamp_test( + &client, + msg_size, + &batch_pattern, + pattern_name, + topic_options, + ) + .await; } } } @@ -71,11 +78,12 @@ async fn run_timestamp_test( message_size: u64, batch_lengths: &[u32], pattern_name: &str, + topic_options: &TopicCreateOptions, ) { let stream_name = format!("test-stream-ts-{}-{}", message_size, pattern_name); let topic_name = format!("test-topic-ts-{}-{}", message_size, pattern_name); - init_system(client, &stream_name, &topic_name).await; + init_system(client, &stream_name, &topic_name, topic_options).await; let total_messages_count: u32 = batch_lengths.iter().sum(); @@ -133,17 +141,18 @@ async fn run_timestamp_test( cleanup(client, &stream_name).await; } -async fn init_system(client: &IggyClient, stream_name: &str, topic_name: &str) { +async fn init_system( + client: &IggyClient, + stream_name: &str, + topic_name: &str, + topic_options: &TopicCreateOptions, +) { client.create_stream(stream_name).await.unwrap(); client .create_topic( &Identifier::named(stream_name).unwrap(), topic_name, - 1, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + topic_options, ) .await .unwrap(); diff --git a/core/integration/tests/server/scenarios/user_scenario.rs b/core/integration/tests/server/scenarios/user_scenario.rs index 1fedad5cb9..91ef52ecbb 100644 --- a/core/integration/tests/server/scenarios/user_scenario.rs +++ b/core/integration/tests/server/scenarios/user_scenario.rs @@ -19,6 +19,7 @@ use crate::server::scenarios::create_client; use iggy::prelude::Identifier; use iggy::prelude::PersonalAccessTokenExpiry; use iggy::prelude::UserStatus; +use iggy::prelude::UserUpdateOptions; use iggy::prelude::defaults::DEFAULT_ROOT_USERNAME; use iggy::prelude::{GlobalPermissions, Permissions}; use iggy::prelude::{PersonalAccessTokenClient, SEC_IN_MICRO, SystemClient, UserClient}; @@ -209,6 +210,7 @@ pub async fn run(harness: &TestHarness) { &Identifier::named(test_user).unwrap(), Some(updated_test_user), Some(UserStatus::Inactive), + &UserUpdateOptions::default(), ) .await .unwrap(); diff --git a/core/integration/tests/server/scenarios/websocket_tls_scenario.rs b/core/integration/tests/server/scenarios/websocket_tls_scenario.rs index f1c4353ac0..b0d3f64229 100644 --- a/core/integration/tests/server/scenarios/websocket_tls_scenario.rs +++ b/core/integration/tests/server/scenarios/websocket_tls_scenario.rs @@ -35,11 +35,11 @@ pub async fn run(client: &IggyClient) { .create_topic( &Identifier::named(stream_name).unwrap(), topic_name, - 1, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .unwrap(); diff --git a/core/integration/tests/server/specific.rs b/core/integration/tests/server/specific.rs index d0508a4e9d..276143049d 100644 --- a/core/integration/tests/server/specific.rs +++ b/core/integration/tests/server/specific.rs @@ -54,7 +54,7 @@ async fn message_size_scenario(harness: &TestHarness) { message_size_scenario::run(harness).await; } -#[iggy_harness(server(partition.messages_required_to_save = "10000"))] +#[iggy_harness] async fn should_handle_single_message_per_batch_with_delayed_persistence(harness: &TestHarness) { single_message_per_batch_scenario::run(harness, 5).await; } @@ -90,38 +90,26 @@ async fn consumer_reconnect_after_server_restart(harness: &mut TestHarness) { reconnect_after_restart_scenario::run_consumer(harness).await; } -#[iggy_harness(server( - partition.messages_required_to_save = "1", - partition.enforce_fsync = true -))] +#[iggy_harness] async fn single_message_restart_offset_zero(harness: &mut TestHarness) { reconnect_after_restart_scenario::run_single_message_offset_zero_restart(harness).await; } // Exercises the rejoin probe's election fallback across all replicas, which a // plain single-node restart does not reach. -#[iggy_harness(server( - partition.messages_required_to_save = "1", - partition.enforce_fsync = true -))] +#[iggy_harness] async fn full_cluster_restart_recovers_and_serves(harness: &mut TestHarness) { reconnect_after_restart_scenario::run_full_cluster_restart(harness).await; } // Exercises `RangeEvicted` + the commit floor: the rejoin window exceeds the // peers' evicted ring, so journal repair alone cannot cover it. -#[iggy_harness(server( - partition.messages_required_to_save = "1", - partition.enforce_fsync = true -))] +#[iggy_harness] async fn rejoin_window_exceeding_evicted_ring(harness: &mut TestHarness) { reconnect_after_restart_scenario::run_ring_overflow_rejoin(harness).await; } -#[iggy_harness(server( - partition.messages_required_to_save = "1", - partition.enforce_fsync = true -))] +#[iggy_harness] async fn consumer_offset_ahead_after_crash(harness: &mut TestHarness) { reconnect_after_restart_scenario::run_consumer_offset_ahead_after_crash(harness).await; } @@ -134,12 +122,7 @@ async fn consumer_offset_ahead_after_crash(harness: &mut TestHarness) { /// Config: high messages_required_to_save so post-restart messages accumulate in /// the journal (exposing the base_offset=0 bug). message_saver flushes pre-restart /// data before the restart. -#[iggy_harness(server( - partition.messages_required_to_save = "10000", - partition.enforce_fsync = false, - message_saver.enabled = true, - message_saver.interval = "1s" -))] +#[iggy_harness(server(message_saver.enabled = true, message_saver.interval = "1s"))] async fn restart_offset_skip(harness: &mut TestHarness) { restart_offset_skip_scenario::run(harness).await; } @@ -150,7 +133,8 @@ async fn restart_offset_skip(harness: &mut TestHarness) { /// and handle_full_segment. /// /// Server configuration: -/// - Very small segment size (512B) to trigger frequent rotations +/// - Smallest segment size a topic may declare (1 MiB), plus a payload sized +/// to keep rotations frequent at that floor (~240 rolls per run) /// - Short message_saver interval (1s) to add concurrent persist operations /// - Small messages_required_to_save (32) to trigger more frequent saves /// - cache_indexes = none to trigger clear_active_indexes path @@ -161,9 +145,7 @@ async fn restart_offset_skip(harness: &mut TestHarness) { // Concurrency race test: runs over the three VSR transports (TCP/QUIC/ // WebSocket -- HTTP/REST carries no VSR framing). #[iggy_harness(server( - segment.size = "512B", message_saver.interval = "1s", - partition.messages_required_to_save = "32", segment.cache_indexes = "none", tcp.socket_migration = false, tcp.socket.override_defaults = true, diff --git a/core/integration/tests/server/topic_admission_vsr.rs b/core/integration/tests/server/topic_admission_vsr.rs index dac97dd969..eb8e2977bf 100644 --- a/core/integration/tests/server/topic_admission_vsr.rs +++ b/core/integration/tests/server/topic_admission_vsr.rs @@ -46,11 +46,13 @@ async fn create_topic_with( .create_topic( stream_id, name, - partitions_count, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - max_topic_size, + &TopicCreateOptions { + partitions_count: Some(partitions_count), + message_expiry: Some(IggyExpiry::NeverExpire), + max_topic_size: (max_topic_size != MaxTopicSize::ServerDefault) + .then_some(max_topic_size), + ..TopicCreateOptions::default() + }, ) .await } @@ -145,10 +147,11 @@ async fn given_updated_topic_when_getting_topic_should_echo_stored_values(harnes stream_id, topic_id, "echo-topic", - CompressionAlgorithm::None, - None, - message_expiry, - max_topic_size, + &TopicUpdateOptions { + message_expiry: Some(message_expiry), + max_topic_size: Some(max_topic_size), + ..TopicUpdateOptions::default() + }, ) .await .expect("update topic"); @@ -161,15 +164,16 @@ async fn given_updated_topic_when_getting_topic_should_echo_stored_values(harnes } }; - // The server echoes both stored sentinels as wire 0 (legacy parity). The - // SDK decodes a topic-response size 0 as `ServerDefault` but an expiry 0 - // as `NeverExpire` (`wire_conversions`), so that is the legacy-identical - // client-visible read-back; the node default must NOT leak into either. + // Settings ride the options block and 0 is its "resolve the default" + // sentinel, so a `ServerDefault` on update carries no key at all: the topic + // keeps what it already had. Resetting a setting back to the node default + // is deliberately not expressible -- an update states the values it wants, + // and everything it omits survives. + let created_size = MaxTopicSize::Custom(IggyByteSize::from_str("2GiB").expect("byte size")); assert_eq!( update_topic(MaxTopicSize::ServerDefault, IggyExpiry::ServerDefault).await, - (MaxTopicSize::ServerDefault, IggyExpiry::NeverExpire), - "an update to ServerDefault must echo the stored sentinel, \ - not the node default frozen at update time" + (created_size, IggyExpiry::NeverExpire), + "a sentinel carries no key, so the value set at creation survives" ); let custom_size = MaxTopicSize::Custom(IggyByteSize::from_str("3GiB").expect("byte size")); let custom_expiry = IggyExpiry::ExpireDuration(IggyDuration::from_str("5s").expect("duration")); @@ -185,6 +189,118 @@ async fn given_updated_topic_when_getting_topic_should_echo_stored_values(harnes ); } +#[iggy_harness( + test_client_transport = [Tcp], + server(tcp.socket.override_defaults = true, tcp.socket.nodelay = true) +)] +async fn given_update_below_one_segment_when_updating_topic_should_reject_typed( + harness: &TestHarness, +) { + let client = harness.tcp_root_client().await.expect("tcp root client"); + client + .create_stream("update-bounds-stream") + .await + .expect("create stream"); + let stream_id = Identifier::from_str_value("update-bounds-stream").expect("stream identifier"); + create_topic_with( + &client, + &stream_id, + "update-bounds-topic", + 1, + MaxTopicSize::Unlimited, + ) + .await + .expect("create topic"); + let topic_id = Identifier::from_str_value("update-bounds-topic").expect("topic identifier"); + + // Create refuses a cap under one segment; an update has to refuse the same + // value, or the stored map reports a size the topic can never enforce. + let tiny = MaxTopicSize::Custom(IggyByteSize::from_str("10KiB").expect("byte size")); + let result = client + .update_topic( + &stream_id, + &topic_id, + "update-bounds-topic", + &TopicUpdateOptions { + max_topic_size: Some(tiny), + ..TopicUpdateOptions::default() + }, + ) + .await; + let invalid_size = IggyError::InvalidTopicSize(tiny, IggyByteSize::default()).as_code(); + assert!( + matches!(&result, Err(error) if error.as_code() == invalid_size), + "update to a cap below one segment must deny with InvalidTopicSize, got {result:?}" + ); + + let topic = client + .get_topic(&stream_id, &topic_id) + .await + .expect("get topic") + .expect("topic exists"); + assert_eq!( + topic.max_topic_size, + MaxTopicSize::Unlimited, + "a denied update must leave the stored cap alone" + ); +} + +#[iggy_harness( + test_client_transport = [Tcp], + server(tcp.socket.override_defaults = true, tcp.socket.nodelay = true) +)] +async fn given_sentinel_option_when_creating_topic_should_store_the_resolved_default( + harness: &TestHarness, +) { + let client = harness.tcp_root_client().await.expect("tcp root client"); + client + .create_stream("sentinel-stream") + .await + .expect("create stream"); + let stream_id = Identifier::from_str_value("sentinel-stream").expect("stream identifier"); + + // `server_default` reaches the wire as a literal 0 through the raw map, the + // same shape the CLI's `--set max_topic_size=server_default` produces. The + // stored map has to report the resolved default rather than that 0, or one + // GetTopic response contradicts itself. + client + .create_topic( + &stream_id, + "sentinel-topic", + &TopicCreateOptions { + partitions_count: Some(1), + raw: std::collections::BTreeMap::from([( + "max_topic_size".to_string(), + "server_default".to_string(), + )]), + ..TopicCreateOptions::default() + }, + ) + .await + .expect("create topic with a sentinel option"); + + let topic_id = Identifier::from_str_value("sentinel-topic").expect("topic identifier"); + let topic = client + .get_topic(&stream_id, &topic_id) + .await + .expect("get topic") + .expect("topic exists"); + let stored = topic + .options + .get(&HeaderKey::from_str("max_topic_size").expect("option key")) + .expect("max_topic_size is stored"); + assert_eq!( + u64::from(topic.max_topic_size), + iggy_common::DEFAULT_MAX_TOPIC_SIZE, + "the typed field resolves to the node default" + ); + assert_eq!( + stored.value.as_bytes(), + &iggy_common::DEFAULT_MAX_TOPIC_SIZE.to_le_bytes(), + "the options map must agree with the typed field" + ); +} + #[iggy_harness( test_client_transport = [Tcp], server(tcp.socket.override_defaults = true, tcp.socket.nodelay = true) diff --git a/core/metadata/src/impls/metadata.rs b/core/metadata/src/impls/metadata.rs index 34c80fbc29..94b372262f 100644 --- a/core/metadata/src/impls/metadata.rs +++ b/core/metadata/src/impls/metadata.rs @@ -47,10 +47,12 @@ use iggy_binary_protocol::{ PrepareOkHeader, ProtocolVersion, ReplyHeader, RoutedRequestHeader, WireDecode, WireEncode, WireName, }; -use iggy_common::IggyError; -use iggy_common::UserId; use iggy_common::calculate_checksum; use iggy_common::variadic; +use iggy_common::{ + IggyByteSize, IggyError, IggyExpiry, MaxTopicSize, TopicCreateOptions, TopicRuntimeDefaults, + UserId, topic_option_keys, validate_topic_segment_size, +}; use journal::local_gate::LocalGate; use journal::superblock::{ PingPongSuperblock, SUPERBLOCK_RETRY_BACKOFF_BASE_MICROS, SUPERBLOCK_RETRY_BACKOFF_MAX_MICROS, @@ -746,18 +748,6 @@ pub struct IggyMetadata { /// [`Self::set_commit_notifier`] runs (the server bootstrap on shard /// 0 sets it; peer shards and tests leave it `None`). commit_notifier: RefCell>, - /// Resolved byte value for `MaxTopicSize::ServerDefault` (`0` on the - /// wire). Primary admission rewrites the sentinel to this value before - /// replication so the committed state carries a concrete size and every - /// replica resolves identically regardless of local config. Set from - /// server config at bootstrap ([`Self::set_default_max_topic_size`]); - /// defaults to unlimited, matching the shipped server config. - default_max_topic_size: Cell, - /// Resolved micros value for `IggyExpiry::ServerDefault` (`0` on the wire). - /// Same admission-time sentinel resolution as [`Self::default_max_topic_size`]; - /// set from server config at bootstrap ([`Self::set_default_message_expiry`]). - /// Defaults to never-expire, matching the shipped server config. - default_message_expiry: Cell, /// Client-table mutations at or below this op are already reflected in a /// state-transferred table, so the tail-repair commit walk must skip /// them (re-running `commit_register` would double-bump epochs). `0` @@ -805,8 +795,6 @@ where journal_gate: LocalGate::new(), client_table: RefCell::new(ClientTable::new(CLIENTS_TABLE_MAX)), commit_notifier: RefCell::new(None), - default_max_topic_size: Cell::new(u64::MAX), - default_message_expiry: Cell::new(u64::MAX), client_table_frontier: Cell::new(0), transfer_offer_cache: RefCell::new(None), } @@ -894,23 +882,6 @@ impl IggyMetadata { op > self.client_table_frontier.get() } - /// Install the resolved byte value used for `MaxTopicSize::ServerDefault`. - /// Server-ng bootstrap calls this with `system.topic.max_size` on every - /// shard (responses read it too); only shard 0's copy feeds admission. - pub fn set_default_max_topic_size(&self, max_topic_size_bytes: u64) { - self.default_max_topic_size.set(max_topic_size_bytes); - } - - /// Byte value a stored `MaxTopicSize::ServerDefault` resolves to on this - /// node. Read by the per-shard segment cleaner, which enforces retention - /// locally and so must resolve the sentinel at enforcement time: create - /// admission rewrites it before replication, but an UPDATE back to - /// `ServerDefault` leaves the sentinel in committed state. - #[must_use] - pub const fn default_max_topic_size(&self) -> u64 { - self.default_max_topic_size.get() - } - /// Raise the forced-checkpoint margin to cover a configured /// prepare-queue depth (`[metadata] prepare_queue_depth`). Clamped to /// the built-in floor by the coordinator; no-op on shards without a @@ -929,13 +900,6 @@ impl IggyMetadata { self.client_table.borrow_mut().set_capacity(max_clients); } - /// Install the resolved micros value used for `IggyExpiry::ServerDefault`. - /// Server-ng bootstrap calls this with `system.topic.message_expiry`; only - /// shard 0's copy feeds admission. - pub fn set_default_message_expiry(&self, message_expiry_micros: u64) { - self.default_message_expiry.set(message_expiry_micros); - } - /// Fire post-commit notifier. Clones the `Rc` out under a short /// borrow so a re-entrant `set_commit_notifier` from inside the /// closure cannot panic on `borrow_mut`. @@ -3309,15 +3273,69 @@ where Operation::CreateTopic => { let mut request = WireCreateTopicRequest::decode_from(body) .map_err(|_| IggyError::InvalidCommand)?; - // Resolve the `ServerDefault` sentinel (0) against server config - // here, at primary admission, so the replicated payload carries a - // concrete size and every replica commits the same value. - if request.max_topic_size == 0 { - request.max_topic_size = self.default_max_topic_size.get(); + // Resolve every absent catalog key against server config here, + // at primary admission, so the replicated payload carries + // concrete values and every replica commits the same state + // regardless of local config. Resolved defaults ride a separate + // derived block, preserving per-key provenance for `GetTopic`. + let explicit = TopicCreateOptions::parse(&request.options)?; + // Re-encode the explicit block from the parse rather than + // forwarding the client's bytes. Parsing normalizes a zero + // sentinel to "absent", so the resolved value goes in the + // derived block -- but apply merges with explicit winning, so a + // forwarded literal `0` would land back on top as the stored + // effective value. `GetTopic` would report 0, and a restart + // would re-parse that 0 to absent and fall back to whatever the + // node default is by then, not the value resolved at creation. + // Re-encoding also canonicalizes kinds (a `"128MiB"` string + // becomes `Uint64`), so the stored map reads back uniformly. + request.options = explicit.to_wire()?; + let resolved_segment_size = explicit + .segment_size + .unwrap_or_else(|| IggyByteSize::from(iggy_common::DEFAULT_SEGMENT_SIZE)); + let resolved_max_topic_size = explicit + .max_topic_size + .unwrap_or_else(|| MaxTopicSize::from(iggy_common::DEFAULT_MAX_TOPIC_SIZE)); + // Backstop for the transport-side typed checks: an explicit + // segment size outside its bounds, or a topic cap below one + // segment, must never enter the WAL. + if let Some(segment_size) = explicit.segment_size { + validate_topic_segment_size( + segment_size.as_bytes_u64(), + iggy_common::MAX_TOPIC_SEGMENT_SIZE, + )?; } - if request.message_expiry == 0 { - request.message_expiry = self.default_message_expiry.get(); + if resolved_max_topic_size.as_bytes_u64() < resolved_segment_size.as_bytes_u64() { + return Err(IggyError::InvalidOptionValue( + topic_option_keys::MAX_TOPIC_SIZE.to_string(), + )); } + let derived_options = explicit.derived_block( + explicit.compression_algorithm.unwrap_or_default(), + explicit + .message_expiry + .unwrap_or_else(|| IggyExpiry::from(iggy_common::DEFAULT_MESSAGE_EXPIRY)), + resolved_max_topic_size, + TopicRuntimeDefaults { + segment_size: resolved_segment_size, + enforce_fsync: explicit + .enforce_fsync + .unwrap_or(iggy_common::DEFAULT_ENFORCE_FSYNC), + messages_required_to_save: explicit + .messages_required_to_save + .unwrap_or(iggy_common::DEFAULT_MESSAGES_REQUIRED_TO_SAVE), + size_of_messages_required_to_save: explicit + .size_of_messages_required_to_save + .unwrap_or_else(|| { + IggyByteSize::from( + iggy_common::DEFAULT_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE, + ) + }), + preallocate_segments: explicit + .preallocate_segments + .unwrap_or(iggy_common::DEFAULT_PREALLOCATE_SEGMENTS), + }, + )?; let partitions = self .allocator .allocate_many(request.partitions_count as usize) @@ -3333,6 +3351,7 @@ where .collect::, _>>()?; let body = PersistedCreateTopicRequest { request, + derived_options, partitions, } .to_bytes(); @@ -3944,6 +3963,7 @@ mod tests { use crate::stm::stream::Streams; use crate::stm::user::Users; use consensus::LocalPipeline; + use iggy_binary_protocol::WireOptions; use iggy_binary_protocol::requests::topics::CreateTopicRequest; use iggy_common::variadic; use journal::prepare_journal::PrepareJournal; @@ -4314,11 +4334,8 @@ mod tests { let body = CreateTopicRequest { stream_id: WireIdentifier::numeric(1), partitions_count: 1, - compression_algorithm: 0, - message_expiry: 0, - max_topic_size: 0, - replication_factor: 1, name: WireName::new("t").unwrap(), + options: WireOptions::empty(), } .to_bytes(); let header_size = size_of::(); @@ -4379,32 +4396,44 @@ mod tests { #[test] fn prepare_request_stamps_create_topic_message_expiry_default() { - // A `CreateTopic` carrying the `ServerDefault` sentinel (0) must be - // rewritten at primary admission to the configured default, so the - // replicated prepare -- and thus every replica's commit -- holds a - // concrete expiry. Mirrors the `max_topic_size` sentinel resolution. + // A `CreateTopic` without an explicit `message_expiry` option must be + // resolved at primary admission to the build default, riding the + // derived block, so the replicated prepare -- and thus every + // replica's commit -- holds a concrete expiry. const CLIENT: u128 = 1; const SESSION: u64 = 10; const ACTING_USER: u32 = 7; - const CONFIGURED_EXPIRY_MICROS: u64 = 7_200_000_000; let plane = metadata_plane(); - plane.set_default_message_expiry(CONFIGURED_EXPIRY_MICROS); plane.client_table.borrow_mut().commit_register( CLIENT, ACTING_USER, register_reply(CLIENT, SESSION), ); - // `create_topic_request` builds the body with `message_expiry == 0`. + // `create_topic_request` builds the body with no options at all. let prepare = plane .prepare_request(create_topic_request(CLIENT, ACTING_USER)) .expect("CreateTopic is client-allowed"); let body = &prepare.as_slice()[size_of::()..prepare.header().size as usize]; let persisted = PersistedCreateTopicRequest::decode_from(body) .expect("create topic with assignments prepare must decode"); + assert!( + persisted.request.options.is_empty(), + "a client that sent no options gets an empty explicit block" + ); + let derived = iggy_common::TopicCreateOptions::parse(&persisted.derived_options) + .expect("derived block parses against the catalog"); assert_eq!( - persisted.request.message_expiry, CONFIGURED_EXPIRY_MICROS, - "ServerDefault expiry must be stamped to the configured default at admission" + derived.message_expiry, + Some(iggy_common::IggyExpiry::from( + iggy_common::DEFAULT_MESSAGE_EXPIRY + )), + "ServerDefault expiry must be resolved into the derived block at admission" + ); + assert_eq!( + persisted.partitions.len(), + 1, + "absent partitions_count defaults to one partition" ); } @@ -4586,6 +4615,7 @@ mod tests { ) -> Message { let body = iggy_binary_protocol::requests::streams::CreateStreamRequest { name: WireName::new(name).unwrap(), + options: WireOptions::empty(), } .to_bytes(); let header_size = size_of::(); diff --git a/core/metadata/src/stm/authz.rs b/core/metadata/src/stm/authz.rs index bbd09fa52c..4f235ff052 100644 --- a/core/metadata/src/stm/authz.rs +++ b/core/metadata/src/stm/authz.rs @@ -409,7 +409,7 @@ mod tests { use iggy_binary_protocol::requests::streams::CreateStreamRequest; use iggy_binary_protocol::requests::topics::CreateTopicRequest; use iggy_binary_protocol::requests::users::CreateUserRequest; - use iggy_binary_protocol::{Command2, WireEncode, WireName}; + use iggy_binary_protocol::{Command2, WireEncode, WireName, WireOptions}; use iggy_common::UserStatus; use server_common::iobuf::Owned; @@ -473,6 +473,7 @@ mod tests { password: "hash".to_string(), status: UserStatus::Active.as_code(), permissions: Some(WirePermissions { global, streams }), + options: WireOptions::empty(), } .to_bytes() } @@ -480,6 +481,7 @@ mod tests { fn create_stream_body(name: &str) -> bytes::Bytes { CreateStreamRequest { name: WireName::new(name).unwrap(), + options: WireOptions::empty(), } .to_bytes() } @@ -489,12 +491,10 @@ mod tests { request: CreateTopicRequest { stream_id: WireIdentifier::numeric(stream_id), partitions_count: 1, - compression_algorithm: 0, - message_expiry: 0, - max_topic_size: 0, - replication_factor: 1, name: WireName::new(name).unwrap(), + options: WireOptions::empty(), }, + derived_options: WireOptions::empty(), partitions: vec![CreatedPartitionAssignment { partition_id: 0, consensus_group_id: 1, diff --git a/core/metadata/src/stm/consumer_group.rs b/core/metadata/src/stm/consumer_group.rs index 1daf6a72f7..fd757c2d4c 100644 --- a/core/metadata/src/stm/consumer_group.rs +++ b/core/metadata/src/stm/consumer_group.rs @@ -37,7 +37,7 @@ use bytes::Bytes; use bytes::{BufMut, BytesMut}; use iggy_binary_protocol::WireIdentifier; use iggy_binary_protocol::codec::{ - WireDecode, WireEncode, capped_capacity, read_u32_le, read_u64_le, read_u128_le, + WireDecode, WireEncode, bounded_capacity, read_u32_le, read_u64_le, read_u128_le, }; use iggy_binary_protocol::requests::consumer_groups::{ CreateConsumerGroupRequest, DeleteConsumerGroupRequest, @@ -435,7 +435,7 @@ impl WireDecode for JoinConsumerGroupRequest { // corrupt/bit-rotted count (near u32::MAX) allocate gigabytes and abort // every backup that applies it. let mut in_flight = - Vec::with_capacity(capped_capacity(count, buf.len().saturating_sub(pos), 4)); + Vec::with_capacity(bounded_capacity(count, buf.len().saturating_sub(pos), 4)); for _ in 0..count { in_flight.push(read_u32_le(buf, pos)?); pos += 4; @@ -947,12 +947,12 @@ impl ConsumerGroupSnapshot { #[cfg(test)] mod tests { use super::*; - use iggy_binary_protocol::WireName; use iggy_binary_protocol::primitives::partition_assignment::CreatedPartitionAssignment; use iggy_binary_protocol::requests::streams::CreateStreamRequest; use iggy_binary_protocol::requests::topics::{ CreateTopicRequest, CreateTopicWithAssignmentsRequest, }; + use iggy_binary_protocol::{WireName, WireOptions}; use iggy_common::IggyTimestamp; // Groups co-locate in the topic node, so an apply resolves its parent through @@ -963,6 +963,7 @@ mod tests { let _ = StateHandler::apply( &CreateStreamRequest { name: WireName::new("stream").unwrap(), + options: WireOptions::empty(), }, &mut inner, IggyTimestamp::now(), @@ -971,12 +972,10 @@ mod tests { request: CreateTopicRequest { stream_id: WireIdentifier::numeric(0), partitions_count: 1, - compression_algorithm: 0, - message_expiry: 0, - max_topic_size: 0, - replication_factor: 1, name: WireName::new("topic").unwrap(), + options: WireOptions::empty(), }, + derived_options: WireOptions::empty(), partitions: vec![CreatedPartitionAssignment { partition_id: 0, consensus_group_id: 1, diff --git a/core/metadata/src/stm/result.rs b/core/metadata/src/stm/result.rs index 6a610bf3a5..a1be91ff64 100644 --- a/core/metadata/src/stm/result.rs +++ b/core/metadata/src/stm/result.rs @@ -146,10 +146,14 @@ macro_rules! result_enum { } // Streams. -result_enum!(CreateStreamResult { NameAlreadyExists = 1012 }); +result_enum!(CreateStreamResult { + NameAlreadyExists = 1012, + InvalidOptionValue = 4042, +}); result_enum!(UpdateStreamResult { StreamNotFound = 1009, NameAlreadyExists = 1012, + InvalidOptionValue = 4042, }); result_enum!(DeleteStreamResult { StreamNotFound = 1009 }); result_enum!(PurgeStreamResult { StreamNotFound = 1009 }); @@ -158,11 +162,13 @@ result_enum!(PurgeStreamResult { StreamNotFound = 1009 }); result_enum!(CreateTopicResult { StreamNotFound = 1009, NameAlreadyExists = 2013, + InvalidOptionValue = 4042, }); result_enum!(UpdateTopicResult { StreamNotFound = 1009, TopicNotFound = 2010, NameAlreadyExists = 2013, + InvalidOptionValue = 4042, }); result_enum!(DeleteTopicResult { StreamNotFound = 1009, @@ -199,11 +205,13 @@ result_enum!(TruncatePartitionResult { result_enum!(CreateUserResult { InvalidUsername = 43, UserAlreadyExists = 46, + InvalidOptionValue = 4042, }); result_enum!(UpdateUserResult { UserNotFound = 20, InvalidUsername = 43, UsernameAlreadyExists = 46, + InvalidOptionValue = 4042, }); result_enum!(DeleteUserResult { UserNotFound = 20, diff --git a/core/metadata/src/stm/snapshot.rs b/core/metadata/src/stm/snapshot.rs index 26b28b660a..89ef62b878 100644 --- a/core/metadata/src/stm/snapshot.rs +++ b/core/metadata/src/stm/snapshot.rs @@ -44,7 +44,15 @@ use crate::stm::user::UsersSnapshot; /// Version 2: `status` sits at reply-header offset 216 (version 1 carried a /// `namespace` word before it), which the client table's cached replies embed as raw /// wire bytes msgpack cannot introspect. -pub const SNAPSHOT_FORMAT_VERSION: u32 = 2; +/// +/// Version 3: `TopicSnapshot` dropped `replication_factor` from the middle of the +/// struct and gained `options`. Both changes land in the same element count, so a +/// version 2 file decoded under this shape does not fail cleanly: msgpack reads the +/// old `replication_factor` byte as `message_expiry` and walks every later field one +/// position out of place. The bump is what turns that into +/// `UnsupportedFormatVersion` instead of an opaque deserializer error, or worse a +/// silent misread. +pub const SNAPSHOT_FORMAT_VERSION: u32 = 3; /// The release that wrote a snapshot: the packed `iggy_binary_protocol` semver of /// this build. [`iggy_binary_protocol::ProtocolVersion`] documents the packing and @@ -521,7 +529,10 @@ macro_rules! impl_fill_restore { mod tests { use super::*; use crate::stm::stream::{PartitionSnapshot, StatsSnapshot, StreamSnapshot, TopicSnapshot}; - use iggy_common::{CompressionAlgorithm, IggyExpiry, IggyTimestamp, MaxTopicSize}; + use crate::stm::user::UserSnapshot; + use iggy_common::{ + CompressionAlgorithm, IggyExpiry, IggyTimestamp, MaxTopicSize, ResourceOptions, + }; #[test] fn test_metadata_snapshot_roundtrip() { @@ -547,7 +558,7 @@ mod tests { // operator's boot reading one field's bytes as another's. Changing either // number is the reminder to change the other. const FIELD_COUNT: u32 = 7; - const PINNED_VERSION: u32 = 2; + const PINNED_VERSION: u32 = 3; let encoded = MetadataSnapshot::new(0).encode().unwrap(); let mut cursor = encoded.as_slice(); @@ -562,6 +573,77 @@ mod tests { ); } + #[test] + fn nested_snapshot_shapes_are_pinned_too() { + // The top-level count above is blind to everything under it, which is how + // `TopicSnapshot` once swapped a removed field for an added one and kept + // the same element count: a version 2 file then decoded field-by-field + // one position out of place instead of failing. The types the STM + // actually grows need their own pins. + const TOPIC_FIELD_COUNT: u32 = 11; + const STREAM_FIELD_COUNT: u32 = 6; + const USER_FIELD_COUNT: u32 = 7; + + let topic = TopicSnapshot { + id: 0, + name: String::new(), + created_at: IggyTimestamp::default(), + message_expiry: IggyExpiry::default(), + compression_algorithm: CompressionAlgorithm::default(), + max_topic_size: MaxTopicSize::default(), + stats: StatsSnapshot { + size_bytes: 0, + messages_count: 0, + segments_count: 0, + }, + partitions: Vec::new(), + consumer_groups: Vec::new(), + next_consumer_group_id: 0, + options: ResourceOptions::new(), + }; + let encoded = rmp_serde::to_vec(&topic).unwrap(); + assert_eq!( + rmp::decode::read_array_len(&mut encoded.as_slice()).unwrap(), + TOPIC_FIELD_COUNT, + "TopicSnapshot's field count changed; bump SNAPSHOT_FORMAT_VERSION with it" + ); + + let stream = StreamSnapshot { + id: 0, + name: String::new(), + created_at: IggyTimestamp::default(), + stats: StatsSnapshot { + size_bytes: 0, + messages_count: 0, + segments_count: 0, + }, + topics: Vec::new(), + options: ResourceOptions::new(), + }; + let encoded = rmp_serde::to_vec(&stream).unwrap(); + assert_eq!( + rmp::decode::read_array_len(&mut encoded.as_slice()).unwrap(), + STREAM_FIELD_COUNT, + "StreamSnapshot's field count changed; bump SNAPSHOT_FORMAT_VERSION with it" + ); + + let user = crate::stm::user::UserSnapshot { + id: 0, + username: String::new(), + password_hash: String::new(), + status: iggy_common::UserStatus::Active, + created_at: IggyTimestamp::default(), + permissions: None, + options: ResourceOptions::new(), + }; + let encoded = rmp_serde::to_vec(&user).unwrap(); + assert_eq!( + rmp::decode::read_array_len(&mut encoded.as_slice()).unwrap(), + USER_FIELD_COUNT, + "UserSnapshot's field count changed; bump SNAPSHOT_FORMAT_VERSION with it" + ); + } + #[test] fn a_build_decodes_what_it_writes() { // Exact-equality versioning makes this the one thing that could go wrong @@ -685,7 +767,6 @@ mod tests { id: 0, name: "topic".to_string(), created_at: ts, - replication_factor: 1, message_expiry: IggyExpiry::default(), compression_algorithm: CompressionAlgorithm::default(), max_topic_size: MaxTopicSize::default(), @@ -707,8 +788,10 @@ mod tests { // roundtrip assert below proves the field survives // instead of matching a default. next_consumer_group_id: 5, + options: ResourceOptions::default(), }, )], + options: ResourceOptions::default(), }, )], }); @@ -738,38 +821,39 @@ mod tests { assert_eq!(topic.next_consumer_group_id, 5); } + fn user_snapshot_fixture(id: u32, username: &str, password_hash: &str) -> UserSnapshot { + use iggy_common::UserStatus; + UserSnapshot { + id, + username: username.to_string(), + password_hash: password_hash.to_string(), + status: UserStatus::Active, + created_at: IggyTimestamp::from(1_694_968_446_131_680_u64), + permissions: None, + options: ResourceOptions::default(), + } + } + + fn stream_snapshot_fixture(id: usize, name: &str, stats: StatsSnapshot) -> StreamSnapshot { + StreamSnapshot { + id, + name: name.to_string(), + created_at: IggyTimestamp::from(1_694_968_446_131_680_u64), + stats, + topics: vec![], + options: ResourceOptions::default(), + } + } + #[test] fn roundtrip_with_slab_gaps() { use crate::stm::stream::StreamsSnapshot; - use crate::stm::user::{PermissionerSnapshot, UserSnapshot, UsersSnapshot}; - use iggy_common::UserStatus; - - let ts = IggyTimestamp::from(1_694_968_446_131_680_u64); + use crate::stm::user::{PermissionerSnapshot, UsersSnapshot}; let users_snap = UsersSnapshot { items: vec![ - ( - 0, - UserSnapshot { - id: 0, - username: "alice".to_string(), - password_hash: "hash_a".to_string(), - status: UserStatus::Active, - created_at: ts, - permissions: None, - }, - ), - ( - 2, - UserSnapshot { - id: 2, - username: "charlie".to_string(), - password_hash: "hash_c".to_string(), - status: UserStatus::Active, - created_at: ts, - permissions: None, - }, - ), + (0, user_snapshot_fixture(0, "alice", "hash_a")), + (2, user_snapshot_fixture(2, "charlie", "hash_c")), ], personal_access_tokens: vec![], permissioner: PermissionerSnapshot { @@ -787,31 +871,27 @@ mod tests { items: vec![ ( 0, - StreamSnapshot { - id: 0, - name: "stream-0".to_string(), - created_at: ts, - stats: StatsSnapshot { + stream_snapshot_fixture( + 0, + "stream-0", + StatsSnapshot { size_bytes: 100, messages_count: 10, segments_count: 1, }, - topics: vec![], - }, + ), ), ( 3, - StreamSnapshot { - id: 3, - name: "stream-3".to_string(), - created_at: ts, - stats: StatsSnapshot { + stream_snapshot_fixture( + 3, + "stream-3", + StatsSnapshot { size_bytes: 200, messages_count: 20, segments_count: 2, }, - topics: vec![], - }, + ), ), ], }; diff --git a/core/metadata/src/stm/stream.rs b/core/metadata/src/stm/stream.rs index e4dfc1b12c..e1a7b3572f 100644 --- a/core/metadata/src/stm/stream.rs +++ b/core/metadata/src/stm/stream.rs @@ -34,6 +34,8 @@ use iggy_binary_protocol::codec::{WireDecode, WireEncode}; // keep the imports under the same gate so a production build does not see them // as unused. The test module re-imports them independently. #[cfg(any(test, feature = "simulator"))] +use iggy_binary_protocol::primitives::options::WireOptions; +#[cfg(any(test, feature = "simulator"))] use iggy_binary_protocol::primitives::partition_assignment::CreatedPartitionAssignment; use iggy_binary_protocol::requests::consumer_groups::{ CreateConsumerGroupRequest, DeleteConsumerGroupRequest, @@ -58,9 +60,10 @@ use iggy_binary_protocol::responses::streams::StreamResponse; use iggy_binary_protocol::responses::streams::get_stream::{GetStreamResponse, TopicHeader}; use iggy_binary_protocol::responses::topics::get_topic::PartitionResponse; use iggy_binary_protocol::{WireIdentifier, WireName}; +use iggy_common::wire_conversions::{resource_options_from_wire, resource_options_to_wire_split}; use iggy_common::{ - CompressionAlgorithm, IggyExpiry, IggyTimestamp, MaxTopicSize, PartitionStats, StreamStats, - TopicStats, + CompressionAlgorithm, IggyByteSize, IggyExpiry, IggyTimestamp, MaxTopicSize, PartitionStats, + ResourceOptions, StreamStats, TopicCreateOptions, TopicRuntimeOptions, TopicStats, }; use serde::{Deserialize, Serialize}; use server_common::sharding::IggyNamespace; @@ -144,7 +147,6 @@ pub struct TopicSnapshot { pub id: usize, pub name: String, pub created_at: IggyTimestamp, - pub replication_factor: u8, pub message_expiry: IggyExpiry, pub compression_algorithm: CompressionAlgorithm, pub max_topic_size: MaxTopicSize, @@ -158,6 +160,8 @@ pub struct TopicSnapshot { pub consumer_groups: Vec<(u64, ConsumerGroupSnapshot)>, #[serde(default)] pub next_consumer_group_id: u64, + #[serde(default)] + pub options: ResourceOptions, } #[derive(Debug, Clone)] @@ -165,10 +169,13 @@ pub struct Topic { pub id: usize, pub name: Arc, pub created_at: IggyTimestamp, - pub replication_factor: u8, pub message_expiry: IggyExpiry, pub compression_algorithm: CompressionAlgorithm, pub max_topic_size: MaxTopicSize, + /// Resolved creation options: the client's explicit keys plus the + /// defaults derived at admission. `partitions_count` is never stored + /// here; the partitions vec is the authority. + pub options: ResourceOptions, pub stats: Arc, pub partitions: Vec, @@ -197,10 +204,10 @@ impl Default for Topic { id: 0, name: Arc::from(""), created_at: IggyTimestamp::default(), - replication_factor: 1, message_expiry: IggyExpiry::default(), compression_algorithm: CompressionAlgorithm::default(), max_topic_size: MaxTopicSize::default(), + options: ResourceOptions::new(), stats: Arc::new(TopicStats::default()), partitions: Vec::new(), round_robin_counter: Arc::new(AtomicUsize::new(0)), @@ -215,7 +222,6 @@ impl Topic { pub fn new( name: Arc, created_at: IggyTimestamp, - replication_factor: u8, message_expiry: IggyExpiry, compression_algorithm: CompressionAlgorithm, max_topic_size: MaxTopicSize, @@ -225,10 +231,10 @@ impl Topic { id: 0, name, created_at, - replication_factor, message_expiry, compression_algorithm, max_topic_size, + options: ResourceOptions::new(), stats: Arc::new(TopicStats::new(stream_stats)), partitions: Vec::new(), round_robin_counter: Arc::new(AtomicUsize::new(0)), @@ -275,6 +281,8 @@ pub struct StreamSnapshot { pub created_at: IggyTimestamp, pub stats: StatsSnapshot, pub topics: Vec<(usize, TopicSnapshot)>, + #[serde(default)] + pub options: ResourceOptions, } #[derive(Debug)] @@ -282,6 +290,7 @@ pub struct Stream { pub id: usize, pub name: Arc, pub created_at: IggyTimestamp, + pub options: ResourceOptions, pub stats: Arc, pub topics: Slab, @@ -294,6 +303,7 @@ impl Default for Stream { id: 0, name: Arc::from(""), created_at: IggyTimestamp::default(), + options: ResourceOptions::default(), stats: Arc::new(StreamStats::default()), topics: Slab::new(), topic_index: AHashMap::default(), @@ -307,6 +317,7 @@ impl Clone for Stream { id: self.id, name: self.name.clone(), created_at: self.created_at, + options: self.options.clone(), stats: self.stats.clone(), topics: self.topics.clone(), topic_index: self.topic_index.clone(), @@ -321,6 +332,7 @@ impl Stream { id: 0, name, created_at, + options: ResourceOptions::new(), stats: Arc::new(StreamStats::default()), topics: Slab::new(), topic_index: AHashMap::default(), @@ -333,6 +345,7 @@ impl Stream { id: 0, name, created_at, + options: ResourceOptions::new(), stats, topics: Slab::new(), topic_index: AHashMap::default(), @@ -870,6 +883,26 @@ impl Streams { }) } + /// A topic's explicitly set segment size, or `None` when the stream or + /// topic is unknown or the topic left the key to the node default. + /// + /// Update admission reads it for the floor `max_topic_size` has to clear: + /// `segment_size` is create-only, so the stored value is the one every one + /// of the topic's partitions is already rotating at. + #[must_use] + pub fn topic_segment_size( + &self, + stream_id: &WireIdentifier, + topic_id: &WireIdentifier, + ) -> Option { + self.inner.read(|inner| { + let stream_slab = inner.resolve_stream_id(stream_id)?; + let topic_slab = inner.resolve_topic_id(stream_slab, topic_id)?; + let topic = inner.items.get(stream_slab)?.topics.get(topic_slab)?; + TopicRuntimeOptions::from_resource_options(&topic.options).segment_size + }) + } + /// Build the `ConsumerGroupDetailsResponse` for a group (members + their /// round-robin partition assignment). `partitions_count` is the topic's /// total partition count. `None` if the stream/topic/group is unknown. @@ -1328,6 +1361,7 @@ impl Streams { CreateStreamRequest { name: WireName::new(format!("sim-stream-{slab}")) .expect("sim stream name is valid"), + options: WireOptions::empty(), }, IggyTimestamp::from(1), )) @@ -1368,13 +1402,11 @@ impl Streams { stream_id: stream_wire.clone(), partitions_count: u32::try_from(partitions.len()) .expect("sim partition count fits u32"), - compression_algorithm: 0, - message_expiry: 0, - max_topic_size: 0, - replication_factor: 1, name: WireName::new(format!("sim-topic-{stream_slab}-{slab}")) .expect("sim topic name is valid"), + options: WireOptions::empty(), }, + derived_options: WireOptions::empty(), partitions, }, IggyTimestamp::from(1), @@ -1495,6 +1527,9 @@ impl StateHandler for CreateStreamRequest { if state.index.contains_key(&name_arc) { return ApplyReply::err(CreateStreamResult::NameAlreadyExists); } + let Ok(options) = resource_options_from_wire(&self.options, true) else { + return ApplyReply::err(CreateStreamResult::InvalidOptionValue); + }; // Share one `Arc` across both left-right buffers via the // registry (see `StatsRegistry`). The id the next insert will use is @@ -1505,6 +1540,7 @@ impl StateHandler for CreateStreamRequest { id, name: name_arc.clone(), created_at: timestamp, + options, stats: stream_stats, topics: Slab::new(), topic_index: AHashMap::default(), @@ -1525,6 +1561,7 @@ impl StateHandler for CreateStreamRequest { size_bytes: 0, messages_count: 0, name: self.name.clone(), + options: self.options.clone(), }, topics: Vec::new(), } @@ -1550,8 +1587,17 @@ impl StateHandler for UpdateStreamRequest { return ApplyReply::err(UpdateStreamResult::NameAlreadyExists); } + // Decoded before any mutation: a malformed block must leave the stream + // untouched rather than half-renamed. + let Ok(updated_options) = resource_options_from_wire(&self.options, true) else { + return ApplyReply::err(UpdateStreamResult::InvalidOptionValue); + }; + state.index.remove(&stream.name); stream.name = new_name_arc.clone(); + // Patch, never replace: keys the client did not send keep their + // current value, so a client that predates a key cannot erase it. + stream.options.extend(updated_options); state.index.insert(new_name_arc, stream_id); ApplyReply::ok(Bytes::new()) } @@ -1635,6 +1681,42 @@ impl StateHandler for CreateTopicWithAssignmentsRequest { } } + // Both blocks were validated and resolved at admission, so apply reads + // them leniently: a key this build does not know is skipped, not + // refused. Refusing would make the verdict depend on the build, and a + // replica that predates a key would then be missing a topic its peers + // committed. Decoded before the revision bump and the stats-registry + // insert below, so a block this build cannot read at all leaves no + // orphaned registry entry and wakes no reconciler. + let explicit = TopicCreateOptions::parse_committed(&self.request.options); + let derived = TopicCreateOptions::parse_committed(&self.derived_options); + let (Ok(explicit_map), Ok(derived_map)) = ( + resource_options_from_wire(&self.request.options, true), + resource_options_from_wire(&self.derived_options, false), + ) else { + return ApplyReply::err(CreateTopicResult::InvalidOptionValue); + }; + let resolved = explicit.resolved_over(&derived); + let Ok(resolved_map) = resolved.to_option_map() else { + return ApplyReply::err(CreateTopicResult::InvalidOptionValue); + }; + + // Explicit wins on collision. `partitions_count` cannot appear here at + // all: it is a fixed field of the command, not an option key. + let mut options = derived_map; + options.extend(explicit_map); + // A client may send the literal server-default sentinel (0) for a typed + // key. The typed fields normalize it to absent and resolve the node + // default, so the map has to report the resolved value too: otherwise + // the merge above drops the derived entry and one `GetTopic` response + // carries the resolved value in the fixed field and 0 in the options + // block. Provenance is left alone - the client did name the key. + for (key, resolved_value) in resolved_map { + if let Some(option) = options.get_mut(&key) { + option.value = resolved_value.value; + } + } + // Past validation: this commit adds partitions, so bump the // monotonic revision and stamp every new partition with it. let new_revision = state.revision.wrapping_add(1); @@ -1659,23 +1741,16 @@ impl StateHandler for CreateTopicWithAssignmentsRequest { return ApplyReply::err(CreateTopicResult::StreamNotFound); }; - let replication_factor = if self.request.replication_factor == 0 { - 1 - } else { - self.request.replication_factor - }; - let topic = Topic { id: topic_id, name: name_arc.clone(), created_at: timestamp, - replication_factor, - message_expiry: IggyExpiry::from(self.request.message_expiry), - compression_algorithm: CompressionAlgorithm::from_code( - self.request.compression_algorithm, - ) - .unwrap_or_default(), - max_topic_size: MaxTopicSize::from(self.request.max_topic_size), + message_expiry: resolved.message_expiry.unwrap_or(IggyExpiry::ServerDefault), + compression_algorithm: resolved.compression_algorithm.unwrap_or_default(), + max_topic_size: resolved + .max_topic_size + .unwrap_or(MaxTopicSize::ServerDefault), + options, stats: topic_stats, partitions: Vec::new(), round_robin_counter: Arc::new(AtomicUsize::new(0)), @@ -1705,7 +1780,11 @@ impl StateHandler for CreateTopicWithAssignmentsRequest { let Some(topic) = stream.topics.get(topic_id) else { return ApplyReply::err(CreateTopicResult::StreamNotFound); }; - ApplyReply::ok(encode_create_topic_reply(&self.request, topic_id, topic)) + ApplyReply::ok(encode_create_topic_reply( + &self.request.name, + topic_id, + topic, + )) } } @@ -1714,28 +1793,28 @@ impl StateHandler for CreateTopicWithAssignmentsRequest { /// reply deserializes without a schema break. Returns empty bytes on a /// `u32` overflow (same contract as a validation rejection) rather than /// saturating to `u32::MAX`. -fn encode_create_topic_reply( - request: &CreateTopicRequest, - topic_id: usize, - topic: &Topic, -) -> Bytes { +fn encode_create_topic_reply(name: &WireName, topic_id: usize, topic: &Topic) -> Bytes { let Ok(topic_id_u32) = u32::try_from(topic_id) else { return Bytes::new(); }; let Ok(partitions_count_u32) = u32::try_from(topic.partitions.len()) else { return Bytes::new(); }; + let Ok((options, derived_options)) = resource_options_to_wire_split(&topic.options) else { + return Bytes::new(); + }; let header = TopicHeader { id: topic_id_u32, created_at: topic.created_at.into(), partitions_count: partitions_count_u32, - message_expiry: request.message_expiry, - compression_algorithm: request.compression_algorithm, - max_topic_size: request.max_topic_size, - replication_factor: topic.replication_factor, + message_expiry: u64::from(topic.message_expiry), + compression_algorithm: topic.compression_algorithm.as_code(), + max_topic_size: u64::from(topic.max_topic_size), size_bytes: 0, messages_count: 0, - name: request.name.clone(), + name: name.clone(), + options, + derived_options, }; let Ok(partitions_resp) = topic .partitions @@ -1793,15 +1872,33 @@ impl StateHandler for UpdateTopicRequest { return ApplyReply::err(UpdateTopicResult::NameAlreadyExists); } + // Decoded before any mutation: a malformed block must leave the topic + // untouched rather than half-renamed. + let Ok(updated_options) = resource_options_from_wire(&self.options, true) else { + return ApplyReply::err(UpdateTopicResult::InvalidOptionValue); + }; + // Read leniently, like every other committed op: a key this build does + // not know is skipped rather than failing an operation its peers + // accepted. + let updated = TopicCreateOptions::parse_committed(&self.options); + stream.topic_index.remove(&topic.name); topic.name = new_name_arc.clone(); - topic.compression_algorithm = - CompressionAlgorithm::from_code(self.compression_algorithm).unwrap_or_default(); - topic.message_expiry = IggyExpiry::from(self.message_expiry); - topic.max_topic_size = MaxTopicSize::from(self.max_topic_size); - if self.replication_factor != 0 { - topic.replication_factor = self.replication_factor; + // Settings arrive only through the options block now, so the typed + // fields are a projection of it and cannot drift. Absent means absent: + // a client that sends just a rename leaves every setting alone, and one + // built before a key existed cannot erase it. + if let Some(compression_algorithm) = updated.compression_algorithm { + topic.compression_algorithm = compression_algorithm; + } + if let Some(message_expiry) = updated.message_expiry { + topic.message_expiry = message_expiry; } + if let Some(max_topic_size) = updated.max_topic_size { + topic.max_topic_size = max_topic_size; + } + // Patch, never replace, for the stored map too. + topic.options.extend(updated_options); stream.topic_index.insert(new_name_arc, topic_id); ApplyReply::ok(Bytes::new()) } @@ -2037,10 +2134,10 @@ impl Snapshotable for Streams { id: topic.id, name: topic.name.to_string(), created_at: topic.created_at, - replication_factor: topic.replication_factor, message_expiry: topic.message_expiry, compression_algorithm: topic.compression_algorithm, max_topic_size: topic.max_topic_size, + options: topic.options.clone(), stats: StatsSnapshot { size_bytes: t_size, messages_count: t_msgs, @@ -2082,6 +2179,7 @@ impl Snapshotable for Streams { segments_count, }, topics, + options: stream.options.clone(), }, ) }) @@ -2158,10 +2256,10 @@ impl StreamsInner { id: topic_snap.id, name: topic_name.clone(), created_at: topic_snap.created_at, - replication_factor: topic_snap.replication_factor, message_expiry: topic_snap.message_expiry, compression_algorithm: topic_snap.compression_algorithm, max_topic_size: topic_snap.max_topic_size, + options: topic_snap.options, stats: topic_stats, partitions: topic_snap .partitions @@ -2207,6 +2305,7 @@ impl StreamsInner { id: stream_snap.id, name: stream_name.clone(), created_at: stream_snap.created_at, + options: stream_snap.options, stats: stream_stats, topics, topic_index, @@ -2269,10 +2368,168 @@ mod tests { fn create_stream(inner: &mut StreamsInner, name: &str) { let request = CreateStreamRequest { name: WireName::new(name).unwrap(), + options: WireOptions::empty(), }; let _ = StateHandler::apply(&request, inner, IggyTimestamp::now()); } + #[test] + fn create_topic_stores_merged_options_and_typed_fields() { + use iggy_common::{HeaderKey, HeaderKind, TopicCreateOptions, topic_option_keys}; + use std::str::FromStr; + + let mut inner = StreamsInner::new(); + create_stream(&mut inner, "s"); + + // Client explicitly pins message_expiry; admission derives the rest. + let explicit = TopicCreateOptions { + message_expiry: Some(IggyExpiry::from(5_000_000u64)), + partitions_count: Some(1), + ..TopicCreateOptions::default() + }; + let derived = TopicCreateOptions { + max_topic_size: Some(MaxTopicSize::from(10_000_000_000u64)), + compression_algorithm: Some(CompressionAlgorithm::None), + ..TopicCreateOptions::default() + }; + let request = CreateTopicWithAssignmentsRequest { + request: WireCreateTopicRequest { + stream_id: WireIdentifier::numeric(0), + partitions_count: 1, + name: WireName::new("t").unwrap(), + options: explicit.to_wire().unwrap(), + }, + derived_options: derived.to_wire().unwrap(), + partitions: vec![CreatedPartitionAssignment { + partition_id: 0, + consensus_group_id: 1, + }], + }; + let reply = StateHandler::apply(&request, &mut inner, IggyTimestamp::from(1)); + assert_eq!(reply.code, 0, "create must succeed"); + + let stream = inner.items.get(0).unwrap(); + let (_, topic) = stream.topics.iter().next().unwrap(); + assert_eq!(topic.message_expiry, IggyExpiry::from(5_000_000u64)); + assert_eq!(topic.max_topic_size, MaxTopicSize::from(10_000_000_000u64)); + assert_eq!(topic.compression_algorithm, CompressionAlgorithm::None); + + // partitions_count is create-consumed, never persisted. + assert_eq!(topic.options.len(), 3); + let expiry_key = HeaderKey::from_str(topic_option_keys::MESSAGE_EXPIRY).unwrap(); + let expiry = topic.options.get(&expiry_key).unwrap(); + assert!(expiry.explicit, "client-sent key keeps its provenance"); + assert_eq!(expiry.value.kind(), HeaderKind::Uint64); + let size_key = HeaderKey::from_str(topic_option_keys::MAX_TOPIC_SIZE).unwrap(); + assert!( + !topic.options.get(&size_key).unwrap().explicit, + "derived key is marked derived" + ); + } + + /// A client may send the literal 0 that means "resolve the default". The + /// merge lets that explicit entry win over the derived one, so the map has + /// to be rewritten from the resolved value: otherwise the persisted map + /// reports 0 while the typed field reports the resolved default, and the + /// single `GetTopic` response contradicts itself. + #[test] + fn create_topic_replaces_a_client_sent_sentinel_with_the_resolved_default() { + use iggy_common::{HeaderKey, TopicCreateOptions, topic_option_keys}; + use std::str::FromStr; + + let mut inner = StreamsInner::new(); + create_stream(&mut inner, "s"); + + let explicit = TopicCreateOptions { + max_topic_size: Some(MaxTopicSize::from(0u64)), + ..TopicCreateOptions::default() + }; + let derived = TopicCreateOptions { + max_topic_size: Some(MaxTopicSize::from(10_000_000_000u64)), + ..TopicCreateOptions::default() + }; + // `to_wire` normalizes the sentinel away, so the block is hand-built to + // carry the literal 0 the CLI's `--set max_topic_size=server_default` + // puts on the wire through the raw map. + let mut sentinel = bytes::BytesMut::new(); + iggy_binary_protocol::primitives::user_headers::encode_user_headers( + &[( + 2, + topic_option_keys::MAX_TOPIC_SIZE.as_bytes(), + 12, + &0u64.to_le_bytes(), + )], + &mut sentinel, + ); + let request = CreateTopicWithAssignmentsRequest { + request: WireCreateTopicRequest { + stream_id: WireIdentifier::numeric(0), + partitions_count: 1, + name: WireName::new("t").unwrap(), + options: WireOptions::from_bytes(sentinel.freeze()).unwrap(), + }, + derived_options: derived.to_wire().unwrap(), + partitions: vec![CreatedPartitionAssignment { + partition_id: 0, + consensus_group_id: 1, + }], + }; + assert!(explicit.max_topic_size.is_some(), "sentinel was sent"); + + let reply = StateHandler::apply(&request, &mut inner, IggyTimestamp::from(1)); + assert_eq!(reply.code, 0, "create must succeed"); + + let stream = inner.items.get(0).unwrap(); + let (_, topic) = stream.topics.iter().next().unwrap(); + assert_eq!(topic.max_topic_size, MaxTopicSize::from(10_000_000_000u64)); + let size_key = HeaderKey::from_str(topic_option_keys::MAX_TOPIC_SIZE).unwrap(); + let stored = topic.options.get(&size_key).unwrap(); + assert_eq!( + stored.value.as_bytes(), + &10_000_000_000u64.to_le_bytes(), + "the map must carry the resolved value, not the sentinel" + ); + } + + #[test] + fn create_stream_options_survive_snapshot_roundtrip() { + use crate::stm::snapshot::FillSnapshot; + use iggy_binary_protocol::primitives::user_headers::encode_user_headers; + + let mut headers = bytes::BytesMut::new(); + encode_user_headers(&[(2, b"future_key", 2, b"future_value")], &mut headers); + let request = CreateStreamRequest { + name: WireName::new("stream-with-options").unwrap(), + options: WireOptions::from_bytes(headers.freeze()).unwrap(), + }; + + let streams = Streams::default(); + streams + .inner + .try_apply(StreamsCommand::CreateStream( + request, + IggyTimestamp::from(1), + )) + .expect("create stream applies"); + + let mut snapshot = MetadataSnapshot::new(1); + streams.fill_snapshot(&mut snapshot).unwrap(); + let encoded = snapshot.encode().unwrap(); + let decoded = MetadataSnapshot::decode(&encoded).unwrap(); + let restored: Streams = crate::stm::snapshot::RestoreSnapshot::restore_snapshot(&decoded) + .expect("streams section restores"); + + let restored_options = restored.read(|inner| { + let (_, stream) = inner.items.iter().next().expect("stream restored"); + stream.options.clone() + }); + assert_eq!(restored_options.len(), 1); + let (key, option) = restored_options.iter().next().unwrap(); + assert_eq!(key.as_bytes(), b"future_key"); + assert_eq!(option.value.as_bytes(), b"future_value"); + assert!(option.explicit); + } + fn make_topic_request( stream_id: u32, partitions_count: u32, @@ -2281,11 +2538,8 @@ mod tests { WireCreateTopicRequest { stream_id: WireIdentifier::numeric(stream_id), partitions_count, - compression_algorithm: 0, - message_expiry: 0, - max_topic_size: 0, - replication_factor: 1, name: WireName::new(name).unwrap(), + options: WireOptions::empty(), } } @@ -2307,6 +2561,7 @@ mod tests { for topic_name in ["logs", "events"] { let create_topic = CreateTopicWithAssignmentsRequest { request: make_topic_request(stream_id, 2, topic_name), + derived_options: WireOptions::empty(), partitions: vec![ CreatedPartitionAssignment { partition_id: 0, @@ -2374,6 +2629,7 @@ mod tests { create_stream(&mut inner, "stream"); let create_topic = CreateTopicWithAssignmentsRequest { request: make_topic_request(0, 2, "topic"), + derived_options: WireOptions::empty(), partitions: vec![ CreatedPartitionAssignment { partition_id: 0, @@ -2401,6 +2657,7 @@ mod tests { create_stream(&mut inner, "stream"); let create_topic = CreateTopicWithAssignmentsRequest { request: make_topic_request(0, 2, "topic"), + derived_options: WireOptions::empty(), partitions: vec![ CreatedPartitionAssignment { partition_id: 0, @@ -2455,6 +2712,7 @@ mod tests { create_stream(&mut inner, "stream"); let create_topic = CreateTopicWithAssignmentsRequest { request: make_topic_request(0, 2, "topic"), + derived_options: WireOptions::empty(), partitions: vec![ CreatedPartitionAssignment { partition_id: 0, @@ -2488,6 +2746,7 @@ mod tests { create_stream(&mut inner, "stream"); let create_topic = CreateTopicWithAssignmentsRequest { request: make_topic_request(0, 2, "topic"), + derived_options: WireOptions::empty(), partitions: vec![ CreatedPartitionAssignment { partition_id: 0, @@ -2582,6 +2841,7 @@ mod tests { create_stream(&mut inner, "stream"); let create_topic = CreateTopicWithAssignmentsRequest { request: make_topic_request(0, partitions_count, "topic"), + derived_options: WireOptions::empty(), partitions: (0..partitions_count) .map(|partition_id| CreatedPartitionAssignment { partition_id, @@ -2636,6 +2896,7 @@ mod tests { create_stream(&mut inner, "stream"); let create_topic = CreateTopicWithAssignmentsRequest { request: make_topic_request(0, 1, "topic"), + derived_options: WireOptions::empty(), partitions: vec![CreatedPartitionAssignment { partition_id: 0, consensus_group_id: 1, @@ -2734,6 +2995,7 @@ mod tests { let mut inner = inner_with_registered_partition(); let create_topic = CreateTopicWithAssignmentsRequest { request: make_topic_request(0, 1, "metrics"), + derived_options: WireOptions::empty(), partitions: vec![CreatedPartitionAssignment { partition_id: 0, consensus_group_id: 2, @@ -2831,6 +3093,7 @@ mod tests { create_stream(&mut inner, "alpha"); let create_topic = CreateTopicWithAssignmentsRequest { request: make_topic_request(0, 1, "logs"), + derived_options: WireOptions::empty(), partitions: vec![CreatedPartitionAssignment { partition_id: 0, consensus_group_id: 1, @@ -2918,6 +3181,7 @@ mod tests { let body = CreateStreamRequest { name: WireName::new(name).unwrap(), + options: WireOptions::empty(), } .to_bytes(); let header_size = size_of::(); @@ -2943,6 +3207,7 @@ mod tests { create_stream(&mut inner, "alpha"); let create_topic = CreateTopicWithAssignmentsRequest { request: make_topic_request(0, 1, "logs"), + derived_options: WireOptions::empty(), partitions: vec![CreatedPartitionAssignment { partition_id: 0, consensus_group_id: 1, diff --git a/core/metadata/src/stm/user.rs b/core/metadata/src/stm/user.rs index 00011b8797..086627eebe 100644 --- a/core/metadata/src/stm/user.rs +++ b/core/metadata/src/stm/user.rs @@ -34,11 +34,12 @@ use iggy_binary_protocol::requests::users::{ }; use iggy_binary_protocol::responses::users::get_user::UserDetailsResponse; use iggy_binary_protocol::responses::users::user_response::UserResponse; -use iggy_binary_protocol::{WireIdentifier, WireName}; +use iggy_binary_protocol::{WireIdentifier, WireName, WireOptions}; use iggy_common::defaults::{DEFAULT_ROOT_USER_ID, MAX_USERNAME_LENGTH, MIN_USERNAME_LENGTH}; +use iggy_common::wire_conversions::resource_options_from_wire; use iggy_common::{ GlobalPermissions, IggyError, IggyExpiry, IggyTimestamp, Permissions, PersonalAccessToken, - StreamPermissions, UserId, UserStatus, + ResourceOptions, StreamPermissions, UserId, UserStatus, }; use serde::{Deserialize, Serialize}; use slab::Slab; @@ -57,6 +58,7 @@ pub struct User { pub status: UserStatus, pub created_at: IggyTimestamp, pub permissions: Option>, + pub options: ResourceOptions, } impl Default for User { @@ -68,6 +70,7 @@ impl Default for User { status: UserStatus::default(), created_at: IggyTimestamp::default(), permissions: None, + options: ResourceOptions::new(), } } } @@ -88,6 +91,7 @@ impl User { status, created_at, permissions, + options: ResourceOptions::new(), } } } @@ -299,6 +303,7 @@ impl Users { }, streams: Vec::new(), }), + options: WireOptions::empty(), }, IggyTimestamp::from(1), )) @@ -440,6 +445,9 @@ impl StateHandler for CreateUserRequest { .permissions .as_ref() .map(|p| Arc::new(Permissions::from(p.clone()))); + let Ok(options) = resource_options_from_wire(&self.options, true) else { + return ApplyReply::err(CreateUserResult::InvalidOptionValue); + }; let user = User { id: 0, @@ -448,6 +456,7 @@ impl StateHandler for CreateUserRequest { status, created_at: timestamp, permissions, + options, }; let id = state.items.insert(user); @@ -474,6 +483,7 @@ impl StateHandler for CreateUserRequest { created_at: timestamp.as_micros(), status: self.status, username: self.username.clone(), + options: self.options.clone(), }, permissions: self.permissions.clone(), } @@ -494,6 +504,12 @@ impl StateHandler for UpdateUserRequest { return ApplyReply::err(UpdateUserResult::UserNotFound); }; + // Decoded before any mutation: a malformed block must leave the user + // untouched rather than half-renamed. + let Ok(updated_options) = resource_options_from_wire(&self.options, true) else { + return ApplyReply::err(UpdateUserResult::InvalidOptionValue); + }; + if let Some(new_username) = &self.username { // Same bound as CreateUser apply: a rename must not smuggle in a // username the edges reject. Rejected before any mutation. @@ -518,6 +534,9 @@ impl StateHandler for UpdateUserRequest { { user.status = new_status; } + // Patch, never replace: keys the client did not send keep their + // current value, so a client that predates a key cannot erase it. + user.options.extend(updated_options); ApplyReply::ok(Bytes::new()) } } @@ -730,6 +749,8 @@ pub struct UserSnapshot { pub status: UserStatus, pub created_at: IggyTimestamp, pub permissions: Option, + #[serde(default)] + pub options: ResourceOptions, } /// Personal access token snapshot representation for serialization. @@ -783,6 +804,7 @@ impl Snapshotable for Users { status: user.status, created_at: user.created_at, permissions: user.permissions.as_ref().map(|p| (**p).clone()), + options: user.options.clone(), }, ) }) @@ -868,6 +890,7 @@ impl UsersInner { status: user_snap.status, created_at: user_snap.created_at, permissions: user_snap.permissions.map(Arc::new), + options: user_snap.options, }; index.insert(username, slab_key as UserId); @@ -1154,6 +1177,7 @@ mod tests { password: "hash".to_owned(), status: 1, permissions: None, + options: WireOptions::empty(), }; let apply = StateHandler::apply(&request, users, IggyTimestamp::now()); assert_eq!(apply.code, 0); @@ -1168,6 +1192,7 @@ mod tests { password: "hash".to_owned(), status: 1, permissions: None, + options: WireOptions::empty(), }; let apply = StateHandler::apply(&request, &mut users, IggyTimestamp::now()); assert_eq!(apply.code, u32::from(CreateUserResult::UserAlreadyExists)); @@ -1301,6 +1326,7 @@ mod tests { password: "hash".to_owned(), status: 1, permissions, + options: WireOptions::empty(), }; let reply = StateHandler::apply(&request, users, IggyTimestamp::now()); assert_eq!(reply.code, 0); @@ -1536,6 +1562,7 @@ mod tests { password: "hash".to_owned(), status: 1, permissions: None, + options: WireOptions::empty(), }; let reply = StateHandler::apply(&request, &mut users, IggyTimestamp::now()); assert_eq!(reply.code, u32::from(CreateUserResult::InvalidUsername)); @@ -1551,6 +1578,7 @@ mod tests { password: "hash".to_owned(), status: 1, permissions: None, + options: WireOptions::empty(), }; let reply = StateHandler::apply(&request, &mut users, IggyTimestamp::now()); assert_eq!(reply.code, 0); @@ -1568,6 +1596,7 @@ mod tests { user_id: WireIdentifier::numeric(alice_id), username: Some(WireName::new(&short).unwrap()), status: None, + options: WireOptions::empty(), }; let reply = StateHandler::apply(&rename, &mut users, IggyTimestamp::now()); assert_eq!(reply.code, u32::from(UpdateUserResult::InvalidUsername)); diff --git a/core/partitions/src/iggy_partition.rs b/core/partitions/src/iggy_partition.rs index 8dc9761699..afcb560e8e 100644 --- a/core/partitions/src/iggy_partition.rs +++ b/core/partitions/src/iggy_partition.rs @@ -58,6 +58,7 @@ use iggy_binary_protocol::{PrepareOkHeader, RoutedRequestHeader}; use iggy_common::{ ConsumerGroupId, ConsumerGroupOffsets, ConsumerKind, ConsumerOffset, ConsumerOffsets, IggyByteSize, IggyError, IggyExpiry, IggyTimestamp, PartitionStats, PollingKind, + TopicRuntimeOptions, }; use journal::Journal as _; use journal::local_gate::LocalGate; @@ -89,7 +90,8 @@ use tracing::{debug, warn}; // // Note: there is no per-client write dedup at the partition plane. // `SendMessages` retries are at-least-once and may commit multiple times. -// Consumers handle duplicate messages via `server_common::MessageDeduplicator` +// Duplicate suppression is a consensus-layer concern: the VSR client table +// dedups by request id (at-most-once), so the data plane needs no message-id set. // (message-id based) if they care. pub struct IggyPartition where @@ -123,6 +125,11 @@ where /// `None` only for in-memory (simulated) partitions. pub(crate) partition_dir: Option, pub(crate) consumer_offset_enforce_fsync: bool, + /// This topic's runtime knobs, resolved at topic admission and carried + /// here by the builder. Every `None` field falls back to the shard-wide + /// `PartitionsConfig` value (simulator and tests build partitions with + /// no resolved options at all). + pub(crate) runtime_options: TopicRuntimeOptions, /// In-flight journal repair: /// set when the recovery handshake finds this replica behind the group's /// commit frontier, cleared when `RepairDone` completes the walk. @@ -465,6 +472,7 @@ where consumer_group_offsets_path: None, partition_dir: None, consumer_offset_enforce_fsync: false, + runtime_options: TopicRuntimeOptions::default(), repair: None, recovered_durable_offset: None, installed_frontier: None, @@ -996,6 +1004,59 @@ where self.transfer_attempts } + /// Install this topic's runtime knobs, as resolved at topic admission. + /// Unset fields keep the shard-wide configured values. + pub const fn set_runtime_options(&mut self, runtime_options: TopicRuntimeOptions) { + self.runtime_options = runtime_options; + } + + #[must_use] + pub const fn runtime_options(&self) -> TopicRuntimeOptions { + self.runtime_options + } + + /// Segment size this partition rolls at: the per-topic value when the + /// topic was created with one, else the shard-wide configured size. + #[must_use] + pub fn effective_segment_size(&self, config: &PartitionsConfig) -> IggyByteSize { + self.runtime_options + .segment_size + .unwrap_or(config.segment_size) + } + + /// Whether this partition's writes fsync. + #[must_use] + pub fn effective_enforce_fsync(&self, config: &PartitionsConfig) -> bool { + self.runtime_options + .enforce_fsync + .unwrap_or(config.enforce_fsync) + } + + /// Message-count threshold that flushes this partition's journal. + #[must_use] + pub fn effective_messages_required_to_save(&self, config: &PartitionsConfig) -> u32 { + self.runtime_options + .messages_required_to_save + .unwrap_or(config.messages_required_to_save) + } + + /// Whether this partition's segments reserve their bytes on open. + #[must_use] + pub fn effective_preallocate_segments(&self, config: &PartitionsConfig) -> bool { + self.runtime_options + .preallocate_segments + .unwrap_or(config.preallocate_segments) + } + + /// Byte threshold that flushes this partition's journal. + #[must_use] + pub fn effective_size_of_messages_required_to_save(&self, config: &PartitionsConfig) -> u64 { + self.runtime_options + .size_of_messages_required_to_save + .unwrap_or(config.size_of_messages_required_to_save) + .as_bytes_u64() + } + pub fn configure_consumer_offset_storage( &mut self, consumer_offsets_path: String, @@ -2798,9 +2859,9 @@ where // only" - safe, since the flush still writes only committed bytes. let is_full = self.log.active_segment().is_full(); let unsaved_messages_count_exceeded = - journal_info.messages_count >= config.messages_required_to_save; + journal_info.messages_count >= self.effective_messages_required_to_save(config); let unsaved_messages_size_exceeded = journal_info.size.as_bytes_u64() - >= config.size_of_messages_required_to_save.as_bytes_u64(); + >= self.effective_size_of_messages_required_to_save(config); let should_persist = is_full || unsaved_messages_count_exceeded || unsaved_messages_size_exceeded; if !force && !should_persist { @@ -3663,7 +3724,10 @@ where active_segment.sealed = true; let start_offset = active_segment.end_offset + 1; - let segment = Segment::new(start_offset, config.segment_size); + let segment_size = self.effective_segment_size(config); + let enforce_fsync = self.effective_enforce_fsync(config); + let preallocate_segments = self.effective_preallocate_segments(config); + let segment = Segment::new(start_offset, segment_size); // `PartitionsConfig::get_messages_path` is a stub (`/tmp/iggy_stub`); // the partition's real directory is only known to the server config // that created the initial segment, so derive the rotated paths from @@ -3698,8 +3762,8 @@ where &index_path, 0, 0, - config.enforce_fsync, - config.enforce_fsync, + enforce_fsync, + enforce_fsync, false, ) .await @@ -3713,9 +3777,9 @@ where MessagesWriter::new( &messages_path, messages_size_bytes, - config.enforce_fsync, + enforce_fsync, false, - config.preallocate_segments.then_some(config.segment_size), + preallocate_segments.then_some(segment_size), ) .await .map_err(|_| IggyError::CannotCreateSegmentLogFile(messages_path.clone()))?, @@ -3726,7 +3790,7 @@ where .ok_or_else(|| IggyError::CannotCreateSegmentIndexFile(index_path.clone()))? .size_counter(); let index_writer = Rc::new( - IggyIndexWriter::new(&index_path, index_size_bytes, config.enforce_fsync, false) + IggyIndexWriter::new(&index_path, index_size_bytes, enforce_fsync, false) .await .map_err(|_| IggyError::CannotCreateSegmentIndexFile(index_path.clone()))?, ); @@ -3944,14 +4008,17 @@ where ) }, ); - let segment = Segment::new(start_offset, config.segment_size); + let segment_size = self.effective_segment_size(config); + let enforce_fsync = self.effective_enforce_fsync(config); + let preallocate_segments = self.effective_preallocate_segments(config); + let segment = Segment::new(start_offset, segment_size); let storage = SegmentStorage::new( &messages_path, &index_path, 0, 0, - config.enforce_fsync, - config.enforce_fsync, + enforce_fsync, + enforce_fsync, false, ) .await @@ -3965,9 +4032,9 @@ where MessagesWriter::new( &messages_path, messages_size_bytes, - config.enforce_fsync, + enforce_fsync, false, - config.preallocate_segments.then_some(config.segment_size), + preallocate_segments.then_some(segment_size), ) .await .map_err(|_| IggyError::CannotCreateSegmentLogFile(messages_path.clone()))?, @@ -3978,7 +4045,7 @@ where .ok_or_else(|| IggyError::CannotCreateSegmentIndexFile(index_path.clone()))? .size_counter(); let index_writer = Rc::new( - IggyIndexWriter::new(&index_path, index_size_bytes, config.enforce_fsync, false) + IggyIndexWriter::new(&index_path, index_size_bytes, enforce_fsync, false) .await .map_err(|_| IggyError::CannotCreateSegmentIndexFile(index_path.clone()))?, ); diff --git a/core/partitions/src/messages_writer.rs b/core/partitions/src/messages_writer.rs index cf6fc33650..4ecaadf509 100644 --- a/core/partitions/src/messages_writer.rs +++ b/core/partitions/src/messages_writer.rs @@ -21,8 +21,6 @@ use compio::{ }; use iggy_common::{IggyByteSize, IggyError}; use server_common::iobuf::Frozen; -#[cfg(target_os = "linux")] -use std::os::fd::AsFd; use std::{ rc::Rc, sync::atomic::{AtomicU64, Ordering}, @@ -66,9 +64,6 @@ impl MessagesWriter { .map_err(|_| IggyError::CannotReadFile)?; if let Some(preallocate_size) = preallocate_size { - #[cfg(target_os = "linux")] - preallocate_file(&file, file_path, preallocate_size.as_bytes_u64()).await; - #[cfg(not(target_os = "linux"))] preallocate_file(&file, file_path, preallocate_size.as_bytes_u64()); } @@ -162,7 +157,7 @@ impl MessagesWriter { } #[cfg(target_os = "linux")] -async fn preallocate_file(file: &File, file_path: &str, len: u64) { +fn preallocate_file(file: &File, file_path: &str, len: u64) { let Ok(len) = i64::try_from(len) else { warn!( target: "iggy.partitions.storage", @@ -173,27 +168,24 @@ async fn preallocate_file(file: &File, file_path: &str, len: u64) { return; }; - let file = match file.as_fd().try_clone_to_owned() { - Ok(file) => file, - Err(error) => { - warn!( - target: "iggy.partitions.storage", - file = file_path, - preallocate_len = len, - %error, - "file descriptor duplication failed, using buffered allocation" - ); - return; - } - }; - - // Remote filesystems can make fallocate block. The duplicated descriptor - // lets the blocking pool reserve extents without stalling the shard thread. - let result = compio::runtime::spawn_blocking(move || { - fallocate(file, FallocateFlags::FALLOC_FL_KEEP_SIZE, 0, len) - }) - .await; - if let Err(error) = result { + // Runs INLINE on the shard thread, deliberately. `server_common::executor` + // sets `thread_pool_limit(0)` on the shard proactor, so `spawn_blocking` + // has no worker to park a task on and compio panics the shard outright with + // "the thread pool is needed but no worker thread is running". (That limit + // is skipped on macOS/aarch64, where the pool does exist -- see the FIXME + // there -- so the panic is Linux-and-most-targets, not universal. This arm + // is Linux-only regardless.) + // + // The cost is acceptable only because of what this call is: a metadata-only + // extent reservation, microseconds on the local filesystems this option + // exists for, and an immediate `EOPNOTSUPP` where the filesystem cannot do + // it. Where it can genuinely block -- NFSv4.2 `ALLOCATE`, FUSE, a badly + // fragmented extent tree forcing a journal commit -- it stalls the whole + // core, not one partition, because nothing here yields. Preallocation is + // opt-in per topic at creation for that reason; on such a deployment, + // create topics without `preallocate_segments` rather than reintroducing a + // pool the shard runtime does not have. + if let Err(error) = fallocate(file, FallocateFlags::FALLOC_FL_KEEP_SIZE, 0, len) { warn!( target: "iggy.partitions.storage", file = file_path, diff --git a/core/partitions/src/state_transfer.rs b/core/partitions/src/state_transfer.rs index 75e36fc56c..31802d2657 100644 --- a/core/partitions/src/state_transfer.rs +++ b/core/partitions/src/state_transfer.rs @@ -2263,14 +2263,15 @@ where // sweep itself is right (a chain the live state does not know // about would resurrect at boot), so one retry against a // transient open failure is the only cheap save available. + let enforce_fsync = self.effective_enforce_fsync(config); let open = || { SegmentStorage::new( &log_final, &index_final, meta.size, meta.index_size, - config.enforce_fsync, - config.enforce_fsync, + enforce_fsync, + enforce_fsync, true, ) }; @@ -2283,7 +2284,7 @@ where source, })?, }; - let mut segment = Segment::new(meta.start_offset, config.segment_size); + let mut segment = Segment::new(meta.start_offset, self.effective_segment_size(config)); segment.sealed = true; segment.start_timestamp = meta.start_timestamp; segment.end_timestamp = meta.end_timestamp; @@ -2318,6 +2319,9 @@ where source, })?; } else { + let enforce_fsync = self.effective_enforce_fsync(config); + let segment_size = self.effective_segment_size(config); + let preallocate_segments = self.effective_preallocate_segments(config); let last = self.log.segments().len() - 1; let storage = self.log.storages()[last].clone(); if let (Some(messages_reader), Some(index_reader), Some(messages_w), Some(index_w)) = ( @@ -2329,9 +2333,9 @@ where let messages_writer = MessagesWriter::new( &messages_reader.path(), messages_w.size_counter(), - config.enforce_fsync, + enforce_fsync, true, - config.preallocate_segments.then_some(config.segment_size), + preallocate_segments.then_some(segment_size), ) .await .map_err(|source| PartitionInstallError::SegmentOpen { @@ -2341,7 +2345,7 @@ where let index_writer = IggyIndexWriter::new( &index_reader.path(), index_w.size_counter(), - config.enforce_fsync, + enforce_fsync, true, ) .await diff --git a/core/sdk/src/client_wrappers/binary_stream_client.rs b/core/sdk/src/client_wrappers/binary_stream_client.rs index 76d68292bc..bf35ebbe32 100644 --- a/core/sdk/src/client_wrappers/binary_stream_client.rs +++ b/core/sdk/src/client_wrappers/binary_stream_client.rs @@ -18,6 +18,7 @@ use crate::client_wrappers::client_wrapper::ClientWrapper; use async_trait::async_trait; use iggy_common::StreamClient; +use iggy_common::StreamUpdateOptions; use iggy_common::{Identifier, IggyError, Stream, StreamDetails}; #[async_trait] @@ -52,13 +53,20 @@ impl StreamClient for ClientWrapper { } } - async fn update_stream(&self, stream_id: &Identifier, name: &str) -> Result<(), IggyError> { + async fn update_stream( + &self, + stream_id: &Identifier, + name: &str, + options: &StreamUpdateOptions, + ) -> Result<(), IggyError> { match self { - ClientWrapper::Iggy(client) => client.update_stream(stream_id, name).await, - ClientWrapper::Http(client) => client.update_stream(stream_id, name).await, - ClientWrapper::Tcp(client) => client.update_stream(stream_id, name).await, - ClientWrapper::Quic(client) => client.update_stream(stream_id, name).await, - ClientWrapper::WebSocket(client) => client.update_stream(stream_id, name).await, + ClientWrapper::Iggy(client) => client.update_stream(stream_id, name, options).await, + ClientWrapper::Http(client) => client.update_stream(stream_id, name, options).await, + ClientWrapper::Tcp(client) => client.update_stream(stream_id, name, options).await, + ClientWrapper::Quic(client) => client.update_stream(stream_id, name, options).await, + ClientWrapper::WebSocket(client) => { + client.update_stream(stream_id, name, options).await + } } } diff --git a/core/sdk/src/client_wrappers/binary_system_client.rs b/core/sdk/src/client_wrappers/binary_system_client.rs index e4e34d9adc..e474377492 100644 --- a/core/sdk/src/client_wrappers/binary_system_client.rs +++ b/core/sdk/src/client_wrappers/binary_system_client.rs @@ -19,8 +19,8 @@ use crate::client_wrappers::client_wrapper::ClientWrapper; use async_trait::async_trait; use iggy_common::SystemClient; use iggy_common::{ - ClientInfo, ClientInfoDetails, IggyDuration, IggyError, Snapshot, SnapshotCompression, Stats, - SystemSnapshotType, + ClientInfo, ClientInfoDetails, IggyDuration, IggyError, OptionSpec, OptionsScope, Snapshot, + SnapshotCompression, Stats, SystemSnapshotType, }; #[async_trait] @@ -65,6 +65,16 @@ impl SystemClient for ClientWrapper { } } + async fn describe_options(&self, scope: OptionsScope) -> Result, IggyError> { + match self { + ClientWrapper::Iggy(client) => client.describe_options(scope).await, + ClientWrapper::Http(client) => client.describe_options(scope).await, + ClientWrapper::Tcp(client) => client.describe_options(scope).await, + ClientWrapper::Quic(client) => client.describe_options(scope).await, + ClientWrapper::WebSocket(client) => client.describe_options(scope).await, + } + } + async fn ping(&self) -> Result<(), IggyError> { match self { ClientWrapper::Iggy(client) => client.ping().await, diff --git a/core/sdk/src/client_wrappers/binary_topic_client.rs b/core/sdk/src/client_wrappers/binary_topic_client.rs index 44a4689300..52c12a71f8 100644 --- a/core/sdk/src/client_wrappers/binary_topic_client.rs +++ b/core/sdk/src/client_wrappers/binary_topic_client.rs @@ -19,7 +19,7 @@ use crate::client_wrappers::client_wrapper::ClientWrapper; use async_trait::async_trait; use iggy_common::TopicClient; use iggy_common::{ - CompressionAlgorithm, Identifier, IggyError, IggyExpiry, MaxTopicSize, Topic, TopicDetails, + Identifier, IggyError, Topic, TopicCreateOptions, TopicDetails, TopicUpdateOptions, }; #[async_trait] @@ -52,78 +52,14 @@ impl TopicClient for ClientWrapper { &self, stream_id: &Identifier, name: &str, - partitions_count: u32, - compression_algorithm: CompressionAlgorithm, - replication_factor: Option, - message_expiry: IggyExpiry, - max_topic_size: MaxTopicSize, + options: &TopicCreateOptions, ) -> Result { match self { - ClientWrapper::Iggy(client) => { - client - .create_topic( - stream_id, - name, - partitions_count, - compression_algorithm, - replication_factor, - message_expiry, - max_topic_size, - ) - .await - } - ClientWrapper::Http(client) => { - client - .create_topic( - stream_id, - name, - partitions_count, - compression_algorithm, - replication_factor, - message_expiry, - max_topic_size, - ) - .await - } - ClientWrapper::Tcp(client) => { - client - .create_topic( - stream_id, - name, - partitions_count, - compression_algorithm, - replication_factor, - message_expiry, - max_topic_size, - ) - .await - } - ClientWrapper::Quic(client) => { - client - .create_topic( - stream_id, - name, - partitions_count, - compression_algorithm, - replication_factor, - message_expiry, - max_topic_size, - ) - .await - } - ClientWrapper::WebSocket(client) => { - client - .create_topic( - stream_id, - name, - partitions_count, - compression_algorithm, - replication_factor, - message_expiry, - max_topic_size, - ) - .await - } + ClientWrapper::Iggy(client) => client.create_topic(stream_id, name, options).await, + ClientWrapper::Http(client) => client.create_topic(stream_id, name, options).await, + ClientWrapper::Tcp(client) => client.create_topic(stream_id, name, options).await, + ClientWrapper::Quic(client) => client.create_topic(stream_id, name, options).await, + ClientWrapper::WebSocket(client) => client.create_topic(stream_id, name, options).await, } } @@ -132,75 +68,32 @@ impl TopicClient for ClientWrapper { stream_id: &Identifier, topic_id: &Identifier, name: &str, - compression_algorithm: CompressionAlgorithm, - replication_factor: Option, - message_expiry: IggyExpiry, - max_topic_size: MaxTopicSize, + options: &TopicUpdateOptions, ) -> Result<(), IggyError> { match self { ClientWrapper::Iggy(client) => { client - .update_topic( - stream_id, - topic_id, - name, - compression_algorithm, - replication_factor, - message_expiry, - max_topic_size, - ) + .update_topic(stream_id, topic_id, name, options) .await } ClientWrapper::Http(client) => { client - .update_topic( - stream_id, - topic_id, - name, - compression_algorithm, - replication_factor, - message_expiry, - max_topic_size, - ) + .update_topic(stream_id, topic_id, name, options) .await } ClientWrapper::Tcp(client) => { client - .update_topic( - stream_id, - topic_id, - name, - compression_algorithm, - replication_factor, - message_expiry, - max_topic_size, - ) + .update_topic(stream_id, topic_id, name, options) .await } ClientWrapper::Quic(client) => { client - .update_topic( - stream_id, - topic_id, - name, - compression_algorithm, - replication_factor, - message_expiry, - max_topic_size, - ) + .update_topic(stream_id, topic_id, name, options) .await } ClientWrapper::WebSocket(client) => { client - .update_topic( - stream_id, - topic_id, - name, - compression_algorithm, - replication_factor, - message_expiry, - max_topic_size, - ) + .update_topic(stream_id, topic_id, name, options) .await } } diff --git a/core/sdk/src/client_wrappers/binary_user_client.rs b/core/sdk/src/client_wrappers/binary_user_client.rs index 9cd6142781..dc38d26653 100644 --- a/core/sdk/src/client_wrappers/binary_user_client.rs +++ b/core/sdk/src/client_wrappers/binary_user_client.rs @@ -18,6 +18,7 @@ use crate::client_wrappers::client_wrapper::ClientWrapper; use async_trait::async_trait; use iggy_common::UserClient; +use iggy_common::UserUpdateOptions; use iggy_common::{ Identifier, IdentityInfo, IggyError, Permissions, UserInfo, UserInfoDetails, UserStatus, }; @@ -95,13 +96,24 @@ impl UserClient for ClientWrapper { user_id: &Identifier, username: Option<&str>, status: Option, + options: &UserUpdateOptions, ) -> Result<(), IggyError> { match self { - ClientWrapper::Http(client) => client.update_user(user_id, username, status).await, - ClientWrapper::Tcp(client) => client.update_user(user_id, username, status).await, - ClientWrapper::Quic(client) => client.update_user(user_id, username, status).await, - ClientWrapper::Iggy(client) => client.update_user(user_id, username, status).await, - ClientWrapper::WebSocket(client) => client.update_user(user_id, username, status).await, + ClientWrapper::Http(client) => { + client.update_user(user_id, username, status, options).await + } + ClientWrapper::Tcp(client) => { + client.update_user(user_id, username, status, options).await + } + ClientWrapper::Quic(client) => { + client.update_user(user_id, username, status, options).await + } + ClientWrapper::Iggy(client) => { + client.update_user(user_id, username, status, options).await + } + ClientWrapper::WebSocket(client) => { + client.update_user(user_id, username, status, options).await + } } } diff --git a/core/sdk/src/clients/binary_streams.rs b/core/sdk/src/clients/binary_streams.rs index 4b646a9d97..4213b876bf 100644 --- a/core/sdk/src/clients/binary_streams.rs +++ b/core/sdk/src/clients/binary_streams.rs @@ -18,6 +18,7 @@ use crate::prelude::IggyClient; use async_trait::async_trait; use iggy_common::StreamClient; +use iggy_common::StreamUpdateOptions; use iggy_common::locking::IggyRwLockFn; use iggy_common::{Identifier, IggyError, Stream, StreamDetails}; @@ -35,11 +36,16 @@ impl StreamClient for IggyClient { self.client.read().await.create_stream(name).await } - async fn update_stream(&self, stream_id: &Identifier, name: &str) -> Result<(), IggyError> { + async fn update_stream( + &self, + stream_id: &Identifier, + name: &str, + options: &StreamUpdateOptions, + ) -> Result<(), IggyError> { self.client .read() .await - .update_stream(stream_id, name) + .update_stream(stream_id, name, options) .await } diff --git a/core/sdk/src/clients/binary_system.rs b/core/sdk/src/clients/binary_system.rs index 3398be4b6b..a2681696f7 100644 --- a/core/sdk/src/clients/binary_system.rs +++ b/core/sdk/src/clients/binary_system.rs @@ -20,8 +20,8 @@ use async_trait::async_trait; use iggy_common::SystemClient; use iggy_common::locking::IggyRwLockFn; use iggy_common::{ - ClientInfo, ClientInfoDetails, IggyDuration, IggyError, Snapshot, SnapshotCompression, Stats, - SystemSnapshotType, + ClientInfo, ClientInfoDetails, IggyDuration, IggyError, OptionSpec, OptionsScope, Snapshot, + SnapshotCompression, Stats, SystemSnapshotType, }; #[async_trait] @@ -42,6 +42,10 @@ impl SystemClient for IggyClient { self.client.read().await.get_clients().await } + async fn describe_options(&self, scope: OptionsScope) -> Result, IggyError> { + self.client.read().await.describe_options(scope).await + } + async fn ping(&self) -> Result<(), IggyError> { self.client.read().await.ping().await } diff --git a/core/sdk/src/clients/binary_topics.rs b/core/sdk/src/clients/binary_topics.rs index f74c678764..60182eb60e 100644 --- a/core/sdk/src/clients/binary_topics.rs +++ b/core/sdk/src/clients/binary_topics.rs @@ -20,7 +20,7 @@ use async_trait::async_trait; use iggy_common::TopicClient; use iggy_common::locking::IggyRwLockFn; use iggy_common::{ - CompressionAlgorithm, Identifier, IggyError, IggyExpiry, MaxTopicSize, Topic, TopicDetails, + Identifier, IggyError, Topic, TopicCreateOptions, TopicDetails, TopicUpdateOptions, }; #[async_trait] @@ -45,24 +45,12 @@ impl TopicClient for IggyClient { &self, stream_id: &Identifier, name: &str, - partitions_count: u32, - compression_algorithm: CompressionAlgorithm, - replication_factor: Option, - message_expiry: IggyExpiry, - max_topic_size: MaxTopicSize, + options: &TopicCreateOptions, ) -> Result { self.client .read() .await - .create_topic( - stream_id, - name, - partitions_count, - compression_algorithm, - replication_factor, - message_expiry, - max_topic_size, - ) + .create_topic(stream_id, name, options) .await } @@ -71,23 +59,12 @@ impl TopicClient for IggyClient { stream_id: &Identifier, topic_id: &Identifier, name: &str, - compression_algorithm: CompressionAlgorithm, - replication_factor: Option, - message_expiry: IggyExpiry, - max_topic_size: MaxTopicSize, + options: &TopicUpdateOptions, ) -> Result<(), IggyError> { self.client .read() .await - .update_topic( - stream_id, - topic_id, - name, - compression_algorithm, - replication_factor, - message_expiry, - max_topic_size, - ) + .update_topic(stream_id, topic_id, name, options) .await } diff --git a/core/sdk/src/clients/binary_users.rs b/core/sdk/src/clients/binary_users.rs index 4e85826bcb..cab16bcf61 100644 --- a/core/sdk/src/clients/binary_users.rs +++ b/core/sdk/src/clients/binary_users.rs @@ -18,6 +18,7 @@ use crate::client_wrappers::client_wrapper::ClientWrapper; use crate::prelude::IggyClient; use async_trait::async_trait; +use iggy_common::UserUpdateOptions; use iggy_common::locking::IggyRwLockFn; use iggy_common::{Client, UserClient}; use iggy_common::{ @@ -58,11 +59,12 @@ impl UserClient for IggyClient { user_id: &Identifier, username: Option<&str>, status: Option, + options: &UserUpdateOptions, ) -> Result<(), IggyError> { self.client .read() .await - .update_user(user_id, username, status) + .update_user(user_id, username, status, options) .await } diff --git a/core/sdk/src/clients/producer.rs b/core/sdk/src/clients/producer.rs index 6d93cc334d..04912b1daa 100644 --- a/core/sdk/src/clients/producer.rs +++ b/core/sdk/src/clients/producer.rs @@ -24,10 +24,10 @@ use crate::clients::producer_dispatcher::ProducerDispatcher; use bytes::Bytes; use futures_util::StreamExt; use iggy_common::locking::{IggyRwLock, IggyRwLockFn}; -use iggy_common::{Client, MessageClient, StreamClient, TopicClient}; +use iggy_common::{Client, MessageClient, StreamClient, TopicClient, TopicCreateOptions}; use iggy_common::{ - CompressionAlgorithm, DiagnosticEvent, EncryptorKind, IdKind, Identifier, IggyDuration, - IggyError, IggyExpiry, IggyMessage, IggyTimestamp, MaxTopicSize, Partitioner, Partitioning, + DiagnosticEvent, EncryptorKind, IdKind, Identifier, IggyDuration, IggyError, IggyExpiry, + IggyMessage, IggyTimestamp, MaxTopicSize, Partitioner, Partitioning, SendMessagesConfirmationResponse, SendMessagesResponse, }; use std::sync::Arc; @@ -94,7 +94,6 @@ pub struct ProducerCore { create_stream_if_not_exists: bool, create_topic_if_not_exists: bool, topic_partitions_count: u32, - topic_replication_factor: Option, topic_message_expiry: IggyExpiry, topic_max_size: MaxTopicSize, default_partitioning: Arc, @@ -154,11 +153,14 @@ impl ProducerCore { .create_topic( &self.stream_id, &self.topic_name, - self.topic_partitions_count, - CompressionAlgorithm::None, - self.topic_replication_factor, - self.topic_message_expiry, - self.topic_max_size, + &TopicCreateOptions { + partitions_count: Some(self.topic_partitions_count), + message_expiry: (self.topic_message_expiry != IggyExpiry::ServerDefault) + .then_some(self.topic_message_expiry), + max_topic_size: (self.topic_max_size != MaxTopicSize::ServerDefault) + .then_some(self.topic_max_size), + ..TopicCreateOptions::default() + }, ) .await?; } @@ -491,7 +493,6 @@ impl IggyProducer { create_stream_if_not_exists: bool, create_topic_if_not_exists: bool, topic_partitions_count: u32, - topic_replication_factor: Option, topic_message_expiry: IggyExpiry, topic_max_size: MaxTopicSize, send_retries_count: Option, @@ -512,7 +513,6 @@ impl IggyProducer { create_stream_if_not_exists, create_topic_if_not_exists, topic_partitions_count, - topic_replication_factor, topic_message_expiry, topic_max_size, default_partitioning: Arc::new(Partitioning::balanced()), diff --git a/core/sdk/src/clients/producer_builder.rs b/core/sdk/src/clients/producer_builder.rs index 576bbe9a06..7734e2e778 100644 --- a/core/sdk/src/clients/producer_builder.rs +++ b/core/sdk/src/clients/producer_builder.rs @@ -46,7 +46,6 @@ pub struct IggyProducerBuilder { create_stream_if_not_exists: bool, create_topic_if_not_exists: bool, topic_partitions_count: u32, - topic_replication_factor: Option, send_retries_count: Option, send_retries_interval: Option, topic_message_expiry: IggyExpiry, @@ -78,7 +77,6 @@ impl IggyProducerBuilder { create_stream_if_not_exists: true, create_topic_if_not_exists: true, topic_partitions_count: 1, - topic_replication_factor: None, topic_message_expiry: IggyExpiry::ServerDefault, topic_max_size: MaxTopicSize::ServerDefault, send_retries_count: Some(3), @@ -165,14 +163,12 @@ impl IggyProducerBuilder { pub fn create_topic_if_not_exists( self, partitions_count: u32, - replication_factor: Option, message_expiry: IggyExpiry, max_size: MaxTopicSize, ) -> Self { Self { create_topic_if_not_exists: true, topic_partitions_count: partitions_count, - topic_replication_factor: replication_factor, topic_message_expiry: message_expiry, topic_max_size: max_size, ..self @@ -226,7 +222,6 @@ impl IggyProducerBuilder { self.create_stream_if_not_exists, self.create_topic_if_not_exists, self.topic_partitions_count, - self.topic_replication_factor, self.topic_message_expiry, self.topic_max_size, self.send_retries_count, diff --git a/core/sdk/src/http/streams.rs b/core/sdk/src/http/streams.rs index 4ac8023146..a382aef3ea 100644 --- a/core/sdk/src/http/streams.rs +++ b/core/sdk/src/http/streams.rs @@ -21,6 +21,7 @@ use crate::prelude::Identifier; use crate::prelude::IggyError; use async_trait::async_trait; use iggy_common::StreamClient; +use iggy_common::StreamUpdateOptions; use iggy_common::create_stream::CreateStream; use iggy_common::update_stream::UpdateStream; use iggy_common::{Stream, StreamDetails}; @@ -71,12 +72,18 @@ impl StreamClient for HttpClient { Ok(stream) } - async fn update_stream(&self, stream_id: &Identifier, name: &str) -> Result<(), IggyError> { + async fn update_stream( + &self, + stream_id: &Identifier, + name: &str, + options: &StreamUpdateOptions, + ) -> Result<(), IggyError> { self.put( &get_details_path(&stream_id.as_cow_str()), &UpdateStream { stream_id: stream_id.clone(), name: name.to_string(), + options: options.raw.clone(), }, ) .await?; diff --git a/core/sdk/src/http/system.rs b/core/sdk/src/http/system.rs index 3742351830..4634c9daf5 100644 --- a/core/sdk/src/http/system.rs +++ b/core/sdk/src/http/system.rs @@ -24,6 +24,7 @@ use iggy_common::Stats; use iggy_common::SystemClient; use iggy_common::get_snapshot::GetSnapshot; use iggy_common::{ClientInfo, ClientInfoDetails}; +use iggy_common::{OptionSpec, OptionsScope}; use iggy_common::{SnapshotCompression, SystemSnapshotType}; const PING: &str = "/ping"; @@ -72,6 +73,14 @@ impl SystemClient for HttpClient { Ok(clients) } + async fn describe_options(&self, scope: OptionsScope) -> Result, IggyError> { + let response = self.get(&format!("/options/{scope}")).await?; + response + .json() + .await + .map_err(|_| IggyError::InvalidJsonResponse) + } + async fn ping(&self) -> Result<(), IggyError> { self.get(PING).await?; Ok(()) diff --git a/core/sdk/src/http/topics.rs b/core/sdk/src/http/topics.rs index 0cd6ffb756..365707b44e 100644 --- a/core/sdk/src/http/topics.rs +++ b/core/sdk/src/http/topics.rs @@ -17,12 +17,14 @@ use crate::http::http_client::HttpClient; use crate::http::http_transport::HttpTransport; -use crate::prelude::{CompressionAlgorithm, Identifier, IggyError, IggyExpiry, MaxTopicSize}; +use crate::prelude::{Identifier, IggyError, IggyExpiry, MaxTopicSize}; use async_trait::async_trait; use iggy_common::TopicClient; use iggy_common::create_topic::CreateTopic; use iggy_common::update_topic::UpdateTopic; -use iggy_common::{Topic, TopicDetails}; +use iggy_common::{ + DEFAULT_PARTITIONS_COUNT, Topic, TopicCreateOptions, TopicDetails, TopicUpdateOptions, +}; #[async_trait] impl TopicClient for HttpClient { @@ -65,11 +67,7 @@ impl TopicClient for HttpClient { &self, stream_id: &Identifier, name: &str, - partitions_count: u32, - compression_algorithm: CompressionAlgorithm, - replication_factor: Option, - message_expiry: IggyExpiry, - max_topic_size: MaxTopicSize, + options: &TopicCreateOptions, ) -> Result { let response = self .post( @@ -77,11 +75,17 @@ impl TopicClient for HttpClient { &CreateTopic { stream_id: stream_id.clone(), name: name.to_string(), - partitions_count, - compression_algorithm, - replication_factor, - message_expiry, - max_topic_size, + partitions_count: options.partitions_count.unwrap_or(DEFAULT_PARTITIONS_COUNT), + compression_algorithm: options.compression_algorithm.unwrap_or_default(), + message_expiry: options.message_expiry.unwrap_or(IggyExpiry::ServerDefault), + max_topic_size: options + .max_topic_size + .unwrap_or(MaxTopicSize::ServerDefault), + // Carries the runtime knobs too, not just `raw`: this body + // has no dedicated field for them, and dropping them here + // gave one transport a topic without the fsync the caller + // asked for while the other honored it. + options: options.to_string_options(), }, ) .await?; @@ -97,10 +101,7 @@ impl TopicClient for HttpClient { stream_id: &Identifier, topic_id: &Identifier, name: &str, - compression_algorithm: CompressionAlgorithm, - replication_factor: Option, - message_expiry: IggyExpiry, - max_topic_size: MaxTopicSize, + options: &TopicUpdateOptions, ) -> Result<(), IggyError> { self.put( &get_details_path(&stream_id.as_cow_str(), &topic_id.as_cow_str()), @@ -108,10 +109,10 @@ impl TopicClient for HttpClient { stream_id: stream_id.clone(), topic_id: topic_id.clone(), name: name.to_string(), - compression_algorithm, - replication_factor, - message_expiry, - max_topic_size, + compression_algorithm: options.compression_algorithm, + message_expiry: options.message_expiry, + max_topic_size: options.max_topic_size, + options: options.raw.clone(), }, ) .await?; diff --git a/core/sdk/src/http/users.rs b/core/sdk/src/http/users.rs index c6514917dc..0d6b82b03b 100644 --- a/core/sdk/src/http/users.rs +++ b/core/sdk/src/http/users.rs @@ -20,6 +20,7 @@ use crate::http::http_transport::HttpTransport; use crate::prelude::{Identifier, IggyError}; use async_trait::async_trait; use iggy_common::UserClient; +use iggy_common::UserUpdateOptions; use iggy_common::change_password::ChangePassword; use iggy_common::create_user::CreateUser; use iggy_common::login_user::LoginUser; @@ -94,6 +95,7 @@ impl UserClient for HttpClient { user_id: &Identifier, username: Option<&str>, status: Option, + options: &UserUpdateOptions, ) -> Result<(), IggyError> { self.put( &format!("{PATH}/{}", user_id.as_cow_str()), @@ -101,6 +103,7 @@ impl UserClient for HttpClient { user_id: user_id.clone(), username: username.map(|s| s.to_string()), status, + options: options.raw.clone(), }, ) .await?; diff --git a/core/sdk/src/prelude.rs b/core/sdk/src/prelude.rs index 51b34e34ab..f3d4db5d78 100644 --- a/core/sdk/src/prelude.rs +++ b/core/sdk/src/prelude.rs @@ -55,15 +55,17 @@ pub use iggy_common::{ HeaderKind, HeaderValue, HttpClientConfig, HttpClientConfigBuilder, HttpMethod, IdKind, Identifier, IdentityInfo, IggyByteSize, IggyDuration, IggyError, IggyExpiry, IggyIndexView, IggyMessage, IggyMessageHeader, IggyMessageHeaderView, IggyMessageView, - IggyMessageViewIterator, IggyTimestamp, MaxTopicSize, Partition, Partitioner, Partitioning, - Permissions, PersonalAccessTokenExpiry, PollMessages, PolledMessages, PollingKind, - PollingStrategy, QuicClientConfig, QuicClientConfigBuilder, QuicClientReconnectionConfig, - SendMessages, SendMessagesConfirmationResponse, SendMessagesResponse, Sizeable, - SnapshotCompression, Stats, Stream, StreamDetails, StreamPermissions, SystemSnapshotType, - TcpClientConfig, TcpClientConfigBuilder, TcpClientReconnectionConfig, Topic, TopicDetails, - TopicPermissions, TransportEndpoints, TransportProtocol, UserId, UserInfo, UserInfoDetails, - UserStatus, Validatable, WebSocketClientConfig, WebSocketClientConfigBuilder, - WebSocketClientReconnectionConfig, defaults, locking, + IggyMessageViewIterator, IggyTimestamp, MaxTopicSize, OptionSpec, OptionValue, OptionsScope, + Partition, Partitioner, Partitioning, Permissions, PersonalAccessTokenExpiry, PollMessages, + PolledMessages, PollingKind, PollingStrategy, QuicClientConfig, QuicClientConfigBuilder, + QuicClientReconnectionConfig, ResourceOptions, SendMessages, SendMessagesConfirmationResponse, + SendMessagesResponse, Sizeable, SnapshotCompression, Stats, Stream, StreamDetails, + StreamPermissions, StreamUpdateOptions, SystemSnapshotType, TcpClientConfig, + TcpClientConfigBuilder, TcpClientReconnectionConfig, Topic, TopicCreateOptions, TopicDetails, + TopicPermissions, TopicUpdateOptions, TransportEndpoints, TransportProtocol, UserId, UserInfo, + UserInfoDetails, UserStatus, UserUpdateOptions, Validatable, WebSocketClientConfig, + WebSocketClientConfigBuilder, WebSocketClientReconnectionConfig, defaults, locking, + topic_option_keys, }; pub use iggy_common::{ Client, ClusterClient, ConsumerGroupClient, ConsumerOffsetClient, MessageClient, diff --git a/core/sdk/src/stream_builder/build/build_iggy_producer.rs b/core/sdk/src/stream_builder/build/build_iggy_producer.rs index 9522452bb9..c9b425ef3a 100644 --- a/core/sdk/src/stream_builder/build/build_iggy_producer.rs +++ b/core/sdk/src/stream_builder/build/build_iggy_producer.rs @@ -47,7 +47,6 @@ pub(crate) async fn build_iggy_producer( let stream = config.stream_name(); let topic = config.topic_name(); let topic_partitions_count = config.topic_partitions_count(); - let topic_replication_factor = config.topic_replication_factor(); let batch_length = config.batch_length(); let linger_time = config.linger_time(); let partitioning = config.partitioning().to_owned(); @@ -62,7 +61,6 @@ pub(crate) async fn build_iggy_producer( .send_retries(send_retries, send_retries_interval) .create_topic_if_not_exists( topic_partitions_count, - topic_replication_factor, IggyExpiry::ServerDefault, MaxTopicSize::ServerDefault, ) diff --git a/core/sdk/src/stream_builder/build/build_stream_topic.rs b/core/sdk/src/stream_builder/build/build_stream_topic.rs index a3dd09b0b1..beebbcd770 100644 --- a/core/sdk/src/stream_builder/build/build_stream_topic.rs +++ b/core/sdk/src/stream_builder/build/build_stream_topic.rs @@ -17,8 +17,7 @@ */ use crate::prelude::{ - CompressionAlgorithm, IdKind, Identifier, IggyClient, IggyError, IggyExpiry, MaxTopicSize, - StreamClient, TopicClient, + IdKind, Identifier, IggyClient, IggyError, StreamClient, TopicClient, TopicCreateOptions, }; use crate::stream_builder::IggyConsumerConfig; @@ -83,7 +82,6 @@ pub(crate) async fn build_iggy_stream_topic_if_not_exists( let stream_id = config.stream_id(); let stream_name = config.stream_name(); let topic_partitions_count = config.partitions_count(); - let topic_replication_factor = config.replication_factor(); let (name, _id) = extract_name_id_from_identifier(topic_id, topic_name)?; trace!("Create topic: {name} for stream: {}", stream_name); @@ -91,11 +89,10 @@ pub(crate) async fn build_iggy_stream_topic_if_not_exists( .create_topic( stream_id, topic_name, - topic_partitions_count, - CompressionAlgorithm::None, - topic_replication_factor, - IggyExpiry::ServerDefault, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(topic_partitions_count), + ..TopicCreateOptions::default() + }, ) .await?; } diff --git a/core/sdk/src/stream_builder/config/config_iggy_consumer.rs b/core/sdk/src/stream_builder/config/config_iggy_consumer.rs index e0b136567c..4f872b20fb 100644 --- a/core/sdk/src/stream_builder/config/config_iggy_consumer.rs +++ b/core/sdk/src/stream_builder/config/config_iggy_consumer.rs @@ -50,7 +50,6 @@ pub struct IggyConsumerConfig { /// Sets the number of partitions for ConsumerKind `Consumer`. Does not apply to `ConsumerGroup`. partitions_count: u32, /// Sets the replication factor for the consumed topic. - replication_factor: Option, /// The polling interval for messages. polling_interval: IggyDuration, /// `PollingStrategy` specifies from where to start polling messages. See `PollingStrategy` for details. @@ -85,7 +84,6 @@ impl Default for IggyConsumerConfig { polling_interval: IggyDuration::from_str("5ms").unwrap(), polling_strategy: PollingStrategy::last(), partitions_count: 1, - replication_factor: None, encryptor: None, polling_retry_interval: IggyDuration::new_from_secs(1), init_retries: Some(5), @@ -112,7 +110,6 @@ impl IggyConsumerConfig { /// * `polling_interval` - The interval between polling for new messages. /// * `polling_strategy` - The polling strategy. /// * `partitions_count` - The number of partitions. - /// * `replication_factor` - The replication factor. /// * `encryptor` - The encryptor. /// * `polling_retry_interval` - The polling retry interval. /// * `init_retries` - The number of init retries. @@ -137,7 +134,6 @@ impl IggyConsumerConfig { polling_interval: IggyDuration, polling_strategy: PollingStrategy, partitions_count: u32, - replication_factor: Option, encryptor: Option>, polling_retry_interval: IggyDuration, init_retries: Option, @@ -157,7 +153,6 @@ impl IggyConsumerConfig { polling_interval, polling_strategy, partitions_count, - replication_factor, encryptor, polling_retry_interval, init_retries, @@ -200,7 +195,6 @@ impl IggyConsumerConfig { polling_interval, polling_strategy: PollingStrategy::last(), partitions_count: 1, - replication_factor: None, encryptor: None, polling_retry_interval: IggyDuration::new_from_secs(1), init_retries: Some(5), @@ -261,10 +255,6 @@ impl IggyConsumerConfig { self.partitions_count } - pub fn replication_factor(&self) -> Option { - self.replication_factor - } - pub fn encryptor(&self) -> Option> { self.encryptor.clone() } @@ -371,7 +361,6 @@ mod tests { ); assert_eq!(config.polling_strategy(), PollingStrategy::last()); assert_eq!(config.partitions_count(), 1); - assert_eq!(config.replication_factor(), None); assert_eq!(config.polling_retry_interval(), IggyDuration::ONE_SECOND); assert_eq!(config.init_retries(), Some(5)); @@ -395,7 +384,6 @@ mod tests { PollingStrategy::last(), 1, None, - None, IggyDuration::new_from_secs(1), Some(3), IggyDuration::new_from_secs(3), @@ -425,7 +413,6 @@ mod tests { ); assert_eq!(config.polling_strategy(), PollingStrategy::last()); assert_eq!(config.partitions_count(), 1); - assert_eq!(config.replication_factor(), None); assert_eq!( config.polling_retry_interval(), @@ -459,6 +446,5 @@ mod tests { ); assert_eq!(config.polling_strategy(), PollingStrategy::last()); assert_eq!(config.partitions_count(), 1); - assert_eq!(config.replication_factor(), None); } } diff --git a/core/sdk/src/stream_builder/config/config_iggy_producer.rs b/core/sdk/src/stream_builder/config/config_iggy_producer.rs index f7f9be4a8b..19dc4cea7b 100644 --- a/core/sdk/src/stream_builder/config/config_iggy_producer.rs +++ b/core/sdk/src/stream_builder/config/config_iggy_producer.rs @@ -34,7 +34,6 @@ pub struct IggyProducerConfig { /// Sets the number of partitions to create for the topic topic_partitions_count: u32, /// Set the topic replication factor - topic_replication_factor: Option, /// The max number of messages to send in a batch. Must be greater than 0. batch_length: u32, /// Sets the interval between sending the messages, can be combined with `batch_length`. @@ -64,7 +63,6 @@ impl Default for IggyProducerConfig { linger_time: IggyDuration::from_str("5ms").unwrap(), partitioning: Partitioning::balanced(), topic_partitions_count: 1, - topic_replication_factor: None, encryptor: None, send_retries_count: Some(3), send_retries_interval: Some(IggyDuration::new_from_secs(1)), @@ -82,7 +80,6 @@ impl IggyProducerConfig { /// * `topic_id` - The topic identifier. /// * `topic_name` - The topic name. /// * `topic_partitions_count` - The number of partitions to create. - /// * `topic_replication_factor` - The replication factor to use. /// * `batch_length` - The max number of messages to send in a batch. /// * `linger_time` - The interval between messages sent. /// * `partitioning` - The partitioning strategy to use. @@ -100,7 +97,6 @@ impl IggyProducerConfig { topic_id: Identifier, topic_name: String, topic_partitions_count: u32, - topic_replication_factor: Option, batch_length: u32, linger_time: IggyDuration, partitioning: Partitioning, @@ -114,7 +110,6 @@ impl IggyProducerConfig { topic_id, topic_name, topic_partitions_count, - topic_replication_factor, batch_length, linger_time, partitioning, @@ -155,7 +150,6 @@ impl IggyProducerConfig { linger_time, partitioning: Partitioning::balanced(), topic_partitions_count: 1, - topic_replication_factor: None, encryptor: None, send_retries_count: Some(3), send_retries_interval: Some(IggyDuration::new_from_secs(1)), @@ -196,10 +190,6 @@ impl IggyProducerConfig { self.topic_partitions_count } - pub fn topic_replication_factor(&self) -> Option { - self.topic_replication_factor - } - pub fn encryptor(&self) -> Option> { self.encryptor.clone() } @@ -250,7 +240,6 @@ mod tests { assert_eq!(config.linger_time(), IggyDuration::from_str("5ms").unwrap()); assert_eq!(config.partitioning(), &Partitioning::balanced()); assert_eq!(config.topic_partitions_count(), 3); - assert_eq!(config.topic_replication_factor(), None); assert_eq!(config.send_retries_count(), Some(3)); assert_eq!( config.send_retries_interval(), @@ -272,7 +261,6 @@ mod tests { assert_eq!(config.linger_time(), IggyDuration::from_str("5ms").unwrap()); assert_eq!(config.partitioning(), &Partitioning::balanced()); assert_eq!(config.topic_partitions_count(), 1); - assert_eq!(config.topic_replication_factor(), None); assert_eq!(config.send_retries_count(), Some(3)); assert_eq!( config.send_retries_interval(), @@ -291,7 +279,6 @@ mod tests { topic_id.clone(), String::from("test_topic"), 3, - None, 100, IggyDuration::from_str("5ms").unwrap(), Partitioning::balanced(), @@ -307,7 +294,6 @@ mod tests { assert_eq!(config.linger_time(), IggyDuration::from_str("5ms").unwrap()); assert_eq!(config.partitioning(), &Partitioning::balanced()); assert_eq!(config.topic_partitions_count(), 3); - assert_eq!(config.topic_replication_factor(), None); assert_eq!(config.send_retries_count(), None); assert_eq!(config.send_retries_interval(), None); } @@ -335,6 +321,5 @@ mod tests { assert_eq!(config.linger_time(), IggyDuration::from_str("5ms").unwrap()); assert_eq!(config.partitioning(), &Partitioning::balanced()); assert_eq!(config.topic_partitions_count(), 1); - assert_eq!(config.topic_replication_factor(), None); } } diff --git a/core/sdk/src/vsr.rs b/core/sdk/src/vsr.rs index e0c0efcbdc..b79f17ae6c 100644 --- a/core/sdk/src/vsr.rs +++ b/core/sdk/src/vsr.rs @@ -351,7 +351,7 @@ mod tests { use iggy_binary_protocol::requests::streams::CreateStreamRequest; use iggy_binary_protocol::requests::users::LoginRegisterRequest; use iggy_binary_protocol::version::IGGY_PROTOCOL_VERSION; - use iggy_binary_protocol::{ClientVersionInfo, WireEncode, WireName}; + use iggy_binary_protocol::{ClientVersionInfo, WireEncode, WireName, WireOptions}; use secrecy::SecretString; fn decode_request_header(bytes: &Bytes) -> RequestHeader { @@ -474,6 +474,7 @@ mod tests { session.bind(99); let payload = CreateStreamRequest { name: WireName::new("stream").unwrap(), + options: WireOptions::empty(), } .to_bytes(); diff --git a/core/server/config.toml b/core/server/config.toml index 418d4e9fd3..6b2c423f73 100644 --- a/core/server/config.toml +++ b/core/server/config.toml @@ -492,28 +492,18 @@ default_algorithm = "none" # Specifies the directory where stream data is stored, relative to `system.path`. path = "streams" -# Topic configuration - default settings for new topics +# Topic configuration [system.topic] # Path for storing topic-related data, relative to `stream.path`. path = "topics" -# Messages can be deleted based on two independent policies: -# 1. Size-based: delete oldest segments when topic exceeds max_size -# 2. Time-based: delete segments older than message_expiry -# Both can be active simultaneously. Per-topic overrides via CreateTopic/UpdateTopic. - -# Maximum topic size before oldest segments are deleted. -# "unlimited" or "0" = no size limit (messages kept indefinitely). -# When 90% of this limit is reached, oldest segments are removed to make room. -# Applies to sealed segments only (active segment is protected). -# Example: "10 GiB" -max_size = "unlimited" - -# Maximum age of messages before segments are deleted. -# "none" = no time limit (messages kept indefinitely). -# Applies to sealed segments only (active segment is protected). -# Example: "7 days", "2 days 4 hours 15 minutes" -message_expiry = "none" +# Retention is per topic, set at CreateTopic and readable on GetTopic: +# max_topic_size - delete oldest sealed segments past this size +# ("unlimited" when unset) +# message_expiry - delete sealed segments older than this ("none" when unset) +# Both policies can be active at once; the active segment is never touched. +# Call GET /options/topic (or the SDK's describe_options) for the full catalog +# with this server's defaults. # Partition configuration [system.partition] @@ -521,11 +511,6 @@ message_expiry = "none" # Specifies the directory where partition data is stored, relative to `topic.path`. path = "partitions" -# Determines whether to enforce file synchronization on partition updates (boolean). -# `true` ensures immediate writing of data to disk for durability. -# `false` allows the OS to manage write operations, which can improve performance. -enforce_fsync = false - # Enables checksum validation for data integrity (boolean). # `true` re-hashes every batch a disk poll reads and fails the poll closed on a # mismatch, so a segment damaged at rest is reported instead of served. @@ -534,26 +519,22 @@ enforce_fsync = false # somewhere else in the stack. validate_checksum = true -# The count threshold of buffered messages before triggering a save to disk. -# Together with `size_of_messages_required_to_save` it defines the threshold. -# This is a soft limit - actual count may be higher depending on last batch size. -# Minimum value is 1. -messages_required_to_save = 1024 - -# The size threshold of buffered messages before triggering a save to disk. -# Together with `messages_required_to_save` it defines the threshold. -# This is a soft limit - actual size may be higher depending on last batch size. -size_of_messages_required_to_save = "1 MiB" +# Durability and flush cadence are per topic, set at CreateTopic: +# enforce_fsync - fsync each write (false when unset) +# messages_required_to_save - flush after this many messages (1024) +# size_of_messages_required_to_save - flush after this many bytes (1 MiB) # Segment configuration [system.segment] -# Defines the soft limit for the size of a storage segment. -# When a segment reaches this size, a new segment is created for subsequent data. -# Example: if `size` is set "1GiB", the actual segment size may be 1GiB + the size of remaining messages in received batch. -# Maximum size is 1 GiB. Size has to be a multiple of 512 B. -size = "1 GiB" -# Reserves segment space in advance when supported by the local filesystem. -preallocate = true +# Segment size is per topic, set at CreateTopic as `segment_size` (1 GiB when +# unset). It is a soft limit: a segment may close one whole batch past it. +# Bounds are enforced at creation - a 512 B multiple, at least 1 MiB, and no +# larger than 1 GiB. +# +# `preallocate_segments` is per topic too: it reserves exactly `segment_size` +# up front where the filesystem supports it. Off unless a topic asks for it -- +# with the default 1 GiB segment size it costs 1 GiB of real disk per +# partition at creation. # Configures whether expired segments are archived (boolean) or just deleted without archiving. # Unsupported: setting this to `true` aborts boot. @@ -568,19 +549,6 @@ archive_expired = false # warns when set. cache_indexes = "open_segment" -# Message deduplication configuration -[system.message_deduplication] -# Controls whether message deduplication is enabled (boolean). -# `true` activates deduplication, ignoring messages with duplicate IDs. -# `false` treats each message as unique, even if IDs are duplicated. -# Unsupported: setting this to `true` aborts boot. -enabled = false -# Maximum number of ID entries in the deduplication cache (u64). -max_entries = 10000 -# Maximum age of ID entries in the deduplication cache in human-readable format. -expiry = "1 m" - -# Recovery configuration in case of lost data [system.recovery] # Controls whether streams/topics/partitions should be recreated if the expected data for existing state is missing (boolean). # Unsupported: setting this to `true` aborts boot. @@ -984,13 +952,19 @@ transfer_served_cache_bytes_max = "2176 MiB" # Alloc ceiling for ONE received state-transfer artifact, per shard. The # receiver holds it resident through verify, walk and staging write, and up to -# four transfers run at once. MUST cover system.segment.size plus -# message_bus.max_message_size (a segment may close one whole batch past its -# cap): under that, a legal segment is refused, the whole manifest with it, and -# the partition livelocks re-requesting it from every peer. Boot validates the -# floor. Raising this above the floor for headroom also DIVIDES the serving -# concurrency derived from transfer_served_cache_bytes_max above, so raise that -# in step. Must be > 0 and <= "64 GiB". +# four transfers run at once. MUST cover the largest segment any topic may be +# created with (a fixed 1 GiB, since segment_size is a per-topic option now) +# plus message_bus.max_message_size, because a segment may close one whole batch +# past its cap: under that, a legal segment is refused, the whole manifest with +# it, and the partition livelocks re-requesting it from every peer. Boot +# validates the floor. +# +# The shipped value sits EXACTLY at that floor: 1024 MiB + the shipped 64 MiB +# max_message_size. Raising max_message_size alone therefore refuses boot -- +# raise this one by the same amount in the same edit. Raising it above the floor +# for headroom also DIVIDES the serving concurrency derived from +# transfer_served_cache_bytes_max above, so raise that in step. Must be > 0 and +# <= "64 GiB". transfer_artifact_bytes_max = "1088 MiB" # Message bus configuration. diff --git a/core/server/src/bootstrap.rs b/core/server/src/bootstrap.rs index db46480702..475a8b36cb 100644 --- a/core/server/src/bootstrap.rs +++ b/core/server/src/bootstrap.rs @@ -50,7 +50,9 @@ use iggy_common::defaults::{ DEFAULT_ROOT_PASSWORD, DEFAULT_ROOT_USERNAME, MAX_PASSWORD_LENGTH, MAX_USERNAME_LENGTH, MIN_PASSWORD_LENGTH, MIN_USERNAME_LENGTH, }; -use iggy_common::{Aes256GcmEncryptor, EncryptorKind, IggyByteSize, PartitionStats, variadic}; +use iggy_common::{ + Aes256GcmEncryptor, EncryptorKind, IggyByteSize, PartitionStats, TopicRuntimeOptions, variadic, +}; use journal::prepare_journal::PrepareJournal; use journal::superblock::{PingPongSuperblock, SuperblockStore}; use journal::{Journal, JournalHandle}; @@ -1125,10 +1127,6 @@ async fn shard_main( // view-change superblock write records the real (checkpoint_op, checksum) // instead of (0, 0). No-op on peer shards, which have no coordinator. metadata.seed_checkpoint_ref(checkpoint_seed.0, checkpoint_seed.1); - // Shard 0's copy resolves the `ServerDefault` sentinels (max topic size and - // message expiry) at create admission; responses echo stored values verbatim. - metadata.set_default_max_topic_size(config.system.topic.max_size.as_bytes_u64()); - metadata.set_default_message_expiry(u64::from(config.system.topic.message_expiry)); // Keep the forced-checkpoint margin >= the configured prepare-queue // depth: ops already pipelined while a checkpoint runs append into that // margin (config validation keeps journal_slots >= 4x this). @@ -1778,15 +1776,14 @@ async fn build_shard_for_thread( let partitions = IggyPartitions::with_capacity( shard_local_id, PartitionsConfig { - messages_required_to_save: config.system.partition.messages_required_to_save, - size_of_messages_required_to_save: config - .system - .partition - .size_of_messages_required_to_save, - enforce_fsync: config.system.partition.enforce_fsync, + messages_required_to_save: iggy_common::DEFAULT_MESSAGES_REQUIRED_TO_SAVE, + size_of_messages_required_to_save: IggyByteSize::from( + iggy_common::DEFAULT_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE, + ), + enforce_fsync: iggy_common::DEFAULT_ENFORCE_FSYNC, validate_checksum: config.system.partition.validate_checksum, - segment_size: config.system.segment.size, - preallocate_segments: config.system.segment.preallocate, + segment_size: IggyByteSize::from(iggy_common::DEFAULT_SEGMENT_SIZE), + preallocate_segments: iggy_common::DEFAULT_PREALLOCATE_SEGMENTS, encryptor, }, owned_partitions_capacity, @@ -1815,7 +1812,13 @@ async fn build_shard_for_thread( partition.id, topic.stats.clone(), ); - owned.push((stream.id, topic_id, stats, partition.clone())); + owned.push(( + stream.id, + topic_id, + stats, + partition.clone(), + TopicRuntimeOptions::from_resource_options(&topic.options), + )); } else { shards_table.insert( namespace, @@ -1835,7 +1838,7 @@ async fn build_shard_for_thread( // bundle was broadcast (see `MetadataHandoff::Owner`). All shards // here only add their per-partition deltas, so the shared // `Arc` atomics race only against other atomic adds. - for (stream_id, topic_id, partition_stats, partition_metadata) in owned { + for (stream_id, topic_id, partition_stats, partition_metadata, topic_runtime) in owned { validate_namespace_bounds(config, stream_id, topic_id, partition_metadata.id)?; let namespace = IggyNamespace::new(stream_id, topic_id, partition_metadata.id); let partition = match load_partition( @@ -1843,6 +1846,7 @@ async fn build_shard_for_thread( namespace, Arc::clone(&partition_stats), &partition_metadata, + topic_runtime, topology.cluster_id, topology.self_replica_id, topology.replica_count, @@ -1911,6 +1915,7 @@ async fn build_shard_for_thread( namespace, partition_stats, partition_metadata.created_revision, + topic_runtime, topology.cluster_id, topology.self_replica_id, topology.replica_count, @@ -2385,23 +2390,21 @@ fn restore_metadata_consensus( } #[allow(clippy::too_many_arguments)] -async fn load_partition( +/// Build the ticked-and-bounded consensus a loaded partition group joins +/// with; the fresh-create path configures its own inside +/// `build_partition_fresh`. +fn loaded_partition_consensus( config: &ServerConfig, namespace: IggyNamespace, - stats: Arc, - partition_metadata: &Partition, cluster_id: u128, self_replica_id: u8, replica_count: u8, bus: Rc, -) -> Result>, ServerError> { - let stream_id = namespace.stream_id(); - let topic_id = namespace.topic_id(); - let partition_id = namespace.partition_id(); +) -> VsrConsensus> { // Request queue holds 2x the prepare depth (buffered requests drain as // prepares commit); depth is the per-partition `[partition]` knob. let prepare_queue_depth = config.partition.prepare_queue_depth; - let mut consensus = VsrConsensus::new( + let consensus = VsrConsensus::new( cluster_id, self_replica_id, replica_count, @@ -2416,6 +2419,72 @@ async fn load_partition( consensus.set_view_change_status_ticks(view_change_status_ticks(config)); consensus.set_request_start_view_ticks(request_start_view_ticks(config)); consensus.set_probe_attempts_max(config.cluster.view_probe_attempts_max); + consensus +} + +/// Recover this partition's persisted segment chain, stamping each segment +/// with the topic's effective segment size (the per-topic value when the +/// topic was created with one, else the shard-wide configured size). +async fn recover_partition_segments( + config: &ServerConfig, + namespace: IggyNamespace, + runtime_options: TopicRuntimeOptions, + stats: &PartitionStats, +) -> Result, ServerError> { + let stream_id = namespace.stream_id(); + let topic_id = namespace.topic_id(); + let partition_id = namespace.partition_id(); + let segment_size = runtime_options + .segment_size + .unwrap_or_else(|| IggyByteSize::from(iggy_common::DEFAULT_SEGMENT_SIZE)); + let enforce_fsync = runtime_options + .enforce_fsync + .unwrap_or(iggy_common::DEFAULT_ENFORCE_FSYNC); + load_persisted_segments( + config, + stream_id, + topic_id, + partition_id, + segment_size, + enforce_fsync, + stats, + ) + .await + .map_err(|source| { + error!( + stream_id, + topic_id, + partition_id, + error = %source, + "failed to load partition log during server bootstrap" + ); + source + }) +} + +#[allow(clippy::too_many_arguments)] +async fn load_partition( + config: &ServerConfig, + namespace: IggyNamespace, + stats: Arc, + partition_metadata: &Partition, + runtime_options: TopicRuntimeOptions, + cluster_id: u128, + self_replica_id: u8, + replica_count: u8, + bus: Rc, +) -> Result>, ServerError> { + let stream_id = namespace.stream_id(); + let topic_id = namespace.topic_id(); + let partition_id = namespace.partition_id(); + let mut consensus = loaded_partition_consensus( + config, + namespace, + cluster_id, + self_replica_id, + replica_count, + bus, + ); // (view, log_view) come from the group's durable superblock when present; // a present but unverifiable record already refused boot inside @@ -2464,20 +2533,10 @@ async fn load_partition( // regress persisted `base_timestamp`. let recovered_segments = - load_persisted_segments(config, stream_id, topic_id, partition_id, &stats) - .await - .map_err(|source| { - error!( - stream_id, - topic_id, - partition_id, - error = %source, - "failed to load partition log during server bootstrap" - ); - source - })?; + recover_partition_segments(config, namespace, runtime_options, &stats).await?; let mut partition = IggyPartition::new(stats.clone(), consensus); + partition.set_runtime_options(runtime_options); partition.set_superblock(superblock, recovered_state.as_ref()); // Recovered partitions honor the same config-surfaced ring ceilings as the // fresh-create path (build_partition_fresh). Retention is already off for @@ -2493,7 +2552,6 @@ async fn load_partition( partition.hydrate_applied_purge_generation().await?; hydrate_partition_log( &mut partition, - config, stream_id, topic_id, partition_id, @@ -2553,14 +2611,31 @@ async fn load_partition( Ok(partition) } +/// Reopen writers over a recovered segment chain. +/// +/// Takes no `&ServerConfig`: every knob it needs is the partition's own +/// resolved topic option now, which is the whole point of the per-topic move. async fn hydrate_partition_log( partition: &mut IggyPartition>, - config: &ServerConfig, stream_id: usize, topic_id: usize, partition_id: usize, recovered_segments: Vec, ) -> Result<(), ServerError> { + // The partition's own resolved knobs, not the shard-wide config: a topic + // created with `enforce_fsync` or a per-topic `segment_size` must get them + // on the writers reopened over its recovered chain too, or a restart would + // silently drop back to the node defaults. + let runtime = partition.runtime_options(); + let enforce_fsync = runtime + .enforce_fsync + .unwrap_or(iggy_common::DEFAULT_ENFORCE_FSYNC); + let segment_size = runtime + .segment_size + .unwrap_or_else(|| IggyByteSize::from(iggy_common::DEFAULT_SEGMENT_SIZE)); + let preallocate_segments = runtime + .preallocate_segments + .unwrap_or(iggy_common::DEFAULT_PREALLOCATE_SEGMENTS); for RecoveredSegment { segment, storage } in recovered_segments { partition .log @@ -2590,13 +2665,9 @@ async fn hydrate_partition_log( MessagesWriter::new( &messages_reader.path(), messages_size_counter, - config.system.partition.enforce_fsync, + enforce_fsync, true, - config - .system - .segment - .preallocate - .then_some(config.system.segment.size), + preallocate_segments.then_some(segment_size), ) .await .map_err(|source| { @@ -2612,24 +2683,19 @@ async fn hydrate_partition_log( })?, )); partition.log.index_writers_mut()[active_index] = Some(Rc::new( - IggyIndexWriter::new( - &index_path, - index_size_counter, - config.system.partition.enforce_fsync, - true, - ) - .await - .map_err(|source| { - error!( - stream_id, - topic_id, - partition_id, - path = %index_path, - error = %source, - "failed to initialize persisted sparse index writer" - ); - source - })?, + IggyIndexWriter::new(&index_path, index_size_counter, enforce_fsync, true) + .await + .map_err(|source| { + error!( + stream_id, + topic_id, + partition_id, + path = %index_path, + error = %source, + "failed to initialize persisted sparse index writer" + ); + source + })?, )); } } diff --git a/core/server/src/dispatch.rs b/core/server/src/dispatch.rs index 14684c3945..8ad3c146cd 100644 --- a/core/server/src/dispatch.rs +++ b/core/server/src/dispatch.rs @@ -74,10 +74,13 @@ use iggy_binary_protocol::requests::partitions::{ CreatePartitionsRequest, DeletePartitionsRequest, }; use iggy_binary_protocol::requests::segments::DeleteSegmentsRequest; +use iggy_binary_protocol::requests::streams::{CreateStreamRequest, UpdateStreamRequest}; use iggy_binary_protocol::requests::system::get_client::GetClientRequest; use iggy_binary_protocol::requests::system::get_snapshot::GetSnapshotRequest; -use iggy_binary_protocol::requests::topics::CreateTopicRequest; -use iggy_binary_protocol::requests::users::{LoginRegisterRequest, LoginRegisterWithPatRequest}; +use iggy_binary_protocol::requests::topics::{CreateTopicRequest, UpdateTopicRequest}; +use iggy_binary_protocol::requests::users::{ + CreateUserRequest, LoginRegisterRequest, LoginRegisterWithPatRequest, UpdateUserRequest, +}; use iggy_binary_protocol::responses::clients::client_response::ConsumerGroupInfoResponse; use iggy_binary_protocol::responses::clients::get_client::ClientDetailsResponse; use iggy_binary_protocol::responses::clients::get_clients::GetClientsResponse; @@ -88,10 +91,13 @@ use iggy_binary_protocol::{ ForwardLogoutOutcome, ForwardLogoutResultHeader, ForwardRegisterHeader, ForwardRegisterOutcome, ForwardRegisterResultHeader, GenericHeader, HEADER_SIZE, KIND_CONSUMER_GROUP, MAX_PARTITIONS_PER_REQUEST, Operation, ProtocolVersion, RequestHeader, RoutedRequestHeader, - WireDecode, WireEncode, WireIdentifier, is_protocol_compatible, + WireDecode, WireEncode, WireIdentifier, WireOptions, is_protocol_compatible, }; use iggy_common::{ - IggyError, MaxTopicSize, PollingStrategy, SnapshotCompression, SystemSnapshotType, + IggyByteSize, IggyError, MaxTopicSize, PollingStrategy, SnapshotCompression, + SystemSnapshotType, TopicCreateOptions, UPDATABLE_STREAM_OPTION_KEYS, + UPDATABLE_TOPIC_OPTION_KEYS, UPDATABLE_USER_OPTION_KEYS, validate_preallocated_topic_bytes, + validate_topic_segment_size, }; use journal::superblock::SuperblockStore; use journal::{Journal, JournalHandle}; @@ -808,23 +814,51 @@ pub(crate) const fn validate_partitions_change_count( /// and `prepare_request` errors evict the session instead of denying typed. /// `ServerDefault` is exempt from the size floor (it resolves against server /// config at admission, matching legacy); `Unlimited` passes numerically. +/// `segment_size_bytes` is the topic's RESOLVED segment size (explicit +/// option, else this node's default), so a per-topic segment above the +/// global default still floors the topic cap. pub(crate) fn validate_topic_bounds( - system_config: &ServerSystemConfig, partitions_count: u32, max_topic_size: MaxTopicSize, + segment_size_bytes: u64, ) -> Result<(), IggyError> { validate_partitions_count(partitions_count)?; + validate_topic_size_floor(max_topic_size, segment_size_bytes) +} + +/// A topic cap below one segment can never be enforced: the first segment +/// already exceeds it. Split out of [`validate_topic_bounds`] because update +/// admission checks the cap without a partitions count to check. +pub(crate) fn validate_topic_size_floor( + max_topic_size: MaxTopicSize, + segment_size_bytes: u64, +) -> Result<(), IggyError> { if !matches!(max_topic_size, MaxTopicSize::ServerDefault) - && max_topic_size.as_bytes_u64() < system_config.segment.size.as_bytes_u64() + && max_topic_size.as_bytes_u64() < segment_size_bytes { return Err(IggyError::InvalidTopicSize( max_topic_size, - system_config.segment.size, + IggyByteSize::from(segment_size_bytes), )); } Ok(()) } +/// Reject option keys outside the resource's catalog, pre-consensus. Unknown +/// keys are rejected rather than skipped: a silently ignored knob would hand +/// the client server defaults without it ever learning. Streams and users +/// have no catalog keys yet, so `known` is empty for both until one lands. +pub(crate) fn validate_option_keys(options: &WireOptions, known: &[&str]) -> Result<(), IggyError> { + for entry in options { + // Wire validation already enforced UTF-8 string keys. + let key = String::from_utf8_lossy(entry.key); + if !known.contains(&key.as_ref()) { + return Err(IggyError::UnsupportedOptionKey(key.into_owned())); + } + } + Ok(()) +} + /// Reject a request before it reaches consensus: warn, then send the typed /// deny reply. A silent drop would wedge every later request on the /// connection until the socket read timeout. `context` labels the rejection @@ -1122,10 +1156,31 @@ async fn handle_client_request( Operation::CreateTopic => CreateTopicRequest::decode_from(request_body(&request)) .map_err(|_| IggyError::InvalidCommand) .and_then(|create_topic| { + // `parse` doubles as the catalog gate: an unknown key or a + // malformed value denies typed here, pre-consensus. + let options = TopicCreateOptions::parse(&create_topic.options)?; + if let Some(segment_size) = options.segment_size { + validate_topic_segment_size( + segment_size.as_bytes_u64(), + iggy_common::MAX_TOPIC_SEGMENT_SIZE, + )?; + } + let segment_size = options.segment_size.map_or_else( + || iggy_common::DEFAULT_SEGMENT_SIZE, + |segment_size| segment_size.as_bytes_u64(), + ); + if options + .preallocate_segments + .unwrap_or(iggy_common::DEFAULT_PREALLOCATE_SEGMENTS) + { + validate_preallocated_topic_bytes(segment_size, create_topic.partitions_count)?; + } validate_topic_bounds( - system_config, create_topic.partitions_count, - MaxTopicSize::from(create_topic.max_topic_size), + options + .max_topic_size + .unwrap_or(MaxTopicSize::ServerDefault), + segment_size, ) }), Operation::CreatePartitions => CreatePartitionsRequest::decode_from(request_body(&request)) @@ -1138,6 +1193,48 @@ async fn handle_client_request( .and_then(|delete_partitions| { validate_partitions_change_count(delete_partitions.partitions_count) }), + // Only the updatable subset: the create-time knobs are pushed to + // partitions when the topic is built and nothing re-pushes them, so + // accepting one here would store a value no partition ever sees. + Operation::UpdateTopic => UpdateTopicRequest::decode_from(request_body(&request)) + .map_err(|_| IggyError::InvalidCommand) + .and_then(|update_topic| { + validate_option_keys(&update_topic.options, UPDATABLE_TOPIC_OPTION_KEYS)?; + let options = TopicCreateOptions::parse(&update_topic.options)?; + let Some(max_topic_size) = options.max_topic_size else { + return Ok(()); + }; + // An update can lower the cap below one segment just as a + // create can, and the stored map would then report a size the + // topic can never enforce. The floor is this topic's own + // segment size, since that key is create-only. + let metadata = shard.plane.metadata(); + let segment_size = metadata + .mux_stm + .streams() + .topic_segment_size(&update_topic.stream_id, &update_topic.topic_id) + .map_or_else( + || iggy_common::DEFAULT_SEGMENT_SIZE, + |segment_size| segment_size.as_bytes_u64(), + ); + validate_topic_size_floor(max_topic_size, segment_size) + }), + Operation::UpdateStream => UpdateStreamRequest::decode_from(request_body(&request)) + .map_err(|_| IggyError::InvalidCommand) + .and_then(|update_stream| { + validate_option_keys(&update_stream.options, UPDATABLE_STREAM_OPTION_KEYS) + }), + Operation::UpdateUser => UpdateUserRequest::decode_from(request_body(&request)) + .map_err(|_| IggyError::InvalidCommand) + .and_then(|update_user| { + validate_option_keys(&update_user.options, UPDATABLE_USER_OPTION_KEYS) + }), + Operation::CreateStream => CreateStreamRequest::decode_from(request_body(&request)) + .map_err(|_| IggyError::InvalidCommand) + .and_then(|create_stream| validate_option_keys(&create_stream.options, &[])), + Operation::CreateUser => CreateUserRequest::decode_from(request_body(&request)) + .map_err(|_| IggyError::InvalidCommand) + .and_then(|create_user| validate_option_keys(&create_user.options, &[])), _ => Ok(()), }; if let Err(error) = bounds { @@ -4017,6 +4114,7 @@ mod tests { // below by driving `on_ack` by hand.) let create_body = CreateStreamRequest { name: iggy_binary_protocol::primitives::identifier::WireName::new("s1").unwrap(), + options: WireOptions::empty(), } .to_bytes(); let prepare = prepare_message(Operation::CreateStream, CLIENT_B, 1, &create_body); @@ -4110,6 +4208,7 @@ mod tests { md.mux_stm.users().ensure_root_user("iggy", "hash"); let create_stream = CreateStreamRequest { name: WireName::new("stream").unwrap(), + options: WireOptions::empty(), }; md.mux_stm .update(prepare_message( @@ -4123,12 +4222,10 @@ mod tests { request: CreateTopicRequest { stream_id: WireIdentifier::numeric(0), partitions_count: 1, - compression_algorithm: 0, - message_expiry: 0, - max_topic_size: 0, - replication_factor: 1, name: WireName::new("topic").unwrap(), + options: WireOptions::empty(), }, + derived_options: WireOptions::empty(), partitions: vec![CreatedPartitionAssignment { partition_id: 0, consensus_group_id: 1, @@ -4830,15 +4927,14 @@ mod tests { #[test] fn create_topic_bounds_deny_pre_consensus() { - let config = ServerSystemConfig::default(); - let segment_size = config.segment.size.as_bytes_u64(); + let segment_size = iggy_common::DEFAULT_SEGMENT_SIZE; assert!(segment_size > 0, "default segment size must be nonzero"); assert!( validate_topic_bounds( - &config, MAX_PARTITIONS_PER_REQUEST, - MaxTopicSize::ServerDefault + MaxTopicSize::ServerDefault, + segment_size ) .is_ok(), "the partition cap itself is admissible" @@ -4846,9 +4942,9 @@ mod tests { assert!( matches!( validate_topic_bounds( - &config, MAX_PARTITIONS_PER_REQUEST + 1, - MaxTopicSize::ServerDefault + MaxTopicSize::ServerDefault, + segment_size ), Err(IggyError::TooManyPartitions) ), @@ -4856,20 +4952,20 @@ mod tests { ); // ServerDefault is numerically 0 yet exempt from the segment-size // floor: it resolves against server config, matching legacy. - assert!(validate_topic_bounds(&config, 1, MaxTopicSize::ServerDefault).is_ok()); - assert!(validate_topic_bounds(&config, 1, MaxTopicSize::Unlimited).is_ok()); + assert!(validate_topic_bounds(1, MaxTopicSize::ServerDefault, segment_size).is_ok()); + assert!(validate_topic_bounds(1, MaxTopicSize::Unlimited, segment_size).is_ok()); let below_floor = MaxTopicSize::Custom((segment_size - 1).into()); assert!( matches!( - validate_topic_bounds(&config, 1, below_floor), + validate_topic_bounds(1, below_floor, segment_size), Err(IggyError::InvalidTopicSize(size, floor)) - if size == below_floor && floor == config.segment.size + if size == below_floor && floor == IggyByteSize::from(segment_size) ), "custom size below the segment size must deny with the bounds" ); - let at_floor = MaxTopicSize::Custom(config.segment.size); + let at_floor = MaxTopicSize::Custom(IggyByteSize::from(segment_size)); assert!( - validate_topic_bounds(&config, 1, at_floor).is_ok(), + validate_topic_bounds(1, at_floor, segment_size).is_ok(), "a topic exactly one segment large is admissible" ); } diff --git a/core/server/src/dispatch/authz.rs b/core/server/src/dispatch/authz.rs index 78129156a3..df4047b7bb 100644 --- a/core/server/src/dispatch/authz.rs +++ b/core/server/src/dispatch/authz.rs @@ -29,9 +29,9 @@ use std::rc::Rc; use consensus::MetadataHandle; use iggy_binary_protocol::codes::{ - GET_CLUSTER_METADATA_CODE, GET_CONSUMER_GROUP_CODE, GET_CONSUMER_GROUPS_CODE, - GET_PERSONAL_ACCESS_TOKENS_CODE, GET_STATS_CODE, GET_STREAM_CODE, GET_STREAMS_CODE, - GET_TOPIC_CODE, GET_TOPICS_CODE, GET_USER_CODE, GET_USERS_CODE, + DESCRIBE_OPTIONS_CODE, GET_CLUSTER_METADATA_CODE, GET_CONSUMER_GROUP_CODE, + GET_CONSUMER_GROUPS_CODE, GET_PERSONAL_ACCESS_TOKENS_CODE, GET_STATS_CODE, GET_STREAM_CODE, + GET_STREAMS_CODE, GET_TOPIC_CODE, GET_TOPICS_CODE, GET_USER_CODE, GET_USERS_CODE, }; use iggy_binary_protocol::requests::consumer_groups::{ GetConsumerGroupRequest, GetConsumerGroupsRequest, @@ -284,6 +284,9 @@ where // Self-scoped: lists only the caller's own tokens, so there is no // permissioner rule to run (legacy runs none either). GET_PERSONAL_ACCESS_TOKENS_CODE => user_id.map(|_| ()).ok_or(IggyError::Unauthenticated), + // Static catalog plus node defaults; nothing resource-scoped to gate + // beyond authentication. + DESCRIBE_OPTIONS_CODE => user_id.map(|_| ()).ok_or(IggyError::Unauthenticated), // Defence in depth: `handle_client_request` already denies an unbound // transport with an `Unauthenticated` Reply before it reaches the // builder, so this arm only ever fires if that gate is bypassed. diff --git a/core/server/src/http.rs b/core/server/src/http.rs index d3e10ca58a..b58498cb90 100644 --- a/core/server/src/http.rs +++ b/core/server/src/http.rs @@ -69,9 +69,9 @@ use crate::cluster_meta::ClusterRoster; use crate::http::handlers::{ change_password, create_cg, create_partitions, create_pat, create_stream, create_topic, create_user, delete_cg, delete_consumer_offset, delete_partitions, delete_pat, delete_segments, - delete_stream, delete_topic, delete_user, get_cg, get_cgs, get_client, get_clients, - get_cluster_metadata, get_consumer_offset, get_pats, get_snapshot, get_stats, get_stream, - get_streams, get_topic, get_topics, get_user, get_users, login_user, + delete_stream, delete_topic, delete_user, describe_options, get_cg, get_cgs, get_client, + get_clients, get_cluster_metadata, get_consumer_offset, get_pats, get_snapshot, get_stats, + get_stream, get_streams, get_topic, get_topics, get_user, get_users, login_user, login_with_personal_access_token, logout_user, ping, poll_messages, purge_stream, purge_topic, refresh_token, send_messages, store_consumer_offset, update_permissions, update_stream, update_topic, update_user, @@ -302,6 +302,7 @@ fn router( delete(delete_consumer_offset), ) .route("/stats", get(get_stats)) + .route("/options/{scope}", get(describe_options)) .route("/snapshot", post(get_snapshot)) .route("/cluster/metadata", get(get_cluster_metadata)) .route("/clients", get(get_clients)) diff --git a/core/server/src/http/handlers.rs b/core/server/src/http/handlers.rs index cc5d501c56..f63549491a 100644 --- a/core/server/src/http/handlers.rs +++ b/core/server/src/http/handlers.rs @@ -19,6 +19,7 @@ //! the control-plane writes, and the data-plane produce / poll / consumer-offset //! routes the router binds. +use std::str::FromStr; use std::sync::Arc; use axum::Json; @@ -29,9 +30,9 @@ use axum::response::{IntoResponse, Response}; use chrono::Local; use consensus::{MetadataHandle, PartitionsHandle}; use iggy_binary_protocol::codes::{ - GET_CONSUMER_GROUP_CODE, GET_CONSUMER_GROUPS_CODE, GET_PERSONAL_ACCESS_TOKENS_CODE, - GET_STATS_CODE, GET_STREAM_CODE, GET_STREAMS_CODE, GET_TOPIC_CODE, GET_TOPICS_CODE, - GET_USER_CODE, GET_USERS_CODE, + DESCRIBE_OPTIONS_CODE, GET_CONSUMER_GROUP_CODE, GET_CONSUMER_GROUPS_CODE, + GET_PERSONAL_ACCESS_TOKENS_CODE, GET_STATS_CODE, GET_STREAM_CODE, GET_STREAMS_CODE, + GET_TOPIC_CODE, GET_TOPICS_CODE, GET_USER_CODE, GET_USERS_CODE, }; use iggy_binary_protocol::requests::consumer_groups::{ CreateConsumerGroupRequest, DeleteConsumerGroupRequest, GetConsumerGroupRequest, @@ -49,6 +50,7 @@ use iggy_binary_protocol::requests::streams::{ CreateStreamRequest, DeleteStreamRequest, GetStreamRequest, GetStreamsRequest, PurgeStreamRequest, UpdateStreamRequest, }; +use iggy_binary_protocol::requests::system::DescribeOptionsRequest; use iggy_binary_protocol::requests::system::GetStatsRequest; use iggy_binary_protocol::requests::topics::{ CreateTopicRequest, DeleteTopicRequest, GetTopicRequest, GetTopicsRequest, PurgeTopicRequest, @@ -66,12 +68,15 @@ use iggy_binary_protocol::responses::consumer_groups::get_consumer_groups::GetCo use iggy_binary_protocol::responses::personal_access_tokens::GetPersonalAccessTokensResponse; use iggy_binary_protocol::responses::streams::get_stream::GetStreamResponse; use iggy_binary_protocol::responses::streams::get_streams::GetStreamsResponse; +use iggy_binary_protocol::responses::system::DescribeOptionsResponse; use iggy_binary_protocol::responses::system::get_stats::StatsResponse; use iggy_binary_protocol::responses::topics::get_topic::GetTopicResponse; use iggy_binary_protocol::responses::topics::get_topics::GetTopicsResponse; use iggy_binary_protocol::responses::users::get_user::UserDetailsResponse; use iggy_binary_protocol::responses::users::get_users::GetUsersResponse; -use iggy_binary_protocol::{Operation, WireDecode, WireEncode, WireIdentifier, WireName}; +use iggy_binary_protocol::{ + Operation, WireDecode, WireEncode, WireIdentifier, WireName, WireOptions, +}; use iggy_common::change_password::ChangePassword; use iggy_common::create_consumer_group::CreateConsumerGroup; use iggy_common::create_partitions::CreatePartitions; @@ -92,14 +97,19 @@ use iggy_common::update_stream::UpdateStream; use iggy_common::update_topic::UpdateTopic; use iggy_common::update_user::UpdateUser; use iggy_common::wire_conversions::{ - clients_from_wire, consumer_groups_from_wire, identifier_to_wire, permissions_to_wire, - personal_access_tokens_from_wire, streams_from_wire, topics_from_wire, users_from_wire, + clients_from_wire, consumer_groups_from_wire, identifier_to_wire, option_specs_from_wire, + permissions_to_wire, personal_access_tokens_from_wire, streams_from_wire, topics_from_wire, + users_from_wire, }; use iggy_common::{ - ClientInfo, ClientInfoDetails, ClusterMetadata, Consumer, ConsumerGroup, ConsumerGroupDetails, - ConsumerOffsetInfo, Identifier, IdentityInfo, IggyError, PersonalAccessTokenInfo, PollMessages, - PolledMessages, RawPersonalAccessToken, SendMessages, SendMessagesConfirmations, Stats, Stream, - StreamDetails, TokenInfo, Topic, TopicDetails, UserInfo, UserInfoDetails, Validatable, + ClientInfo, ClientInfoDetails, ClusterMetadata, CompressionAlgorithm, Consumer, ConsumerGroup, + ConsumerGroupDetails, ConsumerOffsetInfo, Identifier, IdentityInfo, IggyError, IggyExpiry, + MaxTopicSize, OptionSpec, OptionsScope, PersonalAccessTokenInfo, PollMessages, PolledMessages, + RawPersonalAccessToken, SendMessages, SendMessagesConfirmations, Stats, Stream, StreamDetails, + StreamUpdateOptions, TokenInfo, Topic, TopicCreateOptions, TopicDetails, TopicUpdateOptions, + UPDATABLE_STREAM_OPTION_KEYS, UPDATABLE_TOPIC_OPTION_KEYS, UPDATABLE_USER_OPTION_KEYS, + UserInfo, UserInfoDetails, UserUpdateOptions, Validatable, validate_preallocated_topic_bytes, + validate_topic_segment_size, }; use metadata::impls::metadata::StreamsFrontend; use metadata::permissioner::Permissioner; @@ -110,11 +120,12 @@ use shard::{PartitionRead, PartitionReadReply}; use crate::auth::{verify_login_credentials, verify_pat_credentials}; use crate::dispatch::{ - resolve_consumer_offset_request, resolve_poll_request, validate_topic_bounds, + resolve_consumer_offset_request, resolve_poll_request, validate_option_keys, + validate_topic_bounds, validate_topic_size_floor, }; use crate::http::error::{ - ConsistencyQuery, CustomError, PartitionWriteError, ProduceAck, ProduceQuery, ReadError, - WriteError, + Consistency, ConsistencyQuery, CustomError, PartitionWriteError, ProduceAck, ProduceQuery, + ReadError, WriteError, }; use crate::http::extractor::{Authenticated, Identity}; use crate::http::reads::{ @@ -256,6 +267,36 @@ pub(in crate::http) async fn logout_user( StatusCode::NO_CONTENT } +/// `GET /options/{scope}`: describe the option catalog for one resource +/// scope (`topic`, `stream`, `user`) as `Vec` JSON. A +/// consensus-free local read via [`read_local`], gated on authentication +/// only (the catalog is not resource-scoped). +pub(in crate::http) async fn describe_options( + State(state): State, + identity: Identity, + Path(scope): Path, +) -> Result>, ReadError> { + let scope = OptionsScope::from_str(&scope).map_err(ReadError::Rejected)?; + let body = DescribeOptionsRequest { + scope: scope.as_code(), + } + .to_bytes(); + let bytes = SendWrapper::new(read_local( + &state, + &identity, + Consistency::default(), + DESCRIBE_OPTIONS_CODE, + &body, + |_, _| Ok(()), + )) + .await?; + let response = DescribeOptionsResponse::decode_from(&bytes) + .map_err(|_| ReadError::Rejected(IggyError::InvalidCommand))?; + Ok(Json( + option_specs_from_wire(response).map_err(ReadError::Rejected)?, + )) +} + /// `GET /streams`: list every stream as the same `Vec` JSON the legacy /// server returns. A consensus-free local STM read via [`read_local`]. pub(in crate::http) async fn get_streams( @@ -275,7 +316,9 @@ pub(in crate::http) async fn get_streams( .await?; let response = GetStreamsResponse::decode_from(&bytes) .map_err(|_| ReadError::Rejected(IggyError::InvalidCommand))?; - Ok(Json(streams_from_wire(response))) + Ok(Json( + streams_from_wire(response).map_err(ReadError::Rejected)?, + )) } /// `GET /streams/{stream_id}`: fetch one stream by numeric id or name as the @@ -701,6 +744,7 @@ pub(in crate::http) async fn create_stream( let request = CreateStreamRequest { name: WireName::new(command.name) .map_err(|_| WriteError::Rejected(IggyError::InvalidStreamName))?, + options: WireOptions::empty(), }; let body = request.to_bytes(); let payload = SendWrapper::new(submit_write( @@ -722,10 +766,20 @@ pub(in crate::http) async fn update_stream( Json(command): Json, ) -> Result { let stream_id = Identifier::from_str_value(&stream_id).map_err(WriteError::Rejected)?; + let wire_options = StreamUpdateOptions { + raw: command.options, + } + .to_wire() + .map_err(WriteError::Rejected)?; + // Same pre-consensus gate the TCP ingress applies: a key this command may + // not change is denied here by name rather than riding a log entry. + validate_option_keys(&wire_options, UPDATABLE_STREAM_OPTION_KEYS) + .map_err(WriteError::Rejected)?; let request = UpdateStreamRequest { stream_id: identifier_to_wire(&stream_id).map_err(WriteError::Rejected)?, name: WireName::new(command.name) .map_err(|_| WriteError::Rejected(IggyError::InvalidStreamName))?, + options: wire_options, }; let body = request.to_bytes(); SendWrapper::new(submit_write( @@ -794,23 +848,54 @@ pub(in crate::http) async fn create_topic( Json(command): Json, ) -> Result, WriteError> { let stream_id = Identifier::from_str_value(&stream_id).map_err(WriteError::Rejected)?; - // Rejects empty/oversized name, partitions_count > MAX, replication_factor == Some(0). + // Rejects empty/oversized name and partitions_count > MAX. command.validate().map_err(WriteError::Rejected)?; + let options = TopicCreateOptions { + partitions_count: Some(command.partitions_count), + compression_algorithm: (command.compression_algorithm != CompressionAlgorithm::default()) + .then_some(command.compression_algorithm), + message_expiry: (command.message_expiry != IggyExpiry::ServerDefault) + .then_some(command.message_expiry), + max_topic_size: (command.max_topic_size != MaxTopicSize::ServerDefault) + .then_some(command.max_topic_size), + raw: command.options, + ..TopicCreateOptions::default() + }; + let wire_options = options.to_wire().map_err(WriteError::Rejected)?; + // Re-parse the encoded block so `--set`-style raw string entries get the + // same typed pre-consensus checks as native fields; unknown keys deny + // here with the key name. + let parsed = TopicCreateOptions::parse(&wire_options).map_err(WriteError::Rejected)?; + if let Some(segment_size) = parsed.segment_size { + validate_topic_segment_size( + segment_size.as_bytes_u64(), + iggy_common::MAX_TOPIC_SEGMENT_SIZE, + ) + .map_err(WriteError::Rejected)?; + } + let segment_size = parsed.segment_size.map_or_else( + || iggy_common::DEFAULT_SEGMENT_SIZE, + |segment_size| segment_size.as_bytes_u64(), + ); + if parsed + .preallocate_segments + .unwrap_or(iggy_common::DEFAULT_PREALLOCATE_SEGMENTS) + { + validate_preallocated_topic_bytes(segment_size, command.partitions_count) + .map_err(WriteError::Rejected)?; + } validate_topic_bounds( - &state.system_config, command.partitions_count, - command.max_topic_size, + parsed.max_topic_size.unwrap_or(MaxTopicSize::ServerDefault), + segment_size, ) .map_err(WriteError::Rejected)?; let request = CreateTopicRequest { stream_id: identifier_to_wire(&stream_id).map_err(WriteError::Rejected)?, partitions_count: command.partitions_count, - compression_algorithm: command.compression_algorithm.as_code(), - message_expiry: command.message_expiry.into(), - max_topic_size: command.max_topic_size.into(), - replication_factor: command.replication_factor.unwrap_or(0), name: WireName::new(command.name) .map_err(|_| WriteError::Rejected(IggyError::InvalidTopicName))?, + options: wire_options, }; let body = request.to_bytes(); let payload = SendWrapper::new(submit_write( @@ -832,17 +917,46 @@ pub(in crate::http) async fn update_topic( ) -> Result { let stream_id = Identifier::from_str_value(&stream_id).map_err(WriteError::Rejected)?; let topic_id = Identifier::from_str_value(&topic_id).map_err(WriteError::Rejected)?; - // Also rejects replication_factor == Some(0), which `WireName` cannot see. command.validate().map_err(WriteError::Rejected)?; + // The named JSON fields fold into the same option keys the binary protocol + // uses, so REST keeps its ergonomics without giving a setting two homes. + let wire_options = TopicUpdateOptions { + compression_algorithm: command.compression_algorithm, + message_expiry: command.message_expiry, + max_topic_size: command.max_topic_size, + raw: command.options, + } + .to_wire() + .map_err(WriteError::Rejected)?; + // Same pre-consensus gates the TCP ingress applies: a key this command may + // not change is denied here by name rather than riding a log entry, and a + // cap below one of this topic's segments is denied for the same reason it is + // on create. + validate_option_keys(&wire_options, UPDATABLE_TOPIC_OPTION_KEYS) + .map_err(WriteError::Rejected)?; + let stream_wire = identifier_to_wire(&stream_id).map_err(WriteError::Rejected)?; + let topic_wire = identifier_to_wire(&topic_id).map_err(WriteError::Rejected)?; + if let Some(max_topic_size) = TopicCreateOptions::parse(&wire_options) + .map_err(WriteError::Rejected)? + .max_topic_size + { + let metadata = state.shard.plane.metadata(); + let segment_size = metadata + .mux_stm + .streams() + .topic_segment_size(&stream_wire, &topic_wire) + .map_or_else( + || iggy_common::DEFAULT_SEGMENT_SIZE, + |segment_size| segment_size.as_bytes_u64(), + ); + validate_topic_size_floor(max_topic_size, segment_size).map_err(WriteError::Rejected)?; + } let request = UpdateTopicRequest { - stream_id: identifier_to_wire(&stream_id).map_err(WriteError::Rejected)?, - topic_id: identifier_to_wire(&topic_id).map_err(WriteError::Rejected)?, - compression_algorithm: command.compression_algorithm.as_code(), - message_expiry: command.message_expiry.into(), - max_topic_size: command.max_topic_size.into(), - replication_factor: command.replication_factor.unwrap_or(0), + stream_id: stream_wire, + topic_id: topic_wire, name: WireName::new(command.name) .map_err(|_| WriteError::Rejected(IggyError::InvalidTopicName))?, + options: wire_options, }; let body = request.to_bytes(); SendWrapper::new(submit_write( @@ -1371,6 +1485,7 @@ pub(in crate::http) async fn create_user( password: command.password.expose_secret().to_string(), status: command.status.as_code(), permissions: command.permissions.as_ref().map(permissions_to_wire), + options: WireOptions::empty(), }; let body = request.to_bytes(); let payload = SendWrapper::new(submit_write( @@ -1393,6 +1508,13 @@ pub(in crate::http) async fn update_user( let user_id = Identifier::from_str_value(&user_id).map_err(WriteError::Rejected)?; // Rejects an oversized replacement username; a no-op when username is absent. command.validate().map_err(WriteError::Rejected)?; + let user_update_options = UserUpdateOptions { + raw: command.options, + } + .to_wire() + .map_err(WriteError::Rejected)?; + validate_option_keys(&user_update_options, UPDATABLE_USER_OPTION_KEYS) + .map_err(WriteError::Rejected)?; let request = UpdateUserRequest { user_id: identifier_to_wire(&user_id).map_err(WriteError::Rejected)?, username: command @@ -1402,6 +1524,7 @@ pub(in crate::http) async fn update_user( .transpose() .map_err(|_| WriteError::Rejected(IggyError::InvalidUsername))?, status: command.status.map(|status| status.as_code()), + options: user_update_options, }; let body = request.to_bytes(); SendWrapper::new(submit_write( diff --git a/core/server/src/partition_helpers.rs b/core/server/src/partition_helpers.rs index 48ea83a791..e3117fdd37 100644 --- a/core/server/src/partition_helpers.rs +++ b/core/server/src/partition_helpers.rs @@ -30,7 +30,8 @@ use compio::fs::create_dir_all; use configs::server::ServerConfig; use consensus::{LocalPipeline, VsrConsensus, VsrState}; use iggy_common::{ - ConsumerGroupOffsets, ConsumerOffsets, IggyError, IggyTimestamp, PartitionStats, + ConsumerGroupOffsets, ConsumerOffsets, IggyByteSize, IggyError, IggyTimestamp, PartitionStats, + TopicRuntimeOptions, }; use journal::superblock::{PingPongSuperblock, SuperblockContents}; use message_bus::IggyMessageBus; @@ -254,12 +255,19 @@ pub fn configure_consumer_offsets( } } + // Offset files follow the topic's own `enforce_fsync`: they are part of the + // same partition's durability story, and the global knob they used to read + // is gone. + let enforce_fsync = partition + .runtime_options() + .enforce_fsync + .unwrap_or(iggy_common::DEFAULT_ENFORCE_FSYNC); partition.configure_consumer_offset_storage( consumer_offsets_path, consumer_group_offsets_path, consumer_offsets, consumer_group_offsets, - config.system.partition.enforce_fsync, + enforce_fsync, ); Ok(()) } @@ -354,7 +362,16 @@ pub async fn ensure_initial_segment( let index_path = config .system .get_index_path(stream_id, topic_id, partition_id, start_offset); - let enforce_fsync = config.system.partition.enforce_fsync; + let runtime = partition.runtime_options(); + let segment_size = runtime + .segment_size + .unwrap_or_else(|| IggyByteSize::from(iggy_common::DEFAULT_SEGMENT_SIZE)); + let enforce_fsync = runtime + .enforce_fsync + .unwrap_or(iggy_common::DEFAULT_ENFORCE_FSYNC); + let preallocate_segments = runtime + .preallocate_segments + .unwrap_or(iggy_common::DEFAULT_PREALLOCATE_SEGMENTS); // `file_exists = false` TRUNCATES both files, which is load-bearing here: a // fenced-and-rebuilt partition (or one whose quarantine failed) can reach // this with a stale `.index` at offset 0 on disk. The `partitions`-side @@ -393,19 +410,15 @@ pub async fn ensure_initial_segment( .map(|writer| writer.size_counter()) .unwrap_or_default(); partition.log.add_persisted_segment( - Segment::new(start_offset, config.system.segment.size), + Segment::new(start_offset, segment_size), storage, Some(Rc::new( MessagesWriter::new( &messages_path, messages_size_counter, - config.system.partition.enforce_fsync, + enforce_fsync, false, - config - .system - .segment - .preallocate - .then_some(config.system.segment.size), + preallocate_segments.then_some(segment_size), ) .await .map_err(|source| { @@ -421,24 +434,19 @@ pub async fn ensure_initial_segment( })?, )), Some(Rc::new( - IggyIndexWriter::new( - &index_path, - index_size_counter, - config.system.partition.enforce_fsync, - false, - ) - .await - .map_err(|source| { - error!( - stream_id, - topic_id, - partition_id, - path = %index_path, - error = %source, - "failed to initialize initial sparse index writer" - ); - source - })?, + IggyIndexWriter::new(&index_path, index_size_counter, enforce_fsync, false) + .await + .map_err(|source| { + error!( + stream_id, + topic_id, + partition_id, + path = %index_path, + error = %source, + "failed to initialize initial sparse index writer" + ); + source + })?, )), ); partition.stats.increment_segments_count(1); @@ -586,6 +594,7 @@ pub async fn build_partition_fresh( namespace: IggyNamespace, stats: Arc, created_revision: u64, + runtime_options: TopicRuntimeOptions, cluster_id: u128, self_replica_id: u8, replica_count: u8, @@ -677,6 +686,7 @@ pub async fn build_partition_fresh( } let mut partition = IggyPartition::new(stats, consensus); + partition.set_runtime_options(runtime_options); partition.set_superblock(superblock, recovered_state.as_ref()); // Surface the evicted-ring ceilings from config onto the fresh journal. // IggyPartition::new has already disabled retention for single-replica diff --git a/core/server/src/partition_reconciler.rs b/core/server/src/partition_reconciler.rs index 1305bcf7b8..278c2afb3e 100644 --- a/core/server/src/partition_reconciler.rs +++ b/core/server/src/partition_reconciler.rs @@ -641,7 +641,7 @@ async fn reconcile_additions( // built, not once per committed partition every pass. A topic that // vanished between the target snapshot and this read defers to the // next pass. - let Some(partition_stats) = fetch_partition_stats(ctx, ns) else { + let Some((partition_stats, topic_runtime)) = fetch_partition_stats(ctx, ns) else { continue; }; @@ -650,6 +650,7 @@ async fn reconcile_additions( ns, partition_stats, epoch, + topic_runtime, ctx.cluster_id, ctx.self_replica_id, ctx.replica_count, @@ -1049,17 +1050,23 @@ fn current_revision(ctx: &ReconcilerCtx) -> u64 { fn fetch_partition_stats( ctx: &ReconcilerCtx, ns: IggyNamespace, -) -> Option> { +) -> Option<( + Arc, + iggy_common::TopicRuntimeOptions, +)> { ctx.shard.plane.metadata().mux_stm.streams().read(|inner| { let stream = inner.items.get(ns.stream_id())?; let topic = stream.topics.get(ns.topic_id())?; // Get-or-create in the shared registry so the owning shard's counters // are the same `Arc` every shard's `get_topic` reply reads. - Some(inner.stats_registry.partition( - ns.stream_id(), - ns.topic_id(), - ns.partition_id(), - topic.stats.clone(), + Some(( + inner.stats_registry.partition( + ns.stream_id(), + ns.topic_id(), + ns.partition_id(), + topic.stats.clone(), + ), + iggy_common::TopicRuntimeOptions::from_resource_options(&topic.options), )) }) } @@ -1176,6 +1183,7 @@ mod tests { }; use iggy_binary_protocol::{ Command2, Operation, PrepareHeader, ReplyHeader, RoutedRequestHeader, WireIdentifier, + WireOptions, }; use message_bus::IggyMessageBus; use metadata::IggyMetadata; @@ -1321,6 +1329,7 @@ mod tests { fn seed_stream(mux: &TestMux, op: u64, name: &str) { let req = CreateStreamRequest { name: WireName::new(name).expect("test stream name fits WireName"), + options: WireOptions::empty(), }; mux.update(build_prepare(op, Operation::CreateStream, &req)) .expect("CreateStream apply succeeds"); @@ -1337,14 +1346,11 @@ mod tests { let req = CreateTopicWithAssignmentsRequest { request: CreateTopicRequest { stream_id: WireIdentifier::numeric(stream_id), - partitions_count: u32::try_from(assignments.len()) - .expect("partitions count fits u32"), - compression_algorithm: 0, - message_expiry: 0, - max_topic_size: 0, - replication_factor: 1, + partitions_count: 1, name: WireName::new(name).expect("test topic name fits WireName"), + options: WireOptions::empty(), }, + derived_options: WireOptions::empty(), partitions: assignments, }; mux.update(build_prepare( @@ -1458,7 +1464,7 @@ mod tests { size_of_messages_required_to_save: iggy_common::IggyByteSize::from(1024_u64), enforce_fsync: false, validate_checksum: true, - segment_size: config.system.segment.size, + segment_size: iggy_common::IggyByteSize::from(iggy_common::DEFAULT_SEGMENT_SIZE), preallocate_segments: false, encryptor: None, }, @@ -1645,7 +1651,7 @@ mod tests { // `ensure_initial_segment` plants exactly one segment per build and // folds it into the namespace's shared stats, so this counter is the // observable that separates them. - let stats = fetch_partition_stats(&ctx, ns).expect("materialised namespace has stats"); + let (stats, _) = fetch_partition_stats(&ctx, ns).expect("materialised namespace has stats"); assert_eq!( stats.segments_count_inconsistent(), 1, @@ -1682,12 +1688,13 @@ mod tests { let ctx = make_ctx(Rc::clone(&shard), 1, Rc::new(config.clone())); let ns = IggyNamespace::new(0, 0, 0); - let stats = fetch_partition_stats(&ctx, ns).expect("committed namespace has stats"); + let (stats, _) = fetch_partition_stats(&ctx, ns).expect("committed namespace has stats"); let live = build_partition_fresh( &config, ns, Arc::clone(&stats), LIVE_EPOCH, + iggy_common::TopicRuntimeOptions::default(), CLUSTER_ID, 0, 1, @@ -1716,6 +1723,7 @@ mod tests { ns, Arc::clone(&stats), LIVE_EPOCH + 1, + iggy_common::TopicRuntimeOptions::default(), CLUSTER_ID, 0, 1, diff --git a/core/server/src/responses.rs b/core/server/src/responses.rs index 6c6812ab49..5431d677f9 100644 --- a/core/server/src/responses.rs +++ b/core/server/src/responses.rs @@ -31,10 +31,10 @@ use bytes::{Bytes, BytesMut}; use consensus::{MetadataHandle, VsrConsensus}; use iggy_binary_protocol::PrepareHeader; use iggy_binary_protocol::codes::{ - FLUSH_UNSAVED_BUFFER_CODE, GET_CLUSTER_METADATA_CODE, GET_CONSUMER_GROUP_CODE, - GET_CONSUMER_GROUPS_CODE, GET_PERSONAL_ACCESS_TOKENS_CODE, GET_SNAPSHOT_FILE_CODE, - GET_STATS_CODE, GET_STREAM_CODE, GET_STREAMS_CODE, GET_TOPIC_CODE, GET_TOPICS_CODE, - GET_USER_CODE, GET_USERS_CODE, + DESCRIBE_OPTIONS_CODE, FLUSH_UNSAVED_BUFFER_CODE, GET_CLUSTER_METADATA_CODE, + GET_CONSUMER_GROUP_CODE, GET_CONSUMER_GROUPS_CODE, GET_PERSONAL_ACCESS_TOKENS_CODE, + GET_SNAPSHOT_FILE_CODE, GET_STATS_CODE, GET_STREAM_CODE, GET_STREAMS_CODE, GET_TOPIC_CODE, + GET_TOPICS_CODE, GET_USER_CODE, GET_USERS_CODE, }; use iggy_binary_protocol::consensus::{RESULT_COUNT_LEN, result_code}; use iggy_binary_protocol::primitives::consumer::WireConsumer; @@ -49,6 +49,9 @@ use iggy_binary_protocol::requests::messages::SendMessagesHeader; use iggy_binary_protocol::requests::personal_access_tokens::GetPersonalAccessTokensRequest; use iggy_binary_protocol::requests::segments::DeleteSegmentsRequest; use iggy_binary_protocol::requests::streams::{GetStreamRequest, GetStreamsRequest}; +use iggy_binary_protocol::requests::system::{ + DescribeOptionsRequest, OPTIONS_SCOPE_STREAM, OPTIONS_SCOPE_TOPIC, OPTIONS_SCOPE_USER, +}; use iggy_binary_protocol::requests::topics::{GetTopicRequest, GetTopicsRequest}; use iggy_binary_protocol::requests::users::GetUserRequest; use iggy_binary_protocol::responses::clients::client_response::ClientResponse; @@ -68,6 +71,7 @@ use iggy_binary_protocol::responses::system::get_cluster_metadata::{ ClusterMetadataResponse, ClusterNodeResponse, }; use iggy_binary_protocol::responses::system::get_stats::StatsResponse; +use iggy_binary_protocol::responses::system::{DescribeOptionsResponse, OptionDescriptor}; use iggy_binary_protocol::responses::topics::get_topic::{GetTopicResponse, PartitionResponse}; use iggy_binary_protocol::responses::topics::get_topics::GetTopicsResponse; use iggy_binary_protocol::responses::users::LoginRegisterResponse; @@ -78,7 +82,11 @@ use iggy_binary_protocol::{ Command2, GenericHeader, IGGY_PROTOCOL_VERSION, KIND_CONSUMER_GROUP, Operation, ReplyHeader, RoutedRequestHeader, WireDecode, WireEncode, WireIdentifier, WireName, WirePartitioning, }; -use iggy_common::{EncryptorKind, Identifier, IggyError, IggyTimestamp}; +use iggy_common::wire_conversions::{resource_options_to_wire, resource_options_to_wire_split}; +use iggy_common::{ + EncryptorKind, HeaderKind, Identifier, IggyError, IggyTimestamp, OptionsProvenance, + topic_option_keys, +}; use journal::superblock::SuperblockStore; use journal::{Journal, JournalHandle}; use metadata::impls::metadata::StreamsFrontend; @@ -528,6 +536,9 @@ where SB: SuperblockStore + 'static, { match code { + DESCRIBE_OPTIONS_CODE => Ok(NonReplicatedResponse::Bytes( + build_describe_options_response(body)?.to_bytes(), + )), GET_CLUSTER_METADATA_CODE => Ok(NonReplicatedResponse::Bytes( build_cluster_metadata_response(roster, shard, client_ip).to_bytes(), )), @@ -594,34 +605,8 @@ where personal_access_tokens_response(tokens)?.to_bytes(), )) } - GET_CONSUMER_GROUP_CODE => { - let request = GetConsumerGroupRequest::decode_from(body) - .map_err(|_| IggyError::InvalidCommand)?; - ensure_topic_exists(shard, &request.stream_id, &request.topic_id)?; - let response = shard - .plane - .metadata() - .mux_stm - .streams() - .consumer_group_details(&request.stream_id, &request.topic_id, &request.group_id); - Ok(response.map_or(NonReplicatedResponse::Empty, |response| { - NonReplicatedResponse::Bytes(response.to_bytes()) - })) - } - GET_CONSUMER_GROUPS_CODE => { - let request = GetConsumerGroupsRequest::decode_from(body) - .map_err(|_| IggyError::InvalidCommand)?; - ensure_topic_exists(shard, &request.stream_id, &request.topic_id)?; - let groups = shard - .plane - .metadata() - .mux_stm - .streams() - .consumer_group_list(&request.stream_id, &request.topic_id); - Ok(groups.map_or(NonReplicatedResponse::Empty, |groups| { - NonReplicatedResponse::Bytes(GetConsumerGroupsResponse { groups }.to_bytes()) - })) - } + GET_CONSUMER_GROUP_CODE => build_consumer_group_response(shard, body), + GET_CONSUMER_GROUPS_CODE => build_consumer_groups_response(shard, body), // The server has no on-demand flush primitive, so it denies honestly. // The non-replicated catch-all's empty-ok would otherwise attest a // durability guarantee the server never gave. @@ -640,6 +625,56 @@ where } } +fn build_consumer_group_response( + shard: &Rc>, + body: &[u8], +) -> Result +where + B: ShellBus, + MJ: JournalHandle + 'static, + MJ::Target: Journal, Header = PrepareHeader>, + S: 'static, + SB: SuperblockStore + 'static, +{ + let request = + GetConsumerGroupRequest::decode_from(body).map_err(|_| IggyError::InvalidCommand)?; + ensure_topic_exists(shard, &request.stream_id, &request.topic_id)?; + let response = shard + .plane + .metadata() + .mux_stm + .streams() + .consumer_group_details(&request.stream_id, &request.topic_id, &request.group_id); + Ok(response.map_or(NonReplicatedResponse::Empty, |response| { + NonReplicatedResponse::Bytes(response.to_bytes()) + })) +} + +fn build_consumer_groups_response( + shard: &Rc>, + body: &[u8], +) -> Result +where + B: ShellBus, + MJ: JournalHandle + 'static, + MJ::Target: Journal, Header = PrepareHeader>, + S: 'static, + SB: SuperblockStore + 'static, +{ + let request = + GetConsumerGroupsRequest::decode_from(body).map_err(|_| IggyError::InvalidCommand)?; + ensure_topic_exists(shard, &request.stream_id, &request.topic_id)?; + let groups = shard + .plane + .metadata() + .mux_stm + .streams() + .consumer_group_list(&request.stream_id, &request.topic_id); + Ok(groups.map_or(NonReplicatedResponse::Empty, |groups| { + NonReplicatedResponse::Bytes(GetConsumerGroupsResponse { groups }.to_bytes()) + })) +} + /// Build the binary `GetClusterMetadata` reply from the shared roster assembly. /// The leader marking comes from this shard's consensus view; a shard without /// consensus (any shard but 0) still serves the full roster, only with no node @@ -943,6 +978,126 @@ where }) } +/// Every key `CreateTopic` accepts, with the kind, default and bounds of each. +/// +/// Split out of [`build_describe_options_response`] so the descriptions have room +/// to state the bounds each value is checked against: this catalog is the only +/// place an operator learns them. +/// +/// Every default is a build constant: these knobs stopped being config-derived +/// when the `[system.*]` keys became topic options, so the catalog reads them +/// straight from `iggy_common`. +fn topic_option_descriptors() -> Result, IggyError> { + Ok(vec![ + OptionDescriptor { + key: WireName::new(topic_option_keys::COMPRESSION_ALGORITHM) + .map_err(|_| IggyError::InvalidFormat)?, + kind: HeaderKind::String.as_code(), + default_value: Bytes::from_static(b"none"), + description: "Compression algorithm (none, gzip)".to_string(), + }, + OptionDescriptor { + key: WireName::new(topic_option_keys::MESSAGE_EXPIRY) + .map_err(|_| IggyError::InvalidFormat)?, + kind: HeaderKind::Uint64.as_code(), + default_value: Bytes::copy_from_slice( + &iggy_common::DEFAULT_MESSAGE_EXPIRY.to_le_bytes(), + ), + description: "Message expiry in microseconds, or a humantime string \ + (e.g. 7 days)" + .to_string(), + }, + OptionDescriptor { + key: WireName::new(topic_option_keys::MAX_TOPIC_SIZE) + .map_err(|_| IggyError::InvalidFormat)?, + kind: HeaderKind::Uint64.as_code(), + default_value: Bytes::copy_from_slice( + &iggy_common::DEFAULT_MAX_TOPIC_SIZE.to_le_bytes(), + ), + description: "Topic size cap in bytes, or a byte-size string (e.g. 1 GiB); \ + must be at least the segment size" + .to_string(), + }, + OptionDescriptor { + key: WireName::new(topic_option_keys::SEGMENT_SIZE) + .map_err(|_| IggyError::InvalidFormat)?, + kind: HeaderKind::Uint64.as_code(), + default_value: Bytes::copy_from_slice(&iggy_common::DEFAULT_SEGMENT_SIZE.to_le_bytes()), + description: format!( + "Segment size in bytes, or a byte-size string (e.g. 128 MiB); a 512-byte \ + multiple within {}..={}", + iggy_common::MIN_TOPIC_SEGMENT_SIZE, + iggy_common::MAX_TOPIC_SEGMENT_SIZE + ), + }, + OptionDescriptor { + key: WireName::new(topic_option_keys::ENFORCE_FSYNC) + .map_err(|_| IggyError::InvalidFormat)?, + kind: HeaderKind::Bool.as_code(), + default_value: Bytes::copy_from_slice(&[u8::from(iggy_common::DEFAULT_ENFORCE_FSYNC)]), + description: "Whether writes to this topic's partitions fsync".to_string(), + }, + OptionDescriptor { + key: WireName::new(topic_option_keys::MESSAGES_REQUIRED_TO_SAVE) + .map_err(|_| IggyError::InvalidFormat)?, + kind: HeaderKind::Uint32.as_code(), + default_value: Bytes::copy_from_slice( + &iggy_common::DEFAULT_MESSAGES_REQUIRED_TO_SAVE.to_le_bytes(), + ), + description: format!( + "Flush the journal once it holds this many messages; \ + 1..={}. A threshold no segment can reach leaves committed \ + messages in the journal, which a crash does not preserve", + iggy_common::MAX_MESSAGES_REQUIRED_TO_SAVE + ), + }, + OptionDescriptor { + key: WireName::new(topic_option_keys::SIZE_OF_MESSAGES_REQUIRED_TO_SAVE) + .map_err(|_| IggyError::InvalidFormat)?, + kind: HeaderKind::Uint64.as_code(), + default_value: Bytes::copy_from_slice( + &iggy_common::DEFAULT_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE.to_le_bytes(), + ), + description: format!( + "Flush the journal once it holds this many bytes, or a byte-size \ + string; whichever threshold trips first flushes. At most {}", + iggy_common::MAX_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE + ), + }, + OptionDescriptor { + key: WireName::new(topic_option_keys::PREALLOCATE_SEGMENTS) + .map_err(|_| IggyError::InvalidFormat)?, + kind: HeaderKind::Bool.as_code(), + default_value: Bytes::copy_from_slice(&[u8::from( + iggy_common::DEFAULT_PREALLOCATE_SEGMENTS, + )]), + description: format!( + "Reserve each segment's bytes up front where the filesystem supports \ + it; pairs with segment_size. The reservation is real disk and runs \ + inline on the owning shard, at every rotation and once per owned \ + partition at boot, so segment_size * partitions_count is capped at \ + {} bytes", + iggy_common::MAX_PREALLOCATED_TOPIC_BYTES + ), + }, + ]) +} + +/// Serve the option catalog for one resource scope. +/// +/// Streams and users have no catalog keys yet, so their scopes return empty +/// (every key is rejected at create until one lands). +fn build_describe_options_response(body: &[u8]) -> Result { + let request = + DescribeOptionsRequest::decode_from(body).map_err(|_| IggyError::InvalidCommand)?; + let entries = match request.scope { + OPTIONS_SCOPE_TOPIC => topic_option_descriptors()?, + OPTIONS_SCOPE_STREAM | OPTIONS_SCOPE_USER => Vec::new(), + _ => return Err(IggyError::InvalidCommand), + }; + Ok(DescribeOptionsResponse { entries }) +} + #[allow(clippy::cast_possible_truncation)] fn user_response(user: &metadata::stm::user::User) -> Result { Ok(UserResponse { @@ -950,6 +1105,7 @@ fn user_response(user: &metadata::stm::user::User) -> Result Result Result Result { + let (options, derived_options) = resource_options_to_wire_split(&topic.options)?; Ok(StreamTopicHeader { id: usize_to_u32(topic.id)?, created_at: topic.created_at.as_micros(), @@ -1191,10 +1349,11 @@ fn topic_header(topic: &metadata::stm::stream::Topic) -> Result) { let streams = shard.plane.metadata().mux_stm.streams(); // Resolved once per pass, not per partition: the node default is a `Cell` // written at bootstrap and never after. - let default_max_topic_size = shard.plane.metadata().default_max_topic_size(); for namespace in namespaces { let Some((message_expiry, max_topic_size, partition_count)) = streams.topic_retention_config(namespace.stream_id(), namespace.topic_id()) @@ -78,8 +77,11 @@ fn stage_owned_partitions(shard: &Rc) { message_expiry, IggyExpiry::NeverExpire | IggyExpiry::ServerDefault ); - let max_bytes = - per_partition_size_budget(max_topic_size, default_max_topic_size, partition_count); + let max_bytes = per_partition_size_budget( + max_topic_size, + iggy_common::DEFAULT_MAX_TOPIC_SIZE, + partition_count, + ); if !has_expiry && max_bytes.is_none() { continue; diff --git a/core/server/src/segment_recovery.rs b/core/server/src/segment_recovery.rs index ecdeaa1709..b909b78f03 100644 --- a/core/server/src/segment_recovery.rs +++ b/core/server/src/segment_recovery.rs @@ -65,6 +65,8 @@ pub async fn load_persisted_segments( stream_id: usize, topic_id: usize, partition_id: usize, + segment_size: IggyByteSize, + enforce_fsync: bool, stats: &PartitionStats, ) -> Result, ServerError> { let partition_path = config @@ -78,8 +80,7 @@ pub async fn load_persisted_segments( let mut start_offsets = sweep_scratch_files_and_collect_offsets(&partition_path)?; start_offsets.sort_unstable(); - let enforce_fsync = config.system.partition.enforce_fsync; - let max_size = config.system.segment.size; + let max_size = segment_size; let mut recovered = Vec::with_capacity(start_offsets.len()); for start_offset in start_offsets { diff --git a/core/server_common/Cargo.toml b/core/server_common/Cargo.toml index 69d8af0ca4..d0f7057f6f 100644 --- a/core/server_common/Cargo.toml +++ b/core/server_common/Cargo.toml @@ -46,8 +46,6 @@ futures = { workspace = true } human-repr = { workspace = true } iggy_binary_protocol = { workspace = true } iggy_common = { workspace = true } -lending-iterator = { workspace = true } -moka = { workspace = true } opentelemetry = { workspace = true } opentelemetry-appender-tracing = { workspace = true } opentelemetry-otlp = { workspace = true } @@ -73,4 +71,3 @@ nix = { workspace = true } [dev-dependencies] serial_test = { workspace = true } tempfile = { workspace = true } -tokio = { workspace = true } diff --git a/core/server_common/src/deduplication/message_deduplicator.rs b/core/server_common/src/deduplication/message_deduplicator.rs deleted file mode 100644 index 2a11f8e099..0000000000 --- a/core/server_common/src/deduplication/message_deduplicator.rs +++ /dev/null @@ -1,125 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use iggy_common::IggyDuration; -use moka::future::{Cache, CacheBuilder}; - -#[derive(Debug)] -pub struct MessageDeduplicator { - ttl: Option, - max_entries: Option, - cache: Cache, -} - -/// Create deep copy of the `MessageDeduplicator` instance. -/// Regular `Clone` cheap as it only creates thread-safe reference counted -/// pointers to the shared internal data structures. -impl Clone for MessageDeduplicator { - fn clone(&self) -> Self { - let builder = Cache::builder(); - let builder = Self::setup_cache_builder(builder, self.max_entries, self.ttl); - let cache = builder.build(); - - Self { - ttl: self.ttl, - max_entries: self.max_entries, - cache, - } - } -} - -impl MessageDeduplicator { - fn setup_cache_builder( - mut builder: CacheBuilder>, - max_entries: Option, - ttl: Option, - ) -> CacheBuilder> { - if let Some(max_entries) = max_entries { - builder = builder.max_capacity(max_entries); - } - if let Some(ttl) = ttl { - builder = builder.time_to_live(ttl.get_duration()); - } - builder - } - - /// Creates a new message deduplicator with the given max entries and time to live for each ID. - pub fn new(max_entries: Option, ttl: Option) -> Self { - let builder = Cache::builder(); - let builder = Self::setup_cache_builder(builder, max_entries, ttl); - let cache = builder.build(); - - Self { - ttl, - max_entries, - cache, - } - } - - /// Checks if the given ID exists. - pub fn exists(&self, id: u128) -> bool { - self.cache.contains_key(&id) - } - - /// Inserts the given ID. - pub async fn insert(&self, id: u128) { - self.cache.insert(id, true).await - } - - /// Tries to insert the given ID, returns false if it already exists. - pub async fn try_insert(&self, id: u128) -> bool { - if self.exists(id) { - false - } else { - self.insert(id).await; - true - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn message_deduplicator_should_insert_only_unique_identifiers() { - let max_entries = 1000; - let ttl = "1s".parse::().unwrap(); - let deduplicator = MessageDeduplicator::new(Some(max_entries), Some(ttl)); - for i in 0..max_entries { - let id = i as u128; - assert!(deduplicator.try_insert(id).await); - assert!(deduplicator.exists(id)); - assert!(!deduplicator.try_insert(id).await); - } - } - - #[tokio::test] - async fn message_deduplicator_should_evict_identifiers_after_given_time_to_live() { - let max_entries = 3; - let ttl = "100ms".parse::().unwrap(); - let deduplicator = MessageDeduplicator::new(Some(max_entries), Some(ttl)); - for i in 0..max_entries { - let id = i as u128; - assert!(deduplicator.try_insert(id).await); - assert!(deduplicator.exists(id)); - tokio::time::sleep(2 * ttl.get_duration()).await; - assert!(!deduplicator.exists(id)); - assert!(deduplicator.try_insert(id).await); - } - } -} diff --git a/core/server_common/src/deduplication/mod.rs b/core/server_common/src/deduplication/mod.rs deleted file mode 100644 index 3a42ef6118..0000000000 --- a/core/server_common/src/deduplication/mod.rs +++ /dev/null @@ -1,20 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -mod message_deduplicator; - -pub use message_deduplicator::MessageDeduplicator; diff --git a/core/server_common/src/lib.rs b/core/server_common/src/lib.rs index b61ee4ea76..0e94ce3569 100644 --- a/core/server_common/src/lib.rs +++ b/core/server_common/src/lib.rs @@ -20,7 +20,6 @@ mod buffer; mod certificates; mod consensus_message; pub mod crypto; -mod deduplication; pub mod diagnostics; pub mod executor; pub mod fs_utils; @@ -55,7 +54,6 @@ pub use consensus_message::{ ConsensusMessage, FragmentedBacking, MESSAGE_ALIGN, Message, MessageBacking, MessageBag, MutableBacking, RequestBacking, RequestBackingKind, ResponseBacking, ResponseBackingKind, }; -pub use deduplication::MessageDeduplicator; pub use executor::create_shard_executor; pub use in_flight::IggyMessagesBatchSetInFlight; pub use indexes_mut::IggyIndexesMut; diff --git a/core/server_common/src/messages_batch_mut.rs b/core/server_common/src/messages_batch_mut.rs index 6dd71d695a..902eeda518 100644 --- a/core/server_common/src/messages_batch_mut.rs +++ b/core/server_common/src/messages_batch_mut.rs @@ -15,19 +15,15 @@ // specific language governing permissions and limitations // under the License. -use crate::MessageDeduplicator; use crate::PooledBuffer; use crate::indexes_mut::IggyIndexesMut; use iggy_common::{ IGGY_MESSAGE_HEADER_SIZE, INDEX_SIZE, IggyByteSize, IggyError, IggyIndexView, IggyMessage, IggyMessageBoundaries, IggyMessageView, IggyMessageViewIterator, IggyMessageViewMutIterator, - IggyMessagesBatch, IggyTimestamp, MAX_PAYLOAD_SIZE, MAX_USER_HEADERS_SIZE, Sizeable, - Validatable, random_id, + IggyMessagesBatch, MAX_PAYLOAD_SIZE, MAX_USER_HEADERS_SIZE, Sizeable, Validatable, }; -use lending_iterator::prelude::*; use std::ops::Index; -use std::sync::Arc; -use tracing::{error, warn}; +use tracing::error; /// A container for mutable messages that are being prepared for persistence. /// @@ -122,87 +118,6 @@ impl IggyMessagesBatchMut { &self.messages } - /// Prepares all messages in the batch for persistence by setting their offsets, - /// timestamps, and other necessary fields. - /// - /// # Arguments - /// - /// * `start_offset` - The starting offset of the segment - /// * `base_offset` - The base offset for this batch of messages - /// * `current_position` - The current position in the segment - /// - /// # Returns - /// - /// An immutable `IggyMessagesBatch` ready for persistence - pub async fn prepare_for_persistence( - &mut self, - start_offset: u64, - base_offset: u64, - current_position: u32, - deduplicator: Option<&Arc>, - ) { - let messages_count = self.count(); - if messages_count == 0 { - return; - } - - let mut curr_abs_offset = base_offset; - let mut curr_position = current_position; - let mut curr_rel_offset: u32 = 0; - - // Prepare invalid messages indexes if deduplicator is provided, this - // way we avoid creating a new vector if we don't need it. - // The less allocation the better. - let mut invalid_messages_indexes = - deduplicator.map(|_| Vec::with_capacity(messages_count as usize)); - - self.indexes.set_base_position(current_position); - let mut iter: IggyMessageViewMutIterator<'_> = - IggyMessageViewMutIterator::new(&mut self.messages); - let timestamp = IggyTimestamp::now().as_micros(); - - while let Some(mut message) = iter.next() { - message.header_mut().set_offset(curr_abs_offset); - message.header_mut().set_timestamp(timestamp); - if message.header().id() == 0 { - message.header_mut().set_id(random_id::get_uuid()); - } - - if let Some(deduplicator) = deduplicator - && !deduplicator.try_insert(message.header().id()).await - { - warn!( - "Detected duplicate message ID {}, removing...", - message.header().id() - ); - invalid_messages_indexes - .as_mut() - .unwrap() - .push(curr_rel_offset); - } - - message.update_checksum(); - - let message_size = message.size() as u32; - curr_position += message_size; - - let relative_offset = (curr_abs_offset - start_offset) as u32; - self.indexes.set_offset_at(curr_rel_offset, relative_offset); - self.indexes.set_position_at(curr_rel_offset, curr_position); - self.indexes.set_timestamp_at(curr_rel_offset, timestamp); - - curr_abs_offset += 1; - curr_rel_offset += 1; - } - - if let Some(invalid_messages_indexes) = invalid_messages_indexes { - if invalid_messages_indexes.is_empty() { - return; - } - self.remove_messages(&invalid_messages_indexes, current_position); - } - } - /// Returns the first offset in the batch pub fn first_offset(&self) -> Option { if self.is_empty() { diff --git a/core/simulator/src/client.rs b/core/simulator/src/client.rs index 02ea373231..db73cfb517 100644 --- a/core/simulator/src/client.rs +++ b/core/simulator/src/client.rs @@ -44,7 +44,7 @@ use iggy_binary_protocol::requests::users::{ }; use iggy_binary_protocol::{ AckLevel, ClientVersionInfo, IGGY_PROTOCOL_VERSION, Operation, RoutedRequestHeader, WireEncode, - WireIdentifier, WireName, WirePartitioning, WirePollingStrategy, + WireIdentifier, WireName, WireOptions, WirePartitioning, WirePollingStrategy, }; use metadata::stm::user::{CreatePersonalAccessTokenRequest, DeletePersonalAccessTokenRequest}; use secrecy::SecretString; @@ -236,6 +236,7 @@ impl SimClient { pub fn create_stream(&self, name: &str) -> Message { let wire = CreateStreamRequest { name: WireName::new(name).expect("stream name must be valid"), + options: WireOptions::empty(), }; let payload = wire.to_bytes(); @@ -260,6 +261,7 @@ impl SimClient { let wire = UpdateStreamRequest { stream_id: WireIdentifier::named(stream).expect("stream name must be valid"), name: WireName::new(new_name).expect("stream name must be valid"), + options: WireOptions::empty(), }; self.build_request(Operation::UpdateStream, &wire.to_bytes()) } @@ -284,11 +286,8 @@ impl SimClient { let wire = CreateTopicRequest { stream_id: WireIdentifier::named(stream).expect("stream name must be valid"), partitions_count, - compression_algorithm: 0, - message_expiry: 0, - max_topic_size: 0, - replication_factor: 1, name: WireName::new(name).expect("topic name must be valid"), + options: WireOptions::empty(), }; self.build_request(Operation::CreateTopic, &wire.to_bytes()) } @@ -304,11 +303,8 @@ impl SimClient { let wire = UpdateTopicRequest { stream_id: WireIdentifier::named(stream).expect("stream name must be valid"), topic_id: WireIdentifier::named(topic).expect("topic name must be valid"), - compression_algorithm: 0, - message_expiry: 0, - max_topic_size: 0, - replication_factor: 1, name: WireName::new(new_name).expect("topic name must be valid"), + options: WireOptions::empty(), }; self.build_request(Operation::UpdateTopic, &wire.to_bytes()) } @@ -428,6 +424,7 @@ impl SimClient { password: password.to_string(), status, permissions: None, + options: WireOptions::empty(), }; self.build_request(Operation::CreateUser, &wire.to_bytes()) } @@ -445,6 +442,7 @@ impl SimClient { user_id: WireIdentifier::named(user).expect("username must be valid"), username: new_username.map(|n| WireName::new(n).expect("username must be valid")), status, + options: WireOptions::empty(), }; self.build_request(Operation::UpdateUser, &wire.to_bytes()) } diff --git a/core/simulator/src/workload/ops/create_stream.rs b/core/simulator/src/workload/ops/create_stream.rs index cf5e68ace8..685301d5d0 100644 --- a/core/simulator/src/workload/ops/create_stream.rs +++ b/core/simulator/src/workload/ops/create_stream.rs @@ -49,6 +49,9 @@ pub fn sample( Outcome::NameAlreadyExists => Some(Input { name: shadow.pick_stream_name(prng)?, }), + // Not a targeted outcome (absent from `OUTCOMES`); the sim client + // never sends an options block. + Outcome::InvalidOptionValue => None, } } @@ -72,6 +75,6 @@ pub fn predicted_effect(input: &Input, outcome: Outcome) -> Effect { Outcome::Ok => Effect::AddStream { name: input.name.clone(), }, - Outcome::NameAlreadyExists => Effect::None, + Outcome::NameAlreadyExists | Outcome::InvalidOptionValue => Effect::None, } } diff --git a/core/simulator/src/workload/ops/create_topic.rs b/core/simulator/src/workload/ops/create_topic.rs index 07fd654a1d..8eff46fd67 100644 --- a/core/simulator/src/workload/ops/create_topic.rs +++ b/core/simulator/src/workload/ops/create_topic.rs @@ -80,6 +80,9 @@ pub fn sample( partitions_count, }) } + // Not a targeted outcome (absent from `OUTCOMES`); the sim client + // only sends catalog keys with valid values. + Outcome::InvalidOptionValue => None, } } diff --git a/core/simulator/src/workload/ops/create_user.rs b/core/simulator/src/workload/ops/create_user.rs index c5bce99db5..c002d9bbc3 100644 --- a/core/simulator/src/workload/ops/create_user.rs +++ b/core/simulator/src/workload/ops/create_user.rs @@ -47,9 +47,10 @@ pub fn sample( let username = match outcome { Outcome::Ok => shadow.fresh_name("user"), Outcome::UserAlreadyExists => shadow.pick_user_name(prng)?, - // Not a targeted outcome (absent from `OUTCOMES`); the shadow only ever - // mints in-bounds names, so an invalid-length username is never sampled. - Outcome::InvalidUsername => return None, + // Not targeted outcomes (absent from `OUTCOMES`); the shadow only ever + // mints in-bounds names, and the sim client never sends an options + // block. + Outcome::InvalidUsername | Outcome::InvalidOptionValue => return None, }; Some(Input { password: format!("pw-{username}"), @@ -78,6 +79,8 @@ pub fn predicted_effect(input: &Input, outcome: Outcome) -> Effect { Outcome::Ok => Effect::AddUser { name: input.username.clone(), }, - Outcome::UserAlreadyExists | Outcome::InvalidUsername => Effect::None, + Outcome::UserAlreadyExists | Outcome::InvalidUsername | Outcome::InvalidOptionValue => { + Effect::None + } } } diff --git a/core/simulator/src/workload/ops/update_stream.rs b/core/simulator/src/workload/ops/update_stream.rs index f6f2285096..0832fa1e9b 100644 --- a/core/simulator/src/workload/ops/update_stream.rs +++ b/core/simulator/src/workload/ops/update_stream.rs @@ -59,6 +59,9 @@ pub fn sample( Outcome::NameAlreadyExists => { unreachable!("update_stream does not target NameAlreadyExists") } + Outcome::InvalidOptionValue => { + unreachable!("the simulator only sends an empty update options block") + } } } diff --git a/core/simulator/src/workload/ops/update_topic.rs b/core/simulator/src/workload/ops/update_topic.rs index a4e36367e5..6df1a03e43 100644 --- a/core/simulator/src/workload/ops/update_topic.rs +++ b/core/simulator/src/workload/ops/update_topic.rs @@ -73,6 +73,9 @@ pub fn sample( Outcome::NameAlreadyExists => { unreachable!("update_topic does not target NameAlreadyExists") } + Outcome::InvalidOptionValue => { + unreachable!("the simulator only sends an empty update options block") + } } } diff --git a/core/simulator/src/workload/ops/update_user.rs b/core/simulator/src/workload/ops/update_user.rs index c0c15aade4..5b13139f3e 100644 --- a/core/simulator/src/workload/ops/update_user.rs +++ b/core/simulator/src/workload/ops/update_user.rs @@ -70,6 +70,9 @@ pub fn sample( Outcome::InvalidUsername => { unreachable!("update_user does not target InvalidUsername") } + Outcome::InvalidOptionValue => { + unreachable!("the simulator only sends an empty update options block") + } } } @@ -98,8 +101,9 @@ pub fn predicted_effect(input: &Input, outcome: Outcome) -> Effect { new: new.clone(), password: input.current_password.clone(), }), - Outcome::UserNotFound | Outcome::UsernameAlreadyExists | Outcome::InvalidUsername => { - Effect::None - } + Outcome::UserNotFound + | Outcome::UsernameAlreadyExists + | Outcome::InvalidUsername + | Outcome::InvalidOptionValue => Effect::None, } } diff --git a/core/tools/src/data-seeder/seeder.rs b/core/tools/src/data-seeder/seeder.rs index 84199c06b6..14b26e33e1 100644 --- a/core/tools/src/data-seeder/seeder.rs +++ b/core/tools/src/data-seeder/seeder.rs @@ -51,11 +51,11 @@ async fn create_topics(client: &IggyClient, streams: &[(String, u32)]) -> Result .create_topic( &Identifier::named(stream_name).unwrap(), "orders", - 1, - Default::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await?; @@ -63,11 +63,11 @@ async fn create_topics(client: &IggyClient, streams: &[(String, u32)]) -> Result .create_topic( &Identifier::named(stream_name).unwrap(), "users", - 2, - Default::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(2), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await?; @@ -75,11 +75,11 @@ async fn create_topics(client: &IggyClient, streams: &[(String, u32)]) -> Result .create_topic( &Identifier::named(stream_name).unwrap(), "notifications", - 3, - Default::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(3), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await?; @@ -87,11 +87,11 @@ async fn create_topics(client: &IggyClient, streams: &[(String, u32)]) -> Result .create_topic( &Identifier::named(stream_name).unwrap(), "payments", - 2, - Default::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(2), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await?; @@ -99,11 +99,11 @@ async fn create_topics(client: &IggyClient, streams: &[(String, u32)]) -> Result .create_topic( &Identifier::named(stream_name).unwrap(), "deliveries", - 1, - Default::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await?; } diff --git a/examples/go/getting-started/producer/main.go b/examples/go/getting-started/producer/main.go index 129e454fe9..74bd36686a 100644 --- a/examples/go/getting-started/producer/main.go +++ b/examples/go/getting-started/producer/main.go @@ -83,8 +83,7 @@ func initSystem(ctx context.Context, client iggcon.Client) { 1, iggcon.CompressionAlgorithmNone, iggcon.IggyExpiryNeverExpire, - 0, - nil); err != nil { + 0); err != nil { log.Printf("WARN: Topic already exists and will not be created again or error: %v", err) } log.Println("Topic was created.") diff --git a/examples/java/src/main/java/org/apache/iggy/examples/async/AsyncProducer.java b/examples/java/src/main/java/org/apache/iggy/examples/async/AsyncProducer.java index 710089ce70..1cf7b82801 100644 --- a/examples/java/src/main/java/org/apache/iggy/examples/async/AsyncProducer.java +++ b/examples/java/src/main/java/org/apache/iggy/examples/async/AsyncProducer.java @@ -32,7 +32,6 @@ import java.math.BigInteger; import java.util.ArrayList; import java.util.List; -import java.util.Optional; import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicInteger; @@ -142,7 +141,6 @@ private static CompletableFuture setupStreamAndTopic(AsyncIggyTcpClient cl CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, - Optional.empty(), TOPIC_NAME) .thenAccept(created -> log.info("Topic created: {}", created.name())); }); diff --git a/examples/java/src/main/java/org/apache/iggy/examples/gettingstarted/producer/GettingStartedProducer.java b/examples/java/src/main/java/org/apache/iggy/examples/gettingstarted/producer/GettingStartedProducer.java index cfd81e26ad..8f3899e452 100644 --- a/examples/java/src/main/java/org/apache/iggy/examples/gettingstarted/producer/GettingStartedProducer.java +++ b/examples/java/src/main/java/org/apache/iggy/examples/gettingstarted/producer/GettingStartedProducer.java @@ -35,8 +35,6 @@ import java.util.List; import java.util.Optional; -import static java.util.Optional.empty; - public final class GettingStartedProducer { private static final String STREAM_NAME = "sample-stream"; @@ -120,14 +118,7 @@ private static void createTopic(IggyTcpClient client) { return; } client.topics() - .createTopic( - STREAM_ID, - 1L, - CompressionAlgorithm.None, - BigInteger.ZERO, - BigInteger.ZERO, - empty(), - TOPIC_NAME); + .createTopic(STREAM_ID, 1L, CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, TOPIC_NAME); log.info("Topic {} was created.", TOPIC_NAME); } } diff --git a/examples/java/src/main/java/org/apache/iggy/examples/messageenvelope/producer/MessageEnvelopeProducer.java b/examples/java/src/main/java/org/apache/iggy/examples/messageenvelope/producer/MessageEnvelopeProducer.java index 9e55bc27e1..7ef9c3001e 100644 --- a/examples/java/src/main/java/org/apache/iggy/examples/messageenvelope/producer/MessageEnvelopeProducer.java +++ b/examples/java/src/main/java/org/apache/iggy/examples/messageenvelope/producer/MessageEnvelopeProducer.java @@ -78,13 +78,7 @@ public static void main(String[] args) { } else { client.topics() .createTopic( - STREAM_ID, - 1L, - CompressionAlgorithm.None, - BigInteger.ZERO, - BigInteger.ZERO, - Optional.empty(), - TOPIC_NAME); + STREAM_ID, 1L, CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, TOPIC_NAME); log.info("Topic {} was created.", TOPIC_NAME); } diff --git a/examples/java/src/main/java/org/apache/iggy/examples/messageheaders/producer/MessageHeadersProducer.java b/examples/java/src/main/java/org/apache/iggy/examples/messageheaders/producer/MessageHeadersProducer.java index 1039a8bbf9..e56abd5920 100644 --- a/examples/java/src/main/java/org/apache/iggy/examples/messageheaders/producer/MessageHeadersProducer.java +++ b/examples/java/src/main/java/org/apache/iggy/examples/messageheaders/producer/MessageHeadersProducer.java @@ -79,13 +79,7 @@ public static void main(String[] args) { } else { client.topics() .createTopic( - STREAM_ID, - 1L, - CompressionAlgorithm.None, - BigInteger.ZERO, - BigInteger.ZERO, - Optional.empty(), - TOPIC_NAME); + STREAM_ID, 1L, CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, TOPIC_NAME); log.info("Topic {} was created.", TOPIC_NAME); } diff --git a/examples/java/src/main/java/org/apache/iggy/examples/multitenant/producer/MultiTenantProducer.java b/examples/java/src/main/java/org/apache/iggy/examples/multitenant/producer/MultiTenantProducer.java index eb87bac3ef..d2f40942fe 100644 --- a/examples/java/src/main/java/org/apache/iggy/examples/multitenant/producer/MultiTenantProducer.java +++ b/examples/java/src/main/java/org/apache/iggy/examples/multitenant/producer/MultiTenantProducer.java @@ -260,7 +260,6 @@ private static void ensureTopic(IggyTcpClient client, StreamId streamId, TopicId CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, - Optional.empty(), topicId.getName()); log.info("Created topic {} for stream {}", topicId.getName(), streamId.getName()); } diff --git a/examples/java/src/main/java/org/apache/iggy/examples/sinkdataproducer/SinkDataProducer.java b/examples/java/src/main/java/org/apache/iggy/examples/sinkdataproducer/SinkDataProducer.java index c7ffb5c4e3..bd9278314e 100644 --- a/examples/java/src/main/java/org/apache/iggy/examples/sinkdataproducer/SinkDataProducer.java +++ b/examples/java/src/main/java/org/apache/iggy/examples/sinkdataproducer/SinkDataProducer.java @@ -117,7 +117,6 @@ private static void createTopicIfMissing(IggyTcpClient client, StreamId streamId CompressionAlgorithm.None, java.math.BigInteger.ZERO, java.math.BigInteger.ZERO, - java.util.Optional.empty(), topicId.getName()); log.info("Created topic {}.", topicId.getName()); }); diff --git a/examples/java/src/main/java/org/apache/iggy/examples/streambuilder/StreamBasic.java b/examples/java/src/main/java/org/apache/iggy/examples/streambuilder/StreamBasic.java index b8c9929f16..b2523f7432 100644 --- a/examples/java/src/main/java/org/apache/iggy/examples/streambuilder/StreamBasic.java +++ b/examples/java/src/main/java/org/apache/iggy/examples/streambuilder/StreamBasic.java @@ -129,13 +129,7 @@ private static void ensureStreamAndTopic(IggyTcpClient client) { } else { client.topics() .createTopic( - STREAM_ID, - 1L, - CompressionAlgorithm.None, - BigInteger.ZERO, - BigInteger.ZERO, - Optional.empty(), - TOPIC_NAME); + STREAM_ID, 1L, CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, TOPIC_NAME); log.info("Topic {} was created.", TOPIC_NAME); } } diff --git a/examples/java/src/main/java/org/apache/iggy/examples/tcptls/producer/TcpTlsProducer.java b/examples/java/src/main/java/org/apache/iggy/examples/tcptls/producer/TcpTlsProducer.java index 4661bacf2b..043970d72c 100644 --- a/examples/java/src/main/java/org/apache/iggy/examples/tcptls/producer/TcpTlsProducer.java +++ b/examples/java/src/main/java/org/apache/iggy/examples/tcptls/producer/TcpTlsProducer.java @@ -35,8 +35,6 @@ import java.util.List; import java.util.Optional; -import static java.util.Optional.empty; - /** * TCP/TLS Producer Example * @@ -139,14 +137,7 @@ private static void createTopic(IggyTcpClient client) { return; } client.topics() - .createTopic( - STREAM_ID, - 1L, - CompressionAlgorithm.None, - BigInteger.ZERO, - BigInteger.ZERO, - empty(), - TOPIC_NAME); + .createTopic(STREAM_ID, 1L, CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, TOPIC_NAME); log.info("Topic {} was created.", TOPIC_NAME); } } diff --git a/examples/node/src/multi-tenant/producer.ts b/examples/node/src/multi-tenant/producer.ts index 8fa3f32e67..fb914a6c51 100644 --- a/examples/node/src/multi-tenant/producer.ts +++ b/examples/node/src/multi-tenant/producer.ts @@ -74,7 +74,6 @@ async function setupTenants( name: topicName, partitionCount: 2, compressionAlgorithm: 1, // None - replicationFactor: 1, }); log('Topic created for tenant %d with ID: %d', tenantId, topic.id); diff --git a/examples/node/src/sink-data-producer/producer.ts b/examples/node/src/sink-data-producer/producer.ts index 5acd280539..ce2a34c6b3 100644 --- a/examples/node/src/sink-data-producer/producer.ts +++ b/examples/node/src/sink-data-producer/producer.ts @@ -118,7 +118,6 @@ async function produceData(client: Client, streamName: string, topicName: string name: topicName, partitionCount: 1, compressionAlgorithm: 1, - replicationFactor: 1, }); } } catch (error) { @@ -128,7 +127,6 @@ async function produceData(client: Client, streamName: string, topicName: string name: topicName, partitionCount: 1, compressionAlgorithm: 1, - replicationFactor: 1, }); } diff --git a/examples/node/src/stream-builder/example.ts b/examples/node/src/stream-builder/example.ts index 85aea422a0..fb511b366c 100644 --- a/examples/node/src/stream-builder/example.ts +++ b/examples/node/src/stream-builder/example.ts @@ -60,7 +60,6 @@ async function buildClientAndStream(connectionString: string) { name: topicName, partitionCount: 1, compressionAlgorithm: 1, - replicationFactor: 1, }); log(`Stream created: ${stream.id}`); diff --git a/examples/node/src/utils/index.ts b/examples/node/src/utils/index.ts index ad403cef04..df98700e19 100644 --- a/examples/node/src/utils/index.ts +++ b/examples/node/src/utils/index.ts @@ -45,7 +45,6 @@ export async function initSystem(client: Client) { name: `sample-topic-${crypto.randomBytes(4).toString('hex')}`, partitionCount: PARTITION_COUNT, compressionAlgorithm: 1, // None - replicationFactor: 1, }); log('Topic was created successfully.', 'Topic ID: %s', topic?.id); diff --git a/examples/python/basic/producer.py b/examples/python/basic/producer.py index aaa91fc465..cf3e4b84a1 100644 --- a/examples/python/basic/producer.py +++ b/examples/python/basic/producer.py @@ -75,7 +75,6 @@ async def init_system(client: IggyClient): stream=STREAM_NAME, partitions_count=1, name=TOPIC_NAME, - replication_factor=1, ) logger.info("Topic was created successfully.") else: diff --git a/examples/python/getting-started/producer.py b/examples/python/getting-started/producer.py index 23f964fa81..80ab7c6a87 100755 --- a/examples/python/getting-started/producer.py +++ b/examples/python/getting-started/producer.py @@ -149,7 +149,6 @@ async def init_system(client: IggyClient): stream=STREAM_NAME, partitions_count=1, name=TOPIC_NAME, - replication_factor=1, ) logger.info("Topic was created successfully.") else: diff --git a/examples/python/message-headers/common.py b/examples/python/message-headers/common.py index 79640b15ec..967afe930b 100644 --- a/examples/python/message-headers/common.py +++ b/examples/python/message-headers/common.py @@ -132,7 +132,6 @@ async def init_system(client: IggyClient) -> None: stream=STREAM_NAME, partitions_count=1, name=TOPIC_NAME, - replication_factor=1, ) logger.info("Topic was created successfully.") else: diff --git a/examples/rust/src/getting-started/producer/main.rs b/examples/rust/src/getting-started/producer/main.rs index 12f078212a..6e76ef7ff0 100644 --- a/examples/rust/src/getting-started/producer/main.rs +++ b/examples/rust/src/getting-started/producer/main.rs @@ -73,11 +73,11 @@ async fn init_system(client: &IggyClient) -> (u32, u32) { .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - 1, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await { diff --git a/examples/rust/src/message-headers/message-compression/producer/main.rs b/examples/rust/src/message-headers/message-compression/producer/main.rs index 330d24c088..e23b61a5bd 100644 --- a/examples/rust/src/message-headers/message-compression/producer/main.rs +++ b/examples/rust/src/message-headers/message-compression/producer/main.rs @@ -44,11 +44,12 @@ async fn main() -> Result<(), IggyError> { .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - 1, // Number of partitions. - CompressionAlgorithm::None, // NOTE: This configures the compression on the server, not the actual messages in transit! - None, // Replication factor. - IggyExpiry::NeverExpire, // Time until messages expire on the server. - MaxTopicSize::ServerDefault, // Defined in server/config.toml. Defaults to "unlimited". + &TopicCreateOptions { + partitions_count: Some(1), + // Time until messages expire on the server. + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await .expect("Topic was NOT created! Start a fresh server to run this example."); diff --git a/examples/rust/src/multi-tenant/producer/main.rs b/examples/rust/src/multi-tenant/producer/main.rs index c6f82c5db5..d748f7731a 100644 --- a/examples/rust/src/multi-tenant/producer/main.rs +++ b/examples/rust/src/multi-tenant/producer/main.rs @@ -267,7 +267,6 @@ async fn create_producers( .partitioning(Partitioning::balanced()) .create_topic_if_not_exists( partitions_count, - None, IggyExpiry::ServerDefault, MaxTopicSize::ServerDefault, ) diff --git a/examples/rust/src/new-sdk/producer/main.rs b/examples/rust/src/new-sdk/producer/main.rs index 21e3086f75..82c5e96f37 100644 --- a/examples/rust/src/new-sdk/producer/main.rs +++ b/examples/rust/src/new-sdk/producer/main.rs @@ -58,7 +58,6 @@ async fn main() -> anyhow::Result<(), Box> { .partitioning(partitioning) .create_topic_if_not_exists( args.partitions_count, - None, IggyExpiry::ServerDefault, MaxTopicSize::ServerDefault, ) diff --git a/examples/rust/src/shared/system.rs b/examples/rust/src/shared/system.rs index 520099c0e8..6f4358e918 100644 --- a/examples/rust/src/shared/system.rs +++ b/examples/rust/src/shared/system.rs @@ -82,15 +82,18 @@ pub async fn init_by_producer(args: &Args, client: &dyn Client) -> Result<(), Ig info!("Stream does not exist, creating..."); client.create_stream(&args.stream_id).await?; + let compression_algorithm = CompressionAlgorithm::from_code(args.compression_algorithm)?; client .create_topic( &stream_id, &topic_name, - args.partitions_count, - CompressionAlgorithm::from_code(args.compression_algorithm)?, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(args.partitions_count), + compression_algorithm: (compression_algorithm != CompressionAlgorithm::default()) + .then_some(compression_algorithm), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await?; Ok(()) diff --git a/examples/rust/src/stream-builder/stream-producer-config/main.rs b/examples/rust/src/stream-builder/stream-producer-config/main.rs index d2b4e8e8b6..5f7c9a2263 100644 --- a/examples/rust/src/stream-builder/stream-producer-config/main.rs +++ b/examples/rust/src/stream-builder/stream-producer-config/main.rs @@ -37,9 +37,6 @@ async fn main() -> Result<(), IggyError> { // The more clients are reading concurrently, the more partitions you should create. // i.e. if you have 10 clients, you should create 10 partitions .topic_partitions_count(10) - // Optionally, you can set the replication factor for topic redundancy. - // There is a tradeoff between replication factor and performance, so you want to benchmark your setup. - .topic_replication_factor(2) // The max number of messages to send in a batch. The greater the batch size, the higher the throughput for bulk data. // Note, there is a tradeoff between batch size and latency, so you want to benchmark your setup. // Note, this only applies to batch send messages. Single messages are sent immediately. diff --git a/examples/rust/src/tcp-tls/producer/main.rs b/examples/rust/src/tcp-tls/producer/main.rs index 38b04738f1..74cca521c5 100644 --- a/examples/rust/src/tcp-tls/producer/main.rs +++ b/examples/rust/src/tcp-tls/producer/main.rs @@ -92,11 +92,11 @@ async fn init_system(client: &IggyClient) -> (u32, u32) { .create_topic( &Identifier::named(STREAM_NAME).unwrap(), TOPIC_NAME, - 1, - CompressionAlgorithm::default(), - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + ..TopicCreateOptions::default() + }, ) .await { diff --git a/foreign/cpp/include/iggy.hpp b/foreign/cpp/include/iggy.hpp index d874e010de..62362fb963 100644 --- a/foreign/cpp/include/iggy.hpp +++ b/foreign/cpp/include/iggy.hpp @@ -19,6 +19,7 @@ #pragma once +#include #include #include #include @@ -26,6 +27,8 @@ #include #include +#include "lib.rs.h" + namespace iggy { /// Exception raised by the C++ client when an operation fails. @@ -252,4 +255,119 @@ class PollingStrategy final { std::uint64_t polling_strategy_value_; }; +namespace detail { + +/// Numeric option values are little-endian on the wire. Encoded byte by byte so +/// a big-endian host produces the same block as a little-endian one. +template +rust::Vec to_little_endian_bytes(const Value value) { + rust::Vec bytes; + bytes.reserve(sizeof(Value)); + for (std::size_t index = 0; index < sizeof(Value); ++index) { + bytes.push_back(static_cast((value >> (index * 8)) & 0xFF)); + } + + return bytes; +} + +inline rust::Vec to_bool_bytes(const bool value) { + rust::Vec bytes; + bytes.push_back(static_cast(value ? 1 : 0)); + + return bytes; +} + +inline rust::Vec to_key_bytes(const std::string_view key) { + rust::Vec bytes; + bytes.reserve(key.size()); + for (const char character : key) { + bytes.push_back(static_cast(character)); + } + + return bytes; +} + +inline iggy::ffi::HeaderField to_header_field(const iggy::ffi::HeaderKind kind, rust::Vec value) { + iggy::ffi::HeaderField field; + field.kind = static_cast(kind); + field.value = std::move(value); + + return field; +} + +/// An option key is always `String`-kinded. Only the value kind varies per key. +inline iggy::ffi::HeaderEntry to_option_entry(const std::string_view key, + const iggy::ffi::HeaderKind value_kind, + rust::Vec value) { + iggy::ffi::HeaderEntry entry; + entry.key = to_header_field(iggy::ffi::HeaderKind::String, to_key_bytes(key)); + entry.value = to_header_field(value_kind, std::move(value)); + + return entry; +} + +} // namespace detail + +/// Topic option entries for the trailing options block of +/// `Client::create_topic(...)`. +/// +/// Each factory names one key of the server's topic option catalog and encodes +/// its value under that key's catalog kind. Both halves matter: the server +/// refuses a key it does not list, and refuses a listed key whose value arrives +/// under a different kind than the catalog gives it. +/// `Client::describe_options("topic")` enumerates the catalog with each key's +/// kind and default, which is the discovery path over the binary transports: +/// they carry back an error code without naming the refused key. +/// +/// These keys are create-time only. `Client::update_topic(...)` refuses every +/// one of them by name: they describe how a topic's partitions were laid down, +/// so changing one mid-life would leave earlier segments built to the old value. +class TopicOption final { + public: + /// Set the size at which this topic's segments rotate. + /// + /// Must be a multiple of 512 bytes, at least 1 MiB, and no larger than the + /// server's segment ceiling. + static iggy::ffi::HeaderEntry segment_size(const std::uint64_t bytes) { + return detail::to_option_entry("segment_size", iggy::ffi::HeaderKind::Uint64, + detail::to_little_endian_bytes(bytes)); + } + + /// Choose whether writes to this topic's partitions are fsynced. + static iggy::ffi::HeaderEntry enforce_fsync(const bool enabled) { + return detail::to_option_entry("enforce_fsync", iggy::ffi::HeaderKind::Bool, detail::to_bool_bytes(enabled)); + } + + /// Flush the journal once it holds this many messages. + /// + /// Must be non-zero. Paired with + /// `size_of_messages_required_to_save(bytes)`: whichever threshold trips + /// first flushes. + static iggy::ffi::HeaderEntry messages_required_to_save(const std::uint32_t messages) { + return detail::to_option_entry("messages_required_to_save", iggy::ffi::HeaderKind::Uint32, + detail::to_little_endian_bytes(messages)); + } + + /// Flush the journal once it holds this many bytes. + /// + /// Capped at 1 GiB: a threshold above the largest a segment may be never + /// trips, and the journal does not survive a crash. + static iggy::ffi::HeaderEntry size_of_messages_required_to_save(const std::uint64_t bytes) { + return detail::to_option_entry("size_of_messages_required_to_save", iggy::ffi::HeaderKind::Uint64, + detail::to_little_endian_bytes(bytes)); + } + + /// Choose whether a segment's bytes are reserved on disk when it is created. + /// + /// Reserves `segment_size * partitions_count` up front, which the server + /// caps at 64 GiB per topic. + static iggy::ffi::HeaderEntry preallocate_segments(const bool enabled) { + return detail::to_option_entry("preallocate_segments", iggy::ffi::HeaderKind::Bool, + detail::to_bool_bytes(enabled)); + } + + private: + TopicOption() = delete; +}; + } // namespace iggy diff --git a/foreign/cpp/src/client.rs b/foreign/cpp/src/client.rs index 353b76479c..0206aebd4b 100644 --- a/foreign/cpp/src/client.rs +++ b/foreign/cpp/src/client.rs @@ -22,10 +22,12 @@ use iggy::prelude::{ CompressionAlgorithm as RustCompressionAlgorithm, Consumer, ConsumerGroupClient, ConsumerOffsetClient, Identifier as RustIdentifier, IggyClient as RustIggyClient, IggyClientBuilder as RustIggyClientBuilder, IggyExpiry as RustIggyExpiry, IggyMessage, - IggyTimestamp, MaxTopicSize as RustMaxTopicSize, MessageClient, PartitionClient, Partitioning, + IggyTimestamp, MaxTopicSize as RustMaxTopicSize, MessageClient, + OptionsScope as RustOptionsScope, PartitionClient, Partitioning, Permissions as RustPermissions, PollingStrategy, SegmentClient, - SnapshotCompression as RustSnapshotCompression, StreamClient, SystemClient as RustSystemClient, - SystemSnapshotType as RustSystemSnapshotType, TopicClient, UserClient, + SnapshotCompression as RustSnapshotCompression, StreamClient, StreamUpdateOptions, + SystemClient as RustSystemClient, SystemSnapshotType as RustSystemSnapshotType, TopicClient, + TopicCreateOptions, TopicUpdateOptions, UserClient, }; use std::collections::HashSet; use std::convert::TryFrom; @@ -179,7 +181,12 @@ impl Client { RUNTIME.block_on(async { self.inner - .update_stream(&rust_stream_id, &stream_name) + // Streams have no option keys yet. + .update_stream( + &rust_stream_id, + &stream_name, + &StreamUpdateOptions::default(), + ) .await .map_err(|error| { format!( @@ -384,10 +391,10 @@ impl Client { topic_name: String, partitions_count: u32, compression_algorithm: String, - replication_factor: u8, message_expiry_kind: String, message_expiry_value: u64, max_topic_size: String, + options: Vec, ) -> Result { let rust_stream_id = RustIdentifier::try_from(stream_id) .map_err(|error| format!("Could not create topic '{topic_name}': {error}"))?; @@ -399,7 +406,6 @@ impl Client { ) })?, }; - let rust_replication_factor = Some(replication_factor.max(1)); let rust_message_expiry = match message_expiry_kind.as_str() { "" | "server_default" | "default" => RustIggyExpiry::ServerDefault, "never_expire" => RustIggyExpiry::NeverExpire, @@ -421,18 +427,28 @@ impl Client { })?, }; + let raw = crate::type_conversion::ffi_options_to_raw(options) + .map_err(|error| format!("Could not create topic '{topic_name}': {error}"))?; + + // `None` is what tells admission to resolve the server default, so the + // sentinels the string parsers produce must collapse back to it. + let options = TopicCreateOptions { + partitions_count: Some(partitions_count), + compression_algorithm: (rust_compression_algorithm + != RustCompressionAlgorithm::default()) + .then_some(rust_compression_algorithm), + message_expiry: (rust_message_expiry != RustIggyExpiry::ServerDefault) + .then_some(rust_message_expiry), + max_topic_size: (rust_max_topic_size != RustMaxTopicSize::ServerDefault) + .then_some(rust_max_topic_size), + raw, + ..TopicCreateOptions::default() + }; + RUNTIME.block_on(async { let topic_details = self .inner - .create_topic( - &rust_stream_id, - &topic_name, - partitions_count, - rust_compression_algorithm, - rust_replication_factor, - rust_message_expiry, - rust_max_topic_size, - ) + .create_topic(&rust_stream_id, &topic_name, &options) .await .map_err(|error| { format!( @@ -495,10 +511,10 @@ impl Client { topic_id: ffi::Identifier, topic_name: String, compression_algorithm: String, - replication_factor: u8, message_expiry_kind: String, message_expiry_value: u64, max_topic_size: String, + options: Vec, ) -> Result<(), String> { let rust_stream_id = RustIdentifier::try_from(stream_id) .map_err(|error| format!("Could not update topic '{topic_name}': {error}"))?; @@ -512,7 +528,6 @@ impl Client { ) })?, }; - let rust_replication_factor = Some(replication_factor.max(1)); let rust_message_expiry = match message_expiry_kind.as_str() { "" | "server_default" | "default" => RustIggyExpiry::ServerDefault, "never_expire" => RustIggyExpiry::NeverExpire, @@ -534,17 +549,25 @@ impl Client { })?, }; + let raw = crate::type_conversion::ffi_options_to_raw(options) + .map_err(|error| format!("Could not update topic '{topic_name}': {error}"))?; + + // Settings ride the options block; a server-default sentinel means the + // caller did not set the key, so the topic keeps its current value. + let update_options = TopicUpdateOptions { + compression_algorithm: (rust_compression_algorithm + != RustCompressionAlgorithm::default()) + .then_some(rust_compression_algorithm), + message_expiry: (rust_message_expiry != RustIggyExpiry::ServerDefault) + .then_some(rust_message_expiry), + max_topic_size: (rust_max_topic_size != RustMaxTopicSize::ServerDefault) + .then_some(rust_max_topic_size), + raw, + }; + RUNTIME.block_on(async { self.inner - .update_topic( - &rust_stream_id, - &rust_topic_id, - &topic_name, - rust_compression_algorithm, - rust_replication_factor, - rust_message_expiry, - rust_max_topic_size, - ) + .update_topic(&rust_stream_id, &rust_topic_id, &topic_name, &update_options) .await .map_err(|error| { format!( @@ -986,6 +1009,34 @@ impl Client { }) } + /// Serves the option catalog of one scope: "topic", "stream" or "user". + /// + /// A caller learns the keys `create_topic` accepts from here and nowhere + /// else. A key outside the catalog is refused at create, and the binary + /// transports answer that refusal with an error code alone, so the + /// rejection never names the keys that would have worked. + /// + /// A scope whose catalog is still empty answers with an empty vector, so an + /// empty result means the scope takes no keys yet, not that the call failed. + pub fn describe_options(&self, scope: String) -> Result, String> { + let rust_scope = RustOptionsScope::from_str(&scope).map_err(|_| { + format!( + "Could not describe options: invalid scope '{scope}'. Expected 'topic', 'stream' or 'user'." + ) + })?; + + RUNTIME.block_on(async { + let specs = self + .inner + .describe_options(rust_scope) + .await + .map_err(|error| { + format!("Could not describe options for scope '{rust_scope}': {error}") + })?; + Ok(specs.into_iter().map(ffi::OptionSpec::from).collect()) + }) + } + pub fn ping(&self) -> Result<(), String> { RUNTIME.block_on(async { self.inner diff --git a/foreign/cpp/src/lib.rs b/foreign/cpp/src/lib.rs index 3be955d584..3f7eda58d0 100644 --- a/foreign/cpp/src/lib.rs +++ b/foreign/cpp/src/lib.rs @@ -51,9 +51,15 @@ mod ffi { message_expiry: u64, compression_algorithm: String, max_topic_size: u64, - replication_factor: u8, messages_count: u64, partitions_count: u32, + /// Options the creating client set explicitly. Carried as + /// `HeaderEntry` because options ride the user-headers codec: the same + /// TLV a message's `user_headers` uses, with string keys. + options: Vec, + /// Options admission resolved for the keys the client left unset. These + /// would have resolved differently under another server config. + derived_options: Vec, } struct Partition { @@ -73,10 +79,13 @@ mod ffi { message_expiry: u64, compression_algorithm: String, max_topic_size: u64, - replication_factor: u8, messages_count: u64, partitions_count: u32, partitions: Vec, + /// See [`Topic::options`]. + options: Vec, + /// See [`Topic::derived_options`]. + derived_options: Vec, } struct Stream { @@ -86,6 +95,9 @@ mod ffi { size_bytes: u64, messages_count: u64, topics_count: u32, + /// Creation options. Streams have no catalog keys yet, so this is + /// empty until one lands. + options: Vec, } #[repr(u8)] @@ -117,6 +129,23 @@ mod ffi { value: HeaderField, } + /// One key a resource's create command accepts, as served by + /// `describe_options`. + /// + /// This is the discovery surface for the keys `create_topic` takes. A key + /// outside the server catalog is refused at create, and the binary + /// transports carry back only an error code, so nothing in the rejection + /// names the keys that would have worked. + struct OptionSpec { + key: String, + /// Wire kind code the value is encoded under, the same encoding + /// [`HeaderField::kind`] carries. + kind: u8, + /// The default in `kind`'s encoding. Empty when the key has no default. + default_value: Vec, + description: String, + } + struct IggyMessageToSend { id_lo: u64, id_hi: u64, @@ -178,6 +207,8 @@ mod ffi { messages_count: u64, topics_count: u32, topics: Vec, + /// See [`Stream::options`]. + options: Vec, } struct ConsumerGroupMember { @@ -364,10 +395,10 @@ mod ffi { topic_name: String, partitions_count: u32, compression_algorithm: String, - replication_factor: u8, message_expiry_kind: String, message_expiry_value: u64, max_topic_size: String, + options: Vec, ) -> Result; fn get_topic( self: &Client, @@ -382,10 +413,10 @@ mod ffi { topic_id: Identifier, topic_name: String, compression_algorithm: String, - replication_factor: u8, message_expiry_kind: String, message_expiry_value: u64, max_topic_size: String, + options: Vec, ) -> Result<()>; fn delete_topic(self: &Client, stream_id: Identifier, topic_id: Identifier) -> Result<()>; fn purge_topic(self: &Client, stream_id: Identifier, topic_id: Identifier) -> Result<()>; @@ -498,6 +529,10 @@ mod ffi { fn get_me(self: &Client) -> Result; fn get_client(self: &Client, client_id: u32) -> Result; fn get_clients(self: &Client) -> Result>; + /// Serves the option catalog of one scope, named "topic", "stream" or + /// "user". A scope with no keys yet answers with an empty vector, which + /// is an empty catalog rather than a failure. + fn describe_options(self: &Client, scope: String) -> Result>; fn ping(self: &Client) -> Result<()>; fn heartbeat_interval(self: &Client) -> u64; fn snapshot( diff --git a/foreign/cpp/src/type_conversion.rs b/foreign/cpp/src/type_conversion.rs index 3a8ae3ae0b..e2c2866df4 100644 --- a/foreign/cpp/src/type_conversion.rs +++ b/foreign/cpp/src/type_conversion.rs @@ -19,7 +19,7 @@ use crate::ffi; use bytes::Bytes; use iggy::prelude::{ ConsumerGroupDetails as RustConsumerGroupDetails, IdKind, Identifier as RustIdentifier, - IggyMessage as RustIggyMessage, Partition as RustPartition, + IggyMessage as RustIggyMessage, OptionSpec as RustOptionSpec, Partition as RustPartition, PolledMessages as RustPolledMessages, SendMessagesConfirmationResponse as RustSendMessagesConfirmationResponse, SendMessagesResponse as RustSendMessagesResponse, Stream as RustStream, @@ -323,6 +323,66 @@ impl From for ffi::Partition { } } +/// The entries of one provenance, as the `HeaderEntry` the message path uses. +/// +/// Options ride the user-headers codec, so they cross the bridge as the type +/// already there for `user_headers` rather than a second one meaning the same +/// thing. Values keep the kind the server sent, so a `Uint64` stays a `Uint64`. +fn resource_options_to_ffi( + options: &iggy::prelude::ResourceOptions, + explicit: bool, +) -> Vec { + options + .iter() + .filter(|(_, option)| option.explicit == explicit) + .map(|(key, option)| ffi::HeaderEntry { + key: ffi::HeaderField { + kind: key.kind().as_code(), + value: key.as_bytes().to_vec(), + }, + value: ffi::HeaderField { + kind: option.value.kind().as_code(), + value: option.value.as_bytes().to_vec(), + }, + }) + .collect() +} + +/// Render option entries into the string map `TopicCreateOptions::raw` takes. +/// +/// The Rust SDK expresses arbitrary option keys as strings that admission +/// parses by the same rules a config file value goes through, so a typed value +/// handed in here is rendered rather than passed through: sending `Uint64` +/// 134217728 for `segment_size` and sending `"134217728"` land the same stored +/// value. A key this build cannot read at all is dropped rather than guessed. +pub(crate) fn ffi_options_to_raw( + options: Vec, +) -> Result, String> { + let mut raw = std::collections::BTreeMap::new(); + for entry in options { + let RustHeaderEntry { key, value } = RustHeaderEntry::try_from(entry)?; + let key = key + .as_str() + .map_err(|error| format!("Option key is not a string: {error}"))? + .to_owned(); + if raw.insert(key.clone(), value.to_string_value()).is_some() { + return Err(format!("Duplicate option key: {key}")); + } + } + Ok(raw) +} + +impl From for ffi::OptionSpec { + fn from(spec: RustOptionSpec) -> Self { + ffi::OptionSpec { + key: spec.key, + kind: spec.kind.as_code(), + default_value: spec.default_value, + description: spec.description, + } + } +} + impl From for ffi::Topic { fn from(topic: RustTopic) -> Self { ffi::Topic { @@ -333,9 +393,10 @@ impl From for ffi::Topic { message_expiry: u64::from(topic.message_expiry), compression_algorithm: topic.compression_algorithm.to_string(), max_topic_size: u64::from(topic.max_topic_size), - replication_factor: topic.replication_factor, messages_count: topic.messages_count, partitions_count: topic.partitions_count, + options: resource_options_to_ffi(&topic.options, true), + derived_options: resource_options_to_ffi(&topic.options, false), } } } @@ -350,7 +411,6 @@ impl From for ffi::TopicDetails { message_expiry: u64::from(topic.message_expiry), compression_algorithm: topic.compression_algorithm.to_string(), max_topic_size: u64::from(topic.max_topic_size), - replication_factor: topic.replication_factor, messages_count: topic.messages_count, partitions_count: topic.partitions_count, partitions: topic @@ -358,6 +418,8 @@ impl From for ffi::TopicDetails { .into_iter() .map(ffi::Partition::from) .collect(), + options: resource_options_to_ffi(&topic.options, true), + derived_options: resource_options_to_ffi(&topic.options, false), } } } @@ -371,6 +433,7 @@ impl From for ffi::Stream { size_bytes: stream.size.as_bytes_u64(), messages_count: stream.messages_count, topics_count: stream.topics_count, + options: resource_options_to_ffi(&stream.options, true), } } } @@ -385,6 +448,7 @@ impl From for ffi::StreamDetails { messages_count: stream.messages_count, topics_count: stream.topics_count, topics: stream.topics.into_iter().map(ffi::Topic::from).collect(), + options: resource_options_to_ffi(&stream.options, true), } } } diff --git a/foreign/cpp/tests/e2e/client.cpp b/foreign/cpp/tests/e2e/client.cpp index 692481e527..8e5d370d3e 100644 --- a/foreign/cpp/tests/e2e/client.cpp +++ b/foreign/cpp/tests/e2e/client.cpp @@ -572,8 +572,8 @@ TEST_F(LowLevelE2E_Client, FlushUnsavedBufferThrowsForExistingPartition) { ASSERT_NO_THROW(client->create_stream(stream_name)); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); - ASSERT_NO_THROW(client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", - 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, + "server_default", {})); rust::Vec messages; messages.push_back(iggy::ffi::make_message(to_payload("flush-me"), rust::Vec())); @@ -595,8 +595,8 @@ TEST_F(LowLevelE2E_Client, FlushUnsavedBufferThrowsForExistingEmptyPartition) { ASSERT_NO_THROW(client->create_stream(stream_name)); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); - ASSERT_NO_THROW(client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", - 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, + "server_default", {})); ASSERT_THROW(client->flush_unsaved_buffer(make_numeric_identifier(stream.id), make_numeric_identifier(0), 0, true), std::exception); @@ -627,8 +627,8 @@ TEST_F(LowLevelE2E_Client, FlushUnsavedBufferOnNonExistentStreamThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, "never_expire", - 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "never_expire", 0, + "server_default", {})); ASSERT_THROW( client->flush_unsaved_buffer(make_string_identifier(GetRandomName()), make_numeric_identifier(0), 0, true), @@ -643,8 +643,8 @@ TEST_F(LowLevelE2E_Client, FlushUnsavedBufferOnNonExistentTopicThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, "never_expire", - 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "never_expire", 0, + "server_default", {})); ASSERT_THROW(client->flush_unsaved_buffer(make_string_identifier(stream_name), make_string_identifier(GetRandomName()), 0, true), @@ -660,8 +660,8 @@ TEST_F(LowLevelE2E_Client, FlushUnsavedBufferAfterStreamDeletedThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); - ASSERT_NO_THROW(client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", - 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, + "server_default", {})); const std::uint32_t saved_stream_id = stream.id; ASSERT_NO_THROW(client->delete_stream(make_numeric_identifier(saved_stream_id))); @@ -681,8 +681,8 @@ TEST_F(LowLevelE2E_Client, FlushUnsavedBufferAfterTopicDeletedThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); - ASSERT_NO_THROW(client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", - 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, + "server_default", {})); ASSERT_NO_THROW(client->delete_topic(make_numeric_identifier(stream.id), make_string_identifier(topic_name))); ASSERT_THROW( @@ -700,8 +700,8 @@ TEST_F(LowLevelE2E_Client, FlushUnsavedBufferTwiceThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); - ASSERT_NO_THROW(client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", - 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, + "server_default", {})); rust::Vec messages; messages.push_back(iggy::ffi::make_message(to_payload("flush-twice"), rust::Vec())); @@ -723,8 +723,8 @@ TEST_F(LowLevelE2E_Client, FlushUnsavedBufferWithInvalidPartitionIdsThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); - ASSERT_NO_THROW(client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", - 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, + "server_default", {})); const std::uint32_t invalid_partition_ids[] = {1u, 9999u, static_cast(-1)}; for (const std::uint32_t invalid_partition_id : invalid_partition_ids) { @@ -744,8 +744,8 @@ TEST_F(LowLevelE2E_Client, DeleteSegmentsBeforeLoginThrows) { ASSERT_NO_THROW(setup_client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(setup_client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "never_expire", 0, "server_default")); + ASSERT_NO_THROW(setup_client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", + "never_expire", 0, "server_default", {})); iggy::ffi::Client *unauthenticated_client = GetLoggedOutClient(); ASSERT_THROW(unauthenticated_client->delete_segments(make_string_identifier(stream_name), @@ -771,8 +771,8 @@ TEST_F(LowLevelE2E_Client, DeleteSegmentsOnNonExistentStreamThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, "never_expire", - 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "never_expire", 0, + "server_default", {})); ASSERT_THROW( client->delete_segments(make_string_identifier(missing_stream_name), make_string_identifier(topic_name), 0, 1), @@ -788,8 +788,8 @@ TEST_F(LowLevelE2E_Client, DeleteSegmentsOnNonExistentTopicThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, "never_expire", - 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "never_expire", 0, + "server_default", {})); ASSERT_THROW( client->delete_segments(make_string_identifier(stream_name), make_string_identifier(missing_topic_name), 0, 1), @@ -804,8 +804,8 @@ TEST_F(LowLevelE2E_Client, DeleteSegmentsOnNonExistentPartitionThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, "never_expire", - 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "never_expire", 0, + "server_default", {})); ASSERT_THROW( client->delete_segments(make_string_identifier(stream_name), make_string_identifier(topic_name), 999, 1), @@ -820,8 +820,8 @@ TEST_F(LowLevelE2E_Client, DeleteSegmentsWithZeroCountIsNoOp) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, "never_expire", - 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "never_expire", 0, + "server_default", {})); std::uint32_t stream_id = 0; std::uint32_t topic_id = 0; @@ -901,8 +901,8 @@ TEST_F(LowLevelE2E_Client, DeleteSegmentsWhenOnlyActiveSegmentRemainsIsNoOp) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, "never_expire", - 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "never_expire", 0, + "server_default", {})); std::uint32_t stream_id = 0; std::uint32_t topic_id = 0; @@ -1014,12 +1014,12 @@ TEST_F(LowLevelE2E_Client, GetStatsReturnsServerStats) { TrackStream(first_stream_name); ASSERT_NO_THROW(client->create_stream(second_stream_name)); TrackStream(second_stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(first_stream_name), first_topic_name, 1, "none", 0, - "server_default", 0, "server_default")); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(first_stream_name), second_topic_name, 2, "none", 0, - "server_default", 0, "server_default")); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(second_stream_name), third_topic_name, 3, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(first_stream_name), first_topic_name, 1, "none", + "server_default", 0, "server_default", {})); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(first_stream_name), second_topic_name, 2, "none", + "server_default", 0, "server_default", {})); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(second_stream_name), third_topic_name, 3, "none", + "server_default", 0, "server_default", {})); ASSERT_NO_THROW(client->create_partitions(make_string_identifier(first_stream_name), make_string_identifier(first_topic_name), additional_partitions_count)); const auto first_group = client->create_consumer_group(make_string_identifier(first_stream_name), @@ -1167,8 +1167,8 @@ TEST_F(LowLevelE2E_Client, GetMeReflectsConsumerGroupMembershipChanges) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); const auto stream_details = client->get_stream(make_string_identifier(stream_name)); ASSERT_EQ(stream_details.topics.size(), 1u); diff --git a/foreign/cpp/tests/e2e/consumer_group.cpp b/foreign/cpp/tests/e2e/consumer_group.cpp index 7863b66300..5a32d62036 100644 --- a/foreign/cpp/tests/e2e/consumer_group.cpp +++ b/foreign/cpp/tests/e2e/consumer_group.cpp @@ -36,8 +36,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, CreateConsumerGroupSucceeds) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW({ const auto group = client->create_consumer_group(make_string_identifier(stream_name), @@ -59,8 +59,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, CreateConsumerGroupOnNonExistentResourcesThrow ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_THROW(client->create_consumer_group(make_string_identifier(missing_stream_name), make_string_identifier(topic_name), GetRandomName()), @@ -79,8 +79,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, CreateConsumerGroupTwiceOnSameInputThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW(client->create_consumer_group(make_string_identifier(stream_name), make_string_identifier(topic_name), group_name)); TrackConsumerGroup(stream_name, topic_name, group_name); @@ -97,8 +97,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, CreateConsumerGroupWithInvalidNamesThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); const std::string invalid_names[] = {"", std::string(256, 'a')}; for (const std::string &invalid_name : invalid_names) { @@ -118,8 +118,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, CreateConsumerGroupAfterStreamDeletionThrows) ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW(client->delete_stream(make_string_identifier(stream_name))); ForgetTrackedStream(stream_name); @@ -138,8 +138,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, CreateConsumerGroupBeforeLoginThrows) { iggy::ffi::Client *setup_client = GetLoggedInClient(); ASSERT_NO_THROW(setup_client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(setup_client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(setup_client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", + "server_default", 0, "server_default", {})); iggy::ffi::Client *unauthenticated_client = GetLoggedOutClient(); @@ -167,8 +167,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupReturnsSameInfoAsCreateConsume ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); const auto created_group = client->create_consumer_group(make_string_identifier(stream_name), make_string_identifier(topic_name), group_name); @@ -194,8 +194,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupsReturnsCreatedGroups) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW(client->create_consumer_group(make_string_identifier(stream_name), make_string_identifier(topic_name), first_group_name)); @@ -220,8 +220,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupsBeforeLoginThrows) { iggy::ffi::Client *setup_client = GetLoggedInClient(); ASSERT_NO_THROW(setup_client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(setup_client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(setup_client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", + "server_default", 0, "server_default", {})); iggy::ffi::Client *unauthenticated_client = GetLoggedOutClient(); @@ -248,8 +248,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, JoinConsumerGroupSucceeds) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW(client->create_consumer_group(make_string_identifier(stream_name), make_string_identifier(topic_name), group_name)); TrackConsumerGroup(stream_name, topic_name, group_name); @@ -267,8 +267,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, JoinConsumerGroupBeforeLoginThrows) { iggy::ffi::Client *setup_client = GetLoggedInClient(); ASSERT_NO_THROW(setup_client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(setup_client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(setup_client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", + "server_default", 0, "server_default", {})); ASSERT_NO_THROW(setup_client->create_consumer_group(make_string_identifier(stream_name), make_string_identifier(topic_name), group_name)); TrackConsumerGroup(stream_name, topic_name, group_name); @@ -304,8 +304,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, JoinConsumerGroupOnNonExistentResourcesThrows) ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW(client->create_consumer_group(make_string_identifier(stream_name), make_string_identifier(topic_name), created_group_name)); TrackConsumerGroup(stream_name, topic_name, created_group_name); @@ -332,8 +332,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, JoinConsumerGroupAfterStreamDeletionThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW(client->create_consumer_group(make_string_identifier(stream_name), make_string_identifier(topic_name), group_name)); TrackConsumerGroup(stream_name, topic_name, group_name); @@ -354,8 +354,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, JoinConsumerGroupAfterTopicDeletionThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW(client->create_consumer_group(make_string_identifier(stream_name), make_string_identifier(topic_name), group_name)); TrackConsumerGroup(stream_name, topic_name, group_name); @@ -375,8 +375,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, JoinConsumerGroupReflectsInGetConsumerGroup) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); iggy::ffi::ConsumerGroupDetails created_group; ASSERT_NO_THROW({ @@ -407,8 +407,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, JoinConsumerGroupTwiceKeepsSingleMember) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW(client->create_consumer_group(make_string_identifier(stream_name), make_string_identifier(topic_name), group_name)); TrackConsumerGroup(stream_name, topic_name, group_name); @@ -434,8 +434,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, JoinConsumerGroupFromTwoClientsIncreasesMember ASSERT_NO_THROW(first->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(first->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, "server_default", - 0, "server_default")); + ASSERT_NO_THROW(first->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", 0, + "server_default", {})); ASSERT_NO_THROW(first->create_consumer_group(make_string_identifier(stream_name), make_string_identifier(topic_name), group_name)); TrackConsumerGroup(stream_name, topic_name, group_name); @@ -460,8 +460,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, JoinConsumerGroupThenLeaveRestoresMembersCount ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW(client->create_consumer_group(make_string_identifier(stream_name), make_string_identifier(topic_name), group_name)); TrackConsumerGroup(stream_name, topic_name, group_name); @@ -492,8 +492,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, LeaveConsumerGroupReducesMembersCount) { ASSERT_NO_THROW(first->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(first->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, "server_default", - 0, "server_default")); + ASSERT_NO_THROW(first->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", 0, + "server_default", {})); ASSERT_NO_THROW(first->create_consumer_group(make_string_identifier(stream_name), make_string_identifier(topic_name), group_name)); TrackConsumerGroup(stream_name, topic_name, group_name); @@ -526,8 +526,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, LeaveConsumerGroupBeforeLoginThrows) { iggy::ffi::Client *setup_client = GetLoggedInClient(); ASSERT_NO_THROW(setup_client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(setup_client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(setup_client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", + "server_default", 0, "server_default", {})); ASSERT_NO_THROW(setup_client->create_consumer_group(make_string_identifier(stream_name), make_string_identifier(topic_name), group_name)); TrackConsumerGroup(stream_name, topic_name, group_name); @@ -565,8 +565,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, LeaveConsumerGroupOnNonExistentResourcesThrows ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW(client->create_consumer_group(make_string_identifier(stream_name), make_string_identifier(topic_name), created_group_name)); TrackConsumerGroup(stream_name, topic_name, created_group_name); @@ -595,8 +595,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, LeaveConsumerGroupAfterStreamDeletionThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW(client->create_consumer_group(make_string_identifier(stream_name), make_string_identifier(topic_name), group_name)); TrackConsumerGroup(stream_name, topic_name, group_name); @@ -619,8 +619,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, LeaveConsumerGroupAfterTopicDeletionThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW(client->create_consumer_group(make_string_identifier(stream_name), make_string_identifier(topic_name), group_name)); TrackConsumerGroup(stream_name, topic_name, group_name); @@ -643,8 +643,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, LeaveConsumerGroupTwiceThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW(client->create_consumer_group(make_string_identifier(stream_name), make_string_identifier(topic_name), group_name)); TrackConsumerGroup(stream_name, topic_name, group_name); @@ -667,8 +667,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, LeaveConsumerGroupWithoutJoiningThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW(client->create_consumer_group(make_string_identifier(stream_name), make_string_identifier(topic_name), group_name)); TrackConsumerGroup(stream_name, topic_name, group_name); @@ -688,8 +688,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupsReflectsJoinedGroupMembersCou ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); iggy::ffi::ConsumerGroupDetails joined_group; iggy::ffi::ConsumerGroupDetails other_group; @@ -758,8 +758,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupsIsStableAcrossBackToBackCalls ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW(client->create_consumer_group(make_string_identifier(stream_name), make_string_identifier(topic_name), first_group_name)); @@ -795,8 +795,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupsReturnsCorrectNumberOfGroups) ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW(client->create_consumer_group(make_string_identifier(stream_name), make_string_identifier(topic_name), deleted_group_name)); @@ -847,8 +847,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupsAfterStreamDeletionThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW(client->create_consumer_group(make_string_identifier(stream_name), make_string_identifier(topic_name), first_group_name)); TrackConsumerGroup(stream_name, topic_name, first_group_name); @@ -874,8 +874,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupsAfterTopicDeletionThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW(client->create_consumer_group(make_string_identifier(stream_name), make_string_identifier(topic_name), first_group_name)); TrackConsumerGroup(stream_name, topic_name, first_group_name); @@ -900,8 +900,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupBeforeLoginThrows) { iggy::ffi::Client *setup_client = GetLoggedInClient(); ASSERT_NO_THROW(setup_client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(setup_client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(setup_client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", + "server_default", 0, "server_default", {})); ASSERT_NO_THROW(setup_client->create_consumer_group(make_string_identifier(stream_name), make_string_identifier(topic_name), group_name)); TrackConsumerGroup(stream_name, topic_name, group_name); @@ -937,8 +937,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupOnNonExistentResourcesThrows) ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW(client->create_consumer_group(make_string_identifier(stream_name), make_string_identifier(topic_name), created_group_name)); TrackConsumerGroup(stream_name, topic_name, created_group_name); @@ -965,8 +965,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupAfterStreamDeletionThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW(client->create_consumer_group(make_string_identifier(stream_name), make_string_identifier(topic_name), group_name)); ASSERT_NO_THROW(client->delete_stream(make_string_identifier(stream_name))); @@ -986,8 +986,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, DeleteConsumerGroupSucceeds) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW(client->create_consumer_group(make_string_identifier(stream_name), make_string_identifier(topic_name), group_name)); TrackConsumerGroup(stream_name, topic_name, group_name); @@ -1010,8 +1010,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, DeleteConsumerGroupBeforeLoginThrows) { iggy::ffi::Client *setup_client = GetLoggedInClient(); ASSERT_NO_THROW(setup_client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(setup_client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(setup_client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", + "server_default", 0, "server_default", {})); ASSERT_NO_THROW(setup_client->create_consumer_group(make_string_identifier(stream_name), make_string_identifier(topic_name), group_name)); TrackConsumerGroup(stream_name, topic_name, group_name); @@ -1047,8 +1047,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, DeleteConsumerGroupOnNonExistentResourcesThrow ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW(client->create_consumer_group(make_string_identifier(stream_name), make_string_identifier(topic_name), created_group_name)); TrackConsumerGroup(stream_name, topic_name, created_group_name); @@ -1074,8 +1074,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, DeleteConsumerGroupTwiceThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW(client->create_consumer_group(make_string_identifier(stream_name), make_string_identifier(topic_name), group_name)); TrackConsumerGroup(stream_name, topic_name, group_name); @@ -1097,8 +1097,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, DeleteConsumerGroupAfterStreamDeletionThrows) ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW(client->create_consumer_group(make_string_identifier(stream_name), make_string_identifier(topic_name), group_name)); TrackConsumerGroup(stream_name, topic_name, group_name); @@ -1121,8 +1121,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, DeleteConsumerGroupAndRecreateWithSameNameSucc ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW(client->create_consumer_group(make_string_identifier(stream_name), make_string_identifier(topic_name), group_name)); diff --git a/foreign/cpp/tests/e2e/message.cpp b/foreign/cpp/tests/e2e/message.cpp index f805127a34..09b1e2ae1f 100644 --- a/foreign/cpp/tests/e2e/message.cpp +++ b/foreign/cpp/tests/e2e/message.cpp @@ -37,8 +37,8 @@ TEST_F(LowLevelE2E_Message, SendAndPollMessagesRoundTrip) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); rust::Vec messages; for (std::uint32_t i = 0; i < 10; i++) { @@ -79,8 +79,8 @@ TEST_F(LowLevelE2E_Message, PollMessagesVerifyMessageIds) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); rust::Vec messages; auto msg = iggy::ffi::make_message(to_payload("id-test-message"), rust::Vec()); @@ -108,8 +108,8 @@ TEST_F(LowLevelE2E_Message, PollMessagesFromEmptyPartition) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); auto polled = client->poll_messages(make_numeric_identifier(stream.id), make_numeric_identifier(0), 0, "consumer", make_numeric_identifier(1), "offset", 0, 100, false); @@ -181,8 +181,8 @@ TEST_F(LowLevelE2E_Message, SendMessagesWithInvalidPartitioningKind) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); rust::Vec messages; auto msg = iggy::ffi::make_message(to_payload("test"), rust::Vec()); @@ -202,8 +202,8 @@ TEST_F(LowLevelE2E_Message, SendMessagesWithInvalidPartitioningValue) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); rust::Vec messages; auto msg = iggy::ffi::make_message(to_payload("test"), rust::Vec()); @@ -228,8 +228,8 @@ TEST_F(LowLevelE2E_Message, SendMessagesToSpecificPartitionVerified) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 3, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 3, "none", "never_expire", 0, "server_default", + {}); rust::Vec messages; for (std::uint32_t i = 0; i < 5; i++) { @@ -261,8 +261,8 @@ TEST_F(LowLevelE2E_Message, SendEmptyMessageVectorThrows) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); rust::Vec empty_messages; @@ -280,8 +280,8 @@ TEST_F(LowLevelE2E_Message, SendMessageWithEmptyPayloadThrows) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); rust::Vec messages; rust::Vec empty_payload; @@ -302,8 +302,8 @@ TEST_F(LowLevelE2E_Message, SendMessageWithOversizedPayloadThrows) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); // Build a payload one byte over the SDK's max payload size (64 MB). constexpr std::uint32_t kOversizedPayloadBytes = 64'000'001u; @@ -331,8 +331,8 @@ TEST_F(LowLevelE2E_Message, SendMessagesPreservesOrder) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); rust::Vec messages; for (std::uint32_t i = 0; i < 50; i++) { @@ -365,8 +365,8 @@ TEST_F(LowLevelE2E_Message, SendMessagesWithDuplicateIds) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); rust::Vec messages; for (std::uint32_t i = 0; i < 3; i++) { @@ -399,8 +399,8 @@ TEST_F(LowLevelE2E_Message, SendMessagesWithVariousPayloads) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); rust::Vec payload_null; payload_null.push_back(0x00); @@ -466,8 +466,8 @@ TEST_F(LowLevelE2E_Message, SendAndPollMessageWithTypedHeadersRoundTrip) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); struct ExpectedHeaderMessage { const char *key; @@ -555,8 +555,8 @@ TEST_F(LowLevelE2E_Message, SendMessageWithDuplicateTypedHeaderKeysThrows) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); rust::Vec headers; headers.push_back(make_header_entry(make_header_field(iggy::ffi::HeaderKind::String, to_payload("dup-key")), @@ -581,8 +581,8 @@ TEST_F(LowLevelE2E_Message, SendMessageWithWrongFixedWidthHeaderBytesThrows) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); struct FixedWidthHeaderCase { const char *key; @@ -627,8 +627,8 @@ TEST_F(LowLevelE2E_Message, SendMessageWithInvalidTypedHeaderKindThrows) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); iggy::ffi::HeaderField invalid_key; invalid_key.kind = 255; @@ -655,8 +655,8 @@ TEST_F(LowLevelE2E_Message, SendMessageWithInvalidTypedHeaderSizesThrows) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); rust::Vec empty_key_headers; empty_key_headers.push_back( @@ -720,8 +720,8 @@ TEST_F(LowLevelE2E_Message, SendMessageAtUserHeadersSizeBoundary) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); constexpr std::uint32_t kMaxUserHeadersBytes = 100'000u; constexpr std::uint32_t kFullHeaderEncodedBytes = 267u; // 10 bytes framing + 2-byte key + 255-byte value @@ -850,8 +850,8 @@ TEST_F(LowLevelE2E_Message, PollMessagesWithInvalidConsumerKindThrows) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); ASSERT_THROW(client->poll_messages(make_numeric_identifier(stream.id), make_numeric_identifier(0), 0, "invalid", make_numeric_identifier(1), "offset", 0, 10, false), @@ -867,8 +867,8 @@ TEST_F(LowLevelE2E_Message, PollMessagesWithInvalidStrategyKindThrows) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); ASSERT_THROW(client->poll_messages(make_numeric_identifier(stream.id), make_numeric_identifier(0), 0, "consumer", make_numeric_identifier(1), "invalid", 0, 10, false), @@ -884,8 +884,8 @@ TEST_F(LowLevelE2E_Message, PollMessagesCountLessThanAvailable) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); rust::Vec messages; for (std::uint32_t i = 0; i < 10; i++) { @@ -912,8 +912,8 @@ TEST_F(LowLevelE2E_Message, PollMessagesWithLargeOffset) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); rust::Vec messages; for (std::uint32_t i = 0; i < 5; i++) { @@ -940,8 +940,8 @@ TEST_F(LowLevelE2E_Message, PollMessagesFirstStrategy) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); rust::Vec messages; for (std::uint32_t i = 0; i < 10; i++) { @@ -975,8 +975,8 @@ TEST_F(LowLevelE2E_Message, PollMessagesLastStrategy) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); rust::Vec messages; for (std::uint32_t i = 0; i < 10; i++) { @@ -1011,8 +1011,8 @@ TEST_F(LowLevelE2E_Message, PollMessagesNextStrategyNoAutoCommit) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); rust::Vec messages; for (std::uint32_t i = 0; i < 5; i++) { @@ -1053,8 +1053,8 @@ TEST_F(LowLevelE2E_Message, PollMessagesNextStrategyAutoCommit) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); rust::Vec messages; for (std::uint32_t i = 0; i < 10; i++) { @@ -1101,8 +1101,8 @@ TEST_F(LowLevelE2E_Message, PollMessagesConsumerIdIndependence) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); rust::Vec messages; for (std::uint32_t i = 0; i < 5; i++) { @@ -1135,8 +1135,8 @@ TEST_F(LowLevelE2E_Message, PollMessagesMultipleSendsThenPollOrder) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); rust::Vec batch1; for (std::uint32_t i = 0; i < 5; i++) { @@ -1184,8 +1184,8 @@ TEST_F(LowLevelE2E_Message, PollMessagesMultipleCustomIds) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); const std::uint64_t id_values[] = {100, 200, 300, 400, 500}; rust::Vec messages; @@ -1218,8 +1218,8 @@ TEST_F(LowLevelE2E_Message, PollMessagesAfterStreamDeletedThrows) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); rust::Vec messages; auto msg = iggy::ffi::make_message(to_payload("test"), rust::Vec()); @@ -1246,8 +1246,8 @@ TEST_F(LowLevelE2E_Message, PollMessagesWithInvalidPartitionIdThrows) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); ASSERT_THROW(client->poll_messages(make_numeric_identifier(stream.id), make_numeric_identifier(0), 9999, "consumer", make_numeric_identifier(1), "offset", 0, 10, false), @@ -1263,8 +1263,8 @@ TEST_F(LowLevelE2E_Message, PollMessagesWithCountZeroThrows) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); ASSERT_THROW(client->poll_messages(make_numeric_identifier(stream.id), make_numeric_identifier(0), 0, "consumer", make_numeric_identifier(1), "offset", 0, 0, false), @@ -1281,8 +1281,8 @@ TEST_F(LowLevelE2E_Message, PollMessagesWithoutSpecifyingPartition) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); rust::Vec messages; for (std::uint32_t i = 0; i < 5; i++) { @@ -1317,8 +1317,8 @@ TEST_F(LowLevelE2E_Message, PollMessagesTimestampStrategy) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); rust::Vec batch1; for (std::uint32_t i = 0; i < 5; i++) { @@ -1380,8 +1380,8 @@ TEST_F(LowLevelE2E_Message, PollMessagesMonotonicOffsets) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); rust::Vec messages; for (std::uint32_t i = 0; i < 20; i++) { @@ -1419,8 +1419,8 @@ TEST_F(LowLevelE2E_Message, SendMessagesLargeBatch) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); rust::Vec messages; for (std::uint32_t i = 0; i < 1000; i++) { @@ -1480,8 +1480,8 @@ TEST_F(LowLevelE2E_Message, PollMessagesWithInvalidConsumerIdThrows) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); iggy::ffi::Identifier invalid_id; invalid_id.kind = "invalid"; @@ -1502,8 +1502,8 @@ TEST_F(LowLevelE2E_Message, ConsumerGroupCreateJoinAndPollMessages) { auto stream = client->get_stream(make_string_identifier(stream_name)); TrackStream(stream.id); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); const std::string group_name = GetRandomName(); auto group = diff --git a/foreign/cpp/tests/e2e/partition.cpp b/foreign/cpp/tests/e2e/partition.cpp index e429c6d0b8..8f305db971 100644 --- a/foreign/cpp/tests/e2e/partition.cpp +++ b/foreign/cpp/tests/e2e/partition.cpp @@ -36,8 +36,8 @@ TEST_F(LowLevelE2E_Partition, CreatePartitionsSucceeds) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW( client->create_partitions(make_string_identifier(stream_name), make_string_identifier(topic_name), 43)); @@ -79,8 +79,8 @@ TEST_F(LowLevelE2E_Partition, CreatePartitionsOnNonExistentResourcesThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_THROW( client->create_partitions(make_string_identifier(missing_stream_name), make_string_identifier(topic_name), 1), @@ -99,8 +99,8 @@ TEST_F(LowLevelE2E_Partition, CreatePartitionsWithInvalidIdentifiersThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); iggy::ffi::Identifier invalid_stream_kind_id; invalid_stream_kind_id.kind = "invalid"; @@ -159,8 +159,8 @@ TEST_F(LowLevelE2E_Partition, CreatePartitionsWithBoundaryPartitionsCountValues) for (const auto &test_case : test_cases) { SCOPED_TRACE(test_case.topic_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), test_case.topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), test_case.topic_name, 1, "none", + "server_default", 0, "server_default", {})); if (test_case.should_succeed) { ASSERT_NO_THROW(client->create_partitions(make_string_identifier(stream_name), @@ -201,8 +201,8 @@ TEST_F(LowLevelE2E_Partition, CreatePartitionsWithNumericIdentifiersSucceeds) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); const auto stream_details = client->get_stream(make_string_identifier(stream_name)); ASSERT_EQ(stream_details.topics.size(), 1u); @@ -228,8 +228,8 @@ TEST_F(LowLevelE2E_Partition, DeletePartitionsSucceeds) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 44, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 44, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW( client->delete_partitions(make_string_identifier(stream_name), make_string_identifier(topic_name), 43)); @@ -269,8 +269,8 @@ TEST_F(LowLevelE2E_Partition, DeleteMorePartitionsThanExistingThrows) { for (const auto &test_case : test_cases) { SCOPED_TRACE(test_case.topic_name); ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), test_case.topic_name, - test_case.initial_partitions, "none", 0, "server_default", 0, - "server_default")); + test_case.initial_partitions, "none", "server_default", 0, + "server_default", {})); if (test_case.should_succeed) { ASSERT_NO_THROW(client->delete_partitions(make_string_identifier(stream_name), @@ -311,8 +311,8 @@ TEST_F(LowLevelE2E_Partition, DeletePartitionsBeforeCreatingAdditionalPartitions ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 3, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 3, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW( client->delete_partitions(make_string_identifier(stream_name), make_string_identifier(topic_name), 1)); @@ -333,8 +333,8 @@ TEST_F(LowLevelE2E_Partition, DeletePartitionsFromTopicWithZeroPartitionsThrows) ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 0, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 0, "none", "server_default", + 0, "server_default", {})); ASSERT_THROW(client->delete_partitions(make_string_identifier(stream_name), make_string_identifier(topic_name), 1), std::exception); @@ -377,8 +377,8 @@ TEST_F(LowLevelE2E_Partition, DeletePartitionsOnNonExistentResourcesThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 3, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 3, "none", "server_default", + 0, "server_default", {})); ASSERT_THROW( client->delete_partitions(make_string_identifier(missing_stream_name), make_string_identifier(topic_name), 1), @@ -397,8 +397,8 @@ TEST_F(LowLevelE2E_Partition, DeletePartitionsWithInvalidIdentifiersThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 3, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 3, "none", "server_default", + 0, "server_default", {})); iggy::ffi::Identifier invalid_stream_kind_id; invalid_stream_kind_id.kind = "invalid"; @@ -438,8 +438,8 @@ TEST_F(LowLevelE2E_Partition, DeletePartitionsTwiceForSameTopicSucceeds) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 45, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 45, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW( client->delete_partitions(make_string_identifier(stream_name), make_string_identifier(topic_name), 20)); ASSERT_NO_THROW( @@ -462,8 +462,8 @@ TEST_F(LowLevelE2E_Partition, DeletePartitionsAfterStreamDeletionThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 3, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 3, "none", "server_default", + 0, "server_default", {})); const auto stream_details = client->get_stream(make_string_identifier(stream_name)); ASSERT_EQ(stream_details.topics.size(), 1u); diff --git a/foreign/cpp/tests/e2e/stream.cpp b/foreign/cpp/tests/e2e/stream.cpp index 405c59c47f..c4f33e8731 100644 --- a/foreign/cpp/tests/e2e/stream.cpp +++ b/foreign/cpp/tests/e2e/stream.cpp @@ -289,8 +289,8 @@ TEST_F(LowLevelE2E_Stream, UpdateStreamOnlyChangesName) { ForgetTrackedStream(stream_name); TrackStream(stream_id); - ASSERT_NO_THROW(client->create_topic(make_numeric_identifier(stream_id), topic_name, 2, "none", 0, "never_expire", - 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_numeric_identifier(stream_id), topic_name, 2, "none", "never_expire", 0, + "server_default", {})); rust::Vec messages; for (std::uint32_t i = 0; i < 3; ++i) { @@ -336,7 +336,6 @@ TEST_F(LowLevelE2E_Stream, UpdateStreamOnlyChangesName) { EXPECT_EQ(after_topic.message_expiry, before_topic.message_expiry); EXPECT_EQ(after_topic.compression_algorithm, before_topic.compression_algorithm); EXPECT_EQ(after_topic.max_topic_size, before_topic.max_topic_size); - EXPECT_EQ(after_topic.replication_factor, before_topic.replication_factor); EXPECT_EQ(after_topic.messages_count, before_topic.messages_count); EXPECT_EQ(after_topic.partitions_count, before_topic.partitions_count); @@ -573,8 +572,8 @@ TEST_F(LowLevelE2E_Stream, GetStreamsFieldsVerification) { TrackStream(stream_name); auto stream = client->get_stream(make_string_identifier(stream_name)); const std::string topic_name = GetRandomName(); - client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, - "server_default"); + client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", "never_expire", 0, "server_default", + {}); rust::Vec messages; for (std::uint32_t i = 0; i < 5; i++) { @@ -725,10 +724,10 @@ TEST_F(LowLevelE2E_Stream, PurgeStreamPreservesStreamMetadata) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), first_topic_name, 2, "gzip", 1, - "duration", 1000, "1GiB")); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), second_topic_name, 3, "none", 0, - "never_expire", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), first_topic_name, 2, "gzip", "duration", + 1000, "1GiB", {})); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), second_topic_name, 3, "none", + "never_expire", 0, "server_default", {})); const auto stream_before_purge = client->get_stream(make_string_identifier(stream_name)); ASSERT_EQ(stream_before_purge.topics.size(), 2u); @@ -751,7 +750,6 @@ TEST_F(LowLevelE2E_Stream, PurgeStreamPreservesStreamMetadata) { std::uint64_t message_expiry; std::string compression_algorithm; std::uint64_t max_topic_size; - std::uint8_t replication_factor; std::uint32_t partitions_count; }; std::unordered_map topics_before_purge; @@ -763,7 +761,6 @@ TEST_F(LowLevelE2E_Stream, PurgeStreamPreservesStreamMetadata) { topic.message_expiry, static_cast(topic.compression_algorithm), topic.max_topic_size, - topic.replication_factor, topic.partitions_count}; } @@ -787,7 +784,6 @@ TEST_F(LowLevelE2E_Stream, PurgeStreamPreservesStreamMetadata) { EXPECT_EQ(topic.message_expiry, metadata.message_expiry); EXPECT_EQ(topic.compression_algorithm, metadata.compression_algorithm); EXPECT_EQ(topic.max_topic_size, metadata.max_topic_size); - EXPECT_EQ(topic.replication_factor, metadata.replication_factor); EXPECT_EQ(topic.partitions_count, metadata.partitions_count); } } @@ -801,10 +797,10 @@ TEST_F(LowLevelE2E_Stream, PurgeStreamRemovesMessagesAndPreservesTopics) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), first_topic_name, 1, "none", 0, - "server_default", 0, "server_default")); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), second_topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), first_topic_name, 1, "none", + "server_default", 0, "server_default", {})); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), second_topic_name, 1, "none", + "server_default", 0, "server_default", {})); const auto created_stream = client->get_stream(make_string_identifier(stream_name)); ASSERT_EQ(created_stream.topics.size(), 2u); @@ -904,10 +900,10 @@ TEST_F(LowLevelE2E_Stream, PurgeStreamAcrossMultipleTopicsAndPartitionsClearsEve ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), first_topic_name, 2, "none", 0, - "server_default", 0, "server_default")); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), second_topic_name, 3, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), first_topic_name, 2, "none", + "server_default", 0, "server_default", {})); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), second_topic_name, 3, "none", + "server_default", 0, "server_default", {})); const auto created_stream = client->get_stream(make_string_identifier(stream_name)); ASSERT_EQ(created_stream.topics.size(), 2u); @@ -975,8 +971,8 @@ TEST_F(LowLevelE2E_Stream, PurgeStreamThenSendMessagesAgainSucceeds) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); const auto created_stream = client->get_stream(make_string_identifier(stream_name)); ASSERT_EQ(created_stream.topics.size(), 1u); @@ -1011,8 +1007,8 @@ TEST_F(LowLevelE2E_Stream, PurgeStreamTwiceKeepsStreamEmptyAndTopicsIntact) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); const auto created_stream = client->get_stream(make_string_identifier(stream_name)); ASSERT_EQ(created_stream.topics.size(), 1u); diff --git a/foreign/cpp/tests/e2e/topic.cpp b/foreign/cpp/tests/e2e/topic.cpp index 500a1a8089..3c863a185e 100644 --- a/foreign/cpp/tests/e2e/topic.cpp +++ b/foreign/cpp/tests/e2e/topic.cpp @@ -17,6 +17,7 @@ * under the License. */ +#include #include #include #include @@ -26,9 +27,31 @@ #include +#include "iggy.hpp" #include "lib.rs.h" #include "tests/e2e/test_helpers.hpp" +namespace { + +rust::Vec little_endian_bytes(const std::uint64_t value, const std::size_t width) { + rust::Vec bytes; + bytes.reserve(width); + for (std::size_t index = 0; index < width; ++index) { + bytes.push_back(static_cast((value >> (index * 8)) & 0xFF)); + } + + return bytes; +} + +rust::Vec bool_bytes(const bool value) { + rust::Vec bytes; + bytes.push_back(static_cast(value ? 1 : 0)); + + return bytes; +} + +} // namespace + class LowLevelE2E_Topic : public E2ETestFixture {}; TEST_F(LowLevelE2E_Topic, CreateTopicWithAllOptionCombinations) { @@ -42,7 +65,6 @@ TEST_F(LowLevelE2E_Topic, CreateTopicWithAllOptionCombinations) { TrackStream(stream_name); const std::vector compression_algorithms = {"none", "gzip"}; - const std::vector replication_factors = {0, 1, 27}; struct ExpiryOption { std::string kind; std::uint64_t value; @@ -57,21 +79,18 @@ TEST_F(LowLevelE2E_Topic, CreateTopicWithAllOptionCombinations) { std::size_t expected_topics_count = 0; std::unordered_set expected_topic_names; for (const auto &compression_algorithm : compression_algorithms) { - for (const auto replication_factor : replication_factors) { - for (const auto &expiry_option : expiry_options) { - for (const auto &max_topic_size : max_topic_sizes) { - const std::string topic_name = GetRandomName(); - SCOPED_TRACE( - "compression=" + compression_algorithm + ", replication=" + std::to_string(replication_factor) + - ", expiry_kind=" + expiry_option.kind + - ", expiry_value=" + std::to_string(expiry_option.value) + ", max_topic_size=" + max_topic_size); - - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, - compression_algorithm, replication_factor, expiry_option.kind, - expiry_option.value, max_topic_size)); - ++expected_topics_count; - expected_topic_names.insert(topic_name); - } + for (const auto &expiry_option : expiry_options) { + for (const auto &max_topic_size : max_topic_sizes) { + const std::string topic_name = GetRandomName(); + SCOPED_TRACE("compression=" + compression_algorithm + ", expiry_kind=" + expiry_option.kind + + ", expiry_value=" + std::to_string(expiry_option.value) + + ", max_topic_size=" + max_topic_size); + + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, + compression_algorithm, expiry_option.kind, expiry_option.value, + max_topic_size, {})); + ++expected_topics_count; + expected_topic_names.insert(topic_name); } } } @@ -102,12 +121,12 @@ TEST_F(LowLevelE2E_Topic, CreateTopicWithBoundaryPartitionsCountValues) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), zero_partitions_topic_name, 0, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), zero_partitions_topic_name, 0, "none", + "server_default", 0, "server_default", {})); ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), max_partitions_topic_name, 1000, "none", - 0, "server_default", 0, "server_default")); - ASSERT_THROW(client->create_topic(make_string_identifier(stream_name), overflow_topic_name, 1001, "none", 0, - "server_default", 0, "server_default"), + "server_default", 0, "server_default", {})); + ASSERT_THROW(client->create_topic(make_string_identifier(stream_name), overflow_topic_name, 1001, "none", + "server_default", 0, "server_default", {}), std::exception); const auto stream_details = client->get_stream(make_string_identifier(stream_name)); @@ -139,14 +158,14 @@ TEST_F(LowLevelE2E_Topic, CreateTopicWithInvalidNamesThrows) { }; for (const auto &topic_name : illegal_topic_names) { SCOPED_TRACE(topic_name); - ASSERT_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default"), + ASSERT_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {}), std::exception); } const std::string max_length_name(255, 'a'); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), max_length_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), max_length_name, 1, "none", + "server_default", 0, "server_default", {})); } TEST_F(LowLevelE2E_Topic, CreateDuplicateTopicThrows) { @@ -158,10 +177,10 @@ TEST_F(LowLevelE2E_Topic, CreateDuplicateTopicThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); - ASSERT_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, "server_default", - 0, "server_default"), + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); + ASSERT_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", 0, + "server_default", {}), std::exception); } @@ -178,10 +197,10 @@ TEST_F(LowLevelE2E_Topic, CreateSameTopicNameInDifferentStreamsSucceeds) { ASSERT_NO_THROW(client->create_stream(second_stream_name)); TrackStream(second_stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(first_stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(second_stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(first_stream_name), topic_name, 1, "none", + "server_default", 0, "server_default", {})); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(second_stream_name), topic_name, 1, "none", + "server_default", 0, "server_default", {})); } TEST_F(LowLevelE2E_Topic, CreateTopicWithInvalidOptionsThrows) { @@ -197,16 +216,149 @@ TEST_F(LowLevelE2E_Topic, CreateTopicWithInvalidOptionsThrows) { TrackStream(stream_name); ASSERT_THROW(client->create_topic(make_string_identifier(stream_name), invalid_compression_topic_name, 1, - "invalid-compression", 0, "server_default", 0, "server_default"), + "invalid-compression", "server_default", 0, "server_default", {}), std::exception); - ASSERT_THROW(client->create_topic(make_string_identifier(stream_name), invalid_expiry_topic_name, 1, "none", 0, - "invalid-expiry-kind", 0, "server_default"), + ASSERT_THROW(client->create_topic(make_string_identifier(stream_name), invalid_expiry_topic_name, 1, "none", + "invalid-expiry-kind", 0, "server_default", {}), std::exception); - ASSERT_THROW(client->create_topic(make_string_identifier(stream_name), invalid_max_size_topic_name, 1, "none", 0, - "server_default", 0, "not-a-size"), + ASSERT_THROW(client->create_topic(make_string_identifier(stream_name), invalid_max_size_topic_name, 1, "none", + "server_default", 0, "not-a-size", {}), std::exception); } +TEST_F(LowLevelE2E_Topic, CreateTopicWithOptionsReturnsCanonicalKindAndDerivedRemainder) { + RecordProperty("description", + "Returns an explicitly set option in its canonical kind, derives the keys left unset, and rejects " + "an option key outside the server catalog."); + const std::string stream_name = GetRandomName(); + const std::string topic_name = GetRandomName(); + const std::string unknown_option_topic = GetRandomName(); + + iggy::ffi::Client *client = GetLoggedInClient(); + + ASSERT_NO_THROW(client->create_stream(stream_name)); + TrackStream(stream_name); + + rust::Vec options; + options.push_back(make_header_entry(make_header_field(iggy::ffi::HeaderKind::String, to_payload("enforce_fsync")), + make_header_field(iggy::ffi::HeaderKind::String, to_payload("true")))); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", std::move(options))); + + const auto topic_details = + client->get_topic(make_string_identifier(stream_name), make_string_identifier(topic_name)); + + // Admission re-encodes the block from its own parse, so a value comes back + // in its key's catalog kind rather than in the kind that was sent. + rust::Vec enforce_fsync_enabled; + enforce_fsync_enabled.push_back(1); + EXPECT_TRUE(has_header(topic_details.options, static_cast(iggy::ffi::HeaderKind::String), + to_payload("enforce_fsync"), static_cast(iggy::ffi::HeaderKind::Bool), + enforce_fsync_enabled)); + + EXPECT_FALSE(topic_details.derived_options.empty()); + std::unordered_set derived_option_keys; + for (const auto &derived_option : topic_details.derived_options) { + derived_option_keys.insert(std::string(derived_option.key.value.begin(), derived_option.key.value.end())); + } + EXPECT_EQ(derived_option_keys.count("max_topic_size"), 1u); + EXPECT_EQ(derived_option_keys.count("enforce_fsync"), 0u); + + rust::Vec unknown_options; + unknown_options.push_back( + make_header_entry(make_header_field(iggy::ffi::HeaderKind::String, to_payload("not_a_real_option")), + make_header_field(iggy::ffi::HeaderKind::String, to_payload("true")))); + ASSERT_THROW(client->create_topic(make_string_identifier(stream_name), unknown_option_topic, 1, "none", + "server_default", 0, "server_default", std::move(unknown_options)), + std::exception); +} + +TEST_F(LowLevelE2E_Topic, CreateTopicWithTypedOptionHelpersReportsThemAsExplicitOptions) { + RecordProperty("description", + "Creates a topic with every typed option helper and verifies each key comes back as an explicit " + "option in its catalog kind."); + const std::string stream_name = GetRandomName(); + const std::string topic_name = GetRandomName(); + + iggy::ffi::Client *client = GetLoggedInClient(); + + ASSERT_NO_THROW(client->create_stream(stream_name)); + TrackStream(stream_name); + + constexpr std::uint64_t segment_size_bytes = 8ULL * 1024ULL * 1024ULL; + constexpr std::uint32_t messages_required_to_save = 512; + constexpr std::uint64_t size_of_messages_required_to_save = 2ULL * 1024ULL * 1024ULL; + + rust::Vec options; + options.push_back(iggy::TopicOption::segment_size(segment_size_bytes)); + options.push_back(iggy::TopicOption::enforce_fsync(true)); + options.push_back(iggy::TopicOption::messages_required_to_save(messages_required_to_save)); + options.push_back(iggy::TopicOption::size_of_messages_required_to_save(size_of_messages_required_to_save)); + options.push_back(iggy::TopicOption::preallocate_segments(false)); + + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", std::move(options))); + + const auto topic_details = + client->get_topic(make_string_identifier(stream_name), make_string_identifier(topic_name)); + + const auto key_kind = static_cast(iggy::ffi::HeaderKind::String); + const auto bool_kind = static_cast(iggy::ffi::HeaderKind::Bool); + const auto uint32_kind = static_cast(iggy::ffi::HeaderKind::Uint32); + const auto uint64_kind = static_cast(iggy::ffi::HeaderKind::Uint64); + + EXPECT_TRUE(has_header(topic_details.options, key_kind, to_payload("segment_size"), uint64_kind, + little_endian_bytes(segment_size_bytes, 8))); + EXPECT_TRUE(has_header(topic_details.options, key_kind, to_payload("enforce_fsync"), bool_kind, bool_bytes(true))); + EXPECT_TRUE(has_header(topic_details.options, key_kind, to_payload("messages_required_to_save"), uint32_kind, + little_endian_bytes(messages_required_to_save, 4))); + EXPECT_TRUE(has_header(topic_details.options, key_kind, to_payload("size_of_messages_required_to_save"), + uint64_kind, little_endian_bytes(size_of_messages_required_to_save, 8))); + EXPECT_TRUE( + has_header(topic_details.options, key_kind, to_payload("preallocate_segments"), bool_kind, bool_bytes(false))); + + for (const auto &derived_option : topic_details.derived_options) { + const std::string derived_key(derived_option.key.value.begin(), derived_option.key.value.end()); + EXPECT_NE(derived_key, "segment_size") << "segment_size was set explicitly, so it cannot be derived"; + } +} + +TEST_F(LowLevelE2E_Topic, DescribeOptionsServesTopicCatalogAndRejectsUnknownScope) { + RecordProperty("description", + "Serves the topic option catalog with each key's kind, default and description, returns an empty " + "catalog for the stream scope, and rejects an unknown scope name."); + + iggy::ffi::Client *client = GetLoggedInClient(); + + rust::Vec topic_options; + ASSERT_NO_THROW({ topic_options = client->describe_options("topic"); }); + + const iggy::ffi::OptionSpec *segment_size = nullptr; + bool found_enforce_fsync = false; + for (const auto &option : topic_options) { + const std::string key = static_cast(option.key); + if (key == "segment_size") { + segment_size = &option; + } else if (key == "enforce_fsync") { + found_enforce_fsync = true; + } + } + + ASSERT_NE(segment_size, nullptr) << "Topic catalog is missing segment_size"; + EXPECT_TRUE(found_enforce_fsync) << "Topic catalog is missing enforce_fsync"; + EXPECT_EQ(segment_size->kind, static_cast(iggy::ffi::HeaderKind::Uint64)); + EXPECT_FALSE(segment_size->default_value.empty()); + EXPECT_FALSE(segment_size->description.empty()); + + // Streams take no option keys yet, which is an empty catalog rather than a failure. + ASSERT_NO_THROW({ + const auto stream_options = client->describe_options("stream"); + EXPECT_TRUE(stream_options.empty()); + }); + + ASSERT_THROW(client->describe_options("not_a_scope"), std::exception); +} + TEST_F(LowLevelE2E_Topic, CreateTopicWithMaxTopicSizeBelowSegmentSizeThrows) { RecordProperty("description", "Rejects topic creation when the maximum topic size is smaller than the segment size."); @@ -217,8 +369,8 @@ TEST_F(LowLevelE2E_Topic, CreateTopicWithMaxTopicSizeBelowSegmentSizeThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, "server_default", - 0, "1024"), + ASSERT_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", 0, + "1024", {}), std::exception); } @@ -229,8 +381,8 @@ TEST_F(LowLevelE2E_Topic, CreateTopicOnNonExistentStreamThrows) { iggy::ffi::Client *client = GetLoggedInClient(); - ASSERT_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, "server_default", - 0, "server_default"), + ASSERT_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", 0, + "server_default", {}), std::exception); } @@ -246,8 +398,8 @@ TEST_F(LowLevelE2E_Topic, CreateTopicAfterStreamDeletionThrows) { ASSERT_NO_THROW(client->delete_stream(make_string_identifier(stream_name))); ForgetTrackedStream(stream_name); - ASSERT_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, "server_default", - 0, "server_default"), + ASSERT_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", 0, + "server_default", {}), std::exception); } @@ -266,16 +418,16 @@ TEST_F(LowLevelE2E_Topic, CreateTopicWithInvalidStreamIdentifierThrows) { invalid_kind_id.kind = "invalid"; invalid_kind_id.length = 4; invalid_kind_id.value = {1, 0, 0, 0}; - ASSERT_THROW(client->create_topic(std::move(invalid_kind_id), first_topic_name, 1, "none", 0, "server_default", 0, - "server_default"), + ASSERT_THROW(client->create_topic(std::move(invalid_kind_id), first_topic_name, 1, "none", "server_default", 0, + "server_default", {}), std::exception); iggy::ffi::Identifier invalid_numeric_id; invalid_numeric_id.kind = "numeric"; invalid_numeric_id.length = 1; invalid_numeric_id.value.push_back(1); - ASSERT_THROW(client->create_topic(std::move(invalid_numeric_id), second_topic_name, 1, "none", 0, "server_default", - 0, "server_default"), + ASSERT_THROW(client->create_topic(std::move(invalid_numeric_id), second_topic_name, 1, "none", "server_default", 0, + "server_default", {}), std::exception); } @@ -292,13 +444,13 @@ TEST_F(LowLevelE2E_Topic, CreateTopicBeforeLoginThrows) { iggy::ffi::Client *unauthenticated_client = GetLoggedOutClient(); ASSERT_NO_THROW(unauthenticated_client->connect()); - ASSERT_THROW(unauthenticated_client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default"), + ASSERT_THROW(unauthenticated_client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", + "server_default", 0, "server_default", {}), std::exception); ASSERT_NO_THROW(unauthenticated_client->login_user("iggy", "iggy")); ASSERT_NO_THROW(unauthenticated_client->disconnect()); - ASSERT_THROW(unauthenticated_client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default"), + ASSERT_THROW(unauthenticated_client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", + "server_default", 0, "server_default", {}), std::exception); } @@ -311,8 +463,8 @@ TEST_F(LowLevelE2E_Topic, DeleteTopicAfterCreate) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW(client->delete_topic(make_string_identifier(stream_name), make_string_identifier(topic_name))); @@ -356,8 +508,8 @@ TEST_F(LowLevelE2E_Topic, DeleteTopicTwiceThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW(client->delete_topic(make_string_identifier(stream_name), make_string_identifier(topic_name))); ASSERT_THROW(client->delete_topic(make_string_identifier(stream_name), make_string_identifier(topic_name)), @@ -373,8 +525,8 @@ TEST_F(LowLevelE2E_Topic, DeleteTopicAfterStreamDeletionThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW(client->delete_stream(make_string_identifier(stream_name))); ForgetTrackedStream(stream_name); @@ -391,8 +543,8 @@ TEST_F(LowLevelE2E_Topic, DeleteTopicBeforeLoginThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); iggy::ffi::Client *unauthenticated_client = GetLoggedOutClient(); @@ -419,8 +571,8 @@ TEST_F(LowLevelE2E_Topic, DeleteTopicWithInvalidStreamIdentifierThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); iggy::ffi::Identifier invalid_kind_id; invalid_kind_id.kind = "invalid"; @@ -445,8 +597,8 @@ TEST_F(LowLevelE2E_Topic, DeleteTopicWithInvalidTopicIdentifierThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); iggy::ffi::Identifier invalid_kind_id; invalid_kind_id.kind = "invalid"; @@ -472,7 +624,7 @@ TEST_F(LowLevelE2E_Topic, GetTopicReturnsTopicForExistingTopic) { TrackStream(stream_name); ASSERT_NO_THROW( - client->create_topic(make_string_identifier(stream_name), topic_name, 3, "gzip", 1, "duration", 1000, "1GiB")); + client->create_topic(make_string_identifier(stream_name), topic_name, 3, "gzip", "duration", 1000, "1GiB", {})); ASSERT_NO_THROW({ const auto topic_details = @@ -481,7 +633,6 @@ TEST_F(LowLevelE2E_Topic, GetTopicReturnsTopicForExistingTopic) { EXPECT_EQ(topic_details.partitions_count, 3u); EXPECT_EQ(topic_details.partitions.size(), 3u); EXPECT_EQ(topic_details.compression_algorithm, "gzip"); - EXPECT_EQ(topic_details.replication_factor, 1u); EXPECT_EQ(topic_details.message_expiry, 1000u); EXPECT_EQ(topic_details.max_topic_size, 1024ULL * 1024ULL * 1024ULL); }); @@ -495,8 +646,8 @@ TEST_F(LowLevelE2E_Topic, GetTopicBeforeLoginThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); iggy::ffi::Client *unauthenticated_client = GetLoggedOutClient(); @@ -525,8 +676,8 @@ TEST_F(LowLevelE2E_Topic, GetTopicWithWrongStreamIdThrows) { TrackStream(first_stream_name); ASSERT_NO_THROW(client->create_stream(second_stream_name)); TrackStream(second_stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(first_stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(first_stream_name), topic_name, 1, "none", + "server_default", 0, "server_default", {})); ASSERT_THROW(client->get_topic(make_string_identifier(second_stream_name), make_string_identifier(topic_name)), std::exception); @@ -541,8 +692,8 @@ TEST_F(LowLevelE2E_Topic, GetTopicWithWrongTopicThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_THROW(client->get_topic(make_string_identifier(stream_name), make_string_identifier(wrong_topic_name)), std::exception); @@ -556,8 +707,8 @@ TEST_F(LowLevelE2E_Topic, GetTopicAfterStreamDeletionThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW(client->delete_stream(make_string_identifier(stream_name))); ForgetTrackedStream(stream_name); @@ -573,8 +724,8 @@ TEST_F(LowLevelE2E_Topic, GetTopicAfterTopicDeletionThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW(client->delete_topic(make_string_identifier(stream_name), make_string_identifier(topic_name))); ASSERT_THROW(client->get_topic(make_string_identifier(stream_name), make_string_identifier(topic_name)), @@ -589,8 +740,8 @@ TEST_F(LowLevelE2E_Topic, GetTopicReturnsEmptyPartitionsForZeroPartitionTopic) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 0, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 0, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW({ const auto topic_details = @@ -609,8 +760,8 @@ TEST_F(LowLevelE2E_Topic, GetTopicReturnsMaxBoundaryPartitionCount) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1000, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1000, "none", + "server_default", 0, "server_default", {})); ASSERT_NO_THROW({ const auto topic_details = @@ -630,7 +781,7 @@ TEST_F(LowLevelE2E_Topic, GetTopicIsStableAcrossBackToBackCalls) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); ASSERT_NO_THROW( - client->create_topic(make_string_identifier(stream_name), topic_name, 3, "gzip", 1, "duration", 1000, "1GiB")); + client->create_topic(make_string_identifier(stream_name), topic_name, 3, "gzip", "duration", 1000, "1GiB", {})); iggy::ffi::TopicDetails first_topic{}; iggy::ffi::TopicDetails second_topic{}; @@ -644,7 +795,6 @@ TEST_F(LowLevelE2E_Topic, GetTopicIsStableAcrossBackToBackCalls) { EXPECT_EQ(static_cast(second_topic.compression_algorithm), static_cast(first_topic.compression_algorithm)); EXPECT_EQ(second_topic.max_topic_size, first_topic.max_topic_size); - EXPECT_EQ(second_topic.replication_factor, first_topic.replication_factor); EXPECT_EQ(second_topic.partitions_count, first_topic.partitions_count); EXPECT_EQ(second_topic.partitions.size(), first_topic.partitions.size()); } @@ -658,7 +808,7 @@ TEST_F(LowLevelE2E_Topic, GetTopicAgreesWithGetStreamTopicSummary) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); ASSERT_NO_THROW( - client->create_topic(make_string_identifier(stream_name), topic_name, 3, "gzip", 1, "duration", 1000, "1GiB")); + client->create_topic(make_string_identifier(stream_name), topic_name, 3, "gzip", "duration", 1000, "1GiB", {})); ASSERT_NO_THROW({ const auto stream_details = client->get_stream(make_string_identifier(stream_name)); @@ -673,7 +823,6 @@ TEST_F(LowLevelE2E_Topic, GetTopicAgreesWithGetStreamTopicSummary) { EXPECT_EQ(static_cast(topic_details.compression_algorithm), static_cast(topic_summary.compression_algorithm)); EXPECT_EQ(topic_details.max_topic_size, topic_summary.max_topic_size); - EXPECT_EQ(topic_details.replication_factor, topic_summary.replication_factor); EXPECT_EQ(topic_details.partitions_count, topic_summary.partitions_count); EXPECT_EQ(topic_details.partitions.size(), topic_summary.partitions_count); }); @@ -690,23 +839,22 @@ TEST_F(LowLevelE2E_Topic, GetTopicsReturnsCreatedTopicInputFields) { std::string compression_algorithm; std::uint64_t message_expiry; std::uint64_t max_topic_size; - std::uint8_t replication_factor; }; const std::unordered_map expected_topics = { - {first_topic_name, {2, "gzip", 1000, 1024ULL * 1024ULL * 1024ULL, 1}}, + {first_topic_name, {2, "gzip", 1000, 1024ULL * 1024ULL * 1024ULL}}, {second_topic_name, - {0, "none", std::numeric_limits::max(), std::numeric_limits::max(), 1}}, + {0, "none", std::numeric_limits::max(), std::numeric_limits::max()}}, }; iggy::ffi::Client *client = GetLoggedInClient(); ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), first_topic_name, 2, "gzip", 1, - "duration", 1000, "1GiB")); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), second_topic_name, 0, "none", 1, - "never_expire", 0, "unlimited")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), first_topic_name, 2, "gzip", "duration", + 1000, "1GiB", {})); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), second_topic_name, 0, "none", + "never_expire", 0, "unlimited", {})); ASSERT_NO_THROW({ const auto topics = client->get_topics(make_string_identifier(stream_name)); @@ -725,7 +873,6 @@ TEST_F(LowLevelE2E_Topic, GetTopicsReturnsCreatedTopicInputFields) { EXPECT_EQ(topic.compression_algorithm, expected->second.compression_algorithm); EXPECT_EQ(topic.message_expiry, expected->second.message_expiry); EXPECT_EQ(topic.max_topic_size, expected->second.max_topic_size); - EXPECT_EQ(topic.replication_factor, expected->second.replication_factor); found_topic_names.insert(topic_name); } EXPECT_EQ(found_topic_names.size(), expected_topics.size()); @@ -775,10 +922,10 @@ TEST_F(LowLevelE2E_Topic, GetTopicsAfterTopicDeletionReturnsRemainingTopics) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), deleted_topic, 1, "none", 0, - "server_default", 0, "server_default")); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), remaining_topic, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), deleted_topic, 1, "none", + "server_default", 0, "server_default", {})); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), remaining_topic, 1, "none", + "server_default", 0, "server_default", {})); ASSERT_NO_THROW(client->delete_topic(make_string_identifier(stream_name), make_string_identifier(deleted_topic))); ASSERT_NO_THROW({ @@ -798,10 +945,10 @@ TEST_F(LowLevelE2E_Topic, GetTopicsAfterTopicUpdateReturnsUpdatedInputFields) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), original_topic, 2, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), original_topic, 2, "none", + "server_default", 0, "server_default", {})); ASSERT_NO_THROW(client->update_topic(make_string_identifier(stream_name), make_string_identifier(original_topic), - updated_topic_name, "gzip", 1, "duration", 1000, "1GiB")); + updated_topic_name, "gzip", "duration", 1000, "1GiB", {})); ASSERT_NO_THROW({ const auto topics = client->get_topics(make_string_identifier(stream_name)); @@ -809,7 +956,6 @@ TEST_F(LowLevelE2E_Topic, GetTopicsAfterTopicUpdateReturnsUpdatedInputFields) { EXPECT_EQ(topics.front().name, updated_topic_name); EXPECT_EQ(topics.front().partitions_count, 2u); EXPECT_EQ(topics.front().compression_algorithm, "gzip"); - EXPECT_EQ(topics.front().replication_factor, 1u); EXPECT_EQ(topics.front().message_expiry, 1000u); EXPECT_EQ(topics.front().max_topic_size, 1024ULL * 1024ULL * 1024ULL); }); @@ -825,10 +971,10 @@ TEST_F(LowLevelE2E_Topic, UpdateTopicWorksCorrectly) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), original_topic, 2, "none", 1, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), original_topic, 2, "none", + "server_default", 0, "server_default", {})); ASSERT_NO_THROW(client->update_topic(make_string_identifier(stream_name), make_string_identifier(original_topic), - updated_topic_name, "gzip", 27, "duration", 1000, "1GiB")); + updated_topic_name, "gzip", "duration", 1000, "1GiB", {})); ASSERT_NO_THROW({ const auto topic_details = @@ -844,7 +990,6 @@ TEST_F(LowLevelE2E_Topic, UpdateTopicWorksCorrectly) { EXPECT_EQ(topic_summary.message_expiry, topic_details.message_expiry); EXPECT_EQ(topic_summary.compression_algorithm, topic_details.compression_algorithm); EXPECT_EQ(topic_summary.max_topic_size, topic_details.max_topic_size); - EXPECT_EQ(topic_summary.replication_factor, topic_details.replication_factor); EXPECT_EQ(topic_summary.messages_count, topic_details.messages_count); EXPECT_EQ(topic_summary.partitions_count, topic_details.partitions_count); }); @@ -862,10 +1007,10 @@ TEST_F(LowLevelE2E_Topic, UpdateTopicDoesNotChangePartitionsCount) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), original_topic, partitions_count, "none", - 1, "server_default", 0, "server_default")); + "server_default", 0, "server_default", {})); ASSERT_NO_THROW(client->update_topic(make_string_identifier(stream_name), make_string_identifier(original_topic), - updated_topic_name, "gzip", 27, "duration", 1000, "1GiB")); + updated_topic_name, "gzip", "duration", 1000, "1GiB", {})); ASSERT_NO_THROW({ const auto topic_details = @@ -885,8 +1030,8 @@ TEST_F(LowLevelE2E_Topic, UpdateTopicDoesNotChangeMessages) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), original_topic, 1, "none", 1, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), original_topic, 1, "none", + "server_default", 0, "server_default", {})); const auto created_stream = client->get_stream(make_string_identifier(stream_name)); ASSERT_EQ(created_stream.topics.size(), 1u); @@ -899,7 +1044,7 @@ TEST_F(LowLevelE2E_Topic, UpdateTopicDoesNotChangeMessages) { "partition_id", partition_id_bytes(0), std::move(messages))); ASSERT_NO_THROW(client->update_topic(make_string_identifier(stream_name), make_string_identifier(original_topic), - updated_topic_name, "gzip", 27, "duration", 1000, "1GiB")); + updated_topic_name, "gzip", "duration", 1000, "1GiB", {})); ASSERT_NO_THROW({ const auto polled = client->poll_messages(make_numeric_identifier(created_stream.id), @@ -919,7 +1064,6 @@ TEST_F(LowLevelE2E_Topic, UpdateTopicWithAllOptionCombinationsUpdatesInputFields std::string topic_name = GetRandomName(); const std::vector compression_algorithms = {"none", "gzip"}; - const std::vector replication_factors = {0, 1, 27}; struct ExpiryOption { std::string kind; std::uint64_t value; @@ -935,25 +1079,21 @@ TEST_F(LowLevelE2E_Topic, UpdateTopicWithAllOptionCombinationsUpdatesInputFields ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 2, "none", 1, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 2, "none", "server_default", + 0, "server_default", {})); for (const auto &compression_algorithm : compression_algorithms) { - for (const auto replication_factor : replication_factors) { - for (const auto &expiry_option : expiry_options) { - for (const auto &max_topic_size : max_topic_sizes) { - const std::string updated_topic_name = GetRandomName(); - SCOPED_TRACE( - "compression=" + compression_algorithm + ", replication=" + std::to_string(replication_factor) + - ", expiry_kind=" + expiry_option.kind + - ", expiry_value=" + std::to_string(expiry_option.value) + ", max_topic_size=" + max_topic_size); - - ASSERT_NO_THROW(client->update_topic(make_string_identifier(stream_name), - make_string_identifier(topic_name), updated_topic_name, - compression_algorithm, replication_factor, expiry_option.kind, - expiry_option.value, max_topic_size)); - topic_name = updated_topic_name; - } + for (const auto &expiry_option : expiry_options) { + for (const auto &max_topic_size : max_topic_sizes) { + const std::string updated_topic_name = GetRandomName(); + SCOPED_TRACE("compression=" + compression_algorithm + ", expiry_kind=" + expiry_option.kind + + ", expiry_value=" + std::to_string(expiry_option.value) + + ", max_topic_size=" + max_topic_size); + + ASSERT_NO_THROW(client->update_topic( + make_string_identifier(stream_name), make_string_identifier(topic_name), updated_topic_name, + compression_algorithm, expiry_option.kind, expiry_option.value, max_topic_size, {})); + topic_name = updated_topic_name; } } } @@ -969,19 +1109,19 @@ TEST_F(LowLevelE2E_Topic, UpdateTopicWithSameOptionsIsIdempotent) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), original_topic, 2, "none", 1, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), original_topic, 2, "none", + "server_default", 0, "server_default", {})); const auto created_topic = client->get_topic(make_string_identifier(stream_name), make_string_identifier(original_topic)); ASSERT_NO_THROW(client->update_topic(make_string_identifier(stream_name), make_numeric_identifier(created_topic.id), - updated_topic_name, "gzip", 1, "duration", 1000, "1GiB")); + updated_topic_name, "gzip", "duration", 1000, "1GiB", {})); const auto first_update = client->get_topic(make_string_identifier(stream_name), make_numeric_identifier(created_topic.id)); ASSERT_NO_THROW(client->update_topic(make_string_identifier(stream_name), make_numeric_identifier(created_topic.id), - updated_topic_name, "gzip", 1, "duration", 1000, "1GiB")); + updated_topic_name, "gzip", "duration", 1000, "1GiB", {})); const auto second_update = client->get_topic(make_string_identifier(stream_name), make_numeric_identifier(created_topic.id)); @@ -989,7 +1129,6 @@ TEST_F(LowLevelE2E_Topic, UpdateTopicWithSameOptionsIsIdempotent) { EXPECT_EQ(second_update.name, first_update.name); EXPECT_EQ(second_update.partitions_count, first_update.partitions_count); EXPECT_EQ(second_update.compression_algorithm, first_update.compression_algorithm); - EXPECT_EQ(second_update.replication_factor, first_update.replication_factor); EXPECT_EQ(second_update.message_expiry, first_update.message_expiry); EXPECT_EQ(second_update.max_topic_size, first_update.max_topic_size); } @@ -1004,13 +1143,13 @@ TEST_F(LowLevelE2E_Topic, UpdateTopicWithDuplicateTopicNameThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), first_topic_name, 1, "none", 1, - "server_default", 0, "server_default")); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), second_topic_name, 1, "none", 1, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), first_topic_name, 1, "none", + "server_default", 0, "server_default", {})); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), second_topic_name, 1, "none", + "server_default", 0, "server_default", {})); ASSERT_THROW(client->update_topic(make_string_identifier(stream_name), make_string_identifier(first_topic_name), - second_topic_name, "gzip", 1, "duration", 1000, "1GiB"), + second_topic_name, "gzip", "duration", 1000, "1GiB", {}), std::exception); } @@ -1023,8 +1162,8 @@ TEST_F(LowLevelE2E_Topic, UpdateTopicWithInvalidNamesThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 1, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); const std::vector invalid_topic_names = { "", @@ -1035,7 +1174,7 @@ TEST_F(LowLevelE2E_Topic, UpdateTopicWithInvalidNamesThrows) { SCOPED_TRACE("invalid_topic_name_length=" + std::to_string(invalid_topic_name.size())); ASSERT_THROW(client->update_topic(make_string_identifier(stream_name), make_string_identifier(topic_name), - invalid_topic_name, "gzip", 1, "duration", 1000, "1GiB"), + invalid_topic_name, "gzip", "duration", 1000, "1GiB", {}), std::exception); } } @@ -1051,13 +1190,13 @@ TEST_F(LowLevelE2E_Topic, UpdateTopicFailedValidationDoesNotMutateTopic) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); ASSERT_NO_THROW( - client->create_topic(make_string_identifier(stream_name), topic_name, 2, "gzip", 1, "duration", 1000, "1GiB")); + client->create_topic(make_string_identifier(stream_name), topic_name, 2, "gzip", "duration", 1000, "1GiB", {})); const auto topic_before_update = client->get_topic(make_string_identifier(stream_name), make_string_identifier(topic_name)); ASSERT_THROW(client->update_topic(make_string_identifier(stream_name), make_string_identifier(topic_name), - updated_topic_name, "none", 1, "duration", 2000, "not-a-size"), + updated_topic_name, "none", "duration", 2000, "not-a-size", {}), std::exception); const auto topic_after_failed_update = @@ -1067,7 +1206,6 @@ TEST_F(LowLevelE2E_Topic, UpdateTopicFailedValidationDoesNotMutateTopic) { EXPECT_EQ(topic_after_failed_update.name, topic_before_update.name); EXPECT_EQ(topic_after_failed_update.partitions_count, topic_before_update.partitions_count); EXPECT_EQ(topic_after_failed_update.compression_algorithm, topic_before_update.compression_algorithm); - EXPECT_EQ(topic_after_failed_update.replication_factor, topic_before_update.replication_factor); EXPECT_EQ(topic_after_failed_update.message_expiry, topic_before_update.message_expiry); EXPECT_EQ(topic_after_failed_update.max_topic_size, topic_before_update.max_topic_size); @@ -1085,25 +1223,25 @@ TEST_F(LowLevelE2E_Topic, UpdateTopicBeforeLoginThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 1, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); iggy::ffi::Client *unauthenticated_client = GetLoggedOutClient(); ASSERT_THROW( unauthenticated_client->update_topic(make_string_identifier(stream_name), make_string_identifier(topic_name), - updated_topic_name, "gzip", 1, "duration", 1000, "1GiB"), + updated_topic_name, "gzip", "duration", 1000, "1GiB", {}), std::exception); ASSERT_NO_THROW(unauthenticated_client->connect()); ASSERT_THROW( unauthenticated_client->update_topic(make_string_identifier(stream_name), make_string_identifier(topic_name), - updated_topic_name, "gzip", 1, "duration", 1000, "1GiB"), + updated_topic_name, "gzip", "duration", 1000, "1GiB", {}), std::exception); ASSERT_NO_THROW(unauthenticated_client->login_user("iggy", "iggy")); ASSERT_NO_THROW(unauthenticated_client->disconnect()); ASSERT_THROW( unauthenticated_client->update_topic(make_string_identifier(stream_name), make_string_identifier(topic_name), - updated_topic_name, "gzip", 1, "duration", 1000, "1GiB"), + updated_topic_name, "gzip", "duration", 1000, "1GiB", {}), std::exception); } @@ -1116,7 +1254,7 @@ TEST_F(LowLevelE2E_Topic, UpdateTopicOnNonExistentStreamThrows) { iggy::ffi::Client *client = GetLoggedInClient(); ASSERT_THROW(client->update_topic(make_string_identifier(stream_name), make_string_identifier(topic_name), - updated_topic_name, "gzip", 1, "duration", 1000, "1GiB"), + updated_topic_name, "gzip", "duration", 1000, "1GiB", {}), std::exception); } @@ -1132,7 +1270,7 @@ TEST_F(LowLevelE2E_Topic, UpdateTopicOnNonExistentTopicThrows) { TrackStream(stream_name); ASSERT_THROW(client->update_topic(make_string_identifier(stream_name), make_string_identifier(topic_name), - updated_topic_name, "gzip", 1, "duration", 1000, "1GiB"), + updated_topic_name, "gzip", "duration", 1000, "1GiB", {}), std::exception); } @@ -1145,8 +1283,8 @@ TEST_F(LowLevelE2E_Topic, GetTopicsAfterStreamDeletionReturnsEmpty) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW(client->delete_stream(make_string_identifier(stream_name))); ForgetTrackedStream(stream_name); @@ -1176,8 +1314,8 @@ TEST_F(LowLevelE2E_Topic, PurgeTopicAfterStreamDeletionThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); ASSERT_NO_THROW(client->delete_stream(make_string_identifier(stream_name))); ForgetTrackedStream(stream_name); @@ -1208,8 +1346,8 @@ TEST_F(LowLevelE2E_Topic, PurgeTopicWithInvalidStreamIdentifierThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); iggy::ffi::Identifier invalid_kind_id; invalid_kind_id.kind = "invalid"; @@ -1234,8 +1372,8 @@ TEST_F(LowLevelE2E_Topic, PurgeTopicWithInvalidTopicIdentifierThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); iggy::ffi::Identifier invalid_kind_id; invalid_kind_id.kind = "invalid"; @@ -1261,7 +1399,7 @@ TEST_F(LowLevelE2E_Topic, PurgeTopicPreservesTopicMetadata) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); ASSERT_NO_THROW( - client->create_topic(make_string_identifier(stream_name), topic_name, 3, "gzip", 1, "duration", 1000, "1GiB")); + client->create_topic(make_string_identifier(stream_name), topic_name, 3, "gzip", "duration", 1000, "1GiB", {})); auto stream_before_purge = client->get_stream(make_string_identifier(stream_name)); ASSERT_EQ(stream_before_purge.topics.size(), 1u); @@ -1292,7 +1430,6 @@ TEST_F(LowLevelE2E_Topic, PurgeTopicPreservesTopicMetadata) { EXPECT_EQ(topic_after_purge.message_expiry, topic_with_messages.message_expiry); EXPECT_EQ(topic_after_purge.compression_algorithm, topic_with_messages.compression_algorithm); EXPECT_EQ(topic_after_purge.max_topic_size, topic_with_messages.max_topic_size); - EXPECT_EQ(topic_after_purge.replication_factor, topic_with_messages.replication_factor); EXPECT_EQ(topic_after_purge.partitions_count, topic_with_messages.partitions_count); } @@ -1306,10 +1443,10 @@ TEST_F(LowLevelE2E_Topic, PurgeTopicRemovesOnlyTargetTopicMessages) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), first_topic_name, 1, "none", 0, - "server_default", 0, "server_default")); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), second_topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), first_topic_name, 1, "none", + "server_default", 0, "server_default", {})); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), second_topic_name, 1, "none", + "server_default", 0, "server_default", {})); const auto created_stream = client->get_stream(make_string_identifier(stream_name)); ASSERT_EQ(created_stream.topics.size(), 2u); @@ -1390,8 +1527,8 @@ TEST_F(LowLevelE2E_Topic, PurgeTopicAcrossMultiplePartitionsClearsAllPartitions) ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 3, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 3, "none", "server_default", + 0, "server_default", {})); const auto created_stream = client->get_stream(make_string_identifier(stream_name)); ASSERT_EQ(created_stream.topics.size(), 1u); @@ -1434,8 +1571,8 @@ TEST_F(LowLevelE2E_Topic, PurgeTopicThenSendMessagesAgainSucceeds) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); const auto created_stream = client->get_stream(make_string_identifier(stream_name)); ASSERT_EQ(created_stream.topics.size(), 1u); @@ -1475,10 +1612,10 @@ TEST_F(LowLevelE2E_Topic, PurgeTopicTwiceKeepsTargetTopicEmptyAndOtherTopicsUnto ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), first_topic_name, 1, "none", 0, - "server_default", 0, "server_default")); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), second_topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), first_topic_name, 1, "none", + "server_default", 0, "server_default", {})); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), second_topic_name, 1, "none", + "server_default", 0, "server_default", {})); const auto created_stream = client->get_stream(make_string_identifier(stream_name)); ASSERT_EQ(created_stream.topics.size(), 2u); @@ -1562,8 +1699,8 @@ TEST_F(LowLevelE2E_Topic, PurgeTopicBeforeLoginThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", 0, - "server_default", 0, "server_default")); + ASSERT_NO_THROW(client->create_topic(make_string_identifier(stream_name), topic_name, 1, "none", "server_default", + 0, "server_default", {})); iggy::ffi::Client *unauthenticated_client = GetLoggedOutClient(); diff --git a/foreign/cpp/tests/unit/unit_tests.cpp b/foreign/cpp/tests/unit/unit_tests.cpp index 8eaac4e857..6f4bc7590f 100644 --- a/foreign/cpp/tests/unit/unit_tests.cpp +++ b/foreign/cpp/tests/unit/unit_tests.cpp @@ -17,12 +17,30 @@ * under the License. */ +#include #include +#include #include #include "iggy.hpp" +namespace { + +std::string option_key(const iggy::ffi::HeaderEntry &entry) { + return std::string(entry.key.value.begin(), entry.key.value.end()); +} + +std::vector option_value_bytes(const iggy::ffi::HeaderEntry &entry) { + return std::vector(entry.value.value.begin(), entry.value.value.end()); +} + +constexpr std::uint8_t kind_code(const iggy::ffi::HeaderKind kind) { + return static_cast(kind); +} + +} // namespace + TEST(CompressionAlgorithmTest, ReturnsExpectedValues) { EXPECT_EQ(iggy::CompressionAlgorithm::none().compression_algorithm_value(), "none"); EXPECT_EQ(iggy::CompressionAlgorithm::gzip().compression_algorithm_value(), "gzip"); @@ -96,6 +114,71 @@ TEST(ExpiryTest, ReturnsExpectedKindAndValue) { EXPECT_EQ(duration.expiry_value(), static_cast(15)); } +TEST(TopicOptionTest, SegmentSizeEncodesLittleEndianUint64) { + const auto option = iggy::TopicOption::segment_size(0x0102030405060708ULL); + + EXPECT_EQ(option.key.kind, kind_code(iggy::ffi::HeaderKind::String)); + EXPECT_EQ(option_key(option), "segment_size"); + EXPECT_EQ(option.value.kind, kind_code(iggy::ffi::HeaderKind::Uint64)); + EXPECT_EQ(option_value_bytes(option), (std::vector{0x08, 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01})); +} + +TEST(TopicOptionTest, EnforceFsyncEncodesSingleBoolByte) { + const auto enabled = iggy::TopicOption::enforce_fsync(true); + + EXPECT_EQ(enabled.key.kind, kind_code(iggy::ffi::HeaderKind::String)); + EXPECT_EQ(option_key(enabled), "enforce_fsync"); + EXPECT_EQ(enabled.value.kind, kind_code(iggy::ffi::HeaderKind::Bool)); + EXPECT_EQ(option_value_bytes(enabled), (std::vector{1})); + + const auto disabled = iggy::TopicOption::enforce_fsync(false); + + EXPECT_EQ(option_key(disabled), "enforce_fsync"); + EXPECT_EQ(disabled.value.kind, kind_code(iggy::ffi::HeaderKind::Bool)); + EXPECT_EQ(option_value_bytes(disabled), (std::vector{0})); +} + +TEST(TopicOptionTest, MessagesRequiredToSaveEncodesLittleEndianUint32) { + const auto option = iggy::TopicOption::messages_required_to_save(0x01020304U); + + EXPECT_EQ(option.key.kind, kind_code(iggy::ffi::HeaderKind::String)); + EXPECT_EQ(option_key(option), "messages_required_to_save"); + EXPECT_EQ(option.value.kind, kind_code(iggy::ffi::HeaderKind::Uint32)); + EXPECT_EQ(option_value_bytes(option), (std::vector{0x04, 0x03, 0x02, 0x01})); +} + +TEST(TopicOptionTest, SizeOfMessagesRequiredToSaveEncodesLittleEndianUint64) { + const auto option = iggy::TopicOption::size_of_messages_required_to_save(1024ULL * 1024ULL); + + EXPECT_EQ(option.key.kind, kind_code(iggy::ffi::HeaderKind::String)); + EXPECT_EQ(option_key(option), "size_of_messages_required_to_save"); + EXPECT_EQ(option.value.kind, kind_code(iggy::ffi::HeaderKind::Uint64)); + EXPECT_EQ(option_value_bytes(option), (std::vector{0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00})); +} + +TEST(TopicOptionTest, PreallocateSegmentsEncodesSingleBoolByte) { + const auto enabled = iggy::TopicOption::preallocate_segments(true); + + EXPECT_EQ(enabled.key.kind, kind_code(iggy::ffi::HeaderKind::String)); + EXPECT_EQ(option_key(enabled), "preallocate_segments"); + EXPECT_EQ(enabled.value.kind, kind_code(iggy::ffi::HeaderKind::Bool)); + EXPECT_EQ(option_value_bytes(enabled), (std::vector{1})); + + const auto disabled = iggy::TopicOption::preallocate_segments(false); + + EXPECT_EQ(option_value_bytes(disabled), (std::vector{0})); +} + +TEST(TopicOptionTest, MaximumValuesFillEveryValueByte) { + const auto segment_size = iggy::TopicOption::segment_size(std::numeric_limits::max()); + EXPECT_EQ(option_value_bytes(segment_size), + (std::vector{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF})); + + const auto messages_required_to_save = + iggy::TopicOption::messages_required_to_save(std::numeric_limits::max()); + EXPECT_EQ(option_value_bytes(messages_required_to_save), (std::vector{0xFF, 0xFF, 0xFF, 0xFF})); +} + TEST(IggyExceptionTest, StoresMessage) { const iggy::IggyException from_cstr("boom"); EXPECT_EQ(std::string(from_cstr.what()), "boom"); diff --git a/foreign/csharp/Benchmarks/Program.cs b/foreign/csharp/Benchmarks/Program.cs index 030433baf2..ea796fa3de 100644 --- a/foreign/csharp/Benchmarks/Program.cs +++ b/foreign/csharp/Benchmarks/Program.cs @@ -71,7 +71,6 @@ await clients[0].CreateTopicAsync(Identifier.Numeric(startingStreamId + i), compressionAlgorithm: CompressionAlgorithm.None, messageExpiry: TimeSpan.Zero, maxTopicSize: 2_000_000_000, - replicationFactor: 3, partitionsCount: 1); } } diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/VsrCluster.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/VsrCluster.cs index 512b86a0e2..23001c9da5 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/VsrCluster.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/VsrCluster.cs @@ -226,7 +226,6 @@ private IContainer BuildNodeContainer(int node, IReadOnlyList nodeAddres .WithName($"iggy-vsr-{_name}-{node}-{_idSuffix}") .WithEnvironment("IGGY_ROOT_USERNAME", "iggy") .WithEnvironment("IGGY_ROOT_PASSWORD", "iggy") - .WithEnvironment("IGGY_SYSTEM_TOPIC_MESSAGE_EXPIRY", "10m") .WithEnvironment("IGGY_SYSTEM_PATH", $"local_data_vsr_{node}") .WithEnvironment("IGGY_TCP_ADDRESS", $"0.0.0.0:{ports.Tcp}") .WithEnvironment("IGGY_HTTP_ADDRESS", $"0.0.0.0:{ports.Http}") diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/TopicsTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/TopicsTests.cs index 88d2c23f36..050b3f9b87 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/TopicsTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/TopicsTests.cs @@ -43,7 +43,7 @@ public async Task Create_NewTopic_Should_Return_Successfully(Protocol protocol) var response = await client.CreateTopicAsync(Identifier.String(streamName), "Test Topic", 2, CompressionAlgorithm.Gzip, - 1, TimeSpan.FromMinutes(10), 2_000_000_000); + TimeSpan.FromMinutes(10), 2_000_000_000); response.ShouldNotBeNull(); response.Id.ShouldBeGreaterThanOrEqualTo(0u); @@ -54,7 +54,6 @@ public async Task Create_NewTopic_Should_Return_Successfully(Protocol protocol) response.MessageExpiry.ShouldBe(TimeSpan.FromMinutes(10)); response.Size.ShouldBe(0u); response.PartitionsCount.ShouldBe(2u); - response.ReplicationFactor.ShouldBe((byte?)1); response.MaxTopicSize.ShouldBe(2_000_000_000u); response.MessagesCount.ShouldBe(0u); } @@ -82,7 +81,7 @@ public async Task Get_ExistingTopic_Should_ReturnValidResponse(Protocol protocol var streamName = $"topic-get-{Guid.NewGuid():N}"; await client.CreateStreamAsync(streamName); await client.CreateTopicAsync(Identifier.String(streamName), "Get Topic", 2, - CompressionAlgorithm.Gzip, 1, TimeSpan.FromMinutes(10), 2_000_000_000); + CompressionAlgorithm.Gzip, TimeSpan.FromMinutes(10), 2_000_000_000); var response = await client.GetTopicByIdAsync(Identifier.String(streamName), Identifier.Numeric(0)); @@ -95,7 +94,6 @@ await client.CreateTopicAsync(Identifier.String(streamName), "Get Topic", 2, response.MessageExpiry.ShouldBe(TimeSpan.FromMinutes(10)); response.Size.ShouldBe(0u); response.PartitionsCount.ShouldBe(2u); - response.ReplicationFactor.ShouldBe((byte?)1); response.MaxTopicSize.ShouldBe(2_000_000_000u); response.MessagesCount.ShouldBe(0u); } @@ -109,7 +107,7 @@ public async Task Get_ExistingTopic_ByName_Should_ReturnValidResponse(Protocol p var streamName = $"topic-getname-{Guid.NewGuid():N}"; await client.CreateStreamAsync(streamName); await client.CreateTopicAsync(Identifier.String(streamName), "Name Topic", 2, - CompressionAlgorithm.Gzip, 1, TimeSpan.FromMinutes(10), 2_000_000_000); + CompressionAlgorithm.Gzip, TimeSpan.FromMinutes(10), 2_000_000_000); var response = await client.GetTopicByIdAsync(Identifier.String(streamName), Identifier.String("Name Topic")); @@ -122,7 +120,6 @@ await client.CreateTopicAsync(Identifier.String(streamName), "Name Topic", 2, response.MessageExpiry.ShouldBe(TimeSpan.FromMinutes(10)); response.Size.ShouldBe(0u); response.PartitionsCount.ShouldBe(2u); - response.ReplicationFactor.ShouldBe((byte?)1); response.MaxTopicSize.ShouldBe(2_000_000_000u); response.MessagesCount.ShouldBe(0u); } @@ -136,9 +133,9 @@ public async Task Get_ExistingTopics_Should_ReturnValidResponse(Protocol protoco var streamName = $"topic-list-{Guid.NewGuid():N}"; await client.CreateStreamAsync(streamName); await client.CreateTopicAsync(Identifier.String(streamName), "List Topic 1", 2, - CompressionAlgorithm.Gzip, 1, TimeSpan.FromMinutes(10), 2_000_000_000); + CompressionAlgorithm.Gzip, TimeSpan.FromMinutes(10), 2_000_000_000); await client.CreateTopicAsync(Identifier.String(streamName), "List Topic 2", 2, - CompressionAlgorithm.Gzip, 1, TimeSpan.FromMinutes(10), 2_000_000_000); + CompressionAlgorithm.Gzip, TimeSpan.FromMinutes(10), 2_000_000_000); IReadOnlyList response = await client.GetTopicsAsync(Identifier.String(streamName)); @@ -200,7 +197,7 @@ public async Task Update_ExistingTopic_Should_UpdateTopic_Successfully(Protocol await Should.NotThrowAsync(client.UpdateTopicAsync(Identifier.String(streamName), Identifier.Numeric(topicToUpdate.Id), "Updated Topic", - CompressionAlgorithm.Gzip, 3_000_000_000, TimeSpan.FromMinutes(10), 3)); + CompressionAlgorithm.Gzip, 3_000_000_000, TimeSpan.FromMinutes(10))); var result = await client.GetTopicByIdAsync(Identifier.String(streamName), Identifier.Numeric(topicToUpdate.Id)); @@ -209,7 +206,6 @@ await Should.NotThrowAsync(client.UpdateTopicAsync(Identifier.String(streamName) result.MessageExpiry.ShouldBe(TimeSpan.FromMinutes(10)); result.CompressionAlgorithm.ShouldBe(CompressionAlgorithm.Gzip); result.MaxTopicSize.ShouldBe(3_000_000_000u); - result.ReplicationFactor.ShouldBe((byte?)3); } [Test] diff --git a/foreign/csharp/Iggy_SDK/Contracts/Auth/UserResponse.cs b/foreign/csharp/Iggy_SDK/Contracts/Auth/UserResponse.cs index 756294b725..aa20b1e8d6 100644 --- a/foreign/csharp/Iggy_SDK/Contracts/Auth/UserResponse.cs +++ b/foreign/csharp/Iggy_SDK/Contracts/Auth/UserResponse.cs @@ -15,7 +15,9 @@ // specific language governing permissions and limitations // under the License. +using System.Text.Json.Serialization; using Apache.Iggy.Enums; +using Apache.Iggy.Headers; namespace Apache.Iggy.Contracts.Auth; @@ -48,4 +50,10 @@ public sealed class UserResponse /// Optional user permissions. /// public Permissions? Permissions { get; init; } + + ///

+ /// Options attached to the user at creation. TCP transport only. + /// + [JsonIgnore] + public Dictionary? Options { get; init; } } diff --git a/foreign/csharp/Iggy_SDK/Contracts/Http/CreateTopicRequest.cs b/foreign/csharp/Iggy_SDK/Contracts/Http/CreateTopicRequest.cs index d12083edf0..3ad058538c 100644 --- a/foreign/csharp/Iggy_SDK/Contracts/Http/CreateTopicRequest.cs +++ b/foreign/csharp/Iggy_SDK/Contracts/Http/CreateTopicRequest.cs @@ -26,9 +26,15 @@ internal sealed class CreateTopicRequest public CompressionAlgorithm CompressionAlgorithm { get; set; } = CompressionAlgorithm.None; public ulong MessageExpiry { get; set; } public uint PartitionsCount { get; set; } = 1; - public byte? ReplicationFactor { get; set; } = 1; public ulong MaxTopicSize { get; set; } + /// + /// Option keys with no property of their own, as strings the server parses by the same rules a + /// config file value goes through. The REST body carries them this way; the binary transports + /// send a typed TLV block. + /// + public Dictionary Options { get; set; } = new(); + public CreateTopicRequest() { } @@ -38,14 +44,14 @@ public CreateTopicRequest(string name, CompressionAlgorithm compressionAlgorithm, ulong messageExpiry, uint partitionsCount, - byte? replicationFactor, - ulong maxTopicSize) + ulong maxTopicSize, + Dictionary? options = null) { Name = name; CompressionAlgorithm = compressionAlgorithm; MessageExpiry = messageExpiry; PartitionsCount = partitionsCount; - ReplicationFactor = replicationFactor; MaxTopicSize = maxTopicSize; + Options = options ?? new Dictionary(); } } diff --git a/foreign/csharp/Iggy_SDK/Contracts/Http/UpdateTopicRequest.cs b/foreign/csharp/Iggy_SDK/Contracts/Http/UpdateTopicRequest.cs index 8e5daffe16..bef38de81f 100644 --- a/foreign/csharp/Iggy_SDK/Contracts/Http/UpdateTopicRequest.cs +++ b/foreign/csharp/Iggy_SDK/Contracts/Http/UpdateTopicRequest.cs @@ -19,9 +19,13 @@ namespace Apache.Iggy.Contracts.Http; +/// +/// A topic update over REST. carries keys with no property of their +/// own, as strings the server parses by the same rules a config file value goes through. +/// internal record UpdateTopicRequest( string Name, CompressionAlgorithm CompressionAlgorithm, ulong MaxTopicSize, ulong MessageExpiry, - byte? ReplicationFactor); + Dictionary Options); diff --git a/foreign/csharp/Iggy_SDK/Contracts/OptionSpec.cs b/foreign/csharp/Iggy_SDK/Contracts/OptionSpec.cs new file mode 100644 index 0000000000..7eb9f96343 --- /dev/null +++ b/foreign/csharp/Iggy_SDK/Contracts/OptionSpec.cs @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Apache.Iggy.Headers; + +namespace Apache.Iggy.Contracts; + +/// +/// Resource whose option catalog the server describes. +/// +public enum OptionsScope +{ + /// + /// Keys a CreateTopic accepts. + /// + Topic = 1, + + /// + /// Keys a CreateStream accepts; none yet. + /// + Stream = 2, + + /// + /// Keys a CreateUser accepts; none yet. + /// + User = 3 +} + +/// +/// One entry of a resource's option catalog, as served by DescribeOptions. +/// +public class OptionSpec +{ + /// + /// Option key the create command accepts. + /// + public required string Key { get; set; } + + /// + /// Canonical kind for this key: what the server encodes its default under, and + /// what a value set at create is stored as whatever kind the client sent, since + /// create admission re-encodes the block from its own parse. An update stores + /// the client's bytes verbatim and is the exception. + /// + public required HeaderKind Kind { get; set; } + + /// + /// The key's default in 's encoding; empty when the key + /// has no default. + /// + public required byte[] DefaultValue { get; set; } = []; + + /// + /// What the option does, including any bounds it is checked against. + /// + public required string Description { get; set; } +} diff --git a/foreign/csharp/Iggy_SDK/Contracts/StreamResponse.cs b/foreign/csharp/Iggy_SDK/Contracts/StreamResponse.cs index b9d3172af6..ee23a0ec6f 100644 --- a/foreign/csharp/Iggy_SDK/Contracts/StreamResponse.cs +++ b/foreign/csharp/Iggy_SDK/Contracts/StreamResponse.cs @@ -17,6 +17,7 @@ using System.Text.Json.Serialization; +using Apache.Iggy.Headers; using Apache.Iggy.JsonConverters; namespace Apache.Iggy.Contracts; @@ -62,4 +63,10 @@ public sealed class StreamResponse /// List of topics in the stream. /// public IEnumerable Topics { get; init; } = []; + + /// + /// Options attached to the stream at creation. TCP transport only. + /// + [JsonIgnore] + public Dictionary? Options { get; init; } } diff --git a/foreign/csharp/Iggy_SDK/Contracts/Tcp/TcpContracts.cs b/foreign/csharp/Iggy_SDK/Contracts/Tcp/TcpContracts.cs index e96e6ddd87..994fefe3e9 100644 --- a/foreign/csharp/Iggy_SDK/Contracts/Tcp/TcpContracts.cs +++ b/foreign/csharp/Iggy_SDK/Contracts/Tcp/TcpContracts.cs @@ -640,37 +640,89 @@ internal static byte[] GetGroup(Identifier streamId, Identifier topicId, Identif } internal static byte[] UpdateTopic(Identifier streamId, Identifier topicId, string name, - CompressionAlgorithm compressionAlgorithm, ulong maxTopicSize, ulong messageExpiry, byte? replicationFactor) + CompressionAlgorithm compressionAlgorithm, ulong maxTopicSize, ulong messageExpiry, + IReadOnlyDictionary? extraOptions = null) { - Span bytes = stackalloc byte[4 + streamId.Length + topicId.Length + 19 + name.Length]; + // Settings ride the options block. A default value means the caller did + // not set the key, so it is omitted and the server leaves the topic's + // current value alone. + var options = new Dictionary(); + // Caller keys first, so a named argument overwrites one of them. + if (extraOptions is not null) + { + foreach (var (key, value) in extraOptions) + { + options[HeaderKey.FromString(key)] = value; + } + } + + if (compressionAlgorithm != CompressionAlgorithm.None) + { + options[HeaderKey.FromString("compression_algorithm")] + = HeaderValue.FromString(compressionAlgorithm.ToString().ToLowerInvariant()); + } + + if (messageExpiry != 0) + { + options[HeaderKey.FromString("message_expiry")] = HeaderValue.FromUInt64(messageExpiry); + } + + if (maxTopicSize != 0) + { + options[HeaderKey.FromString("max_topic_size")] = HeaderValue.FromUInt64(maxTopicSize); + } + + var optionsLength = HeadersByteLength(options); + Span bytes = + stackalloc byte[4 + streamId.Length + topicId.Length + 1 + name.Length + optionsLength]; bytes.WriteBytesFromStreamAndTopicIdentifiers(streamId, topicId); var position = 4 + streamId.Length + topicId.Length; - bytes[position] = (byte)compressionAlgorithm; - position += 1; - BinaryPrimitives.WriteUInt64LittleEndian(bytes[position..(position + 8)], - messageExpiry); - BinaryPrimitives.WriteUInt64LittleEndian(bytes[(position + 8)..(position + 16)], - maxTopicSize); - bytes[position + 16] = replicationFactor ?? 0; - bytes[position + 17] = (byte)name.Length; - Encoding.UTF8.GetBytes(name, bytes[(position + 18)..]); + bytes[position] = (byte)name.Length; + Encoding.UTF8.GetBytes(name, bytes[(position + 1)..(position + 1 + name.Length)]); + WriteHeadersTo(bytes[(position + 1 + name.Length)..], options); return bytes.ToArray(); } internal static byte[] CreateTopic(Identifier streamId, string name, uint partitionCount, - CompressionAlgorithm compressionAlgorithm, byte? replicationFactor, ulong messageExpiry, - ulong maxTopicSize) + CompressionAlgorithm compressionAlgorithm, ulong messageExpiry, + ulong maxTopicSize, IReadOnlyDictionary? extraOptions = null) { - Span bytes = stackalloc byte[2 + streamId.Length + 23 + name.Length]; + var options = new Dictionary(); + // Caller keys go in first so a named argument overwrites one of them: the + // block must not carry a key twice, or the server refuses it whole. + if (extraOptions is not null) + { + foreach (var (key, value) in extraOptions) + { + options[HeaderKey.FromString(key)] = value; + } + } + + if (compressionAlgorithm != CompressionAlgorithm.None) + { + options[HeaderKey.FromString("compression_algorithm")] + = HeaderValue.FromString(compressionAlgorithm.ToString().ToLowerInvariant()); + } + + if (messageExpiry != 0) + { + options[HeaderKey.FromString("message_expiry")] = HeaderValue.FromUInt64(messageExpiry); + } + + if (maxTopicSize != 0) + { + options[HeaderKey.FromString("max_topic_size")] = HeaderValue.FromUInt64(maxTopicSize); + } + + var optionsLength = HeadersByteLength(options); + Span bytes = stackalloc byte[2 + streamId.Length + 4 + 1 + name.Length + optionsLength]; bytes.WriteBytesFromIdentifier(streamId); var position = 2 + streamId.Length; BinaryPrimitives.WriteUInt32LittleEndian(bytes[position..(position + 4)], partitionCount); - bytes[position + 4] = (byte)compressionAlgorithm; - BinaryPrimitives.WriteUInt64LittleEndian(bytes[(position + 5)..(position + 13)], messageExpiry); - BinaryPrimitives.WriteUInt64LittleEndian(bytes[(position + 13)..(position + 21)], maxTopicSize); - bytes[position + 21] = replicationFactor ?? 0; - bytes[position + 22] = (byte)name.Length; - Encoding.UTF8.GetBytes(name, bytes[(position + 23)..]); + position += 4; + bytes[position] = (byte)name.Length; + Encoding.UTF8.GetBytes(name, bytes[(position + 1)..(position + 1 + name.Length)]); + WriteHeadersTo(bytes[(position + 1 + name.Length)..], options); return bytes.ToArray(); } diff --git a/foreign/csharp/Iggy_SDK/Contracts/TopicOptions.cs b/foreign/csharp/Iggy_SDK/Contracts/TopicOptions.cs new file mode 100644 index 0000000000..db3572d44e --- /dev/null +++ b/foreign/csharp/Iggy_SDK/Contracts/TopicOptions.cs @@ -0,0 +1,106 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Apache.Iggy.Headers; +using Apache.Iggy.IggyClient; + +namespace Apache.Iggy.Contracts; + +/// +/// Topic options that have no named parameter of their own, typed instead of keyed by hand. +/// +/// +/// Hand the result of to the options parameter of +/// . Every key here is settable at creation only: an +/// update takes just the options that have a parameter of their own and refuses the rest by +/// name, the same way it refuses a key outside the server's catalog, so a mistyped key fails +/// the call rather than being ignored. +/// enumerates the keys a given server accepts, with the kind and default of each. +/// A property left null emits no key, which leaves the server default in place. +/// +public sealed class TopicOptions +{ + private const string SegmentSizeKey = "segment_size"; + private const string EnforceFsyncKey = "enforce_fsync"; + private const string MessagesRequiredToSaveKey = "messages_required_to_save"; + private const string SizeOfMessagesRequiredToSaveKey = "size_of_messages_required_to_save"; + private const string PreallocateSegmentsKey = "preallocate_segments"; + + /// + /// Size of a single segment in bytes. The server checks it against its own bounds, a + /// multiple of 512 within the range it reports for the key. + /// + public ulong? SegmentSize { get; init; } + + /// + /// Whether writes to this topic's partitions are fsynced. + /// + public bool? EnforceFsync { get; init; } + + /// + /// Flush the journal once it holds this many messages. Must be non-zero. + /// + public uint? MessagesRequiredToSave { get; init; } + + /// + /// Flush the journal once it holds this many bytes. Paired with + /// : whichever threshold trips first flushes. + /// + public ulong? SizeOfMessagesRequiredToSave { get; init; } + + /// + /// Reserve a segment's bytes up front on a filesystem that supports it. Reserves exactly + /// , so the two belong to one decision. + /// + public bool? PreallocateSegments { get; init; } + + /// + /// Renders the options that were set, each under the kind the server's catalog gives its key. + /// + /// Option values keyed by option name, empty when nothing was set. + public Dictionary ToDictionary() + { + var options = new Dictionary(); + + if (SegmentSize is { } segmentSize) + { + options[SegmentSizeKey] = HeaderValue.FromUInt64(segmentSize); + } + + if (EnforceFsync is { } enforceFsync) + { + options[EnforceFsyncKey] = HeaderValue.FromBool(enforceFsync); + } + + if (MessagesRequiredToSave is { } messagesRequiredToSave) + { + options[MessagesRequiredToSaveKey] = HeaderValue.FromUInt32(messagesRequiredToSave); + } + + if (SizeOfMessagesRequiredToSave is { } sizeOfMessagesRequiredToSave) + { + options[SizeOfMessagesRequiredToSaveKey] = HeaderValue.FromUInt64(sizeOfMessagesRequiredToSave); + } + + if (PreallocateSegments is { } preallocateSegments) + { + options[PreallocateSegmentsKey] = HeaderValue.FromBool(preallocateSegments); + } + + return options; + } +} diff --git a/foreign/csharp/Iggy_SDK/Contracts/TopicResponse.cs b/foreign/csharp/Iggy_SDK/Contracts/TopicResponse.cs index 68297de8ac..877aec6178 100644 --- a/foreign/csharp/Iggy_SDK/Contracts/TopicResponse.cs +++ b/foreign/csharp/Iggy_SDK/Contracts/TopicResponse.cs @@ -18,6 +18,7 @@ using System.Text.Json.Serialization; using Apache.Iggy.Enums; +using Apache.Iggy.Headers; using Apache.Iggy.JsonConverters; namespace Apache.Iggy.Contracts; @@ -75,13 +76,43 @@ public sealed class TopicResponse /// public required uint PartitionsCount { get; init; } - /// - /// Replication factor of the topic. - /// - public required byte? ReplicationFactor { get; init; } /// /// List of partitions in the topic. /// public IEnumerable? Partitions { get; init; } + + /// + /// Options explicitly set by the client at topic creation. Over REST the values arrive in + /// their readable string form, so every one of them is ; + /// the binary transports carry the server's canonical kind per key. + /// + [JsonIgnore] + public Dictionary? Options { get; init; } + + /// + /// Options resolved from server defaults at topic creation. Over REST the values arrive in + /// their readable string form, so every one of them is ; + /// the binary transports carry the server's canonical kind per key. + /// + [JsonIgnore] + public Dictionary? DerivedOptions { get; init; } + + /// + /// REST renders both provenances into one object keyed by option name, so one JSON property + /// fills and . Those two stay ignored by + /// the serializer: a is not a JSON string key, and one of them would + /// claim this property's name. + /// + [JsonInclude] + [JsonPropertyName("options")] + [JsonConverter(typeof(ResourceOptionsConverter))] + internal ResourceOptions RestOptions + { + init + { + Options = value.Explicit; + DerivedOptions = value.Derived; + } + } } diff --git a/foreign/csharp/Iggy_SDK/Headers/HeaderValue.cs b/foreign/csharp/Iggy_SDK/Headers/HeaderValue.cs index 681849fc48..d551965e52 100644 --- a/foreign/csharp/Iggy_SDK/Headers/HeaderValue.cs +++ b/foreign/csharp/Iggy_SDK/Headers/HeaderValue.cs @@ -145,6 +145,20 @@ public static HeaderValue FromGuid(Guid value) }; } + /// + /// Creates a header value from a byte. + /// + /// Header value + /// + public static HeaderValue FromUInt8(byte value) + { + return new HeaderValue + { + Kind = HeaderKind.Uint8, + Value = [value] + }; + } + /// /// Creates a header value from a uint. /// diff --git a/foreign/csharp/Iggy_SDK/IggyClient/IIggySystem.cs b/foreign/csharp/Iggy_SDK/IggyClient/IIggySystem.cs index aab60f1c6c..4b505eb10a 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/IIggySystem.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/IIggySystem.cs @@ -72,6 +72,24 @@ public interface IIggySystem /// A task that represents the asynchronous operation and returns server statistics, or null if unavailable. Task GetStatsAsync(CancellationToken token = default); + /// + /// Retrieves the option catalog for a resource scope: every key its create + /// command accepts, with the kind, default and bounds of each. + /// + /// + /// This is the discovery surface for options. A key outside the catalog is + /// refused at create, and binary transports carry only the error code, so a + /// client has no other way to learn which keys this server knows. Available + /// only for TCP. + /// + /// The resource whose catalog to describe. + /// The cancellation token to cancel the operation. + /// + /// A task that represents the asynchronous operation and returns the catalog + /// entries, empty for a scope with no keys yet. + /// + Task> DescribeOptionsAsync(OptionsScope scope, CancellationToken token = default); + /// /// Retrieves cluster metadata including node information and connection information. /// diff --git a/foreign/csharp/Iggy_SDK/IggyClient/IIggyTopic.cs b/foreign/csharp/Iggy_SDK/IggyClient/IIggyTopic.cs index 5298d25bdc..391fdff6e2 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/IIggyTopic.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/IIggyTopic.cs @@ -17,6 +17,7 @@ using Apache.Iggy.Contracts; using Apache.Iggy.Enums; +using Apache.Iggy.Headers; namespace Apache.Iggy.IggyClient; @@ -57,24 +58,31 @@ public interface IIggyTopic /// The unique name of the topic (max 255 characters). /// The number of partitions for the topic (max 1000). /// The compression algorithm to use for messages (default: None). - /// The replication factor for the topic (optional). /// The message expiry period (0 for server default, MaxValue for never expire). /// The maximum size of the topic in bytes (0 = unlimited). + /// + /// Option keys with no parameter of their own, keyed by option name. Reaches a key the server + /// catalog gained after this build shipped; a named parameter above wins on collision, and a key + /// outside the catalog is refused by name. builds this map for the + /// keys the catalog ships with. Call + /// to see which keys a server accepts. + /// /// The cancellation token to cancel the operation. /// /// A task that represents the asynchronous operation and returns the created topic information, or null if /// creation failed. /// Task CreateTopicAsync(Identifier streamId, string name, uint partitionsCount, - CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, byte? replicationFactor = null, - TimeSpan? messageExpiry = null, ulong maxTopicSize = 0, CancellationToken token = default); + CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, + TimeSpan? messageExpiry = null, ulong maxTopicSize = 0, + IReadOnlyDictionary? options = null, CancellationToken token = default); /// /// Updates the configuration of an existing topic. /// /// - /// This method allows updating topic properties such as name, compression algorithm, size limits, message expiry, and - /// replication factor. + /// This method allows updating topic properties such as name, compression algorithm, size limits and message + /// expiry. /// /// The identifier of the stream containing the topic (numeric ID or name). /// The identifier of the topic to update (numeric ID or name). @@ -82,12 +90,16 @@ public interface IIggyTopic /// The new compression algorithm to use (default: None). /// The new maximum size of the topic in bytes (0 = unlimited). /// The new message expiry period (0 for server default, MaxValue for never expire). - /// The new replication factor (optional). + /// + /// Option keys with no parameter of their own. The server refuses any key an update may not + /// change, by name; a key left out keeps its current value. + /// /// The cancellation token to cancel the operation. /// A task that represents the asynchronous operation. Task UpdateTopicAsync(Identifier streamId, Identifier topicId, string name, CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, ulong maxTopicSize = 0, - TimeSpan? messageExpiry = null, byte? replicationFactor = null, CancellationToken token = default); + TimeSpan? messageExpiry = null, IReadOnlyDictionary? options = null, + CancellationToken token = default); /// /// Deletes an existing topic and all its associated messages and partitions. diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs index b8f51c7a5e..63b80b44e9 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs @@ -31,6 +31,8 @@ using Apache.Iggy.Encryption; using Apache.Iggy.Enums; using Apache.Iggy.Exceptions; +using Apache.Iggy.Headers; +using Apache.Iggy.JsonConverters; using Apache.Iggy.Kinds; using Apache.Iggy.Mappers; using Apache.Iggy.Messages; @@ -155,8 +157,9 @@ public async Task> GetStreamsAsync(CancellationTok /// public async Task CreateTopicAsync(Identifier streamId, string name, uint partitionsCount, - CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, byte? replicationFactor = null, - TimeSpan? messageExpiry = null, ulong maxTopicSize = 0, CancellationToken token = default) + CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, + TimeSpan? messageExpiry = null, ulong maxTopicSize = 0, + IReadOnlyDictionary? options = null, CancellationToken token = default) { var json = JsonSerializer.Serialize(new CreateTopicRequest { @@ -165,7 +168,7 @@ public async Task> GetStreamsAsync(CancellationTok MaxTopicSize = maxTopicSize, MessageExpiry = DurationHelpers.ToDuration(messageExpiry), PartitionsCount = partitionsCount, - ReplicationFactor = replicationFactor + Options = ToStringOptions(options) }, _jsonSerializerOptions); var data = new StringContent(json, Encoding.UTF8, "application/json"); @@ -184,12 +187,13 @@ public async Task> GetStreamsAsync(CancellationTok /// public async Task UpdateTopicAsync(Identifier streamId, Identifier topicId, string name, CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, - ulong maxTopicSize = 0, TimeSpan? messageExpiry = null, byte? replicationFactor = null, + ulong maxTopicSize = 0, TimeSpan? messageExpiry = null, + IReadOnlyDictionary? options = null, CancellationToken token = default) { - var json = JsonSerializer.Serialize(new UpdateTopicRequest(name, compressionAlgorithm, maxTopicSize, - DurationHelpers.ToDuration(messageExpiry), - replicationFactor), + var json = JsonSerializer.Serialize( + new UpdateTopicRequest(name, compressionAlgorithm, maxTopicSize, + DurationHelpers.ToDuration(messageExpiry), ToStringOptions(options)), _jsonSerializerOptions); var data = new StringContent(json, Encoding.UTF8, "application/json"); var response = await _httpClient.PutAsync($"/streams/{streamId}/topics/{topicId}", data, token); @@ -471,6 +475,70 @@ var response throw new FeatureUnavailableException(); } + /// + public async Task> DescribeOptionsAsync(OptionsScope scope, + CancellationToken token = default) + { + var response = await _httpClient.GetAsync($"/options/{scope.ToString().ToLowerInvariant()}", token); + if (response.IsSuccessStatusCode) + { + var specs = await response.Content.ReadFromJsonAsync>(_jsonSerializerOptions, token); + return specs?.Select(spec => spec.ToOptionSpec()).ToList() ?? []; + } + + await HandleResponseAsync(response); + + return []; + } + + /// + /// The REST shape of a catalog entry: the kind arrives as its name and the default as raw + /// bytes, where the binary transport sends a kind code. Mapping it here keeps + /// the one shape a caller sees on either transport. + /// + private sealed record HttpOptionSpec(string Key, string Kind, byte[] DefaultValue, string Description) + { + internal OptionSpec ToOptionSpec() + { + return new OptionSpec + { + Key = Key, + Kind = UserHeadersConverter.ParseHeaderKind(Kind), + DefaultValue = DefaultValue, + Description = Description + }; + } + } + + /// + /// Renders option values as the strings the REST body carries them in. + /// + /// The binary transports send a typed TLV block, but the JSON body takes a plain string map the + /// server parses by the same rules a config file value goes through, so a typed value handed in + /// here is rendered rather than passed through. + /// + private static Dictionary ToStringOptions(IReadOnlyDictionary? options) + { + if (options is null) + { + return new Dictionary(); + } + + return options.ToDictionary(entry => entry.Key, entry => ToStringValue(entry.Value)); + } + + private static string ToStringValue(HeaderValue value) + { + // A Bool header renders as "1" or "0", which the server's option parser refuses. It takes + // the words a config file would carry. + if (value.Kind is HeaderKind.Bool) + { + return value.ToBool() ? "true" : "false"; + } + + return value.ToString(); + } + /// public async Task GetStatsAsync(CancellationToken token = default) { diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs index c68ad62e69..75189e84c3 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs @@ -31,6 +31,7 @@ using Apache.Iggy.Encryption; using Apache.Iggy.Enums; using Apache.Iggy.Exceptions; +using Apache.Iggy.Headers; using Apache.Iggy.Kinds; using Apache.Iggy.Mappers; using Apache.Iggy.Messages; @@ -253,12 +254,13 @@ public async Task> GetTopicsAsync(Identifier stream /// public async Task CreateTopicAsync(Identifier streamId, string name, uint partitionsCount, - CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, byte? replicationFactor = null, - TimeSpan? messageExpiry = null, ulong maxTopicSize = 0, CancellationToken token = default) + CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, + TimeSpan? messageExpiry = null, ulong maxTopicSize = 0, + IReadOnlyDictionary? options = null, CancellationToken token = default) { var messageExpiryValue = DurationHelpers.ToDuration(messageExpiry); var message = TcpContracts.CreateTopic(streamId, name, partitionsCount, compressionAlgorithm, - replicationFactor, messageExpiryValue, maxTopicSize); + messageExpiryValue, maxTopicSize, options); var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_TOPIC_CODE); @@ -275,12 +277,13 @@ public async Task> GetTopicsAsync(Identifier stream /// public async Task UpdateTopicAsync(Identifier streamId, Identifier topicId, string name, CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, - ulong maxTopicSize = 0, TimeSpan? messageExpiry = null, byte? replicationFactor = null, + ulong maxTopicSize = 0, TimeSpan? messageExpiry = null, + IReadOnlyDictionary? options = null, CancellationToken token = default) { var messageExpiryValue = DurationHelpers.ToDuration(messageExpiry); var message = TcpContracts.UpdateTopic(streamId, topicId, name, compressionAlgorithm, maxTopicSize, - messageExpiryValue, replicationFactor); + messageExpiryValue, options); var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.UPDATE_TOPIC_CODE); @@ -577,6 +580,24 @@ public async Task DeleteSegmentsAsync(Identifier streamId, Identifier topicId, u return BinaryMapper.MapStats(responseBuffer.Memory.Span); } + /// + public async Task> DescribeOptionsAsync(OptionsScope scope, + CancellationToken token = default) + { + var message = new[] { (byte)scope }; + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DESCRIBE_OPTIONS_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return []; + } + + return BinaryMapper.MapOptionSpecs(responseBuffer.Memory.Span); + } + /// public async Task GetClusterMetadataAsync(CancellationToken token = default) { diff --git a/foreign/csharp/Iggy_SDK/JsonConverters/ResourceOptionsConverter.cs b/foreign/csharp/Iggy_SDK/JsonConverters/ResourceOptionsConverter.cs new file mode 100644 index 0000000000..adc1aadcb9 --- /dev/null +++ b/foreign/csharp/Iggy_SDK/JsonConverters/ResourceOptionsConverter.cs @@ -0,0 +1,140 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Text.Json; +using System.Text.Json.Serialization; +using Apache.Iggy.Headers; + +namespace Apache.Iggy.JsonConverters; + +/// +/// A resource's options split by provenance, the way the binary transports send them: what a +/// client set at creation, and what the server resolved from its own defaults. +/// +internal readonly record struct ResourceOptions( + Dictionary Explicit, + Dictionary Derived); + +/// +/// Reads the one object a REST response renders both provenances into, keyed by option name, +/// each entry a readable value plus the flag saying who set it: +/// {"segment_size":{"value":"1073741824","explicit":false}}. The value arrives as the +/// string a config file or a create body would carry it in, so every value maps to +/// and the server's canonical kind for the key is not +/// recoverable from it. +/// +internal sealed class ResourceOptionsConverter : JsonConverter +{ + public override ResourceOptions Read(ref Utf8JsonReader reader, Type typeToConvert, + JsonSerializerOptions options) + { + var explicitOptions = new Dictionary(); + var derivedOptions = new Dictionary(); + + if (reader.TokenType == JsonTokenType.Null) + { + return new ResourceOptions(explicitOptions, derivedOptions); + } + + if (reader.TokenType != JsonTokenType.StartObject) + { + throw new JsonException($"Expected start of object for options but got {reader.TokenType}."); + } + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndObject) + { + return new ResourceOptions(explicitOptions, derivedOptions); + } + + if (reader.TokenType != JsonTokenType.PropertyName) + { + throw new JsonException($"Expected option name but got {reader.TokenType}."); + } + + var key = reader.GetString(); + if (string.IsNullOrEmpty(key)) + { + throw new JsonException("Option name must not be empty."); + } + + reader.Read(); + var (value, isExplicit) = ReadOptionValue(ref reader, key); + var target = isExplicit ? explicitOptions : derivedOptions; + target[HeaderKey.FromString(key)] = value; + } + + throw new JsonException("Unexpected end of options object."); + } + + /// + /// Options travel out as the plain string map a create or update body takes, never in this + /// shape, so nothing serializes through here. + /// + public override void Write(Utf8JsonWriter writer, ResourceOptions value, JsonSerializerOptions options) + { + throw new NotSupportedException("Resource options are read from a response, not written to a request."); + } + + private static (HeaderValue value, bool isExplicit) ReadOptionValue(ref Utf8JsonReader reader, string key) + { + if (reader.TokenType != JsonTokenType.StartObject) + { + throw new JsonException($"Expected start of object for option '{key}' but got {reader.TokenType}."); + } + + string? value = null; + bool? isExplicit = null; + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndObject) + { + break; + } + + if (reader.TokenType != JsonTokenType.PropertyName) + { + throw new JsonException($"Expected property name for option '{key}'."); + } + + var propertyName = reader.GetString(); + reader.Read(); + + switch (propertyName) + { + case "value": + value = reader.GetString(); + break; + case "explicit": + isExplicit = reader.GetBoolean(); + break; + default: + reader.Skip(); + break; + } + } + + if (value is null || isExplicit is null) + { + throw new JsonException($"Option '{key}' must have both 'value' and 'explicit' properties."); + } + + return (HeaderValue.FromString(value), isExplicit.Value); + } +} diff --git a/foreign/csharp/Iggy_SDK/JsonConverters/UserHeadersConverter.cs b/foreign/csharp/Iggy_SDK/JsonConverters/UserHeadersConverter.cs index 35aa32688a..2ce8e958f4 100644 --- a/foreign/csharp/Iggy_SDK/JsonConverters/UserHeadersConverter.cs +++ b/foreign/csharp/Iggy_SDK/JsonConverters/UserHeadersConverter.cs @@ -179,7 +179,7 @@ private static HeaderValue ReadHeaderValue(ref Utf8JsonReader reader) return new HeaderValue { Kind = kind.Value, Value = value }; } - private static HeaderKind ParseHeaderKind(string? kindStr) + internal static HeaderKind ParseHeaderKind(string? kindStr) { return kindStr switch { diff --git a/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs b/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs index f5be657f20..124e9bc70a 100644 --- a/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs +++ b/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs @@ -112,7 +112,8 @@ internal static UserResponse MapUser(ReadOnlySpan payload) Id = response.Id, CreatedAt = response.CreatedAt, Username = response.Username, - Status = response.Status + Status = response.Status, + Options = response.Options }; } @@ -122,7 +123,8 @@ internal static UserResponse MapUser(ReadOnlySpan payload) CreatedAt = response.CreatedAt, Username = response.Username, Status = response.Status, - Permissions = null + Permissions = null, + Options = response.Options }; } @@ -228,13 +230,16 @@ private static (UserResponse response, int position) MapToUserResponse(ReadOnlyS var usernameLength = payload[position + 13]; var username = Encoding.UTF8.GetString(payload[(position + 14)..(position + 14 + usernameLength)]); var readBytes = 4 + 8 + 1 + 1 + usernameLength; + var options = MapOptions(payload, position + readBytes, out var optionsReadBytes); + readBytes += optionsReadBytes; return (new UserResponse { Id = id, CreatedAt = createdAt, Status = userStatus, - Username = username + Username = username, + Options = options }, readBytes); } @@ -705,7 +710,7 @@ headers[new HeaderKey return headers; } - private static HeaderKind MapHeaderKind(byte value) + internal static HeaderKind MapHeaderKind(byte value) { return value switch { @@ -740,6 +745,98 @@ private static bool TryMapHeaderKind(byte value, out HeaderKind kind) return false; } + private static Dictionary MapOptions(ReadOnlySpan payload, int position, + out int readBytes) + { + // Every length here is server-controlled. Read the block length as long + // so a value above int.MaxValue cannot wrap negative, and bound each + // entry against the block before slicing: an entry that overruns `end` + // would otherwise be accepted and silently consume the response bytes + // that follow the block. + var optionsLength = BinaryPrimitives.ReadUInt32LittleEndian(payload[position..(position + 4)]); + var available = (long)payload.Length - (position + 4); + if (optionsLength > available) + { + throw new MalformedResponseException( + $"Malformed options block at byte {position}: declared length {optionsLength} exceeds the " + + $"{available} bytes remaining in the payload."); + } + + readBytes = 4 + (int)optionsLength; + + var options = new Dictionary(); + var cursor = position + 4; + var end = cursor + (int)optionsLength; + while (cursor < end) + { + var keyKind = MapHeaderKind(ReadOptionByte(payload, ref cursor, end, position)); + var key = ReadOptionField(payload, ref cursor, end, position, "key"); + + var valueKind = MapHeaderKind(ReadOptionByte(payload, ref cursor, end, position)); + var value = ReadOptionField(payload, ref cursor, end, position, "value"); + + options[new HeaderKey + { + Kind = keyKind, + Value = key + }] = new HeaderValue + { + Kind = valueKind, + Value = value + }; + } + + if (cursor != end) + { + throw new MalformedResponseException( + $"Malformed options block at byte {position}: entries ended at {cursor}, block ends at {end}."); + } + + return options; + } + + private static byte ReadOptionByte(ReadOnlySpan payload, ref int cursor, int end, int blockStart) + { + if (cursor + 1 > end) + { + throw new MalformedResponseException( + $"Malformed options block at byte {blockStart}: entry kind runs past the end of the block."); + } + + var value = payload[cursor]; + cursor += 1; + return value; + } + + private static byte[] ReadOptionField(ReadOnlySpan payload, ref int cursor, int end, int blockStart, + string field) + { + if (cursor + 4 > end) + { + throw new MalformedResponseException( + $"Malformed options block at byte {blockStart}: {field} length runs past the end of the block."); + } + + var length = BinaryPrimitives.ReadUInt32LittleEndian(payload[cursor..(cursor + 4)]); + cursor += 4; + if (length is < 1 or > 255) + { + throw new MalformedResponseException( + $"Malformed options block at byte {blockStart}: {field} length {length} is outside 1..=255."); + } + + if (cursor + (int)length > end) + { + throw new MalformedResponseException( + $"Malformed options block at byte {blockStart}: {field} of {length} bytes runs past the end of " + + "the block."); + } + + var bytes = payload[cursor..(cursor + (int)length)].ToArray(); + cursor += (int)length; + return bytes; + } + internal static IReadOnlyList MapStreams(ReadOnlySpan payload) { List streams = new(); @@ -800,10 +897,11 @@ internal static SendMessagesResponse MapSendMessages(ReadOnlySpan payload) internal static StreamResponse MapStream(ReadOnlySpan payload) { var (stream, position) = MapToStream(payload, 0); - List topics = new(); - var length = payload.Length; - while (position < length) + // Count-driven: topic elements carry variable-length options blocks, + // so "consume until the buffer ends" no longer delimits them. + List topics = new(stream.TopicsCount); + for (var i = 0; i < stream.TopicsCount; i++) { var (topic, readBytes) = MapToTopic(payload, position); topics.Add(topic); @@ -818,7 +916,8 @@ internal static StreamResponse MapStream(ReadOnlySpan payload) Topics = topics, CreatedAt = stream.CreatedAt, MessagesCount = stream.MessagesCount, - Size = stream.Size + Size = stream.Size, + Options = stream.Options }; } @@ -833,6 +932,8 @@ private static (StreamResponse stream, int readBytes) MapToStream(ReadOnlySpan MapTopics(ReadOnlySpan payload) { - List topics = new(); - var length = payload.Length; - var position = 0; + var topicsCount = BinaryPrimitives.ReadUInt32LittleEndian(payload[..4]); + List topics = new((int)topicsCount); + var position = 4; - while (position < length) + for (var i = 0; i < topicsCount; i++) { var (topic, readBytes) = MapToTopic(payload, position); topics.Add(topic); @@ -885,9 +987,10 @@ internal static TopicResponse MapTopic(ReadOnlySpan payload) MessageExpiry = topic.MessageExpiry, MessagesCount = topic.MessagesCount, Size = topic.Size, - ReplicationFactor = topic.ReplicationFactor, MaxTopicSize = topic.MaxTopicSize, - Partitions = partitions + Partitions = partitions, + Options = topic.Options, + DerivedOptions = topic.DerivedOptions }; } @@ -899,12 +1002,15 @@ private static (TopicResponse topic, int readBytes) MapToTopic(ReadOnlySpan MapOptionSpecs(ReadOnlySpan payload) + { + var count = BinaryPrimitives.ReadUInt32LittleEndian(payload[..4]); + var position = 4; + var specs = new List(); + for (var i = 0; i < count; i++) + { + var keyLength = payload[position]; + position += 1; + EnsureFits(payload, position, keyLength, "option key"); + var key = Encoding.UTF8.GetString(payload[position..(position + keyLength)]); + position += keyLength; + + var kind = payload[position]; + position += 1; + + var defaultLength = (int)BinaryPrimitives.ReadUInt32LittleEndian(payload[position..(position + 4)]); + position += 4; + EnsureFits(payload, position, defaultLength, "option default value"); + var defaultValue = payload[position..(position + defaultLength)].ToArray(); + position += defaultLength; + + var descriptionLength = (int)BinaryPrimitives.ReadUInt32LittleEndian(payload[position..(position + 4)]); + position += 4; + EnsureFits(payload, position, descriptionLength, "option description"); + var description = Encoding.UTF8.GetString(payload[position..(position + descriptionLength)]); + position += descriptionLength; + + specs.Add(new OptionSpec + { + Key = key, + Kind = MapHeaderKind(kind), + DefaultValue = defaultValue, + Description = description + }); + } + + return specs; + } + + private static void EnsureFits(ReadOnlySpan payload, int position, int length, string what) + { + if (position + length > payload.Length) + { + throw new InvalidOperationException( + $"Malformed DescribeOptions response: {what} of {length} bytes at offset {position} " + + $"overruns the {payload.Length}-byte payload"); + } + } + internal static ClusterMetadata MapClusterMetadata(ReadOnlySpan payload) { var nameLength = BinaryPrimitives.ReadUInt32LittleEndian(payload[..4]); diff --git a/foreign/csharp/Iggy_SDK/Publishers/IggyPublisher.cs b/foreign/csharp/Iggy_SDK/Publishers/IggyPublisher.cs index 7ec63bebf7..ee2510bb44 100644 --- a/foreign/csharp/Iggy_SDK/Publishers/IggyPublisher.cs +++ b/foreign/csharp/Iggy_SDK/Publishers/IggyPublisher.cs @@ -250,14 +250,14 @@ private async Task CreateTopicIfNeededAsync(CancellationToken ct) if (Config.TopicId.Kind is IdKind.String) { await Client.CreateTopicAsync(Config.StreamId, Config.TopicId.GetString(), - Config.TopicPartitionsCount, Config.TopicCompressionAlgorithm, Config.TopicReplicationFactor, - Config.TopicMessageExpiry, Config.TopicMaxTopicSize, ct); + Config.TopicPartitionsCount, Config.TopicCompressionAlgorithm, + Config.TopicMessageExpiry, Config.TopicMaxTopicSize, token: ct); } else { await Client.CreateTopicAsync(Config.StreamId, Config.TopicName, Config.TopicPartitionsCount, - Config.TopicCompressionAlgorithm, Config.TopicReplicationFactor, - Config.TopicMessageExpiry, Config.TopicMaxTopicSize, ct); + Config.TopicCompressionAlgorithm, + Config.TopicMessageExpiry, Config.TopicMaxTopicSize, token: ct); } LogTopicCreated(Config.TopicId, Config.StreamId); diff --git a/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherBuilder.cs b/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherBuilder.cs index b69b7e99db..270bba9304 100644 --- a/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherBuilder.cs +++ b/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherBuilder.cs @@ -162,7 +162,6 @@ public IggyPublisherBuilder CreateStreamIfNotExists(string name) /// The name to use when creating the topic. /// The number of partitions for the topic. Default is 1. /// The compression algorithm to use for messages in the topic. Default is None. - /// The replication factor for the topic. Null means server default. /// /// The message expiry time. TimeSpan.Zero uses the server default, TimeSpan.MaxValue never /// expires. Default is TimeSpan.Zero. @@ -170,14 +169,13 @@ public IggyPublisherBuilder CreateStreamIfNotExists(string name) /// The maximum size of the topic in bytes (0 for unlimited). Default is 0. /// The builder instance for method chaining. public IggyPublisherBuilder CreateTopicIfNotExists(string name, uint topicPartitionsCount = 1, - CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, byte? replicationFactor = null, + CompressionAlgorithm compressionAlgorithm = CompressionAlgorithm.None, TimeSpan messageExpiry = default, ulong maxTopicSize = 0) { Config.CreateTopic = true; Config.TopicName = name; Config.TopicPartitionsCount = topicPartitionsCount; Config.TopicCompressionAlgorithm = compressionAlgorithm; - Config.TopicReplicationFactor = replicationFactor; Config.TopicMessageExpiry = messageExpiry; Config.TopicMaxTopicSize = maxTopicSize; diff --git a/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherConfig.cs b/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherConfig.cs index 5354069af1..a4ef3571f4 100644 --- a/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherConfig.cs +++ b/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherConfig.cs @@ -147,13 +147,6 @@ public class IggyPublisherConfig /// public CompressionAlgorithm TopicCompressionAlgorithm { get; set; } - /// - /// Gets or sets the replication factor for the topic. - /// Determines how many replicas of each partition are maintained. - /// Only used when is true. - /// - public byte? TopicReplicationFactor { get; set; } - /// /// Gets or sets the message expiry time (0 for server default, TimeSpan.MaxValue for no expiry). /// Messages older than this will be automatically deleted. diff --git a/foreign/csharp/Iggy_SDK/Utils/CommandCodes.cs b/foreign/csharp/Iggy_SDK/Utils/CommandCodes.cs index c7a3051e59..71e945a2ff 100644 --- a/foreign/csharp/Iggy_SDK/Utils/CommandCodes.cs +++ b/foreign/csharp/Iggy_SDK/Utils/CommandCodes.cs @@ -23,6 +23,7 @@ internal static class CommandCodes internal const int GET_STATS_CODE = 10; internal const int GET_SNAPSHOT_CODE = 11; internal const int GET_CLUSTER_METADATA_CODE = 12; + internal const int DESCRIBE_OPTIONS_CODE = 13; internal const int GET_ME_CODE = 20; internal const int GET_CLIENT_CODE = 21; internal const int GET_CLIENTS_CODE = 22; diff --git a/foreign/csharp/Iggy_SDK_Tests/ClientTests/HttpTopicOptionsTests.cs b/foreign/csharp/Iggy_SDK_Tests/ClientTests/HttpTopicOptionsTests.cs new file mode 100644 index 0000000000..a83cb8f2a1 --- /dev/null +++ b/foreign/csharp/Iggy_SDK_Tests/ClientTests/HttpTopicOptionsTests.cs @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Net; +using System.Text; +using Apache.Iggy.Contracts; +using Apache.Iggy.Headers; +using Apache.Iggy.IggyClient.Implementations; + +namespace Apache.Iggy.Tests.ClientTests; + +public sealed class HttpTopicOptionsTests +{ + private const string TopicResponseJson = """ + { + "id": 1, + "created_at": 1750000000000000, + "name": "topic", + "size": "0 B", + "message_expiry": 0, + "compression_algorithm": "none", + "max_topic_size": 0, + "messages_count": 0, + "partitions_count": 1, + "partitions": [], + "options": { + "enforce_fsync": { "value": "true", "explicit": true }, + "segment_size": { "value": "134217728", "explicit": false } + } + } + """; + + private static readonly Identifier StreamId = Identifier.Numeric(1); + + [Fact] + public async Task CreateTopic_SplitsTheResponseOptionsByProvenance() + { + var handler = new StubHandler(TopicResponseJson); + var client = new HttpMessageStream(new HttpClient(handler) { BaseAddress = new Uri("http://localhost") }); + + var topic = await client.CreateTopicAsync(StreamId, "topic", 1, + token: TestContext.Current.CancellationToken); + + Assert.NotNull(topic); + var explicitOption = Assert.Single(topic.Options!); + Assert.Equal("enforce_fsync", explicitOption.Key.AsString()); + Assert.Equal("true", explicitOption.Value.ToString()); + + var derivedOption = Assert.Single(topic.DerivedOptions!); + Assert.Equal("segment_size", derivedOption.Key.AsString()); + Assert.Equal("134217728", derivedOption.Value.ToString()); + } + + [Fact] + public async Task CreateTopic_SendsABoolOptionAsTheWordTheServerParses() + { + var handler = new StubHandler(TopicResponseJson); + var client = new HttpMessageStream(new HttpClient(handler) { BaseAddress = new Uri("http://localhost") }); + + await client.CreateTopicAsync(StreamId, "topic", 1, + options: new TopicOptions { EnforceFsync = true, SegmentSize = 134217728 }.ToDictionary(), + token: TestContext.Current.CancellationToken); + + Assert.Contains("\"enforce_fsync\":\"true\"", handler.RequestBody); + Assert.Contains("\"segment_size\":\"134217728\"", handler.RequestBody); + } + + [Fact] + public async Task GetTopicById_SplitsTheResponseOptionsByProvenance() + { + var handler = new StubHandler(TopicResponseJson); + var client = new HttpMessageStream(new HttpClient(handler) { BaseAddress = new Uri("http://localhost") }); + + var topic = await client.GetTopicByIdAsync(StreamId, Identifier.Numeric(1), + TestContext.Current.CancellationToken); + + Assert.Equal(HeaderKind.String, Assert.Single(topic!.Options!).Value.Kind); + Assert.Equal(HeaderKind.String, Assert.Single(topic.DerivedOptions!).Value.Kind); + } + + private sealed class StubHandler(string json) : HttpMessageHandler + { + internal string RequestBody { get; private set; } = string.Empty; + + protected override async Task SendAsync(HttpRequestMessage request, + CancellationToken ct) + { + if (request.Content is not null) + { + RequestBody = await request.Content.ReadAsStringAsync(ct); + } + + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(json, Encoding.UTF8, "application/json") + }; + } + } +} diff --git a/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs b/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs index e37aa35750..461556c3d4 100644 --- a/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs +++ b/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs @@ -17,12 +17,14 @@ using System.Buffers.Binary; using System.Security.Cryptography; +using System.Text; using Apache.Iggy.Contracts; using Apache.Iggy.Contracts.Auth; using Apache.Iggy.Encryption; using Apache.Iggy.Enums; using Apache.Iggy.Exceptions; using Apache.Iggy.Extensions; +using Apache.Iggy.Headers; using Apache.Iggy.IggyClient.Implementations; using Apache.Iggy.Shared; using Apache.Iggy.Tests.Utils; @@ -146,11 +148,13 @@ var payload2 public void MapStream_ReturnsValidStreamResponse() { // Arrange - var (id, topicsCount, sizeBytes, messagesCount, name, createdAt) = StreamFactory.CreateStreamsResponseFields(); + var (id, _, sizeBytes, messagesCount, name, createdAt) = StreamFactory.CreateStreamsResponseFields(); + // Topics are decoded count-driven, so the header count must match the appended topics. + var topicsCount = 1; var streamPayload = BinaryFactory.CreateStreamPayload(id, topicsCount, name, sizeBytes, messagesCount, createdAt); var (topicId1, partitionsCount1, topicName1, messageExpiry1, topicSizeBytes1, messagesCountTopic1, - createdAtTopic, replicationFactor, maxTopicSize) = + createdAtTopic, maxTopicSize) = TopicFactory.CreateTopicResponseFields(); var topicPayload1 = BinaryFactory.CreateTopicPayload(topicId1, partitionsCount1, @@ -159,7 +163,6 @@ var streamPayload topicSizeBytes1, messagesCountTopic1, createdAt, - replicationFactor, maxTopicSize, 1); @@ -196,19 +199,21 @@ public void MapTopics_ReturnsValidTopicsResponses() { // Arrange var (id1, partitionsCount1, name1, messageExpiry1, sizeBytesTopic1, messagesCountTopic1, createdAt, - replicationFactor1, maxTopicSize1) = + maxTopicSize1) = TopicFactory.CreateTopicResponseFields(); var payload1 = BinaryFactory.CreateTopicPayload(id1, partitionsCount1, messageExpiry1, name1, - sizeBytesTopic1, messagesCountTopic1, createdAt, replicationFactor1, maxTopicSize1, 1); + sizeBytesTopic1, messagesCountTopic1, createdAt, maxTopicSize1, 1); var (id2, partitionsCount2, name2, messageExpiry2, sizeBytesTopic2, messagesCountTopic2, createdAt2, - replicationFactor2, maxTopicSize2) = + maxTopicSize2) = TopicFactory.CreateTopicResponseFields(); var payload2 = BinaryFactory.CreateTopicPayload(id2, partitionsCount2, messageExpiry2, name2, - sizeBytesTopic2, messagesCountTopic2, createdAt2, replicationFactor2, maxTopicSize2, 2); + sizeBytesTopic2, messagesCountTopic2, createdAt2, maxTopicSize2, 2); - var combinedPayload = new byte[payload1.Length + payload2.Length]; - payload1.CopyTo(combinedPayload.AsSpan()); - payload2.CopyTo(combinedPayload.AsSpan(payload1.Length)); + // GetTopics replies start with the topics count. + var combinedPayload = new byte[4 + payload1.Length + payload2.Length]; + BinaryPrimitives.WriteUInt32LittleEndian(combinedPayload.AsSpan(0, 4), 2); + payload1.CopyTo(combinedPayload.AsSpan(4)); + payload2.CopyTo(combinedPayload.AsSpan(4 + payload1.Length)); // Act IReadOnlyList responses = Mappers.BinaryMapper.MapTopics(combinedPayload); @@ -238,10 +243,10 @@ public void MapTopics_ReturnsValidTopicsResponses() public void MapTopic_ReturnsValidTopicResponse() { // Arrange - var (topicId, partitionsCount, topicName, messageExpiry, sizeBytes, messagesCount, createdAt2, replicationFactor - , maxTopicSize) = TopicFactory.CreateTopicResponseFields(); + var (topicId, partitionsCount, topicName, messageExpiry, sizeBytes, messagesCount, createdAt2, + maxTopicSize) = TopicFactory.CreateTopicResponseFields(); var topicPayload = BinaryFactory.CreateTopicPayload(topicId, partitionsCount, messageExpiry, topicName, - sizeBytes, messagesCount, createdAt2, replicationFactor, maxTopicSize, 1); + sizeBytes, messagesCount, createdAt2, maxTopicSize, 1); var combinedPayload = new byte[topicPayload.Length]; topicPayload.CopyTo(combinedPayload.AsSpan()); @@ -259,6 +264,51 @@ public void MapTopic_ReturnsValidTopicResponse() Assert.Equal(CompressionAlgorithm.None, response.CompressionAlgorithm); } + [Fact] + public void MapOptionSpecs_ReturnsTheCatalogWithKindsAndDefaults() + { + // Arrange: [count][key_len][key][kind][default_len][default][description_len][description] + const string key = "segment_size"; + const string description = "Segment size in bytes"; + var defaultValue = new byte[8]; + BinaryPrimitives.WriteUInt64LittleEndian(defaultValue, 1024UL * 1024 * 1024); + + var payload = new List(); + payload.AddRange(BitConverter.GetBytes(1u)); + payload.Add((byte)key.Length); + payload.AddRange(Encoding.UTF8.GetBytes(key)); + payload.Add(12); // Uint64 wire code + payload.AddRange(BitConverter.GetBytes((uint)defaultValue.Length)); + payload.AddRange(defaultValue); + payload.AddRange(BitConverter.GetBytes((uint)description.Length)); + payload.AddRange(Encoding.UTF8.GetBytes(description)); + + // Act + var specs = Mappers.BinaryMapper.MapOptionSpecs(payload.ToArray()); + + // Assert + var spec = Assert.Single(specs); + Assert.Equal(key, spec.Key); + Assert.Equal(HeaderKind.Uint64, spec.Kind); + Assert.Equal(defaultValue, spec.DefaultValue); + Assert.Equal(description, spec.Description); + } + + [Fact] + public void MapOptionSpecs_RejectsAnEntryThatOverrunsThePayload() + { + // A declared length past the end must not read adjacent memory. + var payload = new List(); + payload.AddRange(BitConverter.GetBytes(1u)); + payload.Add(4); + payload.AddRange(Encoding.UTF8.GetBytes("size")); + payload.Add(12); + payload.AddRange(BitConverter.GetBytes(64u)); // claims 64 bytes that are not there + + Assert.Throws(() => + Mappers.BinaryMapper.MapOptionSpecs(payload.ToArray())); + } + [Fact] public void MapConsumerGroups_ReturnsValidConsumerGroupsResponses() { diff --git a/foreign/csharp/Iggy_SDK_Tests/MapperTests/ResourceOptionsConverterTests.cs b/foreign/csharp/Iggy_SDK_Tests/MapperTests/ResourceOptionsConverterTests.cs new file mode 100644 index 0000000000..3c8fb7cd5c --- /dev/null +++ b/foreign/csharp/Iggy_SDK_Tests/MapperTests/ResourceOptionsConverterTests.cs @@ -0,0 +1,187 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Text.Json; +using System.Text.Json.Serialization; +using Apache.Iggy.Contracts; +using Apache.Iggy.Headers; +using Apache.Iggy.JsonConverters; + +namespace Apache.Iggy.Tests.MapperTests; + +public sealed class ResourceOptionsConverterTests +{ + private static readonly JsonSerializerOptions HttpOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, + Converters = { new JsonStringEnumConverter(JsonNamingPolicy.SnakeCaseLower) } + }; + + private static readonly JsonSerializerOptions ConverterOptions = new() + { + Converters = { new ResourceOptionsConverter() } + }; + + [Fact] + public void DeserializeTopic_SplitsOptionsByProvenance() + { + // Arrange: the object a REST topic response renders both provenances into. + const string json = """ + { + "id": 1, + "created_at": 1750000000000000, + "name": "topic", + "size": "0 B", + "message_expiry": 0, + "compression_algorithm": "none", + "max_topic_size": 0, + "messages_count": 0, + "partitions_count": 1, + "partitions": [], + "options": { + "enforce_fsync": { "value": "true", "explicit": true }, + "segment_size": { "value": "134217728", "explicit": true }, + "messages_required_to_save": { "value": "1024", "explicit": false }, + "preallocate_segments": { "value": "false", "explicit": false } + } + } + """; + + // Act + var topic = JsonSerializer.Deserialize(json, HttpOptions); + + // Assert + Assert.NotNull(topic); + Assert.NotNull(topic.Options); + Assert.NotNull(topic.DerivedOptions); + + Assert.Equal(2, topic.Options.Count); + Assert.Equal("true", topic.Options[HeaderKey.FromString("enforce_fsync")].ToString()); + Assert.Equal("134217728", topic.Options[HeaderKey.FromString("segment_size")].ToString()); + + Assert.Equal(2, topic.DerivedOptions.Count); + Assert.Equal("1024", topic.DerivedOptions[HeaderKey.FromString("messages_required_to_save")].ToString()); + Assert.Equal("false", topic.DerivedOptions[HeaderKey.FromString("preallocate_segments")].ToString()); + } + + [Fact] + public void DeserializeTopic_ReadsEveryValueAsAString() + { + // A byte count arrives rendered, so its canonical Uint64 kind is not recoverable over REST. + const string json = """ + { + "id": 1, + "created_at": 1750000000000000, + "name": "topic", + "size": "0 B", + "max_topic_size": 0, + "messages_count": 0, + "partitions_count": 1, + "options": { "segment_size": { "value": "134217728", "explicit": true } } + } + """; + + var topic = JsonSerializer.Deserialize(json, HttpOptions); + + var value = Assert.Single(topic!.Options!).Value; + Assert.Equal(HeaderKind.String, value.Kind); + Assert.Throws(() => value.ToUInt64()); + } + + [Fact] + public void DeserializeTopic_WithoutOptionsLeavesBothDictionariesNull() + { + const string json = """ + { + "id": 1, + "created_at": 1750000000000000, + "name": "topic", + "size": "0 B", + "max_topic_size": 0, + "messages_count": 0, + "partitions_count": 1 + } + """; + + var topic = JsonSerializer.Deserialize(json, HttpOptions); + + Assert.Null(topic!.Options); + Assert.Null(topic.DerivedOptions); + } + + [Fact] + public void Read_EmptyObjectYieldsTwoEmptyDictionaries() + { + var options = JsonSerializer.Deserialize("{}", ConverterOptions); + + Assert.Empty(options.Explicit); + Assert.Empty(options.Derived); + } + + [Fact] + public void Read_RejectsAnEntryWithoutProvenance() + { + const string json = """{ "segment_size": { "value": "134217728" } }"""; + + Assert.Throws(() => JsonSerializer.Deserialize(json, ConverterOptions)); + } + + [Fact] + public void Read_RejectsAValueThatIsNotAnEntryObject() + { + const string json = """{ "segment_size": "134217728" }"""; + + Assert.Throws(() => JsonSerializer.Deserialize(json, ConverterOptions)); + } + + [Fact] + public void DeserializeTopics_SplitsOptionsOfEveryTopicInAList() + { + const string json = """ + [ + { + "id": 1, + "created_at": 1750000000000000, + "name": "first", + "size": "0 B", + "max_topic_size": 0, + "messages_count": 0, + "partitions_count": 1, + "options": { "enforce_fsync": { "value": "true", "explicit": true } } + }, + { + "id": 2, + "created_at": 1750000000000000, + "name": "second", + "size": "0 B", + "max_topic_size": 0, + "messages_count": 0, + "partitions_count": 1, + "options": { "enforce_fsync": { "value": "false", "explicit": false } } + } + ] + """; + + var topics = JsonSerializer.Deserialize>(json, HttpOptions); + + Assert.NotNull(topics); + Assert.Single(topics[0].Options!); + Assert.Empty(topics[0].DerivedOptions!); + Assert.Empty(topics[1].Options!); + Assert.Single(topics[1].DerivedOptions!); + } +} diff --git a/foreign/csharp/Iggy_SDK_Tests/PublisherTests/IggyTypedPublisherTests.cs b/foreign/csharp/Iggy_SDK_Tests/PublisherTests/IggyTypedPublisherTests.cs index 2537b2f3bb..755e6c56cc 100644 --- a/foreign/csharp/Iggy_SDK_Tests/PublisherTests/IggyTypedPublisherTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/PublisherTests/IggyTypedPublisherTests.cs @@ -345,8 +345,7 @@ private static TopicResponse TopicResponse() CreatedAt = default, MaxTopicSize = 0, MessagesCount = 0, - PartitionsCount = 1, - ReplicationFactor = 1 + PartitionsCount = 1 }; } diff --git a/foreign/csharp/Iggy_SDK_Tests/UtilityTests/TopicOptionsTests.cs b/foreign/csharp/Iggy_SDK_Tests/UtilityTests/TopicOptionsTests.cs new file mode 100644 index 0000000000..e1595c380b --- /dev/null +++ b/foreign/csharp/Iggy_SDK_Tests/UtilityTests/TopicOptionsTests.cs @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Apache.Iggy.Contracts; +using Apache.Iggy.Headers; + +namespace Apache.Iggy.Tests.UtilityTests; + +public sealed class TopicOptionsTests +{ + [Fact] + public void ToDictionary_EmitsEveryKeyUnderItsCatalogKind() + { + var options = new TopicOptions + { + SegmentSize = 134217728, + EnforceFsync = true, + MessagesRequiredToSave = 1024, + SizeOfMessagesRequiredToSave = 1048576, + PreallocateSegments = false + }.ToDictionary(); + + Assert.Equal(5, options.Count); + + Assert.Equal(HeaderKind.Uint64, options["segment_size"].Kind); + Assert.Equal(134217728UL, options["segment_size"].ToUInt64()); + + Assert.Equal(HeaderKind.Bool, options["enforce_fsync"].Kind); + Assert.True(options["enforce_fsync"].ToBool()); + + Assert.Equal(HeaderKind.Uint32, options["messages_required_to_save"].Kind); + Assert.Equal(1024U, options["messages_required_to_save"].ToUInt32()); + + Assert.Equal(HeaderKind.Uint64, options["size_of_messages_required_to_save"].Kind); + Assert.Equal(1048576UL, options["size_of_messages_required_to_save"].ToUInt64()); + + Assert.Equal(HeaderKind.Bool, options["preallocate_segments"].Kind); + Assert.False(options["preallocate_segments"].ToBool()); + } + + [Fact] + public void ToDictionary_EmitsOnlyTheKeysThatWereSet() + { + var options = new TopicOptions { EnforceFsync = false }.ToDictionary(); + + var entry = Assert.Single(options); + Assert.Equal("enforce_fsync", entry.Key); + Assert.False(entry.Value.ToBool()); + } + + [Fact] + public void ToDictionary_WithNothingSetIsEmpty() + { + Assert.Empty(new TopicOptions().ToDictionary()); + } +} diff --git a/foreign/csharp/Iggy_SDK_Tests/Utils/BinaryFactory.cs b/foreign/csharp/Iggy_SDK_Tests/Utils/BinaryFactory.cs index 78e5c595e6..9dd385a369 100644 --- a/foreign/csharp/Iggy_SDK_Tests/Utils/BinaryFactory.cs +++ b/foreign/csharp/Iggy_SDK_Tests/Utils/BinaryFactory.cs @@ -67,7 +67,7 @@ internal static byte[] CreateStreamPayload(uint id, int topicsCount, string name ulong messagesCount, ulong createdAt) { var nameBytes = Encoding.UTF8.GetBytes(name); - var totalSize = 4 + 4 + 8 + 8 + 1 + 8 + nameBytes.Length; + var totalSize = 4 + 4 + 8 + 8 + 1 + 8 + nameBytes.Length + 4; var payload = new byte[totalSize]; BinaryPrimitives.WriteUInt32LittleEndian(payload, id); BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(4), createdAt); @@ -76,15 +76,16 @@ internal static byte[] CreateStreamPayload(uint id, int topicsCount, string name BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(24), messagesCount); payload[32] = (byte)nameBytes.Length; nameBytes.CopyTo(payload.AsSpan(33)); + // Empty length-prefixed options block. + BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(33 + nameBytes.Length), 0); return payload; } internal static byte[] CreateTopicPayload(uint id, uint partitionsCount, uint messageExpiry, string name, - ulong sizeBytes, ulong messagesCount, ulong createdAt, byte replicationFactor, ulong maxTopicSize, - int compressionType) + ulong sizeBytes, ulong messagesCount, ulong createdAt, ulong maxTopicSize, int compressionType) { var nameBytes = Encoding.UTF8.GetBytes(name); - var totalSize = 4 + 8 + 4 + 8 + 1 + 8 + 8 + 8 + 1 + 1 + name.Length; + var totalSize = 4 + 8 + 4 + 8 + 1 + 8 + 8 + 8 + 1 + nameBytes.Length + 4 + 4; var payload = new byte[totalSize]; BinaryPrimitives.WriteUInt32LittleEndian(payload, id); @@ -93,11 +94,13 @@ internal static byte[] CreateTopicPayload(uint id, uint partitionsCount, uint me BinaryPrimitives.WriteInt64LittleEndian(payload.AsSpan(16), messageExpiry); payload[24] = (byte)compressionType; BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(25), maxTopicSize); - payload[33] = replicationFactor; - BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(34), sizeBytes); - BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(42), messagesCount); - payload[50] = (byte)nameBytes.Length; - nameBytes.CopyTo(payload.AsSpan(51)); + BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(33), sizeBytes); + BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(41), messagesCount); + payload[49] = (byte)nameBytes.Length; + nameBytes.CopyTo(payload.AsSpan(50)); + // Empty length-prefixed explicit and derived options blocks. + BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(50 + nameBytes.Length), 0); + BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(54 + nameBytes.Length), 0); return payload; } diff --git a/foreign/csharp/Iggy_SDK_Tests/Utils/Topics/TopicFactory.cs b/foreign/csharp/Iggy_SDK_Tests/Utils/Topics/TopicFactory.cs index 32d6a537e9..914f3b1680 100644 --- a/foreign/csharp/Iggy_SDK_Tests/Utils/Topics/TopicFactory.cs +++ b/foreign/csharp/Iggy_SDK_Tests/Utils/Topics/TopicFactory.cs @@ -23,7 +23,7 @@ namespace Apache.Iggy.Tests.Utils.Topics; internal static class TopicFactory { internal static (uint topicId, uint partitionsCount, string topicName, uint messageExpriy, ulong sizeBytes, ulong - messagesCount, ulong createdAt, byte replicationFactor, ulong maxTopicSize) + messagesCount, ulong createdAt, ulong maxTopicSize) CreateTopicResponseFields() { var topicId = (uint)Random.Shared.Next(1, 69); @@ -34,8 +34,7 @@ internal static (uint topicId, uint partitionsCount, string topicName, uint mess var messagesCount = (ulong)Random.Shared.Next(69, 42069); var createdAt = (ulong)Random.Shared.Next(69, 42069); var maxTopicSize = (ulong)Random.Shared.NextInt64(2_000_000_000, 10_000_000_000); - var replicationFactor = (byte)Random.Shared.Next(1, 8); return (topicId, partitionsCount, topicName, messageExpiry, sizeBytes, messagesCount, createdAt, - replicationFactor, maxTopicSize); + maxTopicSize); } } diff --git a/foreign/csharp/README.md b/foreign/csharp/README.md index 508990e391..6f60264b44 100644 --- a/foreign/csharp/README.md +++ b/foreign/csharp/README.md @@ -284,7 +284,6 @@ await client.CreateTopicAsync( name: "my-topic", partitionsCount: 3, compressionAlgorithm: CompressionAlgorithm.None, - replicationFactor: 1, messageExpiry: 0, // 0 = never expire maxTopicSize: 0 // 0 = unlimited ); diff --git a/foreign/go/benchmarks/send_messages_benchmark_test.go b/foreign/go/benchmarks/send_messages_benchmark_test.go index 0e5761dfba..20322d5893 100644 --- a/foreign/go/benchmarks/send_messages_benchmark_test.go +++ b/foreign/go/benchmarks/send_messages_benchmark_test.go @@ -126,7 +126,6 @@ func ensureInfrastructureIsInitialized(cli iggcon.Client, streamId uint32) error iggcon.CompressionAlgorithmNone, iggcon.IggyExpiryServerDefault, 1, - nil, ) if topicErr != nil { diff --git a/foreign/go/binary_serialization/binary_response_deserializer.go b/foreign/go/binary_serialization/binary_response_deserializer.go index 35cf03ac90..bf50e86803 100644 --- a/foreign/go/binary_serialization/binary_response_deserializer.go +++ b/foreign/go/binary_serialization/binary_response_deserializer.go @@ -57,8 +57,12 @@ func DeserializeStream(payload []byte) (*iggcon.StreamDetails, error) { if err != nil { return nil, err } - topics := make([]iggcon.Topic, 0) - for pos < len(payload) { + // Count-driven: a topic element carries variable-length options blocks, + // so "consume until the buffer ends" no longer delimits it. The declared + // count is an unvalidated wire u32, so the allocation hint is capped by + // what the remaining body could possibly hold. + topics := make([]iggcon.Topic, 0, boundedCapacity(stream.TopicsCount, len(payload)-pos, topicMinimumSize)) + for i := uint32(0); i < stream.TopicsCount; i++ { topic, readBytes, err := DeserializeToTopic(payload, pos) if err != nil { return nil, err @@ -110,15 +114,19 @@ func DeserializeToStream(payload []byte, position int) (iggcon.Stream, int, erro messagesCount := binary.LittleEndian.Uint64(payload[position+24 : position+32]) nameLength := int(payload[position+32]) - totalSize := streamFixedSize + nameLength - if remaining < totalSize { + if remaining < streamFixedSize+nameLength { return iggcon.Stream{}, 0, fmt.Errorf( "not enough data to read stream name: need %d bytes, got %d", - totalSize, remaining) + streamFixedSize+nameLength, remaining) } name := string(payload[position+33 : position+33+nameLength]) + options, optionsSize, err := deserializeOptions(payload, position+streamFixedSize+nameLength) + if err != nil { + return iggcon.Stream{}, 0, fmt.Errorf("failed to read stream options: %w", err) + } + return iggcon.Stream{ Id: id, TopicsCount: topicsCount, @@ -126,7 +134,36 @@ func DeserializeToStream(payload []byte, position int) (iggcon.Stream, int, erro SizeBytes: sizeBytes, MessagesCount: messagesCount, CreatedAt: createdAt, - }, totalSize, nil + Options: options, + }, streamFixedSize + nameLength + optionsSize, nil +} + +// deserializeOptions reads a u32-length-prefixed options TLV block at +// position and returns the decoded options with the total bytes consumed +// (prefix included). A zero-length block decodes to a nil map. +func deserializeOptions(payload []byte, position int) (map[string]iggcon.HeaderValue, int, error) { + if len(payload) < position+4 { + return nil, 0, errors.New("not enough data to read options length") + } + optionsLength := int(binary.LittleEndian.Uint32(payload[position : position+4])) + if len(payload) < position+4+optionsLength { + return nil, 0, fmt.Errorf( + "not enough data to read options block: need %d bytes, got %d", + optionsLength, len(payload)-position-4) + } + if optionsLength == 0 { + return nil, 4, nil + } + + entries, err := iggcon.DeserializeHeaders(payload[position+4 : position+4+optionsLength]) + if err != nil { + return nil, 0, err + } + options := make(map[string]iggcon.HeaderValue, len(entries)) + for _, entry := range entries { + options[string(entry.Key.Value)] = entry.Value + } + return options, 4 + optionsLength, nil } // pollBatchHeaderLength covers [partition_id u32][current_offset u64][count u32]. @@ -219,12 +256,86 @@ func DeserializeFetchMessagesResponse(payload []byte, compression iggcon.IggyMes }, nil } +// optionSpecMinimumSize is the smallest catalog entry: a one-character name +// (a length byte plus at least one byte), a kind byte, and empty +// length-prefixed default and description. +const optionSpecMinimumSize = 2 + 1 + 4 + 4 + +// DeserializeOptionSpecs reads a DescribeOptions response. +// +// Wire format: [count:u32][ [key_len:u8][key][kind:u8][default_len:u32][default] +// [description_len:u32][description] ]* +func DeserializeOptionSpecs(payload []byte) ([]iggcon.OptionSpec, error) { + if len(payload) < 4 { + return nil, fmt.Errorf( + "not enough data to read option count: need 4 bytes, got %d", len(payload)) + } + count := binary.LittleEndian.Uint32(payload[0:4]) + position := 4 + + specs := make([]iggcon.OptionSpec, 0, + boundedCapacity(count, len(payload)-position, optionSpecMinimumSize)) + for i := uint32(0); i < count; i++ { + if len(payload)-position < 1 { + return nil, fmt.Errorf("truncated option key length at offset %d", position) + } + keyLength := int(payload[position]) + position++ + if len(payload)-position < keyLength+1 { + return nil, fmt.Errorf("truncated option key at offset %d", position) + } + key := string(payload[position : position+keyLength]) + position += keyLength + + kind := payload[position] + position++ + + defaultValue, read, err := readLengthPrefixed(payload, position, "option default value") + if err != nil { + return nil, err + } + position += read + + description, read, err := readLengthPrefixed(payload, position, "option description") + if err != nil { + return nil, err + } + position += read + + specs = append(specs, iggcon.OptionSpec{ + Key: key, + DefaultValue: iggcon.HeaderValue{Kind: iggcon.HeaderKind(kind), Value: defaultValue}, + Description: string(description), + }) + } + + return specs, nil +} + +func readLengthPrefixed(payload []byte, position int, field string) ([]byte, int, error) { + if len(payload)-position < 4 { + return nil, 0, fmt.Errorf("truncated length prefix for %s at offset %d", field, position) + } + length := int(binary.LittleEndian.Uint32(payload[position : position+4])) + position += 4 + if length < 0 || len(payload)-position < length { + return nil, 0, fmt.Errorf("truncated %s at offset %d", field, position) + } + return payload[position : position+length], 4 + length, nil +} + func DeserializeTopics(payload []byte) ([]iggcon.Topic, error) { - topics := make([]iggcon.Topic, 0) - length := len(payload) - position := 0 + if len(payload) < 4 { + return nil, fmt.Errorf( + "not enough data to read topics count: need 4 bytes, got %d", len(payload)) + } + topicsCount := binary.LittleEndian.Uint32(payload[0:4]) + position := 4 - for position < length { + // The declared count is server-controlled; the allocation hint is capped + // by what the body could possibly hold. + topics := make([]iggcon.Topic, 0, boundedCapacity(topicsCount, len(payload)-position, topicMinimumSize)) + for i := uint32(0); i < topicsCount; i++ { topic, readBytes, err := DeserializeToTopic(payload, position) if err != nil { return nil, err @@ -256,7 +367,40 @@ func DeserializeTopic(payload []byte) (*iggcon.TopicDetails, error) { }, nil } +// topicFixedSize covers the fields before the name: +// id + created_at + partitions_count + message_expiry + compression + +// max_topic_size + size_bytes + messages_count + name_len. +const topicFixedSize = 4 + 8 + 4 + 8 + 1 + 8 + 8 + 8 + 1 // 50 bytes + +// topicMinimumSize is the smallest possible topic element: fixed fields, a +// one-character name (the server rejects an empty one), and the two u32 +// options-length prefixes (explicit and derived), both zero. +const topicMinimumSize = topicFixedSize + 1 + 4 + 4 + +// boundedCapacity caps a wire-declared element count by what the remaining +// bytes could possibly hold. +// +// The count is an unvalidated u32: at max it asks for a multi-hundred-gigabyte +// reservation, and a Go allocation failure cannot be recovered from. +func boundedCapacity(declared uint32, remaining int, minItemSize int) int { + if remaining <= 0 || minItemSize <= 0 { + return 0 + } + capacity := remaining / minItemSize + if uint64(declared) < uint64(capacity) { + return int(declared) + } + return capacity +} + func DeserializeToTopic(payload []byte, position int) (iggcon.Topic, int, error) { + remaining := len(payload) - position + if remaining < topicFixedSize { + return iggcon.Topic{}, 0, fmt.Errorf( + "not enough data to read topic header: need %d bytes, got %d", + topicFixedSize, remaining) + } + topic := iggcon.Topic{} topic.Id = binary.LittleEndian.Uint32(payload[position : position+4]) topic.CreatedAt = binary.LittleEndian.Uint64(payload[position+4 : position+12]) @@ -264,14 +408,34 @@ func DeserializeToTopic(payload []byte, position int) (iggcon.Topic, int, error) topic.MessageExpiry = iggcon.Duration(binary.LittleEndian.Uint64(payload[position+16 : position+24])) topic.CompressionAlgorithm = payload[position+24] topic.MaxTopicSize = binary.LittleEndian.Uint64(payload[position+25 : position+33]) - topic.ReplicationFactor = payload[position+33] - topic.Size = binary.LittleEndian.Uint64(payload[position+34 : position+42]) - topic.MessagesCount = binary.LittleEndian.Uint64(payload[position+42 : position+50]) + topic.Size = binary.LittleEndian.Uint64(payload[position+33 : position+41]) + topic.MessagesCount = binary.LittleEndian.Uint64(payload[position+41 : position+49]) + // Replication factor left the wire protocol together with the old fixed + // layout; every topic reports the single-copy default. + + nameLength := int(payload[position+49]) + if remaining < topicFixedSize+nameLength { + return iggcon.Topic{}, 0, fmt.Errorf( + "not enough data to read topic name: need %d bytes, got %d", + topicFixedSize+nameLength, remaining) + } + topic.Name = string(payload[position+50 : position+50+nameLength]) + + readBytes := topicFixedSize + nameLength + options, optionsSize, err := deserializeOptions(payload, position+readBytes) + if err != nil { + return iggcon.Topic{}, 0, fmt.Errorf("failed to read topic options: %w", err) + } + topic.Options = options + readBytes += optionsSize - nameLength := int(payload[position+50]) - topic.Name = string(payload[position+51 : position+51+nameLength]) + derivedOptions, derivedSize, err := deserializeOptions(payload, position+readBytes) + if err != nil { + return iggcon.Topic{}, 0, fmt.Errorf("failed to read topic derived options: %w", err) + } + topic.DerivedOptions = derivedOptions + readBytes += derivedSize - readBytes := 4 + 8 + 4 + 8 + 8 + 8 + 8 + 1 + 1 + 1 + nameLength return topic, readBytes, nil } @@ -395,6 +559,7 @@ func DeserializeUser(payload []byte) (*iggcon.UserInfoDetails, error) { CreatedAt: response.CreatedAt, Username: response.Username, Status: response.Status, + Options: response.Options, } if hasPermissions == 1 { permissionLength := binary.LittleEndian.Uint32(payload[position+1 : position+5]) @@ -521,12 +686,18 @@ func deserializeToUser(payload []byte, position int) (*iggcon.UserInfo, int, err username := string(payload[position+14 : position+14+int(usernameLength)]) readBytes := 4 + 8 + 1 + 1 + int(usernameLength) + options, optionsSize, err := deserializeOptions(payload, position+readBytes) + if err != nil { + return nil, 0, fmt.Errorf("failed to read user options: %w", err) + } + readBytes += optionsSize return &iggcon.UserInfo{ Id: id, CreatedAt: createdAt, Status: userStatus, Username: username, + Options: options, }, readBytes, nil } diff --git a/foreign/go/binary_serialization/binary_response_deserializer_test.go b/foreign/go/binary_serialization/binary_response_deserializer_test.go index 9165c9946a..010a64ef11 100644 --- a/foreign/go/binary_serialization/binary_response_deserializer_test.go +++ b/foreign/go/binary_serialization/binary_response_deserializer_test.go @@ -89,8 +89,9 @@ func TestDeserializeFetchMessages_EmptyPayload(t *testing.T) { } func encodeStream(id uint32, createdAt uint64, topicsCount uint32, sizeBytes, messagesCount uint64, name string) []byte { + // Trailing 4 zero bytes: the u32 length prefix of an empty options block. nameBytes := []byte(name) - buf := make([]byte, streamFixedSize+len(nameBytes)) + buf := make([]byte, streamFixedSize+len(nameBytes)+4) binary.LittleEndian.PutUint32(buf[0:4], id) binary.LittleEndian.PutUint64(buf[4:12], createdAt) binary.LittleEndian.PutUint32(buf[12:16], topicsCount) @@ -153,6 +154,41 @@ func TestDeserializeToStream_TruncatedName(t *testing.T) { } } +func TestDeserializeToStream_TruncatedOptions(t *testing.T) { + full := encodeStream(1, 100, 0, 0, 0, "s") + for cut := 1; cut <= 4; cut++ { + _, _, err := DeserializeToStream(full[:len(full)-cut], 0) + if err == nil { + t.Fatalf("expected error for options prefix cut by %d bytes, got nil", cut) + } + } +} + +func TestDeserializeToStream_WithOptions(t *testing.T) { + options := iggcon.GetHeadersBytes([]iggcon.HeaderEntry{{ + Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte("future_key")}, + Value: iggcon.HeaderValue{Kind: iggcon.String, Value: []byte("value")}, + }}) + payload := encodeStream(7, 500, 0, 0, 0, "with-options") + binary.LittleEndian.PutUint32(payload[len(payload)-4:], uint32(len(options))) + payload = append(payload, options...) + + stream, readBytes, err := DeserializeToStream(payload, 0) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if readBytes != len(payload) { + t.Fatalf("readBytes = %d, want %d", readBytes, len(payload)) + } + value, ok := stream.Options["future_key"] + if !ok { + t.Fatalf("Options = %v, want key %q", stream.Options, "future_key") + } + if value.Kind != iggcon.String || string(value.Value) != "value" { + t.Errorf("Options[future_key] = %+v, want String %q", value, "value") + } +} + func TestDeserializeStreams_Empty(t *testing.T) { streams, err := DeserializeStreams([]byte{}) if err != nil { @@ -246,3 +282,35 @@ func TestDeserializeStreams_MaxLengthName(t *testing.T) { t.Errorf("Name length = %d, want 255", len(streams[0].Name)) } } + +func TestBoundedCapacity_CapsDeclaredCount(t *testing.T) { + tests := []struct { + name string + declared uint32 + remaining int + minItemSize int + want int + }{ + {"declared below what fits", 2, 10 * topicMinimumSize, topicMinimumSize, 2}, + {"declared above what fits", 1 << 31, 3 * topicMinimumSize, topicMinimumSize, 3}, + {"nothing left to read", 1 << 31, 0, topicMinimumSize, 0}, + {"zero item size", 1 << 31, 1024, 0, 0}, + } + for _, test := range tests { + got := boundedCapacity(test.declared, test.remaining, test.minItemSize) + if got != test.want { + t.Errorf("%s: boundedCapacity(%d, %d, %d) = %d, want %d", + test.name, test.declared, test.remaining, test.minItemSize, got, test.want) + } + } +} + +func TestDeserializeStream_LyingTopicsCountDoesNotPreallocate(t *testing.T) { + // A u32 topics count with no topics behind it: the decode must fail on the + // missing element rather than reserve a slice for the declared count. + payload := encodeStream(1, 100, 1<<31, 0, 0, "s") + + if _, err := DeserializeStream(payload); err == nil { + t.Fatal("expected error for topics count with no elements, got nil") + } +} diff --git a/foreign/go/binary_serialization/vsr_response_deserializer_test.go b/foreign/go/binary_serialization/vsr_response_deserializer_test.go index 7a43ab14ce..e2b4df14bc 100644 --- a/foreign/go/binary_serialization/vsr_response_deserializer_test.go +++ b/foreign/go/binary_serialization/vsr_response_deserializer_test.go @@ -241,20 +241,34 @@ func TestDeserializeFetchMessagesResponse_RejectsOverrunningUserHeaders(t *testi } func TestDeserializeToTopic_PinsTheFieldOrderOfTheWireLayout(t *testing.T) { - const nameOffset = 50 + const nameLenOffset = 49 name := "orders" - payload := make([]byte, nameOffset+1+len(name)) + + partitionsValue := binary.LittleEndian.AppendUint32(nil, 3) + explicitOptions := iggcon.GetHeadersBytes([]iggcon.HeaderEntry{{ + Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte("partitions_count")}, + Value: iggcon.HeaderValue{Kind: iggcon.Uint32, Value: partitionsValue}, + }}) + derivedOptions := iggcon.GetHeadersBytes([]iggcon.HeaderEntry{{ + Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte("compression_algorithm")}, + Value: iggcon.HeaderValue{Kind: iggcon.String, Value: []byte("none")}, + }}) + + payload := make([]byte, nameLenOffset+1+len(name)) binary.LittleEndian.PutUint32(payload[0:4], 11) binary.LittleEndian.PutUint64(payload[4:12], 1700000000) binary.LittleEndian.PutUint32(payload[12:16], 3) binary.LittleEndian.PutUint64(payload[16:24], 60_000_000) payload[24] = 2 binary.LittleEndian.PutUint64(payload[25:33], 1<<30) - payload[33] = 1 - binary.LittleEndian.PutUint64(payload[34:42], 4096) - binary.LittleEndian.PutUint64(payload[42:50], 12) - payload[nameOffset] = byte(len(name)) - copy(payload[nameOffset+1:], name) + binary.LittleEndian.PutUint64(payload[33:41], 4096) + binary.LittleEndian.PutUint64(payload[41:49], 12) + payload[nameLenOffset] = byte(len(name)) + copy(payload[nameLenOffset+1:], name) + payload = binary.LittleEndian.AppendUint32(payload, uint32(len(explicitOptions))) + payload = append(payload, explicitOptions...) + payload = binary.LittleEndian.AppendUint32(payload, uint32(len(derivedOptions))) + payload = append(payload, derivedOptions...) topic, readBytes, err := DeserializeToTopic(payload, 0) require.NoError(t, err) @@ -267,9 +281,39 @@ func TestDeserializeToTopic_PinsTheFieldOrderOfTheWireLayout(t *testing.T) { assert.Equal(t, uint8(2), topic.CompressionAlgorithm, "compression is the u8 at +24") assert.Equal(t, uint64(1<<30), topic.MaxTopicSize) - assert.Equal(t, uint8(1), topic.ReplicationFactor) assert.Equal(t, uint64(4096), topic.Size) assert.Equal(t, uint64(12), topic.MessagesCount) assert.Equal(t, name, topic.Name) + assert.Equal(t, map[string]iggcon.HeaderValue{ + "partitions_count": {Kind: iggcon.Uint32, Value: partitionsValue}, + }, topic.Options) + assert.Equal(t, map[string]iggcon.HeaderValue{ + "compression_algorithm": {Kind: iggcon.String, Value: []byte("none")}, + }, topic.DerivedOptions) assert.Equal(t, len(payload), readBytes) } + +func TestDeserializeToTopic_RejectsEveryTruncationOfTheOptionsBlocks(t *testing.T) { + const nameLenOffset = 49 + name := "t" + fixed := make([]byte, nameLenOffset+1+len(name)) + fixed[nameLenOffset] = byte(len(name)) + copy(fixed[nameLenOffset+1:], name) + + options := iggcon.GetHeadersBytes([]iggcon.HeaderEntry{{ + Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte("max_topic_size")}, + Value: iggcon.HeaderValue{Kind: iggcon.Uint64, Value: binary.LittleEndian.AppendUint64(nil, 1<<30)}, + }}) + payload := binary.LittleEndian.AppendUint32(fixed, uint32(len(options))) + payload = append(payload, options...) + payload = binary.LittleEndian.AppendUint32(payload, 0) + + _, readBytes, err := DeserializeToTopic(payload, 0) + require.NoError(t, err) + require.Equal(t, len(payload), readBytes) + + for i := len(fixed); i < len(payload); i++ { + _, _, err := DeserializeToTopic(payload[:i], 0) + assert.Error(t, err, "expected error for truncation at byte %d", i) + } +} diff --git a/foreign/go/client/tcp/tcp_core_test.go b/foreign/go/client/tcp/tcp_core_test.go index 1ae9a438fa..ffb63fdbc2 100644 --- a/foreign/go/client/tcp/tcp_core_test.go +++ b/foreign/go/client/tcp/tcp_core_test.go @@ -553,16 +553,18 @@ func TestCanReplay_RefusesOnlyReplicatedRequestsWithAnUnknownOutcome(t *testing. } // topicDetailsBody builds the reply body of GetTopic for a topic with the -// given partition count and no partitions listed. +// given partition count and no partitions listed. The trailing 8 zero bytes +// are the u32 length prefixes of the empty explicit and derived options +// blocks. func topicDetailsBody(t *testing.T, partitionsCount uint32) []byte { t.Helper() - const nameOffset = 50 + const nameLenOffset = 49 name := "orders" - body := make([]byte, nameOffset+1+len(name)) + body := make([]byte, nameLenOffset+1+len(name)+4+4) binary.LittleEndian.PutUint32(body[0:4], 1) binary.LittleEndian.PutUint32(body[12:16], partitionsCount) - body[nameOffset] = byte(len(name)) - copy(body[nameOffset+1:], name) + body[nameLenOffset] = byte(len(name)) + copy(body[nameLenOffset+1:], name) return body } diff --git a/foreign/go/client/tcp/tcp_stream_management_test.go b/foreign/go/client/tcp/tcp_stream_management_test.go index 949860a717..f46033b89d 100644 --- a/foreign/go/client/tcp/tcp_stream_management_test.go +++ b/foreign/go/client/tcp/tcp_stream_management_test.go @@ -30,7 +30,8 @@ import ( ) // streamDetailsBody builds a stream reply with no topics: -// [id u32][created_at u64][topics u32][size u64][messages u64][name_len u8][name]. +// [id u32][created_at u64][topics u32][size u64][messages u64][name_len u8][name] +// [options_len u32 = 0]. func streamDetailsBody(id uint32, name string) []byte { body := binary.LittleEndian.AppendUint32(nil, id) body = binary.LittleEndian.AppendUint64(body, 1700000000) @@ -38,7 +39,8 @@ func streamDetailsBody(id uint32, name string) []byte { body = binary.LittleEndian.AppendUint64(body, 0) body = binary.LittleEndian.AppendUint64(body, 0) body = append(body, byte(len(name))) - return append(body, name...) + body = append(body, name...) + return binary.LittleEndian.AppendUint32(body, 0) } func TestCreateStream_DecodesTheCreatedStream(t *testing.T) { diff --git a/foreign/go/client/tcp/tcp_topic_management.go b/foreign/go/client/tcp/tcp_topic_management.go index bc98fd8524..f2d678b3b6 100644 --- a/foreign/go/client/tcp/tcp_topic_management.go +++ b/foreign/go/client/tcp/tcp_topic_management.go @@ -60,7 +60,7 @@ func (c *IggyTcpClient) CreateTopic( compressionAlgorithm iggcon.CompressionAlgorithm, messageExpiry iggcon.Duration, maxTopicSize uint64, - replicationFactor *uint8, + options ...iggcon.HeaderEntry, ) (*iggcon.TopicDetails, error) { if len(name) == 0 || len(name) > MaxStringLength { return nil, ierror.ErrInvalidTopicName @@ -68,10 +68,6 @@ func (c *IggyTcpClient) CreateTopic( if partitionsCount > MaxPartitionCount { return nil, ierror.ErrTooManyPartitions } - if replicationFactor != nil && *replicationFactor == 0 { - return nil, ierror.ErrInvalidReplicationFactor - } - buffer, err := c.do(ctx, &command.CreateTopic{ StreamId: streamId, Name: name, @@ -79,7 +75,7 @@ func (c *IggyTcpClient) CreateTopic( CompressionAlgorithm: compressionAlgorithm, MessageExpiry: messageExpiry, MaxTopicSize: maxTopicSize, - ReplicationFactor: replicationFactor, + Options: options, }) if err != nil { return nil, err @@ -96,22 +92,19 @@ func (c *IggyTcpClient) UpdateTopic( compressionAlgorithm iggcon.CompressionAlgorithm, messageExpiry iggcon.Duration, maxTopicSize uint64, - replicationFactor *uint8, + options ...iggcon.HeaderEntry, ) error { if len(name) == 0 || len(name) > MaxStringLength { return ierror.ErrInvalidTopicName } - if replicationFactor != nil && *replicationFactor == 0 { - return ierror.ErrInvalidReplicationFactor - } _, err := c.do(ctx, &command.UpdateTopic{ StreamId: streamId, TopicId: topicId, CompressionAlgorithm: compressionAlgorithm, MessageExpiry: messageExpiry, MaxTopicSize: maxTopicSize, - ReplicationFactor: replicationFactor, Name: name, + Options: options, }) return err } diff --git a/foreign/go/client/tcp/tcp_topic_management_test.go b/foreign/go/client/tcp/tcp_topic_management_test.go index b9c0700b8f..93a531c234 100644 --- a/foreign/go/client/tcp/tcp_topic_management_test.go +++ b/foreign/go/client/tcp/tcp_topic_management_test.go @@ -19,6 +19,7 @@ package tcp import ( "context" + "encoding/binary" "strings" "testing" @@ -37,7 +38,7 @@ func TestCreateTopic_DecodesTheCreatedTopic(t *testing.T) { }) topic, err := client.CreateTopic(context.Background(), - numericIdentifier(t, 1), "orders", 3, 0, iggcon.Duration(0), 0, nil) + numericIdentifier(t, 1), "orders", 3, 0, iggcon.Duration(0), 0) require.NoError(t, err) assert.Equal(t, uint32(3), topic.PartitionsCount) assert.Equal(t, "orders", topic.Name) @@ -50,13 +51,11 @@ func TestCreateTopic_RejectsInvalidArgumentsWithoutWriting(t *testing.T) { }) streamId := numericIdentifier(t, 1) - zeroReplication := uint8(0) tests := []struct { - name string - topicName string - partitions uint32 - replication *uint8 - want error + name string + topicName string + partitions uint32 + want error }{ {name: "empty name", topicName: "", partitions: 1, want: ierror.ErrInvalidTopicName}, @@ -64,13 +63,11 @@ func TestCreateTopic_RejectsInvalidArgumentsWithoutWriting(t *testing.T) { partitions: 1, want: ierror.ErrInvalidTopicName}, {name: "too many partitions", topicName: "orders", partitions: MaxPartitionCount + 1, want: ierror.ErrTooManyPartitions}, - {name: "zero replication factor", topicName: "orders", partitions: 1, - replication: &zeroReplication, want: ierror.ErrInvalidReplicationFactor}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { _, err := client.CreateTopic(context.Background(), streamId, - test.topicName, test.partitions, 0, iggcon.Duration(0), 0, test.replication) + test.topicName, test.partitions, 0, iggcon.Duration(0), 0) assert.ErrorIs(t, err, test.want) }) } @@ -85,14 +82,10 @@ func TestUpdateTopic_RejectsInvalidArgumentsWithoutWriting(t *testing.T) { streamId := numericIdentifier(t, 1) topicId := numericIdentifier(t, 2) - zeroReplication := uint8(0) err := client.UpdateTopic(context.Background(), streamId, topicId, - "", 0, iggcon.Duration(0), 0, nil) + "", 0, iggcon.Duration(0), 0) assert.ErrorIs(t, err, ierror.ErrInvalidTopicName) - err = client.UpdateTopic(context.Background(), streamId, topicId, - "orders", 0, iggcon.Duration(0), 0, &zeroReplication) - assert.ErrorIs(t, err, ierror.ErrInvalidReplicationFactor) assert.Empty(t, server.recorded()) } @@ -104,12 +97,14 @@ func TestUpdateTopic_AcceptsACommittedUpdate(t *testing.T) { assert.NoError(t, client.UpdateTopic(context.Background(), numericIdentifier(t, 1), numericIdentifier(t, 2), - "renamed", 0, iggcon.Duration(0), 0, nil)) + "renamed", 0, iggcon.Duration(0), 0)) } func TestGetTopics_DecodesTheRoster(t *testing.T) { client, serverConn := newPipeClient(t) - body := append(topicDetailsBody(t, 3), topicDetailsBody(t, 5)...) + body := binary.LittleEndian.AppendUint32(nil, 2) + body = append(body, topicDetailsBody(t, 3)...) + body = append(body, topicDetailsBody(t, 5)...) serve(serverConn, func(_ int, _ request) []byte { return replyFrame(vsr.OperationNonReplicated, body) }) diff --git a/foreign/go/client/tcp/tcp_utilities.go b/foreign/go/client/tcp/tcp_utilities.go index aea90ac19e..76ab2454da 100644 --- a/foreign/go/client/tcp/tcp_utilities.go +++ b/foreign/go/client/tcp/tcp_utilities.go @@ -20,10 +20,24 @@ package tcp import ( "context" + binaryserialization "github.com/apache/iggy/foreign/go/binary_serialization" iggcon "github.com/apache/iggy/foreign/go/contracts" "github.com/apache/iggy/foreign/go/internal/command" ) +// DescribeOptions returns the option catalog for one resource scope. +// +// This is how a client learns which option keys the server accepts: a key +// outside the catalog is refused at create, and the binary transports carry +// only the error code back. Scopes with no keys yet return an empty slice. +func (c *IggyTcpClient) DescribeOptions(ctx context.Context, scope iggcon.OptionsScope) ([]iggcon.OptionSpec, error) { + buffer, err := c.do(ctx, &command.DescribeOptions{Scope: scope}) + if err != nil { + return nil, err + } + return binaryserialization.DeserializeOptionSpecs(buffer) +} + func (c *IggyTcpClient) GetStats(ctx context.Context) (*iggcon.Stats, error) { buffer, err := c.do(ctx, &command.GetStats{}) if err != nil { diff --git a/foreign/go/contracts/client.go b/foreign/go/contracts/client.go index 6bcf71cd55..55b5890be3 100644 --- a/foreign/go/contracts/client.go +++ b/foreign/go/contracts/client.go @@ -61,8 +61,17 @@ type Client interface { // Authentication is required, and the permission to read the topics. GetTopics(ctx context.Context, streamId Identifier) ([]Topic, error) + // DescribeOptions get the option catalog for a resource scope: every key its + // create command accepts, with the kind, default and bounds of each. + // Authentication is required. + DescribeOptions(ctx context.Context, scope OptionsScope) ([]OptionSpec, error) + // CreateTopic create a new topic. // Authentication is required, and the permission to manage the topics. + // + // options carries topic option keys with no parameter of their own, for a key + // the server catalog gained after this build shipped; call DescribeOptions to + // see which keys a server accepts. A parameter above wins on collision. CreateTopic( ctx context.Context, streamId Identifier, @@ -71,11 +80,14 @@ type Client interface { compressionAlgorithm CompressionAlgorithm, messageExpiry Duration, maxTopicSize uint64, - replicationFactor *uint8, + options ...HeaderEntry, ) (*TopicDetails, error) // UpdateTopic update a topic by unique ID or name. // Authentication is required, and the permission to manage the topics. + // + // options carries keys with no parameter of their own. The server refuses any + // key an update may not change, by name. UpdateTopic( ctx context.Context, streamId Identifier, @@ -84,7 +96,7 @@ type Client interface { compressionAlgorithm CompressionAlgorithm, messageExpiry Duration, maxTopicSize uint64, - replicationFactor *uint8, + options ...HeaderEntry, ) error // DeleteTopic delete a topic by unique ID or name. diff --git a/foreign/go/contracts/compression_algorithm.go b/foreign/go/contracts/compression_algorithm.go index 389318a1fe..76dc6c7f8a 100644 --- a/foreign/go/contracts/compression_algorithm.go +++ b/foreign/go/contracts/compression_algorithm.go @@ -17,9 +17,44 @@ package iggcon +import "fmt" + type CompressionAlgorithm uint8 const ( - CompressionAlgorithmNone CompressionAlgorithm = 1 - CompressionAlgorithmGzip CompressionAlgorithm = 2 + // CompressionAlgorithmDefault is the zero value of the field: the server + // resolves the algorithm from its own configuration. + CompressionAlgorithmDefault CompressionAlgorithm = 0 + CompressionAlgorithmNone CompressionAlgorithm = 1 + CompressionAlgorithmGzip CompressionAlgorithm = 2 ) + +// OptionValue renders the algorithm for the server's `compression_algorithm` +// topic option, or "" when the option should be omitted so the server resolves +// its own default. +// +// An unrecognized code is an error rather than a fall back to the default: the +// topic would otherwise be created with the server's algorithm and no +// diagnostic, where the old fixed-field wire format put the raw byte on the +// wire and let the server reject it. +func (c CompressionAlgorithm) OptionValue() (string, error) { + switch c { + case CompressionAlgorithmDefault, CompressionAlgorithmNone: + return "", nil + case CompressionAlgorithmGzip: + return "gzip", nil + default: + return "", fmt.Errorf("unknown compression algorithm code %d", uint8(c)) + } +} + +func (c CompressionAlgorithm) String() string { + switch c { + case CompressionAlgorithmDefault, CompressionAlgorithmNone: + return "none" + case CompressionAlgorithmGzip: + return "gzip" + default: + return fmt.Sprintf("unknown(%d)", uint8(c)) + } +} diff --git a/foreign/go/contracts/compression_algorithm_test.go b/foreign/go/contracts/compression_algorithm_test.go new file mode 100644 index 0000000000..693fb0f1e2 --- /dev/null +++ b/foreign/go/contracts/compression_algorithm_test.go @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package iggcon + +import ( + "bytes" + "encoding/binary" + "testing" +) + +func TestCompressionAlgorithm_OptionValue(t *testing.T) { + tests := []struct { + algorithm CompressionAlgorithm + want string + wantErr bool + }{ + {CompressionAlgorithmDefault, "", false}, + {CompressionAlgorithmNone, "", false}, + {CompressionAlgorithmGzip, "gzip", false}, + {CompressionAlgorithm(200), "", true}, + } + for _, test := range tests { + got, err := test.algorithm.OptionValue() + if (err != nil) != test.wantErr { + t.Errorf("CompressionAlgorithm(%d).OptionValue() error = %v, wantErr %v", + uint8(test.algorithm), err, test.wantErr) + } + if got != test.want { + t.Errorf("CompressionAlgorithm(%d).OptionValue() = %q, want %q", + uint8(test.algorithm), got, test.want) + } + } +} + +func TestCompressionAlgorithm_StringNamesUnknownCode(t *testing.T) { + if got := CompressionAlgorithm(200).String(); got != "unknown(200)" { + t.Errorf("String() = %q, want %q", got, "unknown(200)") + } +} + +// goldenOptionsBlock is the cross-SDK golden vector for an options block. +// +// Rust pins the identical bytes in core/binary_protocol/src/primitives/options.rs, +// as do the Node and Java SDKs. Round-tripping through this SDK's own decoder +// proves nothing about interoperability; these bytes are the contract. +var goldenOptionsBlock = []byte{ + 2, 13, 0, 0, 0, + 'e', 'n', 'f', 'o', 'r', 'c', 'e', '_', 'f', 's', 'y', 'n', 'c', + 3, 1, 0, 0, 0, 1, + 2, 12, 0, 0, 0, + 's', 'e', 'g', 'm', 'e', 'n', 't', '_', 's', 'i', 'z', 'e', + 12, 8, 0, 0, 0, + 0, 0, 0, 64, 0, 0, 0, 0, +} + +func TestGetHeadersBytes_MatchesTheCrossSdkGoldenVector(t *testing.T) { + segmentSize := make([]byte, 8) + binary.LittleEndian.PutUint64(segmentSize, 1073741824) + entries := []HeaderEntry{ + { + Key: HeaderKey{Kind: String, Value: []byte("enforce_fsync")}, + Value: HeaderValue{Kind: Bool, Value: []byte{1}}, + }, + { + Key: HeaderKey{Kind: String, Value: []byte("segment_size")}, + Value: HeaderValue{Kind: Uint64, Value: segmentSize}, + }, + } + + got := GetHeadersBytes(entries) + + if !bytes.Equal(got, goldenOptionsBlock) { + t.Errorf("GetHeadersBytes() = %v, want %v", got, goldenOptionsBlock) + } +} diff --git a/foreign/go/contracts/options.go b/foreign/go/contracts/options.go new file mode 100644 index 0000000000..ef256a0ee3 --- /dev/null +++ b/foreign/go/contracts/options.go @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package iggcon + +import "fmt" + +// OptionsScope is the resource whose option catalog DescribeOptions serves. +type OptionsScope uint8 + +const ( + OptionsScopeTopic OptionsScope = 1 + OptionsScopeStream OptionsScope = 2 + OptionsScopeUser OptionsScope = 3 +) + +// Validate rejects a scope outside the three the server knows. +func (s OptionsScope) Validate() error { + switch s { + case OptionsScopeTopic, OptionsScopeStream, OptionsScopeUser: + return nil + default: + return fmt.Errorf("unknown options scope %d", uint8(s)) + } +} + +func (s OptionsScope) String() string { + switch s { + case OptionsScopeTopic: + return "topic" + case OptionsScopeStream: + return "stream" + case OptionsScopeUser: + return "user" + default: + return fmt.Sprintf("unknown(%d)", uint8(s)) + } +} + +// OptionSpec is one entry of a resource's option catalog. +// +// This is the discovery surface for the option keys a create command accepts: a +// key outside the catalog is refused at create, and the binary transports carry +// only the error code back, so there is no other way to learn which keys a +// given server knows. +type OptionSpec struct { + // Key is the option key the create command accepts. + Key string + // DefaultValue is the key's default, carrying the kind the server encodes + // it under. Empty for a key with no default. Reuses HeaderValue because + // options ride the user-headers codec. + DefaultValue HeaderValue + // Description says what the option does, including the bounds its value is + // checked against. + Description string +} diff --git a/foreign/go/contracts/stream.go b/foreign/go/contracts/stream.go index 523d5d7e25..03091a513b 100644 --- a/foreign/go/contracts/stream.go +++ b/foreign/go/contracts/stream.go @@ -23,12 +23,13 @@ type CreateStreamRequest struct { } type Stream struct { - Id uint32 `json:"id"` - Name string `json:"name"` - SizeBytes uint64 `json:"sizeBytes"` - CreatedAt uint64 `json:"createdAt"` - MessagesCount uint64 `json:"messagesCount"` - TopicsCount uint32 `json:"topicsCount"` + Id uint32 `json:"id"` + Name string `json:"name"` + SizeBytes uint64 `json:"sizeBytes"` + CreatedAt uint64 `json:"createdAt"` + MessagesCount uint64 `json:"messagesCount"` + TopicsCount uint32 `json:"topicsCount"` + Options map[string]HeaderValue `json:"options,omitempty"` } type StreamDetails struct { diff --git a/foreign/go/contracts/topic_options.go b/foreign/go/contracts/topic_options.go new file mode 100644 index 0000000000..7199fbf530 --- /dev/null +++ b/foreign/go/contracts/topic_options.go @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package iggcon + +import "encoding/binary" + +// Topic option keys that CreateTopic has no parameter of its own for. +// +// The constructors below are the spelling to use for them: create admission +// refuses a key outside the server's catalog by name, and refuses a value whose +// kind is neither the kind the catalog gives that key nor a String that key +// parses. DescribeOptions enumerates the catalog one server serves, with the +// bounds each value is checked against. Every key here is create-time only, so +// UpdateTopic refuses it by name. +const ( + topicOptionSegmentSize = "segment_size" + topicOptionEnforceFsync = "enforce_fsync" + topicOptionMessagesRequiredToSave = "messages_required_to_save" + topicOptionSizeOfMessagesRequiredToSave = "size_of_messages_required_to_save" + topicOptionPreallocateSegments = "preallocate_segments" +) + +// SegmentSizeOption sets how large a partition segment grows before it rotates. +// The server takes a 512-byte multiple inside the bounds its catalog reports. +// Zero is not refused: it leaves the key to the server default. +func SegmentSizeOption(bytes uint64) HeaderEntry { + return uint64Option(topicOptionSegmentSize, bytes) +} + +// EnforceFsyncOption makes writes to the topic's partitions fsync. +func EnforceFsyncOption(enabled bool) HeaderEntry { + return boolOption(topicOptionEnforceFsync, enabled) +} + +// MessagesRequiredToSaveOption flushes the journal once it holds this many +// messages. Zero is refused. Pairs with SizeOfMessagesRequiredToSaveOption: +// whichever threshold trips first flushes. +func MessagesRequiredToSaveOption(messages uint32) HeaderEntry { + return uint32Option(topicOptionMessagesRequiredToSave, messages) +} + +// SizeOfMessagesRequiredToSaveOption flushes the journal once it holds this many +// bytes. A threshold above the largest a segment may be is refused, since it +// could never trip. Zero leaves the key to the server default. +func SizeOfMessagesRequiredToSaveOption(bytes uint64) HeaderEntry { + return uint64Option(topicOptionSizeOfMessagesRequiredToSave, bytes) +} + +// PreallocateSegmentsOption reserves each segment's bytes on disk up front. A +// topic reserves segment_size for every partition it has, and the server refuses +// a create whose product crosses its preallocation cap. +func PreallocateSegmentsOption(enabled bool) HeaderEntry { + return boolOption(topicOptionPreallocateSegments, enabled) +} + +func uint64Option(key string, value uint64) HeaderEntry { + buf := make([]byte, 8) + binary.LittleEndian.PutUint64(buf, value) + return HeaderEntry{ + Key: HeaderKey{Kind: String, Value: []byte(key)}, + Value: HeaderValue{Kind: Uint64, Value: buf}, + } +} + +func uint32Option(key string, value uint32) HeaderEntry { + buf := make([]byte, 4) + binary.LittleEndian.PutUint32(buf, value) + return HeaderEntry{ + Key: HeaderKey{Kind: String, Value: []byte(key)}, + Value: HeaderValue{Kind: Uint32, Value: buf}, + } +} + +func boolOption(key string, value bool) HeaderEntry { + encoded := byte(0) + if value { + encoded = 1 + } + return HeaderEntry{ + Key: HeaderKey{Kind: String, Value: []byte(key)}, + Value: HeaderValue{Kind: Bool, Value: []byte{encoded}}, + } +} diff --git a/foreign/go/contracts/topic_options_test.go b/foreign/go/contracts/topic_options_test.go new file mode 100644 index 0000000000..885812e670 --- /dev/null +++ b/foreign/go/contracts/topic_options_test.go @@ -0,0 +1,135 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package iggcon + +import ( + "bytes" + "testing" +) + +// Keys and value bytes are spelled out rather than taken from the constants and +// binary.LittleEndian: an expectation built the same way as the code under test +// would agree with it even byte-swapped or misnamed. These literals are the wire +// contract each constructor is pinned to. +func TestTopicOptions_ConstructorsEncodeKeyAndValue(t *testing.T) { + tests := []struct { + name string + entry HeaderEntry + wantKey string + wantKind HeaderKind + wantValue []byte + }{ + { + name: "segment size", + entry: SegmentSizeOption(1 << 30), + wantKey: "segment_size", + wantKind: Uint64, + wantValue: []byte{0, 0, 0, 64, 0, 0, 0, 0}, + }, + { + name: "enforce fsync on", + entry: EnforceFsyncOption(true), + wantKey: "enforce_fsync", + wantKind: Bool, + wantValue: []byte{1}, + }, + { + name: "enforce fsync off", + entry: EnforceFsyncOption(false), + wantKey: "enforce_fsync", + wantKind: Bool, + wantValue: []byte{0}, + }, + { + name: "messages required to save", + entry: MessagesRequiredToSaveOption(1024), + wantKey: "messages_required_to_save", + wantKind: Uint32, + wantValue: []byte{0, 4, 0, 0}, + }, + { + name: "size of messages required to save", + entry: SizeOfMessagesRequiredToSaveOption(1 << 20), + wantKey: "size_of_messages_required_to_save", + wantKind: Uint64, + wantValue: []byte{0, 0, 16, 0, 0, 0, 0, 0}, + }, + { + name: "preallocate segments on", + entry: PreallocateSegmentsOption(true), + wantKey: "preallocate_segments", + wantKind: Bool, + wantValue: []byte{1}, + }, + { + name: "preallocate segments off", + entry: PreallocateSegmentsOption(false), + wantKey: "preallocate_segments", + wantKind: Bool, + wantValue: []byte{0}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if test.entry.Key.Kind != String { + t.Errorf("key kind = %d, want String", test.entry.Key.Kind) + } + if got := string(test.entry.Key.Value); got != test.wantKey { + t.Errorf("key = %q, want %q", got, test.wantKey) + } + if test.entry.Value.Kind != test.wantKind { + t.Errorf("value kind = %d, want %d", test.entry.Value.Kind, test.wantKind) + } + if !bytes.Equal(test.entry.Value.Value, test.wantValue) { + t.Errorf("value = %v, want %v", test.entry.Value.Value, test.wantValue) + } + if expected := test.wantKind.ExpectedSize(); len(test.entry.Value.Value) != expected { + t.Errorf("value length = %d, want %d for kind %d", + len(test.entry.Value.Value), expected, test.wantKind) + } + }) + } +} + +func TestTopicOptions_ConstructorEntriesSurviveTheHeaderCodec(t *testing.T) { + entries := []HeaderEntry{ + SegmentSizeOption(1 << 20), + EnforceFsyncOption(true), + MessagesRequiredToSaveOption(7), + SizeOfMessagesRequiredToSaveOption(4096), + PreallocateSegmentsOption(false), + } + + decoded, err := DeserializeHeaders(GetHeadersBytes(entries)) + if err != nil { + t.Fatalf("options block is not valid TLV: %v", err) + } + if len(decoded) != len(entries) { + t.Fatalf("decoded %d entries, want %d", len(decoded), len(entries)) + } + for index, entry := range entries { + got := decoded[index] + if !bytes.Equal(got.Key.Value, entry.Key.Value) || got.Key.Kind != entry.Key.Kind { + t.Errorf("entry %d key = %+v, want %+v", index, got.Key, entry.Key) + } + if !bytes.Equal(got.Value.Value, entry.Value.Value) || got.Value.Kind != entry.Value.Kind { + t.Errorf("entry %d value = %+v, want %+v", index, got.Value, entry.Value) + } + } +} diff --git a/foreign/go/contracts/topics.go b/foreign/go/contracts/topics.go index 554be7ee40..65fed72c4b 100644 --- a/foreign/go/contracts/topics.go +++ b/foreign/go/contracts/topics.go @@ -26,7 +26,6 @@ type CreateTopicRequest struct { CompressionAlgorithm uint8 `json:"compressionAlgorithm"` MessageExpiry time.Duration `json:"messageExpiry"` MaxTopicSize uint64 `json:"maxTopicSize"` - ReplicationFactor uint8 `json:"replicationFactor"` Name string `json:"name"` } @@ -36,21 +35,21 @@ type UpdateTopicRequest struct { CompressionAlgorithm uint8 `json:"compressionAlgorithm"` MessageExpiry time.Duration `json:"messageExpiry"` MaxTopicSize uint64 `json:"maxTopicSize"` - ReplicationFactor uint8 `json:"replicationFactor"` Name string `json:"name"` } type Topic struct { - Id uint32 `json:"id"` - CreatedAt uint64 `json:"createdAt"` - Name string `json:"name"` - Size uint64 `json:"size"` - MessageExpiry Duration `json:"messageExpiry"` - CompressionAlgorithm uint8 `json:"compressionAlgorithm"` - MaxTopicSize uint64 `json:"maxTopicSize"` - ReplicationFactor uint8 `json:"replicationFactor"` - MessagesCount uint64 `json:"messagesCount"` - PartitionsCount uint32 `json:"partitionsCount"` + Id uint32 `json:"id"` + CreatedAt uint64 `json:"createdAt"` + Name string `json:"name"` + Size uint64 `json:"size"` + MessageExpiry Duration `json:"messageExpiry"` + CompressionAlgorithm uint8 `json:"compressionAlgorithm"` + MaxTopicSize uint64 `json:"maxTopicSize"` + MessagesCount uint64 `json:"messagesCount"` + PartitionsCount uint32 `json:"partitionsCount"` + Options map[string]HeaderValue `json:"options,omitempty"` + DerivedOptions map[string]HeaderValue `json:"derivedOptions,omitempty"` } type TopicDetails struct { diff --git a/foreign/go/contracts/users.go b/foreign/go/contracts/users.go index 7bcfc108ca..8dbafe7746 100644 --- a/foreign/go/contracts/users.go +++ b/foreign/go/contracts/users.go @@ -20,10 +20,11 @@ package iggcon import "encoding/binary" type UserInfo struct { - Id uint32 `json:"Id"` - CreatedAt uint64 `json:"CreatedAt"` - Status UserStatus `json:"Status"` - Username string `json:"Username"` + Id uint32 `json:"Id"` + CreatedAt uint64 `json:"CreatedAt"` + Status UserStatus `json:"Status"` + Username string `json:"Username"` + Options map[string]HeaderValue `json:"Options,omitempty"` } type UserInfoDetails struct { diff --git a/foreign/go/internal/command/code.go b/foreign/go/internal/command/code.go index 9cc6fff112..057a829693 100644 --- a/foreign/go/internal/command/code.go +++ b/foreign/go/internal/command/code.go @@ -24,6 +24,7 @@ const ( GetStatsCode Code = 10 GetSnapshotFileCode Code = 11 GetClusterMetadataCode Code = 12 + DescribeOptionsCode Code = 13 GetMeCode Code = 20 GetClientCode Code = 21 GetClientsCode Code = 22 diff --git a/foreign/go/internal/command/stream.go b/foreign/go/internal/command/stream.go index d3c3c4098f..210296609b 100644 --- a/foreign/go/internal/command/stream.go +++ b/foreign/go/internal/command/stream.go @@ -65,6 +65,10 @@ func (g *GetStreams) MarshalBinary() ([]byte, error) { type UpdateStream struct { StreamId iggcon.Identifier `json:"streamId"` Name string `json:"name"` + // Options carries the trailing options block. Streams have no catalog + // keys yet, so the server rejects every key; the block is the extension + // point for the first one. + Options []iggcon.HeaderEntry `json:"-"` } func (u *UpdateStream) Code() Code { @@ -77,11 +81,14 @@ func (u *UpdateStream) MarshalBinary() ([]byte, error) { return nil, err } nameLength := len(u.Name) - bytes := make([]byte, len(streamIdBytes)+1+nameLength) + optionsBytes := iggcon.GetHeadersBytes(u.Options) + bytes := make([]byte, len(streamIdBytes)+1+nameLength+len(optionsBytes)) copy(bytes[0:len(streamIdBytes)], streamIdBytes) position := len(streamIdBytes) bytes[position] = byte(nameLength) - copy(bytes[position+1:], u.Name) + position++ + position += copy(bytes[position:], u.Name) + copy(bytes[position:], optionsBytes) return bytes, nil } diff --git a/foreign/go/internal/command/system.go b/foreign/go/internal/command/system.go index 778d7043f9..78cb6b59fb 100644 --- a/foreign/go/internal/command/system.go +++ b/foreign/go/internal/command/system.go @@ -17,7 +17,11 @@ package command -import "encoding/binary" +import ( + "encoding/binary" + + iggcon "github.com/apache/iggy/foreign/go/contracts" +) type GetClient struct { ClientID uint32 @@ -54,6 +58,22 @@ func (m *GetClusterMetadata) MarshalBinary() ([]byte, error) { return []byte{}, nil } +// DescribeOptions asks for the option catalog of one resource scope. +type DescribeOptions struct { + Scope iggcon.OptionsScope +} + +func (d *DescribeOptions) Code() Code { + return DescribeOptionsCode +} + +func (d *DescribeOptions) MarshalBinary() ([]byte, error) { + if err := d.Scope.Validate(); err != nil { + return nil, err + } + return []byte{byte(d.Scope)}, nil +} + type GetStats struct{} func (c *GetStats) Code() Code { diff --git a/foreign/go/internal/command/topic.go b/foreign/go/internal/command/topic.go index 1ce8a988d0..9add5dd986 100644 --- a/foreign/go/internal/command/topic.go +++ b/foreign/go/internal/command/topic.go @@ -19,10 +19,17 @@ package command import ( "encoding/binary" + "fmt" "github.com/apache/iggy/foreign/go/contracts" ) +const ( + topicOptionCompressionAlgorithm = "compression_algorithm" + topicOptionMessageExpiry = "message_expiry" + topicOptionMaxTopicSize = "max_topic_size" +) + type CreateTopic struct { StreamId iggcon.Identifier `json:"streamId"` PartitionsCount uint32 `json:"partitionsCount"` @@ -30,7 +37,10 @@ type CreateTopic struct { MessageExpiry iggcon.Duration `json:"messageExpiry"` MaxTopicSize uint64 `json:"maxTopicSize"` Name string `json:"name"` - ReplicationFactor *uint8 `json:"replicationFactor"` + // Options carries keys with no field above, for a key the server catalog + // gained after this build shipped. Entries reuse HeaderEntry because options + // ride the user-headers codec. A field above wins on collision. + Options []iggcon.HeaderEntry `json:"-"` } func (t *CreateTopic) Code() Code { @@ -38,58 +48,92 @@ func (t *CreateTopic) Code() Code { } func (t *CreateTopic) MarshalBinary() ([]byte, error) { - if t.ReplicationFactor == nil { - t.ReplicationFactor = new(uint8) - } - streamIdBytes, err := t.StreamId.MarshalBinary() if err != nil { return nil, err } nameBytes := []byte(t.Name) + options, err := t.options() + if err != nil { + return nil, err + } + optionsBytes := iggcon.GetHeadersBytes(options) - totalLength := len(streamIdBytes) + // StreamId - 4 + // PartitionsCount - 1 + // CompressionAlgorithm - 8 + // MessageExpiry - 8 + // MaxTopicSize - 1 + // ReplicationFactor - 1 + // Name length - len(nameBytes) // Name - bytes := make([]byte, totalLength) - - position := 0 - - // StreamId - copy(bytes[position:], streamIdBytes) - position += len(streamIdBytes) - - // PartitionsCount - binary.LittleEndian.PutUint32(bytes[position:], t.PartitionsCount) - position += 4 - - // CompressionAlgorithm - bytes[position] = byte(t.CompressionAlgorithm) - position++ + bytes := make([]byte, 0, len(streamIdBytes)+4+1+len(nameBytes)+len(optionsBytes)) + bytes = append(bytes, streamIdBytes...) + bytes = binary.LittleEndian.AppendUint32(bytes, t.PartitionsCount) + bytes = append(bytes, byte(len(nameBytes))) + bytes = append(bytes, nameBytes...) + bytes = append(bytes, optionsBytes...) - // MessageExpiry - binary.LittleEndian.PutUint64(bytes[position:], uint64(t.MessageExpiry)) - position += 8 + return bytes, nil +} - // MaxTopicSize - binary.LittleEndian.PutUint64(bytes[position:], t.MaxTopicSize) - position += 8 +// options builds the trailing options block. partitions_count is not an +// option: it fills the command's own fixed field. Keys carrying the +// server-default sentinel (expiry 0, size 0, compression none) are omitted so +// the server derives them and returns them as derived entries. +func (t *CreateTopic) options() ([]iggcon.HeaderEntry, error) { + var options []iggcon.HeaderEntry + compression, err := t.CompressionAlgorithm.OptionValue() + if err != nil { + return nil, err + } + if compression != "" { + options = append(options, stringOption(topicOptionCompressionAlgorithm, compression)) + } + if t.MessageExpiry != 0 { + options = append(options, uint64Option(topicOptionMessageExpiry, uint64(t.MessageExpiry))) + } + if t.MaxTopicSize != 0 { + options = append(options, uint64Option(topicOptionMaxTopicSize, t.MaxTopicSize)) + } + return mergeOptions(options, t.Options) +} - // ReplicationFactor - bytes[position] = *t.ReplicationFactor - position++ +// mergeOptions appends the caller's entries to the ones the typed fields +// produced, dropping any that would duplicate a typed key. +// +// The block must not carry a key twice: wire validation refuses the whole +// request for a duplicate. The typed field wins because it is the specific +// argument, mirroring how the Rust SDK inserts typed values after raw ones. +func mergeOptions(typed, extra []iggcon.HeaderEntry) ([]iggcon.HeaderEntry, error) { + if len(extra) == 0 { + return typed, nil + } + seen := make(map[string]struct{}, len(typed)+len(extra)) + for _, entry := range typed { + seen[string(entry.Key.Value)] = struct{}{} + } + merged := typed + for _, entry := range extra { + if entry.Key.Kind != iggcon.String { + return nil, fmt.Errorf("option key kind %d is not a string", entry.Key.Kind) + } + key := string(entry.Key.Value) + if _, duplicate := seen[key]; duplicate { + continue + } + seen[key] = struct{}{} + merged = append(merged, entry) + } + return merged, nil +} - // Name - bytes[position] = byte(len(nameBytes)) - position++ - copy(bytes[position:], nameBytes) +func uint64Option(key string, value uint64) iggcon.HeaderEntry { + buf := make([]byte, 8) + binary.LittleEndian.PutUint64(buf, value) + return iggcon.HeaderEntry{ + Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte(key)}, + Value: iggcon.HeaderValue{Kind: iggcon.Uint64, Value: buf}, + } +} - return bytes, nil +func stringOption(key string, value string) iggcon.HeaderEntry { + return iggcon.HeaderEntry{ + Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte(key)}, + Value: iggcon.HeaderValue{Kind: iggcon.String, Value: []byte(value)}, + } } type GetTopic struct { @@ -131,23 +175,46 @@ func (d *DeleteTopic) MarshalBinary() ([]byte, error) { } type UpdateTopic struct { - StreamId iggcon.Identifier `json:"streamId"` - TopicId iggcon.Identifier `json:"topicId"` + StreamId iggcon.Identifier `json:"streamId"` + TopicId iggcon.Identifier `json:"topicId"` + Name string `json:"name"` + // Options carries keys with no field below. The server refuses any key an + // update may not change, by name. + Options []iggcon.HeaderEntry `json:"-"` + // Settings ride the options block. A zero value means "leave it alone", + // so a rename does not silently reset the rest. CompressionAlgorithm iggcon.CompressionAlgorithm `json:"compressionAlgorithm"` MessageExpiry iggcon.Duration `json:"messageExpiry"` MaxTopicSize uint64 `json:"maxTopicSize"` - ReplicationFactor *uint8 `json:"replicationFactor"` - Name string `json:"name"` } func (u *UpdateTopic) Code() Code { return UpdateTopicCode } -func (u *UpdateTopic) MarshalBinary() ([]byte, error) { - if u.ReplicationFactor == nil { - u.ReplicationFactor = new(uint8) +// options builds the trailing options block. Only keys an update may change are +// allowed; the server rejects the create-time knobs by name. A zero value means +// the caller did not set the key, so it is omitted and the server leaves the +// topic's current value alone. +func (u *UpdateTopic) options() ([]iggcon.HeaderEntry, error) { + var options []iggcon.HeaderEntry + compression, err := u.CompressionAlgorithm.OptionValue() + if err != nil { + return nil, err + } + if compression != "" { + options = append(options, stringOption(topicOptionCompressionAlgorithm, compression)) + } + if u.MessageExpiry != 0 { + options = append(options, uint64Option(topicOptionMessageExpiry, uint64(u.MessageExpiry))) + } + if u.MaxTopicSize != 0 { + options = append(options, uint64Option(topicOptionMaxTopicSize, u.MaxTopicSize)) } + return mergeOptions(options, u.Options) +} + +func (u *UpdateTopic) MarshalBinary() ([]byte, error) { streamIdBytes, err := u.StreamId.MarshalBinary() if err != nil { return nil, err @@ -157,29 +224,23 @@ func (u *UpdateTopic) MarshalBinary() ([]byte, error) { return nil, err } - buffer := make([]byte, 19+len(streamIdBytes)+len(topicIdBytes)+len(u.Name)) + options, err := u.options() + if err != nil { + return nil, err + } + optionsBytes := iggcon.GetHeadersBytes(options) + buffer := make([]byte, 1+len(streamIdBytes)+len(topicIdBytes)+len(u.Name)+len(optionsBytes)) offset := 0 offset += copy(buffer[offset:], streamIdBytes) offset += copy(buffer[offset:], topicIdBytes) - buffer[offset] = byte(u.CompressionAlgorithm) - offset++ - - binary.LittleEndian.PutUint64(buffer[offset:], uint64(u.MessageExpiry)) - offset += 8 - - binary.LittleEndian.PutUint64(buffer[offset:], u.MaxTopicSize) - offset += 8 - - buffer[offset] = *u.ReplicationFactor - offset++ - buffer[offset] = uint8(len(u.Name)) offset++ - copy(buffer[offset:], u.Name) + offset += copy(buffer[offset:], u.Name) + copy(buffer[offset:], optionsBytes) return buffer, nil } diff --git a/foreign/go/internal/command/topic_test.go b/foreign/go/internal/command/topic_test.go index 12357ed4d6..233408603d 100644 --- a/foreign/go/internal/command/topic_test.go +++ b/foreign/go/internal/command/topic_test.go @@ -19,11 +19,97 @@ package command import ( "bytes" + "encoding/binary" "testing" iggcon "github.com/apache/iggy/foreign/go/contracts" ) +func TestSerialize_CreateTopic_ServerDefaults(t *testing.T) { + streamId, _ := iggcon.NewIdentifier("stream") + request := CreateTopic{ + StreamId: streamId, + Name: "topic", + PartitionsCount: 2, + CompressionAlgorithm: iggcon.CompressionAlgorithmNone, + } + + serialized, err := request.MarshalBinary() + if err != nil { + t.Fatalf("Failed to serialize CreateTopic: %v", err) + } + + expected := []byte{ + 0x02, // StreamId Kind (StringId) + 0x06, // StreamId Length (6) + 0x73, 0x74, 0x72, 0x65, 0x61, 0x6D, // StreamId Value ("stream") + 0x02, 0x00, 0x00, 0x00, // PartitionsCount (2) + 0x05, // Name Length (5) + 0x74, 0x6F, 0x70, 0x69, 0x63, // Name ("topic") + // options: empty, every default is derived server-side + } + + if !bytes.Equal(serialized, expected) { + t.Errorf("CreateTopic serialization failed. \nExpected:\t%v\nGot:\t\t%v", expected, serialized) + } +} + +func TestSerialize_CreateTopic_NonDefaultsBecomeOptions(t *testing.T) { + streamId, _ := iggcon.NewIdentifier(uint32(1)) + request := CreateTopic{ + StreamId: streamId, + Name: "topic", + PartitionsCount: 4, + CompressionAlgorithm: iggcon.CompressionAlgorithmGzip, + MessageExpiry: 100 * iggcon.Microsecond, + MaxTopicSize: 1 << 30, + } + + serialized, err := request.MarshalBinary() + if err != nil { + t.Fatalf("Failed to serialize CreateTopic: %v", err) + } + + // [stream_id: kind+len+4][partitions_count:4][name_len][name] then options to end. + const streamIdSize = 2 + 4 + if partitions := binary.LittleEndian.Uint32(serialized[streamIdSize:]); partitions != 4 { + t.Errorf("partitions_count fixed field = %d, want 4", partitions) + } + + optionsOffset := streamIdSize + 4 + 1 + len(request.Name) + options, err := iggcon.DeserializeHeaders(serialized[optionsOffset:]) + if err != nil { + t.Fatalf("Options block is not valid TLV: %v", err) + } + + byKey := make(map[string]iggcon.HeaderValue, len(options)) + for _, entry := range options { + if entry.Key.Kind != iggcon.String { + t.Errorf("Option key %q has kind %d, want String", entry.Key.Value, entry.Key.Kind) + } + byKey[string(entry.Key.Value)] = entry.Value + } + + if len(byKey) != 3 { + t.Fatalf("expected 3 options, got %d: %v", len(byKey), byKey) + } + if _, found := byKey["partitions_count"]; found { + t.Error("partitions_count rides the fixed field, not the options block") + } + compression := byKey[topicOptionCompressionAlgorithm] + if compression.Kind != iggcon.String || string(compression.Value) != "gzip" { + t.Errorf("compression_algorithm = %+v, want String %q", compression, "gzip") + } + expiry := byKey[topicOptionMessageExpiry] + if expiry.Kind != iggcon.Uint64 || binary.LittleEndian.Uint64(expiry.Value) != 100 { + t.Errorf("message_expiry = %+v, want Uint64 100", expiry) + } + maxSize := byKey[topicOptionMaxTopicSize] + if maxSize.Kind != iggcon.Uint64 || binary.LittleEndian.Uint64(maxSize.Value) != 1<<30 { + t.Errorf("max_topic_size = %+v, want Uint64 %d", maxSize, 1<<30) + } +} + func TestSerialize_UpdateTopic(t *testing.T) { streamId, _ := iggcon.NewIdentifier("stream") topicId, _ := iggcon.NewIdentifier(uint32(1)) @@ -47,15 +133,150 @@ func TestSerialize_UpdateTopic(t *testing.T) { 0x01, // TopicId Kind (NumericId) 0x04, // TopicId Length (4) 0x01, 0x00, 0x00, 0x00, // TopicId Value (1) - 0x00, // compression algorithm - 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // Message Expiry (100) - 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // Max Topic Size (100) - 0x00, // Replication factor 0x0C, // Name Length (12) 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5F, 0x74, 0x6F, 0x70, 0x69, 0x63, // Name ("update_topic") + // Settings ride the options block; compression is None so it is omitted. + 0x02, 0x0E, 0x00, 0x00, 0x00, // key kind (String), key length (14) + 0x6D, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x5F, 0x65, 0x78, 0x70, 0x69, 0x72, 0x79, // "message_expiry" + 0x0C, 0x08, 0x00, 0x00, 0x00, // value kind (Uint64), value length (8) + 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 100 + 0x02, 0x0E, 0x00, 0x00, 0x00, // key kind (String), key length (14) + 0x6D, 0x61, 0x78, 0x5F, 0x74, 0x6F, 0x70, 0x69, 0x63, 0x5F, 0x73, 0x69, 0x7A, 0x65, // "max_topic_size" + 0x0C, 0x08, 0x00, 0x00, 0x00, // value kind (Uint64), value length (8) + 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 100 } if !bytes.Equal(serialized1, expected) { t.Errorf("Test case 1 failed. \nExpected:\t%v\nGot:\t\t%v", expected, serialized1) } } + +func TestCreateTopic_CallerOptionsRideTheBlockAndTypedFieldsWin(t *testing.T) { + streamId, err := iggcon.NewIdentifier(uint32(1)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + command := CreateTopic{ + StreamId: streamId, + Name: "t", + PartitionsCount: 1, + MaxTopicSize: 4096, + Options: []iggcon.HeaderEntry{ + { + Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte("enforce_fsync")}, + Value: iggcon.HeaderValue{Kind: iggcon.Bool, Value: []byte{1}}, + }, + // The typed field already covers this key, so the caller's entry is + // dropped: a duplicate key makes the server refuse the whole block. + { + Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte("max_topic_size")}, + Value: iggcon.HeaderValue{Kind: iggcon.String, Value: []byte("1 GiB")}, + }, + }, + } + + options, err := command.options() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + byKey := map[string]iggcon.HeaderValue{} + for _, entry := range options { + key := string(entry.Key.Value) + if _, duplicate := byKey[key]; duplicate { + t.Fatalf("key %q appears twice in the options block", key) + } + byKey[key] = entry.Value + } + if _, ok := byKey["enforce_fsync"]; !ok { + t.Error("a caller-supplied key must reach the options block") + } + if got := byKey["max_topic_size"]; got.Kind != iggcon.Uint64 { + t.Errorf("max_topic_size kind = %v, want the typed field's Uint64", got.Kind) + } +} + +func TestCreateTopic_TypedOptionConstructorsReachTheBlock(t *testing.T) { + streamId, err := iggcon.NewIdentifier(uint32(1)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + command := CreateTopic{ + StreamId: streamId, + Name: "topic", + PartitionsCount: 1, + Options: []iggcon.HeaderEntry{ + iggcon.SegmentSizeOption(1 << 20), + iggcon.EnforceFsyncOption(true), + iggcon.MessagesRequiredToSaveOption(7), + iggcon.SizeOfMessagesRequiredToSaveOption(4096), + iggcon.PreallocateSegmentsOption(false), + }, + } + + serialized, err := command.MarshalBinary() + if err != nil { + t.Fatalf("failed to serialize CreateTopic: %v", err) + } + + const streamIdSize = 2 + 4 + optionsOffset := streamIdSize + 4 + 1 + len(command.Name) + options, err := iggcon.DeserializeHeaders(serialized[optionsOffset:]) + if err != nil { + t.Fatalf("options block is not valid TLV: %v", err) + } + byKey := make(map[string]iggcon.HeaderValue, len(options)) + for _, entry := range options { + byKey[string(entry.Key.Value)] = entry.Value + } + + // Literal keys and value bytes: they are the wire contract the constructors + // are pinned to, so restating them here is the assertion. + want := []struct { + key string + kind iggcon.HeaderKind + value []byte + }{ + {"segment_size", iggcon.Uint64, []byte{0, 0, 16, 0, 0, 0, 0, 0}}, + {"enforce_fsync", iggcon.Bool, []byte{1}}, + {"messages_required_to_save", iggcon.Uint32, []byte{7, 0, 0, 0}}, + {"size_of_messages_required_to_save", iggcon.Uint64, []byte{0, 16, 0, 0, 0, 0, 0, 0}}, + {"preallocate_segments", iggcon.Bool, []byte{0}}, + } + if len(byKey) != len(want) { + t.Fatalf("options block carries %d keys, want %d: %v", len(byKey), len(want), byKey) + } + for _, expected := range want { + got, found := byKey[expected.key] + if !found { + t.Errorf("key %q is missing from the options block", expected.key) + continue + } + if got.Kind != expected.kind { + t.Errorf("key %q kind = %d, want %d", expected.key, got.Kind, expected.kind) + } + if !bytes.Equal(got.Value, expected.value) { + t.Errorf("key %q value = %v, want %v", expected.key, got.Value, expected.value) + } + } +} + +func TestCreateTopic_NonStringOptionKeyIsRejected(t *testing.T) { + streamId, err := iggcon.NewIdentifier(uint32(1)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + command := CreateTopic{ + StreamId: streamId, + Name: "t", + PartitionsCount: 1, + Options: []iggcon.HeaderEntry{{ + Key: iggcon.HeaderKey{Kind: iggcon.Uint32, Value: []byte{1, 0, 0, 0}}, + Value: iggcon.HeaderValue{Kind: iggcon.String, Value: []byte("x")}, + }}, + } + + if _, err := command.MarshalBinary(); err == nil { + t.Error("expected an error for a non-string option key, got nil") + } +} diff --git a/foreign/go/internal/command/update_user.go b/foreign/go/internal/command/update_user.go index a20d647fb7..452b4db771 100644 --- a/foreign/go/internal/command/update_user.go +++ b/foreign/go/internal/command/update_user.go @@ -23,6 +23,10 @@ type UpdateUser struct { UserID iggcon.Identifier `json:"-"` Username *string `json:"username"` Status *iggcon.UserStatus `json:"userStatus"` + // Options carries the trailing options block. Users have no catalog keys + // yet, so the server rejects every key; the block is the extension point + // for the first one. + Options []iggcon.HeaderEntry `json:"-"` } func (u *UpdateUser) Code() Code { @@ -34,23 +38,24 @@ func (u *UpdateUser) MarshalBinary() ([]byte, error) { if err != nil { return nil, err } - length := len(userIdBytes) - - if u.Username == nil { - u.Username = new(string) + username := "" + if u.Username != nil { + username = *u.Username } - username := *u.Username - + // Both presence flags are always written; only the payload behind a flag is + // conditional. The options block decodes to the end of the payload, so a + // byte of slack here is a stray zero-kind entry the server rejects. + length := len(userIdBytes) + 2 if len(username) != 0 { - length += 2 + len(username) + length += 1 + len(username) } - if u.Status != nil { - length += 2 + length++ } - bytes := make([]byte, length+1) + optionsBytes := iggcon.GetHeadersBytes(u.Options) + bytes := make([]byte, length+len(optionsBytes)) position := 0 copy(bytes[position:position+len(userIdBytes)], userIdBytes) @@ -79,9 +84,13 @@ func (u *UpdateUser) MarshalBinary() ([]byte, error) { statusByte = 2 } bytes[position] = statusByte + position++ } else { bytes[position] = 0 + position++ } + copy(bytes[position:], optionsBytes) + return bytes, nil } diff --git a/foreign/go/internal/command/update_user_test.go b/foreign/go/internal/command/update_user_test.go new file mode 100644 index 0000000000..5cead96447 --- /dev/null +++ b/foreign/go/internal/command/update_user_test.go @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package command + +import ( + "testing" + + iggcon "github.com/apache/iggy/foreign/go/contracts" +) + +// The options block decodes to the end of the payload, so any slack the sizing +// leaves behind reaches the server as an entry with kind 0, which it rejects. +func TestUpdateUser_MarshalBinaryLeavesNoSlack(t *testing.T) { + username := "user" + status := iggcon.Active + userID, err := iggcon.NewIdentifier(uint32(7)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + userIDBytes, err := userID.MarshalBinary() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + tests := []struct { + name string + username *string + status *iggcon.UserStatus + want int + }{ + {"both set", &username, &status, len(userIDBytes) + 2 + len(username) + 2}, + {"username only", &username, nil, len(userIDBytes) + 2 + len(username) + 1}, + {"status only", nil, &status, len(userIDBytes) + 1 + 2}, + {"neither set", nil, nil, len(userIDBytes) + 2}, + } + for _, test := range tests { + command := UpdateUser{UserID: userID, Username: test.username, Status: test.status} + payload, err := command.MarshalBinary() + if err != nil { + t.Fatalf("%s: unexpected error: %v", test.name, err) + } + if len(payload) != test.want { + t.Errorf("%s: payload length = %d, want %d", test.name, len(payload), test.want) + } + } +} + +func TestUpdateUser_MarshalBinaryDoesNotMutateUsername(t *testing.T) { + userID, err := iggcon.NewIdentifier(uint32(1)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + command := UpdateUser{UserID: userID} + + if _, err = command.MarshalBinary(); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if command.Username != nil { + t.Errorf("Username = %v, want nil", command.Username) + } +} diff --git a/foreign/go/tests/e2e_helpers_test.go b/foreign/go/tests/e2e_helpers_test.go index 75e680c24d..8a4549b748 100644 --- a/foreign/go/tests/e2e_helpers_test.go +++ b/foreign/go/tests/e2e_helpers_test.go @@ -137,7 +137,7 @@ func scratchTopic(t *testing.T, connected iggcon.Client, partitionsCount uint32) t.Cleanup(func() { _ = connected.DeleteStream(context.Background(), streamId) }) topic, err := connected.CreateTopic(ctx, streamId, name, partitionsCount, - iggcon.CompressionAlgorithmNone, iggcon.Duration(0), 0, nil) + iggcon.CompressionAlgorithmNone, iggcon.Duration(0), 0) require.NoError(t, err) topicId, err := iggcon.NewIdentifier(topic.Id) require.NoError(t, err) diff --git a/foreign/go/tests/e2e_test.go b/foreign/go/tests/e2e_test.go index 93a7c02cbb..48c2cfc909 100644 --- a/foreign/go/tests/e2e_test.go +++ b/foreign/go/tests/e2e_test.go @@ -20,7 +20,9 @@ package tests_test import ( "context" "encoding/binary" + "fmt" "testing" + "time" "github.com/apache/iggy/foreign/go/client/tcp" iggcon "github.com/apache/iggy/foreign/go/contracts" @@ -309,3 +311,143 @@ func TestE2E_TLSRoundTrip(t *testing.T) { require.NoError(t, err) assert.Len(t, polled.Messages, 3) } + +func TestE2E_TopicOptionsRoundTripAndCatalog(t *testing.T) { + connected := connect(t) + ctx := context.Background() + + specs, err := connected.DescribeOptions(ctx, iggcon.OptionsScopeTopic) + require.NoError(t, err) + byKey := map[string]iggcon.OptionSpec{} + for _, spec := range specs { + byKey[spec.Key] = spec + } + require.Contains(t, byKey, "enforce_fsync", "the catalog lists the keys create accepts") + require.Contains(t, byKey, "segment_size") + assert.NotEmpty(t, byKey["segment_size"].Description) + assert.Equal(t, iggcon.Uint64, byKey["segment_size"].DefaultValue.Kind) + + // Scopes with no catalog keys answer with an empty list rather than failing. + streamSpecs, err := connected.DescribeOptions(ctx, iggcon.OptionsScopeStream) + require.NoError(t, err) + assert.Empty(t, streamSpecs) + + name := fmt.Sprintf("go-e2e-options-%d", time.Now().UnixNano()) + stream, err := connected.CreateStream(ctx, name) + require.NoError(t, err) + streamId, err := iggcon.NewIdentifier(stream.Id) + require.NoError(t, err) + t.Cleanup(func() { _ = connected.DeleteStream(context.Background(), streamId) }) + + // A key with no parameter of its own rides the variadic options block. + created, err := connected.CreateTopic(ctx, streamId, name, 1, + iggcon.CompressionAlgorithmNone, iggcon.Duration(0), 0, + iggcon.HeaderEntry{ + Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte("enforce_fsync")}, + Value: iggcon.HeaderValue{Kind: iggcon.String, Value: []byte("true")}, + }) + require.NoError(t, err) + + topicId, err := iggcon.NewIdentifier(created.Id) + require.NoError(t, err) + topic, err := connected.GetTopic(ctx, streamId, topicId) + require.NoError(t, err) + + fsync, ok := topic.Options["enforce_fsync"] + require.True(t, ok, "an explicitly set key is reported as explicit, got %v", topic.Options) + // Create admission re-encodes the block from its own parse, so the stored + // value carries the key's canonical kind whatever kind the client sent it + // as: this string "true" comes back as a Bool. + assert.Equal(t, iggcon.Bool, fsync.Kind) + assert.Equal(t, []byte{1}, fsync.Value) + // Keys the client left alone are resolved by admission and reported apart. + require.Contains(t, topic.DerivedOptions, "max_topic_size") + assert.NotContains(t, topic.DerivedOptions, "enforce_fsync") + + // A key outside the catalog is refused by name. + _, err = connected.CreateTopic(ctx, streamId, name+"-bad", 1, + iggcon.CompressionAlgorithmNone, iggcon.Duration(0), 0, + iggcon.HeaderEntry{ + Key: iggcon.HeaderKey{Kind: iggcon.String, Value: []byte("not_a_real_option")}, + Value: iggcon.HeaderValue{Kind: iggcon.String, Value: []byte("1")}, + }) + require.Error(t, err) +} + +func TestE2E_TypedTopicOptionsMatchTheCatalog(t *testing.T) { + connected := connect(t) + ctx := context.Background() + + // Values inside the bounds the catalog reports: a 512-byte multiple segment + // size at the floor, a non-zero message threshold, preallocation off so + // nothing is reserved on disk. + typed := []iggcon.HeaderEntry{ + iggcon.SegmentSizeOption(1024 * 1024), + iggcon.EnforceFsyncOption(true), + iggcon.MessagesRequiredToSaveOption(7), + iggcon.SizeOfMessagesRequiredToSaveOption(4096), + iggcon.PreallocateSegmentsOption(false), + } + + specs, err := connected.DescribeOptions(ctx, iggcon.OptionsScopeTopic) + require.NoError(t, err) + catalog := map[string]iggcon.OptionSpec{} + for _, spec := range specs { + catalog[spec.Key] = spec + } + for _, entry := range typed { + key := string(entry.Key.Value) + spec, found := catalog[key] + require.True(t, found, "%q is not a catalog key", key) + assert.Equal(t, spec.DefaultValue.Kind, entry.Value.Kind, + "%q must be sent in the kind the catalog gives it", key) + } + + name := fmt.Sprintf("go-e2e-typed-options-%d", time.Now().UnixNano()) + stream, err := connected.CreateStream(ctx, name) + require.NoError(t, err) + streamId, err := iggcon.NewIdentifier(stream.Id) + require.NoError(t, err) + t.Cleanup(func() { _ = connected.DeleteStream(context.Background(), streamId) }) + + created, err := connected.CreateTopic(ctx, streamId, name, 1, + iggcon.CompressionAlgorithmNone, iggcon.Duration(0), 0, typed...) + require.NoError(t, err) + + topicId, err := iggcon.NewIdentifier(created.Id) + require.NoError(t, err) + topic, err := connected.GetTopic(ctx, streamId, topicId) + require.NoError(t, err) + + for _, entry := range typed { + key := string(entry.Key.Value) + stored, found := topic.Options[key] + require.True(t, found, "%q was set explicitly, got %v", key, topic.Options) + // Admission re-encodes the block in each key's canonical kind, which is + // the kind the constructor already sent, so the bytes survive unchanged. + assert.Equal(t, entry.Value.Kind, stored.Kind, key) + assert.Equal(t, entry.Value.Value, stored.Value, key) + assert.NotContains(t, topic.DerivedOptions, key) + } + + // A flush threshold of zero messages can never trip, so it is refused. + _, err = connected.CreateTopic(ctx, streamId, name+"-zero-flush", 1, + iggcon.CompressionAlgorithmNone, iggcon.Duration(0), 0, + iggcon.MessagesRequiredToSaveOption(0)) + require.Error(t, err) + + // A segment size off the 512-byte grid is refused, but zero is not: it + // leaves the key derived. + _, err = connected.CreateTopic(ctx, streamId, name+"-odd-segment", 1, + iggcon.CompressionAlgorithmNone, iggcon.Duration(0), 0, + iggcon.SegmentSizeOption(1024*1024+1)) + require.Error(t, err) + + zeroSegment := iggcon.SegmentSizeOption(0) + derived, err := connected.CreateTopic(ctx, streamId, name+"-zero-segment", 1, + iggcon.CompressionAlgorithmNone, iggcon.Duration(0), 0, zeroSegment) + require.NoError(t, err) + segmentSizeKey := string(zeroSegment.Key.Value) + assert.NotContains(t, derived.Options, segmentSizeKey) + assert.Contains(t, derived.DerivedOptions, segmentSizeKey) +} diff --git a/foreign/java/bench/src/main/java/org/apache/iggy/bench/common/provision/ResourceProvisioner.java b/foreign/java/bench/src/main/java/org/apache/iggy/bench/common/provision/ResourceProvisioner.java index 717d694bc3..8fdae2b9d2 100644 --- a/foreign/java/bench/src/main/java/org/apache/iggy/bench/common/provision/ResourceProvisioner.java +++ b/foreign/java/bench/src/main/java/org/apache/iggy/bench/common/provision/ResourceProvisioner.java @@ -32,7 +32,6 @@ import java.math.BigInteger; import java.util.ArrayList; import java.util.List; -import java.util.Optional; public final class ResourceProvisioner { @@ -87,7 +86,6 @@ public ProvisionedResources provisionResources( CompressionAlgorithm.None, BigInteger.valueOf(pinnedProducerCliArgs.messageExpiry()), BigInteger.valueOf(pinnedProducerCliArgs.maxTopicSize()), - Optional.empty(), topicNames.get(0)); } } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/SystemClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/SystemClient.java index 2ac3f0b3a9..6eab11ad08 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/SystemClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/SystemClient.java @@ -22,6 +22,8 @@ import org.apache.iggy.cluster.ClusterMetadata; import org.apache.iggy.system.ClientInfo; import org.apache.iggy.system.ClientInfoDetails; +import org.apache.iggy.system.OptionSpec; +import org.apache.iggy.system.OptionsScope; import org.apache.iggy.system.Stats; import java.util.List; @@ -46,6 +48,18 @@ public interface SystemClient { */ CompletableFuture getClusterMetadata(); + /** + * Describes the option catalog for a resource scope asynchronously. + * + *

This is how a client learns which option keys a create command accepts: a key outside the + * catalog is refused at create, and the binary transports carry only the error code back. A scope + * with no keys yet answers with an empty list. + * + * @param scope the resource whose catalog to describe + * @return A CompletableFuture containing the catalog entries + */ + CompletableFuture> describeOptions(OptionsScope scope); + /** * Gets information about the current client asynchronously. * diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/TopicsClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/TopicsClient.java index fcd8761d66..711bdb7baa 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/TopicsClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/TopicsClient.java @@ -21,12 +21,14 @@ import org.apache.iggy.identifier.StreamId; import org.apache.iggy.identifier.TopicId; +import org.apache.iggy.message.HeaderValue; import org.apache.iggy.topic.CompressionAlgorithm; import org.apache.iggy.topic.Topic; import org.apache.iggy.topic.TopicDetails; import java.math.BigInteger; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -44,7 +46,7 @@ * // Create a topic with 3 partitions and no message expiry * topics.createTopic( * StreamId.of(1L), 3L, CompressionAlgorithm.none(), - * BigInteger.ZERO, BigInteger.ZERO, Optional.empty(), "events") + * BigInteger.ZERO, BigInteger.ZERO, "events") * .thenAccept(details -> System.out.println("Topic created: " + details.name())); * * // List all topics in a stream @@ -92,21 +94,42 @@ public interface TopicsClient { * means messages never expire * @param maxTopicSize maximum topic size in bytes; {@link BigInteger#ZERO} * means unlimited - * @param replicationFactor optional replication factor for the topic; if empty, * the server default is used * @param name the topic name (must be unique within the stream) * @return a {@link CompletableFuture} that completes with the created {@link TopicDetails} * @throws org.apache.iggy.exception.IggyException if the stream does not exist or a * topic with the same name already exists */ + default CompletableFuture createTopic( + StreamId streamId, + Long partitionsCount, + CompressionAlgorithm compressionAlgorithm, + BigInteger messageExpiry, + BigInteger maxTopicSize, + String name) { + return createTopic( + streamId, partitionsCount, compressionAlgorithm, messageExpiry, maxTopicSize, name, Map.of()); + } + + /** + * Creates a topic, carrying option keys that have no parameter of their own. + * + *

{@code options} is keyed by option name and reuses {@link HeaderValue} because options ride + * the user-headers codec. It reaches keys the server catalog gained after this build shipped; a + * parameter above wins on collision, and a key outside the catalog is refused by name. Call + * {@code describeOptions} on the server to see which keys it accepts. + * + * @param options option keys with no parameter of their own + * @return a {@link CompletableFuture} that completes with the created {@link TopicDetails} + */ CompletableFuture createTopic( StreamId streamId, Long partitionsCount, CompressionAlgorithm compressionAlgorithm, BigInteger messageExpiry, BigInteger maxTopicSize, - Optional replicationFactor, - String name); + String name, + Map options); /** * Updates the configuration of an existing topic. @@ -119,19 +142,37 @@ CompletableFuture createTopic( * @param compressionAlgorithm the new compression algorithm * @param messageExpiry the new message expiry in microseconds * @param maxTopicSize the new maximum topic size in bytes - * @param replicationFactor optional new replication factor * @param name the new topic name * @return a {@link CompletableFuture} that completes when the update is done * @throws org.apache.iggy.exception.IggyException if the topic does not exist */ + default CompletableFuture updateTopic( + StreamId streamId, + TopicId topicId, + CompressionAlgorithm compressionAlgorithm, + BigInteger messageExpiry, + BigInteger maxTopicSize, + String name) { + return updateTopic(streamId, topicId, compressionAlgorithm, messageExpiry, maxTopicSize, name, Map.of()); + } + + /** + * Updates a topic, carrying option keys that have no parameter of their own. + * + *

The server refuses any key an update may not change, by name. A key left out keeps its + * current value. + * + * @param options option keys with no parameter of their own + * @return a {@link CompletableFuture} that completes when the update is done + */ CompletableFuture updateTopic( StreamId streamId, TopicId topicId, CompressionAlgorithm compressionAlgorithm, BigInteger messageExpiry, BigInteger maxTopicSize, - Optional replicationFactor, - String name); + String name, + Map options); /** * Deletes a topic and all of its partitions and messages. diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/StreamsTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/StreamsTcpClient.java index 91f37e4a54..77957a5b47 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/StreamsTcpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/StreamsTcpClient.java @@ -23,6 +23,8 @@ import io.netty.util.ReferenceCounted; import org.apache.iggy.client.async.StreamsClient; import org.apache.iggy.identifier.StreamId; +import org.apache.iggy.message.HeaderKey; +import org.apache.iggy.message.HeaderValue; import org.apache.iggy.serde.BytesSerializer; import org.apache.iggy.serde.CommandCode; import org.apache.iggy.stream.StreamBase; @@ -30,6 +32,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.function.Supplier; @@ -105,6 +108,9 @@ public CompletableFuture updateStream(StreamId streamId, String name) { payload.writeBytes(idBytes); payload.writeBytes(BytesSerializer.toBytes(name)); + // Trailing options block. Streams have no catalog keys yet, so the + // server rejects every key; the empty block is the extension point. + payload.writeBytes(BytesSerializer.toBytes(Map.of())); return connection().send(CommandCode.Stream.UPDATE.getValue(), payload).thenAccept(ReferenceCounted::release); } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/SystemTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/SystemTcpClient.java index 8692a2065f..d8b19d1924 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/SystemTcpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/SystemTcpClient.java @@ -26,6 +26,8 @@ import org.apache.iggy.serde.CommandCode; import org.apache.iggy.system.ClientInfo; import org.apache.iggy.system.ClientInfoDetails; +import org.apache.iggy.system.OptionSpec; +import org.apache.iggy.system.OptionsScope; import org.apache.iggy.system.Stats; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -68,6 +70,23 @@ public CompletableFuture getStats() { }); } + @Override + public CompletableFuture> describeOptions(OptionsScope scope) { + var payload = Unpooled.buffer(1).writeByte(scope.asCode()); + + log.debug("Describing {} options", scope); + + return connection() + .send(CommandCode.System.DESCRIBE_OPTIONS.getValue(), payload) + .thenApply(response -> { + try { + return BytesDeserializer.readOptionSpecs(response); + } finally { + response.release(); + } + }); + } + @Override public CompletableFuture getClusterMetadata() { var payload = Unpooled.EMPTY_BUFFER; diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/TopicsTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/TopicsTcpClient.java index 737f0a3410..ff06d86b9b 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/TopicsTcpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/TopicsTcpClient.java @@ -19,10 +19,13 @@ package org.apache.iggy.client.async.tcp; +import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import org.apache.iggy.client.async.TopicsClient; import org.apache.iggy.identifier.StreamId; import org.apache.iggy.identifier.TopicId; +import org.apache.iggy.message.HeaderKey; +import org.apache.iggy.message.HeaderValue; import org.apache.iggy.serde.BytesDeserializer; import org.apache.iggy.serde.BytesSerializer; import org.apache.iggy.serde.CommandCode; @@ -32,19 +35,24 @@ import java.math.BigInteger; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.function.Supplier; import static org.apache.iggy.serde.BytesSerializer.toBytes; -import static org.apache.iggy.serde.BytesSerializer.toBytesAsU64; /** * Async TCP implementation of TopicsClient using Netty for non-blocking I/O. */ public class TopicsTcpClient implements TopicsClient { + private static final String COMPRESSION_ALGORITHM_OPTION = "compression_algorithm"; + private static final String MESSAGE_EXPIRY_OPTION = "message_expiry"; + private static final String MAX_TOPIC_SIZE_OPTION = "max_topic_size"; + private final Supplier connectionSupplier; public TopicsTcpClient(Supplier connectionSupplier) { @@ -79,8 +87,9 @@ public CompletableFuture> getTopics(StreamId streamId) { return connection().send(CommandCode.Topic.GET_ALL.getValue(), payload).thenApply(response -> { try { - List topics = new ArrayList<>(); - while (response.isReadable()) { + var topicsCount = response.readUnsignedIntLE(); + List topics = new ArrayList<>(Math.toIntExact(topicsCount)); + for (long i = 0; i < topicsCount; i++) { topics.add(BytesDeserializer.readTopic(response)); } return topics; @@ -97,19 +106,11 @@ public CompletableFuture createTopic( CompressionAlgorithm compressionAlgorithm, BigInteger messageExpiry, BigInteger maxTopicSize, - Optional replicationFactor, - String name) { + String name, + Map options) { - var streamIdBytes = toBytes(streamId); - var payload = Unpooled.buffer(23 + streamIdBytes.readableBytes() + name.length()); - - payload.writeBytes(streamIdBytes); - payload.writeIntLE(partitionsCount.intValue()); - payload.writeByte(compressionAlgorithm.asCode()); - payload.writeBytes(toBytesAsU64(messageExpiry)); - payload.writeBytes(toBytesAsU64(maxTopicSize)); - payload.writeByte(replicationFactor.orElse((short) 0)); - payload.writeBytes(BytesSerializer.toBytes(name)); + var payload = createTopicPayload( + streamId, partitionsCount, compressionAlgorithm, messageExpiry, maxTopicSize, name, options); return connection().send(CommandCode.Topic.CREATE.getValue(), payload).thenApply(response -> { try { @@ -120,6 +121,53 @@ public CompletableFuture createTopic( }); } + static ByteBuf createTopicPayload( + StreamId streamId, + Long partitionsCount, + CompressionAlgorithm compressionAlgorithm, + BigInteger messageExpiry, + BigInteger maxTopicSize, + String name, + Map options) { + // partitions_count is a fixed field of the command, never an option: + // admission consumes it to compute partition assignments. + var payload = Unpooled.buffer(); + payload.writeBytes(toBytes(streamId)); + payload.writeIntLE(partitionsCount.intValue()); + payload.writeBytes(BytesSerializer.toBytes(name)); + payload.writeBytes(BytesSerializer.toBytes( + createTopicOptions(compressionAlgorithm, messageExpiry, maxTopicSize, options))); + return payload; + } + + private static Map createTopicOptions( + CompressionAlgorithm compressionAlgorithm, + BigInteger messageExpiry, + BigInteger maxTopicSize, + Map extra) { + // Server-default sentinels (compression none, expiry 0, size 0) are + // omitted so the admitting server resolves them from its config. + Map options = new LinkedHashMap<>(); + // Caller keys go in first so a parameter above overwrites one of them: + // the block must not carry a key twice, or the server refuses it whole. + extra.forEach((key, value) -> options.put(HeaderKey.fromString(key), value)); + if (compressionAlgorithm != CompressionAlgorithm.None) { + var compressionName = + switch (compressionAlgorithm) { + case None -> "none"; + case Gzip -> "gzip"; + }; + options.put(HeaderKey.fromString(COMPRESSION_ALGORITHM_OPTION), HeaderValue.fromString(compressionName)); + } + if (messageExpiry.signum() != 0) { + options.put(HeaderKey.fromString(MESSAGE_EXPIRY_OPTION), HeaderValue.fromUint64(messageExpiry)); + } + if (maxTopicSize.signum() != 0) { + options.put(HeaderKey.fromString(MAX_TOPIC_SIZE_OPTION), HeaderValue.fromUint64(maxTopicSize)); + } + return options; + } + @Override public CompletableFuture updateTopic( StreamId streamId, @@ -127,17 +175,18 @@ public CompletableFuture updateTopic( CompressionAlgorithm compressionAlgorithm, BigInteger messageExpiry, BigInteger maxTopicSize, - Optional replicationFactor, - String name) { + String name, + Map options) { var payload = Unpooled.buffer(); payload.writeBytes(toBytes(streamId)); payload.writeBytes(toBytes(topicId)); - payload.writeByte(compressionAlgorithm.asCode()); - payload.writeBytes(toBytesAsU64(messageExpiry)); - payload.writeBytes(toBytesAsU64(maxTopicSize)); - payload.writeByte(replicationFactor.orElse((short) 0)); payload.writeBytes(BytesSerializer.toBytes(name)); + // Settings ride the options block. A default value means the caller did + // not set the key, so it is omitted and the server leaves the topic's + // current value alone. + payload.writeBytes(BytesSerializer.toBytes( + createTopicOptions(compressionAlgorithm, messageExpiry, maxTopicSize, options))); return connection() .send(CommandCode.Topic.UPDATE.getValue(), payload) diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/UsersTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/UsersTcpClient.java index f6e3be1f2d..b6e7ab1940 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/UsersTcpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/UsersTcpClient.java @@ -23,6 +23,8 @@ import org.apache.iggy.IggyVersion; import org.apache.iggy.client.async.UsersClient; import org.apache.iggy.identifier.UserId; +import org.apache.iggy.message.HeaderKey; +import org.apache.iggy.message.HeaderValue; import org.apache.iggy.serde.BytesDeserializer; import org.apache.iggy.serde.CommandCode; import org.apache.iggy.user.IdentityInfo; @@ -34,6 +36,7 @@ import org.slf4j.LoggerFactory; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.function.Supplier; @@ -115,6 +118,9 @@ public CompletableFuture updateUser(UserId userId, Optional userna payload.writeByte(s.asCode()); }, () -> payload.writeByte(0)); + // Trailing options block. Users have no catalog keys yet, so the + // server rejects every key; the empty block is the extension point. + payload.writeBytes(toBytes(Map.of())); return connection().sendAndRelease(CommandCode.User.UPDATE, payload); } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/SystemClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/SystemClient.java index abb7b42601..612c8c1f98 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/SystemClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/SystemClient.java @@ -22,6 +22,8 @@ import org.apache.iggy.cluster.ClusterMetadata; import org.apache.iggy.system.ClientInfo; import org.apache.iggy.system.ClientInfoDetails; +import org.apache.iggy.system.OptionSpec; +import org.apache.iggy.system.OptionsScope; import org.apache.iggy.system.Stats; import java.util.List; @@ -32,6 +34,16 @@ public interface SystemClient { ClusterMetadata getClusterMetadata(); + /** + * Describes the option catalog for a resource scope: every key its create command accepts, with + * the kind, default and bounds of each. + * + *

A key outside the catalog is refused at create and the binary transports carry only the + * error code back, so this is the only way to learn which keys a server knows. A scope with no + * keys yet returns an empty list. + */ + List describeOptions(OptionsScope scope); + ClientInfoDetails getMe(); ClientInfoDetails getClient(Long clientId); diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/TopicsClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/TopicsClient.java index 18936f94f3..57c7bd8f7c 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/TopicsClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/TopicsClient.java @@ -21,12 +21,14 @@ import org.apache.iggy.identifier.StreamId; import org.apache.iggy.identifier.TopicId; +import org.apache.iggy.message.HeaderValue; import org.apache.iggy.topic.CompressionAlgorithm; import org.apache.iggy.topic.Topic; import org.apache.iggy.topic.TopicDetails; import java.math.BigInteger; import java.util.List; +import java.util.Map; import java.util.Optional; public interface TopicsClient { @@ -49,26 +51,37 @@ default TopicDetails createTopic( CompressionAlgorithm compressionAlgorithm, BigInteger messageExpiry, BigInteger maxTopicSize, - Optional replicationFactor, String name) { return createTopic( - StreamId.of(streamId), - partitionsCount, - compressionAlgorithm, - messageExpiry, - maxTopicSize, - replicationFactor, - name); + StreamId.of(streamId), partitionsCount, compressionAlgorithm, messageExpiry, maxTopicSize, name); } + default TopicDetails createTopic( + StreamId streamId, + Long partitionsCount, + CompressionAlgorithm compressionAlgorithm, + BigInteger messageExpiry, + BigInteger maxTopicSize, + String name) { + return createTopic( + streamId, partitionsCount, compressionAlgorithm, messageExpiry, maxTopicSize, name, Map.of()); + } + + /** + * Creates a topic, carrying option keys that have no parameter of their own. + * + *

{@code options} is keyed by option name and reuses {@link HeaderValue} because options ride + * the user-headers codec. It reaches keys the server catalog gained after this build shipped; a + * parameter above wins on collision, and a key outside the catalog is refused by name. + */ TopicDetails createTopic( StreamId streamId, Long partitionsCount, CompressionAlgorithm compressionAlgorithm, BigInteger messageExpiry, BigInteger maxTopicSize, - Optional replicationFactor, - String name); + String name, + Map options); default void updateTopic( Long streamId, @@ -76,26 +89,35 @@ default void updateTopic( CompressionAlgorithm compressionAlgorithm, BigInteger messageExpiry, BigInteger maxTopicSize, - Optional replicationFactor, String name) { updateTopic( - StreamId.of(streamId), - TopicId.of(topicId), - compressionAlgorithm, - messageExpiry, - maxTopicSize, - replicationFactor, - name); + StreamId.of(streamId), TopicId.of(topicId), compressionAlgorithm, messageExpiry, maxTopicSize, name); + } + + default void updateTopic( + StreamId streamId, + TopicId topicId, + CompressionAlgorithm compressionAlgorithm, + BigInteger messageExpiry, + BigInteger maxTopicSize, + String name) { + updateTopic(streamId, topicId, compressionAlgorithm, messageExpiry, maxTopicSize, name, Map.of()); } + /** + * Updates a topic, carrying option keys that have no parameter of their own. + * + *

The server refuses any key an update may not change, by name. A key left out keeps its + * current value. + */ void updateTopic( StreamId streamId, TopicId topicId, CompressionAlgorithm compressionAlgorithm, BigInteger messageExpiry, BigInteger maxTopicSize, - Optional replicationFactor, - String name); + String name, + Map options); default void deleteTopic(Long streamId, Long topicId) { deleteTopic(StreamId.of(streamId), TopicId.of(topicId)); diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/StreamsHttpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/StreamsHttpClient.java index 6929bc2934..74b629fa28 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/StreamsHttpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/StreamsHttpClient.java @@ -25,6 +25,7 @@ import org.apache.iggy.stream.StreamDetails; import tools.jackson.core.type.TypeReference; +import java.math.BigInteger; import java.util.List; import java.util.Optional; @@ -40,7 +41,9 @@ public StreamsHttpClient(InternalHttpClient httpClient) { @Override public Optional getStream(StreamId streamId) { var request = httpClient.prepareGetRequest(STREAMS + "/" + streamId); - return httpClient.executeWithOptionalResponse(request, StreamDetails.class); + return httpClient + .executeWithOptionalResponse(request, HttpStreamDetails.class) + .map(HttpStreamDetails::toStreamDetails); } @Override @@ -70,4 +73,36 @@ public void deleteStream(StreamId streamId) { record CreateStream(String name) {} record UpdateStream(String name) {} + + /** + * The REST shape of a stream with its topics. + * + *

Only the nested topics need mapping: they carry options, which REST reports in one map with a + * per-entry provenance flag rather than as the two blocks the binary protocol sends. See + * {@link TopicsHttpClient.HttpTopic}. + */ + record HttpStreamDetails( + Long id, + BigInteger createdAt, + String name, + String size, + BigInteger messagesCount, + Long topicsCount, + List topics) { + + StreamDetails toStreamDetails() { + return new StreamDetails( + id, + createdAt, + name, + size, + messagesCount, + topicsCount, + topics == null + ? List.of() + : topics.stream() + .map(TopicsHttpClient.HttpTopic::toTopic) + .toList()); + } + } } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/SystemHttpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/SystemHttpClient.java index d238c32f01..1ac0c7232e 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/SystemHttpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/SystemHttpClient.java @@ -22,12 +22,17 @@ import org.apache.iggy.client.blocking.SystemClient; import org.apache.iggy.cluster.ClusterMetadata; import org.apache.iggy.exception.IggyOperationNotSupportedException; +import org.apache.iggy.message.HeaderKind; +import org.apache.iggy.message.HeaderValue; import org.apache.iggy.system.ClientInfo; import org.apache.iggy.system.ClientInfoDetails; +import org.apache.iggy.system.OptionSpec; +import org.apache.iggy.system.OptionsScope; import org.apache.iggy.system.Stats; import tools.jackson.core.type.TypeReference; import java.util.List; +import java.util.Locale; class SystemHttpClient implements SystemClient { @@ -35,6 +40,7 @@ class SystemHttpClient implements SystemClient { private static final String CLUSTER_METADATA = "/cluster/metadata"; private static final String CLIENTS = "/clients"; private static final String PING = "/ping"; + private static final String OPTIONS = "/options"; private final InternalHttpClient httpClient; public SystemHttpClient(InternalHttpClient httpClient) { @@ -53,6 +59,13 @@ public ClusterMetadata getClusterMetadata() { return httpClient.execute(request, ClusterMetadata.class); } + @Override + public List describeOptions(OptionsScope scope) { + var request = httpClient.prepareGetRequest(OPTIONS + "/" + scope.name().toLowerCase(Locale.ROOT)); + List specs = httpClient.execute(request, new TypeReference<>() {}); + return specs.stream().map(HttpOptionSpec::toOptionSpec).toList(); + } + @Override public ClientInfoDetails getMe() { throw new IggyOperationNotSupportedException("getMe", "HTTP"); @@ -75,4 +88,15 @@ public String ping() { var request = httpClient.prepareGetRequest(PING); return httpClient.executeWithStringResponse(request); } + + /** + * The REST shape of a catalog entry: the kind arrives as its lowercase name and the default as a + * JSON array, where the binary transport sends a kind code and raw bytes. Mapping it here keeps + * {@link OptionSpec} the one shape a caller sees on either transport. + */ + record HttpOptionSpec(String key, HeaderKind kind, byte[] defaultValue, String description) { + OptionSpec toOptionSpec() { + return new OptionSpec(key, new HeaderValue(kind, defaultValue), description); + } + } } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/TopicsHttpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/TopicsHttpClient.java index e343058a8d..6c1eec972c 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/TopicsHttpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/TopicsHttpClient.java @@ -22,13 +22,17 @@ import org.apache.iggy.client.blocking.TopicsClient; import org.apache.iggy.identifier.StreamId; import org.apache.iggy.identifier.TopicId; +import org.apache.iggy.message.HeaderValue; +import org.apache.iggy.partition.Partition; import org.apache.iggy.topic.CompressionAlgorithm; import org.apache.iggy.topic.Topic; import org.apache.iggy.topic.TopicDetails; import tools.jackson.core.type.TypeReference; import java.math.BigInteger; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Optional; class TopicsHttpClient implements TopicsClient { @@ -44,13 +48,16 @@ public TopicsHttpClient(InternalHttpClient httpClient) { @Override public Optional getTopic(StreamId streamId, TopicId topicId) { var request = httpClient.prepareGetRequest(STREAMS + "/" + streamId + TOPICS + "/" + topicId); - return httpClient.executeWithOptionalResponse(request, TopicDetails.class); + return httpClient + .executeWithOptionalResponse(request, HttpTopicDetails.class) + .map(HttpTopicDetails::toTopicDetails); } @Override public List getTopics(StreamId streamId) { var request = httpClient.prepareGetRequest(STREAMS + "/" + streamId + TOPICS); - return httpClient.execute(request, new TypeReference<>() {}); + List topics = httpClient.execute(request, new TypeReference<>() {}); + return topics.stream().map(HttpTopic::toTopic).toList(); } @Override @@ -60,13 +67,18 @@ public TopicDetails createTopic( CompressionAlgorithm compressionAlgorithm, BigInteger messageExpiry, BigInteger maxTopicSize, - Optional replicationFactor, - String name) { + String name, + Map options) { var request = httpClient.preparePostRequest( STREAMS + "/" + streamId + TOPICS, new CreateTopic( - partitionsCount, compressionAlgorithm, messageExpiry, maxTopicSize, replicationFactor, name)); - return httpClient.execute(request, new TypeReference<>() {}); + partitionsCount, + compressionAlgorithm, + messageExpiry, + maxTopicSize, + name, + toStringOptions(options))); + return httpClient.execute(request, HttpTopicDetails.class).toTopicDetails(); } @Override @@ -76,11 +88,11 @@ public void updateTopic( CompressionAlgorithm compressionAlgorithm, BigInteger messageExpiry, BigInteger maxTopicSize, - Optional replicationFactor, - String name) { + String name, + Map options) { var request = httpClient.preparePutRequest( STREAMS + "/" + streamId + TOPICS + "/" + topicId, - new UpdateTopic(compressionAlgorithm, messageExpiry, maxTopicSize, replicationFactor, name)); + new UpdateTopic(compressionAlgorithm, messageExpiry, maxTopicSize, name, toStringOptions(options))); httpClient.execute(request); } @@ -90,18 +102,119 @@ public void deleteTopic(StreamId streamId, TopicId topicId) { httpClient.execute(request); } + /** + * Renders option values as the strings the REST body carries them in. + * + *

The binary transports send a typed TLV block, but the JSON body takes a plain string map + * that the server parses by the same rules a config file value goes through, so a typed value + * handed in here is rendered: sending a {@code Uint64} 134217728 and sending "134217728" land + * the same stored value. + */ + private static Map toStringOptions(Map options) { + Map rendered = new LinkedHashMap<>(); + options.forEach((key, value) -> rendered.put(key, value.toStringValue())); + return rendered; + } + + private static Map optionsOf(Map options, boolean explicit) { + if (options == null) { + return Map.of(); + } + Map selected = new LinkedHashMap<>(); + options.forEach((key, option) -> { + if (option.explicit() == explicit) { + selected.put(key, HeaderValue.fromString(option.value())); + } + }); + return Map.copyOf(selected); + } + record CreateTopic( Long partitionsCount, CompressionAlgorithm compressionAlgorithm, BigInteger messageExpiry, BigInteger maxTopicSize, - Optional replicationFactor, - String name) {} + String name, + Map options) {} record UpdateTopic( CompressionAlgorithm compressionAlgorithm, BigInteger messageExpiry, BigInteger maxTopicSize, - Optional replicationFactor, - String name) {} + String name, + Map options) {} + + /** + * One option entry as REST renders it: the value in the string form the create body takes, plus + * the provenance flag that the binary protocol carries as two separate blocks instead. + */ + record HttpOptionValue(String value, boolean explicit) {} + + /** + * The REST shape of a topic. + * + *

It differs from {@link Topic} in one place: REST reports every option in a single map with a + * per-entry {@code explicit} flag, where the binary protocol sends an explicit block and a derived + * block. Splitting here is what lets a caller read {@code options()} and {@code derivedOptions()} + * the same way on either transport. Values arrive in string form, so each is String-kinded rather + * than carrying the kind the server stored. + */ + record HttpTopic( + Long id, + BigInteger createdAt, + String name, + String size, + BigInteger messageExpiry, + CompressionAlgorithm compressionAlgorithm, + BigInteger maxTopicSize, + BigInteger messagesCount, + Long partitionsCount, + Map options) { + + Topic toTopic() { + return new Topic( + id, + createdAt, + name, + size, + messageExpiry, + compressionAlgorithm, + maxTopicSize, + messagesCount, + partitionsCount, + optionsOf(options, true), + optionsOf(options, false)); + } + } + + /** The REST shape of a topic with its partitions. See {@link HttpTopic}. */ + record HttpTopicDetails( + Long id, + BigInteger createdAt, + String name, + String size, + BigInteger messageExpiry, + CompressionAlgorithm compressionAlgorithm, + BigInteger maxTopicSize, + BigInteger messagesCount, + Long partitionsCount, + List partitions, + Map options) { + + TopicDetails toTopicDetails() { + return new TopicDetails( + id, + createdAt, + name, + size, + messageExpiry, + compressionAlgorithm, + maxTopicSize, + messagesCount, + partitionsCount, + partitions == null ? List.of() : partitions, + optionsOf(options, true), + optionsOf(options, false)); + } + } } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/SystemTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/SystemTcpClient.java index 4d00cfa147..59f4336859 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/SystemTcpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/SystemTcpClient.java @@ -23,6 +23,8 @@ import org.apache.iggy.cluster.ClusterMetadata; import org.apache.iggy.system.ClientInfo; import org.apache.iggy.system.ClientInfoDetails; +import org.apache.iggy.system.OptionSpec; +import org.apache.iggy.system.OptionsScope; import org.apache.iggy.system.Stats; import java.util.List; @@ -45,6 +47,11 @@ public ClusterMetadata getClusterMetadata() { return FutureUtil.resolve(delegate.getClusterMetadata()); } + @Override + public List describeOptions(OptionsScope scope) { + return FutureUtil.resolve(delegate.describeOptions(scope)); + } + @Override public ClientInfoDetails getMe() { return FutureUtil.resolve(delegate.getMe()); diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/TopicsTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/TopicsTcpClient.java index 1900cab15b..d81196cde0 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/TopicsTcpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/TopicsTcpClient.java @@ -22,12 +22,14 @@ import org.apache.iggy.client.blocking.TopicsClient; import org.apache.iggy.identifier.StreamId; import org.apache.iggy.identifier.TopicId; +import org.apache.iggy.message.HeaderValue; import org.apache.iggy.topic.CompressionAlgorithm; import org.apache.iggy.topic.Topic; import org.apache.iggy.topic.TopicDetails; import java.math.BigInteger; import java.util.List; +import java.util.Map; import java.util.Optional; final class TopicsTcpClient implements TopicsClient { @@ -55,10 +57,10 @@ public TopicDetails createTopic( CompressionAlgorithm compressionAlgorithm, BigInteger messageExpiry, BigInteger maxTopicSize, - Optional replicationFactor, - String name) { + String name, + Map options) { return FutureUtil.resolve(delegate.createTopic( - streamId, partitionsCount, compressionAlgorithm, messageExpiry, maxTopicSize, replicationFactor, name)); + streamId, partitionsCount, compressionAlgorithm, messageExpiry, maxTopicSize, name, options)); } @Override @@ -68,10 +70,10 @@ public void updateTopic( CompressionAlgorithm compressionAlgorithm, BigInteger messageExpiry, BigInteger maxTopicSize, - Optional replicationFactor, - String name) { + String name, + Map options) { FutureUtil.resolve(delegate.updateTopic( - streamId, topicId, compressionAlgorithm, messageExpiry, maxTopicSize, replicationFactor, name)); + streamId, topicId, compressionAlgorithm, messageExpiry, maxTopicSize, name, options)); } @Override diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/HeaderValue.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/HeaderValue.java index ee4177da04..11a52c5d0e 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/HeaderValue.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/HeaderValue.java @@ -24,6 +24,7 @@ import org.apache.commons.lang3.builder.HashCodeBuilder; import org.apache.iggy.exception.IggyInvalidArgumentException; +import java.math.BigInteger; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; @@ -93,6 +94,15 @@ public static HeaderValue fromUint32(long val) { return new HeaderValue(HeaderKind.Uint32, buffer.array()); } + public static HeaderValue fromUint64(BigInteger val) { + if (val.signum() < 0 || val.bitLength() > 64) { + throw new IggyInvalidArgumentException("Value must be between 0 and 18446744073709551615"); + } + ByteBuffer buffer = ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN); + buffer.putLong(val.longValue()); + return new HeaderValue(HeaderKind.Uint64, buffer.array()); + } + public static HeaderValue fromFloat32(float val) { ByteBuffer buffer = ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN); buffer.putFloat(val); @@ -175,6 +185,14 @@ public long asUint32() { return (ByteBuffer.wrap(value).order(ByteOrder.LITTLE_ENDIAN).getInt() & 0xFFFFFFFFL); } + public BigInteger asUint64() { + if (kind != HeaderKind.Uint64) { + throw new IggyInvalidArgumentException("Header value is not a uint64, kind: " + kind); + } + long raw = ByteBuffer.wrap(value).order(ByteOrder.LITTLE_ENDIAN).getLong(); + return new BigInteger(Long.toUnsignedString(raw)); + } + public float asFloat32() { if (kind != HeaderKind.Float32) { throw new IggyInvalidArgumentException("Header value is not a float32, kind: " + kind); @@ -218,7 +236,13 @@ public int hashCode() { return new HashCodeBuilder(17, 37).append(kind).append(value).toHashCode(); } - private String toStringValue() { + /** + * Renders this value as the string a config file or a REST body carries it in. + * + *

The server parses those strings by the same rules, so a rendered value and the typed one + * it came from resolve to the same stored value. + */ + public String toStringValue() { if (kind == HeaderKind.String) { return asString(); } @@ -234,11 +258,18 @@ private String numericOrRawToString() { case Int16 -> String.valueOf(asInt16()); case Int32 -> String.valueOf(asInt32()); case Int64 -> String.valueOf(asInt64()); + case Float32 -> String.valueOf(asFloat32()); + case Float64 -> String.valueOf(asFloat64()); + default -> unsignedOrRawToString(); + }; + } + + private String unsignedOrRawToString() { + return switch (kind) { case Uint8 -> String.valueOf(asUint8()); case Uint16 -> String.valueOf(asUint16()); case Uint32 -> String.valueOf(asUint32()); - case Float32 -> String.valueOf(asFloat32()); - case Float64 -> String.valueOf(asFloat64()); + case Uint64 -> asUint64().toString(); default -> Base64.getEncoder().encodeToString(value); }; } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesDeserializer.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesDeserializer.java index 623d62b36e..8a1b3c5b35 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesDeserializer.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesDeserializer.java @@ -31,6 +31,7 @@ import org.apache.iggy.consumergroup.ConsumerGroupDetails; import org.apache.iggy.consumergroup.ConsumerGroupMember; import org.apache.iggy.consumeroffset.ConsumerOffsetInfo; +import org.apache.iggy.exception.IggyInvalidArgumentException; import org.apache.iggy.exception.IggyMalformedResponseException; import org.apache.iggy.message.BytesMessageId; import org.apache.iggy.message.HeaderKey; @@ -51,6 +52,7 @@ import org.apache.iggy.system.ClientInfo; import org.apache.iggy.system.ClientInfoDetails; import org.apache.iggy.system.ConsumerGroupInfo; +import org.apache.iggy.system.OptionSpec; import org.apache.iggy.system.Stats; import org.apache.iggy.topic.CompressionAlgorithm; import org.apache.iggy.topic.Topic; @@ -67,6 +69,7 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -80,6 +83,12 @@ public final class BytesDeserializer { private static final int CONSUMER_GROUP_ASSIGNMENT_ENTRY_BYTES = Integer.BYTES; private static final int SEND_CONFIRMATION_BYTES = 3 * Integer.BYTES + Long.BYTES; private static final int MIN_CLUSTER_NODE_BYTES = 18; + // A one-character key (length byte plus a byte of name), a kind byte, and + // empty length-prefixed default and description. + private static final int MIN_OPTION_SPEC_BYTES = 2 + 1 + 4 + 4; + // 50-byte fixed part + a one-character name (the server rejects an empty + // one) + two u32 options-block length prefixes. + private static final int MIN_TOPIC_BYTES = 59; private BytesDeserializer() {} @@ -89,17 +98,22 @@ public static StreamBase readStreamBase(ByteBuf response) { var topicsCount = response.readUnsignedIntLE(); var size = readU64AsBigInteger(response); var messagesCount = readU64AsBigInteger(response); - var nameLength = response.readByte(); + var nameLength = response.readUnsignedByte(); var name = response.readCharSequence(nameLength, StandardCharsets.UTF_8).toString(); + var options = readOptionsBlock(response, "stream options"); - return new StreamBase(streamId, createdAt, name, size.toString(), messagesCount, topicsCount); + return new StreamBase(streamId, createdAt, name, size.toString(), messagesCount, topicsCount, options); } public static StreamDetails readStreamDetails(ByteBuf response) { var streamBase = readStreamBase(response); - List topics = new ArrayList<>(); - while (response.isReadable()) { + // Count-driven: a topic element carries variable-length options + // blocks, so "consume until the buffer ends" no longer delimits it. + int topicsCount = validatedCollectionSize( + streamBase.topicsCount(), response.readableBytes(), MIN_TOPIC_BYTES, "Stream topics count"); + List topics = new ArrayList<>(topicsCount); + for (int i = 0; i < topicsCount; i++) { topics.add(readTopic(response)); } @@ -134,11 +148,12 @@ public static Topic readTopic(ByteBuf response) { var messageExpiry = readU64AsBigInteger(response); var compressionAlgorithmCode = response.readByte(); var maxTopicSize = readU64AsBigInteger(response); - var replicationFactor = response.readByte(); var size = readU64AsBigInteger(response); var messagesCount = readU64AsBigInteger(response); - var nameLength = response.readByte(); + var nameLength = response.readUnsignedByte(); var name = response.readCharSequence(nameLength, StandardCharsets.UTF_8).toString(); + var options = readOptionsBlock(response, "topic explicit options"); + var derivedOptions = readOptionsBlock(response, "topic derived options"); return new Topic( topicId, createdAt, @@ -147,9 +162,10 @@ public static Topic readTopic(ByteBuf response) { messageExpiry, CompressionAlgorithm.fromCode(compressionAlgorithmCode), maxTopicSize, - (short) replicationFactor, messagesCount, - partitionsCount); + partitionsCount, + options, + derivedOptions); } public static ConsumerGroupDetails readConsumerGroupDetails(ByteBuf response) { @@ -177,7 +193,7 @@ public static ConsumerGroup readConsumerGroup(ByteBuf response) { var groupId = response.readUnsignedIntLE(); var partitionsCount = response.readUnsignedIntLE(); var membersCount = response.readUnsignedIntLE(); - var nameLength = response.readByte(); + var nameLength = response.readUnsignedByte(); var name = response.readCharSequence(nameLength, StandardCharsets.UTF_8).toString(); return new ConsumerGroup(groupId, name, partitionsCount, membersCount); } @@ -401,6 +417,36 @@ public static ConsumerGroupInfo readConsumerGroupInfo(ByteBuf response) { return new ConsumerGroupInfo(streamId, topicId, groupId); } + /** + * Reads a {@code DescribeOptions} response. + * + *

Wire format: {@code [count:u32][key_len:u8][key][kind:u8][default_len:u32][default] + * [description_len:u32][description]}*. A scope with no catalog keys answers with a zero count + * rather than an error. + */ + public static List readOptionSpecs(ByteBuf response) { + var count = response.readUnsignedIntLE(); + int specsCount = + validatedCollectionSize(count, response.readableBytes(), MIN_OPTION_SPEC_BYTES, "Option count"); + List specs = new ArrayList<>(specsCount); + for (int i = 0; i < specsCount; i++) { + var keyLength = response.readUnsignedByte(); + if (keyLength > response.readableBytes()) { + throw new IggyMalformedResponseException("Truncated option key at entry " + i); + } + var key = + response.readCharSequence(keyLength, StandardCharsets.UTF_8).toString(); + + var kindCode = response.readUnsignedByte(); + var defaultValue = readU32PrefixedBytes(response, "option default value for '" + key + "'"); + var description = readU32PrefixedString(response, "option description for '" + key + "'"); + + specs.add(new OptionSpec(key, new HeaderValue(HeaderKind.fromCode(kindCode), defaultValue), description)); + } + + return specs; + } + public static ClusterMetadata readClusterMetadata(ByteBuf response) { var name = readU32PrefixedString(response, "cluster name"); var nodesCount = response.readUnsignedIntLE(); @@ -435,6 +481,10 @@ public static UserInfoDetails readUserInfoDetails(ByteBuf response) { if (response.readBoolean()) { var permissions = readPermissions(response); permissionsOptional = Optional.of(permissions); + } else { + // No-permissions marker is u32_le(0): the flag byte above was its + // first byte, skip the remaining three zero bytes. + response.skipBytes(3); } return new UserInfoDetails(userInfo, permissionsOptional); @@ -506,27 +556,104 @@ public static UserInfo readUserInfo(ByteBuf response) { var createdAt = readU64AsBigInteger(response); var statusCode = response.readByte(); var status = UserStatus.fromCode(statusCode); - var usernameLength = response.readByte(); + var usernameLength = response.readUnsignedByte(); var username = response.readCharSequence(usernameLength, StandardCharsets.UTF_8) .toString(); + // Validated and dropped: users have no catalog keys yet, so the server + // refuses every one and the block is always empty. + readOptionsBlock(response, "user options"); return new UserInfo(userId, createdAt, status, username); } public static RawPersonalAccessToken readRawPersonalAccessToken(ByteBuf response) { - var tokenLength = response.readByte(); + var tokenLength = response.readUnsignedByte(); var token = response.readCharSequence(tokenLength, StandardCharsets.UTF_8).toString(); return new RawPersonalAccessToken(token); } public static PersonalAccessTokenInfo readPersonalAccessTokenInfo(ByteBuf response) { - var nameLength = response.readByte(); + var nameLength = response.readUnsignedByte(); var name = response.readCharSequence(nameLength, StandardCharsets.UTF_8).toString(); var expiry = readU64AsBigInteger(response); Optional expiryOptional = expiry.equals(BigInteger.ZERO) ? Optional.empty() : Optional.of(expiry); return new PersonalAccessTokenInfo(name, expiryOptional); } + /** + * Reads a {@code u32}-length-prefixed options block into its entries. + * + *

Keys are always UTF-8 strings; values keep the kind the server sent, so a kind this + * build has no name for is dropped rather than failing the whole response - the wire + * contract forwards unknown value kinds so a mixed-version cluster can round-trip them. + */ + private static Map readOptionsBlock(ByteBuf buffer, String field) { + if (buffer.readableBytes() < Integer.BYTES) { + throw new IggyMalformedResponseException("Missing length prefix for " + field); + } + var optionsLength = buffer.readUnsignedIntLE(); + if (optionsLength > buffer.readableBytes()) { + throw new IggyMalformedResponseException("Length " + optionsLength + " for " + field + + " exceeds remaining payload of " + buffer.readableBytes() + " bytes"); + } + if (optionsLength == 0) { + return Map.of(); + } + + ByteBuf options = buffer.readSlice(toInt(optionsLength)); + Map entries = new LinkedHashMap<>(); + while (options.isReadable()) { + readOptionEntry(options, field, entries); + } + return entries; + } + + private static void readOptionEntry(ByteBuf options, String field, Map entries) { + // The key kind is read and dropped: wire validation already enforces that + // every option key is a UTF-8 string. + readOptionFieldKind(options, field, "key"); + var key = new String(readOptionFieldValue(options, field, "key"), StandardCharsets.UTF_8); + + var valueKindCode = readOptionFieldKind(options, field, "value for '" + key + "'"); + byte[] value = readOptionFieldValue(options, field, "value for '" + key + "'"); + try { + entries.put(key, new HeaderValue(HeaderKind.fromCode(valueKindCode), value)); + } catch (IggyInvalidArgumentException unknownKind) { + // A newer peer's value kind: keep every other entry readable. + } + } + + private static short readOptionFieldKind(ByteBuf options, String field, String what) { + if (options.readableBytes() < 1 + Integer.BYTES) { + throw new IggyMalformedResponseException("Truncated " + what + " header in " + field); + } + return options.readUnsignedByte(); + } + + private static byte[] readOptionFieldValue(ByteBuf options, String field, String what) { + var length = options.readUnsignedIntLE(); + if (length > options.readableBytes()) { + throw new IggyMalformedResponseException("Truncated " + what + " in " + field); + } + byte[] value = newByteArray(length); + options.readBytes(value); + return value; + } + + private static byte[] readU32PrefixedBytes(ByteBuf buffer, String field) { + if (buffer.readableBytes() < Integer.BYTES) { + throw new IggyMalformedResponseException("Missing length prefix for " + field); + } + var length = buffer.readUnsignedIntLE(); + if (length > buffer.readableBytes()) { + throw new IggyMalformedResponseException("Length " + length + " for " + field + + " exceeds remaining payload of " + buffer.readableBytes() + " bytes"); + } + byte[] value = newByteArray(length); + buffer.readBytes(value); + return value; + } + private static String readU32PrefixedString(ByteBuf buffer, String field) { if (buffer.readableBytes() < Integer.BYTES) { throw new IggyMalformedResponseException("Missing length prefix for " + field); diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/CommandCode.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/CommandCode.java index 4c2e56ceac..36ac31266d 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/CommandCode.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/CommandCode.java @@ -31,6 +31,7 @@ enum System implements CommandCode { PING(1), GET_STATS(10), GET_CLUSTER_METADATA(12), + DESCRIBE_OPTIONS(13), GET_ME(20), GET_CLIENT(21), GET_ALL_CLIENTS(22); diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/stream/StreamBase.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/stream/StreamBase.java index 4efc17f35b..16aa3c5895 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/stream/StreamBase.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/stream/StreamBase.java @@ -19,7 +19,23 @@ package org.apache.iggy.stream; +import org.apache.iggy.message.HeaderValue; + import java.math.BigInteger; +import java.util.Map; +/** + * A stream as the server reports it. + * + *

{@code options} carries the creation options, keyed by option name. Streams have no + * catalog keys yet, so it is empty until one lands; the field is read rather than skipped + * so the first key needs no client change. + */ public record StreamBase( - Long id, BigInteger createdAt, String name, String size, BigInteger messagesCount, Long topicsCount) {} + Long id, + BigInteger createdAt, + String name, + String size, + BigInteger messagesCount, + Long topicsCount, + Map options) {} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/system/OptionSpec.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/system/OptionSpec.java new file mode 100644 index 0000000000..9f69010bac --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/system/OptionSpec.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.system; + +import org.apache.iggy.message.HeaderValue; + +/** + * One entry of a resource's option catalog. + * + *

This is the discovery surface for the option keys a create command accepts: a key outside the + * catalog is refused at create, and the binary transports carry only the error code back, so there + * is no other way to learn which keys a given server knows. + * + * @param key the option key the create command accepts + * @param defaultValue the key's default, carrying the kind the server encodes it under, or empty for + * a key with no default. Reuses {@link HeaderValue} because options ride the user-headers codec. + * @param description what the option does, including the bounds its value is checked against + */ +public record OptionSpec(String key, HeaderValue defaultValue, String description) {} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/system/OptionsScope.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/system/OptionsScope.java new file mode 100644 index 0000000000..d3675b91d7 --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/system/OptionsScope.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.system; + +/** + * The resource whose option catalog {@code describeOptions} serves. + */ +public enum OptionsScope { + Topic(1), + Stream(2), + User(3); + + private final int code; + + OptionsScope(int code) { + this.code = code; + } + + public int asCode() { + return code; + } +} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/Topic.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/Topic.java index f6a1ef7cfe..8a88eb88bc 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/Topic.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/Topic.java @@ -19,8 +19,24 @@ package org.apache.iggy.topic; +import org.apache.iggy.message.HeaderValue; + import java.math.BigInteger; +import java.util.Map; +/** + * A topic as the server reports it. + * + *

{@code options} carries the keys the creating client set explicitly; + * {@code derivedOptions} carries the values admission resolved for the keys it did not. + * Both are keyed by option name, and the split is the provenance: a value in + * {@code derivedOptions} would have resolved differently under another server config. + * The fixed fields above repeat three of those keys and stay for compatibility. + * + *

Both transports populate them. The binary protocol carries each value in the kind the server + * stored it under; REST renders values in a readable string form, so over HTTP every value is + * String-kinded. + */ public record Topic( Long id, BigInteger createdAt, @@ -29,6 +45,12 @@ public record Topic( BigInteger messageExpiry, CompressionAlgorithm compressionAlgorithm, BigInteger maxTopicSize, - Short replicationFactor, BigInteger messagesCount, - Long partitionsCount) {} + Long partitionsCount, + Map options, + Map derivedOptions) { + public Topic { + options = options == null ? Map.of() : options; + derivedOptions = derivedOptions == null ? Map.of() : derivedOptions; + } +} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/TopicDetails.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/TopicDetails.java index 1ab20ef688..78dfd34d74 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/TopicDetails.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/TopicDetails.java @@ -19,11 +19,18 @@ package org.apache.iggy.topic; +import org.apache.iggy.message.HeaderValue; import org.apache.iggy.partition.Partition; import java.math.BigInteger; import java.util.List; +import java.util.Map; +/** + * A topic and its partitions, as the server reports them. + * + *

See {@link Topic} for what {@code options} and {@code derivedOptions} carry. + */ public record TopicDetails( Long id, BigInteger createdAt, @@ -32,10 +39,16 @@ public record TopicDetails( BigInteger messageExpiry, CompressionAlgorithm compressionAlgorithm, BigInteger maxTopicSize, - Short replicationFactor, BigInteger messagesCount, Long partitionsCount, - List partitions) { + List partitions, + Map options, + Map derivedOptions) { + public TopicDetails { + options = options == null ? Map.of() : options; + derivedOptions = derivedOptions == null ? Map.of() : derivedOptions; + } + public TopicDetails(Topic topic, List partitions) { this( topic.id(), @@ -45,9 +58,10 @@ public TopicDetails(Topic topic, List partitions) { topic.messageExpiry(), topic.compressionAlgorithm(), topic.maxTopicSize(), - topic.replicationFactor(), topic.messagesCount(), topic.partitionsCount(), - partitions); + partitions, + topic.options(), + topic.derivedOptions()); } } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/TopicOptions.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/TopicOptions.java new file mode 100644 index 0000000000..27e081bb0a --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/TopicOptions.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.topic; + +import org.apache.iggy.message.HeaderValue; + +import java.math.BigInteger; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Typed builder for the topic option keys that have no parameter of their own on + * {@code createTopic}. + * + *

Compression, message expiry and max topic size are named parameters already. The five keys here + * would otherwise have to be spelled as raw strings paired with hand-encoded values, where a typo in + * the key or the wrong kind for the value is only caught by the server refusing the create. + * + *

These are create-time keys: an update refuses them by name, because they describe how a + * partition's storage was laid down. {@code describeOptions} enumerates what a given server accepts. + * + *

{@code
+ * var options = TopicOptions.builder()
+ *         .segmentSize(BigInteger.valueOf(134_217_728))
+ *         .enforceFsync(true)
+ *         .build();
+ * topicsClient.createTopic(streamId, 1L, CompressionAlgorithm.None,
+ *         BigInteger.ZERO, BigInteger.ZERO, "orders", options);
+ * }
+ */ +public final class TopicOptions { + + private TopicOptions() {} + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder { + + private final Map options = new LinkedHashMap<>(); + + private Builder() {} + + /** Per-topic segment size in bytes: a 512-byte multiple within the server's bounds. */ + public Builder segmentSize(BigInteger bytes) { + options.put("segment_size", HeaderValue.fromUint64(bytes)); + return this; + } + + /** Whether writes to this topic's partitions fsync. */ + public Builder enforceFsync(boolean enabled) { + options.put("enforce_fsync", HeaderValue.fromBool(enabled)); + return this; + } + + /** Flush the journal once it holds this many messages. Must be non-zero. */ + public Builder messagesRequiredToSave(long messages) { + options.put("messages_required_to_save", HeaderValue.fromUint32(messages)); + return this; + } + + /** Flush the journal once it holds this many bytes. */ + public Builder sizeOfMessagesRequiredToSave(BigInteger bytes) { + options.put("size_of_messages_required_to_save", HeaderValue.fromUint64(bytes)); + return this; + } + + /** + * Reserve each segment's bytes up front where the filesystem supports it. + * + *

The reservation is real disk and runs inline on the owning shard, so the server caps + * {@code segment_size * partitions_count} at admission. + */ + public Builder preallocateSegments(boolean enabled) { + options.put("preallocate_segments", HeaderValue.fromBool(enabled)); + return this; + } + + /** The keys set on this builder, in the order they were set. */ + public Map build() { + return Map.copyOf(options); + } + } +} diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncClientIntegrationTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncClientIntegrationTest.java index 4f1496f4da..fee52d7c04 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncClientIntegrationTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncClientIntegrationTest.java @@ -91,13 +91,7 @@ void shouldPollMessagesAndVerifyContent() throws Exception { client.streams().createStream(streamName).get(TIMEOUT_SECONDS, TimeUnit.SECONDS); client.topics() .createTopic( - streamId, - 2L, - CompressionAlgorithm.None, - BigInteger.ZERO, - BigInteger.ZERO, - Optional.empty(), - "test-topic") + streamId, 2L, CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, "test-topic") .get(TIMEOUT_SECONDS, TimeUnit.SECONDS); List messages = new ArrayList<>(); @@ -146,13 +140,7 @@ void shouldHandleConcurrentSendsAndPolls() throws Exception { client.streams().createStream(streamName).get(TIMEOUT_SECONDS, TimeUnit.SECONDS); client.topics() .createTopic( - streamId, - 2L, - CompressionAlgorithm.None, - BigInteger.ZERO, - BigInteger.ZERO, - Optional.empty(), - "test-topic") + streamId, 2L, CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, "test-topic") .get(TIMEOUT_SECONDS, TimeUnit.SECONDS); // when — send messages concurrently from multiple threads @@ -209,13 +197,7 @@ void shouldSendAndPollLargeVolume() throws Exception { client.streams().createStream(streamName).get(TIMEOUT_SECONDS, TimeUnit.SECONDS); client.topics() .createTopic( - streamId, - 2L, - CompressionAlgorithm.None, - BigInteger.ZERO, - BigInteger.ZERO, - Optional.empty(), - "test-topic") + streamId, 2L, CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, "test-topic") .get(TIMEOUT_SECONDS, TimeUnit.SECONDS); // when — send messages in concurrent batches diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConnectionPoolAuthTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConnectionPoolAuthTest.java index 17a750792c..bce4d1340f 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConnectionPoolAuthTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConnectionPoolAuthTest.java @@ -37,7 +37,6 @@ import java.math.BigInteger; import java.util.ArrayList; import java.util.List; -import java.util.Optional; import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -162,7 +161,6 @@ void shouldAuthenticatePoolChannelsLazily() throws Exception { CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, - Optional.empty(), "test-topic") .get(5, TimeUnit.SECONDS); @@ -203,7 +201,6 @@ void shouldReAuthenticateStaleChannelsAfterReLogin() throws Exception { CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, - Optional.empty(), "test-topic") .get(5, TimeUnit.SECONDS); diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConsumerGroupsTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConsumerGroupsTest.java index 5a60691cd9..e7dec176bc 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConsumerGroupsTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConsumerGroupsTest.java @@ -82,14 +82,7 @@ public static void setup() throws Exception { client.streams().createStream(TEST_STREAM).get(TIMEOUT_SECONDS, TimeUnit.SECONDS); client.topics() - .createTopic( - STREAM_ID, - 1L, - CompressionAlgorithm.None, - BigInteger.ZERO, - BigInteger.ZERO, - Optional.empty(), - TEST_TOPIC) + .createTopic(STREAM_ID, 1L, CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, TEST_TOPIC) .get(TIMEOUT_SECONDS, TimeUnit.SECONDS); } @@ -268,14 +261,7 @@ void shouldHandleConcurrentGroupCreations() throws Exception { try { client.streams().createStream(streamName).get(TIMEOUT_SECONDS, TimeUnit.SECONDS); client.topics() - .createTopic( - streamId, - 1L, - CompressionAlgorithm.None, - BigInteger.ZERO, - BigInteger.ZERO, - Optional.empty(), - topicName) + .createTopic(streamId, 1L, CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, topicName) .get(TIMEOUT_SECONDS, TimeUnit.SECONDS); // when diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/TopicsTcpClientPayloadTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/TopicsTcpClientPayloadTest.java new file mode 100644 index 0000000000..a58b72ad74 --- /dev/null +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/TopicsTcpClientPayloadTest.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.client.async.tcp; + +import io.netty.buffer.ByteBuf; +import org.apache.iggy.identifier.StreamId; +import org.apache.iggy.message.HeaderKind; +import org.apache.iggy.topic.CompressionAlgorithm; +import org.junit.jupiter.api.Test; + +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.util.LinkedHashMap; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * CreateTopic wire format: + * [stream_id:Identifier][partitions_count:u32_le][name_len:u8][name:N][options TLV to end] + */ +class TopicsTcpClientPayloadTest { + + @Test + void shouldWritePartitionsCountAsFixedFieldBeforeName() { + ByteBuf payload = TopicsTcpClient.createTopicPayload( + StreamId.of(1L), 3L, CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, "orders", Map.of()); + + assertThat(payload.readUnsignedByte()).isEqualTo((short) 1); // numeric identifier kind + assertThat(payload.readUnsignedByte()).isEqualTo((short) 4); // identifier length + assertThat(payload.readUnsignedIntLE()).isEqualTo(1L); + assertThat(payload.readUnsignedIntLE()).isEqualTo(3L); + assertThat(readString(payload)).isEqualTo("orders"); + assertThat(payload.isReadable()).isFalse(); + } + + @Test + void shouldOmitServerDefaultOptions() { + ByteBuf payload = TopicsTcpClient.createTopicPayload( + StreamId.of("my-stream"), + 1L, + CompressionAlgorithm.None, + BigInteger.ZERO, + BigInteger.ZERO, + "events", + Map.of()); + + payload.skipBytes(2 + "my-stream".length()); + assertThat(payload.readUnsignedIntLE()).isEqualTo(1L); + assertThat(readString(payload)).isEqualTo("events"); + assertThat(payload.isReadable()).isFalse(); + } + + @Test + void shouldEncodeExplicitOptionsWithoutPartitionsCount() { + ByteBuf payload = TopicsTcpClient.createTopicPayload( + StreamId.of(1L), + 2L, + CompressionAlgorithm.Gzip, + BigInteger.valueOf(60_000_000L), + BigInteger.valueOf(1024L), + "orders", + Map.of()); + + payload.skipBytes(6); + assertThat(payload.readUnsignedIntLE()).isEqualTo(2L); + readString(payload); + + Map options = readOptions(payload); + assertThat(options).containsOnlyKeys("compression_algorithm", "message_expiry", "max_topic_size"); + assertThat(options.get("compression_algorithm").kind()).isEqualTo(HeaderKind.String); + assertThat(new String(options.get("compression_algorithm").value(), StandardCharsets.UTF_8)) + .isEqualTo("gzip"); + assertThat(options.get("message_expiry").kind()).isEqualTo(HeaderKind.Uint64); + assertThat(options.get("max_topic_size").kind()).isEqualTo(HeaderKind.Uint64); + } + + private static String readString(ByteBuf buffer) { + byte[] bytes = new byte[buffer.readUnsignedByte()]; + buffer.readBytes(bytes); + return new String(bytes, StandardCharsets.UTF_8); + } + + private static Map readOptions(ByteBuf buffer) { + Map options = new LinkedHashMap<>(); + while (buffer.isReadable()) { + buffer.readUnsignedByte(); // key kind + byte[] key = new byte[Math.toIntExact(buffer.readUnsignedIntLE())]; + buffer.readBytes(key); + HeaderKind valueKind = HeaderKind.fromCode(buffer.readUnsignedByte()); + byte[] value = new byte[Math.toIntExact(buffer.readUnsignedIntLE())]; + buffer.readBytes(value); + options.put(new String(key, StandardCharsets.UTF_8), new TypedValue(valueKind, value)); + } + return options; + } + + private record TypedValue(HeaderKind kind, byte[] value) {} +} diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/IntegrationTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/IntegrationTest.java index c41b0f75aa..ecd2f0d81e 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/IntegrationTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/IntegrationTest.java @@ -29,7 +29,6 @@ import java.util.ArrayList; import java.util.List; -import static java.util.Optional.empty; import static org.apache.iggy.TestConstants.STREAM_NAME; import static org.apache.iggy.TestConstants.TOPIC_NAME; @@ -98,7 +97,6 @@ protected void setUpStreamAndTopic() { CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, - empty(), TOPIC_NAME.getName()); } diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/SystemClientBaseTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/SystemClientBaseTest.java index 1e4f096f33..5330397deb 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/SystemClientBaseTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/SystemClientBaseTest.java @@ -21,9 +21,14 @@ import org.apache.iggy.cluster.ClusterNodeRole; import org.apache.iggy.cluster.ClusterNodeStatus; +import org.apache.iggy.message.HeaderKind; +import org.apache.iggy.system.OptionSpec; +import org.apache.iggy.system.OptionsScope; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import java.util.stream.Collectors; + import static org.assertj.core.api.Assertions.assertThat; public abstract class SystemClientBaseTest extends IntegrationTest { @@ -37,6 +42,28 @@ void beforeEachBase() { login(); } + @Test + void shouldDescribeTheTopicOptionCatalog() { + // when + var specs = systemClient.describeOptions(OptionsScope.Topic); + + // then + var byKey = specs.stream().collect(Collectors.toMap(OptionSpec::key, spec -> spec)); + assertThat(byKey).containsKeys("segment_size", "enforce_fsync"); + var segmentSize = byKey.get("segment_size"); + assertThat(segmentSize.defaultValue().kind()).isEqualTo(HeaderKind.Uint64); + assertThat(segmentSize.defaultValue().value()).isNotEmpty(); + assertThat(segmentSize.description()).isNotBlank(); + } + + @Test + void shouldDescribeEmptyCatalogsForScopesWithoutKeys() { + // Streams and users have no catalog keys yet, so the scope answers with + // an empty list rather than an error. + assertThat(systemClient.describeOptions(OptionsScope.Stream)).isEmpty(); + assertThat(systemClient.describeOptions(OptionsScope.User)).isEmpty(); + } + @Test void shouldGetStats() { // when diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/TopicsClientBaseTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/TopicsClientBaseTest.java index 80ea9914e5..9e6657ecbc 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/TopicsClientBaseTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/TopicsClientBaseTest.java @@ -20,15 +20,17 @@ package org.apache.iggy.client.blocking; import org.apache.iggy.identifier.TopicId; +import org.apache.iggy.message.HeaderValue; import org.apache.iggy.topic.CompressionAlgorithm; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.math.BigInteger; +import java.util.Map; -import static java.util.Optional.empty; import static org.apache.iggy.TestConstants.STREAM_NAME; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; public abstract class TopicsClientBaseTest extends IntegrationTest { @@ -46,7 +48,7 @@ void beforeEachBase() { void shouldCreateAndDeleteTopic() { // when var topicDetails = topicsClient.createTopic( - STREAM_NAME, 1L, CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, empty(), "test-topic"); + STREAM_NAME, 1L, CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, "test-topic"); var topicId = TopicId.of("test-topic"); var topicOptional = topicsClient.getTopic(STREAM_NAME, topicId); @@ -66,7 +68,7 @@ void shouldCreateAndDeleteTopic() { void shouldUpdateTopic() { // given var topic = topicsClient.createTopic( - STREAM_NAME, 1L, CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, empty(), "test-topic"); + STREAM_NAME, 1L, CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, "test-topic"); // when topicsClient.updateTopic( @@ -75,7 +77,6 @@ void shouldUpdateTopic() { CompressionAlgorithm.None, BigInteger.valueOf(5000), BigInteger.ZERO, - empty(), "new-name"); // then @@ -85,6 +86,43 @@ void shouldUpdateTopic() { assertThat(updatedTopic.messageExpiry()).isEqualTo(BigInteger.valueOf(5000)); } + @Test + void shouldCarryOptionKeysWithNoParameterOfTheirOwn() { + // when + var created = topicsClient.createTopic( + STREAM_NAME, + 1L, + CompressionAlgorithm.None, + BigInteger.ZERO, + BigInteger.ZERO, + "options-topic", + Map.of("enforce_fsync", HeaderValue.fromString("true"))); + + // then + var topic = topicsClient.getTopic(STREAM_NAME, TopicId.of(created.id())).orElseThrow(); + // Asserted through the string rendering rather than the kind: the binary + // transport reports the key's canonical Bool while REST renders values as + // strings, and both mean the same setting. + assertThat(topic.options()).containsKey("enforce_fsync"); + assertThat(topic.options().get("enforce_fsync").toStringValue()).isEqualTo("true"); + // Keys the client left alone are resolved by admission and reported apart. + assertThat(topic.derivedOptions()).containsKey("max_topic_size"); + assertThat(topic.derivedOptions()).doesNotContainKey("enforce_fsync"); + } + + @Test + void shouldRejectAnOptionKeyOutsideTheCatalog() { + assertThatThrownBy(() -> topicsClient.createTopic( + STREAM_NAME, + 1L, + CompressionAlgorithm.None, + BigInteger.ZERO, + BigInteger.ZERO, + "bad-options-topic", + Map.of("not_a_real_option", HeaderValue.fromString("1")))) + .isInstanceOf(RuntimeException.class); + } + @Test void shouldReturnEmptyForNonExistingTopic() { // when diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/TlsConnectionTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/TlsConnectionTest.java index e4605229ed..f284ad5c81 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/TlsConnectionTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/TlsConnectionTest.java @@ -43,7 +43,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import static java.util.Optional.empty; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -131,14 +130,7 @@ void sendAndReceiveOverTlsShouldWork() { assertThat(stream).isNotNull(); client.topics() - .createTopic( - streamId, - 1L, - CompressionAlgorithm.None, - BigInteger.ZERO, - BigInteger.ZERO, - empty(), - topicName); + .createTopic(streamId, 1L, CompressionAlgorithm.None, BigInteger.ZERO, BigInteger.ZERO, topicName); List messages = List.of(Message.of("tls-message-1"), Message.of("tls-message-2"), Message.of("tls-message-3")); diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/TopicsTcpClientTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/TopicsTcpClientTest.java index 6e5e80f14d..cce977599c 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/TopicsTcpClientTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/TopicsTcpClientTest.java @@ -21,6 +21,18 @@ import org.apache.iggy.client.blocking.IggyBaseClient; import org.apache.iggy.client.blocking.TopicsClientBaseTest; +import org.apache.iggy.identifier.TopicId; +import org.apache.iggy.message.HeaderKind; +import org.apache.iggy.message.HeaderValue; +import org.apache.iggy.topic.CompressionAlgorithm; +import org.junit.jupiter.api.Test; + +import java.math.BigInteger; +import java.util.Map; + +import static org.apache.iggy.TestConstants.STREAM_NAME; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; class TopicsTcpClientTest extends TopicsClientBaseTest { @@ -28,4 +40,35 @@ class TopicsTcpClientTest extends TopicsClientBaseTest { protected IggyBaseClient getClient() { return TcpClientFactory.create(serverHost(), serverTcpPort()); } + + @Test + void shouldStoreAnOptionInItsCanonicalKind() { + // Create admission re-encodes the block from its own parse, so a value + // arrives back in the key's catalog kind whatever kind it was sent as. + // Only the binary transport carries kinds: REST renders values as strings. + var created = topicsClient.createTopic( + STREAM_NAME, + 1L, + CompressionAlgorithm.None, + BigInteger.ZERO, + BigInteger.ZERO, + "canonical-kind-topic", + Map.of("enforce_fsync", HeaderValue.fromString("true"))); + + var topic = topicsClient.getTopic(STREAM_NAME, TopicId.of(created.id())).orElseThrow(); + assertThat(topic.options().get("enforce_fsync").kind()).isEqualTo(HeaderKind.Bool); + } + + @Test + void shouldRejectAnOptionKeyOutsideTheCatalog() { + assertThatThrownBy(() -> topicsClient.createTopic( + STREAM_NAME, + 1L, + CompressionAlgorithm.None, + BigInteger.ZERO, + BigInteger.ZERO, + "bad-options-topic", + Map.of("not_a_real_option", HeaderValue.fromString("1")))) + .isInstanceOf(RuntimeException.class); + } } diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/message/HeaderValueTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/message/HeaderValueTest.java index b7e550e42f..4cbcabdaf7 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/message/HeaderValueTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/message/HeaderValueTest.java @@ -26,6 +26,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; +import java.math.BigInteger; import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; @@ -143,6 +144,30 @@ void fromUint32ReturnsExpectedHeaderValueWhenValueIsValid() { assertThat(headerValue.value()).isEqualTo(new byte[] {-1, -1, -1, -1}); } + @ParameterizedTest + @ValueSource(strings = {"-1", "18446744073709551616"}) + void fromUint64ThrowsIggyInvalidArgumentExceptionWhenValueOutOfBounds(String value) { + assertThatThrownBy(() -> HeaderValue.fromUint64(new BigInteger(value))) + .isInstanceOf(IggyInvalidArgumentException.class); + } + + @Test + void fromUint64ReturnsExpectedHeaderValueWhenValueIsValid() { + var maxUint64 = new BigInteger("18446744073709551615"); + var headerValue = HeaderValue.fromUint64(maxUint64); + + assertThat(headerValue.kind()).isEqualTo(HeaderKind.Uint64); + assertThat(headerValue.value()).isEqualTo(new byte[] {-1, -1, -1, -1, -1, -1, -1, -1}); + assertThat(headerValue.asUint64()).isEqualTo(maxUint64); + } + + @Test + void fromUint64EncodesLittleEndian() { + var headerValue = HeaderValue.fromUint64(BigInteger.valueOf(258)); + + assertThat(headerValue.value()).isEqualTo(new byte[] {2, 1, 0, 0, 0, 0, 0, 0}); + } + @Test void fromFloat32ReturnsExpectedHeaderValue() { var headerValue = HeaderValue.fromFloat32(123.4f); diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/BytesDeserializerTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/BytesDeserializerTest.java index ee835f5b39..d36cdb5b88 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/BytesDeserializerTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/BytesDeserializerTest.java @@ -28,6 +28,7 @@ import org.apache.iggy.exception.IggyMalformedResponseException; import org.apache.iggy.message.HeaderKey; import org.apache.iggy.message.HeaderKind; +import org.apache.iggy.message.HeaderValue; import org.apache.iggy.system.CacheMetricsKey; import org.apache.iggy.topic.CompressionAlgorithm; import org.apache.iggy.user.UserStatus; @@ -37,6 +38,7 @@ import java.math.BigInteger; import java.nio.charset.StandardCharsets; import java.util.HexFormat; +import java.util.Map; import static org.apache.iggy.serde.BytesDeserializer.readClientInfo; import static org.apache.iggy.serde.BytesDeserializer.readClientInfoDetails; @@ -87,11 +89,48 @@ private static void writeTopicData(ByteBuf buffer) { writeU64(buffer, BigInteger.ZERO); // message expiry buffer.writeByte(CompressionAlgorithm.None.asCode()); // compression writeU64(buffer, BigInteger.valueOf(10000)); // max topic size - buffer.writeByte(1); // replication factor writeU64(buffer, BigInteger.valueOf(500)); // size writeU64(buffer, BigInteger.valueOf(50)); // messages count buffer.writeByte(4); // name length buffer.writeBytes("test".getBytes()); + writeOptionsBlock( + buffer, Map.of(HeaderKey.fromString("max_topic_size"), HeaderValue.fromUint64(BigInteger.TEN))); + writeOptionsBlock(buffer, Map.of(HeaderKey.fromString("segment_size"), HeaderValue.fromString("1 GiB"))); + } + + private static void writeTopicDataWithUnknownOptionKind(ByteBuf buffer) { + buffer.writeIntLE(10); + writeU64(buffer, BigInteger.valueOf(1000)); + buffer.writeIntLE(4); + writeU64(buffer, BigInteger.ZERO); + buffer.writeByte(CompressionAlgorithm.None.asCode()); + writeU64(buffer, BigInteger.valueOf(10000)); + writeU64(buffer, BigInteger.valueOf(500)); + writeU64(buffer, BigInteger.valueOf(50)); + buffer.writeByte(4); + buffer.writeBytes("test".getBytes()); + + ByteBuf options = Unpooled.buffer(); + var known = BytesSerializer.toBytes( + Map.of(HeaderKey.fromString("max_topic_size"), HeaderValue.fromUint64(BigInteger.TEN))); + options.writeBytes(known); + // Hand-rolled entry: a string key with a value kind no `HeaderKind` names. + options.writeByte(HeaderKind.String.asCode()); + options.writeIntLE("from_the_future".length()); + options.writeBytes("from_the_future".getBytes()); + options.writeByte(200); + options.writeIntLE(2); + options.writeBytes(new byte[] {1, 2}); + buffer.writeIntLE(options.readableBytes()); + buffer.writeBytes(options); + + writeOptionsBlock(buffer, Map.of()); + } + + private static void writeOptionsBlock(ByteBuf buffer, Map options) { + var optionsTlv = BytesSerializer.toBytes(options); + buffer.writeIntLE(optionsTlv.readableBytes()); + buffer.writeBytes(optionsTlv); } private static void writePartitionData(ByteBuf buffer) { @@ -163,6 +202,7 @@ void shouldDeserializeStreamBase() { writeU64(buffer, BigInteger.valueOf(100)); // messages count buffer.writeByte(11); // name length buffer.writeBytes("test-stream".getBytes(StandardCharsets.UTF_8)); + writeOptionsBlock(buffer, Map.of()); // when var stream = readStreamBase(buffer); @@ -186,6 +226,7 @@ void shouldDeserializeStreamDetails() { writeU64(buffer, BigInteger.valueOf(100)); buffer.writeByte(6); buffer.writeBytes("stream".getBytes()); + writeOptionsBlock(buffer, Map.of()); // Write one topic writeTopicData(buffer); @@ -214,6 +255,23 @@ void shouldDeserializeTopic() { assertThat(topic.id()).isEqualTo(10L); assertThat(topic.name()).isEqualTo("test"); assertThat(topic.partitionsCount()).isEqualTo(4L); + assertThat(topic.options()).containsOnlyKeys("max_topic_size"); + assertThat(topic.options().get("max_topic_size").kind()).isEqualTo(HeaderKind.Uint64); + assertThat(topic.derivedOptions()).containsOnlyKeys("segment_size"); + assertThat(new String(topic.derivedOptions().get("segment_size").value())) + .isEqualTo("1 GiB"); + } + + @Test + void shouldKeepReadableOptionsWhenOneValueKindIsUnknown() { + // The wire contract forwards value kinds a client build has no name + // for, so one of them must not cost the whole response. + ByteBuf buffer = Unpooled.buffer(); + writeTopicDataWithUnknownOptionKind(buffer); + + var topic = readTopic(buffer); + + assertThat(topic.options()).containsOnlyKeys("max_topic_size"); } @Test @@ -803,6 +861,7 @@ void shouldDeserializeUserInfo() { buffer.writeByte(UserStatus.Active.asCode()); // status buffer.writeByte(4); // username length buffer.writeBytes("user".getBytes()); + writeOptionsBlock(buffer, Map.of()); // when var userInfo = readUserInfo(buffer); @@ -823,7 +882,8 @@ void shouldDeserializeUserInfoDetailsWithoutPermissions() { buffer.writeByte(UserStatus.Active.asCode()); buffer.writeByte(5); buffer.writeBytes("admin".getBytes()); - buffer.writeBoolean(false); // no permissions + writeOptionsBlock(buffer, Map.of()); + buffer.writeIntLE(0); // no-permissions marker: u32_le(0) // when var userInfoDetails = readUserInfoDetails(buffer); @@ -842,6 +902,7 @@ void shouldDeserializeUserInfoDetailsWithPermissions() { buffer.writeByte(UserStatus.Active.asCode()); buffer.writeByte(5); buffer.writeBytes("admin".getBytes()); + writeOptionsBlock(buffer, Map.of()); buffer.writeBoolean(true); // has permissions buffer.writeIntLE(10); // permissions length (ignored but required) // Write global permissions (10 booleans) diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/OptionsBlockGoldenVectorTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/OptionsBlockGoldenVectorTest.java new file mode 100644 index 0000000000..ad327861cb --- /dev/null +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/OptionsBlockGoldenVectorTest.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.serde; + +import io.netty.buffer.ByteBuf; +import org.apache.iggy.message.HeaderKey; +import org.apache.iggy.message.HeaderValue; +import org.junit.jupiter.api.Test; + +import java.math.BigInteger; +import java.util.LinkedHashMap; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * The cross-SDK golden vector for an options block. + * + *

Rust pins the identical bytes in {@code core/binary_protocol/src/primitives/options.rs}, as do + * the Node and Go SDKs. Round-tripping a block through this SDK's own decoder proves nothing about + * interoperability; these bytes are the contract, and a change to the TLV layout has to break every + * copy of them together. + * + *

{@code enforce_fsync} (a one-byte {@code Bool}) and {@code segment_size} (an eight-byte + * {@code Uint64}) cover both value widths, in the sorted key order every encoder has to produce. + */ +class OptionsBlockGoldenVectorTest { + + private static final byte[] GOLDEN_OPTIONS_BLOCK = { + 2, 13, 0, 0, 0, 'e', 'n', 'f', 'o', 'r', 'c', 'e', '_', 'f', 's', 'y', 'n', 'c', 3, 1, 0, 0, 0, 1, 2, 12, 0, 0, + 0, 's', 'e', 'g', 'm', 'e', 'n', 't', '_', 's', 'i', 'z', 'e', 12, 8, 0, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0 + }; + + @Test + void shouldEncodeTheCrossSdkGoldenVector() { + Map options = new LinkedHashMap<>(); + options.put(HeaderKey.fromString("enforce_fsync"), HeaderValue.fromBool(true)); + options.put(HeaderKey.fromString("segment_size"), HeaderValue.fromUint64(BigInteger.valueOf(1_073_741_824L))); + + ByteBuf encoded = BytesSerializer.toBytes(options); + + byte[] bytes = new byte[encoded.readableBytes()]; + encoded.readBytes(bytes); + assertThat(bytes).isEqualTo(GOLDEN_OPTIONS_BLOCK); + } +} diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/stream/StreamDetailsTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/stream/StreamDetailsTest.java index a0e1adf0da..e8bc7fb986 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/stream/StreamDetailsTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/stream/StreamDetailsTest.java @@ -25,13 +25,14 @@ import java.math.BigInteger; import java.util.List; +import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; class StreamDetailsTest { @Test void constructorWithStreamBaseCreatesExpectedStreamDetails() { - var base = new StreamBase(10L, BigInteger.valueOf(500L), "name", "size", BigInteger.ZERO, 1L); + var base = new StreamBase(10L, BigInteger.valueOf(500L), "name", "size", BigInteger.ZERO, 1L, Map.of()); var topics = List.of(new Topic( 1L, BigInteger.ZERO, @@ -40,9 +41,10 @@ void constructorWithStreamBaseCreatesExpectedStreamDetails() { BigInteger.TEN, CompressionAlgorithm.None, BigInteger.ONE, - (short) 2, BigInteger.ZERO, - 2L)); + 2L, + Map.of(), + Map.of())); var streamDetails = new StreamDetails(base, topics); diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/topic/TopicDetailsTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/topic/TopicDetailsTest.java index b605f90108..49eabd9e99 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/topic/TopicDetailsTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/topic/TopicDetailsTest.java @@ -24,6 +24,7 @@ import java.math.BigInteger; import java.util.List; +import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; @@ -38,9 +39,10 @@ void constructorWithTopicCreatesTopicDetailsWithExpectedValues() { BigInteger.valueOf(10000L), CompressionAlgorithm.Gzip, BigInteger.TWO, - (short) 12, BigInteger.ZERO, - 1L); + 1L, + Map.of(), + Map.of()); var partitions = List.of(new Partition(1L, BigInteger.TEN, 2L, BigInteger.ZERO, "size", BigInteger.ONE)); var topicDetails = new TopicDetails(topic, partitions); @@ -52,7 +54,6 @@ void constructorWithTopicCreatesTopicDetailsWithExpectedValues() { assertThat(topicDetails.messageExpiry()).isEqualTo(BigInteger.valueOf(10000L)); assertThat(topicDetails.compressionAlgorithm()).isEqualTo(CompressionAlgorithm.Gzip); assertThat(topicDetails.maxTopicSize()).isEqualTo(BigInteger.TWO); - assertThat(topicDetails.replicationFactor()).isEqualTo((short) 12); assertThat(topicDetails.messagesCount()).isEqualTo(BigInteger.ZERO); assertThat(topicDetails.partitionsCount()).isEqualTo(1L); assertThat(topicDetails.partitions()).isEqualTo(partitions); diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/topic/TopicOptionsTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/topic/TopicOptionsTest.java new file mode 100644 index 0000000000..e7153f65f8 --- /dev/null +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/topic/TopicOptionsTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.topic; + +import org.apache.iggy.message.HeaderKind; +import org.junit.jupiter.api.Test; + +import java.math.BigInteger; + +import static org.assertj.core.api.Assertions.assertThat; + +class TopicOptionsTest { + + @Test + void shouldEmitOnlyTheKeysThatWereSet() { + var options = TopicOptions.builder().enforceFsync(true).build(); + + assertThat(options).containsOnlyKeys("enforce_fsync"); + assertThat(options.get("enforce_fsync").kind()).isEqualTo(HeaderKind.Bool); + assertThat(options.get("enforce_fsync").value()).containsExactly(1); + } + + @Test + void shouldEncodeEveryKeyInItsCatalogKind() { + var options = TopicOptions.builder() + .segmentSize(BigInteger.valueOf(134_217_728)) + .enforceFsync(false) + .messagesRequiredToSave(1024) + .sizeOfMessagesRequiredToSave(BigInteger.valueOf(1_048_576)) + .preallocateSegments(true) + .build(); + + assertThat(options) + .containsOnlyKeys( + "segment_size", + "enforce_fsync", + "messages_required_to_save", + "size_of_messages_required_to_save", + "preallocate_segments"); + assertThat(options.get("segment_size").kind()).isEqualTo(HeaderKind.Uint64); + assertThat(options.get("messages_required_to_save").kind()).isEqualTo(HeaderKind.Uint32); + assertThat(options.get("size_of_messages_required_to_save").kind()).isEqualTo(HeaderKind.Uint64); + assertThat(options.get("preallocate_segments").kind()).isEqualTo(HeaderKind.Bool); + // Little-endian, so the low byte of 128 MiB leads and the high bytes are zero. + assertThat(options.get("segment_size").value()).containsExactly(0, 0, 0, 8, 0, 0, 0, 0); + } +} diff --git a/foreign/node/src/e2e/tcp.topic.e2e.ts b/foreign/node/src/e2e/tcp.topic.e2e.ts index c5239c9461..43c09f341d 100644 --- a/foreign/node/src/e2e/tcp.topic.e2e.ts +++ b/foreign/node/src/e2e/tcp.topic.e2e.ts @@ -35,8 +35,7 @@ describe('e2e -> topic', async () => { name: topicName, partitionCount: 0, compressionAlgorithm: 1, - messageExpiry: 0n, - replicationFactor: 1 + messageExpiry: 0n }); assert.ok(TOPIC); }); diff --git a/foreign/node/src/wire/command-set.ts b/foreign/node/src/wire/command-set.ts index a7789cb7f8..1588306d6b 100644 --- a/foreign/node/src/wire/command-set.ts +++ b/foreign/node/src/wire/command-set.ts @@ -68,6 +68,7 @@ import { deletePartition } from './partition/delete-partition.command.js'; import { deleteSegments } from './segment/delete-segments.command.js'; +import { describeOptions } from './system/describe-options.command.js'; import { getStats } from './system/get-stats.command.js'; import { ping } from './system/ping.command.js'; @@ -190,7 +191,8 @@ type MessageAPI = ReturnType; const systemAPI = (c: ClientProvider) => ({ ping: ping(c), - getStats: getStats(c) + getStats: getStats(c), + describeOptions: describeOptions(c) }); type SystemAPI = ReturnType; diff --git a/foreign/node/src/wire/command.code.ts b/foreign/node/src/wire/command.code.ts index 2b63a552c5..972f00e332 100644 --- a/foreign/node/src/wire/command.code.ts +++ b/foreign/node/src/wire/command.code.ts @@ -23,6 +23,7 @@ export const COMMAND_CODE = { GetStats: 10, GetSnapshot: 11, // @TODO GET_SNAPSHOT_FILE_CODE: u32 = 11 GetClusterMetadata: 12, + DescribeOptions: 13, GetMe: 20, GetClient: 21, GetClients: 22, diff --git a/foreign/node/src/wire/options.utils.test.ts b/foreign/node/src/wire/options.utils.test.ts new file mode 100644 index 0000000000..423e8577ff --- /dev/null +++ b/foreign/node/src/wire/options.utils.test.ts @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { + serializeOptions, + deserializePrefixedOptions +} from './options.utils.js'; +import { HeaderValue } from './message/header.utils.js'; + +const prefixed = (block: Buffer): Buffer => { + const length = Buffer.alloc(4); + length.writeUInt32LE(block.length, 0); + return Buffer.concat([length, block]); +}; + +/** + * The cross-SDK golden vector for an options block. + * + * Rust pins the identical bytes in `core/binary_protocol/src/primitives/options.rs`, + * as do the Go and Java SDKs. Round-tripping through this SDK's own decoder proves + * nothing about interoperability; these bytes are the contract. + */ +const GOLDEN_OPTIONS_BLOCK = Buffer.from([ + 2, 13, 0, 0, 0, + ...Buffer.from('enforce_fsync'), + 3, 1, 0, 0, 0, 1, + 2, 12, 0, 0, 0, + ...Buffer.from('segment_size'), + 12, 8, 0, 0, 0, + 0, 0, 0, 64, 0, 0, 0, 0 +]); + +describe('serializeOptions', () => { + it('encodes the cross-SDK golden vector byte for byte', () => { + const encoded = serializeOptions([ + { key: 'enforce_fsync', value: HeaderValue.Bool(true) }, + { key: 'segment_size', value: HeaderValue.Uint64(1_073_741_824n) } + ]); + + assert.deepEqual(encoded, GOLDEN_OPTIONS_BLOCK); + }); +}); + +describe('deserializePrefixedOptions', () => { + it('reads a whole block and reports the bytes it consumed', () => { + const block = prefixed(serializeOptions([ + { key: 'segment_size', value: HeaderValue.Uint64(1_048_576n) } + ])); + + const { bytesRead, options } = deserializePrefixedOptions(block); + + assert.equal(bytesRead, block.length); + assert.deepEqual(options, { segment_size: 1_048_576n }); + }); + + it('rejects a block whose declared length runs past the payload', () => { + // `subarray` clamps instead of throwing, so without the bounds check the + // truncated value comes back as raw bytes through the forward-compat catch + // and `bytesRead` over-reports, shifting every later field. + const block = prefixed(serializeOptions([ + { key: 'segment_size', value: HeaderValue.Uint64(1_048_576n) } + ])); + + assert.throws( + () => deserializePrefixedOptions(block.subarray(0, block.length - 4)), + /overruns the payload/ + ); + }); +}); diff --git a/foreign/node/src/wire/options.utils.ts b/foreign/node/src/wire/options.utils.ts new file mode 100644 index 0000000000..46e4a82cea --- /dev/null +++ b/foreign/node/src/wire/options.utils.ts @@ -0,0 +1,173 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +import { HeaderKind } from './message/header.type.js'; +import { + serializeHeaders, + deserializeHeaderValue, + HeaderKeyFactory, + type HeaderValue, + type ParsedHeaderValue, +} from './message/header.utils.js'; + +/** Maximum number of key-value entries in one options block. */ +export const MAX_OPTIONS = 1024; + +/** + * Maximum total byte length of an encoded options block. + * + * Mirrors the Rust `MAX_OPTIONS_BYTES`, which in turn mirrors the user-headers + * budget: options ride that codec and inherit its limit. + */ +export const MAX_OPTIONS_BYTES = 100 * 1000; + +/** + * Key and value length bound, inherited from the header-field codec rather + * than being an options-specific rule (`serializeHeaders` enforces the same + * range on the way out). + */ +const MAX_HEADER_FIELD_LENGTH = 255; + +/** A resource option entry: UTF-8 string key with a typed value. */ +export type OptionEntry = { + key: string, + value: HeaderValue +}; + +/** Deserialized options block keyed by option name. */ +export type ParsedOptions = Record; + +/** Result of deserializing a length-prefixed options block. */ +export type OptionsDeserialized = { + /** Number of bytes consumed, length prefix included */ + bytesRead: number, + /** Deserialized options */ + options: ParsedOptions +}; + +/** + * Keeps the last entry for each key, preserving first-seen order. + * + * A block carrying a key twice is refused whole by wire validation, so callers + * that append their own entries ahead of the typed ones rely on this to let the + * typed value win. + */ +export const dedupeOptions = (options: OptionEntry[]): OptionEntry[] => { + const byKey = new Map(); + for (const entry of options) + byKey.set(entry.key, entry); + return [...byKey.values()]; +}; + +/** + * Serializes resource options into a TLV block. + * Reuses the user-headers TLV encoding: each field is + * `[kind:u8][len:u32_le][bytes]`, alternating key, value. + * Empty options serialize to zero bytes. + * + * @param options - Option entries to serialize + * @returns Serialized options block + * @throws Error if an options constraint is violated + */ +export const serializeOptions = (options: OptionEntry[]): Buffer => { + if (options.length > MAX_OPTIONS) + throw new Error( + `Options block has ${options.length} entries, exceeds maximum ${MAX_OPTIONS}`); + + // No key-length check here: `serializeHeaders` already bounds every field + // to 1..=255, so an options-specific cap would only duplicate it. + const block = serializeHeaders(options.map(({ key, value }) => + ({ key: HeaderKeyFactory.String(key), value }))); + + if (block.length > MAX_OPTIONS_BYTES) + throw new Error( + `Options block is ${block.length} bytes, exceeds maximum ${MAX_OPTIONS_BYTES}`); + + return block; +}; + +/** + * Deserializes a bare options TLV block spanning `[pos, end)`. + * + * @param p - Buffer containing the options block + * @param pos - Starting position of the block + * @param end - End position of the block (exclusive) + * @returns Deserialized options keyed by option name + * @throws Error if a key is not a string or the block is malformed + */ +export const deserializeOptions = ( + p: Buffer, pos = 0, end = p.length +): ParsedOptions => { + const options: ParsedOptions = {}; + while (pos < end) { + const keyKind = p.readUInt8(pos); + if (keyKind !== HeaderKind.String) + throw new Error(`Option key kind ${keyKind} is not a string`); + const keyLength = p.readUInt32LE(pos + 1); + if (keyLength < 1 || keyLength > MAX_HEADER_FIELD_LENGTH) + throw new Error( + `Invalid option key length: ${keyLength}, ` + + `must be between 1 and ${MAX_HEADER_FIELD_LENGTH}`); + if (pos + 5 + keyLength > end) + throw new Error('Option key overruns the block'); + const key = p.subarray(pos + 5, pos + 5 + keyLength).toString(); + pos += 5 + keyLength; + + const valueKind = p.readUInt8(pos); + const valueLength = p.readUInt32LE(pos + 1); + if (pos + 5 + valueLength > end) + throw new Error(`Option value for key '${key}' overruns the block`); + const valueBytes = p.subarray(pos + 5, pos + 5 + valueLength); + pos += 5 + valueLength; + + let value: ParsedHeaderValue; + try { + value = deserializeHeaderValue(valueKind, valueBytes); + } catch { + // Unknown value kinds stay raw bytes, mirroring the wire + // forward-compatibility contract for options. + value = valueBytes; + } + options[key] = value; + } + return options; +}; + +/** + * Deserializes a `u32_le`-length-prefixed options block at `pos`. + * + * @param p - Buffer containing `[options_len:u32_le][options TLV]` + * @param pos - Starting position of the length prefix + * @returns Bytes consumed (prefix included) and deserialized options + */ +export const deserializePrefixedOptions = ( + p: Buffer, pos = 0 +): OptionsDeserialized => { + const length = p.readUInt32LE(pos); + const end = pos + 4 + length; + // Without this, `subarray` clamps a truncated block silently: a known-kind + // value throws inside `deserializeHeaderValue`, the forward-compat catch + // swallows it and hands back raw bytes, and `bytesRead` over-reports so every + // later field decodes from the wrong offset. + if (end > p.length) + throw new Error( + `Options block overruns the payload: ${length} bytes declared at ${pos}, ` + + `${p.length - pos - 4} available`); + const options = deserializeOptions(p, pos + 4, end); + return { bytesRead: 4 + length, options }; +}; diff --git a/foreign/node/src/wire/stream/stream.utils.ts b/foreign/node/src/wire/stream/stream.utils.ts index f767b57458..c7e3f019e6 100644 --- a/foreign/node/src/wire/stream/stream.utils.ts +++ b/foreign/node/src/wire/stream/stream.utils.ts @@ -17,6 +17,7 @@ // import { toDate } from '../serialize.utils.js'; +import { deserializePrefixedOptions, type ParsedOptions } from '../options.utils.js'; /** * Stream information returned from the server. @@ -33,7 +34,9 @@ export type Stream = { /** Total number of messages in the stream */ messagesCount: bigint, /** Stream creation timestamp */ - createdAt: Date + createdAt: Date, + /** Options the client explicitly sent at create */ + options: ParsedOptions } /** @@ -65,11 +68,13 @@ export const deserializeToStream = (r: Buffer, pos = 0): StreamDeserialized => { const messagesCount = r.readBigUint64LE(pos + 24); const nameLength = r.readUInt8(pos + 32); const name = r.subarray(pos + 33, pos + 33 + nameLength).toString(); + const { bytesRead: optionsBytes, options } = + deserializePrefixedOptions(r, pos + 33 + nameLength); return { - bytesRead: 33 + nameLength, + bytesRead: 33 + nameLength + optionsBytes, data: { - id, name, topicsCount, messagesCount, sizeBytes, createdAt + id, name, topicsCount, messagesCount, sizeBytes, createdAt, options } }; }; diff --git a/foreign/node/src/wire/system/describe-options.command.ts b/foreign/node/src/wire/system/describe-options.command.ts new file mode 100644 index 0000000000..70707f89c6 --- /dev/null +++ b/foreign/node/src/wire/system/describe-options.command.ts @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +import type { CommandResponse } from '../../client/index.js'; +import { COMMAND_CODE } from '../command.code.js'; +import { wrapCommand } from '../command.utils.js'; + +/** Resource whose option catalog the server serves. */ +export const OptionsScope = { + Topic: 1, + Stream: 2, + User: 3 +} as const; + +export type OptionsScope = typeof OptionsScope[keyof typeof OptionsScope]; + +/** + * One catalog entry: the key a create command accepts, the kind the server + * encodes its default under, that default, and what the option does. + * + * `kind` is this key's canonical kind: what the server encodes its default + * under, and what a value set at create is stored as whatever kind it was sent + * in, since create admission re-encodes the block from its own parse. An update + * stores the client's bytes verbatim and is the exception. + */ +export type OptionSpec = { + key: string, + kind: number, + defaultValue: Buffer, + description: string +}; + +export type DescribeOptions = { + scope: OptionsScope +}; + +const deserializeDescribeOptions = (b: Buffer): OptionSpec[] => { + const count = b.readUInt32LE(0); + let position = 4; + const specs: OptionSpec[] = []; + for (let i = 0; i < count; i++) { + const keyLength = b.readUInt8(position); + position += 1; + if (position + keyLength > b.length) + throw new Error(`Option key overruns the response at offset ${position}`); + const key = b.subarray(position, position + keyLength).toString(); + position += keyLength; + + const kind = b.readUInt8(position); + position += 1; + + const defaultLength = b.readUInt32LE(position); + position += 4; + if (position + defaultLength > b.length) + throw new Error(`Default value for '${key}' overruns the response`); + const defaultValue = Buffer.from( + b.subarray(position, position + defaultLength) + ); + position += defaultLength; + + const descriptionLength = b.readUInt32LE(position); + position += 4; + if (position + descriptionLength > b.length) + throw new Error(`Description for '${key}' overruns the response`); + const description = b.subarray( + position, position + descriptionLength + ).toString(); + position += descriptionLength; + + specs.push({ key, kind, defaultValue, description }); + } + return specs; +}; + +export const DESCRIBE_OPTIONS = { + code: COMMAND_CODE.DescribeOptions, + + serialize: ({ scope }: DescribeOptions) => { + return Buffer.from([scope]); + }, + + deserialize: (r: CommandResponse): OptionSpec[] => + deserializeDescribeOptions(r.data) +}; + +export const describeOptions = wrapCommand( + DESCRIBE_OPTIONS +); diff --git a/foreign/node/src/wire/system/index.ts b/foreign/node/src/wire/system/index.ts index 5decb6ef48..3326db31a7 100644 --- a/foreign/node/src/wire/system/index.ts +++ b/foreign/node/src/wire/system/index.ts @@ -16,5 +16,6 @@ // under the License. // +export * from './describe-options.command.js'; export * from './get-stats.command.js'; export * from './ping.command.js'; diff --git a/foreign/node/src/wire/topic/create-topic.command.test.ts b/foreign/node/src/wire/topic/create-topic.command.test.ts index 9c362efa6b..4235d91bfb 100644 --- a/foreign/node/src/wire/topic/create-topic.command.test.ts +++ b/foreign/node/src/wire/topic/create-topic.command.test.ts @@ -19,6 +19,8 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; import { CREATE_TOPIC } from './create-topic.command.js'; +import { deserializeOptions } from '../options.utils.js'; +import { HeaderValue } from '../message/header.utils.js'; describe('CreateTopic', () => { @@ -30,17 +32,87 @@ describe('CreateTopic', () => { partitionCount: 1, compressionAlgorithm: 1, // 1 = None, 2 = Gzip messageExpiry: 0n, - maxTopicSize: 0n, - replicationFactor: 1 + maxTopicSize: 0n }; - it('serialize 1 name into buffer', () => { + // TLV field: [kind:u8][len:u32_le][bytes] + const tlvSize = (bytes: number) => 1 + 4 + bytes; + const identifierSize = 1 + 1 + 4; // numeric stream id + const fixedSize = identifierSize + 4 + 1; // + partitions_count + name_len + + it('serialize name and default options into buffer', () => { + // Server-default sentinels are omitted, leaving an empty options block. assert.deepEqual( CREATE_TOPIC.serialize(t1).length, - 6 + 4 + 1 + 8 + 8 + 1 + 1 + t1.name.length + fixedSize + t1.name.length + ); + }); + + it('serialize partitionCount as a fixed u32 before the name', () => { + const t = { ...t1, partitionCount: 7 }; + const b = CREATE_TOPIC.serialize(t); + assert.equal(b.readUInt32LE(identifierSize), 7); + assert.equal(b.readUInt8(identifierSize + 4), t.name.length); + assert.equal(b.subarray(fixedSize).toString(), t.name); + }); + + it('serialize non-default options into buffer', () => { + const t = { + ...t1, + compressionAlgorithm: 2, + messageExpiry: 42n, + maxTopicSize: 1024n + }; + assert.deepEqual( + CREATE_TOPIC.serialize(t).length, + fixedSize + t1.name.length + + tlvSize('compression_algorithm'.length) + tlvSize('gzip'.length) + + tlvSize('message_expiry'.length) + tlvSize(8) + + tlvSize('max_topic_size'.length) + tlvSize(8) ); }); + it('serialize segment and save-trigger options into buffer', () => { + const t = { + ...t1, + segmentSize: 1048576n, + enforceFsync: true, + messagesRequiredToSave: 1000, + sizeOfMessagesRequiredToSave: 4096n, + preallocateSegments: false + }; + assert.deepEqual( + CREATE_TOPIC.serialize(t).length, + fixedSize + t1.name.length + + tlvSize('segment_size'.length) + tlvSize(8) + + tlvSize('enforce_fsync'.length) + tlvSize(1) + + tlvSize('messages_required_to_save'.length) + tlvSize(4) + + tlvSize('size_of_messages_required_to_save'.length) + tlvSize(8) + + tlvSize('preallocate_segments'.length) + tlvSize(1) + ); + }); + + it('serialize caller-supplied option keys, typed fields winning', () => { + const t = { + ...t1, + maxTopicSize: 4096n, + options: [ + { key: 'enforce_fsync', value: HeaderValue.Bool(true) }, + // The typed field covers this key, so the caller's entry is dropped: + // a duplicate key makes the server refuse the whole block. + { key: 'max_topic_size', value: HeaderValue.String('1 GiB') } + ] + }; + + const b = CREATE_TOPIC.serialize(t); + // The create payload runs its options block to the end, unprefixed. + const options = deserializeOptions(b, fixedSize + t.name.length); + + assert.deepEqual(Object.keys(options).sort(), ['enforce_fsync', 'max_topic_size']); + assert.equal(options.enforce_fsync, true); + assert.equal(options.max_topic_size, 4096n); + }); + it('throw on name < 1', () => { const t = { ...t1, name: '' }; assert.throws( @@ -62,20 +134,6 @@ describe('CreateTopic', () => { ); }); - it('throw on replication_factor < 1', () => { - const t = { ...t1, replicationFactor: 0 }; - assert.throws( - () => CREATE_TOPIC.serialize(t), - ); - }); - - it('throw on replication_factor > 255', () => { - const t = { ...t1, replicationFactor: 257 }; - assert.throws( - () => CREATE_TOPIC.serialize(t), - ); - }); - it('accept compressionAlgorithm = 2 (gzip)', () => { const t = { ...t1, compressionAlgorithm: 2 }; assert.doesNotThrow( diff --git a/foreign/node/src/wire/topic/create-topic.command.ts b/foreign/node/src/wire/topic/create-topic.command.ts index 297e369e3b..8da4fec9b3 100644 --- a/foreign/node/src/wire/topic/create-topic.command.ts +++ b/foreign/node/src/wire/topic/create-topic.command.ts @@ -20,8 +20,11 @@ import type { CommandResponse } from '../../client/client.type.js'; import { serializeIdentifier, type Id } from '../identifier.utils.js'; import { wrapCommand } from '../command.utils.js'; import { COMMAND_CODE } from '../command.code.js'; +import { dedupeOptions, serializeOptions, type OptionEntry } from '../options.utils.js'; +import { HeaderValue } from '../message/header.utils.js'; import { isValidCompressionAlgorithm, CompressionAlgorithm, + compressionAlgorithmName, deserializeTopic, type Topic, type CompressionAlgorithm as CompressionAlgorithmT @@ -44,13 +47,30 @@ export type CreateTopic = { messageExpiry?: bigint, /** Maximum topic size in bytes (0 = unlimited) */ maxTopicSize?: bigint, - /** Replication factor (1-255) */ - replicationFactor?: number + /** Segment size in bytes: 512-byte multiple between 1 MiB and 1 GiB */ + segmentSize?: bigint, + /** Fsync every write instead of leaving it to the page cache */ + enforceFsync?: boolean, + /** Message count that triggers a save (must be non-zero) */ + messagesRequiredToSave?: number, + /** Accumulated message bytes that trigger a save */ + sizeOfMessagesRequiredToSave?: bigint, + /** Preallocate segment files when the topic is created */ + preallocateSegments?: boolean, + /** + * Option keys with no field of their own, for a key the server catalog gained + * after this build shipped. A field above wins on collision, since the block + * must not carry a key twice. Call `describeOptions` for the keys a server + * accepts. + */ + options?: OptionEntry[] }; /** * Create topic command definition. * Creates a new topic within a stream. + * Layout: `[stream_id][partitions_count:u32_le][name_len:u8][name]` + * followed by the options TLV block running to the end of the payload. */ export const CREATE_TOPIC = { code: COMMAND_CODE.CreateTopic, @@ -62,32 +82,75 @@ export const CREATE_TOPIC = { compressionAlgorithm = CompressionAlgorithm.None, messageExpiry = 0n, maxTopicSize = 0n, - replicationFactor = 1 + segmentSize, + enforceFsync, + messagesRequiredToSave, + sizeOfMessagesRequiredToSave, + preallocateSegments, + options: extraOptions = [] }: CreateTopic ) => { // Topic ID is now auto-assigned by the server, not sent in the protocol const streamIdentifier = serializeIdentifier(streamId); const bName = Buffer.from(name) - if (replicationFactor < 1 || replicationFactor > 255) - throw new Error('Topic replication factor should be between 1 and 255'); if (bName.length < 1 || bName.length > 255) throw new Error('Topic name should be between 1 and 255 bytes'); if(!isValidCompressionAlgorithm(compressionAlgorithm)) throw new Error(`createTopic: invalid compressionAlgorithm (${compressionAlgorithm})`); - const b = Buffer.allocUnsafe(4 + 1 + 8 + 8 + 1 + 1); + // partitions_count rides the command's own fixed field, never the + // options block: the server rejects it as an unsupported option key. + // Server-default sentinels (expiry 0, size 0, compression none) and + // unset optionals are omitted so the server resolves them from its own + // config and reports them back as derived options. + // Caller keys first so a typed field below overwrites one of them. + const options: OptionEntry[] = [...extraOptions]; + if (compressionAlgorithm !== CompressionAlgorithm.None) + options.push({ + key: 'compression_algorithm', + value: HeaderValue.String(compressionAlgorithmName(compressionAlgorithm)) + }); + if (messageExpiry !== 0n) + options.push({ + key: 'message_expiry', value: HeaderValue.Uint64(messageExpiry) + }); + if (maxTopicSize !== 0n) + options.push({ + key: 'max_topic_size', value: HeaderValue.Uint64(maxTopicSize) + }); + if (segmentSize !== undefined) + options.push({ + key: 'segment_size', value: HeaderValue.Uint64(segmentSize) + }); + if (enforceFsync !== undefined) + options.push({ + key: 'enforce_fsync', value: HeaderValue.Bool(enforceFsync) + }); + if (messagesRequiredToSave !== undefined) + options.push({ + key: 'messages_required_to_save', + value: HeaderValue.Uint32(messagesRequiredToSave) + }); + if (sizeOfMessagesRequiredToSave !== undefined) + options.push({ + key: 'size_of_messages_required_to_save', + value: HeaderValue.Uint64(sizeOfMessagesRequiredToSave) + }); + if (preallocateSegments !== undefined) + options.push({ + key: 'preallocate_segments', + value: HeaderValue.Bool(preallocateSegments) + }); + const b = Buffer.allocUnsafe(4 + 1); b.writeUInt32LE(partitionCount, 0); - b.writeUInt8(compressionAlgorithm, 4); - b.writeBigUInt64LE(messageExpiry, 5); // 0 is unlimited - b.writeBigUInt64LE(maxTopicSize, 13); // optional, 0 is null - b.writeUInt8(replicationFactor, 21); // must be > 0 - b.writeUInt8(bName.length, 22); + b.writeUInt8(bName.length, 4); return Buffer.concat([ streamIdentifier, b, bName, + serializeOptions(dedupeOptions(options)), ]); }, diff --git a/foreign/node/src/wire/topic/topic.utils.test.ts b/foreign/node/src/wire/topic/topic.utils.test.ts index 41753f2dbb..0e012c5a15 100644 --- a/foreign/node/src/wire/topic/topic.utils.test.ts +++ b/foreign/node/src/wire/topic/topic.utils.test.ts @@ -19,27 +19,68 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; import { deserializeBaseTopic } from './topic.utils.js'; +import { serializeOptions } from '../options.utils.js'; +import { HeaderValue } from '../message/header.utils.js'; describe('deserializeBaseTopic', () => { - it('uses the current message-expiry then compression wire layout', () => { - const data = Buffer.alloc(52); - data.writeUInt32LE(7, 0); - data.writeBigUInt64LE(1_710_000_000_000n, 4); - data.writeUInt32LE(3, 12); - data.writeBigUInt64LE(86_400_000_000n, 16); - data.writeUInt8(2, 24); - data.writeBigUInt64LE(1_000_000n, 25); - data.writeUInt8(3, 33); - data.writeBigUInt64LE(4096n, 34); - data.writeBigUInt64LE(12n, 42); - data.writeUInt8(1, 50); - data.write('t', 51); + it('uses the 50-byte fixed layout with two options blocks', () => { + const fixed = Buffer.alloc(51); + fixed.writeUInt32LE(7, 0); + fixed.writeBigUInt64LE(1_710_000_000_000n, 4); + fixed.writeUInt32LE(3, 12); + fixed.writeBigUInt64LE(86_400_000_000n, 16); + fixed.writeUInt8(2, 24); + fixed.writeBigUInt64LE(1_000_000n, 25); + fixed.writeBigUInt64LE(4096n, 33); + fixed.writeBigUInt64LE(12n, 41); + fixed.writeUInt8(1, 49); + fixed.write('t', 50); + + const explicit = serializeOptions([ + { key: 'message_expiry', value: HeaderValue.Uint64(86_400_000_000n) } + ]); + const derived = serializeOptions([ + { key: 'compression_algorithm', value: HeaderValue.String('gzip') }, + { key: 'max_topic_size', value: HeaderValue.Uint64(1_000_000n) } + ]); + const prefix = (block: Buffer) => { + const len = Buffer.alloc(4); + len.writeUInt32LE(block.length, 0); + return Buffer.concat([len, block]); + }; + const data = Buffer.concat([fixed, prefix(explicit), prefix(derived)]); const { bytesRead, data: topic } = deserializeBaseTopic(data); assert.equal(bytesRead, data.length); + assert.equal(topic.id, 7); + assert.equal(topic.partitionsCount, 3); assert.equal(topic.messageExpiry, 86_400_000_000n); assert.equal(topic.compressionAlgorithm, 2); assert.equal(topic.maxTopicSize, 1_000_000n); + assert.equal(topic.sizeBytes, 4096n); + assert.equal(topic.messagesCount, 12n); + assert.equal(topic.name, 't'); + assert.deepEqual(topic.options, { message_expiry: 86_400_000_000n }); + assert.deepEqual(topic.derivedOptions, { + compression_algorithm: 'gzip', + max_topic_size: 1_000_000n + }); + }); + + it('skips empty options blocks', () => { + const fixed = Buffer.alloc(51); + fixed.writeUInt32LE(1, 0); + fixed.writeUInt32LE(2, 12); + fixed.writeUInt8(1, 49); + fixed.write('t', 50); + const data = Buffer.concat([fixed, Buffer.alloc(8)]); + + const { bytesRead, data: topic } = deserializeBaseTopic(data); + + assert.equal(bytesRead, data.length); + assert.equal(topic.partitionsCount, 2); + assert.deepEqual(topic.options, {}); + assert.deepEqual(topic.derivedOptions, {}); }); }); diff --git a/foreign/node/src/wire/topic/topic.utils.ts b/foreign/node/src/wire/topic/topic.utils.ts index 9384ff5ee2..6fae7daa32 100644 --- a/foreign/node/src/wire/topic/topic.utils.ts +++ b/foreign/node/src/wire/topic/topic.utils.ts @@ -18,6 +18,7 @@ import { toDate } from '../serialize.utils.js'; import type { ValueOf } from '../../type.utils.js'; +import { deserializePrefixedOptions, type ParsedOptions } from '../options.utils.js'; /** * Basic topic information without partition details. @@ -37,12 +38,14 @@ export type BaseTopic = { messageExpiry: bigint, /** Maximum topic size in bytes (0 = unlimited) */ maxTopicSize: bigint, - /** Replication factor */ - replicationFactor: number /** Total size of the topic in bytes */ sizeBytes: bigint, /** Total number of messages in the topic */ messagesCount: bigint, + /** Options the client explicitly sent at create */ + options: ParsedOptions, + /** Options resolved from server defaults at create */ + derivedOptions: ParsedOptions, }; /** @@ -113,28 +116,45 @@ export const isValidCompressionAlgorithm = (ca: number): ca is CompressionAlgori Object.values(CompressionAlgorithm).includes(ca); /** - * Deserializes a base topic from a buffer. + * Maps a compression algorithm code to its wire option name. + * + * @param ca - Compression algorithm code + * @returns Option value string ('none' or 'gzip') + */ +export const compressionAlgorithmName = (ca: CompressionAlgorithm): string => + ca === CompressionAlgorithm.Gzip ? 'gzip' : 'none'; + +/** + * Deserializes a topic header from a buffer. + * Layout: 50 fixed bytes, name, then two length-prefixed options blocks + * (client-explicit and server-derived). * * @param p - Buffer containing serialized topic data * @param pos - Starting position in the buffer * @returns Object with bytes read and deserialized topic data */ export const deserializeBaseTopic = (p: Buffer, pos = 0): BaseTopicSerialized => { + const start = pos; const id = p.readUInt32LE(pos); const createdAt = toDate(p.readBigUint64LE(pos + 4)); const partitionsCount = p.readUInt32LE(pos + 12); const messageExpiry = p.readBigUInt64LE(pos + 16); const compressionAlgorithm = p.readUInt8(pos + 24); const maxTopicSize = p.readBigUInt64LE(pos + 25); - const replicationFactor = p.readUInt8(pos + 33); - const sizeBytes = p.readBigUInt64LE(pos + 34); - const messagesCount = p.readBigUInt64LE(pos + 42); + const sizeBytes = p.readBigUInt64LE(pos + 33); + const messagesCount = p.readBigUInt64LE(pos + 41); + + const nameLength = p.readUInt8(pos + 49); + const name = p.subarray(pos + 50, pos + 50 + nameLength).toString(); + pos += 50 + nameLength; - const nameLength = p.readUInt8(pos + 50); - const name = p.subarray(pos + 51, pos + 51 + nameLength).toString(); + const explicit = deserializePrefixedOptions(p, pos); + pos += explicit.bytesRead; + const derived = deserializePrefixedOptions(p, pos); + pos += derived.bytesRead; return { - bytesRead: 4 + 8 + 4 + 1 + 8 + 8 + 1 + 8 + 8 + 1 + nameLength, + bytesRead: pos - start, data: { id, name, @@ -142,10 +162,11 @@ export const deserializeBaseTopic = (p: Buffer, pos = 0): BaseTopicSerialized => partitionsCount, compressionAlgorithm, maxTopicSize, - replicationFactor, messageExpiry, messagesCount, sizeBytes, + options: explicit.options, + derivedOptions: derived.options, } } }; @@ -175,6 +196,9 @@ export const deserializePartition = (p: Buffer, pos = 0): PartitionSerialized => /** * Deserializes a topic with partitions from a buffer. + * Partition elements are count-driven from the topic's partitionsCount: + * the header's variable-length options blocks make greedy consumption + * ambiguous. * * @param p - Buffer containing serialized topic data * @param pos - Starting position in the buffer @@ -189,8 +213,7 @@ export const deserializeTopic = (p: Buffer, pos = 0): TopicSerialized => { const { bytesRead, data } = deserializeBaseTopic(p, pos); pos += bytesRead; const partitions = []; - const end = p.length; - while (pos < end) { + for (let i = 0; i < data.partitionsCount; i++) { const { bytesRead, data } = deserializePartition(p, pos); partitions.push(data); pos += bytesRead; @@ -200,18 +223,20 @@ export const deserializeTopic = (p: Buffer, pos = 0): TopicSerialized => { /** - * Deserializes multiple topics from a buffer. + * Deserializes a GetTopics response: `[topics_count:u32_le]` prefix followed + * by count-driven topic headers (no partition details). * * @param p - Buffer containing serialized topics data * @param pos - Starting position in the buffer * @returns Array of deserialized topics */ export const deserializeTopics = (p: Buffer, pos = 0): Topic[] => { + const topicsCount = p.readUInt32LE(pos); + pos += 4; const topics = []; - const len = p.length; - while (pos < len) { - const { bytesRead, data } = deserializeTopic(p, pos); - topics.push(data); + for (let i = 0; i < topicsCount; i++) { + const { bytesRead, data } = deserializeBaseTopic(p, pos); + topics.push({ ...data, partitions: [] }); pos += bytesRead; } return topics; diff --git a/foreign/node/src/wire/topic/update-topic.command.ts b/foreign/node/src/wire/topic/update-topic.command.ts index 8aa2df20e8..159d8686db 100644 --- a/foreign/node/src/wire/topic/update-topic.command.ts +++ b/foreign/node/src/wire/topic/update-topic.command.ts @@ -17,12 +17,15 @@ // import { serializeIdentifier, type Id } from '../identifier.utils.js'; +import { dedupeOptions, serializeOptions, type OptionEntry } from '../options.utils.js'; +import { HeaderValue } from '../message/header.utils.js'; import { deserializeVoidResponse } from '../../client/client.utils.js'; import { wrapCommand } from '../command.utils.js'; import { COMMAND_CODE } from '../command.code.js'; import { type CompressionAlgorithm as CompressionAlgorithmT, CompressionAlgorithm, + compressionAlgorithmName, isValidCompressionAlgorithm } from './topic.utils.js'; @@ -43,8 +46,11 @@ export type UpdateTopic = { messageExpiry?: bigint, /** Maximum topic size in bytes (0 = unlimited) */ maxTopicSize?: bigint, - /** Replication factor (1-255) */ - replicationFactor?: number, + /** + * Option keys with no field of their own. The server refuses any key an update + * may not change, by name; a key left out keeps its current value. + */ + options?: OptionEntry[] }; /** @@ -61,7 +67,7 @@ export const UPDATE_TOPIC = { compressionAlgorithm = CompressionAlgorithm.None, messageExpiry = 0n, maxTopicSize = 0n, - replicationFactor = 1, + options: extraOptions = [] }: UpdateTopic) => { const streamIdentifier = serializeIdentifier(streamId); const topicIdentifier = serializeIdentifier(topicId); @@ -70,20 +76,37 @@ export const UPDATE_TOPIC = { if (bName.length < 1 || bName.length > 255) throw new Error('Topic name should be between 1 and 255 bytes'); if(!isValidCompressionAlgorithm(compressionAlgorithm)) - throw new Error(`createTopic: invalid compressionAlgorithm (${compressionAlgorithm})`); + throw new Error(`updateTopic: invalid compressionAlgorithm (${compressionAlgorithm})`); - const b = Buffer.allocUnsafe(8 + 8 + 1 + 1 + 1); - b.writeUInt8(compressionAlgorithm, 0); - b.writeBigUInt64LE(messageExpiry, 1); // 0 is unlimited ??? - b.writeBigUInt64LE(maxTopicSize, 9); // optional, 0 is null - b.writeUInt8(replicationFactor, 17); // must be > 0 - b.writeUInt8(bName.length, 18); + // Settings ride the options block. A default value means the caller did not + // set the key, so it is omitted and the server leaves the current value be. + // Caller keys first so a typed field below overwrites one of them. + const options: OptionEntry[] = [...extraOptions]; + if (compressionAlgorithm !== CompressionAlgorithm.None) + options.push({ + key: 'compression_algorithm', + value: HeaderValue.String(compressionAlgorithmName(compressionAlgorithm)) + }); + if (messageExpiry !== 0n) + options.push({ + key: 'message_expiry', + value: HeaderValue.Uint64(messageExpiry) + }); + if (maxTopicSize !== 0n) + options.push({ + key: 'max_topic_size', + value: HeaderValue.Uint64(maxTopicSize) + }); + + const b = Buffer.allocUnsafe(1); + b.writeUInt8(bName.length, 0); return Buffer.concat([ streamIdentifier, topicIdentifier, b, bName, + serializeOptions(dedupeOptions(options)), ]); }, diff --git a/foreign/node/src/wire/user/user.utils.ts b/foreign/node/src/wire/user/user.utils.ts index a12c69410c..cab94001d5 100644 --- a/foreign/node/src/wire/user/user.utils.ts +++ b/foreign/node/src/wire/user/user.utils.ts @@ -18,6 +18,7 @@ import { toDate } from '../serialize.utils.js'; import { deserializePermissions, type UserPermissions } from './permissions.utils.js'; +import { deserializePrefixedOptions, type ParsedOptions } from '../options.utils.js'; /** * Basic user information without permissions. @@ -30,7 +31,9 @@ export type BaseUser = { /** User status (Active/Inactive) */ status: string, /** Username */ - userName: string + userName: string, + /** Options the client explicitly sent at create */ + options: ParsedOptions }; /** @@ -88,14 +91,17 @@ export const deserializeBaseUser = (p: Buffer, pos = 0): BaseUserDeserialized => const status = statusString(p.readUInt8(pos + 12)); const userNameLength = p.readUInt8(pos + 13); const userName = p.subarray(pos + 14, pos + 14 + userNameLength).toString(); + const { bytesRead: optionsBytes, options } = + deserializePrefixedOptions(p, pos + 14 + userNameLength); return { - bytesRead: 14 + userNameLength, + bytesRead: 14 + userNameLength + optionsBytes, data: { id, createdAt, status, userName, + options, } } }; diff --git a/foreign/php/iggy-php.stubs.php b/foreign/php/iggy-php.stubs.php index caf4a7841c..211e6f120f 100644 --- a/foreign/php/iggy-php.stubs.php +++ b/foreign/php/iggy-php.stubs.php @@ -123,18 +123,22 @@ public function createStream(string $name): void {} /** * Creates a topic. * - * message_expiry_micros is null for server default. + * Every option left null resolves against the server default at admission. * * @param mixed $stream * @param string $name * @param int $partitions_count * @param string|null $compression_algorithm - * @param int|null $replication_factor * @param int|null $message_expiry_micros * @param int|null $max_topic_size + * @param int|null $segment_size + * @param bool|null $enforce_fsync + * @param int|null $messages_required_to_save + * @param int|null $size_of_messages_required_to_save + * @param bool|null $preallocate_segments * @return void */ - public function createTopic(mixed $stream, string $name, int $partitions_count, ?string $compression_algorithm = null, ?int $replication_factor = null, ?int $message_expiry_micros = null, ?int $max_topic_size = null): void {} + public function createTopic(mixed $stream, string $name, int $partitions_count, ?string $compression_algorithm = null, ?int $message_expiry_micros = null, ?int $max_topic_size = null, ?int $segment_size = null, ?bool $enforce_fsync = null, ?int $messages_required_to_save = null, ?int $size_of_messages_required_to_save = null, ?bool $preallocate_segments = null): void {} /** * Deletes a stream by id or name. diff --git a/foreign/php/src/client.rs b/foreign/php/src/client.rs index 118775a238..a4413a78c6 100644 --- a/foreign/php/src/client.rs +++ b/foreign/php/src/client.rs @@ -125,7 +125,7 @@ impl IggyClient { /// Creates a topic. /// - /// message_expiry_micros is null for server default. + /// Every option left null resolves against the server default at admission. #[allow(clippy::too_many_arguments)] pub fn create_topic( &self, @@ -133,9 +133,13 @@ impl IggyClient { name: String, partitions_count: u32, compression_algorithm: Option, - replication_factor: Option, message_expiry_micros: Option, max_topic_size: Option, + segment_size: Option, + enforce_fsync: Option, + messages_required_to_save: Option, + size_of_messages_required_to_save: Option, + preallocate_segments: Option, ) -> PhpResult { let compression_algorithm = match compression_algorithm { Some(value) => CompressionAlgorithm::from_str(&value).map_err(to_php_exception)?, @@ -148,17 +152,26 @@ impl IggyClient { let stream: Identifier = stream.try_into()?; let inner = self.inner.clone(); + // `None` is what tells admission to resolve the server default, so the + // sentinels above must collapse back to it. + let options = TopicCreateOptions { + partitions_count: Some(partitions_count), + compression_algorithm: (compression_algorithm != CompressionAlgorithm::default()) + .then_some(compression_algorithm), + message_expiry: (expiry != IggyExpiry::ServerDefault).then_some(expiry), + max_topic_size: (max_size != MaxTopicSize::ServerDefault).then_some(max_size), + segment_size: segment_size.map(IggyByteSize::from), + enforce_fsync, + messages_required_to_save, + size_of_messages_required_to_save: size_of_messages_required_to_save + .map(IggyByteSize::from), + preallocate_segments, + ..TopicCreateOptions::default() + }; + runtime().block_on(async move { inner - .create_topic( - &stream, - &name, - partitions_count, - compression_algorithm, - replication_factor, - expiry, - max_size, - ) + .create_topic(&stream, &name, &options) .await .map(|_| ()) .map_err(to_php_exception) diff --git a/foreign/python/apache_iggy.pyi b/foreign/python/apache_iggy.pyi index ab63e66c5a..7e32fdb418 100644 --- a/foreign/python/apache_iggy.pyi +++ b/foreign/python/apache_iggy.pyi @@ -40,6 +40,7 @@ __all__ = [ "IggyConsumer", "IggyExpiry", "MaxTopicSize", + "OptionSpec", "Partition", "Permissions", "PollingStrategy", @@ -1003,15 +1004,41 @@ class IggyClient: Returns the stream details, or `None` if the stream does not exist. Raises `RuntimeError` on failure. """ + def describe_options( + self, scope: builtins.str + ) -> collections.abc.Awaitable[builtins.list[OptionSpec]]: + r""" + Describe the option catalog for a resource scope. + + This is the discovery surface for the `options` argument on + `create_topic`/`update_topic`: a key outside the catalog is refused at + create, and the binary transports carry only the error code back. + + Args: + scope: One of `"topic"`, `"stream"`, `"user"`. + + Returns: + An awaitable that resolves to `list[OptionSpec]`, empty for a scope + with no keys yet. + + Raises: + ValueError: If the scope name is not one of the three above. + RuntimeError: If the request fails. + """ def create_topic( self, stream: builtins.str | builtins.int, name: builtins.str, partitions_count: builtins.int, compression_algorithm: builtins.str | None = None, - replication_factor: builtins.int | None = None, message_expiry: IggyExpiry | None = None, max_topic_size: MaxTopicSize | None = None, + segment_size: builtins.int | None = None, + enforce_fsync: builtins.bool | None = None, + messages_required_to_save: builtins.int | None = None, + size_of_messages_required_to_save: builtins.int | None = None, + preallocate_segments: builtins.bool | None = None, + options: builtins.dict[builtins.str, builtins.str] | None = None, ) -> collections.abc.Awaitable[None]: r""" Creates a new topic with the given parameters. @@ -1021,9 +1048,18 @@ class IggyClient: name: Topic name as `str`. partitions_count: Number of partitions as `int`. compression_algorithm: Compression algorithm as `str | None`. - replication_factor: Replication factor as `int | None`. message_expiry: Message expiry as `IggyExpiry | None`. max_topic_size: Maximum topic size as `MaxTopicSize | None`. + segment_size: Per-topic segment size in bytes as `int | None`. + enforce_fsync: Per-topic fsync enforcement as `bool | None`. + messages_required_to_save: Message-count flush threshold as `int | None`. + size_of_messages_required_to_save: Byte flush threshold as `int | None`. + preallocate_segments: Reserve segment bytes on open as `bool | None`. + options: Additional option keys as `dict[str, str] | None`, sent + verbatim so a newer server key can be set from this build. + + Every option left as `None` resolves against the server default at + admission. Returns: An awaitable that resolves to `None` when the topic is created. @@ -1063,22 +1099,21 @@ class IggyClient: topic_id: builtins.str | builtins.int, name: builtins.str, compression_algorithm: builtins.str | None = None, - replication_factor: builtins.int | None = None, message_expiry: IggyExpiry | None = None, max_topic_size: MaxTopicSize | None = None, ) -> collections.abc.Awaitable[None]: r""" Update an existing topic. - This is a full replacement: any optional parameter left unset is reset to - its server default rather than preserved. + A patch, not a replacement: every setting rides the options block, so a + field left unset keeps the topic's current value rather than resetting + it to a server default. Args: stream_id: Stream identifier as `str | int`. topic_id: Topic identifier as `str | int`. name: New topic name as `str`. compression_algorithm: Compression algorithm as `str | None`. - replication_factor: Replication factor as `int | None`. message_expiry: Message expiry as `IggyExpiry | None`. max_topic_size: Maximum topic size as `MaxTopicSize | None`. @@ -1494,6 +1529,41 @@ class MaxTopicSize: ... +@typing.final +class OptionSpec: + r""" + One entry of a resource's option catalog, as served by `describe_options`. + """ + @property + def key(self) -> builtins.str: + r""" + The option key a create command accepts. + """ + @property + def kind(self) -> builtins.str: + r""" + Name of this key's canonical kind: what the server encodes its default + under, and what a value set by `create_topic` is stored as whatever kind + it was sent in, since create admission re-encodes the block from its own + parse. `update_topic` stores what the client sent verbatim and is the + exception. + """ + @property + def default_value(self) -> HeaderValue | None: + r""" + The key's default as a `HeaderValue`, or `None` when the key has no + default. + + The same type message user headers use, so the usual accessors read it; + options ride that codec. + """ + @property + def description(self) -> builtins.str: + r""" + What the option does, including the bounds its value is checked against. + """ + def __repr__(self) -> builtins.str: ... + @typing.final class Partition: @property @@ -1968,9 +2038,21 @@ class Topic: The maximum size of the topic. """ @property - def replication_factor(self) -> builtins.int: + def options(self) -> UserHeaders: r""" - Replication factor for the topic. + Options the creating client set explicitly. + + The same `dict[HeaderKey, HeaderValue]` that `ReceiveMessage.user_headers` + returns, since options ride that codec; call `to_scalar_dict()` for the + plain-scalar form. + """ + @property + def derived_options(self) -> UserHeaders: + r""" + Options admission resolved for the keys the client did not send. + + Same shape as `options`. These would have resolved differently under + another server configuration. """ @typing.final @@ -2021,9 +2103,21 @@ class TopicDetails: The maximum size of the topic. """ @property - def replication_factor(self) -> builtins.int: + def options(self) -> UserHeaders: r""" - Replication factor for the topic. + Options the creating client set explicitly. + + The same `dict[HeaderKey, HeaderValue]` that `ReceiveMessage.user_headers` + returns, since options ride that codec; call `to_scalar_dict()` for the + plain-scalar form. + """ + @property + def derived_options(self) -> UserHeaders: + r""" + Options admission resolved for the keys the client did not send. + + Same shape as `options`. These would have resolved differently under + another server configuration. """ @property def partitions(self) -> builtins.list[Partition]: diff --git a/foreign/python/src/client.rs b/foreign/python/src/client.rs index cde56040ff..181fd8b78e 100644 --- a/foreign/python/src/client.rs +++ b/foreign/python/src/client.rs @@ -27,6 +27,7 @@ use pyo3::types::{PyBytes, PyDelta, PyList, PyType}; use pyo3_async_runtimes::tokio::future_into_py; use pyo3_stub_gen::define_stub_info_gatherer; use pyo3_stub_gen::derive::{gen_stub_pyclass, gen_stub_pymethods}; +use std::collections::BTreeMap; use std::str::FromStr; use std::sync::Arc; @@ -37,6 +38,7 @@ use crate::consumer::{ }; use crate::duration::{py_delta_to_iggy_duration, reject_zero}; use crate::identifier::PyIdentifier; +use crate::options::OptionSpec as PyOptionSpec; use crate::permissions::Permissions as PyPermissions; use crate::receive_message::{PollingStrategy, ReceiveMessage}; use crate::send_message::{SendMessage, SendMessagesResponse as PySendMessagesResponse}; @@ -155,6 +157,39 @@ impl IggyClient { }) } + /// Describe the option catalog for a resource scope. + /// + /// This is the discovery surface for the `options` argument on + /// `create_topic`/`update_topic`: a key outside the catalog is refused at + /// create, and the binary transports carry only the error code back. + /// + /// Args: + /// scope: One of `"topic"`, `"stream"`, `"user"`. + /// + /// Returns: + /// An awaitable that resolves to `list[OptionSpec]`, empty for a scope + /// with no keys yet. + /// + /// Raises: + /// ValueError: If the scope name is not one of the three above. + /// RuntimeError: If the request fails. + #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[list[OptionSpec]]", imports=("collections.abc")))] + fn describe_options<'a>(&self, py: Python<'a>, scope: &str) -> PyResult> { + let scope = crate::options::options_scope_from_str(scope)?; + let inner = self.inner.clone(); + + future_into_py(py, async move { + let specs = inner + .describe_options(scope) + .await + .map_err(|e| PyErr::new::(e.to_string()))?; + Ok(specs + .into_iter() + .map(PyOptionSpec::from) + .collect::>()) + }) + } + /// Logs in the user with the given credentials. /// Raises `RuntimeError` on failure. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] @@ -286,7 +321,13 @@ impl IggyClient { future_into_py(py, async move { inner - .update_user(&user_id, username.as_deref(), status) + .update_user( + &user_id, + username.as_deref(), + status, + // Users have no option keys yet. + &UserUpdateOptions::default(), + ) .await .map_err(|e| PyErr::new::(e.to_string()))?; Ok(()) @@ -466,9 +507,18 @@ impl IggyClient { /// name: Topic name as `str`. /// partitions_count: Number of partitions as `int`. /// compression_algorithm: Compression algorithm as `str | None`. - /// replication_factor: Replication factor as `int | None`. /// message_expiry: Message expiry as `IggyExpiry | None`. /// max_topic_size: Maximum topic size as `MaxTopicSize | None`. + /// segment_size: Per-topic segment size in bytes as `int | None`. + /// enforce_fsync: Per-topic fsync enforcement as `bool | None`. + /// messages_required_to_save: Message-count flush threshold as `int | None`. + /// size_of_messages_required_to_save: Byte flush threshold as `int | None`. + /// preallocate_segments: Reserve segment bytes on open as `bool | None`. + /// options: Additional option keys as `dict[str, str] | None`, sent + /// verbatim so a newer server key can be set from this build. + /// + /// Every option left as `None` resolves against the server default at + /// admission. /// /// Returns: /// An awaitable that resolves to `None` when the topic is created. @@ -477,7 +527,7 @@ impl IggyClient { /// ValueError: If `message_expiry` or `max_topic_size` is out of range. /// PyRuntimeError: If another argument is invalid or the request fails. #[pyo3( - signature = (stream, name, partitions_count, compression_algorithm = None, replication_factor = None, message_expiry = None, max_topic_size = None) + signature = (stream, name, partitions_count, compression_algorithm = None, message_expiry = None, max_topic_size = None, segment_size = None, enforce_fsync = None, messages_required_to_save = None, size_of_messages_required_to_save = None, preallocate_segments = None, options = None) )] #[allow(clippy::too_many_arguments)] #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] @@ -490,33 +540,50 @@ impl IggyClient { #[gen_stub(override_type(type_repr = "builtins.str | None"))] compression_algorithm: Option< String, >, - #[gen_stub(override_type(type_repr = "builtins.int | None"))] replication_factor: Option< - u8, - >, #[gen_stub(override_type(type_repr = "IggyExpiry | None"))] message_expiry: Option< &IggyExpiry, >, #[gen_stub(override_type(type_repr = "MaxTopicSize | None"))] max_topic_size: Option< &MaxTopicSize, >, + #[gen_stub(override_type(type_repr = "builtins.int | None"))] segment_size: Option, + #[gen_stub(override_type(type_repr = "builtins.bool | None"))] enforce_fsync: Option, + #[gen_stub(override_type(type_repr = "builtins.int | None"))] + messages_required_to_save: Option, + #[gen_stub(override_type(type_repr = "builtins.int | None"))] + size_of_messages_required_to_save: Option, + #[gen_stub(override_type(type_repr = "builtins.bool | None"))] preallocate_segments: Option< + bool, + >, + #[gen_stub(override_type(type_repr = "builtins.dict[builtins.str, builtins.str] | None"))] + options: Option>, ) -> PyResult> { let (compression_algorithm, expiry, max_size) = resolve_topic_params(compression_algorithm, message_expiry, max_topic_size)?; + let topic_options = TopicCreateOptions { + partitions_count: Some(partitions_count), + // `None` is what tells admission to resolve the server default, so + // the sentinels the resolver returns must collapse back to it. + compression_algorithm: (compression_algorithm != CompressionAlgorithm::default()) + .then_some(compression_algorithm), + message_expiry: (expiry != RustIggyExpiry::ServerDefault).then_some(expiry), + max_topic_size: (max_size != RustMaxTopicSize::ServerDefault).then_some(max_size), + segment_size: segment_size.map(IggyByteSize::from), + enforce_fsync, + messages_required_to_save, + size_of_messages_required_to_save: size_of_messages_required_to_save + .map(IggyByteSize::from), + preallocate_segments, + raw: options.unwrap_or_default(), + }; + let stream = Identifier::try_from(stream)?; let inner = self.inner.clone(); future_into_py(py, async move { inner - .create_topic( - &stream, - &name, - partitions_count, - compression_algorithm, - replication_factor, - expiry, - max_size, - ) + .create_topic(&stream, &name, &topic_options) .await .map_err(|e| PyErr::new::(e.to_string()))?; Ok(()) @@ -576,15 +643,15 @@ impl IggyClient { /// Update an existing topic. /// - /// This is a full replacement: any optional parameter left unset is reset to - /// its server default rather than preserved. + /// A patch, not a replacement: every setting rides the options block, so a + /// field left unset keeps the topic's current value rather than resetting + /// it to a server default. /// /// Args: /// stream_id: Stream identifier as `str | int`. /// topic_id: Topic identifier as `str | int`. /// name: New topic name as `str`. /// compression_algorithm: Compression algorithm as `str | None`. - /// replication_factor: Replication factor as `int | None`. /// message_expiry: Message expiry as `IggyExpiry | None`. /// max_topic_size: Maximum topic size as `MaxTopicSize | None`. /// @@ -595,7 +662,7 @@ impl IggyClient { /// ValueError: If `message_expiry` or `max_topic_size` is out of range. /// PyRuntimeError: If another argument is invalid or the request fails. #[pyo3( - signature = (stream_id, topic_id, name, compression_algorithm = None, replication_factor = None, message_expiry = None, max_topic_size = None) + signature = (stream_id, topic_id, name, compression_algorithm = None, message_expiry = None, max_topic_size = None) )] #[allow(clippy::too_many_arguments)] #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] @@ -608,9 +675,6 @@ impl IggyClient { #[gen_stub(override_type(type_repr = "builtins.str | None"))] compression_algorithm: Option< String, >, - #[gen_stub(override_type(type_repr = "builtins.int | None"))] replication_factor: Option< - u8, - >, #[gen_stub(override_type(type_repr = "IggyExpiry | None"))] message_expiry: Option< &IggyExpiry, >, @@ -618,8 +682,20 @@ impl IggyClient { &MaxTopicSize, >, ) -> PyResult> { - let (compression_algorithm, expiry, max_size) = - resolve_topic_params(compression_algorithm, message_expiry, max_topic_size)?; + // Absent stays absent: a key the caller did not pass is left alone + // server-side rather than reset to a default. + let compression_algorithm = compression_algorithm + .map(|algo| { + CompressionAlgorithm::from_str(&algo) + .map_err(|e| PyErr::new::(e.to_string())) + }) + .transpose()?; + let update_options = TopicUpdateOptions { + compression_algorithm, + message_expiry: message_expiry.map(RustIggyExpiry::try_from).transpose()?, + max_topic_size: max_topic_size.map(RustMaxTopicSize::try_from).transpose()?, + ..TopicUpdateOptions::default() + }; let stream_id = Identifier::try_from(stream_id)?; let topic_id = Identifier::try_from(topic_id)?; @@ -627,15 +703,7 @@ impl IggyClient { future_into_py(py, async move { inner - .update_topic( - &stream_id, - &topic_id, - &name, - compression_algorithm, - replication_factor, - expiry, - max_size, - ) + .update_topic(&stream_id, &topic_id, &name, &update_options) .await .map_err(|e| PyErr::new::(e.to_string()))?; Ok(()) diff --git a/foreign/python/src/lib.rs b/foreign/python/src/lib.rs index 9c7f1efaa7..7910af77b9 100644 --- a/foreign/python/src/lib.rs +++ b/foreign/python/src/lib.rs @@ -20,6 +20,7 @@ mod config; mod consumer; mod duration; mod identifier; +mod options; mod permissions; mod receive_message; mod send_message; @@ -34,6 +35,7 @@ use consumer::{ AutoCommit, AutoCommitAfter, AutoCommitWhen, ConsumerGroup, ConsumerGroupDetails, ConsumerGroupMember, IggyConsumer, ReceiveMessageIterator, }; +use options::OptionSpec; use permissions::{GlobalPermissions, Permissions, StreamPermissions, TopicPermissions}; use pyo3::prelude::*; use receive_message::{PollingStrategy, ReceiveMessage}; @@ -59,6 +61,7 @@ fn apache_iggy(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/foreign/python/src/options.rs b/foreign/python/src/options.rs new file mode 100644 index 0000000000..c3eee8cdd6 --- /dev/null +++ b/foreign/python/src/options.rs @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use iggy::prelude::{ + HeaderValue as RustHeaderValue, OptionSpec as RustOptionSpec, OptionsScope as RustOptionsScope, +}; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use pyo3_stub_gen::derive::{gen_stub_pyclass, gen_stub_pymethods}; +use std::str::FromStr; + +use crate::user_headers::{HeaderValue, rust_header_value_to_py}; + +/// One entry of a resource's option catalog, as served by `describe_options`. +#[gen_stub_pyclass] +#[pyclass] +pub struct OptionSpec { + inner: RustOptionSpec, +} + +impl From for OptionSpec { + fn from(spec: RustOptionSpec) -> Self { + Self { inner: spec } + } +} + +#[gen_stub_pymethods] +#[pymethods] +impl OptionSpec { + /// The option key a create command accepts. + #[getter] + pub fn key(&self) -> String { + self.inner.key.clone() + } + + /// Name of this key's canonical kind: what the server encodes its default + /// under, and what a value set by `create_topic` is stored as whatever kind + /// it was sent in, since create admission re-encodes the block from its own + /// parse. `update_topic` stores what the client sent verbatim and is the + /// exception. + #[getter] + pub fn kind(&self) -> String { + self.inner.kind.to_string() + } + + /// The key's default as a `HeaderValue`, or `None` when the key has no + /// default. + /// + /// The same type message user headers use, so the usual accessors read it; + /// options ride that codec. + #[getter] + pub fn default_value<'a>(&self, py: Python<'a>) -> PyResult>> { + if self.inner.default_value.is_empty() { + return Ok(None); + } + let value = RustHeaderValue::from_raw(self.inner.kind, &self.inner.default_value).map_err( + |error| { + PyValueError::new_err(format!( + "option '{}' has a default this build cannot read: {error}", + self.inner.key + )) + }, + )?; + rust_header_value_to_py(py, &value).map(Some) + } + + /// What the option does, including the bounds its value is checked against. + #[getter] + pub fn description(&self) -> String { + self.inner.description.clone() + } + + fn __repr__(&self) -> String { + format!( + "OptionSpec(key='{}', kind='{}')", + self.inner.key, self.inner.kind + ) + } +} + +/// Resolve the scope a `describe_options` call names. +/// +/// Takes the same names the CLI and the REST path take (`topic`, `stream`, +/// `user`) rather than an enum class, so a scope is one string at the call site. +/// +/// # Errors +/// +/// Raises `ValueError` for a name outside the three scopes. +pub fn options_scope_from_str(scope: &str) -> PyResult { + RustOptionsScope::from_str(scope).map_err(|_| { + PyValueError::new_err(format!( + "unknown options scope '{scope}', expected one of: topic, stream, user" + )) + }) +} diff --git a/foreign/python/src/topic.rs b/foreign/python/src/topic.rs index 40a7db228e..e5f86facd8 100644 --- a/foreign/python/src/topic.rs +++ b/foreign/python/src/topic.rs @@ -17,14 +17,37 @@ use iggy::prelude::{ IggyByteSize, IggyExpiry as RustIggyExpiry, MaxTopicSize as RustMaxTopicSize, - Partition as RustPartition, Topic as RustTopic, TopicDetails as RustTopicDetails, + Partition as RustPartition, ResourceOptions, Topic as RustTopic, + TopicDetails as RustTopicDetails, }; + use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use pyo3::types::PyDelta; use pyo3_stub_gen::derive::{gen_stub_pyclass, gen_stub_pyclass_complex_enum, gen_stub_pymethods}; use crate::duration::{iggy_duration_to_py_delta, py_delta_to_iggy_duration}; +use crate::user_headers::{UserHeaders, rust_user_headers_to_py}; + +/// The entries of one provenance, as the dictionary message user headers come +/// back as. +/// +/// Options ride the user-headers codec, so they are handed back through the +/// same `HeaderKey`/`HeaderValue` types rather than a second shape meaning the +/// same thing: `to_scalar_dict()` works on the result exactly as it does on +/// `ReceiveMessage.user_headers`. +fn options_by_provenance<'a>( + py: Python<'a>, + options: &ResourceOptions, + explicit: bool, +) -> PyResult> { + let selected = options + .iter() + .filter(|(_, option)| option.explicit == explicit) + .map(|(key, option)| (key.clone(), option.value.clone())) + .collect(); + rust_user_headers_to_py(py, selected) +} /// The expiry of the messages in a topic. #[gen_stub_pyclass_complex_enum] @@ -219,10 +242,23 @@ impl Topic { self.inner.max_topic_size.into() } - /// Replication factor for the topic. + /// Options the creating client set explicitly. + /// + /// The same `dict[HeaderKey, HeaderValue]` that `ReceiveMessage.user_headers` + /// returns, since options ride that codec; call `to_scalar_dict()` for the + /// plain-scalar form. + #[getter] + pub fn options<'a>(&self, py: Python<'a>) -> PyResult> { + options_by_provenance(py, &self.inner.options, true) + } + + /// Options admission resolved for the keys the client did not send. + /// + /// Same shape as [`Self::options`]. These would have resolved differently + /// under another server configuration. #[getter] - pub fn replication_factor(&self) -> u8 { - self.inner.replication_factor + pub fn derived_options<'a>(&self, py: Python<'a>) -> PyResult> { + options_by_provenance(py, &self.inner.options, false) } } @@ -297,10 +333,23 @@ impl TopicDetails { self.inner.max_topic_size.into() } - /// Replication factor for the topic. + /// Options the creating client set explicitly. + /// + /// The same `dict[HeaderKey, HeaderValue]` that `ReceiveMessage.user_headers` + /// returns, since options ride that codec; call `to_scalar_dict()` for the + /// plain-scalar form. + #[getter] + pub fn options<'a>(&self, py: Python<'a>) -> PyResult> { + options_by_provenance(py, &self.inner.options, true) + } + + /// Options admission resolved for the keys the client did not send. + /// + /// Same shape as [`Self::options`]. These would have resolved differently + /// under another server configuration. #[getter] - pub fn replication_factor(&self) -> u8 { - self.inner.replication_factor + pub fn derived_options<'a>(&self, py: Python<'a>) -> PyResult> { + options_by_provenance(py, &self.inner.options, false) } /// The collection of partitions in the topic. diff --git a/foreign/python/src/user_headers.rs b/foreign/python/src/user_headers.rs index 3eaa843a30..e3fc21082a 100644 --- a/foreign/python/src/user_headers.rs +++ b/foreign/python/src/user_headers.rs @@ -578,6 +578,18 @@ pub(crate) fn py_user_headers_to_rust( Ok(rust_headers) } +/// Wrap one Rust header value as the `HeaderValue` message headers already use. +/// +/// The option catalog reports each key's default as a typed value, and options +/// ride the user-headers codec, so they hand back the same type rather than a +/// second one that means the same thing. +pub(crate) fn rust_header_value_to_py<'a>( + py: Python<'a>, + value: &RustHeaderValue, +) -> PyResult> { + Bound::::try_from(RustHeaderValueRef { py, value }) +} + pub(crate) fn rust_user_headers_to_py<'a>( py: Python<'a>, headers: RustUserHeaders, diff --git a/foreign/python/tests/test_topic.py b/foreign/python/tests/test_topic.py index 46434525ef..8221ffc6b3 100644 --- a/foreign/python/tests/test_topic.py +++ b/foreign/python/tests/test_topic.py @@ -19,7 +19,7 @@ import pytest -from apache_iggy import IggyClient, IggyExpiry, MaxTopicSize, SendMessage +from apache_iggy import HeaderValue, IggyClient, IggyExpiry, MaxTopicSize, SendMessage from .utils import ( get_server_config, @@ -391,66 +391,6 @@ async def test_create_topic_invalid_max_topic_size( max_topic_size=MaxTopicSize.Custom(max_topic_size_bytes), ) - @pytest.mark.asyncio - @pytest.mark.parametrize( - "replication_factor", - [ - 0, # value for server default replication factor - 1, - 42, - 255, - ], - ) - async def test_create_topic_with_valid_replication_factor( - self, iggy_client: IggyClient, unique_name, replication_factor: int - ): - """Test create_topic accepts a supported replication factor.""" - stream_name = unique_name() - topic_name = unique_name() - - await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - replication_factor=replication_factor, - ) - - topic = await iggy_client.get_topic(stream_name, topic_name) - assert topic is not None - assert topic.name == topic_name - - @pytest.mark.asyncio - @pytest.mark.parametrize( - ("replication_factor", "expected_exception"), - [ - (-1, OverflowError), - (256, OverflowError), - ("1", TypeError), - (1.0, TypeError), - ], - ) - async def test_create_topic_invalid_replication_factor( - self, - iggy_client: IggyClient, - unique_name, - replication_factor, - expected_exception, - ): - """Test create_topic rejects invalid replication factor values.""" - stream_name = unique_name() - topic_name = unique_name() - - await iggy_client.create_stream(stream_name) - - with pytest.raises(expected_exception): - await iggy_client.create_topic( - stream=stream_name, - name=topic_name, - partitions_count=1, - replication_factor=replication_factor, - ) - @pytest.mark.asyncio @pytest.mark.parametrize("partitions_count", [1001, 10000]) async def test_create_topic_invalid_partitions_count( @@ -954,66 +894,6 @@ async def test_update_topic_invalid_message_expiry( message_expiry=invalid_message_expiry, ) - @pytest.mark.asyncio - @pytest.mark.parametrize("replication_factor", [0, 1, 255]) - async def test_update_topic_with_valid_replication_factor( - self, iggy_client: IggyClient, unique_name, replication_factor: int - ): - """Test update_topic accepts a supported replication factor.""" - stream_name = unique_name() - topic_name = unique_name() - - await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) - - await iggy_client.update_topic( - stream_id=stream_name, - topic_id=topic_name, - name=topic_name, - replication_factor=replication_factor, - ) - - topic = await iggy_client.get_topic(stream_name, topic_name) - assert topic is not None - # The server normalizes a replication factor of 0 to 1. - assert topic.replication_factor == (replication_factor or 1) - - @pytest.mark.asyncio - @pytest.mark.parametrize( - ("replication_factor", "expected_exception"), - [ - (-1, OverflowError), - (256, OverflowError), - ("1", TypeError), - (1.0, TypeError), - ], - ) - async def test_update_topic_invalid_replication_factor( - self, - iggy_client: IggyClient, - unique_name, - replication_factor, - expected_exception, - ): - """Test update_topic rejects invalid replication factor values.""" - stream_name = unique_name() - topic_name = unique_name() - - await iggy_client.create_stream(stream_name) - await iggy_client.create_topic( - stream=stream_name, name=topic_name, partitions_count=1 - ) - - with pytest.raises(expected_exception): - await iggy_client.update_topic( - stream_id=stream_name, - topic_id=topic_name, - name=topic_name, - replication_factor=replication_factor, - ) - @pytest.mark.asyncio @pytest.mark.parametrize( ("max_topic_size_bytes", "expected_exception"), @@ -1132,6 +1012,10 @@ async def test_update_topic_with_valid_max_topic_size( stream=stream_name, name=topic_name, partitions_count=1 ) + created = await iggy_client.get_topic(stream_name, topic_name) + assert created is not None + resolved_at_creation = created.max_topic_size + await iggy_client.update_topic( stream_id=stream_name, topic_id=topic_name, @@ -1143,7 +1027,13 @@ async def test_update_topic_with_valid_max_topic_size( assert topic is not None assert topic.name == topic_name if expected_kind == "server_default": - assert isinstance(topic.max_topic_size, MaxTopicSize.ServerDefault) + # Every setting rides the options block and 0 is its "resolve the + # default" sentinel, so a ServerDefault update carries no key at all + # and the topic keeps the value admission resolved when it was + # created. Resetting a setting back to the node default is + # deliberately not expressible. + assert isinstance(topic.max_topic_size, type(resolved_at_creation)) + assert not isinstance(topic.max_topic_size, MaxTopicSize.ServerDefault) elif expected_kind == "unlimited": assert isinstance(topic.max_topic_size, MaxTopicSize.Unlimited) else: @@ -1393,7 +1283,6 @@ async def test_purge_topic_clears_messages_but_keeps_topic( assert after.created_at == before.created_at assert after.partitions_count == before.partitions_count assert after.compression_algorithm == before.compression_algorithm - assert after.replication_factor == before.replication_factor assert isinstance(before.message_expiry, IggyExpiry.NeverExpire) assert isinstance(after.message_expiry, IggyExpiry.NeverExpire) assert isinstance(before.max_topic_size, MaxTopicSize.Unlimited) @@ -1475,3 +1364,69 @@ async def test_purge_topic_requires_connection_and_auth(self, unique_name): await client.connect() with pytest.raises(RuntimeError): await client.purge_topic(unique_name(), unique_name()) + + +class TestTopicOptions: + """Tests for the option catalog and the options a topic reports.""" + + @pytest.mark.asyncio + async def test_topic_options_round_trip(self, iggy_client: IggyClient, unique_name): + """Options a client sets come back readable, split by provenance.""" + stream_name = unique_name() + topic_name = unique_name() + + await iggy_client.create_stream(stream_name) + await iggy_client.create_topic( + stream=stream_name, + name=topic_name, + partitions_count=1, + options={"enforce_fsync": "true", "segment_size": "128 MiB"}, + ) + + topic = await iggy_client.get_topic(stream_name, topic_name) + assert topic is not None + # Options come back through the same typed dict message user headers + # use, so the scalar helper reads them the same way. + explicit = topic.options.to_scalar_dict() + assert explicit["enforce_fsync"] is True + assert explicit["segment_size"] == 128 * 1024 * 1024 + # Keys the client left alone are resolved by admission and reported + # separately, so an operator can tell chosen from defaulted. + derived = topic.derived_options.to_scalar_dict() + assert "max_topic_size" in derived + assert "enforce_fsync" not in derived + + topics = await iggy_client.get_topics(stream_name) + listed = next(entry for entry in topics if entry.name == topic_name) + assert listed.options.to_scalar_dict()["enforce_fsync"] is True + + @pytest.mark.asyncio + async def test_describe_options_lists_the_topic_catalog( + self, iggy_client: IggyClient + ): + """The catalog is what tells a client which keys create accepts.""" + specs = await iggy_client.describe_options("topic") + + by_key = {spec.key: spec for spec in specs} + assert "segment_size" in by_key + assert "enforce_fsync" in by_key + segment_size = by_key["segment_size"] + assert segment_size.kind == "uint64" + # The default is the same HeaderValue type message headers carry, so it + # arrives as the variant matching the key's kind. + default = segment_size.default_value + assert isinstance(default, HeaderValue.UnsignedInt64) + assert default.value == 1024 * 1024 * 1024 + assert segment_size.description + + # Streams and users have no catalog keys yet. + assert await iggy_client.describe_options("stream") == [] + assert await iggy_client.describe_options("user") == [] + + @pytest.mark.asyncio + async def test_describe_options_rejects_an_unknown_scope( + self, iggy_client: IggyClient + ): + """Test describe_options raises ValueError for an unknown scope.""" + with pytest.raises(ValueError): + await iggy_client.describe_options("partition") diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000000..26f23d6c0f --- /dev/null +++ b/package-lock.json @@ -0,0 +1,110 @@ +{ + "name": "iggy", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "dependencies": { + "ccusage": "^20.0.19" + } + }, + "node_modules/@ccusage/ccusage-darwin-arm64": { + "version": "20.0.19", + "resolved": "https://registry.npmjs.org/@ccusage/ccusage-darwin-arm64/-/ccusage-darwin-arm64-20.0.19.tgz", + "integrity": "sha512-8b69h4tuMH1KNW+EYjFRJWpbWHXbMUtFu8cOCyFOQsR6lGxq37g0kzEGHSqG+iZEoUTHaabR5CE7ZHLhW9Q64A==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@ccusage/ccusage-darwin-x64": { + "version": "20.0.19", + "resolved": "https://registry.npmjs.org/@ccusage/ccusage-darwin-x64/-/ccusage-darwin-x64-20.0.19.tgz", + "integrity": "sha512-YTsqIfsPo3qR9OMrwJtnJoq/fil7Jk5mzXV8k0Iejpiq7zysw4FfkjCeHr0UVJG8ihdqtE18o+xIk99p3v+5Ew==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@ccusage/ccusage-linux-arm64": { + "version": "20.0.19", + "resolved": "https://registry.npmjs.org/@ccusage/ccusage-linux-arm64/-/ccusage-linux-arm64-20.0.19.tgz", + "integrity": "sha512-4e+7hV3WrEpM7hQPQkh4/zNLTUSqzpd+vSSWl2y659+xQ+JCDLWqE6gzvXLMOAT/LTtcFtlrduIWDZt8VPceqQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@ccusage/ccusage-linux-x64": { + "version": "20.0.19", + "resolved": "https://registry.npmjs.org/@ccusage/ccusage-linux-x64/-/ccusage-linux-x64-20.0.19.tgz", + "integrity": "sha512-VPbB6onrwOGojTKutFzeahttb98ZgmdsiQpOr/BJet1i3QdrmUGXGmJSt591xXrvVOVVgsC7/5wBELdWM9/bKA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@ccusage/ccusage-win32-arm64": { + "version": "20.0.19", + "resolved": "https://registry.npmjs.org/@ccusage/ccusage-win32-arm64/-/ccusage-win32-arm64-20.0.19.tgz", + "integrity": "sha512-oOv4mZkapxzvQeAMI+jqfTOv/zN2x2AYPAD+F4AnYHQQL4n898axhH9uPd2Ls0U5CDlvfh1zDTyNILz8X9M15A==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@ccusage/ccusage-win32-x64": { + "version": "20.0.19", + "resolved": "https://registry.npmjs.org/@ccusage/ccusage-win32-x64/-/ccusage-win32-x64-20.0.19.tgz", + "integrity": "sha512-KtDlTk07uv8cu6TlizB/RptcDiQePE1YSQe6uNNZTUnwVy9zq5R3fvUUFr9/d3FYHwimSGduoHXRS8Ta+Kmkrw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/ccusage": { + "version": "20.0.19", + "resolved": "https://registry.npmjs.org/ccusage/-/ccusage-20.0.19.tgz", + "integrity": "sha512-vyNIyctcvTKFbrsU/WJuWiII3CVw8WopU/0NcR+lKmRbh62wm0jccmkWrFCG6j7R8agc8mIBz5+YlmGbXH1RnQ==", + "license": "MIT", + "bin": { + "ccusage": "src/cli.js" + }, + "funding": { + "url": "https://github.com/sponsors/ryoppippi" + }, + "optionalDependencies": { + "@ccusage/ccusage-darwin-arm64": "20.0.19", + "@ccusage/ccusage-darwin-x64": "20.0.19", + "@ccusage/ccusage-linux-arm64": "20.0.19", + "@ccusage/ccusage-linux-x64": "20.0.19", + "@ccusage/ccusage-win32-arm64": "20.0.19", + "@ccusage/ccusage-win32-x64": "20.0.19" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000000..b916f62aed --- /dev/null +++ b/package.json @@ -0,0 +1,5 @@ +{ + "dependencies": { + "ccusage": "^20.0.19" + } +} diff --git a/scripts/performance/run-standard-performance-suite.sh b/scripts/performance/run-standard-performance-suite.sh index f99635ff81..10244a5443 100755 --- a/scripts/performance/run-standard-performance-suite.sh +++ b/scripts/performance/run-standard-performance-suite.sh @@ -68,8 +68,12 @@ get_env_vars() { # Specific env vars based on bench type case "$bench_type" in + # fsync is a topic creation option (`enforce_fsync`) now, not server config, + # so the bench command carries `--enforce-fsync` (added by + # `construct_bench_command` off the same remark) and only the cache setting + # is left to the server environment. *"no_cache_fsync"*) - env_vars+=("IGGY_SYSTEM_CACHE_ENABLED=false IGGY_SYSTEM_PARTITION_ENFORCE_FSYNC=true") + env_vars+=("IGGY_SYSTEM_CACHE_ENABLED=false") ;; *"only_cache"*) env_vars+=("IGGY_SYSTEM_CACHE_SIZE=9GB") diff --git a/scripts/performance/utils.sh b/scripts/performance/utils.sh index b8513da2a4..f710b5b80a 100755 --- a/scripts/performance/utils.sh +++ b/scripts/performance/utils.sh @@ -148,6 +148,15 @@ function construct_bench_command() { ;; esac + # fsync is a per-topic option now, not server config, so the fsync variants + # have to ask for it on the bench command line rather than via server env. + local fsync_arg="" + case "$remark" in + *"no_cache_fsync"*) + fsync_arg="--enforce-fsync" + ;; + esac + local commit_hash commit_hash=$(get_git_iggy_server_tag_or_sha1 .) || { echo "Failed to get git commit or tag." @@ -159,7 +168,7 @@ function construct_bench_command() { exit 1 } - echo "$bench_command ${rate_limit:+ --rate-limit ${rate_limit}} \ + echo "$bench_command ${rate_limit:+ --rate-limit ${rate_limit}} ${fsync_arg} \ --message-size ${message_size} \ --messages-per-batch ${messages_per_batch} \ --message-batches ${message_batches} \