diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index 14b019853df..bdc3fa09ab1 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -985,6 +985,18 @@ public class ConfigOptions { + "we would fsync after every message; if it were 5 we would fsync after every " + "five messages."); + public static final ConfigOption LOG_REPLICATION_LEADER_EPOCH_ENABLED = + key("log.replication.leader-epoch.enabled") + .booleanType() + .defaultValue(true) + .withDescription( + "Whether to validate replication log history using leader epochs. " + + "Can be changed dynamically with alterClusterConfigs. Disabled peers remain compatible " + + "through legacy replication, which cannot verify divergent histories. " + + "Disabling discards local epoch history; enabling does not reconstruct " + + "unknown history for existing records. New boundaries are established " + + "by subsequent leader epochs, not by changing this option."); + public static final ConfigOption LOG_FLUSH_OFFSET_CHECKPOINT_INTERVAL = key("log.flush.offset.checkpoint-interval") .durationType() diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/LeaderEpochOffset.java b/fluss-common/src/main/java/org/apache/fluss/metadata/LeaderEpochOffset.java new file mode 100644 index 00000000000..f6c57ce842a --- /dev/null +++ b/fluss-common/src/main/java/org/apache/fluss/metadata/LeaderEpochOffset.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.metadata; + +import org.apache.fluss.annotation.Internal; + +import java.util.Objects; + +/** A leader epoch and its associated log boundary. */ +@Internal +public final class LeaderEpochOffset { + private final int epoch; + private final long offset; + + public LeaderEpochOffset(int epoch, long offset) { + this.epoch = epoch; + this.offset = offset; + } + + public int epoch() { + return epoch; + } + + public long offset() { + return offset; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof LeaderEpochOffset)) { + return false; + } + LeaderEpochOffset that = (LeaderEpochOffset) other; + return epoch == that.epoch && offset == that.offset; + } + + @Override + public int hashCode() { + return Objects.hash(epoch, offset); + } + + @Override + public String toString() { + return "LeaderEpochOffset(" + epoch + ", " + offset + ")"; + } +} diff --git a/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogManifestJsonSerde.java b/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogManifestJsonSerde.java index 00eddd0baef..d1425616342 100644 --- a/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogManifestJsonSerde.java +++ b/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogManifestJsonSerde.java @@ -17,6 +17,7 @@ package org.apache.fluss.remote; +import org.apache.fluss.metadata.LeaderEpochOffset; import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.core.JsonGenerator; @@ -52,6 +53,7 @@ public class RemoteLogManifestJsonSerde private static final String MAX_TIMESTAMP_FIELD = "max_timestamp"; private static final String SEGMENT_SIZE_IN_BYTES_FIELD = "size_in_bytes"; private static final String HIGHEST_COPIED_END_OFFSET_FIELD = "highest_copied_end_offset"; + private static final String LEADER_EPOCHS_FIELD = "leader_epochs"; private static final int SNAPSHOT_VERSION = 1; @Override @@ -99,6 +101,16 @@ public void serialize(RemoteLogManifest manifest, JsonGenerator generator) throw generator.writeNumberField(MAX_TIMESTAMP_FIELD, remoteLogSegment.maxTimestamp()); generator.writeNumberField( SEGMENT_SIZE_IN_BYTES_FIELD, remoteLogSegment.segmentSizeInBytes()); + if (!remoteLogSegment.leaderEpochs().isEmpty()) { + generator.writeArrayFieldStart(LEADER_EPOCHS_FIELD); + for (LeaderEpochOffset epoch : remoteLogSegment.leaderEpochs()) { + generator.writeStartObject(); + generator.writeNumberField("epoch", epoch.epoch()); + generator.writeNumberField(START_OFFSET_FIELD, epoch.offset()); + generator.writeEndObject(); + } + generator.writeEndArray(); + } generator.writeEndObject(); } generator.writeEndArray(); @@ -150,6 +162,17 @@ public RemoteLogManifest deserialize(JsonNode node) { if (logicalEndOffsetNode != null) { segmentBuilder.logicalEndOffset(logicalEndOffsetNode.asLong()); } + JsonNode epochNodes = entryJson.get(LEADER_EPOCHS_FIELD); + if (epochNodes != null) { + List epochs = new ArrayList<>(); + for (JsonNode epoch : epochNodes) { + epochs.add( + new LeaderEpochOffset( + epoch.get("epoch").asInt(), + epoch.get(START_OFFSET_FIELD).asLong())); + } + segmentBuilder.leaderEpochs(epochs); + } snapshotEntries.add(segmentBuilder.build()); } diff --git a/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogSegment.java b/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogSegment.java index e68c5338843..112d9096533 100644 --- a/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogSegment.java +++ b/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogSegment.java @@ -18,11 +18,15 @@ package org.apache.fluss.remote; import org.apache.fluss.annotation.Internal; +import org.apache.fluss.metadata.LeaderEpochOffset; import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.TableBucket; import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.Objects; import java.util.UUID; @@ -58,6 +62,7 @@ public class RemoteLogSegment { private final long maxTimestamp; private final int segmentSizeInBytes; + private final List leaderEpochs; private RemoteLogSegment( PhysicalTablePath physicalTablePath, @@ -68,7 +73,8 @@ private RemoteLogSegment( @Nullable Long logicalStartOffset, @Nullable Long logicalEndOffset, long maxTimestamp, - int segmentSizeInBytes) { + int segmentSizeInBytes, + List leaderEpochs) { this.physicalTablePath = checkNotNull(physicalTablePath); this.tableBucket = checkNotNull(tableBucket); this.remoteLogSegmentId = checkNotNull(remoteLogSegmentId); @@ -107,6 +113,7 @@ private RemoteLogSegment( } this.maxTimestamp = maxTimestamp; this.segmentSizeInBytes = segmentSizeInBytes; + this.leaderEpochs = Collections.unmodifiableList(new ArrayList<>(leaderEpochs)); } public PhysicalTablePath physicalTablePath() { @@ -161,7 +168,13 @@ public RemoteLogSegment withLogicalRange(long logicalStartOffset, long logicalEn logicalStartOffset, logicalEndOffset, maxTimestamp, - segmentSizeInBytes); + segmentSizeInBytes, + leaderEpochs); + } + + /** Epoch boundaries for this physical segment; empty for legacy manifests. */ + public List leaderEpochs() { + return leaderEpochs; } public long maxTimestamp() { @@ -189,7 +202,8 @@ public boolean equals(Object o) { && maxTimestamp == that.maxTimestamp && Objects.equals(remoteLogSegmentId, that.remoteLogSegmentId) && Objects.equals(physicalTablePath, that.physicalTablePath) - && Objects.equals(tableBucket, that.tableBucket); + && Objects.equals(tableBucket, that.tableBucket) + && leaderEpochs.equals(that.leaderEpochs); } @Override @@ -203,7 +217,8 @@ public int hashCode() { logicalStartOffset, logicalEndOffset, maxTimestamp, - segmentSizeInBytes); + segmentSizeInBytes, + leaderEpochs); } @Override @@ -241,6 +256,7 @@ public static class Builder { private @Nullable Long logicalEndOffset; private long maxTimestamp; private int segmentSizeInBytes; + private List leaderEpochs = Collections.emptyList(); public static Builder builder() { return new Builder(); @@ -291,6 +307,11 @@ public Builder tableBucket(TableBucket tableBucket) { return this; } + public Builder leaderEpochs(List leaderEpochs) { + this.leaderEpochs = leaderEpochs; + return this; + } + public RemoteLogSegment build() { return new RemoteLogSegment( physicalTablePath, @@ -301,7 +322,8 @@ public RemoteLogSegment build() { logicalStartOffset, logicalEndOffset, maxTimestamp, - segmentSizeInBytes); + segmentSizeInBytes, + leaderEpochs); } } } diff --git a/fluss-common/src/test/java/org/apache/fluss/remote/RemoteLogManifestJsonSerdeTest.java b/fluss-common/src/test/java/org/apache/fluss/remote/RemoteLogManifestJsonSerdeTest.java index e0951321589..24f46a925f9 100644 --- a/fluss-common/src/test/java/org/apache/fluss/remote/RemoteLogManifestJsonSerdeTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/remote/RemoteLogManifestJsonSerdeTest.java @@ -17,14 +17,21 @@ package org.apache.fluss.remote; +import org.apache.fluss.metadata.LeaderEpochOffset; import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TablePath; import org.apache.fluss.utils.json.JsonSerdeTestBase; +import org.apache.fluss.utils.json.JsonSerdeUtils; + +import org.junit.jupiter.api.Test; import java.util.Arrays; +import java.util.Collections; import java.util.UUID; +import static org.assertj.core.api.Assertions.assertThat; + /** Tests of {@link RemoteLogManifestJsonSerde}. */ class RemoteLogManifestJsonSerdeTest extends JsonSerdeTestBase { private static final PhysicalTablePath TABLE_PATH1 = @@ -102,6 +109,32 @@ protected RemoteLogManifestJsonSerdeTest() { super(RemoteLogManifestJsonSerde.INSTANCE); } + @Test + void testLeaderEpochHistorySurvivesManifestRoundTrip() throws Exception { + RemoteLogSegment segment = + RemoteLogSegment.Builder.builder() + .physicalTablePath(TABLE_PATH1) + .tableBucket(TABLE_BUCKET1) + .remoteLogSegmentId(UUID.randomUUID()) + .remoteLogStartOffset(10) + .remoteLogEndOffset(30) + .maxTimestamp(0) + .segmentSizeInBytes(100) + .leaderEpochs( + Arrays.asList( + new LeaderEpochOffset(4, 5), new LeaderEpochOffset(7, 20))) + .build(); + RemoteLogManifest manifest = + new RemoteLogManifest( + TABLE_PATH1, TABLE_BUCKET1, Collections.singletonList(segment)); + byte[] json = + JsonSerdeUtils.writeValueAsBytes(manifest, RemoteLogManifestJsonSerde.INSTANCE); + assertThat(JsonSerdeUtils.readValue(json, RemoteLogManifestJsonSerde.INSTANCE)) + .isEqualTo(manifest); + assertThat(segment.withLogicalRange(15, 25).leaderEpochs()) + .containsExactly(new LeaderEpochOffset(4, 5), new LeaderEpochOffset(7, 20)); + } + @Override protected RemoteLogManifest[] createObjects() { return new RemoteLogManifest[] {MANIFEST_SNAPSHOT1, MANIFEST_SNAPSHOT2}; diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/FetchLogEpochInfo.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/FetchLogEpochInfo.java new file mode 100644 index 00000000000..50ce0d43da2 --- /dev/null +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/FetchLogEpochInfo.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.rpc.entity; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.metadata.LeaderEpochOffset; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** Epoch validation and log history attached to a replica fetch response. */ +@Internal +public final class FetchLogEpochInfo { + private final int leaderEpoch; + @Nullable private final LeaderEpochOffset divergingEpoch; + private final List epochStarts; + + public FetchLogEpochInfo( + int leaderEpoch, + @Nullable LeaderEpochOffset divergingEpoch, + List epochStarts) { + this.leaderEpoch = leaderEpoch; + this.divergingEpoch = divergingEpoch; + this.epochStarts = Collections.unmodifiableList(new ArrayList<>(epochStarts)); + } + + public int leaderEpoch() { + return leaderEpoch; + } + + @Nullable + public LeaderEpochOffset divergingEpoch() { + return divergingEpoch; + } + + public List epochStarts() { + return epochStarts; + } +} diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/FetchLogResultForBucket.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/FetchLogResultForBucket.java index 11f6d7e73b8..8a88b6f3f6b 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/FetchLogResultForBucket.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/FetchLogResultForBucket.java @@ -38,6 +38,7 @@ public class FetchLogResultForBucket extends ResultForBucket { private final long highWatermark; private final long filteredEndOffset; private final long minRetainOffset; + @Nullable private final FetchLogEpochInfo epochInfo; private FetchLogResultForBucket( TableBucket tableBucket, @@ -47,7 +48,28 @@ private FetchLogResultForBucket( long filteredEndOffset, long minRetainOffset, ApiError error) { + this( + tableBucket, + remoteLogFetchInfo, + records, + highWatermark, + filteredEndOffset, + minRetainOffset, + error, + null); + } + + private FetchLogResultForBucket( + TableBucket tableBucket, + @Nullable RemoteLogFetchInfo remoteLogFetchInfo, + @Nullable LogRecords records, + long highWatermark, + long filteredEndOffset, + long minRetainOffset, + ApiError error, + @Nullable FetchLogEpochInfo epochInfo) { super(tableBucket, error); + this.epochInfo = epochInfo; this.remoteLogFetchInfo = remoteLogFetchInfo; this.records = records; this.highWatermark = highWatermark; @@ -55,6 +77,24 @@ private FetchLogResultForBucket( this.minRetainOffset = minRetainOffset; } + /** Returns a copy carrying replica epoch information. */ + public FetchLogResultForBucket withEpochInfo(@Nullable FetchLogEpochInfo epochInfo) { + return new FetchLogResultForBucket( + getTableBucket(), + remoteLogFetchInfo, + records, + highWatermark, + filteredEndOffset, + minRetainOffset, + getError(), + epochInfo); + } + + @Nullable + public FetchLogEpochInfo epochInfo() { + return epochInfo; + } + /** Creates a successful local fetch result. */ public static FetchLogResultForBucket records( TableBucket tableBucket, diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java index 37d9f907449..e7082a9f5a1 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java @@ -44,7 +44,7 @@ public enum ApiKeys { UPDATE_METADATA(1013, 0, 0, PRIVATE), // Version 1: Supports original_partition_name in requests and responses for historical writes. PRODUCE_LOG(1014, 0, 1, PUBLIC), - FETCH_LOG(1015, 0, 0, PUBLIC), + FETCH_LOG(1015, 0, 1, PUBLIC), // Version 0: Uses lake's encoder for primary key encoding (legacy behavior). // Version 1: Uses CompactedKeyEncoder for primary key encoding when bucket key differs from diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java index 8733056f1f8..5233357d2b7 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java @@ -17,6 +17,7 @@ package org.apache.fluss.rpc.util; +import org.apache.fluss.metadata.LeaderEpochOffset; import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.ResolvedPartitionSpec; import org.apache.fluss.metadata.TableBucket; @@ -25,12 +26,14 @@ import org.apache.fluss.record.MemoryLogRecords; import org.apache.fluss.remote.RemoteLogFetchInfo; import org.apache.fluss.remote.RemoteLogSegment; +import org.apache.fluss.rpc.entity.FetchLogEpochInfo; import org.apache.fluss.rpc.entity.FetchLogResultForBucket; import org.apache.fluss.rpc.messages.LookupRequest; import org.apache.fluss.rpc.messages.PbAclFilter; import org.apache.fluss.rpc.messages.PbAclInfo; import org.apache.fluss.rpc.messages.PbFetchLogRespForBucket; import org.apache.fluss.rpc.messages.PbKeyValue; +import org.apache.fluss.rpc.messages.PbLeaderEpochOffset; import org.apache.fluss.rpc.messages.PbPartitionSpec; import org.apache.fluss.rpc.messages.PbRemoteLogFetchInfo; import org.apache.fluss.rpc.messages.PbRemoteLogSegment; @@ -208,8 +211,14 @@ public static FetchLogResultForBucket getFetchLogResultForBucket( pbRemoteLogSegment.hasMaxTimestamp() ? pbRemoteLogSegment.getMaxTimestamp() : -1; + List segmentEpochs = new ArrayList<>(); + for (PbLeaderEpochOffset epoch : pbRemoteLogSegment.getLeaderEpochsList()) { + segmentEpochs.add( + new LeaderEpochOffset(epoch.getEpoch(), epoch.getOffset())); + } RemoteLogSegment remoteLogSegment = RemoteLogSegment.Builder.builder() + .leaderEpochs(segmentEpochs) .tableBucket(tb) .physicalTablePath(physicalTablePath) .remoteLogSegmentId( @@ -253,6 +262,22 @@ public static FetchLogResultForBucket getFetchLogResultForBucket( } } + if (respForBucket.hasCurrentLeaderEpoch()) { + List starts = new ArrayList<>(); + for (PbLeaderEpochOffset start : respForBucket.getEpochStartsList()) { + starts.add(new LeaderEpochOffset(start.getEpoch(), start.getOffset())); + } + LeaderEpochOffset divergence = + respForBucket.hasDivergingEpoch() + ? new LeaderEpochOffset( + respForBucket.getDivergingEpoch().getEpoch(), + respForBucket.getDivergingEpoch().getOffset()) + : null; + fetchLogResultForBucket = + fetchLogResultForBucket.withEpochInfo( + new FetchLogEpochInfo( + respForBucket.getCurrentLeaderEpoch(), divergence, starts)); + } return fetchLogResultForBucket; } diff --git a/fluss-rpc/src/main/proto/FlussApi.proto b/fluss-rpc/src/main/proto/FlussApi.proto index b778db8c24a..71c0a2d2c1d 100644 --- a/fluss-rpc/src/main/proto/FlussApi.proto +++ b/fluss-rpc/src/main/proto/FlussApi.proto @@ -934,16 +934,23 @@ message PbFetchLogReqForTable { message PbFetchLogReqForBucket { optional int64 partition_id = 1; required int32 bucket_id = 2; - // TODO leader epoch required int64 fetch_offset = 3; required int32 max_fetch_bytes = 4; optional int32 routing_bucket_count = 5; + // Replication only. Both fields must be present to request epoch validation. + optional int32 current_leader_epoch = 6; + optional int32 last_fetched_epoch = 7; } message PbFetchLogRespForTable { required int64 table_id = 1; repeated PbFetchLogRespForBucket buckets_resp = 2; } +message PbLeaderEpochOffset { + required int32 epoch = 1; + required int64 offset = 2; +} + message PbFetchLogRespForBucket { optional int64 partition_id = 1; required int32 bucket_id = 2; @@ -960,6 +967,9 @@ message PbFetchLogRespForBucket { // The safe local log retention boundary confirmed by a committed KV snapshot. This is only // returned for KV follower fetches and is distinct from the physical log_start_offset. optional int64 min_retain_offset = 10; + optional int32 current_leader_epoch = 11; + optional PbLeaderEpochOffset diverging_epoch = 12; + repeated PbLeaderEpochOffset epoch_starts = 13; } message PbPutKvReqForBucket { @@ -1157,6 +1167,7 @@ message PbRemoteLogSegment { required int64 remote_log_end_offset = 3; required int32 segment_size_in_bytes = 4; optional int64 max_timestamp = 5; + repeated PbLeaderEpochOffset leader_epochs = 6; } message PbPartitionInfo { diff --git a/fluss-rust/crates/fluss/proto/FlussApi.proto b/fluss-rust/crates/fluss/proto/FlussApi.proto index b778db8c24a..71c0a2d2c1d 100644 --- a/fluss-rust/crates/fluss/proto/FlussApi.proto +++ b/fluss-rust/crates/fluss/proto/FlussApi.proto @@ -934,16 +934,23 @@ message PbFetchLogReqForTable { message PbFetchLogReqForBucket { optional int64 partition_id = 1; required int32 bucket_id = 2; - // TODO leader epoch required int64 fetch_offset = 3; required int32 max_fetch_bytes = 4; optional int32 routing_bucket_count = 5; + // Replication only. Both fields must be present to request epoch validation. + optional int32 current_leader_epoch = 6; + optional int32 last_fetched_epoch = 7; } message PbFetchLogRespForTable { required int64 table_id = 1; repeated PbFetchLogRespForBucket buckets_resp = 2; } +message PbLeaderEpochOffset { + required int32 epoch = 1; + required int64 offset = 2; +} + message PbFetchLogRespForBucket { optional int64 partition_id = 1; required int32 bucket_id = 2; @@ -960,6 +967,9 @@ message PbFetchLogRespForBucket { // The safe local log retention boundary confirmed by a committed KV snapshot. This is only // returned for KV follower fetches and is distinct from the physical log_start_offset. optional int64 min_retain_offset = 10; + optional int32 current_leader_epoch = 11; + optional PbLeaderEpochOffset diverging_epoch = 12; + repeated PbLeaderEpochOffset epoch_starts = 13; } message PbPutKvReqForBucket { @@ -1157,6 +1167,7 @@ message PbRemoteLogSegment { required int64 remote_log_end_offset = 3; required int32 segment_size_in_bytes = 4; optional int64 max_timestamp = 5; + repeated PbLeaderEpochOffset leader_epochs = 6; } message PbPartitionInfo { diff --git a/fluss-rust/crates/fluss/src/client/table/scanner.rs b/fluss-rust/crates/fluss/src/client/table/scanner.rs index a8720b5ef79..08f57f63afd 100644 --- a/fluss-rust/crates/fluss/src/client/table/scanner.rs +++ b/fluss-rust/crates/fluss/src/client/table/scanner.rs @@ -2358,6 +2358,8 @@ impl LogFetcher { fetch_offset: offset, max_fetch_bytes: self.fetch_max_bytes_for_bucket, routing_bucket_count: None, + current_leader_epoch: None, + last_fetched_epoch: None, }; fetch_log_req_for_buckets @@ -3000,6 +3002,9 @@ mod tests { records: None, filtered_end_offset, min_retain_offset: None, + current_leader_epoch: None, + diverging_epoch: None, + epoch_starts: Vec::new(), }], }], } @@ -3055,6 +3060,9 @@ mod tests { records: None, filtered_end_offset: None, min_retain_offset: None, + current_leader_epoch: None, + diverging_epoch: None, + epoch_starts: Vec::new(), }], }], }; @@ -3115,6 +3123,9 @@ mod tests { records: None, filtered_end_offset: None, min_retain_offset: None, + current_leader_epoch: None, + diverging_epoch: None, + epoch_starts: Vec::new(), }], }], }; @@ -3465,6 +3476,9 @@ mod tests { records: None, filtered_end_offset: None, min_retain_offset: None, + current_leader_epoch: None, + diverging_epoch: None, + epoch_starts: Vec::new(), }], }], }; diff --git a/fluss-rust/crates/fluss/src/proto/fluss.rs b/fluss-rust/crates/fluss/src/proto/fluss.rs index 4e4e06d4124..c890a4e96cd 100644 --- a/fluss-rust/crates/fluss/src/proto/fluss.rs +++ b/fluss-rust/crates/fluss/src/proto/fluss.rs @@ -1241,13 +1241,17 @@ pub struct PbFetchLogReqForBucket { pub partition_id: ::core::option::Option, #[prost(int32, required, tag = "2")] pub bucket_id: i32, - /// TODO leader epoch #[prost(int64, required, tag = "3")] pub fetch_offset: i64, #[prost(int32, required, tag = "4")] pub max_fetch_bytes: i32, #[prost(int32, optional, tag = "5")] pub routing_bucket_count: ::core::option::Option, + /// Replication only. Both fields must be present to request epoch validation. + #[prost(int32, optional, tag = "6")] + pub current_leader_epoch: ::core::option::Option, + #[prost(int32, optional, tag = "7")] + pub last_fetched_epoch: ::core::option::Option, } #[derive(Clone, PartialEq, ::prost::Message)] pub struct PbFetchLogRespForTable { @@ -1256,6 +1260,13 @@ pub struct PbFetchLogRespForTable { #[prost(message, repeated, tag = "2")] pub buckets_resp: ::prost::alloc::vec::Vec, } +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbLeaderEpochOffset { + #[prost(int32, required, tag = "1")] + pub epoch: i32, + #[prost(int64, required, tag = "2")] + pub offset: i64, +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct PbFetchLogRespForBucket { #[prost(int64, optional, tag = "1")] @@ -1284,6 +1295,12 @@ pub struct PbFetchLogRespForBucket { /// returned for KV follower fetches and is distinct from the physical log_start_offset. #[prost(int64, optional, tag = "10")] pub min_retain_offset: ::core::option::Option, + #[prost(int32, optional, tag = "11")] + pub current_leader_epoch: ::core::option::Option, + #[prost(message, optional, tag = "12")] + pub diverging_epoch: ::core::option::Option, + #[prost(message, repeated, tag = "13")] + pub epoch_starts: ::prost::alloc::vec::Vec, } #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct PbPutKvReqForBucket { @@ -1570,7 +1587,7 @@ pub struct PbRemoteLogFetchInfo { #[prost(int32, optional, tag = "4")] pub first_start_pos: ::core::option::Option, } -#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct PbRemoteLogSegment { #[prost(string, required, tag = "1")] pub remote_log_segment_id: ::prost::alloc::string::String, @@ -1582,6 +1599,8 @@ pub struct PbRemoteLogSegment { pub segment_size_in_bytes: i32, #[prost(int64, optional, tag = "5")] pub max_timestamp: ::core::option::Option, + #[prost(message, repeated, tag = "6")] + pub leader_epochs: ::prost::alloc::vec::Vec, } #[derive(Clone, PartialEq, ::prost::Message)] pub struct PbPartitionInfo { diff --git a/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java b/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java index 5c891dd2edb..3fd548d2bcb 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java @@ -52,6 +52,7 @@ import static org.apache.fluss.config.ConfigOptions.KV_LEADER_REPLICA_MEMORY_RESERVED; import static org.apache.fluss.config.ConfigOptions.KV_SHARED_RATE_LIMITER_BYTES_PER_SEC; import static org.apache.fluss.config.ConfigOptions.KV_SNAPSHOT_INTERVAL; +import static org.apache.fluss.config.ConfigOptions.LOG_REPLICATION_LEADER_EPOCH_ENABLED; import static org.apache.fluss.config.ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER; import static org.apache.fluss.config.ConfigOptions.LOG_RETENTION_ROLL_ACTIVE_SEGMENT_ENABLED; import static org.apache.fluss.config.ConfigOptions.NETTY_SERVER_MAX_QUEUED_HISTORICAL_REQUESTS; @@ -83,6 +84,7 @@ class DynamicServerConfig { Arrays.asList( DATALAKE_FORMAT.key(), LOG_RETENTION_ROLL_ACTIVE_SEGMENT_ENABLED.key(), + LOG_REPLICATION_LEADER_EPOCH_ENABLED.key(), LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER.key(), KV_LEADER_REPLICA_MEMORY_RESERVED.key(), KV_SHARED_RATE_LIMITER_BYTES_PER_SEC.key(), diff --git a/fluss-server/src/main/java/org/apache/fluss/server/entity/FetchReqInfo.java b/fluss-server/src/main/java/org/apache/fluss/server/entity/FetchReqInfo.java index 0f8df680028..102a3f15116 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/entity/FetchReqInfo.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/entity/FetchReqInfo.java @@ -32,6 +32,8 @@ public final class FetchReqInfo { @Nullable private final int[] projectFields; private int maxBytes; + private final int currentLeaderEpoch; + private final int lastFetchedEpoch; public FetchReqInfo(long tableId, long fetchOffset, int maxBytes) { this(tableId, fetchOffset, maxBytes, null); @@ -39,12 +41,32 @@ public FetchReqInfo(long tableId, long fetchOffset, int maxBytes) { public FetchReqInfo( long tableId, long fetchOffset, int maxBytes, @Nullable int[] projectFields) { + this(tableId, fetchOffset, maxBytes, projectFields, -1, -1); + } + + public FetchReqInfo( + long tableId, + long fetchOffset, + int maxBytes, + @Nullable int[] projectFields, + int currentLeaderEpoch, + int lastFetchedEpoch) { + this.currentLeaderEpoch = currentLeaderEpoch; + this.lastFetchedEpoch = lastFetchedEpoch; this.tableId = tableId; this.fetchOffset = fetchOffset; this.maxBytes = maxBytes; this.projectFields = projectFields; } + public int currentLeaderEpoch() { + return currentLeaderEpoch; + } + + public int lastFetchedEpoch() { + return lastFetchedEpoch; + } + public long getTableId() { return tableId; } @@ -97,11 +119,20 @@ public boolean equals(Object o) { return false; } - return fetchOffset == fetchReqInfo.fetchOffset && maxBytes == fetchReqInfo.maxBytes; + return fetchOffset == fetchReqInfo.fetchOffset + && maxBytes == fetchReqInfo.maxBytes + && currentLeaderEpoch == fetchReqInfo.currentLeaderEpoch + && lastFetchedEpoch == fetchReqInfo.lastFetchedEpoch; } @Override public int hashCode() { - return Objects.hash(tableId, fetchOffset, maxBytes, Arrays.hashCode(projectFields)); + return Objects.hash( + tableId, + fetchOffset, + maxBytes, + Arrays.hashCode(projectFields), + currentLeaderEpoch, + lastFetchedEpoch); } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParams.java b/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParams.java index 76a30a1ddc7..87ebade8fa9 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParams.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParams.java @@ -71,7 +71,8 @@ public final class FetchParams { private final int minFetchBytes; private final long maxWaitMs; private final FetchLogReadPreference readPreference; - // TODO: add more params like epoch etc. + private int currentLeaderEpoch = -1; + private int lastFetchedEpoch = -1; public FetchParams(int replicaId, int maxFetchBytes) { this( @@ -134,6 +135,20 @@ public FetchParams( FetchLogReadPreference.LOCAL_FIRST); } + /** Sets replication epoch fields for the bucket currently being fetched. */ + public void setCurrentFetchEpoch(int currentLeaderEpoch, int lastFetchedEpoch) { + this.currentLeaderEpoch = currentLeaderEpoch; + this.lastFetchedEpoch = lastFetchedEpoch; + } + + public int currentLeaderEpoch() { + return currentLeaderEpoch; + } + + public int lastFetchedEpoch() { + return lastFetchedEpoch; + } + public void setCurrentFetch( long tableId, long fetchOffset, diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LeaderEpochHistory.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LeaderEpochHistory.java new file mode 100644 index 00000000000..0a735668eb0 --- /dev/null +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LeaderEpochHistory.java @@ -0,0 +1,253 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.log; + +import org.apache.fluss.exception.LogStorageException; +import org.apache.fluss.metadata.LeaderEpochOffset; +import org.apache.fluss.server.log.checkpoint.CheckpointFile; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Optional; +import java.util.TreeMap; + +import static org.apache.fluss.utils.Preconditions.checkArgument; + +/** + * Persistent epoch boundaries for a tablet's log. Epochs may be skipped, and an epoch may contain + * no records. Offsets before the first boundary have unknown history. + * + *

The caller serializes all reads and mutations with log appends and truncation. Boundaries must + * be persisted before their records are appended. Log truncation must be durable before boundaries + * are removed. + */ +final class LeaderEpochHistory { + private final CheckpointFile checkpoint; + private final NavigableMap epochs = new TreeMap<>(); + private IOException persistenceFailure; + + LeaderEpochHistory(File file) throws IOException { + checkpoint = new CheckpointFile<>(file, 0, new EntryFormatter()); + for (LeaderEpochOffset entry : checkpoint.read()) { + if (entry.epoch() < 0 + || entry.offset() < 0 + || (!epochs.isEmpty() + && (entry.epoch() <= epochs.lastKey() + || entry.offset() < epochs.lastEntry().getValue()))) { + throw new IOException("Invalid leader epoch history in " + file); + } + epochs.put(entry.epoch(), entry.offset()); + } + } + + /** Persists an epoch boundary before publishing it to readers. */ + void assign(int epoch, long startOffset) throws IOException { + ensureUsable(); + checkArgument(epoch >= 0 && startOffset >= 0, "Negative epoch or offset."); + Long existing = epochs.get(epoch); + if (existing != null) { + // A restarted leader can resume the latest epoch at a later log end. + checkArgument(epoch == epochs.lastKey(), "Epoch must not decrease."); + checkArgument(startOffset >= existing, "Epoch already starts at %s", existing); + return; + } + checkArgument(epochs.isEmpty() || epoch > epochs.lastKey(), "Epoch must increase."); + checkArgument( + epochs.isEmpty() || startOffset >= epochs.lastEntry().getValue(), + "Epoch offsets must not decrease."); + NavigableMap updated = new TreeMap<>(epochs); + updated.put(epoch, startOffset); + persist(updated); + } + + /** Returns the epoch of an actual record, excluding empty epochs at the log end. */ + int epochForOffset(long offset, long logEndOffset) { + ensureUsable(); + if (offset < 0 || offset >= logEndOffset) { + return -1; + } + int result = -1; + for (Map.Entry entry : epochs.entrySet()) { + if (entry.getValue() > offset) { + break; + } + result = entry.getKey(); + } + return result; + } + + /** Returns the end of the greatest known epoch no greater than the requested epoch. */ + Optional endOffsetFor(int requestedEpoch, long logEndOffset) { + ensureUsable(); + Map.Entry entry = epochs.floorEntry(requestedEpoch); + if (entry == null || entry.getValue() > logEndOffset) { + return Optional.empty(); + } + Map.Entry next = epochs.higherEntry(entry.getKey()); + return Optional.of( + new LeaderEpochOffset( + entry.getKey(), + next == null ? logEndOffset : Math.min(next.getValue(), logEndOffset))); + } + + /** Removes boundaries for records discarded by a durable log truncation. */ + void truncateFromEnd(long endOffset) throws IOException { + ensureUsable(); + checkArgument(endOffset >= 0, "Negative log end offset."); + NavigableMap updated = new TreeMap<>(epochs); + updated.entrySet().removeIf(entry -> entry.getValue() >= endOffset); + if (!updated.equals(epochs)) { + persist(updated); + } + } + + /** Retains the epoch covering the first retained record and all later boundaries. */ + void truncateFromStart(long startOffset) throws IOException { + ensureUsable(); + int retainedEpoch = epochForOffset(startOffset, Long.MAX_VALUE); + if (retainedEpoch < 0) { + return; + } + NavigableMap updated = new TreeMap<>(epochs.tailMap(retainedEpoch, true)); + if (!updated.equals(epochs)) { + persist(updated); + } + } + + /** + * Discards epoch knowledge without changing WAL bytes, before an untracked append or restore. + */ + void invalidate() throws IOException { + ensureUsable(); + if (!epochs.isEmpty()) { + persist(new TreeMap<>()); + } + } + + /** Returns boundaries covering a fetched range; the first may precede its start. */ + List entries(long startOffset, long endOffset) { + ensureUsable(); + List result = new ArrayList<>(); + if (endOffset <= startOffset) { + return result; + } + int first = epochForOffset(startOffset, Long.MAX_VALUE); + for (Map.Entry entry : epochs.entrySet()) { + if (entry.getValue() >= endOffset) { + break; + } + if (entry.getKey() >= first) { + result.add(new LeaderEpochOffset(entry.getKey(), entry.getValue())); + } + } + return result; + } + + /** Persists the epoch boundaries accompanying a contiguous replication append. */ + void append(List entries, long startOffset, long endOffset) + throws IOException { + ensureUsable(); + if (endOffset <= startOffset) { + return; + } + NavigableMap updated = new TreeMap<>(epochs); + for (LeaderEpochOffset entry : entries) { + if (entry.offset() >= endOffset) { + break; + } + Long existing = updated.get(entry.epoch()); + if (existing != null) { + checkArgument( + existing == entry.offset(), + "Conflicting start offset for epoch %s", + entry.epoch()); + continue; + } + // A source boundary can precede this fetch, but it cannot establish the identity + // of bytes we did not copy. Preserve unknown history until a new boundary is fetched. + if (entry.offset() < startOffset) { + continue; + } + checkArgument(entry.epoch() >= 0 && entry.offset() >= 0, "Negative epoch boundary."); + checkArgument( + updated.isEmpty() + || (entry.epoch() > updated.lastKey() + && entry.offset() >= updated.lastEntry().getValue()), + "Non-monotonic replication epoch history."); + updated.put(entry.epoch(), entry.offset()); + } + if (!updated.equals(epochs)) { + persist(updated); + } + } + + /** Prevents further access after WAL or checkpoint persistence becomes uncertain. */ + void markFailed(IOException cause) { + persistenceFailure = cause; + } + + void ensureUsable() { + if (persistenceFailure != null) { + throw new LogStorageException( + "Leader epoch checkpoint failed; the log must be reloaded before further use.", + persistenceFailure); + } + } + + private void persist(NavigableMap updated) throws IOException { + List entries = new ArrayList<>(); + updated.forEach((epoch, offset) -> entries.add(new LeaderEpochOffset(epoch, offset))); + try { + checkpoint.write(entries); + } catch (IOException e) { + // A failed directory sync can follow a successful atomic replacement. Neither the + // old in-memory history nor a retry is safe until the log has been recovered. + persistenceFailure = e; + throw e; + } + epochs.clear(); + epochs.putAll(updated); + } + + private static final class EntryFormatter + implements CheckpointFile.EntryFormatter { + @Override + public String toString(LeaderEpochOffset entry) { + return entry.epoch() + " " + entry.offset(); + } + + @Override + public Optional fromString(String line) { + String[] fields = line.split(" "); + if (fields.length != 2) { + return Optional.empty(); + } + try { + return Optional.of( + new LeaderEpochOffset( + Integer.parseInt(fields[0]), Long.parseLong(fields[1]))); + } catch (NumberFormatException e) { + return Optional.empty(); + } + } + } +} diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LogManager.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LogManager.java index 4c1a84dd1fa..7186acf77be 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/LogManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LogManager.java @@ -412,6 +412,20 @@ public void truncateTo(TableBucket tableBucket, long offset) throws LogStorageEx } } + /** Applies epoch reconciliation only while its replication mode is enabled. */ + public boolean truncateToWithEpoch(TableBucket tableBucket, long offset) { + return inLock( + logCreationOrDeletionLock, + () -> { + LogTablet log = currentLogs.get(tableBucket); + if (log == null || !log.isLeaderEpochEnabled()) { + return false; + } + truncateTo(tableBucket, offset); + return true; + }); + } + public void truncateFullyAndStartAt(TableBucket tableBucket, long newOffset) { LogTablet logTablet = currentLogs.get(tableBucket); // If the log tablet does not exist, skip it. @@ -568,6 +582,16 @@ public void validate(Configuration newConfig) { @Override public void reconfigure(Configuration newConfig) { + inLock( + logCreationOrDeletionLock, + () -> { + boolean enabled = + newConfig.get(ConfigOptions.LOG_REPLICATION_LEADER_EPOCH_ENABLED); + for (LogTablet log : currentLogs.values()) { + log.setLeaderEpochEnabled(enabled); + } + conf.set(ConfigOptions.LOG_REPLICATION_LEADER_EPOCH_ENABLED, enabled); + }); boolean newRollExpiredActiveSegmentEnabled = newConfig.get(ConfigOptions.LOG_RETENTION_ROLL_ACTIVE_SEGMENT_ENABLED); boolean oldRollExpiredActiveSegmentEnabled = diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LogReadInfo.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LogReadInfo.java index 0d14fb2412e..5947f85ec5d 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/LogReadInfo.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LogReadInfo.java @@ -18,6 +18,9 @@ package org.apache.fluss.server.log; import org.apache.fluss.annotation.Internal; +import org.apache.fluss.rpc.entity.FetchLogEpochInfo; + +import javax.annotation.Nullable; /** Structure used for lower level reads. */ @Internal @@ -27,18 +30,38 @@ public class LogReadInfo { private final long highWatermark; private final long logEndOffset; private final long minRetainOffset; + @Nullable private final FetchLogEpochInfo epochInfo; public LogReadInfo( FetchDataInfo fetchedData, long highWatermark, long logEndOffset, long minRetainOffset) { + this(fetchedData, highWatermark, logEndOffset, minRetainOffset, null); + } + + public LogReadInfo( + FetchDataInfo fetchedData, + long highWatermark, + long logEndOffset, + long minRetainOffset, + @Nullable FetchLogEpochInfo epochInfo) { + this.epochInfo = epochInfo; this.fetchedData = fetchedData; this.highWatermark = highWatermark; this.logEndOffset = logEndOffset; this.minRetainOffset = minRetainOffset; } + @Nullable + public FetchLogEpochInfo epochInfo() { + return epochInfo; + } + + public LogReadInfo withEpochInfo(@Nullable FetchLogEpochInfo info) { + return new LogReadInfo(fetchedData, highWatermark, logEndOffset, minRetainOffset, info); + } + public FetchDataInfo getFetchedData() { return fetchedData; } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java index 47e0e7eb2c6..caf69113ed6 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java @@ -27,6 +27,7 @@ import org.apache.fluss.exception.InvalidTimestampException; import org.apache.fluss.exception.LogOffsetOutOfRangeException; import org.apache.fluss.exception.LogStorageException; +import org.apache.fluss.metadata.LeaderEpochOffset; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.TableBucket; @@ -42,6 +43,7 @@ import org.apache.fluss.server.log.LocalLog.SegmentDeletionReason; import org.apache.fluss.server.metrics.group.BucketMetricGroup; import org.apache.fluss.server.metrics.group.TabletServerMetricGroup; +import org.apache.fluss.utils.FileUtils; import org.apache.fluss.utils.FlussPaths; import org.apache.fluss.utils.clock.Clock; import org.apache.fluss.utils.concurrent.Scheduler; @@ -88,6 +90,9 @@ public final class LogTablet { // Configured local storage root that owns this tablet, for example /data-0. private final File dataDir; + + @GuardedBy("lock") + private final LeaderEpochHistory leaderEpochHistory; // Logical table/partition identity of this tablet. private final PhysicalTablePath physicalPath; @@ -125,6 +130,7 @@ public final class LogTablet { // The minimum offset that should be retained in the local log. This is used to ensure that, // the offset of kv snapshot should be retained, otherwise, kv recovery will fail. private final AtomicLong minRetainOffset; + private volatile boolean leaderEpochEnabled; // tracking the log start offset in remote storage private volatile long remoteLogStartOffset = Long.MAX_VALUE; // tracking the log end offset in remote storage @@ -154,10 +160,22 @@ private LogTablet( WriterStateManager writerStateManager, TableConfig tableConfig, boolean isChangelog, - Clock clock) { + Clock clock) + throws IOException { this.dataDir = dataDir; this.physicalPath = physicalPath; this.localLog = localLog; + this.leaderEpochEnabled = conf.get(ConfigOptions.LOG_REPLICATION_LEADER_EPOCH_ENABLED); + this.leaderEpochHistory = + new LeaderEpochHistory(new File(getLogDir(), "leader-epoch-checkpoint")); + // LogLoader may have removed an incomplete tail. Make that recovery durable before + // removing boundaries that described it. + localLog.getSegments().activeSegment().flush(); + FileUtils.flushDirIfExists(getLogDir().toPath()); + leaderEpochHistory.truncateFromEnd(localLogEndOffset()); + if (!leaderEpochEnabled) { + leaderEpochHistory.invalidate(); + } this.maxSegmentFileSize = (int) conf.get(ConfigOptions.LOG_SEGMENT_FILE_SIZE).getBytes(); this.logFlushIntervalMessages = conf.get(ConfigOptions.LOG_FLUSH_INTERVAL_MESSAGES); int writerExpirationCheckIntervalMs = @@ -504,6 +522,116 @@ public LogAppendInfo appendAsFollower(MemoryLogRecords records) throws Exception return append(records, false); } + /** Whether this tablet participates in epoch-aware replication. */ + public boolean isLeaderEpochEnabled() { + return leaderEpochEnabled; + } + + /** Changes replication mode without assigning an epoch to previously untracked records. */ + void setLeaderEpochEnabled(boolean enabled) { + synchronized (lock) { + if (enabled == leaderEpochEnabled) { + return; + } + try { + // Persist loss of knowledge before publishing the mode change. Enabling cannot + // assign the current epoch a different start offset from its existing replicas. + leaderEpochHistory.invalidate(); + leaderEpochEnabled = enabled; + } catch (IOException e) { + throw new LogStorageException( + "Failed to change leader epoch mode for " + getTableBucket(), e); + } + } + } + + /** Records the boundary of a newly elected leader before accepting its writes. */ + public void assignLeaderEpoch(int epoch) throws IOException { + synchronized (lock) { + if (!leaderEpochEnabled) { + return; + } + leaderEpochHistory.truncateFromEnd(localLogEndOffset()); + leaderEpochHistory.assign(epoch, localLogEndOffset()); + } + } + + /** Returns the epoch of the last record before the given fetch offset, or -1 if unknown. */ + public int lastFetchedEpoch(long fetchOffset) { + synchronized (lock) { + return leaderEpochHistory.epochForOffset(fetchOffset - 1, localLogEndOffset()); + } + } + + /** Returns the end of the last known epoch at or before the requested epoch. */ + public Optional endOffsetForEpoch(int epoch) { + synchronized (lock) { + return leaderEpochHistory.endOffsetFor(epoch, localLogEndOffset()); + } + } + + /** Returns the epoch boundaries covering a replicated range. */ + public List epochEntries(long start, long end) { + synchronized (lock) { + return leaderEpochHistory.entries(start, end); + } + } + + /** Appends records and their source history under the same log lock. */ + public LogAppendInfo appendAsFollower(MemoryLogRecords records, List epochs) + throws Exception { + if (!leaderEpochEnabled) { + return appendAsFollower(records); + } + synchronized (lock) { + long endOffset = localLogEndOffset(); + for (LogRecordBatch batch : records.batches()) { + checkArgument( + batch.baseLogOffset() == endOffset, + "Non-contiguous replication append at %s, expected %s", + batch.baseLogOffset(), + endOffset); + endOffset = batch.lastLogOffset() + 1; + } + return append(records, false, epochs); + } + } + + /** Forgets unverified history before reading records from a legacy replication source. */ + public void invalidateLeaderEpochHistory() throws IOException { + synchronized (lock) { + leaderEpochHistory.invalidate(); + } + } + + /** Installs the writer snapshot and epoch history of a committed remote log prefix. */ + public void restoreFromRemote( + long endOffset, File writerSnapshot, List epochs) { + synchronized (lock) { + try { + truncateFullyAndStartAt(endOffset); + // Force loading the downloaded snapshot even though the WAL already ends here. + writerStateManager.truncateFullyAndStartAt(0L); + File target = FlussPaths.writerSnapshotFile(getLogDir(), endOffset); + FileUtils.flushFileIfExists(writerSnapshot.toPath()); + FileUtils.atomicMoveWithFallback(writerSnapshot.toPath(), target.toPath()); + writerStateManager.reloadSnapshots(); + loadWriterSnapshot(endOffset); + localLog.getSegments().activeSegment().flush(); + FileUtils.flushDirIfExists(getLogDir().toPath()); + if (leaderEpochEnabled) { + leaderEpochHistory.append(epochs, 0, endOffset); + } + } catch (Exception e) { + IOException failure = + e instanceof IOException ? (IOException) e : new IOException(e); + leaderEpochHistory.markFailed(failure); + throw new LogStorageException( + "Failed to restore remote log for " + getTableBucket(), failure); + } + } + } + /** Read messages from the local log without projection or filter. */ public FetchDataInfo read( long readOffset, int maxLength, FetchIsolation fetchIsolation, boolean minOneMessage) @@ -757,6 +885,14 @@ public void updateLakeMaxTimestamp(long lakeMaxTimestamp) { } private void onHighWatermarkUpdated(long previousHighWatermark, long currentHighWatermark) { + long cleanupToOffset = + remoteLogEndOffset == -1L + ? highestCopiedEndOffset + : Math.min(remoteLogEndOffset, highestCopiedEndOffset); + if (previousHighWatermark < cleanupToOffset && currentHighWatermark >= cleanupToOffset) { + // Remote offsets may arrive before a follower learns the committed high watermark. + deleteSegmentsAlreadyExistsInRemote(); + } if (!isDataLakeEnabled) { return; } @@ -922,6 +1058,14 @@ private LogOffsetMetadata convertToOffsetMetadataOrThrow(long offset) throws IOE */ private LogAppendInfo append(MemoryLogRecords records, boolean appendAsLeader) throws Exception { + return append(records, appendAsLeader, null); + } + + private LogAppendInfo append( + MemoryLogRecords records, + boolean appendAsLeader, + @Nullable List sourceEpochs) + throws Exception { LogAppendInfo appendInfo = analyzeAndValidateRecords(records); // return if we have no valid records. @@ -934,6 +1078,7 @@ private LogAppendInfo append(MemoryLogRecords records, boolean appendAsLeader) synchronized (lock) { localLog.checkIfMemoryMappedBufferClosed(); + leaderEpochHistory.ensureUsable(); if (appendAsLeader) { long offset = localLog.getLocalLogEndOffset(); // assign offsets to the message set. @@ -987,11 +1132,34 @@ private LogAppendInfo append(MemoryLogRecords records, boolean appendAsLeader) // Append the records, and increment the local log end offset immediately after // append because write to the transaction index below may fail, and we want to // ensure that the offsets of future appends still grow monotonically. - localLog.append( - appendInfo.lastOffset(), - appendInfo.maxTimestamp(), - appendInfo.startOffsetOfMaxTimestamp(), - validRecords); + if (!appendAsLeader) { + if (!leaderEpochEnabled) { + sourceEpochs = null; + } + if (sourceEpochs == null + || sourceEpochs.isEmpty() + || sourceEpochs.get(0).offset() > localLogEndOffset() + || (sourceEpochs.get(0).offset() < localLogEndOffset() + && leaderEpochHistory.epochForOffset( + localLogEndOffset() - 1, localLogEndOffset()) + != sourceEpochs.get(0).epoch())) { + leaderEpochHistory.invalidate(); + } + if (sourceEpochs != null) { + leaderEpochHistory.append( + sourceEpochs, localLogEndOffset(), appendInfo.lastOffset() + 1); + } + } + try { + localLog.append( + appendInfo.lastOffset(), + appendInfo.maxTimestamp(), + appendInfo.startOffsetOfMaxTimestamp(), + validRecords); + } catch (IOException e) { + leaderEpochHistory.markFailed(e); + throw e; + } updateHighWatermarkWithLogEndOffset(); // update the writer state. @@ -1148,6 +1316,9 @@ public void roll(Optional expectedNextOffset) throws IOException { /** Truncate this log so that it ends with the greatest offset < targetOffset. */ boolean truncateTo(long targetOffset) throws LogStorageException { + synchronized (lock) { + leaderEpochHistory.ensureUsable(); + } if (targetOffset < 0) { throw new IllegalArgumentException( String.format( @@ -1174,6 +1345,9 @@ boolean truncateTo(long targetOffset) throws LogStorageException { truncateFullyAndStartAt(targetOffset); } else { List deletedSegments = localLog.truncateTo(targetOffset); + localLog.getSegments().activeSegment().flush(); + FileUtils.flushDirIfExists(getLogDir().toPath()); + leaderEpochHistory.truncateFromEnd(localLogEndOffset()); deleteWriterSnapshots(deletedSegments, writerStateManager); rebuildWriterState(targetOffset, writerStateManager); @@ -1185,6 +1359,7 @@ boolean truncateTo(long targetOffset) throws LogStorageException { return true; } catch (IOException e) { + leaderEpochHistory.markFailed(e); throw new LogStorageException( String.format( "Error while truncating log for bucket %s to offset %s.", @@ -1200,11 +1375,13 @@ void truncateFullyAndStartAt(long newOffset) throws LogStorageException { LOG.debug("Truncate and start at offset {} for bucket {}", newOffset, getTableBucket()); synchronized (lock) { try { + leaderEpochHistory.invalidate(); localLog.truncateFullyAndStartAt(newOffset); writerStateManager.truncateFullyAndStartAt(newOffset); rebuildWriterState(newOffset, writerStateManager); updateHighWatermark(localLog.getLocalLogEndOffset()); } catch (IOException e) { + leaderEpochHistory.markFailed(e); throw new LogStorageException( String.format( "Error while truncating log for bucket %s to offset %s.", diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java index e650b03d674..cd4fa377045 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java @@ -307,6 +307,8 @@ private long copyLogSegmentFilesToRemote( .remoteLogEndOffset(segmentEndOffset) .maxTimestamp(segment.maxTimestampSoFar()) .segmentSizeInBytes(sizeInBytes) + .leaderEpochs( + log.epochEntries(segment.getBaseOffset(), segmentEndOffset)) .build(); try { remoteLogStorage.copyLogSegmentFiles(copyRemoteLogSegment, logSegmentFiles); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java index a74056ebc95..964416c4ed6 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java @@ -36,6 +36,7 @@ import org.apache.fluss.exception.TooManyScannersException; import org.apache.fluss.fs.FsPath; import org.apache.fluss.metadata.ChangelogImage; +import org.apache.fluss.metadata.LeaderEpochOffset; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.Schema; @@ -53,6 +54,8 @@ import org.apache.fluss.record.LogRecordReadContext; import org.apache.fluss.record.LogRecords; import org.apache.fluss.record.MemoryLogRecords; +import org.apache.fluss.rpc.entity.FetchLogEpochInfo; +import org.apache.fluss.rpc.entity.FetchLogResultForBucket; import org.apache.fluss.rpc.protocol.Errors; import org.apache.fluss.rpc.protocol.MergeMode; import org.apache.fluss.rpc.util.PredicateMessageUtils; @@ -500,9 +503,10 @@ public void makeLeader(NotifyLeaderAndIsrData data) throws IOException { int requestLeaderEpoch = data.getLeaderEpoch(); if (requestLeaderEpoch > leaderEpoch) { - boolean isNewLeader = !isLeader(); + boolean resetFollowerOffsets = !isLeader(); leaderEpoch = requestLeaderEpoch; onBecomeNewLeader(); + logTablet.assignLeaderEpoch(leaderEpoch); leaderReplicaIdOpt.set(localTabletServerId); // onBecomeNewLeader may recover a KV snapshot, so start the ISR lag // grace period after it completes. @@ -513,7 +517,7 @@ public void makeLeader(NotifyLeaderAndIsrData data) throws IOException { followerReplica.resetFollowerReplicaState( currentTimeMs, leaderEndOffset, - isNewLeader, + resetFollowerOffsets, data.getIsr() .contains(followerReplica.getFollowerId())); } @@ -1240,6 +1244,121 @@ public LogAppendInfo appendRecordsToFollower(MemoryLogRecords memoryLogRecords) return logTablet.appendAsFollower(memoryLogRecords); } + /** Applies a fetch response only while the replica still follows its requested leader epoch. */ + public LogAppendInfo appendRecordsToFollower( + FetchLogResultForBucket data, int expectedLeader, int expectedEpoch) throws Exception { + return inReadLock( + leaderIsrUpdateLock, + () -> { + validateFollowerEpoch(expectedLeader, expectedEpoch); + FetchLogEpochInfo epochInfo = + logTablet.isLeaderEpochEnabled() ? data.epochInfo() : null; + MemoryLogRecords records = (MemoryLogRecords) data.recordsOrEmpty(); + LogAppendInfo appended; + if (epochInfo == null) { + appended = logTablet.appendAsFollower(records); + logTablet.updateHighWatermark(logTablet.localLogEndOffset()); + } else { + if (epochInfo.leaderEpoch() != expectedEpoch) { + throw new FencedLeaderEpochException( + "Fetch response refers to a different leader epoch."); + } + appended = logTablet.appendAsFollower(records, epochInfo.epochStarts()); + logTablet.updateHighWatermark( + Math.min(data.getHighWatermark(), logTablet.localLogEndOffset())); + } + if (isKvTable() && data.hasMinRetainOffset()) { + logTablet.updateMinRetainOffset(data.getMinRetainOffset()); + } + return appended; + }); + } + + /** Truncates a divergent follower tail without crossing a concurrent role change. */ + public void truncateFollowerTo(long offset, int expectedLeader, int expectedEpoch) { + inReadLock( + leaderIsrUpdateLock, + () -> { + validateFollowerEpoch(expectedLeader, expectedEpoch); + logManager.truncateTo(tableBucket, offset); + }); + } + + /** Serializes an epoch-based truncation with role changes and dynamic configuration. */ + public boolean truncateFollowerToEpochOffset( + long offset, int expectedLeader, int expectedEpoch) { + return inReadLock( + leaderIsrUpdateLock, + () -> { + validateFollowerEpoch(expectedLeader, expectedEpoch); + return logManager.truncateToWithEpoch(tableBucket, offset); + }); + } + + public void invalidateFollowerEpochHistory(int expectedLeader, int expectedEpoch) + throws IOException { + inReadLock( + leaderIsrUpdateLock, + () -> { + validateFollowerEpoch(expectedLeader, expectedEpoch); + logTablet.invalidateLeaderEpochHistory(); + }); + } + + public void truncateFollowerFullyAndStartAt( + long offset, int expectedLeader, int expectedEpoch) { + inReadLock( + leaderIsrUpdateLock, + () -> { + validateFollowerEpoch(expectedLeader, expectedEpoch); + truncateFullyAndStartAt(offset); + }); + } + + /** Installs a downloaded remote snapshot only if its requested leader is still current. */ + public void restoreFollowerFromRemote( + long endOffset, + File writerSnapshot, + List epochs, + long leaderHighWatermark, + int expectedLeader, + int expectedEpoch) { + inReadLock( + leaderIsrUpdateLock, + () -> { + validateFollowerEpoch(expectedLeader, expectedEpoch); + logTablet.restoreFromRemote(endOffset, writerSnapshot, epochs); + logTablet.updateHighWatermark(Math.min(endOffset, leaderHighWatermark)); + }); + } + + /** Attaches the leader identity to a remote fetch response after the manifest lookup. */ + public FetchLogResultForBucket withRemoteFetchEpoch( + FetchLogResultForBucket result, int expectedEpoch) { + return inReadLock( + leaderIsrUpdateLock, + () -> { + if (!isLeader() || leaderEpoch != expectedEpoch) { + throw new FencedLeaderEpochException( + "Leader changed during remote log lookup."); + } + return logTablet.isLeaderEpochEnabled() + ? result.withEpochInfo( + new FetchLogEpochInfo( + leaderEpoch, null, Collections.emptyList())) + : result; + }); + } + + private void validateFollowerEpoch(int expectedLeader, int expectedEpoch) { + if (isLeader() + || leaderEpoch != expectedEpoch + || !Objects.equals(getLeaderId(), expectedLeader)) { + throw new FencedLeaderEpochException( + "Replica no longer follows the requested leader epoch."); + } + } + /** * Samples the recent backpressure pressure for piggyback on a put response. Also records the * value on this bucket's {@link BucketMetricGroup} for table-level aggregation. @@ -1409,22 +1528,91 @@ public LogReadInfo fetchRecords(FetchParams fetchParams) throws IOException { physicalPath)); } if (fetchParams.isFromFollower()) { - long followerFetchTimeMs = clock.milliseconds(); - LogReadInfo logReadInfo = + LogReadInfo result = inReadLock( leaderIsrUpdateLock, () -> { LogTablet localLog = localLogOrThrow(fetchParams.fetchOnlyLeader()); - return readRecords(fetchParams, localLog); + boolean epochRequest = + localLog.isLeaderEpochEnabled() + && fetchParams.currentLeaderEpoch() >= 0; + if (epochRequest + && fetchParams.currentLeaderEpoch() != leaderEpoch) { + throw new FencedLeaderEpochException( + "Fetch request refers to a different leader epoch."); + } + if (epochRequest + && fetchParams.lastFetchedEpoch() >= 0 + && localLog.lastFetchedEpoch(fetchParams.fetchOffset()) + != fetchParams.lastFetchedEpoch()) { + Optional boundary = + localLog.endOffsetForEpoch( + fetchParams.lastFetchedEpoch()); + if (boundary.isPresent()) { + if (boundary.get().epoch() == fetchParams.lastFetchedEpoch() + && boundary.get().offset() + >= fetchParams.fetchOffset()) { + boundary = + localLog.endOffsetForEpoch( + fetchParams.lastFetchedEpoch() - 1); + } + if (boundary.isPresent()) { + return new LogReadInfo( + new FetchDataInfo(MemoryLogRecords.EMPTY), + localLog.getHighWatermark(), + localLog.localLogEndOffset(), + -1, + new FetchLogEpochInfo( + leaderEpoch, + boundary.get(), + Collections.emptyList())); + } + } + // Preserve legacy interoperability for prefixes without epoch + // proof; their history remains unknown. + } + long fetchTime = clock.milliseconds(); + LogReadInfo info = readRecords(fetchParams, localLog); + FollowerReplica follower = + getFollowerReplicaOrThrown(fetchParams.replicaId()); + follower.updateFetchState( + info.getFetchedData().getFetchOffsetMetadata(), + fetchTime, + info.getLogEndOffset()); + return info.withEpochInfo( + new FetchLogEpochInfo( + leaderEpoch, + null, + epochRequest + ? localLog.epochEntries( + fetchParams.fetchOffset(), + info.getLogEndOffset()) + : Collections.emptyList())); }); - - FollowerReplica followerReplica = getFollowerReplicaOrThrown(fetchParams.replicaId()); - updateFollowerFetchState( - followerReplica, - logReadInfo.getFetchedData().getFetchOffsetMetadata(), - followerFetchTimeMs, - logReadInfo.getLogEndOffset()); - return logReadInfo; + FetchLogEpochInfo epochInfo = result.epochInfo(); + if (epochInfo.divergingEpoch() == null) { + FollowerReplica follower = followerReplicasMap.get(fetchParams.replicaId()); + if (follower != null) { + maybeExpandISr(follower, epochInfo.leaderEpoch()); + boolean incremented = + inReadLock( + leaderIsrUpdateLock, + () -> + isLeader() + && leaderEpoch == epochInfo.leaderEpoch() + && followerReplicasMap.get( + fetchParams.replicaId()) + == follower + && maybeIncrementLeaderHW( + logTablet, clock.milliseconds())); + if (incremented) { + tryCompleteDelayedOperations(); + } + } + } + return logTablet.isLeaderEpochEnabled() && fetchParams.currentLeaderEpoch() >= 0 + ? result + : result.withEpochInfo(null); } else { return inReadLock( leaderIsrUpdateLock, @@ -1558,45 +1746,6 @@ private void updateAssignmentAndIsr( isrState = new IsrState.CommittedIsrState(isr, standbyReplicas); } - private void updateFollowerFetchState( - FollowerReplica followerReplica, - LogOffsetMetadata followerFetchOffsetMetadata, - long followerFetchTimeMs, - long leaderLogEndOffset) - throws IOException { - long prevFollowerEndOffset = followerReplica.stateSnapshot().getLogEndOffset(); - - // Apply read lock here to avoid the race between ISR updates and the fetch requests from - // rebooted follower. It could break the tablet server epoch checks in the ISR expansion. - inReadLock( - leaderIsrUpdateLock, - () -> - followerReplica.updateFetchState( - followerFetchOffsetMetadata, - followerFetchTimeMs, - leaderLogEndOffset)); - - // Check if this in-sync replica needs to be added to the ISR. - maybeExpandISr(followerReplica); - - // check if the HW of the replica can now be incremented since the replica may already be in - // the ISR and its LEO has just incremented - boolean leaderHWIncremented = false; - if (prevFollowerEndOffset != followerReplica.stateSnapshot().getLogEndOffset()) { - leaderHWIncremented = maybeIncrementLeaderHW(logTablet, followerFetchTimeMs); - } - - if (leaderHWIncremented) { - tryCompleteDelayedOperations(); - } - - LOG.debug( - "Recorded replica {} log end offset (LEO) position {} for bucket {}.", - localTabletServerId, - followerFetchOffsetMetadata.getMessageOffset(), - tableBucket); - } - private FollowerReplica getFollowerReplicaOrThrown(int followerId) { FollowerReplica followerReplica = followerReplicasMap.get(followerId); if (followerReplica == null) { @@ -2091,11 +2240,17 @@ private void validateBucketEpoch(int requestBucketEpoch) { * *

This function can be triggered when a replica's LEO has incremented. */ - private void maybeExpandISr(FollowerReplica followerReplica) { + private void maybeExpandISr(FollowerReplica followerReplica, int expectedEpoch) { boolean needsIsrUpdate = inReadLock( leaderIsrUpdateLock, - () -> !isrState.isInflight() && needsExpandIsr(followerReplica)); + () -> + isLeader() + && leaderEpoch == expectedEpoch + && followerReplicasMap.get(followerReplica.getFollowerId()) + == followerReplica + && !isrState.isInflight() + && needsExpandIsr(followerReplica)); if (needsIsrUpdate) { Optional adjustIsrUpdateOpt = @@ -2105,6 +2260,9 @@ private void maybeExpandISr(FollowerReplica followerReplica) { IsrState currentIsrState = isrState; // check if this replica needs to be added to the ISR. if (isLeader() + && leaderEpoch == expectedEpoch + && followerReplicasMap.get(followerReplica.getFollowerId()) + == followerReplica && currentIsrState instanceof IsrState.CommittedIsrState && needsExpandIsr(followerReplica)) { return Optional.of( diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index c918147e4f7..e3a69d50101 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -1553,6 +1553,12 @@ private void makeFollowers( .map(Replica::getTableBucket) .collect(Collectors.toSet())); + for (Replica replica : replicasBecomeFollower) { + if (!replica.getLogTablet().isLeaderEpochEnabled()) { + replica.truncateTo(replica.getLogHighWatermark()); + } + } + replicasBecomeFollower.forEach( replica -> completeDelayedOperations(replica.getTableBucket())); @@ -1564,7 +1570,8 @@ private void makeFollowers( // TODO this logic need to be removed after we introduce leader epoch cache, and fetcher // manager support truncating while fetching. Trace by // https://github.com/apache/fluss/issues/673 - truncateToHighWatermark(replicasBecomeFollower); + // Preserve the tail until the fetch response identifies the common log history. + // A follower's committed watermark can lag behind the previous leader's watermark. // add fetcher for those follower replicas. addFetcherForReplicas(replicasBecomeFollower, result); @@ -1836,6 +1843,8 @@ public Map readFromLog( } } + fetchParams.setCurrentFetchEpoch( + fetchReqInfo.currentLeaderEpoch(), fetchReqInfo.lastFetchedEpoch()); LogReadInfo readInfo = replica.fetchRecords(fetchParams); // Once we read from a non-empty bucket, we stop ignoring request and bucket @@ -1848,15 +1857,16 @@ public Map readFromLog( limitBytes = Math.max(0, limitBytes - recordBatchSize); FetchLogResultForBucket fetchLogResult = FetchLogResultForBucket.records( - tb, - fetchedData.getRecords(), - readInfo.getHighWatermark(), - fetchedData.hasFilteredEndOffset() - ? fetchedData.getFilteredEndOffset() - : -1L, - readInfo.hasMinRetainOffset() - ? readInfo.getMinRetainOffset() - : -1L); + tb, + fetchedData.getRecords(), + readInfo.getHighWatermark(), + fetchedData.hasFilteredEndOffset() + ? fetchedData.getFilteredEndOffset() + : -1L, + readInfo.hasMinRetainOffset() + ? readInfo.getMinRetainOffset() + : -1L) + .withEpochInfo(readInfo.epochInfo()); logReadResult.put( tb, new LogReadResult(fetchLogResult, fetchedData.getFetchOffsetMetadata())); @@ -1882,6 +1892,19 @@ public Map readFromLog( FetchLogResultForBucket result; if (replica != null && e instanceof LogOffsetOutOfRangeException) { result = handleFetchOutOfRangeException(replica, fetchOffset, e); + if (isFromFollower + && fetchReqInfo.currentLeaderEpoch() >= 0 + && result.fetchFromRemote()) { + try { + result = + replica.withRemoteFetchEpoch( + result, fetchReqInfo.currentLeaderEpoch()); + } catch (Exception epochError) { + result = + FetchLogResultForBucket.error( + tb, ApiError.fromThrowable(epochError)); + } + } } else { result = FetchLogResultForBucket.error(tb, ApiError.fromThrowable(e)); } @@ -2145,6 +2168,11 @@ private void maybeAddDelayedFetchLog( } } + if (fetchLogResultForBucket.epochInfo() != null + && fetchLogResultForBucket.epochInfo().divergingEpoch() != null) { + errorReadingData = true; + break; + } if (!fetchLogResultForBucket.fetchFromRemote()) { hasFetchFromLocal = true; bytesReadable += fetchLogResultForBucket.recordsOrEmpty().sizeInBytes(); @@ -2422,19 +2450,6 @@ private void sweepOrphanTabletDirs( LOG.info("Swept orphan tablet directories for bucket {}", tb); } - private void truncateToHighWatermark(List replicas) { - for (Replica replica : replicas) { - long highWatermark = replica.getLogTablet().getHighWatermark(); - LOG.info( - "Truncating the logEndOffset for replica id {} of table bucket {} to local " - + "highWatermark {} as it becomes the follower", - serverId, - replica.getTableBucket(), - highWatermark); - replica.truncateTo(highWatermark); - } - } - private void validateAndApplyCoordinatorEpoch(int requestCoordinatorEpoch, String requestName) { if (requestCoordinatorEpoch < this.coordinatorEpoch) { String errorMessage = diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/FetchLogContext.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/FetchLogContext.java index 7a8db54ac8a..0f8bdfab35f 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/FetchLogContext.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/FetchLogContext.java @@ -17,22 +17,68 @@ package org.apache.fluss.server.replica.fetcher; +import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TablePath; import org.apache.fluss.rpc.messages.FetchLogRequest; +import org.apache.fluss.rpc.messages.PbFetchLogReqForBucket; +import org.apache.fluss.rpc.messages.PbFetchLogReqForTable; +import java.util.Collections; +import java.util.HashMap; import java.util.Map; /** FetchLogContext to fetch log from leader. */ public class FetchLogContext { private final Map tableIdToTablePath; private final FetchLogRequest fetchLogRequest; + private final Map fetchStates; + private final Map leaderEpochs = new HashMap<>(); + private final Map requests = new HashMap<>(); public FetchLogContext( Map tableIdToTablePath, FetchLogRequest fetchLogRequest) { + this(tableIdToTablePath, fetchLogRequest, Collections.emptyMap()); + } + + public FetchLogContext( + Map tableIdToTablePath, + FetchLogRequest fetchLogRequest, + Map fetchStates) { + this.fetchStates = new HashMap<>(fetchStates); + for (PbFetchLogReqForTable table : fetchLogRequest.getTablesReqsList()) { + for (PbFetchLogReqForBucket bucket : table.getBucketsReqsList()) { + requests.put( + new TableBucket( + table.getTableId(), + bucket.hasPartitionId() ? bucket.getPartitionId() : null, + bucket.getBucketId()), + bucket); + } + } this.tableIdToTablePath = tableIdToTablePath; this.fetchLogRequest = fetchLogRequest; } + public FetchLogContext withFetchStates(Map states) { + return new FetchLogContext(tableIdToTablePath, fetchLogRequest, states); + } + + public void setLeaderEpoch(TableBucket bucket, int epoch) { + leaderEpochs.put(bucket, epoch); + } + + public int leaderEpoch(TableBucket bucket) { + return leaderEpochs.get(bucket); + } + + public boolean matches(TableBucket bucket, BucketFetchStatus state) { + return fetchStates.get(bucket) == state; + } + + public PbFetchLogReqForBucket getRequest(TableBucket bucket) { + return requests.get(bucket); + } + public FetchLogRequest getFetchLogRequest() { return fetchLogRequest; } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThread.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThread.java index 45c22ff2bda..47b34ecd6c7 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThread.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThread.java @@ -22,18 +22,20 @@ import org.apache.fluss.exception.InvalidOffsetException; import org.apache.fluss.exception.InvalidRecordException; import org.apache.fluss.exception.OutOfOrderSequenceException; -import org.apache.fluss.exception.RemoteStorageException; import org.apache.fluss.exception.StorageException; +import org.apache.fluss.metadata.LeaderEpochOffset; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TablePath; import org.apache.fluss.record.MemoryLogRecords; import org.apache.fluss.remote.RemoteLogFetchInfo; import org.apache.fluss.remote.RemoteLogSegment; +import org.apache.fluss.rpc.entity.FetchLogEpochInfo; import org.apache.fluss.rpc.entity.FetchLogResultForBucket; import org.apache.fluss.rpc.messages.FetchLogRequest; +import org.apache.fluss.rpc.messages.PbFetchLogReqForBucket; +import org.apache.fluss.rpc.messages.PbFetchLogReqForTable; import org.apache.fluss.server.log.LogAppendInfo; import org.apache.fluss.server.log.LogTablet; -import org.apache.fluss.server.log.remote.RemoteLogManager; import org.apache.fluss.server.log.remote.RemoteLogStorage.IndexType; import org.apache.fluss.server.metrics.group.TabletServerMetricGroup; import org.apache.fluss.server.replica.Replica; @@ -41,8 +43,6 @@ import org.apache.fluss.server.replica.fetcher.LeaderEndpoint.FetchData; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.shaded.netty4.io.netty.util.ReferenceCountUtil; -import org.apache.fluss.utils.FileUtils; -import org.apache.fluss.utils.FlussPaths; import org.apache.fluss.utils.concurrent.ShutdownableThread; import org.apache.fluss.utils.log.FairBucketStatusMap; @@ -53,11 +53,11 @@ import javax.annotation.concurrent.GuardedBy; import java.io.File; -import java.io.IOException; import java.nio.file.Files; import java.nio.file.StandardCopyOption; import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -149,6 +149,49 @@ private void maybeFetch() { fetchLogContext = leader.buildFetchLogContext( fairBucketStatusMap.bucketStatusMap()); + if (fetchLogContext.isPresent()) { + FetchLogContext context = + fetchLogContext + .get() + .withFetchStates( + fairBucketStatusMap.bucketStatusMap()); + for (PbFetchLogReqForTable table : + context.getFetchLogRequest().getTablesReqsList()) { + for (PbFetchLogReqForBucket bucket : + table.getBucketsReqsList()) { + TableBucket tb = + new TableBucket( + table.getTableId(), + bucket.hasPartitionId() + ? bucket.getPartitionId() + : null, + bucket.getBucketId()); + try { + Replica replica = + replicaManager.getReplicaOrException(tb); + int epoch = replica.getLeaderEpoch(); + context.setLeaderEpoch(tb, epoch); + if (replica.getLogTablet().isLeaderEpochEnabled()) { + bucket.setCurrentLeaderEpoch(epoch) + .setLastFetchedEpoch( + replica.getLogTablet() + .lastFetchedEpoch( + bucket + .getFetchOffset())); + } + + } catch (Exception e) { + LOG.error( + "Cannot read replication history for {}", + tb, + e); + removeBucket(tb); + return Optional.empty(); + } + } + } + fetchLogContext = Optional.of(context); + } if (!fetchLogContext.isPresent()) { LOG.trace( "There are no active buckets. Back off for {} ms before " @@ -265,7 +308,8 @@ private void processFetchLogRequest(FetchLogContext fetchLogContext) { if (responseData != null) { bucketStatusMapLock.lock(); try { - handleFetchLogResponse(responseData.getFetchLogResultMap(), bucketsWithError); + handleFetchLogResponse( + fetchLogContext, responseData.getFetchLogResultMap(), bucketsWithError); } finally { // release buffer handle by fetchLogResponse. releaseFetchDataBuffer(responseData); @@ -289,13 +333,16 @@ private static void releaseFetchDataBuffer(FetchData data) { } private void handleFetchLogResponse( + FetchLogContext context, Map responseData, Set replicasWithError) { responseData.forEach( (tableBucket, replicaData) -> { BucketFetchStatus currentFetchStatus = fairBucketStatusMap.statusValue(tableBucket); - if (currentFetchStatus == null || !currentFetchStatus.isReadyForFetch()) { + if (currentFetchStatus == null + || !currentFetchStatus.isReadyForFetch() + || !context.matches(tableBucket, currentFetchStatus)) { return; } @@ -303,10 +350,17 @@ private void handleFetchLogResponse( switch (replicaData.getError().error()) { case NONE: handleFetchLogResponseOfSuccessBucket( - tableBucket, currentFetchStatus, replicaData); + tableBucket, + currentFetchStatus, + replicaData, + context.getRequest(tableBucket), + context.leaderEpoch(tableBucket)); break; case LOG_OFFSET_OUT_OF_RANGE_EXCEPTION: - if (!handleOutOfRangeError(tableBucket, currentFetchStatus)) { + if (!handleOutOfRangeError( + tableBucket, + currentFetchStatus, + context.leaderEpoch(tableBucket))) { replicasWithError.add(tableBucket); } break; @@ -330,28 +384,61 @@ private void handleFetchLogResponse( private void handleFetchLogResponseOfSuccessBucket( TableBucket tableBucket, BucketFetchStatus currentFetchStatus, - FetchLogResultForBucket replicaData) { + FetchLogResultForBucket replicaData, + PbFetchLogReqForBucket request, + int expectedEpoch) { try { long nextFetchOffset = -1L; + FetchLogEpochInfo epochInfo = replicaData.epochInfo(); + Replica currentReplica = replicaManager.getReplicaOrException(tableBucket); + if (epochInfo != null && !currentReplica.getLogTablet().isLeaderEpochEnabled()) { + // A response prepared before disabling must not apply an epoch-based truncation. + return; + } + if (currentReplica.isLeader() + || currentReplica.getLeaderEpoch() != expectedEpoch + || (epochInfo != null && epochInfo.leaderEpoch() != expectedEpoch)) { + return; + } + if (epochInfo != null && epochInfo.divergingEpoch() != null) { + LeaderEpochOffset divergence = epochInfo.divergingEpoch(); + Optional localEnd = + currentReplica.getLogTablet().endOffsetForEpoch(divergence.epoch()); + if (!localEnd.isPresent()) { + currentReplica.invalidateFollowerEpochHistory( + leader.leaderServerId(), expectedEpoch); + return; + } + long truncateOffset = Math.min(divergence.offset(), localEnd.get().offset()); + if (!currentReplica.truncateFollowerToEpochOffset( + truncateOffset, leader.leaderServerId(), expectedEpoch)) { + return; + } + fairBucketStatusMap.updateAndMoveToEnd( + tableBucket, + new BucketFetchStatus( + currentFetchStatus.tableId(), + currentFetchStatus.tablePath(), + truncateOffset, + null)); + return; + } if (replicaData.fetchFromRemote()) { - nextFetchOffset = processFetchResultFromRemoteStorage(tableBucket, replicaData); + nextFetchOffset = + processFetchResultFromRemoteStorage( + tableBucket, replicaData, expectedEpoch); } else { LogAppendInfo logAppendInfo = processFetchResultFromLocalStorage( - tableBucket, currentFetchStatus.fetchOffset(), replicaData); + tableBucket, + currentFetchStatus.fetchOffset(), + replicaData, + expectedEpoch); if (logAppendInfo.validBytes() > 0) { nextFetchOffset = logAppendInfo.lastOffset() + 1; } } - // Propagate the leader's KV snapshot retention boundary to the follower even when the - // successful response contains no records, so obsolete local log segments can be - // cleaned up. - Replica replica = replicaManager.getReplicaOrException(tableBucket); - if (replica.isKvTable() && replicaData.hasMinRetainOffset()) { - replica.getLogTablet().updateMinRetainOffset(replicaData.getMinRetainOffset()); - } - if (nextFetchOffset != -1L && fairBucketStatusMap.contains(tableBucket)) { BucketFetchStatus newFetchStatus = new BucketFetchStatus( @@ -381,8 +468,17 @@ private void handleFetchLogResponseOfSuccessBucket( } else if (e instanceof DuplicateSequenceException || e instanceof OutOfOrderSequenceException || e instanceof InvalidOffsetException) { - // TODO this part of logic need to be removed after we introduce leader epoch cache. - // Trace by https://github.com/apache/fluss/issues/673 + if (replicaData.epochInfo() != null + && request.hasLastFetchedEpoch() + && request.getLastFetchedEpoch() >= 0) { + LOG.error( + "Replication offset or sequence mismatch after epoch validation for {}", + tableBucket, + e); + removeBucket(tableBucket); + return; + } + // Legacy peers and unknown prefixes still use the existing recovery path. LOG.error( "Founding recoverable error while processing data for bucket {} at offset {}, try to " + "truncate to LeaderEndOffsetSnapshot", @@ -390,7 +486,8 @@ private void handleFetchLogResponseOfSuccessBucket( currentFetchStatus.fetchOffset(), e); try { - truncateToLeaderEndOffsetSnapshot(tableBucket, currentFetchStatus.tablePath()); + truncateToLeaderEndOffsetSnapshot( + tableBucket, currentFetchStatus.tablePath(), expectedEpoch); } catch (Exception ex) { LOG.error( "Error while truncating bucket {} at offset {}", @@ -410,8 +507,8 @@ private void handleFetchLogResponseOfSuccessBucket( } } - private void truncateToLeaderEndOffsetSnapshot(TableBucket tableBucket, TablePath tablePath) - throws Exception { + private void truncateToLeaderEndOffsetSnapshot( + TableBucket tableBucket, TablePath tablePath, int expectedEpoch) throws Exception { long leaderLocalEndOffsetWhileBecomeLeader = leader.fetchLeaderEndOffsetSnapshot(tableBucket).get(); long localLogEndOffset = @@ -419,7 +516,7 @@ private void truncateToLeaderEndOffsetSnapshot(TableBucket tableBucket, TablePat if (leaderLocalEndOffsetWhileBecomeLeader != 0L && leaderLocalEndOffsetWhileBecomeLeader < localLogEndOffset) { // truncate to leaderEndOffsetSnapshot to reset follower's WriterState and fetch offset. - truncate(tableBucket, leaderLocalEndOffsetWhileBecomeLeader); + truncate(tableBucket, leaderLocalEndOffsetWhileBecomeLeader, expectedEpoch); // update fetch status. BucketFetchStatus bucketFetchStatus = @@ -432,9 +529,10 @@ private void truncateToLeaderEndOffsetSnapshot(TableBucket tableBucket, TablePat } } - private boolean handleOutOfRangeError(TableBucket tableBucket, BucketFetchStatus fetchStatus) { + private boolean handleOutOfRangeError( + TableBucket tableBucket, BucketFetchStatus fetchStatus, int expectedEpoch) { try { - BucketFetchStatus newFetchStatus = fetchOffsetAndTruncate(tableBucket); + BucketFetchStatus newFetchStatus = fetchOffsetAndTruncate(tableBucket, expectedEpoch); fairBucketStatusMap.updateAndMoveToEnd(tableBucket, newFetchStatus); LOG.info( "Current offset {} for table bucket {} is out of range, which typically implies " @@ -450,7 +548,8 @@ private boolean handleOutOfRangeError(TableBucket tableBucket, BucketFetchStatus } /** Handle a replica whose offset is out of range and return a new fetch offset. */ - private BucketFetchStatus fetchOffsetAndTruncate(TableBucket tableBucket) throws Exception { + private BucketFetchStatus fetchOffsetAndTruncate(TableBucket tableBucket, int expectedEpoch) + throws Exception { Replica replica = replicaManager.getReplicaOrException(tableBucket); long replicaEndOffset = replica.getLocalLogEndOffset(); @@ -476,7 +575,7 @@ private BucketFetchStatus fetchOffsetAndTruncate(TableBucket tableBucket) throws tableBucket, replicaEndOffset, leaderEndOffset); - truncate(tableBucket, leaderEndOffset); + truncate(tableBucket, leaderEndOffset, expectedEpoch); return new BucketFetchStatus( tableBucket.getTableId(), replica.getTablePath(), leaderEndOffset, null); } else { @@ -514,7 +613,8 @@ private BucketFetchStatus fetchOffsetAndTruncate(TableBucket tableBucket) throws // Only truncate log when current leader's log start offset is greater than follower's // log end offset. if (leaderStartOffset > replicaEndOffset) { - truncateFullyAndStartAt(tableBucket, leaderStartOffset); + replica.truncateFollowerFullyAndStartAt( + leaderStartOffset, leader.leaderServerId(), expectedEpoch); } long offsetToFetch = Math.max(leaderStartOffset, replicaEndOffset); @@ -560,7 +660,10 @@ Optional fetchStatus(TableBucket tableBucket) { } private LogAppendInfo processFetchResultFromLocalStorage( - TableBucket tableBucket, long fetchOffset, FetchLogResultForBucket replicaData) + TableBucket tableBucket, + long fetchOffset, + FetchLogResultForBucket replicaData, + int expectedEpoch) throws Exception { Replica replica = replicaManager.getReplicaOrException(tableBucket); LogTablet logTablet = replica.getLogTablet(); @@ -581,103 +684,62 @@ private LogAppendInfo processFetchResultFromLocalStorage( replicaData.getHighWatermark()); // Append the messages to the follower log tablet. - LogAppendInfo logAppendInfo = replica.appendRecordsToFollower(records); + LogAppendInfo logAppendInfo = + replica.appendRecordsToFollower( + replicaData, leader.leaderServerId(), expectedEpoch); LOG.trace( "Follower has replica log end offset {} after appending {} bytes of messages for replica {}", logTablet.localLogEndOffset(), records.sizeInBytes(), tableBucket); - // For the follower replica, we do not need to keep its segment base offset and physical - // position. These values will be computed upon becoming leader or handling a preferred read - // replica fetch. - // TODO, to avoid lose data in case of leader change, we now change to update highWatermark - // first for follower instead of first for leader. The reason why can see - // https://cwiki.apache.org/confluence/display/KAFKA/KIP-101+-+Alter+Replication+Protocol+to+use+Leader+Epoch+rather+than+High+Watermark+for+Truncation - // for more details. However, this is just a temporary solution, if we want to have a strong - // consistency guarantee, we should do as KIP-101 do, trace by: - // https://github.com/apache/fluss/issues/673 - logTablet.updateHighWatermark(logTablet.localLogEndOffset()); - LOG.trace( - "Follower received high watermark {} from the leader for replica {}", - replicaData.getHighWatermark(), - tableBucket); - serverMetricGroup.replicationBytesIn().inc(records.sizeInBytes()); return logAppendInfo; } private long processFetchResultFromRemoteStorage( - TableBucket tb, FetchLogResultForBucket replicaData) { - RemoteLogFetchInfo rlFetchInfo = replicaData.remoteLogFetchInfo(); - checkNotNull(rlFetchInfo, "RemoteLogFetchInfo is null"); + TableBucket tb, FetchLogResultForBucket replicaData, int expectedEpoch) + throws Exception { + RemoteLogFetchInfo info = + checkNotNull(replicaData.remoteLogFetchInfo(), "RemoteLogFetchInfo is null"); Replica replica = replicaManager.getReplicaOrException(tb); - RemoteLogManager rlm = replicaManager.getRemoteLogManager(); - - // TODO after introduce leader epoch cache, we need to rebuild the local leader epoch - // cache. Trace by https://github.com/apache/fluss/issues/673 - - // update next fetch offset and writer id snapshot in local. - RemoteLogSegment remoteLogSegmentWithMaxStartOffset = - rlFetchInfo - .remoteLogSegmentList() - .get(rlFetchInfo.remoteLogSegmentList().size() - 1); - // build writer snapshots until remoteLogSegment.endOffset() and start segment from - // until remoteLogSegment.endOffset(). - long nextFetchOffset = remoteLogSegmentWithMaxStartOffset.remoteLogEndOffset(); - + RemoteLogSegment segment = + info.remoteLogSegmentList().get(info.remoteLogSegmentList().size() - 1); + long nextOffset = segment.remoteLogEndOffset(); + File download = + Files.createTempFile( + replica.getLogTablet().getLogDir().toPath(), + "writer-snapshot-", + ".tmp") + .toFile(); try { - // Truncate the existing local log before restoring the writer id snapshots. - replica.truncateFullyAndStartAt(nextFetchOffset); - - // TODO maybe need increase log start offset. - - LogTablet log = replica.getLogTablet(); - // 1. Perform a truncate before calling buildWriterIdSnapshotFile() to ensure that all - // historical data is completely cleaned up. - log.writerStateManager().truncateFullyAndStartAt(0L); - - // 2. download writer id snapshots from remote storage. - File snapshotFile = FlussPaths.writerSnapshotFile(log.getLogDir(), nextFetchOffset); - buildWriterIdSnapshotFile(snapshotFile, remoteLogSegmentWithMaxStartOffset, rlm); - - // 3. Perform a reloadSnapshots after buildWriterIdSnapshotFile() to load the latest - // downloaded writerId snapshot file into the writerStateManager. - // Note: This must occur after the file is downloaded, so we cannot call - // truncateFullyAndReloadSnapshots() here to avoid deleting the newly downloaded - // writerId snapshot file. - log.writerStateManager().reloadSnapshots(); - log.loadWriterSnapshot(nextFetchOffset); - LOG.info( - "Build the writer snapshots from remote storage for {} with active " - + "writer size: {} and remoteLogEndOffset: {}", - tb, - log.writerStateManager().activeWriters().size(), - nextFetchOffset); - } catch (Exception e) { - LOG.error( - "Failed to truncate and restore writer snapshot for {} while log hash been moved to remote", - tb, - e); + // Download before replacing local state. A network failure must leave the fetch + // position and the existing WAL intact. + try (java.io.InputStream input = + replicaManager + .getRemoteLogManager() + .getRemoteLogStorage() + .fetchIndex(segment, IndexType.WRITER_ID_SNAPSHOT)) { + Files.copy(input, download.toPath(), StandardCopyOption.REPLACE_EXISTING); + } + FetchLogEpochInfo epochInfo = replicaData.epochInfo(); + List epochs = + epochInfo == null ? Collections.emptyList() : segment.leaderEpochs(); + replica.restoreFollowerFromRemote( + nextOffset, + download, + epochs, + epochInfo == null ? nextOffset : replicaData.getHighWatermark(), + leader.leaderServerId(), + expectedEpoch); + return nextOffset; + } finally { + Files.deleteIfExists(download.toPath()); } - return nextFetchOffset; } - private void buildWriterIdSnapshotFile( - File snapshotFile, RemoteLogSegment remoteLogSegment, RemoteLogManager rlm) - throws RemoteStorageException, IOException { - File tmpSnapshotFile = new File(snapshotFile.getAbsolutePath() + ".tmp"); - // Copy it to snapshot file in atomic manner. - Files.copy( - rlm.getRemoteLogStorage() - .fetchIndex(remoteLogSegment, IndexType.WRITER_ID_SNAPSHOT), - tmpSnapshotFile.toPath(), - StandardCopyOption.REPLACE_EXISTING); - FileUtils.atomicMoveWithFallback(tmpSnapshotFile.toPath(), snapshotFile.toPath(), false); - } - - private void truncate(TableBucket tableBucket, long offset) { + private void truncate(TableBucket tableBucket, long offset, int expectedEpoch) { Replica replica = replicaManager.getReplicaOrException(tableBucket); LogTablet log = replica.getLogTablet(); @@ -689,12 +751,7 @@ private void truncate(TableBucket tableBucket, long offset) { log.getHighWatermark()); } - replica.truncateTo(offset); - } - - private void truncateFullyAndStartAt(TableBucket tableBucket, long offset) { - Replica replica = replicaManager.getReplicaOrException(tableBucket); - replica.truncateFullyAndStartAt(offset); + replica.truncateFollowerTo(offset, leader.leaderServerId(), expectedEpoch); } @Override diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java index 242f92a0ea4..0b579f75bdc 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java @@ -36,6 +36,7 @@ import org.apache.fluss.metadata.AggFunctions; import org.apache.fluss.metadata.DatabaseChange; import org.apache.fluss.metadata.DatabaseSummary; +import org.apache.fluss.metadata.LeaderEpochOffset; import org.apache.fluss.metadata.PartitionSpec; import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.ResolvedPartitionSpec; @@ -54,6 +55,7 @@ import org.apache.fluss.record.MemoryLogRecords; import org.apache.fluss.remote.RemoteLogFetchInfo; import org.apache.fluss.remote.RemoteLogSegment; +import org.apache.fluss.rpc.entity.FetchLogEpochInfo; import org.apache.fluss.rpc.entity.FetchLogResultForBucket; import org.apache.fluss.rpc.entity.LimitScanResultForBucket; import org.apache.fluss.rpc.entity.ListOffsetsResultForBucket; @@ -1062,7 +1064,14 @@ public static Map getFetchLogData(FetchLogRequest req tableId, fetchLogReqForBucket.getFetchOffset(), fetchLogReqForBucket.getMaxFetchBytes(), - projectionFields)); + projectionFields, + fetchLogReqForBucket.hasCurrentLeaderEpoch() + && fetchLogReqForBucket.hasLastFetchedEpoch() + ? fetchLogReqForBucket.getCurrentLeaderEpoch() + : -1, + fetchLogReqForBucket.hasLastFetchedEpoch() + ? fetchLogReqForBucket.getLastFetchedEpoch() + : -1)); } } @@ -1083,6 +1092,22 @@ public static FetchLogResponse makeFetchLogResponse( FetchLogResultForBucket bucketResult = entry.getValue(); PbFetchLogRespForBucket fetchLogRespForBucket = new PbFetchLogRespForBucket().setBucketId(tb.getBucket()); + FetchLogEpochInfo epochInfo = bucketResult.epochInfo(); + if (epochInfo != null) { + fetchLogRespForBucket.setCurrentLeaderEpoch(epochInfo.leaderEpoch()); + if (epochInfo.divergingEpoch() != null) { + fetchLogRespForBucket + .setDivergingEpoch() + .setEpoch(epochInfo.divergingEpoch().epoch()) + .setOffset(epochInfo.divergingEpoch().offset()); + } + for (LeaderEpochOffset start : epochInfo.epochStarts()) { + fetchLogRespForBucket + .addEpochStart() + .setEpoch(start.epoch()) + .setOffset(start.offset()); + } + } if (bucketResult.hasFilteredEndOffset()) { fetchLogRespForBucket.setFilteredEndOffset(bucketResult.getFilteredEndOffset()); } @@ -1115,6 +1140,12 @@ public static FetchLogResponse makeFetchLogResponse( .setRemoteLogEndOffset(logSegment.remoteLogEndOffset()) .setSegmentSizeInBytes(logSegment.segmentSizeInBytes()) .setMaxTimestamp(logSegment.maxTimestamp()); + for (LeaderEpochOffset epoch : logSegment.leaderEpochs()) { + pbRemoteLogSegment + .addLeaderEpoch() + .setEpoch(epoch.epoch()) + .setOffset(epoch.offset()); + } remoteLogSegmentList.add(pbRemoteLogSegment); } fetchLogRespForBucket diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/LeaderEpochHistoryTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/LeaderEpochHistoryTest.java new file mode 100644 index 00000000000..b5020d4c9b4 --- /dev/null +++ b/fluss-server/src/test/java/org/apache/fluss/server/log/LeaderEpochHistoryTest.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.log; + +import org.apache.fluss.exception.LogStorageException; +import org.apache.fluss.metadata.LeaderEpochOffset; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.Collections; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests persistent log history used to find a common prefix after leader changes. */ +class LeaderEpochHistoryTest { + @TempDir private File directory; + + @Test + void testUnknownPrefixAndSkippedEpochsSurviveRestart() throws Exception { + File file = new File(directory, "leader-epoch-checkpoint"); + LeaderEpochHistory history = new LeaderEpochHistory(file); + history.assign(41, 100); + history.assign(45, 160); + history = new LeaderEpochHistory(file); + + assertThat(history.epochForOffset(99, 200)).isEqualTo(-1); + assertThat(history.epochForOffset(159, 200)).isEqualTo(41); + assertThat(history.epochForOffset(160, 200)).isEqualTo(45); + assertThat(history.endOffsetFor(40, 200)).isEmpty(); + LeaderEpochOffset preceding = history.endOffsetFor(44, 200).get(); + assertThat(preceding.epoch()).isEqualTo(41); + assertThat(preceding.offset()).isEqualTo(160); + assertThat(history.endOffsetFor(45, 200).get().offset()).isEqualTo(200); + } + + @Test + void testResumingLatestEpochPreservesItsStartAcrossRestart() throws Exception { + File file = new File(directory, "epochs"); + LeaderEpochHistory history = new LeaderEpochHistory(file); + history.assign(1, 0); + history.assign(2, 10); + history = new LeaderEpochHistory(file); + history.assign(2, 20); + history = new LeaderEpochHistory(file); + assertThat(history.epochForOffset(9, 20)).isEqualTo(1); + assertThat(history.epochForOffset(10, 20)).isEqualTo(2); + assertThat(history.endOffsetFor(1, 20).get().offset()).isEqualTo(10); + LeaderEpochHistory recovered = history; + assertThatThrownBy(() -> recovered.assign(1, 20)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> recovered.assign(2, 9)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void testEmptyEpochDoesNotChangeLastRecordEpoch() throws Exception { + LeaderEpochHistory history = new LeaderEpochHistory(new File(directory, "epochs")); + history.assign(1, 0); + history.assign(2, 10); + history.assign(3, 10); + assertThat(history.epochForOffset(9, 10)).isEqualTo(1); + assertThat(history.epochForOffset(10, 10)).isEqualTo(-1); + assertThat(history.endOffsetFor(2, 10).get().offset()).isEqualTo(10); + assertThat(history.epochForOffset(10, 11)).isEqualTo(3); + } + + @Test + void testTruncateAndReplaceTailSurvivesRestart() throws Exception { + File file = new File(directory, "epochs"); + LeaderEpochHistory history = new LeaderEpochHistory(file); + history.assign(1, 0); + history.assign(2, 10); + history.assign(3, 20); + history.truncateFromEnd(10); + history.assign(4, 10); + history = new LeaderEpochHistory(file); + assertThat(history.epochForOffset(9, 15)).isEqualTo(1); + assertThat(history.epochForOffset(10, 15)).isEqualTo(4); + assertThat(history.endOffsetFor(3, 15).get().epoch()).isEqualTo(1); + assertThat(history.endOffsetFor(3, 15).get().offset()).isEqualTo(10); + } + + @Test + void testRetentionPreservesEpochCoveringFirstRecord() throws Exception { + File file = new File(directory, "epochs"); + LeaderEpochHistory history = new LeaderEpochHistory(file); + history.assign(1, 0); + history.assign(2, 10); + history.assign(3, 20); + history.truncateFromStart(15); + history = new LeaderEpochHistory(file); + assertThat(history.endOffsetFor(1, 30)).isEmpty(); + assertThat(history.epochForOffset(15, 30)).isEqualTo(2); + assertThat(history.endOffsetFor(2, 30).get().offset()).isEqualTo(20); + } + + @Test + void testUntrackedAppendInvalidatesHistoryAcrossRestart() throws Exception { + File file = new File(directory, "epochs"); + LeaderEpochHistory history = new LeaderEpochHistory(file); + history.assign(41, 0); + assertThat(history.epochForOffset(9, 10)).isEqualTo(41); + history.invalidate(); + history = new LeaderEpochHistory(file); + assertThat(history.epochForOffset(19, 20)).isEqualTo(-1); + assertThat(history.endOffsetFor(41, 20)).isEmpty(); + history.assign(45, 20); + assertThat(history.epochForOffset(19, 25)).isEqualTo(-1); + assertThat(history.epochForOffset(20, 25)).isEqualTo(45); + } + + @Test + void testFetchedBoundaryDoesNotIdentifyUncopiedPrefix() throws Exception { + LeaderEpochHistory history = new LeaderEpochHistory(new File(directory, "epochs")); + history.append(Collections.singletonList(new LeaderEpochOffset(41, 0)), 10, 10); + assertThat(history.epochForOffset(9, 10)).isEqualTo(-1); + history.append( + Arrays.asList(new LeaderEpochOffset(41, 0), new LeaderEpochOffset(45, 15)), 10, 20); + assertThat(history.epochForOffset(9, 20)).isEqualTo(-1); + assertThat(history.epochForOffset(14, 20)).isEqualTo(-1); + assertThat(history.epochForOffset(15, 20)).isEqualTo(45); + assertThat(history.entries(20, 20)).isEmpty(); + } + + @Test + void testFailedCheckpointStopsHistoryUntilReload() throws Exception { + File file = new File(directory, "epochs"); + LeaderEpochHistory history = new LeaderEpochHistory(file); + history.assign(1, 0); + Files.createDirectory(new File(directory, "epochs.tmp").toPath()); + assertThatThrownBy(() -> history.assign(2, 10)).isInstanceOf(IOException.class); + assertThatThrownBy(() -> history.epochForOffset(10, 15)) + .isInstanceOf(LogStorageException.class); + assertThatThrownBy(() -> history.assign(1, 0)).isInstanceOf(LogStorageException.class); + assertThat(new LeaderEpochHistory(file).epochForOffset(10, 15)).isEqualTo(1); + } +} diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/LogTabletTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/LogTabletTest.java index b133eeccad2..48274293bb3 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/log/LogTabletTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/log/LogTabletTest.java @@ -21,6 +21,7 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.config.MemorySize; import org.apache.fluss.exception.OutOfOrderSequenceException; +import org.apache.fluss.metadata.LeaderEpochOffset; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.record.LogRecord; @@ -42,6 +43,7 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Files; import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; @@ -114,6 +116,135 @@ public void teardown() throws Exception { scheduler.shutdown(); } + @Test + void testDisablingEpochHistorySurvivesReenableWithExistingData() throws Exception { + logTablet.assignLeaderEpoch(3); + logTablet.appendAsLeader(genMemoryLogRecordsWithWriterId(DATA1, 1L, 0, 0L)); + Configuration disabled = new Configuration(conf); + disabled.set(ConfigOptions.LOG_REPLICATION_LEADER_EPOCH_ENABLED, false); + reopenLogTablet(disabled); + assertThat(logTablet.lastFetchedEpoch(10)).isEqualTo(-1); + logTablet.assignLeaderEpoch(4); + logTablet.appendAsLeader(genMemoryLogRecordsWithWriterId(DATA1, 1L, 1, 0L)); + assertThat(logTablet.lastFetchedEpoch(20)).isEqualTo(-1); + reopenLogTablet(conf); + assertThat(logTablet.lastFetchedEpoch(20)).isEqualTo(-1); + logTablet.assignLeaderEpoch(5); + logTablet.appendAsLeader(genMemoryLogRecordsWithWriterId(DATA1, 1L, 2, 0L)); + assertThat(logTablet.lastFetchedEpoch(20)).isEqualTo(-1); + assertThat(logTablet.lastFetchedEpoch(30)).isEqualTo(5); + } + + private void reopenLogTablet(Configuration configuration) throws Exception { + logTablet.close(); + logTablet = + LogTablet.create( + tempDir, + PhysicalTablePath.of(DATA1_TABLE_PATH), + logDir, + configuration, + new AtomicBoolean( + configuration.get( + ConfigOptions.LOG_RETENTION_ROLL_ACTIVE_SEGMENT_ENABLED)), + TestingMetricGroups.TABLET_SERVER_METRICS, + 0, + scheduler, + LogFormat.ARROW, + 1, + false, + SystemClock.getInstance(), + true); + } + + @Test + void testUnverifiedSourceDoesNotExtendKnownLocalEpoch() throws Exception { + logTablet.assignLeaderEpoch(3); + logTablet.appendAsLeader(genMemoryLogRecordsWithWriterId(DATA1, 1L, 0, 0L)); + logTablet.appendAsFollower( + genMemoryLogRecordsWithWriterId(DATA1, 2L, 0, 10L), + Collections.singletonList(new LeaderEpochOffset(5, 5))); + assertThat(logTablet.lastFetchedEpoch(20)).isEqualTo(-1); + assertThat(logTablet.localLogEndOffset()).isEqualTo(20); + } + + @Test + void testDisabledRemoteRestoreDoesNotImportEpochHistory() throws Exception { + logTablet.assignLeaderEpoch(3); + logTablet.appendAsLeader(genMemoryLogRecordsWithWriterId(DATA1, 1L, 0, 0L)); + takeWriterSnapshot(logTablet); + File downloaded = new File(tempDir, "remote-writer-snapshot"); + Files.copy(writerSnapshotFile(logDir, 10).toPath(), downloaded.toPath()); + Configuration disabled = new Configuration(conf); + disabled.set(ConfigOptions.LOG_REPLICATION_LEADER_EPOCH_ENABLED, false); + reopenLogTablet(disabled); + logTablet.restoreFromRemote( + 10, downloaded, Collections.singletonList(new LeaderEpochOffset(3, 0))); + assertThat(logTablet.lastFetchedEpoch(10)).isEqualTo(-1); + reopenLogTablet(conf); + assertThat(logTablet.lastFetchedEpoch(10)).isEqualTo(-1); + logTablet.assignLeaderEpoch(4); + logTablet.appendAsLeader(genMemoryLogRecordsWithWriterId(DATA1, 1L, 1, 0L)); + assertThat(logTablet.localLogEndOffset()).isEqualTo(20); + assertThat(logTablet.lastFetchedEpoch(20)).isEqualTo(4); + } + + @Test + void testRemoteRestoreInstallsWriterStateAndEpochHistory() throws Exception { + logTablet.assignLeaderEpoch(3); + logTablet.appendAsLeader(genMemoryLogRecordsWithWriterId(DATA1, 1L, 0, 0L)); + takeWriterSnapshot(logTablet); + File downloaded = new File(tempDir, "downloaded-writer-snapshot"); + Files.copy(writerSnapshotFile(logDir, 10).toPath(), downloaded.toPath()); + logTablet.assignLeaderEpoch(5); + logTablet.appendAsLeader(genMemoryLogRecordsWithWriterId(DATA1, 2L, 0, 0L)); + + logTablet.restoreFromRemote( + 10, downloaded, Collections.singletonList(new LeaderEpochOffset(3, 0))); + assertThat(logTablet.localLogEndOffset()).isEqualTo(10); + assertThat(latestWriterStateEndOffset(logTablet)).isEqualTo(10); + assertThat(logTablet.lastFetchedEpoch(10)).isEqualTo(3); + assertThat(logTablet.endOffsetForEpoch(5)).contains(new LeaderEpochOffset(3, 10)); + logTablet.assignLeaderEpoch(6); + logTablet.appendAsLeader(genMemoryLogRecordsWithWriterId(DATA1, 1L, 1, 0L)); + assertThat(logTablet.localLogEndOffset()).isEqualTo(20); + assertThat(logTablet.lastFetchedEpoch(20)).isEqualTo(6); + logTablet.close(); + logTablet = + LogTablet.create( + tempDir, + PhysicalTablePath.of(DATA1_TABLE_PATH), + logDir, + conf, + new AtomicBoolean( + conf.get(ConfigOptions.LOG_RETENTION_ROLL_ACTIVE_SEGMENT_ENABLED)), + TestingMetricGroups.TABLET_SERVER_METRICS, + 0, + scheduler, + LogFormat.ARROW, + 1, + false, + SystemClock.getInstance(), + true); + assertThat(logTablet.lastFetchedEpoch(20)).isEqualTo(6); + logTablet.appendAsLeader(genMemoryLogRecordsWithWriterId(DATA1, 1L, 2, 0L)); + assertThat(logTablet.localLogEndOffset()).isEqualTo(30); + } + + @Test + void testRemoteCleanupResumesWhenHighWatermarkCatchesUp() throws Exception { + for (int i = 0; i < 4; i++) { + logTablet.appendAsLeader(genMemoryLogRecordsByObject(DATA1)); + if (i < 3) { + logTablet.roll(Optional.empty()); + } + } + logTablet.updateRemoteLogOffsets(0L, 30L, 30L); + assertThat(logTablet.logSegments()).hasSize(4); + logTablet.updateHighWatermark(30L); + assertThat(logTablet.logSegments()).hasSize(1); + assertThat(logTablet.localLogStartOffset()).isEqualTo(30L); + } + @Test void testRemoteLogOffsetsCanResetAfterEmptyManifest() { logTablet.updateRemoteLogOffsets(0L, 10L, 10L); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogITCase.java b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogITCase.java index 5ac61b0bb92..f8535bcdf15 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogITCase.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogITCase.java @@ -222,6 +222,22 @@ void testFollowerFetchAlreadyMoveToRemoteLog(boolean withWriterId) throws Except // restart follower FLUSS_CLUSTER_EXTENSION.startTabletServer(follower); FLUSS_CLUSTER_EXTENSION.waitUntilReplicaExpandToIsr(tb, follower); + assertThat( + FLUSS_CLUSTER_EXTENSION + .waitAndGetFollowerReplica(tb, follower) + .getLogTablet() + .lastFetchedEpoch(100)) + .isEqualTo( + FLUSS_CLUSTER_EXTENSION + .waitAndGetLeaderReplica(tb) + .getLogTablet() + .lastFetchedEpoch(100)); + assertThat( + FLUSS_CLUSTER_EXTENSION + .waitAndGetFollowerReplica(tb, follower) + .getLogTablet() + .lastFetchedEpoch(100)) + .isGreaterThanOrEqualTo(0); } @Test diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrITCase.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrITCase.java index 846a1f5f56c..97f27ccfe49 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrITCase.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrITCase.java @@ -129,18 +129,8 @@ void testIsrShrinkAndExpand() throws Exception { .getHighWatermark()) .isEqualTo(10L)); - currentLeaderAndIsr = zkClient.getLeaderAndIsr(tb).get(); - LeaderAndIsr newLeaderAndIsr = - new LeaderAndIsr( - currentLeaderAndIsr.leader(), - currentLeaderAndIsr.leaderEpoch() + 1, - isr, - currentLeaderAndIsr.standbyReplicas(), - currentLeaderAndIsr.coordinatorEpoch(), - currentLeaderAndIsr.bucketEpoch()); isr.add(stopFollower); - FLUSS_CLUSTER_EXTENSION.notifyLeaderAndIsr( - stopFollower, DATA1_TABLE_PATH, tb, newLeaderAndIsr, isr); + FLUSS_CLUSTER_EXTENSION.restartTabletServer(stopFollower, new Configuration()); // retry until the stop follower add back to ISR. retry( Duration.ofMinutes(1), diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java index 5d95f19c9a5..a551f1a2a08 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java @@ -1266,6 +1266,8 @@ void testConcurrentLookupWithInsertIfNotExistsAutoIncrement() throws Exception { // Values should be 1, 2, 3 (in any order due to concurrency) assertThat(autoIncrementValues).containsExactlyInAnyOrder(1L, 2L, 3L); + // The insert response returns before the async KV flush publishes the high watermark. + waitUntilHighWatermark(tb, 3); // Verify exactly 3 changelog entries were written (one per unique key) FetchLogResultForBucket logResult = fetchLog(tb, 0L); // Only the first upsert for a given primary key generates changelog records. Subsequent diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/LeaderEpochCompatibilityITCase.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/LeaderEpochCompatibilityITCase.java new file mode 100644 index 00000000000..a035b9d3dfb --- /dev/null +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/LeaderEpochCompatibilityITCase.java @@ -0,0 +1,301 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.replica.fetcher; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.cluster.AlterConfigOpType; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.rpc.messages.AlterClusterConfigsRequest; +import org.apache.fluss.rpc.messages.FetchLogRequest; +import org.apache.fluss.rpc.messages.PbFetchLogReqForBucket; +import org.apache.fluss.rpc.messages.PbFetchLogRespForBucket; +import org.apache.fluss.server.log.FetchIsolation; +import org.apache.fluss.server.replica.Replica; +import org.apache.fluss.server.tablet.TabletServer; +import org.apache.fluss.server.testutils.FlussClusterExtension; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.apache.fluss.record.TestData.DATA1; +import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE; +import static org.apache.fluss.record.TestData.DATA1_SCHEMA; +import static org.apache.fluss.record.TestData.DATA1_TABLE_PATH; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.assertProduceLogResponse; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.createTable; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newFetchLogRequest; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newProduceLogRequest; +import static org.apache.fluss.testutils.DataTestUtils.assertLogRecordsEquals; +import static org.apache.fluss.testutils.DataTestUtils.genMemoryLogRecordsByObject; +import static org.apache.fluss.testutils.common.CommonTestUtils.retry; +import static org.assertj.core.api.Assertions.assertThat; + +/** Exercises production replication modes through real RPC and persistent node restarts. */ +class LeaderEpochCompatibilityITCase { + private FlussClusterExtension cluster; + private TableBucket bucket; + private final List expected = new ArrayList<>(); + + @AfterEach + void close() throws Exception { + if (cluster != null) { + cluster.close(); + } + } + + @ParameterizedTest + @CsvSource({"false,false", "false,true", "true,false", "true,true"}) + void testMixedModesReplicateAndMigrate(boolean leaderEnabled, boolean followerEnabled) + throws Exception { + start(leaderEnabled); + int leader = cluster.waitAndGetLeader(bucket); + for (int id = 0; id < 3; id++) { + if (id != leader && followerEnabled != leaderEnabled) { + restart(id, followerEnabled); + } + } + append(); + assertReplicas(); + for (int id = 0; id < 3; id++) { + int epoch = replica(id).getLogTablet().lastFetchedEpoch(expected.size()); + if (leaderEnabled && (id == leader || followerEnabled)) { + assertThat(epoch).isGreaterThanOrEqualTo(0); + } else { + assertThat(epoch).isEqualTo(-1); + } + } + assertEpochResponse(leader, followerEnabled, leaderEnabled && followerEnabled); + migrateAndAppend(); + assertThat(replica(leader).getLogTablet().isLeaderEpochEnabled()).isEqualTo(leaderEnabled); + for (int id = 0; id < 3; id++) { + if (id != leader) { + assertThat(replica(id).getLogTablet().isLeaderEpochEnabled()) + .isEqualTo(followerEnabled); + } + } + } + + @Test + void testRollingEnableDisableAndReenable() throws Exception { + start(false); + append(); + assertReplicas(); + for (int id = 0; id < 3; id++) { + restart(id, true); + append(); + assertReplicas(); + } + for (int id = 0; id < 3; id++) { + assertThat(replica(id).getLogTablet().lastFetchedEpoch(10)).isEqualTo(-1); + } + int follower = (cluster.waitAndGetLeader(bucket) + 1) % 3; + restart(follower, false); + append(); + assertReplicas(); + long untrackedEnd = expected.size(); + restart(follower, true); + assertThat(replica(follower).getLogTablet().lastFetchedEpoch(untrackedEnd)).isEqualTo(-1); + append(); + assertReplicas(); + migrateAndAppend(); + for (int id = 0; id < 3; id++) { + assertThat(replica(id).getLogTablet().lastFetchedEpoch(expected.size())) + .isGreaterThanOrEqualTo(0); + } + } + + @Test + void testAlterModesWithoutRestartingNodes() throws Exception { + start(true); + append(); + assertReplicas(); + List runningNodes = new ArrayList<>(); + for (int id = 0; id < 3; id++) { + runningNodes.add(cluster.getTabletServerById(id)); + } + alterMode(false); + for (int id = 0; id < 3; id++) { + assertThat(replica(id).getLogTablet().lastFetchedEpoch(10)).isEqualTo(-1); + } + append(); + assertReplicas(); + alterMode(true); + append(); + assertReplicas(); + for (int id = 0; id < 3; id++) { + assertThat(cluster.getTabletServerById(id)).isSameAs(runningNodes.get(id)); + // Enabling within the same leader epoch cannot invent a new epoch boundary. + assertThat(replica(id).getLogTablet().lastFetchedEpoch(expected.size())).isEqualTo(-1); + } + migrateAndAppend(); + for (int id = 0; id < 3; id++) { + assertThat(replica(id).getLogTablet().lastFetchedEpoch(expected.size())) + .isGreaterThanOrEqualTo(0); + } + } + + private void alterMode(boolean enabled) throws Exception { + AlterClusterConfigsRequest request = new AlterClusterConfigsRequest(); + request.addAlterConfig() + .setConfigKey(ConfigOptions.LOG_REPLICATION_LEADER_EPOCH_ENABLED.key()) + .setConfigValue(Boolean.toString(enabled)) + .setOpType(AlterConfigOpType.SET.value()); + cluster.newCoordinatorClient().alterClusterConfigs(request).get(30, TimeUnit.SECONDS); + retry( + Duration.ofSeconds(30), + () -> { + for (int id = 0; id < 3; id++) { + assertThat(replica(id).getLogTablet().isLeaderEpochEnabled()) + .isEqualTo(enabled); + } + }); + } + + private void start(boolean enabled) throws Exception { + Configuration conf = new Configuration(); + conf.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 3); + conf.set(ConfigOptions.LOG_REPLICA_MAX_LAG_TIME, Duration.ofSeconds(2)); + conf.set(ConfigOptions.REMOTE_LOG_TASK_INTERVAL_DURATION, Duration.ZERO); + FlussClusterExtension.Builder builder = + FlussClusterExtension.builder().setNumOfTabletServers(3).setClusterConf(conf); + for (int id = 0; id < 3; id++) { + builder.setTabletServerConf(id, mode(enabled)); + } + cluster = builder.build(); + cluster.start(); + long tableId = + createTable( + cluster, + DATA1_TABLE_PATH, + TableDescriptor.builder() + .schema(DATA1_SCHEMA) + .distributedBy(1, "a") + .build()); + bucket = new TableBucket(tableId, 0); + cluster.waitUntilAllReplicaReady(bucket); + } + + private static Configuration mode(boolean enabled) { + Configuration config = new Configuration(); + config.set(ConfigOptions.LOG_REPLICATION_LEADER_EPOCH_ENABLED, enabled); + return config; + } + + private void restart(int id, boolean enabled) throws Exception { + cluster.restartTabletServer(id, mode(enabled)); + cluster.waitUntilReplicaExpandToIsr(bucket, id); + assertReplicas(); + } + + private void append() throws Exception { + cluster.waitAndGetLeaderReplica(bucket); + int leader = cluster.waitAndGetLeader(bucket); + List batch = new ArrayList<>(); + for (int i = 0; i < DATA1.size(); i++) { + batch.add(new Object[] {expected.size() + i, "row-" + (expected.size() + i)}); + } + assertProduceLogResponse( + cluster.newTabletServerClientForNode(leader) + .produceLog( + newProduceLogRequest( + bucket.getTableId(), + 0, + -1, + genMemoryLogRecordsByObject(batch))) + .get(30, TimeUnit.SECONDS), + 0, + (long) expected.size()); + expected.addAll(batch); + } + + private void assertEpochResponse(int leader, boolean requestEpoch, boolean responseEpoch) + throws Exception { + int follower = (leader + 1) % 3; + FetchLogRequest request = + newFetchLogRequest(follower, bucket.getTableId(), 0, expected.size()); + if (requestEpoch) { + PbFetchLogReqForBucket bucketRequest = + request.getTablesReqsList().get(0).getBucketsReqsList().get(0); + bucketRequest + .setCurrentLeaderEpoch(replica(leader).getLeaderEpoch()) + .setLastFetchedEpoch( + replica(follower).getLogTablet().lastFetchedEpoch(expected.size())); + } + PbFetchLogRespForBucket response = + cluster.newTabletServerClientForNode(leader) + .fetchLog(request) + .get(30, TimeUnit.SECONDS) + .getTablesRespsList() + .get(0) + .getBucketsRespsList() + .get(0); + assertThat(response.hasErrorCode()).isFalse(); + assertThat(response.hasCurrentLeaderEpoch()).isEqualTo(responseEpoch); + assertThat(response.hasDivergingEpoch()).isFalse(); + } + + private void migrateAndAppend() throws Exception { + int oldLeader = cluster.waitAndGetLeader(bucket); + cluster.stopTabletServer(oldLeader); + retry( + Duration.ofSeconds(30), + () -> assertThat(cluster.waitAndGetLeader(bucket)).isNotEqualTo(oldLeader)); + append(); + cluster.startTabletServer(oldLeader); + cluster.waitUntilReplicaExpandToIsr(bucket, oldLeader); + assertReplicas(); + } + + private Replica replica(int id) { + return cluster.getTabletServerById(id).getReplicaManager().getReplicaOrException(bucket); + } + + private void assertReplicas() throws Exception { + retry( + Duration.ofSeconds(30), + () -> { + assertThat(cluster.waitAndGetLeaderReplica(bucket).getIsr()).hasSize(3); + for (int id = 0; id < 3; id++) { + Replica replica = replica(id); + assertThat(replica.getLocalLogEndOffset()).isEqualTo(expected.size()); + assertThat(replica.getLogHighWatermark()).isEqualTo(expected.size()); + if (!expected.isEmpty()) { + assertLogRecordsEquals( + DATA1_ROW_TYPE, + replica.getLogTablet() + .read( + 0, + Integer.MAX_VALUE, + FetchIsolation.LOG_END, + true) + .getRecords(), + expected); + } + } + }); + } +} diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/LeaderMigrationWalTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/LeaderMigrationWalTest.java new file mode 100644 index 00000000000..4190aaaf68e --- /dev/null +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/LeaderMigrationWalTest.java @@ -0,0 +1,662 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.replica.fetcher; + +import org.apache.fluss.cluster.Endpoint; +import org.apache.fluss.cluster.ServerNode; +import org.apache.fluss.cluster.ServerType; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.exception.FencedLeaderEpochException; +import org.apache.fluss.metadata.LeaderEpochOffset; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.record.LogRecord; +import org.apache.fluss.record.LogRecordBatch; +import org.apache.fluss.record.LogRecordReadContext; +import org.apache.fluss.record.TestingSchemaGetter; +import org.apache.fluss.rpc.RpcClient; +import org.apache.fluss.rpc.entity.FetchLogEpochInfo; +import org.apache.fluss.rpc.entity.FetchLogResultForBucket; +import org.apache.fluss.rpc.entity.ProduceLogResultForBucket; +import org.apache.fluss.rpc.messages.ApiMessage; +import org.apache.fluss.rpc.messages.FetchLogRequest; +import org.apache.fluss.rpc.protocol.ApiKeys; +import org.apache.fluss.server.coordinator.LakeCatalogDynamicLoader; +import org.apache.fluss.server.coordinator.MetadataManager; +import org.apache.fluss.server.coordinator.TestCoordinatorGateway; +import org.apache.fluss.server.coordinator.statemachine.ReplicaLeaderElection.ReassignmentLeaderElection; +import org.apache.fluss.server.entity.NotifyLeaderAndIsrData; +import org.apache.fluss.server.entity.NotifyLeaderAndIsrResultForBucket; +import org.apache.fluss.server.kv.KvManager; +import org.apache.fluss.server.kv.scan.ScannerManager; +import org.apache.fluss.server.kv.snapshot.TestingCompletedKvSnapshotCommitter; +import org.apache.fluss.server.log.FetchIsolation; +import org.apache.fluss.server.log.LogManager; +import org.apache.fluss.server.metadata.ClusterMetadata; +import org.apache.fluss.server.metadata.ServerInfo; +import org.apache.fluss.server.metadata.TabletServerMetadataCache; +import org.apache.fluss.server.metrics.group.TestingMetricGroups; +import org.apache.fluss.server.replica.Replica; +import org.apache.fluss.server.replica.ReplicaManager; +import org.apache.fluss.server.storage.LocalDiskManager; +import org.apache.fluss.server.zk.NOPErrorHandler; +import org.apache.fluss.server.zk.ZooKeeperClient; +import org.apache.fluss.server.zk.ZooKeeperExtension; +import org.apache.fluss.server.zk.data.LeaderAndIsr; +import org.apache.fluss.server.zk.data.TableRegistration; +import org.apache.fluss.testutils.common.AllCallbackWrapper; +import org.apache.fluss.utils.CloseableIterator; +import org.apache.fluss.utils.clock.ManualClock; +import org.apache.fluss.utils.concurrent.FlussScheduler; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.File; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +import static org.apache.fluss.record.TestData.ANOTHER_DATA1; +import static org.apache.fluss.record.TestData.DATA1; +import static org.apache.fluss.record.TestData.DATA1_PHYSICAL_TABLE_PATH; +import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE; +import static org.apache.fluss.record.TestData.DATA1_SCHEMA; +import static org.apache.fluss.record.TestData.DATA1_TABLE_DESCRIPTOR; +import static org.apache.fluss.record.TestData.DATA1_TABLE_ID; +import static org.apache.fluss.record.TestData.DATA1_TABLE_PATH; +import static org.apache.fluss.record.TestData.DEFAULT_SCHEMA_ID; +import static org.apache.fluss.server.utils.ServerRpcMessageUtils.makeFetchLogResponse; +import static org.apache.fluss.testutils.DataTestUtils.genMemoryLogRecordsWithWriterId; +import static org.apache.fluss.testutils.common.CommonTestUtils.retry; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests WAL convergence through clean leader migration with controlled RPC delivery. */ +class LeaderMigrationWalTest { + @RegisterExtension + static final AllCallbackWrapper ZK = + new AllCallbackWrapper<>(new ZooKeeperExtension()); + + private static final TableBucket BUCKET = new TableBucket(DATA1_TABLE_ID, 0); + private static final List REPLICAS = Arrays.asList(1, 2, 3); + private final Map servers = new HashMap<>(); + private final ManualClock clock = new ManualClock(System.currentTimeMillis()); + private @TempDir File tempDir; + private ZooKeeperClient zkClient; + private LeaderAndIsr leaderAndIsr; + + @BeforeEach + void setUp() throws Exception { + zkClient = ZK.getCustomExtension().getZooKeeperClient(NOPErrorHandler.INSTANCE); + ZK.getCustomExtension().cleanupRoot(); + zkClient.registerTable( + DATA1_TABLE_PATH, + TableRegistration.newTable( + DATA1_TABLE_ID, + new File(tempDir, "remote").getAbsolutePath(), + DATA1_TABLE_DESCRIPTOR)); + zkClient.registerFirstSchema(DATA1_TABLE_PATH, DATA1_SCHEMA); + for (int id : REPLICAS) { + servers.put(id, new Server(id)); + } + leaderAndIsr = new LeaderAndIsr(1, 0, REPLICAS, Collections.emptyList(), 0, 0); + for (int id : REPLICAS) { + notifyRole(id); + } + } + + @AfterEach + void tearDown() throws Exception { + for (Server server : servers.values()) { + server.rpc.pause(); + } + for (Server server : servers.values()) { + server.close(); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testCleanMigrationReplacesUncommittedFollowerTail(boolean enabled) throws Exception { + Configuration mode = new Configuration(); + mode.set(ConfigOptions.LOG_REPLICATION_LEADER_EPOCH_ENABLED, enabled); + for (Server server : servers.values()) { + server.log.reconfigure(mode); + } + assertMigrationConverges(false, enabled); + } + + @Test + void testCleanMigrationWithCaughtUpTarget() throws Exception { + assertMigrationConverges(true); + } + + @Test + void testDisablingPreventsEpochTruncationOfExistingTail() throws Exception { + append(1, DATA1, 100, 1).get(5, TimeUnit.SECONDS); + deliver(2, 1, 0); + List copied = readWal(2); + Configuration mode = new Configuration(); + mode.set(ConfigOptions.LOG_REPLICATION_LEADER_EPOCH_ENABLED, false); + servers.get(2).log.reconfigure(mode); + assertThat(replica(2).truncateFollowerToEpochOffset(0, 1, 0)).isFalse(); + assertThat(replica(2).getLocalLogEndOffset()).isEqualTo(10); + assertThat(readWal(2)).containsExactlyElementsOf(copied); + assertThat(replica(2).getLogTablet().lastFetchedEpoch(10)).isEqualTo(-1); + } + + @Test + void testLateFetchCannotModifyReplicaAfterLeaderChange() throws Exception { + FetchLogResultForBucket late = + FetchLogResultForBucket.records( + BUCKET, + genMemoryLogRecordsWithWriterId(DATA1, 100, 0, 0), + 10, + -1, + -1) + .withEpochInfo( + new FetchLogEpochInfo( + 0, + null, + Collections.singletonList(new LeaderEpochOffset(0, 0)))); + moveLeader(3); + assertThatThrownBy(() -> replica(2).appendRecordsToFollower(late, 1, 0)) + .isInstanceOf(FencedLeaderEpochException.class); + assertThat(replica(2).getLocalLogEndOffset()).isZero(); + assertThat(replica(2).getLogHighWatermark()).isZero(); + assertThat(replica(2).getLogTablet().lastFetchedEpoch(10)).isEqualTo(-1); + } + + @Test + void testConvergenceAcrossConsecutiveMigrationsAndEmptyEpoch() throws Exception { + assertMigrationConverges(false); + List prefix = readWal(2); + moveLeader(1); + // No records are written in this epoch before the next clean migration. + moveLeader(3); + CompletableFuture> appended = + append(3, DATA1.subList(0, 5), 103, -1); + for (int round = 0; round < 6; round++) { + deliver(1, 3, -1); + deliver(2, 3, -1); + if (appended.isDone() + && replica(1).getLogHighWatermark() == 20 + && replica(2).getLogHighWatermark() == 20) { + break; + } + } + assertThat(appended.get(5, TimeUnit.SECONDS)) + .containsExactly(new ProduceLogResultForBucket(BUCKET, 15, 20)); + for (int i = 0; i < 5; i++) { + prefix.add((15 + i) + ":" + DATA1.get(i)[0] + ":" + DATA1.get(i)[1]); + } + for (int id : REPLICAS) { + assertThat(readWal(id)).containsExactlyElementsOf(prefix); + assertThat(replica(id).getLogTablet().lastFetchedEpoch(20)) + .isEqualTo(leaderAndIsr.leaderEpoch()); + } + } + + private void moveLeader(int target) throws Exception { + for (Server server : servers.values()) { + server.rpc.pause(); + } + List preference = new ArrayList<>(REPLICAS); + preference.remove(Integer.valueOf(target)); + preference.add(0, target); + leaderAndIsr = + new ReassignmentLeaderElection(preference) + .leaderElection(REPLICAS, leaderAndIsr, false) + .get() + .getLeaderAndIsr(); + notifyRole(target); + for (int id : REPLICAS) { + if (id != target) { + notifyRole(id); + } + } + for (Server server : servers.values()) { + server.rpc.resume(); + } + } + + @Test + void testLegacyReplicationRemainsAvailableWithoutInventingHistory() throws Exception { + CompletableFuture> first = append(1, DATA1, 100, -1); + deliver(2, 1, 0, true); + deliver(3, 1, 0); + deliver(2, 1, 10, true); + deliver(3, 1, 10); + first.get(5, TimeUnit.SECONDS); + assertThat(replica(2).getLogTablet().lastFetchedEpoch(10)).isEqualTo(-1); + + CompletableFuture> second = + append(1, ANOTHER_DATA1.subList(0, 5), 101, -1); + // The upgraded peer must not infer the identity of its old prefix from the source's + // epoch boundary, which precedes this fetch. + deliver(2, 1, 10); + deliver(3, 1, 10); + deliver(2, 1, 15); + deliver(3, 1, 15); + second.get(5, TimeUnit.SECONDS); + deliver(2, 1, 15); + assertThat(replica(2).getLogTablet().lastFetchedEpoch(10)).isEqualTo(-1); + assertThat(replica(2).getLogTablet().lastFetchedEpoch(15)).isEqualTo(-1); + assertThat(readWal(2)).containsExactlyElementsOf(readWal(1)); + } + + private void assertMigrationConverges(boolean catchUpTarget) throws Exception { + assertMigrationConverges(catchUpTarget, true); + } + + private void assertMigrationConverges(boolean catchUpTarget, boolean enabled) throws Exception { + CompletableFuture> committed = append(1, DATA1, 100, -1); + deliver(2, 1, 0); + deliver(3, 1, 0); + deliver(2, 1, 10); + deliver(3, 1, 10); + assertThat(committed.get(5, TimeUnit.SECONDS)) + .containsExactly(new ProduceLogResultForBucket(BUCKET, 0, 10)); + assertThat(replica(1).getLogHighWatermark()).isEqualTo(10); + + // Only C receives this uncommitted tail; B remains in the ISR at the committed prefix. + append(1, DATA1.subList(0, 5), 101, 1).get(5, TimeUnit.SECONDS); + deliver(3, 1, 10); + assertThat(replica(2).getLocalLogEndOffset()).isEqualTo(10); + assertThat(replica(3).getLocalLogEndOffset()).isEqualTo(15); + assertThat(replica(1).getLogHighWatermark()).isEqualTo(10); + + if (catchUpTarget) { + deliver(2, 1, 10); + } + long newLeaderStart = catchUpTarget ? 15 : 10; + long newLeaderEnd = newLeaderStart + 5; + assertThat(replica(1).getIsr()).containsExactlyInAnyOrderElementsOf(REPLICAS); + + // Reassignment elects B from the live ISR. No unclean election or file mutation is used. + for (Server server : servers.values()) { + server.rpc.pause(); + } + leaderAndIsr = + new ReassignmentLeaderElection(Arrays.asList(2, 1, 3)) + .leaderElection(REPLICAS, leaderAndIsr, false) + .get() + .getLeaderAndIsr(); + assertThat(leaderAndIsr.leader()).isEqualTo(2); + assertThat(leaderAndIsr.isr()).containsExactlyElementsOf(REPLICAS); + notifyRole(2); + notifyRole(1); + notifyRole(3); + for (Server server : servers.values()) { + server.rpc.resume(); + } + + CompletableFuture> replacement = + append(2, ANOTHER_DATA1.subList(0, 5), 102, -1); + // Allow history reconciliation, data replication, and committed watermark propagation. + for (int round = 0; round < 6; round++) { + deliver(1, 2, -1); + deliver(3, 2, -1); + if (enabled && round == 0) { + assertThat(replica(2).getLogHighWatermark()).isLessThanOrEqualTo(newLeaderStart); + assertThat(replacement).isNotDone(); + } + if (replacement.isDone() + && replica(1).getLogHighWatermark() == newLeaderEnd + && replica(3).getLogHighWatermark() == newLeaderEnd) { + break; + } + } + assertThat(replacement.get(5, TimeUnit.SECONDS)) + .containsExactly( + new ProduceLogResultForBucket(BUCKET, newLeaderStart, newLeaderEnd)); + assertThat(replica(2).getLogHighWatermark()).isEqualTo(newLeaderEnd); + assertThat(replica(3).getLogHighWatermark()).isEqualTo(newLeaderEnd); + assertThat(replica(2).getIsr()).containsExactlyInAnyOrderElementsOf(REPLICAS); + + List expected = new ArrayList<>(); + for (int i = 0; i < DATA1.size(); i++) { + expected.add(i + ":" + DATA1.get(i)[0] + ":" + DATA1.get(i)[1]); + } + if (catchUpTarget) { + for (int i = 0; i < 5; i++) { + expected.add((10 + i) + ":" + DATA1.get(i)[0] + ":" + DATA1.get(i)[1]); + } + } + for (int i = 0; i < 5; i++) { + expected.add( + (newLeaderStart + i) + + ":" + + ANOTHER_DATA1.get(i)[0] + + ":" + + ANOTHER_DATA1.get(i)[1]); + } + assertThat(readWal(2)).containsExactlyElementsOf(expected); + assertThat(readWal(1)).containsExactlyElementsOf(expected); + if (enabled) { + assertThat(readWal(3)) + .as("ISR replicas must contain the acknowledged records at the same offsets") + .containsExactlyElementsOf(expected); + } else { + List divergent = new ArrayList<>(expected.subList(0, 10)); + for (int i = 0; i < 5; i++) { + divergent.add((10 + i) + ":" + DATA1.get(i)[0] + ":" + DATA1.get(i)[1]); + } + assertThat(readWal(3)) + .as("Legacy replication retains the old uncommitted tail") + .containsExactlyElementsOf(divergent) + .isNotEqualTo(readWal(2)); + } + } + + private Replica replica(int id) { + return servers.get(id).manager.getReplicaOrException(BUCKET); + } + + private void notifyRole(int id) throws Exception { + CompletableFuture> result = + new CompletableFuture<>(); + servers.get(id) + .manager + .becomeLeaderOrFollower( + 0, + Collections.singletonList( + new NotifyLeaderAndIsrData( + DATA1_PHYSICAL_TABLE_PATH, BUCKET, REPLICAS, leaderAndIsr)), + result::complete); + assertThat(result.get(5, TimeUnit.SECONDS)) + .containsExactly(new NotifyLeaderAndIsrResultForBucket(BUCKET)); + } + + private CompletableFuture> append( + int id, List rows, long writerId, int acks) throws Exception { + CompletableFuture> result = new CompletableFuture<>(); + servers.get(id) + .manager + .appendRecordsToLog( + 10000, + acks, + Collections.singletonMap( + BUCKET, genMemoryLogRecordsWithWriterId(rows, writerId, 0, 0)), + null, + result::complete); + return result; + } + + private void deliver(int follower, int leader, long offset) throws Exception { + deliver(follower, leader, offset, false); + } + + private void deliver(int follower, int leader, long offset, boolean legacy) throws Exception { + ControlledRpcClient rpc = servers.get(follower).rpc; + PendingRequest pending = rpc.requests.poll(5, TimeUnit.SECONDS); + assertThat(pending).as("fetch from server %s", follower).isNotNull(); + assertThat(pending.destination).isEqualTo(leader); + assertThat(pending.request).isInstanceOf(FetchLogRequest.class); + FetchLogRequest request = (FetchLogRequest) pending.request; + if (offset >= 0) { + assertThat( + request.getTablesReqsList() + .get(0) + .getBucketsReqsList() + .get(0) + .getFetchOffset()) + .isEqualTo(offset); + } + FetchLogRequest serverRequest = new FetchLogRequest().copyFrom(request); + if (legacy) { + serverRequest + .getTablesReqsList() + .forEach( + table -> + table.getBucketsReqsList() + .forEach( + bucket -> { + bucket.clearCurrentLeaderEpoch(); + bucket.clearLastFetchedEpoch(); + })); + } + TestingLeaderEndpoint endpoint = + new TestingLeaderEndpoint( + servers.get(leader).conf, + servers.get(leader).manager, + new ServerNode( + follower, "localhost", 10000 + follower, ServerType.TABLET_SERVER), + leader); + LeaderEndpoint.FetchData data = + endpoint.fetchLog( + new FetchLogContext( + Collections.singletonMap(DATA1_TABLE_ID, DATA1_TABLE_PATH), + serverRequest)) + .get(5, TimeUnit.SECONDS); + pending.result.complete(makeFetchLogResponse(data.getFetchLogResultMap())); + // The next request proves the actual background fetcher has consumed this response. + retry(Duration.ofSeconds(5), () -> assertThat(rpc.requests).isNotEmpty()); + } + + private List readWal(int id) throws Exception { + List records = new ArrayList<>(); + try (LogRecordReadContext context = + LogRecordReadContext.createArrowReadContext( + DATA1_ROW_TYPE, + DEFAULT_SCHEMA_ID, + new TestingSchemaGetter(DEFAULT_SCHEMA_ID, DATA1_SCHEMA))) { + for (LogRecordBatch batch : + replica(id) + .getLogTablet() + .read(0, Integer.MAX_VALUE, FetchIsolation.HIGH_WATERMARK, true) + .getRecords() + .batches()) { + batch.ensureValid(); + try (CloseableIterator iterator = batch.records(context)) { + while (iterator.hasNext()) { + LogRecord record = iterator.next(); + records.add( + record.logOffset() + + ":" + + record.getRow().getInt(0) + + ":" + + record.getRow().getString(1)); + } + } + } + } + return records; + } + + private static final class PendingRequest { + private final int destination; + private final ApiMessage request; + private final CompletableFuture result = new CompletableFuture<>(); + + private PendingRequest(int destination, ApiMessage request) { + this.destination = destination; + this.request = request; + } + } + + /** Holds RPC requests until the test delivers them to the real destination ReplicaManager. */ + private static final class ControlledRpcClient implements RpcClient { + private final BlockingQueue requests = new LinkedBlockingQueue<>(); + private boolean paused; + + @Override + public synchronized CompletableFuture sendRequest( + ServerNode node, ApiKeys apiKey, ApiMessage request) { + PendingRequest pending = new PendingRequest(node.id(), request); + if (paused) { + pending.result.completeExceptionally(new IOException("Connection paused")); + } else { + requests.add(pending); + } + return pending.result; + } + + synchronized void pause() { + paused = true; + PendingRequest request; + while ((request = requests.poll()) != null) { + request.result.completeExceptionally(new IOException("Connection paused")); + } + } + + synchronized void resume() { + paused = false; + } + + @Override + public boolean connect(ServerNode node) { + return true; + } + + @Override + public boolean isReady(String serverUid) { + return true; + } + + @Override + public CompletableFuture disconnect(String serverUid) { + pause(); + return CompletableFuture.completedFuture(null); + } + + @Override + public void close() { + pause(); + } + } + + private final class Server implements AutoCloseable { + private final Configuration conf = new Configuration(); + private final ControlledRpcClient rpc = new ControlledRpcClient(); + private final FlussScheduler scheduler = new FlussScheduler(2); + private final ExecutorService io = Executors.newSingleThreadExecutor(); + private final LocalDiskManager disk; + private final LogManager log; + private final KvManager kv; + private final ScannerManager scanner; + private final ReplicaManager manager; + + private Server(int id) throws Exception { + conf.set(ConfigOptions.TABLET_SERVER_ID, id); + conf.setString( + ConfigOptions.DATA_DIR, new File(tempDir, "server-" + id).getAbsolutePath()); + conf.set(ConfigOptions.REMOTE_DATA_DIR, new File(tempDir, "remote").getAbsolutePath()); + conf.set(ConfigOptions.SERVER_DATA_DISK_WRITE_LIMIT_RATIO, 1.0); + conf.set(ConfigOptions.LOG_REPLICA_FETCH_BACKOFF_INTERVAL, Duration.ofMillis(10)); + conf.set(ConfigOptions.LOG_REPLICA_FETCH_WAIT_MAX_TIME, Duration.ZERO); + scheduler.startup(); + disk = LocalDiskManager.create(conf); + log = + LogManager.create( + conf, + zkClient, + scheduler, + clock, + TestingMetricGroups.TABLET_SERVER_METRICS, + disk); + log.startup(); + kv = + KvManager.create( + conf, + zkClient, + log, + TestingMetricGroups.TABLET_SERVER_METRICS, + disk, + null, + clock); + kv.startup(); + scanner = new ScannerManager(conf, scheduler); + TabletServerMetadataCache metadata = + new TabletServerMetadataCache( + new MetadataManager( + zkClient, + conf, + new LakeCatalogDynamicLoader(conf, null, true))); + List members = new ArrayList<>(); + for (int member : REPLICAS) { + members.add( + new ServerInfo( + member, + null, + Endpoint.fromListenersString( + "FLUSS://localhost:" + (10000 + member)), + ServerType.TABLET_SERVER)); + } + metadata.updateClusterMetadata( + new ClusterMetadata( + new ServerInfo( + 0, + null, + Endpoint.fromListenersString("FLUSS://localhost:9999"), + ServerType.COORDINATOR), + new HashSet<>(members))); + manager = + new ReplicaManager( + conf, + scheduler, + log, + kv, + zkClient, + id, + metadata, + rpc, + new TestCoordinatorGateway(), + new TestingCompletedKvSnapshotCommitter(), + NOPErrorHandler.INSTANCE, + TestingMetricGroups.TABLET_SERVER_METRICS, + TestingMetricGroups.USER_METRICS, + scanner, + clock, + io, + disk, + null); + manager.startup(); + } + + @Override + public void close() throws Exception { + manager.shutdown(); + manager.getRemoteLogManager().close(); + scanner.close(); + kv.shutdown(); + log.shutdown(); + scheduler.shutdown(); + disk.close(); + io.shutdownNow(); + } + } +} diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherITCase.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherITCase.java index 36d3ef567ed..4d72281ec0d 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherITCase.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherITCase.java @@ -27,6 +27,7 @@ import org.apache.fluss.rpc.gateway.TabletServerGateway; import org.apache.fluss.rpc.messages.PbPutKvRespForBucket; import org.apache.fluss.rpc.messages.PutKvResponse; +import org.apache.fluss.rpc.messages.StopReplicaRequest; import org.apache.fluss.server.entity.FetchReqInfo; import org.apache.fluss.server.log.FetchParams; import org.apache.fluss.server.replica.Replica; @@ -66,6 +67,7 @@ import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newFetchLogRequest; import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newLookupRequest; import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newPutKvRequest; +import static org.apache.fluss.server.utils.ServerRpcMessageUtils.makeStopBucketReplica; import static org.apache.fluss.testutils.DataTestUtils.assertLogRecordsEquals; import static org.apache.fluss.testutils.DataTestUtils.assertLogRecordsEqualsWithRowKind; import static org.apache.fluss.testutils.DataTestUtils.genKvRecordBatch; @@ -269,9 +271,21 @@ void testFlushForPutKvNeedAck() throws Exception { .get() .id(); - int leaderEpoch = 0; - // stop the follower replica for the bucket - FLUSS_CLUSTER_EXTENSION.stopReplica(followerToStop, tb, leaderEpoch); + LeaderAndIsr currentLeaderAndIsr = zkClient.getLeaderAndIsr(tb).get(); + // Remove the local follower so it can be recreated in the same leader epoch. + FLUSS_CLUSTER_EXTENSION + .newTabletServerClientForNode(followerToStop) + .stopReplica( + new StopReplicaRequest() + .setCoordinatorEpoch(currentLeaderAndIsr.coordinatorEpoch()) + .addAllStopReplicasReqs( + Collections.singleton( + makeStopBucketReplica( + tb, + true, + false, + currentLeaderAndIsr.leaderEpoch())))) + .get(); // put kv record batch to the leader, // but as one server is killed, the put won't be ack @@ -310,20 +324,17 @@ void testFlushForPutKvNeedAck() throws Exception { // start the follower replica by notify leaderAndIsr, // then the kv should be flushed finally - LeaderAndIsr currentLeaderAndIsr = zkClient.getLeaderAndIsr(tb).get(); - LeaderAndIsr newLeaderAndIsr = - new LeaderAndIsr( - currentLeaderAndIsr.leader(), - currentLeaderAndIsr.leaderEpoch() + 1, - currentLeaderAndIsr.isr(), - Collections.emptyList(), - currentLeaderAndIsr.coordinatorEpoch(), - currentLeaderAndIsr.bucketEpoch()); FLUSS_CLUSTER_EXTENSION.notifyLeaderAndIsr( - followerToStop, DATA1_TABLE_PATH, tb, newLeaderAndIsr, Arrays.asList(0, 1, 2)); + followerToStop, + DATA1_TABLE_PATH_PK, + tb, + currentLeaderAndIsr, + Arrays.asList(0, 1, 2)); // wait until the put future is done - putResponse.get(); + for (PbPutKvRespForBucket result : putResponse.get().getBucketsRespsList()) { + assertThat(result.hasErrorCode()).isFalse(); + } // then we can check all the value for (Tuple2 keyValue : expectedKeyValues) { diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThreadTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThreadTest.java index 8262d080f1f..9dfb856afa4 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThreadTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThreadTest.java @@ -144,7 +144,7 @@ public void setup() throws Exception { ServerNode follower = new ServerNode( followerServerId, "localhost", 10001, ServerType.TABLET_SERVER, "rack2"); - leaderEndpoint = new TestingLeaderEndpoint(conf, leaderRM, follower); + leaderEndpoint = new TestingLeaderEndpoint(conf, leaderRM, follower, leaderServerId); followerFetcher = new ReplicaFetcherThread("test-fetcher-thread", followerRM, leaderEndpoint, 1000); @@ -277,7 +277,7 @@ void testRestoreKvMinRetainOffsetFromDelayedEmptyFetchResponse() throws Exceptio } @Test - void testFollowerHighWatermarkHigherThanOrEqualToLeader() throws Exception { + void testFollowerHighWatermarkConvergesToLeader() throws Exception { Replica leaderReplica = leaderRM.getReplicaOrException(tb); Replica followerReplica = followerRM.getReplicaOrException(tb); @@ -310,7 +310,15 @@ void testFollowerHighWatermarkHigherThanOrEqualToLeader() throws Exception { assertThat(followerReplica.getLocalLogEndOffset()) .isEqualTo(baseOffset + 10L)); assertThat(followerReplica.getLogHighWatermark()) - .isGreaterThanOrEqualTo(leaderReplica.getLogHighWatermark()); + .isLessThanOrEqualTo(leaderReplica.getLogHighWatermark()) + .isLessThanOrEqualTo(followerReplica.getLocalLogEndOffset()); + retry( + Duration.ofSeconds(20), + () -> { + assertThat(leaderReplica.getLogHighWatermark()).isEqualTo(baseOffset + 10L); + assertThat(followerReplica.getLogHighWatermark()) + .isEqualTo(baseOffset + 10L); + }); } } @@ -479,7 +487,7 @@ void testFetchTimeoutReleasesPooledByteBuf() throws Exception { ServerType.TABLET_SERVER, "rack2"); TestingLeaderEndpoint testingEndpoint = - new TestingLeaderEndpoint(conf, leaderRM, followerNode); + new TestingLeaderEndpoint(conf, leaderRM, followerNode, leaderServerId); // Append records to leader so fetch responses carry actual data CompletableFuture> future = new CompletableFuture<>(); @@ -595,6 +603,7 @@ private void makeLeaderAndFollower( private LocalDiskManager createLocalDiskManager(int serverId) throws Exception { Configuration conf = new Configuration(); + conf.set(ConfigOptions.SERVER_DATA_DISK_WRITE_LIMIT_RATIO, 1.0); conf.set(ConfigOptions.TABLET_SERVER_ID, serverId); conf.setString(ConfigOptions.DATA_DIR, tempDir.getAbsolutePath() + "/server-" + serverId); return LocalDiskManager.create(conf); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/TestingLeaderEndpoint.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/TestingLeaderEndpoint.java index ab6d8e5c43f..5c16e2b8d17 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/TestingLeaderEndpoint.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/TestingLeaderEndpoint.java @@ -49,6 +49,7 @@ public class TestingLeaderEndpoint implements LeaderEndpoint { private final ReplicaManager replicaManager; + private final int leaderServerId; private final ServerNode localNode; /** The max size for the fetch response. */ private final int maxFetchSize; @@ -72,8 +73,12 @@ public class TestingLeaderEndpoint implements LeaderEndpoint { new java.util.concurrent.CopyOnWriteArrayList<>(); public TestingLeaderEndpoint( - Configuration conf, ReplicaManager replicaManager, ServerNode localNode) { + Configuration conf, + ReplicaManager replicaManager, + ServerNode localNode, + int leaderServerId) { this.replicaManager = replicaManager; + this.leaderServerId = leaderServerId; this.localNode = localNode; this.maxFetchSize = (int) conf.get(ConfigOptions.LOG_REPLICA_FETCH_MAX_BYTES).getBytes(); this.maxFetchSizeForBucket = @@ -85,7 +90,7 @@ public TestingLeaderEndpoint( @Override public int leaderServerId() { - return localNode.id(); + return leaderServerId; } @Override @@ -213,11 +218,12 @@ private Map processResult( value.hasMinRetainOffset() ? value.getMinRetainOffset() : -1L; FetchLogResultForBucket memoryResult = FetchLogResultForBucket.records( - tb, - memRecords, - value.getHighWatermark(), - filteredEndOffset, - minRetainOffset); + tb, + memRecords, + value.getHighWatermark(), + filteredEndOffset, + minRetainOffset) + .withEpochInfo(value.epochInfo()); result.put(tb, memoryResult); } else { result.put(tb, value); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java b/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java index cff3f788c99..bccc0a3e8d8 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java @@ -137,6 +137,7 @@ public final class FlussClusterExtension private final Map tabletServers; private final Map tabletServerInfos; private final Configuration clusterConf; + private final Map tabletServerConfigs; private final Clock clock; private final String[] racks; private final List remoteDirNames; @@ -153,13 +154,17 @@ private FlussClusterExtension( Configuration clusterConf, Clock clock, String[] racks, - List remoteDirNames) { + List remoteDirNames, + Map tabletServerConfigs) { this.initialNumOfTabletServers = numOfTabletServers; this.tabletServers = new HashMap<>(numOfTabletServers); this.coordinatorServerListeners = coordinatorServerListeners; this.tabletServerListeners = tabletServerListeners; this.tabletServerInfos = new HashMap<>(); this.clusterConf = clusterConf; + this.tabletServerConfigs = new HashMap<>(); + tabletServerConfigs.forEach( + (id, config) -> this.tabletServerConfigs.put(id, new Configuration(config))); this.clock = clock; checkArgument( racks != null && racks.length == numOfTabletServers, @@ -339,7 +344,13 @@ private void startTabletServer(int serverId, @Nullable Configuration overwriteCo tabletServerConf.setString(ConfigOptions.BIND_LISTENERS, tabletServerListeners); tabletServerConf.setDouble(ConfigOptions.SERVER_DATA_DISK_WRITE_LIMIT_RATIO, 1.0); if (overwriteConfig != null) { - tabletServerConf.addAll(overwriteConfig); + tabletServerConfigs + .computeIfAbsent(serverId, id -> new Configuration()) + .addAll(overwriteConfig); + } + Configuration nodeConfig = tabletServerConfigs.get(serverId); + if (nodeConfig != null) { + tabletServerConf.addAll(nodeConfig); } setRemoteDataDir(tabletServerConf); @@ -1017,6 +1028,7 @@ public static class Builder { private List remoteDirNames = Collections.emptyList(); private final Configuration clusterConf = new Configuration(); + private final Map tabletServerConfigs = new HashMap<>(); public Builder() { // reduce testing resources @@ -1046,6 +1058,12 @@ public Builder setClusterConf(Configuration clusterConf) { return this; } + /** Sets persistent configuration overrides for one tablet server, including restarts. */ + public Builder setTabletServerConf(int serverId, Configuration config) { + tabletServerConfigs.put(serverId, new Configuration(config)); + return this; + } + /** Sets the listeners of tablet servers. */ public Builder setTabletServerListeners(String tabletServerListeners) { this.tabletServerListeners = tabletServerListeners; @@ -1091,7 +1109,8 @@ public FlussClusterExtension build() { clusterConf, clock, racks, - remoteDirNames); + remoteDirNames, + tabletServerConfigs); } } }