From 921b5d366ef8aaf47cafc22cdd608d9209f459fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E9=B5=BA?= Date: Thu, 10 Sep 2026 12:08:39 +0800 Subject: [PATCH 1/4] [server] Reconcile divergent replica logs using leader epochs Track durable leader epoch boundaries and exchange optional epoch metadata during replication to reconcile divergent tails after leader migration. Preserve legacy replication compatibility and support a dynamically configurable, default-enabled epoch mode. Cover divergence, mixed-mode replication, dynamic configuration changes, and remote recovery. --- .../apache/fluss/config/ConfigOptions.java | 12 + .../fluss/metadata/LeaderEpochOffset.java | 61 ++ .../remote/RemoteLogManifestJsonSerde.java | 23 + .../apache/fluss/remote/RemoteLogSegment.java | 32 +- .../RemoteLogManifestJsonSerdeTest.java | 33 + .../fluss/rpc/entity/FetchLogEpochInfo.java | 57 ++ .../rpc/entity/FetchLogResultForBucket.java | 40 ++ .../apache/fluss/rpc/protocol/ApiKeys.java | 2 +- .../fluss/rpc/util/CommonRpcMessageUtils.java | 25 + fluss-rpc/src/main/proto/FlussApi.proto | 13 +- .../fluss/server/DynamicServerConfig.java | 2 + .../fluss/server/entity/FetchReqInfo.java | 35 +- .../apache/fluss/server/log/FetchParams.java | 17 +- .../fluss/server/log/LeaderEpochHistory.java | 251 +++++++ .../apache/fluss/server/log/LogManager.java | 24 + .../apache/fluss/server/log/LogReadInfo.java | 23 + .../apache/fluss/server/log/LogTablet.java | 182 ++++- .../server/log/remote/LogTieringTask.java | 2 + .../apache/fluss/server/replica/Replica.java | 267 +++++-- .../fluss/server/replica/ReplicaManager.java | 61 +- .../replica/fetcher/FetchLogContext.java | 46 ++ .../replica/fetcher/ReplicaFetcherThread.java | 293 ++++---- .../server/utils/ServerRpcMessageUtils.java | 33 +- .../server/log/LeaderEpochHistoryTest.java | 139 ++++ .../fluss/server/log/LogTabletTest.java | 116 +++ .../server/log/remote/RemoteLogITCase.java | 16 + .../LeaderEpochCompatibilityITCase.java | 301 ++++++++ .../fetcher/LeaderMigrationWalTest.java | 661 ++++++++++++++++++ .../replica/fetcher/ReplicaFetcherITCase.java | 39 +- .../fetcher/TestingLeaderEndpoint.java | 11 +- .../testutils/FlussClusterExtension.java | 25 +- 31 files changed, 2608 insertions(+), 234 deletions(-) create mode 100644 fluss-common/src/main/java/org/apache/fluss/metadata/LeaderEpochOffset.java create mode 100644 fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/FetchLogEpochInfo.java create mode 100644 fluss-server/src/main/java/org/apache/fluss/server/log/LeaderEpochHistory.java create mode 100644 fluss-server/src/test/java/org/apache/fluss/server/log/LeaderEpochHistoryTest.java create mode 100644 fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/LeaderEpochCompatibilityITCase.java create mode 100644 fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/LeaderMigrationWalTest.java diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index 14b019853df..bdc3fa09ab1 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -985,6 +985,18 @@ public class ConfigOptions { + "we would fsync after every message; if it were 5 we would fsync after every " + "five messages."); + public static final ConfigOption LOG_REPLICATION_LEADER_EPOCH_ENABLED = + key("log.replication.leader-epoch.enabled") + .booleanType() + .defaultValue(true) + .withDescription( + "Whether to validate replication log history using leader epochs. " + + "Can be changed dynamically with alterClusterConfigs. Disabled peers remain compatible " + + "through legacy replication, which cannot verify divergent histories. " + + "Disabling discards local epoch history; enabling does not reconstruct " + + "unknown history for existing records. New boundaries are established " + + "by subsequent leader epochs, not by changing this option."); + public static final ConfigOption LOG_FLUSH_OFFSET_CHECKPOINT_INTERVAL = key("log.flush.offset.checkpoint-interval") .durationType() diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/LeaderEpochOffset.java b/fluss-common/src/main/java/org/apache/fluss/metadata/LeaderEpochOffset.java new file mode 100644 index 00000000000..f6c57ce842a --- /dev/null +++ b/fluss-common/src/main/java/org/apache/fluss/metadata/LeaderEpochOffset.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.metadata; + +import org.apache.fluss.annotation.Internal; + +import java.util.Objects; + +/** A leader epoch and its associated log boundary. */ +@Internal +public final class LeaderEpochOffset { + private final int epoch; + private final long offset; + + public LeaderEpochOffset(int epoch, long offset) { + this.epoch = epoch; + this.offset = offset; + } + + public int epoch() { + return epoch; + } + + public long offset() { + return offset; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof LeaderEpochOffset)) { + return false; + } + LeaderEpochOffset that = (LeaderEpochOffset) other; + return epoch == that.epoch && offset == that.offset; + } + + @Override + public int hashCode() { + return Objects.hash(epoch, offset); + } + + @Override + public String toString() { + return "LeaderEpochOffset(" + epoch + ", " + offset + ")"; + } +} diff --git a/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogManifestJsonSerde.java b/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogManifestJsonSerde.java index 00eddd0baef..d1425616342 100644 --- a/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogManifestJsonSerde.java +++ b/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogManifestJsonSerde.java @@ -17,6 +17,7 @@ package org.apache.fluss.remote; +import org.apache.fluss.metadata.LeaderEpochOffset; import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.shaded.jackson2.com.fasterxml.jackson.core.JsonGenerator; @@ -52,6 +53,7 @@ public class RemoteLogManifestJsonSerde private static final String MAX_TIMESTAMP_FIELD = "max_timestamp"; private static final String SEGMENT_SIZE_IN_BYTES_FIELD = "size_in_bytes"; private static final String HIGHEST_COPIED_END_OFFSET_FIELD = "highest_copied_end_offset"; + private static final String LEADER_EPOCHS_FIELD = "leader_epochs"; private static final int SNAPSHOT_VERSION = 1; @Override @@ -99,6 +101,16 @@ public void serialize(RemoteLogManifest manifest, JsonGenerator generator) throw generator.writeNumberField(MAX_TIMESTAMP_FIELD, remoteLogSegment.maxTimestamp()); generator.writeNumberField( SEGMENT_SIZE_IN_BYTES_FIELD, remoteLogSegment.segmentSizeInBytes()); + if (!remoteLogSegment.leaderEpochs().isEmpty()) { + generator.writeArrayFieldStart(LEADER_EPOCHS_FIELD); + for (LeaderEpochOffset epoch : remoteLogSegment.leaderEpochs()) { + generator.writeStartObject(); + generator.writeNumberField("epoch", epoch.epoch()); + generator.writeNumberField(START_OFFSET_FIELD, epoch.offset()); + generator.writeEndObject(); + } + generator.writeEndArray(); + } generator.writeEndObject(); } generator.writeEndArray(); @@ -150,6 +162,17 @@ public RemoteLogManifest deserialize(JsonNode node) { if (logicalEndOffsetNode != null) { segmentBuilder.logicalEndOffset(logicalEndOffsetNode.asLong()); } + JsonNode epochNodes = entryJson.get(LEADER_EPOCHS_FIELD); + if (epochNodes != null) { + List epochs = new ArrayList<>(); + for (JsonNode epoch : epochNodes) { + epochs.add( + new LeaderEpochOffset( + epoch.get("epoch").asInt(), + epoch.get(START_OFFSET_FIELD).asLong())); + } + segmentBuilder.leaderEpochs(epochs); + } snapshotEntries.add(segmentBuilder.build()); } diff --git a/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogSegment.java b/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogSegment.java index e68c5338843..112d9096533 100644 --- a/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogSegment.java +++ b/fluss-common/src/main/java/org/apache/fluss/remote/RemoteLogSegment.java @@ -18,11 +18,15 @@ package org.apache.fluss.remote; import org.apache.fluss.annotation.Internal; +import org.apache.fluss.metadata.LeaderEpochOffset; import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.TableBucket; import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.Objects; import java.util.UUID; @@ -58,6 +62,7 @@ public class RemoteLogSegment { private final long maxTimestamp; private final int segmentSizeInBytes; + private final List leaderEpochs; private RemoteLogSegment( PhysicalTablePath physicalTablePath, @@ -68,7 +73,8 @@ private RemoteLogSegment( @Nullable Long logicalStartOffset, @Nullable Long logicalEndOffset, long maxTimestamp, - int segmentSizeInBytes) { + int segmentSizeInBytes, + List leaderEpochs) { this.physicalTablePath = checkNotNull(physicalTablePath); this.tableBucket = checkNotNull(tableBucket); this.remoteLogSegmentId = checkNotNull(remoteLogSegmentId); @@ -107,6 +113,7 @@ private RemoteLogSegment( } this.maxTimestamp = maxTimestamp; this.segmentSizeInBytes = segmentSizeInBytes; + this.leaderEpochs = Collections.unmodifiableList(new ArrayList<>(leaderEpochs)); } public PhysicalTablePath physicalTablePath() { @@ -161,7 +168,13 @@ public RemoteLogSegment withLogicalRange(long logicalStartOffset, long logicalEn logicalStartOffset, logicalEndOffset, maxTimestamp, - segmentSizeInBytes); + segmentSizeInBytes, + leaderEpochs); + } + + /** Epoch boundaries for this physical segment; empty for legacy manifests. */ + public List leaderEpochs() { + return leaderEpochs; } public long maxTimestamp() { @@ -189,7 +202,8 @@ public boolean equals(Object o) { && maxTimestamp == that.maxTimestamp && Objects.equals(remoteLogSegmentId, that.remoteLogSegmentId) && Objects.equals(physicalTablePath, that.physicalTablePath) - && Objects.equals(tableBucket, that.tableBucket); + && Objects.equals(tableBucket, that.tableBucket) + && leaderEpochs.equals(that.leaderEpochs); } @Override @@ -203,7 +217,8 @@ public int hashCode() { logicalStartOffset, logicalEndOffset, maxTimestamp, - segmentSizeInBytes); + segmentSizeInBytes, + leaderEpochs); } @Override @@ -241,6 +256,7 @@ public static class Builder { private @Nullable Long logicalEndOffset; private long maxTimestamp; private int segmentSizeInBytes; + private List leaderEpochs = Collections.emptyList(); public static Builder builder() { return new Builder(); @@ -291,6 +307,11 @@ public Builder tableBucket(TableBucket tableBucket) { return this; } + public Builder leaderEpochs(List leaderEpochs) { + this.leaderEpochs = leaderEpochs; + return this; + } + public RemoteLogSegment build() { return new RemoteLogSegment( physicalTablePath, @@ -301,7 +322,8 @@ public RemoteLogSegment build() { logicalStartOffset, logicalEndOffset, maxTimestamp, - segmentSizeInBytes); + segmentSizeInBytes, + leaderEpochs); } } } diff --git a/fluss-common/src/test/java/org/apache/fluss/remote/RemoteLogManifestJsonSerdeTest.java b/fluss-common/src/test/java/org/apache/fluss/remote/RemoteLogManifestJsonSerdeTest.java index e0951321589..24f46a925f9 100644 --- a/fluss-common/src/test/java/org/apache/fluss/remote/RemoteLogManifestJsonSerdeTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/remote/RemoteLogManifestJsonSerdeTest.java @@ -17,14 +17,21 @@ package org.apache.fluss.remote; +import org.apache.fluss.metadata.LeaderEpochOffset; import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TablePath; import org.apache.fluss.utils.json.JsonSerdeTestBase; +import org.apache.fluss.utils.json.JsonSerdeUtils; + +import org.junit.jupiter.api.Test; import java.util.Arrays; +import java.util.Collections; import java.util.UUID; +import static org.assertj.core.api.Assertions.assertThat; + /** Tests of {@link RemoteLogManifestJsonSerde}. */ class RemoteLogManifestJsonSerdeTest extends JsonSerdeTestBase { private static final PhysicalTablePath TABLE_PATH1 = @@ -102,6 +109,32 @@ protected RemoteLogManifestJsonSerdeTest() { super(RemoteLogManifestJsonSerde.INSTANCE); } + @Test + void testLeaderEpochHistorySurvivesManifestRoundTrip() throws Exception { + RemoteLogSegment segment = + RemoteLogSegment.Builder.builder() + .physicalTablePath(TABLE_PATH1) + .tableBucket(TABLE_BUCKET1) + .remoteLogSegmentId(UUID.randomUUID()) + .remoteLogStartOffset(10) + .remoteLogEndOffset(30) + .maxTimestamp(0) + .segmentSizeInBytes(100) + .leaderEpochs( + Arrays.asList( + new LeaderEpochOffset(4, 5), new LeaderEpochOffset(7, 20))) + .build(); + RemoteLogManifest manifest = + new RemoteLogManifest( + TABLE_PATH1, TABLE_BUCKET1, Collections.singletonList(segment)); + byte[] json = + JsonSerdeUtils.writeValueAsBytes(manifest, RemoteLogManifestJsonSerde.INSTANCE); + assertThat(JsonSerdeUtils.readValue(json, RemoteLogManifestJsonSerde.INSTANCE)) + .isEqualTo(manifest); + assertThat(segment.withLogicalRange(15, 25).leaderEpochs()) + .containsExactly(new LeaderEpochOffset(4, 5), new LeaderEpochOffset(7, 20)); + } + @Override protected RemoteLogManifest[] createObjects() { return new RemoteLogManifest[] {MANIFEST_SNAPSHOT1, MANIFEST_SNAPSHOT2}; diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/FetchLogEpochInfo.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/FetchLogEpochInfo.java new file mode 100644 index 00000000000..50ce0d43da2 --- /dev/null +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/FetchLogEpochInfo.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.rpc.entity; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.metadata.LeaderEpochOffset; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** Epoch validation and log history attached to a replica fetch response. */ +@Internal +public final class FetchLogEpochInfo { + private final int leaderEpoch; + @Nullable private final LeaderEpochOffset divergingEpoch; + private final List epochStarts; + + public FetchLogEpochInfo( + int leaderEpoch, + @Nullable LeaderEpochOffset divergingEpoch, + List epochStarts) { + this.leaderEpoch = leaderEpoch; + this.divergingEpoch = divergingEpoch; + this.epochStarts = Collections.unmodifiableList(new ArrayList<>(epochStarts)); + } + + public int leaderEpoch() { + return leaderEpoch; + } + + @Nullable + public LeaderEpochOffset divergingEpoch() { + return divergingEpoch; + } + + public List epochStarts() { + return epochStarts; + } +} diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/FetchLogResultForBucket.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/FetchLogResultForBucket.java index 11f6d7e73b8..8a88b6f3f6b 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/FetchLogResultForBucket.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/entity/FetchLogResultForBucket.java @@ -38,6 +38,7 @@ public class FetchLogResultForBucket extends ResultForBucket { private final long highWatermark; private final long filteredEndOffset; private final long minRetainOffset; + @Nullable private final FetchLogEpochInfo epochInfo; private FetchLogResultForBucket( TableBucket tableBucket, @@ -47,7 +48,28 @@ private FetchLogResultForBucket( long filteredEndOffset, long minRetainOffset, ApiError error) { + this( + tableBucket, + remoteLogFetchInfo, + records, + highWatermark, + filteredEndOffset, + minRetainOffset, + error, + null); + } + + private FetchLogResultForBucket( + TableBucket tableBucket, + @Nullable RemoteLogFetchInfo remoteLogFetchInfo, + @Nullable LogRecords records, + long highWatermark, + long filteredEndOffset, + long minRetainOffset, + ApiError error, + @Nullable FetchLogEpochInfo epochInfo) { super(tableBucket, error); + this.epochInfo = epochInfo; this.remoteLogFetchInfo = remoteLogFetchInfo; this.records = records; this.highWatermark = highWatermark; @@ -55,6 +77,24 @@ private FetchLogResultForBucket( this.minRetainOffset = minRetainOffset; } + /** Returns a copy carrying replica epoch information. */ + public FetchLogResultForBucket withEpochInfo(@Nullable FetchLogEpochInfo epochInfo) { + return new FetchLogResultForBucket( + getTableBucket(), + remoteLogFetchInfo, + records, + highWatermark, + filteredEndOffset, + minRetainOffset, + getError(), + epochInfo); + } + + @Nullable + public FetchLogEpochInfo epochInfo() { + return epochInfo; + } + /** Creates a successful local fetch result. */ public static FetchLogResultForBucket records( TableBucket tableBucket, diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java index 37d9f907449..e7082a9f5a1 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/protocol/ApiKeys.java @@ -44,7 +44,7 @@ public enum ApiKeys { UPDATE_METADATA(1013, 0, 0, PRIVATE), // Version 1: Supports original_partition_name in requests and responses for historical writes. PRODUCE_LOG(1014, 0, 1, PUBLIC), - FETCH_LOG(1015, 0, 0, PUBLIC), + FETCH_LOG(1015, 0, 1, PUBLIC), // Version 0: Uses lake's encoder for primary key encoding (legacy behavior). // Version 1: Uses CompactedKeyEncoder for primary key encoding when bucket key differs from diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java index 8733056f1f8..5233357d2b7 100644 --- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java +++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/util/CommonRpcMessageUtils.java @@ -17,6 +17,7 @@ package org.apache.fluss.rpc.util; +import org.apache.fluss.metadata.LeaderEpochOffset; import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.ResolvedPartitionSpec; import org.apache.fluss.metadata.TableBucket; @@ -25,12 +26,14 @@ import org.apache.fluss.record.MemoryLogRecords; import org.apache.fluss.remote.RemoteLogFetchInfo; import org.apache.fluss.remote.RemoteLogSegment; +import org.apache.fluss.rpc.entity.FetchLogEpochInfo; import org.apache.fluss.rpc.entity.FetchLogResultForBucket; import org.apache.fluss.rpc.messages.LookupRequest; import org.apache.fluss.rpc.messages.PbAclFilter; import org.apache.fluss.rpc.messages.PbAclInfo; import org.apache.fluss.rpc.messages.PbFetchLogRespForBucket; import org.apache.fluss.rpc.messages.PbKeyValue; +import org.apache.fluss.rpc.messages.PbLeaderEpochOffset; import org.apache.fluss.rpc.messages.PbPartitionSpec; import org.apache.fluss.rpc.messages.PbRemoteLogFetchInfo; import org.apache.fluss.rpc.messages.PbRemoteLogSegment; @@ -208,8 +211,14 @@ public static FetchLogResultForBucket getFetchLogResultForBucket( pbRemoteLogSegment.hasMaxTimestamp() ? pbRemoteLogSegment.getMaxTimestamp() : -1; + List segmentEpochs = new ArrayList<>(); + for (PbLeaderEpochOffset epoch : pbRemoteLogSegment.getLeaderEpochsList()) { + segmentEpochs.add( + new LeaderEpochOffset(epoch.getEpoch(), epoch.getOffset())); + } RemoteLogSegment remoteLogSegment = RemoteLogSegment.Builder.builder() + .leaderEpochs(segmentEpochs) .tableBucket(tb) .physicalTablePath(physicalTablePath) .remoteLogSegmentId( @@ -253,6 +262,22 @@ public static FetchLogResultForBucket getFetchLogResultForBucket( } } + if (respForBucket.hasCurrentLeaderEpoch()) { + List starts = new ArrayList<>(); + for (PbLeaderEpochOffset start : respForBucket.getEpochStartsList()) { + starts.add(new LeaderEpochOffset(start.getEpoch(), start.getOffset())); + } + LeaderEpochOffset divergence = + respForBucket.hasDivergingEpoch() + ? new LeaderEpochOffset( + respForBucket.getDivergingEpoch().getEpoch(), + respForBucket.getDivergingEpoch().getOffset()) + : null; + fetchLogResultForBucket = + fetchLogResultForBucket.withEpochInfo( + new FetchLogEpochInfo( + respForBucket.getCurrentLeaderEpoch(), divergence, starts)); + } return fetchLogResultForBucket; } diff --git a/fluss-rpc/src/main/proto/FlussApi.proto b/fluss-rpc/src/main/proto/FlussApi.proto index b778db8c24a..71c0a2d2c1d 100644 --- a/fluss-rpc/src/main/proto/FlussApi.proto +++ b/fluss-rpc/src/main/proto/FlussApi.proto @@ -934,16 +934,23 @@ message PbFetchLogReqForTable { message PbFetchLogReqForBucket { optional int64 partition_id = 1; required int32 bucket_id = 2; - // TODO leader epoch required int64 fetch_offset = 3; required int32 max_fetch_bytes = 4; optional int32 routing_bucket_count = 5; + // Replication only. Both fields must be present to request epoch validation. + optional int32 current_leader_epoch = 6; + optional int32 last_fetched_epoch = 7; } message PbFetchLogRespForTable { required int64 table_id = 1; repeated PbFetchLogRespForBucket buckets_resp = 2; } +message PbLeaderEpochOffset { + required int32 epoch = 1; + required int64 offset = 2; +} + message PbFetchLogRespForBucket { optional int64 partition_id = 1; required int32 bucket_id = 2; @@ -960,6 +967,9 @@ message PbFetchLogRespForBucket { // The safe local log retention boundary confirmed by a committed KV snapshot. This is only // returned for KV follower fetches and is distinct from the physical log_start_offset. optional int64 min_retain_offset = 10; + optional int32 current_leader_epoch = 11; + optional PbLeaderEpochOffset diverging_epoch = 12; + repeated PbLeaderEpochOffset epoch_starts = 13; } message PbPutKvReqForBucket { @@ -1157,6 +1167,7 @@ message PbRemoteLogSegment { required int64 remote_log_end_offset = 3; required int32 segment_size_in_bytes = 4; optional int64 max_timestamp = 5; + repeated PbLeaderEpochOffset leader_epochs = 6; } message PbPartitionInfo { diff --git a/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java b/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java index 5c891dd2edb..3fd548d2bcb 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/DynamicServerConfig.java @@ -52,6 +52,7 @@ import static org.apache.fluss.config.ConfigOptions.KV_LEADER_REPLICA_MEMORY_RESERVED; import static org.apache.fluss.config.ConfigOptions.KV_SHARED_RATE_LIMITER_BYTES_PER_SEC; import static org.apache.fluss.config.ConfigOptions.KV_SNAPSHOT_INTERVAL; +import static org.apache.fluss.config.ConfigOptions.LOG_REPLICATION_LEADER_EPOCH_ENABLED; import static org.apache.fluss.config.ConfigOptions.LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER; import static org.apache.fluss.config.ConfigOptions.LOG_RETENTION_ROLL_ACTIVE_SEGMENT_ENABLED; import static org.apache.fluss.config.ConfigOptions.NETTY_SERVER_MAX_QUEUED_HISTORICAL_REQUESTS; @@ -83,6 +84,7 @@ class DynamicServerConfig { Arrays.asList( DATALAKE_FORMAT.key(), LOG_RETENTION_ROLL_ACTIVE_SEGMENT_ENABLED.key(), + LOG_REPLICATION_LEADER_EPOCH_ENABLED.key(), LOG_REPLICA_MIN_IN_SYNC_REPLICAS_NUMBER.key(), KV_LEADER_REPLICA_MEMORY_RESERVED.key(), KV_SHARED_RATE_LIMITER_BYTES_PER_SEC.key(), diff --git a/fluss-server/src/main/java/org/apache/fluss/server/entity/FetchReqInfo.java b/fluss-server/src/main/java/org/apache/fluss/server/entity/FetchReqInfo.java index 0f8df680028..102a3f15116 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/entity/FetchReqInfo.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/entity/FetchReqInfo.java @@ -32,6 +32,8 @@ public final class FetchReqInfo { @Nullable private final int[] projectFields; private int maxBytes; + private final int currentLeaderEpoch; + private final int lastFetchedEpoch; public FetchReqInfo(long tableId, long fetchOffset, int maxBytes) { this(tableId, fetchOffset, maxBytes, null); @@ -39,12 +41,32 @@ public FetchReqInfo(long tableId, long fetchOffset, int maxBytes) { public FetchReqInfo( long tableId, long fetchOffset, int maxBytes, @Nullable int[] projectFields) { + this(tableId, fetchOffset, maxBytes, projectFields, -1, -1); + } + + public FetchReqInfo( + long tableId, + long fetchOffset, + int maxBytes, + @Nullable int[] projectFields, + int currentLeaderEpoch, + int lastFetchedEpoch) { + this.currentLeaderEpoch = currentLeaderEpoch; + this.lastFetchedEpoch = lastFetchedEpoch; this.tableId = tableId; this.fetchOffset = fetchOffset; this.maxBytes = maxBytes; this.projectFields = projectFields; } + public int currentLeaderEpoch() { + return currentLeaderEpoch; + } + + public int lastFetchedEpoch() { + return lastFetchedEpoch; + } + public long getTableId() { return tableId; } @@ -97,11 +119,20 @@ public boolean equals(Object o) { return false; } - return fetchOffset == fetchReqInfo.fetchOffset && maxBytes == fetchReqInfo.maxBytes; + return fetchOffset == fetchReqInfo.fetchOffset + && maxBytes == fetchReqInfo.maxBytes + && currentLeaderEpoch == fetchReqInfo.currentLeaderEpoch + && lastFetchedEpoch == fetchReqInfo.lastFetchedEpoch; } @Override public int hashCode() { - return Objects.hash(tableId, fetchOffset, maxBytes, Arrays.hashCode(projectFields)); + return Objects.hash( + tableId, + fetchOffset, + maxBytes, + Arrays.hashCode(projectFields), + currentLeaderEpoch, + lastFetchedEpoch); } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParams.java b/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParams.java index 76a30a1ddc7..87ebade8fa9 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParams.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParams.java @@ -71,7 +71,8 @@ public final class FetchParams { private final int minFetchBytes; private final long maxWaitMs; private final FetchLogReadPreference readPreference; - // TODO: add more params like epoch etc. + private int currentLeaderEpoch = -1; + private int lastFetchedEpoch = -1; public FetchParams(int replicaId, int maxFetchBytes) { this( @@ -134,6 +135,20 @@ public FetchParams( FetchLogReadPreference.LOCAL_FIRST); } + /** Sets replication epoch fields for the bucket currently being fetched. */ + public void setCurrentFetchEpoch(int currentLeaderEpoch, int lastFetchedEpoch) { + this.currentLeaderEpoch = currentLeaderEpoch; + this.lastFetchedEpoch = lastFetchedEpoch; + } + + public int currentLeaderEpoch() { + return currentLeaderEpoch; + } + + public int lastFetchedEpoch() { + return lastFetchedEpoch; + } + public void setCurrentFetch( long tableId, long fetchOffset, diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LeaderEpochHistory.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LeaderEpochHistory.java new file mode 100644 index 00000000000..e4554ed4eb2 --- /dev/null +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LeaderEpochHistory.java @@ -0,0 +1,251 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.log; + +import org.apache.fluss.exception.LogStorageException; +import org.apache.fluss.metadata.LeaderEpochOffset; +import org.apache.fluss.server.log.checkpoint.CheckpointFile; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Optional; +import java.util.TreeMap; + +import static org.apache.fluss.utils.Preconditions.checkArgument; + +/** + * Persistent epoch boundaries for a tablet's log. Epochs may be skipped, and an epoch may contain + * no records. Offsets before the first boundary have unknown history. + * + *

The caller serializes all reads and mutations with log appends and truncation. Boundaries must + * be persisted before their records are appended. Log truncation must be durable before boundaries + * are removed. + */ +final class LeaderEpochHistory { + private final CheckpointFile checkpoint; + private final NavigableMap epochs = new TreeMap<>(); + private IOException persistenceFailure; + + LeaderEpochHistory(File file) throws IOException { + checkpoint = new CheckpointFile<>(file, 0, new EntryFormatter()); + for (LeaderEpochOffset entry : checkpoint.read()) { + if (entry.epoch() < 0 + || entry.offset() < 0 + || (!epochs.isEmpty() + && (entry.epoch() <= epochs.lastKey() + || entry.offset() < epochs.lastEntry().getValue()))) { + throw new IOException("Invalid leader epoch history in " + file); + } + epochs.put(entry.epoch(), entry.offset()); + } + } + + /** Persists an epoch boundary before publishing it to readers. */ + void assign(int epoch, long startOffset) throws IOException { + ensureUsable(); + checkArgument(epoch >= 0 && startOffset >= 0, "Negative epoch or offset."); + Long existing = epochs.get(epoch); + if (existing != null) { + checkArgument(existing == startOffset, "Epoch already starts at %s", existing); + return; + } + checkArgument(epochs.isEmpty() || epoch > epochs.lastKey(), "Epoch must increase."); + checkArgument( + epochs.isEmpty() || startOffset >= epochs.lastEntry().getValue(), + "Epoch offsets must not decrease."); + NavigableMap updated = new TreeMap<>(epochs); + updated.put(epoch, startOffset); + persist(updated); + } + + /** Returns the epoch of an actual record, excluding empty epochs at the log end. */ + int epochForOffset(long offset, long logEndOffset) { + ensureUsable(); + if (offset < 0 || offset >= logEndOffset) { + return -1; + } + int result = -1; + for (Map.Entry entry : epochs.entrySet()) { + if (entry.getValue() > offset) { + break; + } + result = entry.getKey(); + } + return result; + } + + /** Returns the end of the greatest known epoch no greater than the requested epoch. */ + Optional endOffsetFor(int requestedEpoch, long logEndOffset) { + ensureUsable(); + Map.Entry entry = epochs.floorEntry(requestedEpoch); + if (entry == null || entry.getValue() > logEndOffset) { + return Optional.empty(); + } + Map.Entry next = epochs.higherEntry(entry.getKey()); + return Optional.of( + new LeaderEpochOffset( + entry.getKey(), + next == null ? logEndOffset : Math.min(next.getValue(), logEndOffset))); + } + + /** Removes boundaries for records discarded by a durable log truncation. */ + void truncateFromEnd(long endOffset) throws IOException { + ensureUsable(); + checkArgument(endOffset >= 0, "Negative log end offset."); + NavigableMap updated = new TreeMap<>(epochs); + updated.entrySet().removeIf(entry -> entry.getValue() >= endOffset); + if (!updated.equals(epochs)) { + persist(updated); + } + } + + /** Retains the epoch covering the first retained record and all later boundaries. */ + void truncateFromStart(long startOffset) throws IOException { + ensureUsable(); + int retainedEpoch = epochForOffset(startOffset, Long.MAX_VALUE); + if (retainedEpoch < 0) { + return; + } + NavigableMap updated = new TreeMap<>(epochs.tailMap(retainedEpoch, true)); + if (!updated.equals(epochs)) { + persist(updated); + } + } + + /** + * Discards epoch knowledge without changing WAL bytes, before an untracked append or restore. + */ + void invalidate() throws IOException { + ensureUsable(); + if (!epochs.isEmpty()) { + persist(new TreeMap<>()); + } + } + + /** Returns boundaries covering a fetched range; the first may precede its start. */ + List entries(long startOffset, long endOffset) { + ensureUsable(); + List result = new ArrayList<>(); + if (endOffset <= startOffset) { + return result; + } + int first = epochForOffset(startOffset, Long.MAX_VALUE); + for (Map.Entry entry : epochs.entrySet()) { + if (entry.getValue() >= endOffset) { + break; + } + if (entry.getKey() >= first) { + result.add(new LeaderEpochOffset(entry.getKey(), entry.getValue())); + } + } + return result; + } + + /** Persists the epoch boundaries accompanying a contiguous replication append. */ + void append(List entries, long startOffset, long endOffset) + throws IOException { + ensureUsable(); + if (endOffset <= startOffset) { + return; + } + NavigableMap updated = new TreeMap<>(epochs); + for (LeaderEpochOffset entry : entries) { + if (entry.offset() >= endOffset) { + break; + } + Long existing = updated.get(entry.epoch()); + if (existing != null) { + checkArgument( + existing == entry.offset(), + "Conflicting start offset for epoch %s", + entry.epoch()); + continue; + } + // A source boundary can precede this fetch, but it cannot establish the identity + // of bytes we did not copy. Preserve unknown history until a new boundary is fetched. + if (entry.offset() < startOffset) { + continue; + } + checkArgument(entry.epoch() >= 0 && entry.offset() >= 0, "Negative epoch boundary."); + checkArgument( + updated.isEmpty() + || (entry.epoch() > updated.lastKey() + && entry.offset() >= updated.lastEntry().getValue()), + "Non-monotonic replication epoch history."); + updated.put(entry.epoch(), entry.offset()); + } + if (!updated.equals(epochs)) { + persist(updated); + } + } + + /** Prevents further access after WAL or checkpoint persistence becomes uncertain. */ + void markFailed(IOException cause) { + persistenceFailure = cause; + } + + void ensureUsable() { + if (persistenceFailure != null) { + throw new LogStorageException( + "Leader epoch checkpoint failed; the log must be reloaded before further use.", + persistenceFailure); + } + } + + private void persist(NavigableMap updated) throws IOException { + List entries = new ArrayList<>(); + updated.forEach((epoch, offset) -> entries.add(new LeaderEpochOffset(epoch, offset))); + try { + checkpoint.write(entries); + } catch (IOException e) { + // A failed directory sync can follow a successful atomic replacement. Neither the + // old in-memory history nor a retry is safe until the log has been recovered. + persistenceFailure = e; + throw e; + } + epochs.clear(); + epochs.putAll(updated); + } + + private static final class EntryFormatter + implements CheckpointFile.EntryFormatter { + @Override + public String toString(LeaderEpochOffset entry) { + return entry.epoch() + " " + entry.offset(); + } + + @Override + public Optional fromString(String line) { + String[] fields = line.split(" "); + if (fields.length != 2) { + return Optional.empty(); + } + try { + return Optional.of( + new LeaderEpochOffset( + Integer.parseInt(fields[0]), Long.parseLong(fields[1]))); + } catch (NumberFormatException e) { + return Optional.empty(); + } + } + } +} diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LogManager.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LogManager.java index 4c1a84dd1fa..7186acf77be 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/LogManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LogManager.java @@ -412,6 +412,20 @@ public void truncateTo(TableBucket tableBucket, long offset) throws LogStorageEx } } + /** Applies epoch reconciliation only while its replication mode is enabled. */ + public boolean truncateToWithEpoch(TableBucket tableBucket, long offset) { + return inLock( + logCreationOrDeletionLock, + () -> { + LogTablet log = currentLogs.get(tableBucket); + if (log == null || !log.isLeaderEpochEnabled()) { + return false; + } + truncateTo(tableBucket, offset); + return true; + }); + } + public void truncateFullyAndStartAt(TableBucket tableBucket, long newOffset) { LogTablet logTablet = currentLogs.get(tableBucket); // If the log tablet does not exist, skip it. @@ -568,6 +582,16 @@ public void validate(Configuration newConfig) { @Override public void reconfigure(Configuration newConfig) { + inLock( + logCreationOrDeletionLock, + () -> { + boolean enabled = + newConfig.get(ConfigOptions.LOG_REPLICATION_LEADER_EPOCH_ENABLED); + for (LogTablet log : currentLogs.values()) { + log.setLeaderEpochEnabled(enabled); + } + conf.set(ConfigOptions.LOG_REPLICATION_LEADER_EPOCH_ENABLED, enabled); + }); boolean newRollExpiredActiveSegmentEnabled = newConfig.get(ConfigOptions.LOG_RETENTION_ROLL_ACTIVE_SEGMENT_ENABLED); boolean oldRollExpiredActiveSegmentEnabled = diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LogReadInfo.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LogReadInfo.java index 0d14fb2412e..5947f85ec5d 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/LogReadInfo.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LogReadInfo.java @@ -18,6 +18,9 @@ package org.apache.fluss.server.log; import org.apache.fluss.annotation.Internal; +import org.apache.fluss.rpc.entity.FetchLogEpochInfo; + +import javax.annotation.Nullable; /** Structure used for lower level reads. */ @Internal @@ -27,18 +30,38 @@ public class LogReadInfo { private final long highWatermark; private final long logEndOffset; private final long minRetainOffset; + @Nullable private final FetchLogEpochInfo epochInfo; public LogReadInfo( FetchDataInfo fetchedData, long highWatermark, long logEndOffset, long minRetainOffset) { + this(fetchedData, highWatermark, logEndOffset, minRetainOffset, null); + } + + public LogReadInfo( + FetchDataInfo fetchedData, + long highWatermark, + long logEndOffset, + long minRetainOffset, + @Nullable FetchLogEpochInfo epochInfo) { + this.epochInfo = epochInfo; this.fetchedData = fetchedData; this.highWatermark = highWatermark; this.logEndOffset = logEndOffset; this.minRetainOffset = minRetainOffset; } + @Nullable + public FetchLogEpochInfo epochInfo() { + return epochInfo; + } + + public LogReadInfo withEpochInfo(@Nullable FetchLogEpochInfo info) { + return new LogReadInfo(fetchedData, highWatermark, logEndOffset, minRetainOffset, info); + } + public FetchDataInfo getFetchedData() { return fetchedData; } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java index 47e0e7eb2c6..14ad0e002c0 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java @@ -27,6 +27,7 @@ import org.apache.fluss.exception.InvalidTimestampException; import org.apache.fluss.exception.LogOffsetOutOfRangeException; import org.apache.fluss.exception.LogStorageException; +import org.apache.fluss.metadata.LeaderEpochOffset; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.TableBucket; @@ -42,6 +43,7 @@ import org.apache.fluss.server.log.LocalLog.SegmentDeletionReason; import org.apache.fluss.server.metrics.group.BucketMetricGroup; import org.apache.fluss.server.metrics.group.TabletServerMetricGroup; +import org.apache.fluss.utils.FileUtils; import org.apache.fluss.utils.FlussPaths; import org.apache.fluss.utils.clock.Clock; import org.apache.fluss.utils.concurrent.Scheduler; @@ -88,6 +90,9 @@ public final class LogTablet { // Configured local storage root that owns this tablet, for example /data-0. private final File dataDir; + + @GuardedBy("lock") + private final LeaderEpochHistory leaderEpochHistory; // Logical table/partition identity of this tablet. private final PhysicalTablePath physicalPath; @@ -125,6 +130,7 @@ public final class LogTablet { // The minimum offset that should be retained in the local log. This is used to ensure that, // the offset of kv snapshot should be retained, otherwise, kv recovery will fail. private final AtomicLong minRetainOffset; + private volatile boolean leaderEpochEnabled; // tracking the log start offset in remote storage private volatile long remoteLogStartOffset = Long.MAX_VALUE; // tracking the log end offset in remote storage @@ -154,10 +160,22 @@ private LogTablet( WriterStateManager writerStateManager, TableConfig tableConfig, boolean isChangelog, - Clock clock) { + Clock clock) + throws IOException { this.dataDir = dataDir; this.physicalPath = physicalPath; this.localLog = localLog; + this.leaderEpochEnabled = conf.get(ConfigOptions.LOG_REPLICATION_LEADER_EPOCH_ENABLED); + this.leaderEpochHistory = + new LeaderEpochHistory(new File(getLogDir(), "leader-epoch-checkpoint")); + // LogLoader may have removed an incomplete tail. Make that recovery durable before + // removing boundaries that described it. + localLog.getSegments().activeSegment().flush(); + FileUtils.flushDirIfExists(getLogDir().toPath()); + leaderEpochHistory.truncateFromEnd(localLogEndOffset()); + if (!leaderEpochEnabled) { + leaderEpochHistory.invalidate(); + } this.maxSegmentFileSize = (int) conf.get(ConfigOptions.LOG_SEGMENT_FILE_SIZE).getBytes(); this.logFlushIntervalMessages = conf.get(ConfigOptions.LOG_FLUSH_INTERVAL_MESSAGES); int writerExpirationCheckIntervalMs = @@ -504,6 +522,116 @@ public LogAppendInfo appendAsFollower(MemoryLogRecords records) throws Exception return append(records, false); } + /** Whether this tablet participates in epoch-aware replication. */ + public boolean isLeaderEpochEnabled() { + return leaderEpochEnabled; + } + + /** Changes replication mode without assigning an epoch to previously untracked records. */ + void setLeaderEpochEnabled(boolean enabled) { + synchronized (lock) { + if (enabled == leaderEpochEnabled) { + return; + } + try { + // Persist loss of knowledge before publishing the mode change. Enabling cannot + // assign the current epoch a different start offset from its existing replicas. + leaderEpochHistory.invalidate(); + leaderEpochEnabled = enabled; + } catch (IOException e) { + throw new LogStorageException( + "Failed to change leader epoch mode for " + getTableBucket(), e); + } + } + } + + /** Records the boundary of a newly elected leader before accepting its writes. */ + public void assignLeaderEpoch(int epoch) throws IOException { + synchronized (lock) { + if (!leaderEpochEnabled) { + return; + } + leaderEpochHistory.truncateFromEnd(localLogEndOffset()); + leaderEpochHistory.assign(epoch, localLogEndOffset()); + } + } + + /** Returns the epoch of the last record before the given fetch offset, or -1 if unknown. */ + public int lastFetchedEpoch(long fetchOffset) { + synchronized (lock) { + return leaderEpochHistory.epochForOffset(fetchOffset - 1, localLogEndOffset()); + } + } + + /** Returns the end of the last known epoch at or before the requested epoch. */ + public Optional endOffsetForEpoch(int epoch) { + synchronized (lock) { + return leaderEpochHistory.endOffsetFor(epoch, localLogEndOffset()); + } + } + + /** Returns the epoch boundaries covering a replicated range. */ + public List epochEntries(long start, long end) { + synchronized (lock) { + return leaderEpochHistory.entries(start, end); + } + } + + /** Appends records and their source history under the same log lock. */ + public LogAppendInfo appendAsFollower(MemoryLogRecords records, List epochs) + throws Exception { + if (!leaderEpochEnabled) { + return appendAsFollower(records); + } + synchronized (lock) { + long endOffset = localLogEndOffset(); + for (LogRecordBatch batch : records.batches()) { + checkArgument( + batch.baseLogOffset() == endOffset, + "Non-contiguous replication append at %s, expected %s", + batch.baseLogOffset(), + endOffset); + endOffset = batch.lastLogOffset() + 1; + } + return append(records, false, epochs); + } + } + + /** Forgets unverified history before reading records from a legacy replication source. */ + public void invalidateLeaderEpochHistory() throws IOException { + synchronized (lock) { + leaderEpochHistory.invalidate(); + } + } + + /** Installs the writer snapshot and epoch history of a committed remote log prefix. */ + public void restoreFromRemote( + long endOffset, File writerSnapshot, List epochs) { + synchronized (lock) { + try { + truncateFullyAndStartAt(endOffset); + // Force loading the downloaded snapshot even though the WAL already ends here. + writerStateManager.truncateFullyAndStartAt(0L); + File target = FlussPaths.writerSnapshotFile(getLogDir(), endOffset); + FileUtils.flushFileIfExists(writerSnapshot.toPath()); + FileUtils.atomicMoveWithFallback(writerSnapshot.toPath(), target.toPath()); + writerStateManager.reloadSnapshots(); + loadWriterSnapshot(endOffset); + localLog.getSegments().activeSegment().flush(); + FileUtils.flushDirIfExists(getLogDir().toPath()); + if (leaderEpochEnabled) { + leaderEpochHistory.append(epochs, 0, endOffset); + } + } catch (Exception e) { + IOException failure = + e instanceof IOException ? (IOException) e : new IOException(e); + leaderEpochHistory.markFailed(failure); + throw new LogStorageException( + "Failed to restore remote log for " + getTableBucket(), failure); + } + } + } + /** Read messages from the local log without projection or filter. */ public FetchDataInfo read( long readOffset, int maxLength, FetchIsolation fetchIsolation, boolean minOneMessage) @@ -922,6 +1050,14 @@ private LogOffsetMetadata convertToOffsetMetadataOrThrow(long offset) throws IOE */ private LogAppendInfo append(MemoryLogRecords records, boolean appendAsLeader) throws Exception { + return append(records, appendAsLeader, null); + } + + private LogAppendInfo append( + MemoryLogRecords records, + boolean appendAsLeader, + @Nullable List sourceEpochs) + throws Exception { LogAppendInfo appendInfo = analyzeAndValidateRecords(records); // return if we have no valid records. @@ -934,6 +1070,7 @@ private LogAppendInfo append(MemoryLogRecords records, boolean appendAsLeader) synchronized (lock) { localLog.checkIfMemoryMappedBufferClosed(); + leaderEpochHistory.ensureUsable(); if (appendAsLeader) { long offset = localLog.getLocalLogEndOffset(); // assign offsets to the message set. @@ -987,11 +1124,35 @@ private LogAppendInfo append(MemoryLogRecords records, boolean appendAsLeader) // Append the records, and increment the local log end offset immediately after // append because write to the transaction index below may fail, and we want to // ensure that the offsets of future appends still grow monotonically. - localLog.append( - appendInfo.lastOffset(), - appendInfo.maxTimestamp(), - appendInfo.startOffsetOfMaxTimestamp(), - validRecords); + if (!appendAsLeader) { + if (!leaderEpochEnabled) { + sourceEpochs = null; + } + if (sourceEpochs == null + || sourceEpochs.isEmpty() + || sourceEpochs.get(0).offset() > localLogEndOffset() + || (sourceEpochs.get(0).offset() < localLogEndOffset() + && leaderEpochHistory.epochForOffset( + localLogEndOffset() - 1, localLogEndOffset()) + != sourceEpochs.get(0).epoch())) { + leaderEpochHistory.invalidate(); + } + if (sourceEpochs != null) { + leaderEpochHistory.append( + sourceEpochs, localLogEndOffset(), appendInfo.lastOffset() + 1); + } + } + try { + localLog.append( + appendInfo.lastOffset(), + appendInfo.maxTimestamp(), + appendInfo.startOffsetOfMaxTimestamp(), + validRecords); + } catch (IOException e) { + leaderEpochHistory.markFailed(e); + throw new LogStorageException( + "Failed to append WAL for " + getTableBucket(), e); + } updateHighWatermarkWithLogEndOffset(); // update the writer state. @@ -1148,6 +1309,9 @@ public void roll(Optional expectedNextOffset) throws IOException { /** Truncate this log so that it ends with the greatest offset < targetOffset. */ boolean truncateTo(long targetOffset) throws LogStorageException { + synchronized (lock) { + leaderEpochHistory.ensureUsable(); + } if (targetOffset < 0) { throw new IllegalArgumentException( String.format( @@ -1174,6 +1338,9 @@ boolean truncateTo(long targetOffset) throws LogStorageException { truncateFullyAndStartAt(targetOffset); } else { List deletedSegments = localLog.truncateTo(targetOffset); + localLog.getSegments().activeSegment().flush(); + FileUtils.flushDirIfExists(getLogDir().toPath()); + leaderEpochHistory.truncateFromEnd(localLogEndOffset()); deleteWriterSnapshots(deletedSegments, writerStateManager); rebuildWriterState(targetOffset, writerStateManager); @@ -1185,6 +1352,7 @@ boolean truncateTo(long targetOffset) throws LogStorageException { return true; } catch (IOException e) { + leaderEpochHistory.markFailed(e); throw new LogStorageException( String.format( "Error while truncating log for bucket %s to offset %s.", @@ -1200,11 +1368,13 @@ void truncateFullyAndStartAt(long newOffset) throws LogStorageException { LOG.debug("Truncate and start at offset {} for bucket {}", newOffset, getTableBucket()); synchronized (lock) { try { + leaderEpochHistory.invalidate(); localLog.truncateFullyAndStartAt(newOffset); writerStateManager.truncateFullyAndStartAt(newOffset); rebuildWriterState(newOffset, writerStateManager); updateHighWatermark(localLog.getLocalLogEndOffset()); } catch (IOException e) { + leaderEpochHistory.markFailed(e); throw new LogStorageException( String.format( "Error while truncating log for bucket %s to offset %s.", diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java index e650b03d674..cd4fa377045 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java @@ -307,6 +307,8 @@ private long copyLogSegmentFilesToRemote( .remoteLogEndOffset(segmentEndOffset) .maxTimestamp(segment.maxTimestampSoFar()) .segmentSizeInBytes(sizeInBytes) + .leaderEpochs( + log.epochEntries(segment.getBaseOffset(), segmentEndOffset)) .build(); try { remoteLogStorage.copyLogSegmentFiles(copyRemoteLogSegment, logSegmentFiles); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java index a74056ebc95..0fbb601a20b 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java @@ -36,6 +36,7 @@ import org.apache.fluss.exception.TooManyScannersException; import org.apache.fluss.fs.FsPath; import org.apache.fluss.metadata.ChangelogImage; +import org.apache.fluss.metadata.LeaderEpochOffset; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.Schema; @@ -53,6 +54,8 @@ import org.apache.fluss.record.LogRecordReadContext; import org.apache.fluss.record.LogRecords; import org.apache.fluss.record.MemoryLogRecords; +import org.apache.fluss.rpc.entity.FetchLogEpochInfo; +import org.apache.fluss.rpc.entity.FetchLogResultForBucket; import org.apache.fluss.rpc.protocol.Errors; import org.apache.fluss.rpc.protocol.MergeMode; import org.apache.fluss.rpc.util.PredicateMessageUtils; @@ -500,9 +503,11 @@ public void makeLeader(NotifyLeaderAndIsrData data) throws IOException { int requestLeaderEpoch = data.getLeaderEpoch(); if (requestLeaderEpoch > leaderEpoch) { - boolean isNewLeader = !isLeader(); + boolean resetFollowerOffsets = + logTablet.isLeaderEpochEnabled() || !isLeader(); leaderEpoch = requestLeaderEpoch; onBecomeNewLeader(); + logTablet.assignLeaderEpoch(leaderEpoch); leaderReplicaIdOpt.set(localTabletServerId); // onBecomeNewLeader may recover a KV snapshot, so start the ISR lag // grace period after it completes. @@ -513,7 +518,7 @@ public void makeLeader(NotifyLeaderAndIsrData data) throws IOException { followerReplica.resetFollowerReplicaState( currentTimeMs, leaderEndOffset, - isNewLeader, + resetFollowerOffsets, data.getIsr() .contains(followerReplica.getFollowerId())); } @@ -1240,6 +1245,121 @@ public LogAppendInfo appendRecordsToFollower(MemoryLogRecords memoryLogRecords) return logTablet.appendAsFollower(memoryLogRecords); } + /** Applies a fetch response only while the replica still follows its requested leader epoch. */ + public LogAppendInfo appendRecordsToFollower( + FetchLogResultForBucket data, int expectedLeader, int expectedEpoch) throws Exception { + return inReadLock( + leaderIsrUpdateLock, + () -> { + validateFollowerEpoch(expectedLeader, expectedEpoch); + FetchLogEpochInfo epochInfo = + logTablet.isLeaderEpochEnabled() ? data.epochInfo() : null; + MemoryLogRecords records = (MemoryLogRecords) data.recordsOrEmpty(); + LogAppendInfo appended; + if (epochInfo == null) { + appended = logTablet.appendAsFollower(records); + logTablet.updateHighWatermark(logTablet.localLogEndOffset()); + } else { + if (epochInfo.leaderEpoch() != expectedEpoch) { + throw new FencedLeaderEpochException( + "Fetch response refers to a different leader epoch."); + } + appended = logTablet.appendAsFollower(records, epochInfo.epochStarts()); + logTablet.updateHighWatermark( + Math.min(data.getHighWatermark(), logTablet.localLogEndOffset())); + } + if (isKvTable() && data.hasMinRetainOffset()) { + logTablet.updateMinRetainOffset(data.getMinRetainOffset()); + } + return appended; + }); + } + + /** Truncates a divergent follower tail without crossing a concurrent role change. */ + public void truncateFollowerTo(long offset, int expectedLeader, int expectedEpoch) { + inReadLock( + leaderIsrUpdateLock, + () -> { + validateFollowerEpoch(expectedLeader, expectedEpoch); + logManager.truncateTo(tableBucket, offset); + }); + } + + /** Serializes an epoch-based truncation with role changes and dynamic configuration. */ + public boolean truncateFollowerToEpochOffset( + long offset, int expectedLeader, int expectedEpoch) { + return inReadLock( + leaderIsrUpdateLock, + () -> { + validateFollowerEpoch(expectedLeader, expectedEpoch); + return logManager.truncateToWithEpoch(tableBucket, offset); + }); + } + + public void invalidateFollowerEpochHistory(int expectedLeader, int expectedEpoch) + throws IOException { + inReadLock( + leaderIsrUpdateLock, + () -> { + validateFollowerEpoch(expectedLeader, expectedEpoch); + logTablet.invalidateLeaderEpochHistory(); + }); + } + + public void truncateFollowerFullyAndStartAt( + long offset, int expectedLeader, int expectedEpoch) { + inReadLock( + leaderIsrUpdateLock, + () -> { + validateFollowerEpoch(expectedLeader, expectedEpoch); + truncateFullyAndStartAt(offset); + }); + } + + /** Installs a downloaded remote snapshot only if its requested leader is still current. */ + public void restoreFollowerFromRemote( + long endOffset, + File writerSnapshot, + List epochs, + long leaderHighWatermark, + int expectedLeader, + int expectedEpoch) { + inReadLock( + leaderIsrUpdateLock, + () -> { + validateFollowerEpoch(expectedLeader, expectedEpoch); + logTablet.restoreFromRemote(endOffset, writerSnapshot, epochs); + logTablet.updateHighWatermark(Math.min(endOffset, leaderHighWatermark)); + }); + } + + /** Attaches the leader identity to a remote fetch response after the manifest lookup. */ + public FetchLogResultForBucket withRemoteFetchEpoch( + FetchLogResultForBucket result, int expectedEpoch) { + return inReadLock( + leaderIsrUpdateLock, + () -> { + if (!isLeader() || leaderEpoch != expectedEpoch) { + throw new FencedLeaderEpochException( + "Leader changed during remote log lookup."); + } + return logTablet.isLeaderEpochEnabled() + ? result.withEpochInfo( + new FetchLogEpochInfo( + leaderEpoch, null, Collections.emptyList())) + : result; + }); + } + + private void validateFollowerEpoch(int expectedLeader, int expectedEpoch) { + if (isLeader() + || leaderEpoch != expectedEpoch + || !Objects.equals(getLeaderId(), expectedLeader)) { + throw new FencedLeaderEpochException( + "Replica no longer follows the requested leader epoch."); + } + } + /** * Samples the recent backpressure pressure for piggyback on a put response. Also records the * value on this bucket's {@link BucketMetricGroup} for table-level aggregation. @@ -1409,22 +1529,91 @@ public LogReadInfo fetchRecords(FetchParams fetchParams) throws IOException { physicalPath)); } if (fetchParams.isFromFollower()) { - long followerFetchTimeMs = clock.milliseconds(); - LogReadInfo logReadInfo = + LogReadInfo result = inReadLock( leaderIsrUpdateLock, () -> { LogTablet localLog = localLogOrThrow(fetchParams.fetchOnlyLeader()); - return readRecords(fetchParams, localLog); + boolean epochRequest = + localLog.isLeaderEpochEnabled() + && fetchParams.currentLeaderEpoch() >= 0; + if (epochRequest + && fetchParams.currentLeaderEpoch() != leaderEpoch) { + throw new FencedLeaderEpochException( + "Fetch request refers to a different leader epoch."); + } + if (epochRequest + && fetchParams.lastFetchedEpoch() >= 0 + && localLog.lastFetchedEpoch(fetchParams.fetchOffset()) + != fetchParams.lastFetchedEpoch()) { + Optional boundary = + localLog.endOffsetForEpoch( + fetchParams.lastFetchedEpoch()); + if (boundary.isPresent()) { + if (boundary.get().epoch() == fetchParams.lastFetchedEpoch() + && boundary.get().offset() + >= fetchParams.fetchOffset()) { + boundary = + localLog.endOffsetForEpoch( + fetchParams.lastFetchedEpoch() - 1); + } + if (boundary.isPresent()) { + return new LogReadInfo( + new FetchDataInfo(MemoryLogRecords.EMPTY), + localLog.getHighWatermark(), + localLog.localLogEndOffset(), + -1, + new FetchLogEpochInfo( + leaderEpoch, + boundary.get(), + Collections.emptyList())); + } + } + // Preserve legacy interoperability for prefixes without epoch + // proof; their history remains unknown. + } + long fetchTime = clock.milliseconds(); + LogReadInfo info = readRecords(fetchParams, localLog); + FollowerReplica follower = + getFollowerReplicaOrThrown(fetchParams.replicaId()); + follower.updateFetchState( + info.getFetchedData().getFetchOffsetMetadata(), + fetchTime, + info.getLogEndOffset()); + return info.withEpochInfo( + new FetchLogEpochInfo( + leaderEpoch, + null, + epochRequest + ? localLog.epochEntries( + fetchParams.fetchOffset(), + info.getLogEndOffset()) + : Collections.emptyList())); }); - - FollowerReplica followerReplica = getFollowerReplicaOrThrown(fetchParams.replicaId()); - updateFollowerFetchState( - followerReplica, - logReadInfo.getFetchedData().getFetchOffsetMetadata(), - followerFetchTimeMs, - logReadInfo.getLogEndOffset()); - return logReadInfo; + FetchLogEpochInfo epochInfo = result.epochInfo(); + if (epochInfo.divergingEpoch() == null) { + FollowerReplica follower = followerReplicasMap.get(fetchParams.replicaId()); + if (follower != null) { + maybeExpandISr(follower, epochInfo.leaderEpoch()); + boolean incremented = + inReadLock( + leaderIsrUpdateLock, + () -> + isLeader() + && leaderEpoch == epochInfo.leaderEpoch() + && followerReplicasMap.get( + fetchParams.replicaId()) + == follower + && maybeIncrementLeaderHW( + logTablet, clock.milliseconds())); + if (incremented) { + tryCompleteDelayedOperations(); + } + } + } + return logTablet.isLeaderEpochEnabled() && fetchParams.currentLeaderEpoch() >= 0 + ? result + : result.withEpochInfo(null); } else { return inReadLock( leaderIsrUpdateLock, @@ -1558,45 +1747,6 @@ private void updateAssignmentAndIsr( isrState = new IsrState.CommittedIsrState(isr, standbyReplicas); } - private void updateFollowerFetchState( - FollowerReplica followerReplica, - LogOffsetMetadata followerFetchOffsetMetadata, - long followerFetchTimeMs, - long leaderLogEndOffset) - throws IOException { - long prevFollowerEndOffset = followerReplica.stateSnapshot().getLogEndOffset(); - - // Apply read lock here to avoid the race between ISR updates and the fetch requests from - // rebooted follower. It could break the tablet server epoch checks in the ISR expansion. - inReadLock( - leaderIsrUpdateLock, - () -> - followerReplica.updateFetchState( - followerFetchOffsetMetadata, - followerFetchTimeMs, - leaderLogEndOffset)); - - // Check if this in-sync replica needs to be added to the ISR. - maybeExpandISr(followerReplica); - - // check if the HW of the replica can now be incremented since the replica may already be in - // the ISR and its LEO has just incremented - boolean leaderHWIncremented = false; - if (prevFollowerEndOffset != followerReplica.stateSnapshot().getLogEndOffset()) { - leaderHWIncremented = maybeIncrementLeaderHW(logTablet, followerFetchTimeMs); - } - - if (leaderHWIncremented) { - tryCompleteDelayedOperations(); - } - - LOG.debug( - "Recorded replica {} log end offset (LEO) position {} for bucket {}.", - localTabletServerId, - followerFetchOffsetMetadata.getMessageOffset(), - tableBucket); - } - private FollowerReplica getFollowerReplicaOrThrown(int followerId) { FollowerReplica followerReplica = followerReplicasMap.get(followerId); if (followerReplica == null) { @@ -2091,11 +2241,17 @@ private void validateBucketEpoch(int requestBucketEpoch) { * *

This function can be triggered when a replica's LEO has incremented. */ - private void maybeExpandISr(FollowerReplica followerReplica) { + private void maybeExpandISr(FollowerReplica followerReplica, int expectedEpoch) { boolean needsIsrUpdate = inReadLock( leaderIsrUpdateLock, - () -> !isrState.isInflight() && needsExpandIsr(followerReplica)); + () -> + isLeader() + && leaderEpoch == expectedEpoch + && followerReplicasMap.get(followerReplica.getFollowerId()) + == followerReplica + && !isrState.isInflight() + && needsExpandIsr(followerReplica)); if (needsIsrUpdate) { Optional adjustIsrUpdateOpt = @@ -2105,6 +2261,9 @@ private void maybeExpandISr(FollowerReplica followerReplica) { IsrState currentIsrState = isrState; // check if this replica needs to be added to the ISR. if (isLeader() + && leaderEpoch == expectedEpoch + && followerReplicasMap.get(followerReplica.getFollowerId()) + == followerReplica && currentIsrState instanceof IsrState.CommittedIsrState && needsExpandIsr(followerReplica)) { return Optional.of( diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java index c918147e4f7..e3a69d50101 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java @@ -1553,6 +1553,12 @@ private void makeFollowers( .map(Replica::getTableBucket) .collect(Collectors.toSet())); + for (Replica replica : replicasBecomeFollower) { + if (!replica.getLogTablet().isLeaderEpochEnabled()) { + replica.truncateTo(replica.getLogHighWatermark()); + } + } + replicasBecomeFollower.forEach( replica -> completeDelayedOperations(replica.getTableBucket())); @@ -1564,7 +1570,8 @@ private void makeFollowers( // TODO this logic need to be removed after we introduce leader epoch cache, and fetcher // manager support truncating while fetching. Trace by // https://github.com/apache/fluss/issues/673 - truncateToHighWatermark(replicasBecomeFollower); + // Preserve the tail until the fetch response identifies the common log history. + // A follower's committed watermark can lag behind the previous leader's watermark. // add fetcher for those follower replicas. addFetcherForReplicas(replicasBecomeFollower, result); @@ -1836,6 +1843,8 @@ public Map readFromLog( } } + fetchParams.setCurrentFetchEpoch( + fetchReqInfo.currentLeaderEpoch(), fetchReqInfo.lastFetchedEpoch()); LogReadInfo readInfo = replica.fetchRecords(fetchParams); // Once we read from a non-empty bucket, we stop ignoring request and bucket @@ -1848,15 +1857,16 @@ public Map readFromLog( limitBytes = Math.max(0, limitBytes - recordBatchSize); FetchLogResultForBucket fetchLogResult = FetchLogResultForBucket.records( - tb, - fetchedData.getRecords(), - readInfo.getHighWatermark(), - fetchedData.hasFilteredEndOffset() - ? fetchedData.getFilteredEndOffset() - : -1L, - readInfo.hasMinRetainOffset() - ? readInfo.getMinRetainOffset() - : -1L); + tb, + fetchedData.getRecords(), + readInfo.getHighWatermark(), + fetchedData.hasFilteredEndOffset() + ? fetchedData.getFilteredEndOffset() + : -1L, + readInfo.hasMinRetainOffset() + ? readInfo.getMinRetainOffset() + : -1L) + .withEpochInfo(readInfo.epochInfo()); logReadResult.put( tb, new LogReadResult(fetchLogResult, fetchedData.getFetchOffsetMetadata())); @@ -1882,6 +1892,19 @@ public Map readFromLog( FetchLogResultForBucket result; if (replica != null && e instanceof LogOffsetOutOfRangeException) { result = handleFetchOutOfRangeException(replica, fetchOffset, e); + if (isFromFollower + && fetchReqInfo.currentLeaderEpoch() >= 0 + && result.fetchFromRemote()) { + try { + result = + replica.withRemoteFetchEpoch( + result, fetchReqInfo.currentLeaderEpoch()); + } catch (Exception epochError) { + result = + FetchLogResultForBucket.error( + tb, ApiError.fromThrowable(epochError)); + } + } } else { result = FetchLogResultForBucket.error(tb, ApiError.fromThrowable(e)); } @@ -2145,6 +2168,11 @@ private void maybeAddDelayedFetchLog( } } + if (fetchLogResultForBucket.epochInfo() != null + && fetchLogResultForBucket.epochInfo().divergingEpoch() != null) { + errorReadingData = true; + break; + } if (!fetchLogResultForBucket.fetchFromRemote()) { hasFetchFromLocal = true; bytesReadable += fetchLogResultForBucket.recordsOrEmpty().sizeInBytes(); @@ -2422,19 +2450,6 @@ private void sweepOrphanTabletDirs( LOG.info("Swept orphan tablet directories for bucket {}", tb); } - private void truncateToHighWatermark(List replicas) { - for (Replica replica : replicas) { - long highWatermark = replica.getLogTablet().getHighWatermark(); - LOG.info( - "Truncating the logEndOffset for replica id {} of table bucket {} to local " - + "highWatermark {} as it becomes the follower", - serverId, - replica.getTableBucket(), - highWatermark); - replica.truncateTo(highWatermark); - } - } - private void validateAndApplyCoordinatorEpoch(int requestCoordinatorEpoch, String requestName) { if (requestCoordinatorEpoch < this.coordinatorEpoch) { String errorMessage = diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/FetchLogContext.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/FetchLogContext.java index 7a8db54ac8a..0f8bdfab35f 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/FetchLogContext.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/FetchLogContext.java @@ -17,22 +17,68 @@ package org.apache.fluss.server.replica.fetcher; +import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TablePath; import org.apache.fluss.rpc.messages.FetchLogRequest; +import org.apache.fluss.rpc.messages.PbFetchLogReqForBucket; +import org.apache.fluss.rpc.messages.PbFetchLogReqForTable; +import java.util.Collections; +import java.util.HashMap; import java.util.Map; /** FetchLogContext to fetch log from leader. */ public class FetchLogContext { private final Map tableIdToTablePath; private final FetchLogRequest fetchLogRequest; + private final Map fetchStates; + private final Map leaderEpochs = new HashMap<>(); + private final Map requests = new HashMap<>(); public FetchLogContext( Map tableIdToTablePath, FetchLogRequest fetchLogRequest) { + this(tableIdToTablePath, fetchLogRequest, Collections.emptyMap()); + } + + public FetchLogContext( + Map tableIdToTablePath, + FetchLogRequest fetchLogRequest, + Map fetchStates) { + this.fetchStates = new HashMap<>(fetchStates); + for (PbFetchLogReqForTable table : fetchLogRequest.getTablesReqsList()) { + for (PbFetchLogReqForBucket bucket : table.getBucketsReqsList()) { + requests.put( + new TableBucket( + table.getTableId(), + bucket.hasPartitionId() ? bucket.getPartitionId() : null, + bucket.getBucketId()), + bucket); + } + } this.tableIdToTablePath = tableIdToTablePath; this.fetchLogRequest = fetchLogRequest; } + public FetchLogContext withFetchStates(Map states) { + return new FetchLogContext(tableIdToTablePath, fetchLogRequest, states); + } + + public void setLeaderEpoch(TableBucket bucket, int epoch) { + leaderEpochs.put(bucket, epoch); + } + + public int leaderEpoch(TableBucket bucket) { + return leaderEpochs.get(bucket); + } + + public boolean matches(TableBucket bucket, BucketFetchStatus state) { + return fetchStates.get(bucket) == state; + } + + public PbFetchLogReqForBucket getRequest(TableBucket bucket) { + return requests.get(bucket); + } + public FetchLogRequest getFetchLogRequest() { return fetchLogRequest; } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThread.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThread.java index 45c22ff2bda..47b34ecd6c7 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThread.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThread.java @@ -22,18 +22,20 @@ import org.apache.fluss.exception.InvalidOffsetException; import org.apache.fluss.exception.InvalidRecordException; import org.apache.fluss.exception.OutOfOrderSequenceException; -import org.apache.fluss.exception.RemoteStorageException; import org.apache.fluss.exception.StorageException; +import org.apache.fluss.metadata.LeaderEpochOffset; import org.apache.fluss.metadata.TableBucket; import org.apache.fluss.metadata.TablePath; import org.apache.fluss.record.MemoryLogRecords; import org.apache.fluss.remote.RemoteLogFetchInfo; import org.apache.fluss.remote.RemoteLogSegment; +import org.apache.fluss.rpc.entity.FetchLogEpochInfo; import org.apache.fluss.rpc.entity.FetchLogResultForBucket; import org.apache.fluss.rpc.messages.FetchLogRequest; +import org.apache.fluss.rpc.messages.PbFetchLogReqForBucket; +import org.apache.fluss.rpc.messages.PbFetchLogReqForTable; import org.apache.fluss.server.log.LogAppendInfo; import org.apache.fluss.server.log.LogTablet; -import org.apache.fluss.server.log.remote.RemoteLogManager; import org.apache.fluss.server.log.remote.RemoteLogStorage.IndexType; import org.apache.fluss.server.metrics.group.TabletServerMetricGroup; import org.apache.fluss.server.replica.Replica; @@ -41,8 +43,6 @@ import org.apache.fluss.server.replica.fetcher.LeaderEndpoint.FetchData; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.shaded.netty4.io.netty.util.ReferenceCountUtil; -import org.apache.fluss.utils.FileUtils; -import org.apache.fluss.utils.FlussPaths; import org.apache.fluss.utils.concurrent.ShutdownableThread; import org.apache.fluss.utils.log.FairBucketStatusMap; @@ -53,11 +53,11 @@ import javax.annotation.concurrent.GuardedBy; import java.io.File; -import java.io.IOException; import java.nio.file.Files; import java.nio.file.StandardCopyOption; import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -149,6 +149,49 @@ private void maybeFetch() { fetchLogContext = leader.buildFetchLogContext( fairBucketStatusMap.bucketStatusMap()); + if (fetchLogContext.isPresent()) { + FetchLogContext context = + fetchLogContext + .get() + .withFetchStates( + fairBucketStatusMap.bucketStatusMap()); + for (PbFetchLogReqForTable table : + context.getFetchLogRequest().getTablesReqsList()) { + for (PbFetchLogReqForBucket bucket : + table.getBucketsReqsList()) { + TableBucket tb = + new TableBucket( + table.getTableId(), + bucket.hasPartitionId() + ? bucket.getPartitionId() + : null, + bucket.getBucketId()); + try { + Replica replica = + replicaManager.getReplicaOrException(tb); + int epoch = replica.getLeaderEpoch(); + context.setLeaderEpoch(tb, epoch); + if (replica.getLogTablet().isLeaderEpochEnabled()) { + bucket.setCurrentLeaderEpoch(epoch) + .setLastFetchedEpoch( + replica.getLogTablet() + .lastFetchedEpoch( + bucket + .getFetchOffset())); + } + + } catch (Exception e) { + LOG.error( + "Cannot read replication history for {}", + tb, + e); + removeBucket(tb); + return Optional.empty(); + } + } + } + fetchLogContext = Optional.of(context); + } if (!fetchLogContext.isPresent()) { LOG.trace( "There are no active buckets. Back off for {} ms before " @@ -265,7 +308,8 @@ private void processFetchLogRequest(FetchLogContext fetchLogContext) { if (responseData != null) { bucketStatusMapLock.lock(); try { - handleFetchLogResponse(responseData.getFetchLogResultMap(), bucketsWithError); + handleFetchLogResponse( + fetchLogContext, responseData.getFetchLogResultMap(), bucketsWithError); } finally { // release buffer handle by fetchLogResponse. releaseFetchDataBuffer(responseData); @@ -289,13 +333,16 @@ private static void releaseFetchDataBuffer(FetchData data) { } private void handleFetchLogResponse( + FetchLogContext context, Map responseData, Set replicasWithError) { responseData.forEach( (tableBucket, replicaData) -> { BucketFetchStatus currentFetchStatus = fairBucketStatusMap.statusValue(tableBucket); - if (currentFetchStatus == null || !currentFetchStatus.isReadyForFetch()) { + if (currentFetchStatus == null + || !currentFetchStatus.isReadyForFetch() + || !context.matches(tableBucket, currentFetchStatus)) { return; } @@ -303,10 +350,17 @@ private void handleFetchLogResponse( switch (replicaData.getError().error()) { case NONE: handleFetchLogResponseOfSuccessBucket( - tableBucket, currentFetchStatus, replicaData); + tableBucket, + currentFetchStatus, + replicaData, + context.getRequest(tableBucket), + context.leaderEpoch(tableBucket)); break; case LOG_OFFSET_OUT_OF_RANGE_EXCEPTION: - if (!handleOutOfRangeError(tableBucket, currentFetchStatus)) { + if (!handleOutOfRangeError( + tableBucket, + currentFetchStatus, + context.leaderEpoch(tableBucket))) { replicasWithError.add(tableBucket); } break; @@ -330,28 +384,61 @@ private void handleFetchLogResponse( private void handleFetchLogResponseOfSuccessBucket( TableBucket tableBucket, BucketFetchStatus currentFetchStatus, - FetchLogResultForBucket replicaData) { + FetchLogResultForBucket replicaData, + PbFetchLogReqForBucket request, + int expectedEpoch) { try { long nextFetchOffset = -1L; + FetchLogEpochInfo epochInfo = replicaData.epochInfo(); + Replica currentReplica = replicaManager.getReplicaOrException(tableBucket); + if (epochInfo != null && !currentReplica.getLogTablet().isLeaderEpochEnabled()) { + // A response prepared before disabling must not apply an epoch-based truncation. + return; + } + if (currentReplica.isLeader() + || currentReplica.getLeaderEpoch() != expectedEpoch + || (epochInfo != null && epochInfo.leaderEpoch() != expectedEpoch)) { + return; + } + if (epochInfo != null && epochInfo.divergingEpoch() != null) { + LeaderEpochOffset divergence = epochInfo.divergingEpoch(); + Optional localEnd = + currentReplica.getLogTablet().endOffsetForEpoch(divergence.epoch()); + if (!localEnd.isPresent()) { + currentReplica.invalidateFollowerEpochHistory( + leader.leaderServerId(), expectedEpoch); + return; + } + long truncateOffset = Math.min(divergence.offset(), localEnd.get().offset()); + if (!currentReplica.truncateFollowerToEpochOffset( + truncateOffset, leader.leaderServerId(), expectedEpoch)) { + return; + } + fairBucketStatusMap.updateAndMoveToEnd( + tableBucket, + new BucketFetchStatus( + currentFetchStatus.tableId(), + currentFetchStatus.tablePath(), + truncateOffset, + null)); + return; + } if (replicaData.fetchFromRemote()) { - nextFetchOffset = processFetchResultFromRemoteStorage(tableBucket, replicaData); + nextFetchOffset = + processFetchResultFromRemoteStorage( + tableBucket, replicaData, expectedEpoch); } else { LogAppendInfo logAppendInfo = processFetchResultFromLocalStorage( - tableBucket, currentFetchStatus.fetchOffset(), replicaData); + tableBucket, + currentFetchStatus.fetchOffset(), + replicaData, + expectedEpoch); if (logAppendInfo.validBytes() > 0) { nextFetchOffset = logAppendInfo.lastOffset() + 1; } } - // Propagate the leader's KV snapshot retention boundary to the follower even when the - // successful response contains no records, so obsolete local log segments can be - // cleaned up. - Replica replica = replicaManager.getReplicaOrException(tableBucket); - if (replica.isKvTable() && replicaData.hasMinRetainOffset()) { - replica.getLogTablet().updateMinRetainOffset(replicaData.getMinRetainOffset()); - } - if (nextFetchOffset != -1L && fairBucketStatusMap.contains(tableBucket)) { BucketFetchStatus newFetchStatus = new BucketFetchStatus( @@ -381,8 +468,17 @@ private void handleFetchLogResponseOfSuccessBucket( } else if (e instanceof DuplicateSequenceException || e instanceof OutOfOrderSequenceException || e instanceof InvalidOffsetException) { - // TODO this part of logic need to be removed after we introduce leader epoch cache. - // Trace by https://github.com/apache/fluss/issues/673 + if (replicaData.epochInfo() != null + && request.hasLastFetchedEpoch() + && request.getLastFetchedEpoch() >= 0) { + LOG.error( + "Replication offset or sequence mismatch after epoch validation for {}", + tableBucket, + e); + removeBucket(tableBucket); + return; + } + // Legacy peers and unknown prefixes still use the existing recovery path. LOG.error( "Founding recoverable error while processing data for bucket {} at offset {}, try to " + "truncate to LeaderEndOffsetSnapshot", @@ -390,7 +486,8 @@ private void handleFetchLogResponseOfSuccessBucket( currentFetchStatus.fetchOffset(), e); try { - truncateToLeaderEndOffsetSnapshot(tableBucket, currentFetchStatus.tablePath()); + truncateToLeaderEndOffsetSnapshot( + tableBucket, currentFetchStatus.tablePath(), expectedEpoch); } catch (Exception ex) { LOG.error( "Error while truncating bucket {} at offset {}", @@ -410,8 +507,8 @@ private void handleFetchLogResponseOfSuccessBucket( } } - private void truncateToLeaderEndOffsetSnapshot(TableBucket tableBucket, TablePath tablePath) - throws Exception { + private void truncateToLeaderEndOffsetSnapshot( + TableBucket tableBucket, TablePath tablePath, int expectedEpoch) throws Exception { long leaderLocalEndOffsetWhileBecomeLeader = leader.fetchLeaderEndOffsetSnapshot(tableBucket).get(); long localLogEndOffset = @@ -419,7 +516,7 @@ private void truncateToLeaderEndOffsetSnapshot(TableBucket tableBucket, TablePat if (leaderLocalEndOffsetWhileBecomeLeader != 0L && leaderLocalEndOffsetWhileBecomeLeader < localLogEndOffset) { // truncate to leaderEndOffsetSnapshot to reset follower's WriterState and fetch offset. - truncate(tableBucket, leaderLocalEndOffsetWhileBecomeLeader); + truncate(tableBucket, leaderLocalEndOffsetWhileBecomeLeader, expectedEpoch); // update fetch status. BucketFetchStatus bucketFetchStatus = @@ -432,9 +529,10 @@ private void truncateToLeaderEndOffsetSnapshot(TableBucket tableBucket, TablePat } } - private boolean handleOutOfRangeError(TableBucket tableBucket, BucketFetchStatus fetchStatus) { + private boolean handleOutOfRangeError( + TableBucket tableBucket, BucketFetchStatus fetchStatus, int expectedEpoch) { try { - BucketFetchStatus newFetchStatus = fetchOffsetAndTruncate(tableBucket); + BucketFetchStatus newFetchStatus = fetchOffsetAndTruncate(tableBucket, expectedEpoch); fairBucketStatusMap.updateAndMoveToEnd(tableBucket, newFetchStatus); LOG.info( "Current offset {} for table bucket {} is out of range, which typically implies " @@ -450,7 +548,8 @@ private boolean handleOutOfRangeError(TableBucket tableBucket, BucketFetchStatus } /** Handle a replica whose offset is out of range and return a new fetch offset. */ - private BucketFetchStatus fetchOffsetAndTruncate(TableBucket tableBucket) throws Exception { + private BucketFetchStatus fetchOffsetAndTruncate(TableBucket tableBucket, int expectedEpoch) + throws Exception { Replica replica = replicaManager.getReplicaOrException(tableBucket); long replicaEndOffset = replica.getLocalLogEndOffset(); @@ -476,7 +575,7 @@ private BucketFetchStatus fetchOffsetAndTruncate(TableBucket tableBucket) throws tableBucket, replicaEndOffset, leaderEndOffset); - truncate(tableBucket, leaderEndOffset); + truncate(tableBucket, leaderEndOffset, expectedEpoch); return new BucketFetchStatus( tableBucket.getTableId(), replica.getTablePath(), leaderEndOffset, null); } else { @@ -514,7 +613,8 @@ private BucketFetchStatus fetchOffsetAndTruncate(TableBucket tableBucket) throws // Only truncate log when current leader's log start offset is greater than follower's // log end offset. if (leaderStartOffset > replicaEndOffset) { - truncateFullyAndStartAt(tableBucket, leaderStartOffset); + replica.truncateFollowerFullyAndStartAt( + leaderStartOffset, leader.leaderServerId(), expectedEpoch); } long offsetToFetch = Math.max(leaderStartOffset, replicaEndOffset); @@ -560,7 +660,10 @@ Optional fetchStatus(TableBucket tableBucket) { } private LogAppendInfo processFetchResultFromLocalStorage( - TableBucket tableBucket, long fetchOffset, FetchLogResultForBucket replicaData) + TableBucket tableBucket, + long fetchOffset, + FetchLogResultForBucket replicaData, + int expectedEpoch) throws Exception { Replica replica = replicaManager.getReplicaOrException(tableBucket); LogTablet logTablet = replica.getLogTablet(); @@ -581,103 +684,62 @@ private LogAppendInfo processFetchResultFromLocalStorage( replicaData.getHighWatermark()); // Append the messages to the follower log tablet. - LogAppendInfo logAppendInfo = replica.appendRecordsToFollower(records); + LogAppendInfo logAppendInfo = + replica.appendRecordsToFollower( + replicaData, leader.leaderServerId(), expectedEpoch); LOG.trace( "Follower has replica log end offset {} after appending {} bytes of messages for replica {}", logTablet.localLogEndOffset(), records.sizeInBytes(), tableBucket); - // For the follower replica, we do not need to keep its segment base offset and physical - // position. These values will be computed upon becoming leader or handling a preferred read - // replica fetch. - // TODO, to avoid lose data in case of leader change, we now change to update highWatermark - // first for follower instead of first for leader. The reason why can see - // https://cwiki.apache.org/confluence/display/KAFKA/KIP-101+-+Alter+Replication+Protocol+to+use+Leader+Epoch+rather+than+High+Watermark+for+Truncation - // for more details. However, this is just a temporary solution, if we want to have a strong - // consistency guarantee, we should do as KIP-101 do, trace by: - // https://github.com/apache/fluss/issues/673 - logTablet.updateHighWatermark(logTablet.localLogEndOffset()); - LOG.trace( - "Follower received high watermark {} from the leader for replica {}", - replicaData.getHighWatermark(), - tableBucket); - serverMetricGroup.replicationBytesIn().inc(records.sizeInBytes()); return logAppendInfo; } private long processFetchResultFromRemoteStorage( - TableBucket tb, FetchLogResultForBucket replicaData) { - RemoteLogFetchInfo rlFetchInfo = replicaData.remoteLogFetchInfo(); - checkNotNull(rlFetchInfo, "RemoteLogFetchInfo is null"); + TableBucket tb, FetchLogResultForBucket replicaData, int expectedEpoch) + throws Exception { + RemoteLogFetchInfo info = + checkNotNull(replicaData.remoteLogFetchInfo(), "RemoteLogFetchInfo is null"); Replica replica = replicaManager.getReplicaOrException(tb); - RemoteLogManager rlm = replicaManager.getRemoteLogManager(); - - // TODO after introduce leader epoch cache, we need to rebuild the local leader epoch - // cache. Trace by https://github.com/apache/fluss/issues/673 - - // update next fetch offset and writer id snapshot in local. - RemoteLogSegment remoteLogSegmentWithMaxStartOffset = - rlFetchInfo - .remoteLogSegmentList() - .get(rlFetchInfo.remoteLogSegmentList().size() - 1); - // build writer snapshots until remoteLogSegment.endOffset() and start segment from - // until remoteLogSegment.endOffset(). - long nextFetchOffset = remoteLogSegmentWithMaxStartOffset.remoteLogEndOffset(); - + RemoteLogSegment segment = + info.remoteLogSegmentList().get(info.remoteLogSegmentList().size() - 1); + long nextOffset = segment.remoteLogEndOffset(); + File download = + Files.createTempFile( + replica.getLogTablet().getLogDir().toPath(), + "writer-snapshot-", + ".tmp") + .toFile(); try { - // Truncate the existing local log before restoring the writer id snapshots. - replica.truncateFullyAndStartAt(nextFetchOffset); - - // TODO maybe need increase log start offset. - - LogTablet log = replica.getLogTablet(); - // 1. Perform a truncate before calling buildWriterIdSnapshotFile() to ensure that all - // historical data is completely cleaned up. - log.writerStateManager().truncateFullyAndStartAt(0L); - - // 2. download writer id snapshots from remote storage. - File snapshotFile = FlussPaths.writerSnapshotFile(log.getLogDir(), nextFetchOffset); - buildWriterIdSnapshotFile(snapshotFile, remoteLogSegmentWithMaxStartOffset, rlm); - - // 3. Perform a reloadSnapshots after buildWriterIdSnapshotFile() to load the latest - // downloaded writerId snapshot file into the writerStateManager. - // Note: This must occur after the file is downloaded, so we cannot call - // truncateFullyAndReloadSnapshots() here to avoid deleting the newly downloaded - // writerId snapshot file. - log.writerStateManager().reloadSnapshots(); - log.loadWriterSnapshot(nextFetchOffset); - LOG.info( - "Build the writer snapshots from remote storage for {} with active " - + "writer size: {} and remoteLogEndOffset: {}", - tb, - log.writerStateManager().activeWriters().size(), - nextFetchOffset); - } catch (Exception e) { - LOG.error( - "Failed to truncate and restore writer snapshot for {} while log hash been moved to remote", - tb, - e); + // Download before replacing local state. A network failure must leave the fetch + // position and the existing WAL intact. + try (java.io.InputStream input = + replicaManager + .getRemoteLogManager() + .getRemoteLogStorage() + .fetchIndex(segment, IndexType.WRITER_ID_SNAPSHOT)) { + Files.copy(input, download.toPath(), StandardCopyOption.REPLACE_EXISTING); + } + FetchLogEpochInfo epochInfo = replicaData.epochInfo(); + List epochs = + epochInfo == null ? Collections.emptyList() : segment.leaderEpochs(); + replica.restoreFollowerFromRemote( + nextOffset, + download, + epochs, + epochInfo == null ? nextOffset : replicaData.getHighWatermark(), + leader.leaderServerId(), + expectedEpoch); + return nextOffset; + } finally { + Files.deleteIfExists(download.toPath()); } - return nextFetchOffset; } - private void buildWriterIdSnapshotFile( - File snapshotFile, RemoteLogSegment remoteLogSegment, RemoteLogManager rlm) - throws RemoteStorageException, IOException { - File tmpSnapshotFile = new File(snapshotFile.getAbsolutePath() + ".tmp"); - // Copy it to snapshot file in atomic manner. - Files.copy( - rlm.getRemoteLogStorage() - .fetchIndex(remoteLogSegment, IndexType.WRITER_ID_SNAPSHOT), - tmpSnapshotFile.toPath(), - StandardCopyOption.REPLACE_EXISTING); - FileUtils.atomicMoveWithFallback(tmpSnapshotFile.toPath(), snapshotFile.toPath(), false); - } - - private void truncate(TableBucket tableBucket, long offset) { + private void truncate(TableBucket tableBucket, long offset, int expectedEpoch) { Replica replica = replicaManager.getReplicaOrException(tableBucket); LogTablet log = replica.getLogTablet(); @@ -689,12 +751,7 @@ private void truncate(TableBucket tableBucket, long offset) { log.getHighWatermark()); } - replica.truncateTo(offset); - } - - private void truncateFullyAndStartAt(TableBucket tableBucket, long offset) { - Replica replica = replicaManager.getReplicaOrException(tableBucket); - replica.truncateFullyAndStartAt(offset); + replica.truncateFollowerTo(offset, leader.leaderServerId(), expectedEpoch); } @Override diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java index 242f92a0ea4..0b579f75bdc 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java @@ -36,6 +36,7 @@ import org.apache.fluss.metadata.AggFunctions; import org.apache.fluss.metadata.DatabaseChange; import org.apache.fluss.metadata.DatabaseSummary; +import org.apache.fluss.metadata.LeaderEpochOffset; import org.apache.fluss.metadata.PartitionSpec; import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.metadata.ResolvedPartitionSpec; @@ -54,6 +55,7 @@ import org.apache.fluss.record.MemoryLogRecords; import org.apache.fluss.remote.RemoteLogFetchInfo; import org.apache.fluss.remote.RemoteLogSegment; +import org.apache.fluss.rpc.entity.FetchLogEpochInfo; import org.apache.fluss.rpc.entity.FetchLogResultForBucket; import org.apache.fluss.rpc.entity.LimitScanResultForBucket; import org.apache.fluss.rpc.entity.ListOffsetsResultForBucket; @@ -1062,7 +1064,14 @@ public static Map getFetchLogData(FetchLogRequest req tableId, fetchLogReqForBucket.getFetchOffset(), fetchLogReqForBucket.getMaxFetchBytes(), - projectionFields)); + projectionFields, + fetchLogReqForBucket.hasCurrentLeaderEpoch() + && fetchLogReqForBucket.hasLastFetchedEpoch() + ? fetchLogReqForBucket.getCurrentLeaderEpoch() + : -1, + fetchLogReqForBucket.hasLastFetchedEpoch() + ? fetchLogReqForBucket.getLastFetchedEpoch() + : -1)); } } @@ -1083,6 +1092,22 @@ public static FetchLogResponse makeFetchLogResponse( FetchLogResultForBucket bucketResult = entry.getValue(); PbFetchLogRespForBucket fetchLogRespForBucket = new PbFetchLogRespForBucket().setBucketId(tb.getBucket()); + FetchLogEpochInfo epochInfo = bucketResult.epochInfo(); + if (epochInfo != null) { + fetchLogRespForBucket.setCurrentLeaderEpoch(epochInfo.leaderEpoch()); + if (epochInfo.divergingEpoch() != null) { + fetchLogRespForBucket + .setDivergingEpoch() + .setEpoch(epochInfo.divergingEpoch().epoch()) + .setOffset(epochInfo.divergingEpoch().offset()); + } + for (LeaderEpochOffset start : epochInfo.epochStarts()) { + fetchLogRespForBucket + .addEpochStart() + .setEpoch(start.epoch()) + .setOffset(start.offset()); + } + } if (bucketResult.hasFilteredEndOffset()) { fetchLogRespForBucket.setFilteredEndOffset(bucketResult.getFilteredEndOffset()); } @@ -1115,6 +1140,12 @@ public static FetchLogResponse makeFetchLogResponse( .setRemoteLogEndOffset(logSegment.remoteLogEndOffset()) .setSegmentSizeInBytes(logSegment.segmentSizeInBytes()) .setMaxTimestamp(logSegment.maxTimestamp()); + for (LeaderEpochOffset epoch : logSegment.leaderEpochs()) { + pbRemoteLogSegment + .addLeaderEpoch() + .setEpoch(epoch.epoch()) + .setOffset(epoch.offset()); + } remoteLogSegmentList.add(pbRemoteLogSegment); } fetchLogRespForBucket diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/LeaderEpochHistoryTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/LeaderEpochHistoryTest.java new file mode 100644 index 00000000000..86fcaaa3661 --- /dev/null +++ b/fluss-server/src/test/java/org/apache/fluss/server/log/LeaderEpochHistoryTest.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.log; + +import org.apache.fluss.exception.LogStorageException; +import org.apache.fluss.metadata.LeaderEpochOffset; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.Collections; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests persistent log history used to find a common prefix after leader changes. */ +class LeaderEpochHistoryTest { + @TempDir private File directory; + + @Test + void testUnknownPrefixAndSkippedEpochsSurviveRestart() throws Exception { + File file = new File(directory, "leader-epoch-checkpoint"); + LeaderEpochHistory history = new LeaderEpochHistory(file); + history.assign(41, 100); + history.assign(45, 160); + history = new LeaderEpochHistory(file); + + assertThat(history.epochForOffset(99, 200)).isEqualTo(-1); + assertThat(history.epochForOffset(159, 200)).isEqualTo(41); + assertThat(history.epochForOffset(160, 200)).isEqualTo(45); + assertThat(history.endOffsetFor(40, 200)).isEmpty(); + LeaderEpochOffset preceding = history.endOffsetFor(44, 200).get(); + assertThat(preceding.epoch()).isEqualTo(41); + assertThat(preceding.offset()).isEqualTo(160); + assertThat(history.endOffsetFor(45, 200).get().offset()).isEqualTo(200); + } + + @Test + void testEmptyEpochDoesNotChangeLastRecordEpoch() throws Exception { + LeaderEpochHistory history = new LeaderEpochHistory(new File(directory, "epochs")); + history.assign(1, 0); + history.assign(2, 10); + history.assign(3, 10); + assertThat(history.epochForOffset(9, 10)).isEqualTo(1); + assertThat(history.epochForOffset(10, 10)).isEqualTo(-1); + assertThat(history.endOffsetFor(2, 10).get().offset()).isEqualTo(10); + assertThat(history.epochForOffset(10, 11)).isEqualTo(3); + } + + @Test + void testTruncateAndReplaceTailSurvivesRestart() throws Exception { + File file = new File(directory, "epochs"); + LeaderEpochHistory history = new LeaderEpochHistory(file); + history.assign(1, 0); + history.assign(2, 10); + history.assign(3, 20); + history.truncateFromEnd(10); + history.assign(4, 10); + history = new LeaderEpochHistory(file); + assertThat(history.epochForOffset(9, 15)).isEqualTo(1); + assertThat(history.epochForOffset(10, 15)).isEqualTo(4); + assertThat(history.endOffsetFor(3, 15).get().epoch()).isEqualTo(1); + assertThat(history.endOffsetFor(3, 15).get().offset()).isEqualTo(10); + } + + @Test + void testRetentionPreservesEpochCoveringFirstRecord() throws Exception { + File file = new File(directory, "epochs"); + LeaderEpochHistory history = new LeaderEpochHistory(file); + history.assign(1, 0); + history.assign(2, 10); + history.assign(3, 20); + history.truncateFromStart(15); + history = new LeaderEpochHistory(file); + assertThat(history.endOffsetFor(1, 30)).isEmpty(); + assertThat(history.epochForOffset(15, 30)).isEqualTo(2); + assertThat(history.endOffsetFor(2, 30).get().offset()).isEqualTo(20); + } + + @Test + void testUntrackedAppendInvalidatesHistoryAcrossRestart() throws Exception { + File file = new File(directory, "epochs"); + LeaderEpochHistory history = new LeaderEpochHistory(file); + history.assign(41, 0); + assertThat(history.epochForOffset(9, 10)).isEqualTo(41); + history.invalidate(); + history = new LeaderEpochHistory(file); + assertThat(history.epochForOffset(19, 20)).isEqualTo(-1); + assertThat(history.endOffsetFor(41, 20)).isEmpty(); + history.assign(45, 20); + assertThat(history.epochForOffset(19, 25)).isEqualTo(-1); + assertThat(history.epochForOffset(20, 25)).isEqualTo(45); + } + + @Test + void testFetchedBoundaryDoesNotIdentifyUncopiedPrefix() throws Exception { + LeaderEpochHistory history = new LeaderEpochHistory(new File(directory, "epochs")); + history.append(Collections.singletonList(new LeaderEpochOffset(41, 0)), 10, 10); + assertThat(history.epochForOffset(9, 10)).isEqualTo(-1); + history.append( + Arrays.asList(new LeaderEpochOffset(41, 0), new LeaderEpochOffset(45, 15)), 10, 20); + assertThat(history.epochForOffset(9, 20)).isEqualTo(-1); + assertThat(history.epochForOffset(14, 20)).isEqualTo(-1); + assertThat(history.epochForOffset(15, 20)).isEqualTo(45); + assertThat(history.entries(20, 20)).isEmpty(); + } + + @Test + void testFailedCheckpointStopsHistoryUntilReload() throws Exception { + File file = new File(directory, "epochs"); + LeaderEpochHistory history = new LeaderEpochHistory(file); + history.assign(1, 0); + Files.createDirectory(new File(directory, "epochs.tmp").toPath()); + assertThatThrownBy(() -> history.assign(2, 10)).isInstanceOf(IOException.class); + assertThatThrownBy(() -> history.epochForOffset(10, 15)) + .isInstanceOf(LogStorageException.class); + assertThatThrownBy(() -> history.assign(1, 0)).isInstanceOf(LogStorageException.class); + assertThat(new LeaderEpochHistory(file).epochForOffset(10, 15)).isEqualTo(1); + } +} diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/LogTabletTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/LogTabletTest.java index b133eeccad2..a74bb0e752e 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/log/LogTabletTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/log/LogTabletTest.java @@ -21,6 +21,7 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.config.MemorySize; import org.apache.fluss.exception.OutOfOrderSequenceException; +import org.apache.fluss.metadata.LeaderEpochOffset; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.PhysicalTablePath; import org.apache.fluss.record.LogRecord; @@ -42,6 +43,7 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Files; import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; @@ -114,6 +116,120 @@ public void teardown() throws Exception { scheduler.shutdown(); } + @Test + void testDisablingEpochHistorySurvivesReenableWithExistingData() throws Exception { + logTablet.assignLeaderEpoch(3); + logTablet.appendAsLeader(genMemoryLogRecordsWithWriterId(DATA1, 1L, 0, 0L)); + Configuration disabled = new Configuration(conf); + disabled.set(ConfigOptions.LOG_REPLICATION_LEADER_EPOCH_ENABLED, false); + reopenLogTablet(disabled); + assertThat(logTablet.lastFetchedEpoch(10)).isEqualTo(-1); + logTablet.assignLeaderEpoch(4); + logTablet.appendAsLeader(genMemoryLogRecordsWithWriterId(DATA1, 1L, 1, 0L)); + assertThat(logTablet.lastFetchedEpoch(20)).isEqualTo(-1); + reopenLogTablet(conf); + assertThat(logTablet.lastFetchedEpoch(20)).isEqualTo(-1); + logTablet.assignLeaderEpoch(5); + logTablet.appendAsLeader(genMemoryLogRecordsWithWriterId(DATA1, 1L, 2, 0L)); + assertThat(logTablet.lastFetchedEpoch(20)).isEqualTo(-1); + assertThat(logTablet.lastFetchedEpoch(30)).isEqualTo(5); + } + + private void reopenLogTablet(Configuration configuration) throws Exception { + logTablet.close(); + logTablet = + LogTablet.create( + tempDir, + PhysicalTablePath.of(DATA1_TABLE_PATH), + logDir, + configuration, + new AtomicBoolean( + configuration.get( + ConfigOptions.LOG_RETENTION_ROLL_ACTIVE_SEGMENT_ENABLED)), + TestingMetricGroups.TABLET_SERVER_METRICS, + 0, + scheduler, + LogFormat.ARROW, + 1, + false, + SystemClock.getInstance(), + true); + } + + @Test + void testUnverifiedSourceDoesNotExtendKnownLocalEpoch() throws Exception { + logTablet.assignLeaderEpoch(3); + logTablet.appendAsLeader(genMemoryLogRecordsWithWriterId(DATA1, 1L, 0, 0L)); + logTablet.appendAsFollower( + genMemoryLogRecordsWithWriterId(DATA1, 2L, 0, 10L), + Collections.singletonList(new LeaderEpochOffset(5, 5))); + assertThat(logTablet.lastFetchedEpoch(20)).isEqualTo(-1); + assertThat(logTablet.localLogEndOffset()).isEqualTo(20); + } + + @Test + void testDisabledRemoteRestoreDoesNotImportEpochHistory() throws Exception { + logTablet.assignLeaderEpoch(3); + logTablet.appendAsLeader(genMemoryLogRecordsWithWriterId(DATA1, 1L, 0, 0L)); + takeWriterSnapshot(logTablet); + File downloaded = new File(tempDir, "remote-writer-snapshot"); + Files.copy(writerSnapshotFile(logDir, 10).toPath(), downloaded.toPath()); + Configuration disabled = new Configuration(conf); + disabled.set(ConfigOptions.LOG_REPLICATION_LEADER_EPOCH_ENABLED, false); + reopenLogTablet(disabled); + logTablet.restoreFromRemote( + 10, downloaded, Collections.singletonList(new LeaderEpochOffset(3, 0))); + assertThat(logTablet.lastFetchedEpoch(10)).isEqualTo(-1); + reopenLogTablet(conf); + assertThat(logTablet.lastFetchedEpoch(10)).isEqualTo(-1); + logTablet.assignLeaderEpoch(4); + logTablet.appendAsLeader(genMemoryLogRecordsWithWriterId(DATA1, 1L, 1, 0L)); + assertThat(logTablet.localLogEndOffset()).isEqualTo(20); + assertThat(logTablet.lastFetchedEpoch(20)).isEqualTo(4); + } + + @Test + void testRemoteRestoreInstallsWriterStateAndEpochHistory() throws Exception { + logTablet.assignLeaderEpoch(3); + logTablet.appendAsLeader(genMemoryLogRecordsWithWriterId(DATA1, 1L, 0, 0L)); + takeWriterSnapshot(logTablet); + File downloaded = new File(tempDir, "downloaded-writer-snapshot"); + Files.copy(writerSnapshotFile(logDir, 10).toPath(), downloaded.toPath()); + logTablet.assignLeaderEpoch(5); + logTablet.appendAsLeader(genMemoryLogRecordsWithWriterId(DATA1, 2L, 0, 0L)); + + logTablet.restoreFromRemote( + 10, downloaded, Collections.singletonList(new LeaderEpochOffset(3, 0))); + assertThat(logTablet.localLogEndOffset()).isEqualTo(10); + assertThat(latestWriterStateEndOffset(logTablet)).isEqualTo(10); + assertThat(logTablet.lastFetchedEpoch(10)).isEqualTo(3); + assertThat(logTablet.endOffsetForEpoch(5)).contains(new LeaderEpochOffset(3, 10)); + logTablet.assignLeaderEpoch(6); + logTablet.appendAsLeader(genMemoryLogRecordsWithWriterId(DATA1, 1L, 1, 0L)); + assertThat(logTablet.localLogEndOffset()).isEqualTo(20); + assertThat(logTablet.lastFetchedEpoch(20)).isEqualTo(6); + logTablet.close(); + logTablet = + LogTablet.create( + tempDir, + PhysicalTablePath.of(DATA1_TABLE_PATH), + logDir, + conf, + new AtomicBoolean( + conf.get(ConfigOptions.LOG_RETENTION_ROLL_ACTIVE_SEGMENT_ENABLED)), + TestingMetricGroups.TABLET_SERVER_METRICS, + 0, + scheduler, + LogFormat.ARROW, + 1, + false, + SystemClock.getInstance(), + true); + assertThat(logTablet.lastFetchedEpoch(20)).isEqualTo(6); + logTablet.appendAsLeader(genMemoryLogRecordsWithWriterId(DATA1, 1L, 2, 0L)); + assertThat(logTablet.localLogEndOffset()).isEqualTo(30); + } + @Test void testRemoteLogOffsetsCanResetAfterEmptyManifest() { logTablet.updateRemoteLogOffsets(0L, 10L, 10L); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogITCase.java b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogITCase.java index 5ac61b0bb92..f8535bcdf15 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogITCase.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogITCase.java @@ -222,6 +222,22 @@ void testFollowerFetchAlreadyMoveToRemoteLog(boolean withWriterId) throws Except // restart follower FLUSS_CLUSTER_EXTENSION.startTabletServer(follower); FLUSS_CLUSTER_EXTENSION.waitUntilReplicaExpandToIsr(tb, follower); + assertThat( + FLUSS_CLUSTER_EXTENSION + .waitAndGetFollowerReplica(tb, follower) + .getLogTablet() + .lastFetchedEpoch(100)) + .isEqualTo( + FLUSS_CLUSTER_EXTENSION + .waitAndGetLeaderReplica(tb) + .getLogTablet() + .lastFetchedEpoch(100)); + assertThat( + FLUSS_CLUSTER_EXTENSION + .waitAndGetFollowerReplica(tb, follower) + .getLogTablet() + .lastFetchedEpoch(100)) + .isGreaterThanOrEqualTo(0); } @Test diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/LeaderEpochCompatibilityITCase.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/LeaderEpochCompatibilityITCase.java new file mode 100644 index 00000000000..a035b9d3dfb --- /dev/null +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/LeaderEpochCompatibilityITCase.java @@ -0,0 +1,301 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.replica.fetcher; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.cluster.AlterConfigOpType; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.rpc.messages.AlterClusterConfigsRequest; +import org.apache.fluss.rpc.messages.FetchLogRequest; +import org.apache.fluss.rpc.messages.PbFetchLogReqForBucket; +import org.apache.fluss.rpc.messages.PbFetchLogRespForBucket; +import org.apache.fluss.server.log.FetchIsolation; +import org.apache.fluss.server.replica.Replica; +import org.apache.fluss.server.tablet.TabletServer; +import org.apache.fluss.server.testutils.FlussClusterExtension; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.apache.fluss.record.TestData.DATA1; +import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE; +import static org.apache.fluss.record.TestData.DATA1_SCHEMA; +import static org.apache.fluss.record.TestData.DATA1_TABLE_PATH; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.assertProduceLogResponse; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.createTable; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newFetchLogRequest; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newProduceLogRequest; +import static org.apache.fluss.testutils.DataTestUtils.assertLogRecordsEquals; +import static org.apache.fluss.testutils.DataTestUtils.genMemoryLogRecordsByObject; +import static org.apache.fluss.testutils.common.CommonTestUtils.retry; +import static org.assertj.core.api.Assertions.assertThat; + +/** Exercises production replication modes through real RPC and persistent node restarts. */ +class LeaderEpochCompatibilityITCase { + private FlussClusterExtension cluster; + private TableBucket bucket; + private final List expected = new ArrayList<>(); + + @AfterEach + void close() throws Exception { + if (cluster != null) { + cluster.close(); + } + } + + @ParameterizedTest + @CsvSource({"false,false", "false,true", "true,false", "true,true"}) + void testMixedModesReplicateAndMigrate(boolean leaderEnabled, boolean followerEnabled) + throws Exception { + start(leaderEnabled); + int leader = cluster.waitAndGetLeader(bucket); + for (int id = 0; id < 3; id++) { + if (id != leader && followerEnabled != leaderEnabled) { + restart(id, followerEnabled); + } + } + append(); + assertReplicas(); + for (int id = 0; id < 3; id++) { + int epoch = replica(id).getLogTablet().lastFetchedEpoch(expected.size()); + if (leaderEnabled && (id == leader || followerEnabled)) { + assertThat(epoch).isGreaterThanOrEqualTo(0); + } else { + assertThat(epoch).isEqualTo(-1); + } + } + assertEpochResponse(leader, followerEnabled, leaderEnabled && followerEnabled); + migrateAndAppend(); + assertThat(replica(leader).getLogTablet().isLeaderEpochEnabled()).isEqualTo(leaderEnabled); + for (int id = 0; id < 3; id++) { + if (id != leader) { + assertThat(replica(id).getLogTablet().isLeaderEpochEnabled()) + .isEqualTo(followerEnabled); + } + } + } + + @Test + void testRollingEnableDisableAndReenable() throws Exception { + start(false); + append(); + assertReplicas(); + for (int id = 0; id < 3; id++) { + restart(id, true); + append(); + assertReplicas(); + } + for (int id = 0; id < 3; id++) { + assertThat(replica(id).getLogTablet().lastFetchedEpoch(10)).isEqualTo(-1); + } + int follower = (cluster.waitAndGetLeader(bucket) + 1) % 3; + restart(follower, false); + append(); + assertReplicas(); + long untrackedEnd = expected.size(); + restart(follower, true); + assertThat(replica(follower).getLogTablet().lastFetchedEpoch(untrackedEnd)).isEqualTo(-1); + append(); + assertReplicas(); + migrateAndAppend(); + for (int id = 0; id < 3; id++) { + assertThat(replica(id).getLogTablet().lastFetchedEpoch(expected.size())) + .isGreaterThanOrEqualTo(0); + } + } + + @Test + void testAlterModesWithoutRestartingNodes() throws Exception { + start(true); + append(); + assertReplicas(); + List runningNodes = new ArrayList<>(); + for (int id = 0; id < 3; id++) { + runningNodes.add(cluster.getTabletServerById(id)); + } + alterMode(false); + for (int id = 0; id < 3; id++) { + assertThat(replica(id).getLogTablet().lastFetchedEpoch(10)).isEqualTo(-1); + } + append(); + assertReplicas(); + alterMode(true); + append(); + assertReplicas(); + for (int id = 0; id < 3; id++) { + assertThat(cluster.getTabletServerById(id)).isSameAs(runningNodes.get(id)); + // Enabling within the same leader epoch cannot invent a new epoch boundary. + assertThat(replica(id).getLogTablet().lastFetchedEpoch(expected.size())).isEqualTo(-1); + } + migrateAndAppend(); + for (int id = 0; id < 3; id++) { + assertThat(replica(id).getLogTablet().lastFetchedEpoch(expected.size())) + .isGreaterThanOrEqualTo(0); + } + } + + private void alterMode(boolean enabled) throws Exception { + AlterClusterConfigsRequest request = new AlterClusterConfigsRequest(); + request.addAlterConfig() + .setConfigKey(ConfigOptions.LOG_REPLICATION_LEADER_EPOCH_ENABLED.key()) + .setConfigValue(Boolean.toString(enabled)) + .setOpType(AlterConfigOpType.SET.value()); + cluster.newCoordinatorClient().alterClusterConfigs(request).get(30, TimeUnit.SECONDS); + retry( + Duration.ofSeconds(30), + () -> { + for (int id = 0; id < 3; id++) { + assertThat(replica(id).getLogTablet().isLeaderEpochEnabled()) + .isEqualTo(enabled); + } + }); + } + + private void start(boolean enabled) throws Exception { + Configuration conf = new Configuration(); + conf.set(ConfigOptions.DEFAULT_REPLICATION_FACTOR, 3); + conf.set(ConfigOptions.LOG_REPLICA_MAX_LAG_TIME, Duration.ofSeconds(2)); + conf.set(ConfigOptions.REMOTE_LOG_TASK_INTERVAL_DURATION, Duration.ZERO); + FlussClusterExtension.Builder builder = + FlussClusterExtension.builder().setNumOfTabletServers(3).setClusterConf(conf); + for (int id = 0; id < 3; id++) { + builder.setTabletServerConf(id, mode(enabled)); + } + cluster = builder.build(); + cluster.start(); + long tableId = + createTable( + cluster, + DATA1_TABLE_PATH, + TableDescriptor.builder() + .schema(DATA1_SCHEMA) + .distributedBy(1, "a") + .build()); + bucket = new TableBucket(tableId, 0); + cluster.waitUntilAllReplicaReady(bucket); + } + + private static Configuration mode(boolean enabled) { + Configuration config = new Configuration(); + config.set(ConfigOptions.LOG_REPLICATION_LEADER_EPOCH_ENABLED, enabled); + return config; + } + + private void restart(int id, boolean enabled) throws Exception { + cluster.restartTabletServer(id, mode(enabled)); + cluster.waitUntilReplicaExpandToIsr(bucket, id); + assertReplicas(); + } + + private void append() throws Exception { + cluster.waitAndGetLeaderReplica(bucket); + int leader = cluster.waitAndGetLeader(bucket); + List batch = new ArrayList<>(); + for (int i = 0; i < DATA1.size(); i++) { + batch.add(new Object[] {expected.size() + i, "row-" + (expected.size() + i)}); + } + assertProduceLogResponse( + cluster.newTabletServerClientForNode(leader) + .produceLog( + newProduceLogRequest( + bucket.getTableId(), + 0, + -1, + genMemoryLogRecordsByObject(batch))) + .get(30, TimeUnit.SECONDS), + 0, + (long) expected.size()); + expected.addAll(batch); + } + + private void assertEpochResponse(int leader, boolean requestEpoch, boolean responseEpoch) + throws Exception { + int follower = (leader + 1) % 3; + FetchLogRequest request = + newFetchLogRequest(follower, bucket.getTableId(), 0, expected.size()); + if (requestEpoch) { + PbFetchLogReqForBucket bucketRequest = + request.getTablesReqsList().get(0).getBucketsReqsList().get(0); + bucketRequest + .setCurrentLeaderEpoch(replica(leader).getLeaderEpoch()) + .setLastFetchedEpoch( + replica(follower).getLogTablet().lastFetchedEpoch(expected.size())); + } + PbFetchLogRespForBucket response = + cluster.newTabletServerClientForNode(leader) + .fetchLog(request) + .get(30, TimeUnit.SECONDS) + .getTablesRespsList() + .get(0) + .getBucketsRespsList() + .get(0); + assertThat(response.hasErrorCode()).isFalse(); + assertThat(response.hasCurrentLeaderEpoch()).isEqualTo(responseEpoch); + assertThat(response.hasDivergingEpoch()).isFalse(); + } + + private void migrateAndAppend() throws Exception { + int oldLeader = cluster.waitAndGetLeader(bucket); + cluster.stopTabletServer(oldLeader); + retry( + Duration.ofSeconds(30), + () -> assertThat(cluster.waitAndGetLeader(bucket)).isNotEqualTo(oldLeader)); + append(); + cluster.startTabletServer(oldLeader); + cluster.waitUntilReplicaExpandToIsr(bucket, oldLeader); + assertReplicas(); + } + + private Replica replica(int id) { + return cluster.getTabletServerById(id).getReplicaManager().getReplicaOrException(bucket); + } + + private void assertReplicas() throws Exception { + retry( + Duration.ofSeconds(30), + () -> { + assertThat(cluster.waitAndGetLeaderReplica(bucket).getIsr()).hasSize(3); + for (int id = 0; id < 3; id++) { + Replica replica = replica(id); + assertThat(replica.getLocalLogEndOffset()).isEqualTo(expected.size()); + assertThat(replica.getLogHighWatermark()).isEqualTo(expected.size()); + if (!expected.isEmpty()) { + assertLogRecordsEquals( + DATA1_ROW_TYPE, + replica.getLogTablet() + .read( + 0, + Integer.MAX_VALUE, + FetchIsolation.LOG_END, + true) + .getRecords(), + expected); + } + } + }); + } +} diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/LeaderMigrationWalTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/LeaderMigrationWalTest.java new file mode 100644 index 00000000000..3dcf25190ed --- /dev/null +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/LeaderMigrationWalTest.java @@ -0,0 +1,661 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.replica.fetcher; + +import org.apache.fluss.cluster.Endpoint; +import org.apache.fluss.cluster.ServerNode; +import org.apache.fluss.cluster.ServerType; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.exception.FencedLeaderEpochException; +import org.apache.fluss.metadata.LeaderEpochOffset; +import org.apache.fluss.metadata.TableBucket; +import org.apache.fluss.record.LogRecord; +import org.apache.fluss.record.LogRecordBatch; +import org.apache.fluss.record.LogRecordReadContext; +import org.apache.fluss.record.TestingSchemaGetter; +import org.apache.fluss.rpc.RpcClient; +import org.apache.fluss.rpc.entity.FetchLogEpochInfo; +import org.apache.fluss.rpc.entity.FetchLogResultForBucket; +import org.apache.fluss.rpc.entity.ProduceLogResultForBucket; +import org.apache.fluss.rpc.messages.ApiMessage; +import org.apache.fluss.rpc.messages.FetchLogRequest; +import org.apache.fluss.rpc.protocol.ApiKeys; +import org.apache.fluss.server.coordinator.LakeCatalogDynamicLoader; +import org.apache.fluss.server.coordinator.MetadataManager; +import org.apache.fluss.server.coordinator.TestCoordinatorGateway; +import org.apache.fluss.server.coordinator.statemachine.ReplicaLeaderElection.ReassignmentLeaderElection; +import org.apache.fluss.server.entity.NotifyLeaderAndIsrData; +import org.apache.fluss.server.entity.NotifyLeaderAndIsrResultForBucket; +import org.apache.fluss.server.kv.KvManager; +import org.apache.fluss.server.kv.scan.ScannerManager; +import org.apache.fluss.server.kv.snapshot.TestingCompletedKvSnapshotCommitter; +import org.apache.fluss.server.log.FetchIsolation; +import org.apache.fluss.server.log.LogManager; +import org.apache.fluss.server.metadata.ClusterMetadata; +import org.apache.fluss.server.metadata.ServerInfo; +import org.apache.fluss.server.metadata.TabletServerMetadataCache; +import org.apache.fluss.server.metrics.group.TestingMetricGroups; +import org.apache.fluss.server.replica.Replica; +import org.apache.fluss.server.replica.ReplicaManager; +import org.apache.fluss.server.storage.LocalDiskManager; +import org.apache.fluss.server.zk.NOPErrorHandler; +import org.apache.fluss.server.zk.ZooKeeperClient; +import org.apache.fluss.server.zk.ZooKeeperExtension; +import org.apache.fluss.server.zk.data.LeaderAndIsr; +import org.apache.fluss.server.zk.data.TableRegistration; +import org.apache.fluss.testutils.common.AllCallbackWrapper; +import org.apache.fluss.utils.CloseableIterator; +import org.apache.fluss.utils.clock.ManualClock; +import org.apache.fluss.utils.concurrent.FlussScheduler; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.File; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +import static org.apache.fluss.record.TestData.ANOTHER_DATA1; +import static org.apache.fluss.record.TestData.DATA1; +import static org.apache.fluss.record.TestData.DATA1_PHYSICAL_TABLE_PATH; +import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE; +import static org.apache.fluss.record.TestData.DATA1_SCHEMA; +import static org.apache.fluss.record.TestData.DATA1_TABLE_DESCRIPTOR; +import static org.apache.fluss.record.TestData.DATA1_TABLE_ID; +import static org.apache.fluss.record.TestData.DATA1_TABLE_PATH; +import static org.apache.fluss.record.TestData.DEFAULT_SCHEMA_ID; +import static org.apache.fluss.server.utils.ServerRpcMessageUtils.makeFetchLogResponse; +import static org.apache.fluss.testutils.DataTestUtils.genMemoryLogRecordsWithWriterId; +import static org.apache.fluss.testutils.common.CommonTestUtils.retry; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests WAL convergence through clean leader migration with controlled RPC delivery. */ +class LeaderMigrationWalTest { + @RegisterExtension + static final AllCallbackWrapper ZK = + new AllCallbackWrapper<>(new ZooKeeperExtension()); + + private static final TableBucket BUCKET = new TableBucket(DATA1_TABLE_ID, 0); + private static final List REPLICAS = Arrays.asList(1, 2, 3); + private final Map servers = new HashMap<>(); + private final ManualClock clock = new ManualClock(System.currentTimeMillis()); + private @TempDir File tempDir; + private ZooKeeperClient zkClient; + private LeaderAndIsr leaderAndIsr; + + @BeforeEach + void setUp() throws Exception { + zkClient = ZK.getCustomExtension().getZooKeeperClient(NOPErrorHandler.INSTANCE); + ZK.getCustomExtension().cleanupRoot(); + zkClient.registerTable( + DATA1_TABLE_PATH, + TableRegistration.newTable( + DATA1_TABLE_ID, + new File(tempDir, "remote").getAbsolutePath(), + DATA1_TABLE_DESCRIPTOR)); + zkClient.registerFirstSchema(DATA1_TABLE_PATH, DATA1_SCHEMA); + for (int id : REPLICAS) { + servers.put(id, new Server(id)); + } + leaderAndIsr = new LeaderAndIsr(1, 0, REPLICAS, Collections.emptyList(), 0, 0); + for (int id : REPLICAS) { + notifyRole(id); + } + } + + @AfterEach + void tearDown() throws Exception { + for (Server server : servers.values()) { + server.rpc.pause(); + } + for (Server server : servers.values()) { + server.close(); + } + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testCleanMigrationReplacesUncommittedFollowerTail(boolean enabled) throws Exception { + Configuration mode = new Configuration(); + mode.set(ConfigOptions.LOG_REPLICATION_LEADER_EPOCH_ENABLED, enabled); + for (Server server : servers.values()) { + server.log.reconfigure(mode); + } + assertMigrationConverges(false, enabled); + } + + @Test + void testCleanMigrationWithCaughtUpTarget() throws Exception { + assertMigrationConverges(true); + } + + @Test + void testDisablingPreventsEpochTruncationOfExistingTail() throws Exception { + append(1, DATA1, 100, 1).get(5, TimeUnit.SECONDS); + deliver(2, 1, 0); + List copied = readWal(2); + Configuration mode = new Configuration(); + mode.set(ConfigOptions.LOG_REPLICATION_LEADER_EPOCH_ENABLED, false); + servers.get(2).log.reconfigure(mode); + assertThat(replica(2).truncateFollowerToEpochOffset(0, 1, 0)).isFalse(); + assertThat(replica(2).getLocalLogEndOffset()).isEqualTo(10); + assertThat(readWal(2)).containsExactlyElementsOf(copied); + assertThat(replica(2).getLogTablet().lastFetchedEpoch(10)).isEqualTo(-1); + } + + @Test + void testLateFetchCannotModifyReplicaAfterLeaderChange() throws Exception { + FetchLogResultForBucket late = + FetchLogResultForBucket.records( + BUCKET, + genMemoryLogRecordsWithWriterId(DATA1, 100, 0, 0), + 10, + -1, + -1) + .withEpochInfo( + new FetchLogEpochInfo( + 0, + null, + Collections.singletonList(new LeaderEpochOffset(0, 0)))); + moveLeader(3); + assertThatThrownBy(() -> replica(2).appendRecordsToFollower(late, 1, 0)) + .isInstanceOf(FencedLeaderEpochException.class); + assertThat(replica(2).getLocalLogEndOffset()).isZero(); + assertThat(replica(2).getLogHighWatermark()).isZero(); + assertThat(replica(2).getLogTablet().lastFetchedEpoch(10)).isEqualTo(-1); + } + + @Test + void testConvergenceAcrossConsecutiveMigrationsAndEmptyEpoch() throws Exception { + assertMigrationConverges(false); + List prefix = readWal(2); + moveLeader(1); + // No records are written in this epoch before the next clean migration. + moveLeader(3); + CompletableFuture> appended = + append(3, DATA1.subList(0, 5), 103, -1); + for (int round = 0; round < 6; round++) { + deliver(1, 3, -1); + deliver(2, 3, -1); + if (appended.isDone() + && replica(1).getLogHighWatermark() == 20 + && replica(2).getLogHighWatermark() == 20) { + break; + } + } + assertThat(appended.get(5, TimeUnit.SECONDS)) + .containsExactly(new ProduceLogResultForBucket(BUCKET, 15, 20)); + for (int i = 0; i < 5; i++) { + prefix.add((15 + i) + ":" + DATA1.get(i)[0] + ":" + DATA1.get(i)[1]); + } + for (int id : REPLICAS) { + assertThat(readWal(id)).containsExactlyElementsOf(prefix); + assertThat(replica(id).getLogTablet().lastFetchedEpoch(20)) + .isEqualTo(leaderAndIsr.leaderEpoch()); + } + } + + private void moveLeader(int target) throws Exception { + for (Server server : servers.values()) { + server.rpc.pause(); + } + List preference = new ArrayList<>(REPLICAS); + preference.remove(Integer.valueOf(target)); + preference.add(0, target); + leaderAndIsr = + new ReassignmentLeaderElection(preference) + .leaderElection(REPLICAS, leaderAndIsr, false) + .get() + .getLeaderAndIsr(); + notifyRole(target); + for (int id : REPLICAS) { + if (id != target) { + notifyRole(id); + } + } + for (Server server : servers.values()) { + server.rpc.resume(); + } + } + + @Test + void testLegacyReplicationRemainsAvailableWithoutInventingHistory() throws Exception { + CompletableFuture> first = append(1, DATA1, 100, -1); + deliver(2, 1, 0, true); + deliver(3, 1, 0); + deliver(2, 1, 10, true); + deliver(3, 1, 10); + first.get(5, TimeUnit.SECONDS); + assertThat(replica(2).getLogTablet().lastFetchedEpoch(10)).isEqualTo(-1); + + CompletableFuture> second = + append(1, ANOTHER_DATA1.subList(0, 5), 101, -1); + // The upgraded peer must not infer the identity of its old prefix from the source's + // epoch boundary, which precedes this fetch. + deliver(2, 1, 10); + deliver(3, 1, 10); + deliver(2, 1, 15); + deliver(3, 1, 15); + second.get(5, TimeUnit.SECONDS); + deliver(2, 1, 15); + assertThat(replica(2).getLogTablet().lastFetchedEpoch(10)).isEqualTo(-1); + assertThat(replica(2).getLogTablet().lastFetchedEpoch(15)).isEqualTo(-1); + assertThat(readWal(2)).containsExactlyElementsOf(readWal(1)); + } + + private void assertMigrationConverges(boolean catchUpTarget) throws Exception { + assertMigrationConverges(catchUpTarget, true); + } + + private void assertMigrationConverges(boolean catchUpTarget, boolean enabled) throws Exception { + CompletableFuture> committed = append(1, DATA1, 100, -1); + deliver(2, 1, 0); + deliver(3, 1, 0); + deliver(2, 1, 10); + deliver(3, 1, 10); + assertThat(committed.get(5, TimeUnit.SECONDS)) + .containsExactly(new ProduceLogResultForBucket(BUCKET, 0, 10)); + assertThat(replica(1).getLogHighWatermark()).isEqualTo(10); + + // Only C receives this uncommitted tail; B remains in the ISR at the committed prefix. + append(1, DATA1.subList(0, 5), 101, 1).get(5, TimeUnit.SECONDS); + deliver(3, 1, 10); + assertThat(replica(2).getLocalLogEndOffset()).isEqualTo(10); + assertThat(replica(3).getLocalLogEndOffset()).isEqualTo(15); + assertThat(replica(1).getLogHighWatermark()).isEqualTo(10); + + if (catchUpTarget) { + deliver(2, 1, 10); + } + long newLeaderStart = catchUpTarget ? 15 : 10; + long newLeaderEnd = newLeaderStart + 5; + assertThat(replica(1).getIsr()).containsExactlyInAnyOrderElementsOf(REPLICAS); + + // Reassignment elects B from the live ISR. No unclean election or file mutation is used. + for (Server server : servers.values()) { + server.rpc.pause(); + } + leaderAndIsr = + new ReassignmentLeaderElection(Arrays.asList(2, 1, 3)) + .leaderElection(REPLICAS, leaderAndIsr, false) + .get() + .getLeaderAndIsr(); + assertThat(leaderAndIsr.leader()).isEqualTo(2); + assertThat(leaderAndIsr.isr()).containsExactlyElementsOf(REPLICAS); + notifyRole(2); + notifyRole(1); + notifyRole(3); + for (Server server : servers.values()) { + server.rpc.resume(); + } + + CompletableFuture> replacement = + append(2, ANOTHER_DATA1.subList(0, 5), 102, -1); + // Allow history reconciliation, data replication, and committed watermark propagation. + for (int round = 0; round < 6; round++) { + deliver(1, 2, -1); + deliver(3, 2, -1); + if (enabled && round == 0) { + assertThat(replica(2).getLogHighWatermark()).isLessThanOrEqualTo(newLeaderStart); + assertThat(replacement).isNotDone(); + } + if (replacement.isDone() + && replica(1).getLogHighWatermark() == newLeaderEnd + && replica(3).getLogHighWatermark() == newLeaderEnd) { + break; + } + } + assertThat(replacement.get(5, TimeUnit.SECONDS)) + .containsExactly( + new ProduceLogResultForBucket(BUCKET, newLeaderStart, newLeaderEnd)); + assertThat(replica(2).getLogHighWatermark()).isEqualTo(newLeaderEnd); + assertThat(replica(3).getLogHighWatermark()).isEqualTo(newLeaderEnd); + assertThat(replica(2).getIsr()).containsExactlyInAnyOrderElementsOf(REPLICAS); + + List expected = new ArrayList<>(); + for (int i = 0; i < DATA1.size(); i++) { + expected.add(i + ":" + DATA1.get(i)[0] + ":" + DATA1.get(i)[1]); + } + if (catchUpTarget) { + for (int i = 0; i < 5; i++) { + expected.add((10 + i) + ":" + DATA1.get(i)[0] + ":" + DATA1.get(i)[1]); + } + } + for (int i = 0; i < 5; i++) { + expected.add( + (newLeaderStart + i) + + ":" + + ANOTHER_DATA1.get(i)[0] + + ":" + + ANOTHER_DATA1.get(i)[1]); + } + assertThat(readWal(2)).containsExactlyElementsOf(expected); + assertThat(readWal(1)).containsExactlyElementsOf(expected); + if (enabled) { + assertThat(readWal(3)) + .as("ISR replicas must contain the acknowledged records at the same offsets") + .containsExactlyElementsOf(expected); + } else { + List divergent = new ArrayList<>(expected.subList(0, 10)); + for (int i = 0; i < 5; i++) { + divergent.add((10 + i) + ":" + DATA1.get(i)[0] + ":" + DATA1.get(i)[1]); + } + assertThat(readWal(3)) + .as("Legacy replication retains the old uncommitted tail") + .containsExactlyElementsOf(divergent) + .isNotEqualTo(readWal(2)); + } + } + + private Replica replica(int id) { + return servers.get(id).manager.getReplicaOrException(BUCKET); + } + + private void notifyRole(int id) throws Exception { + CompletableFuture> result = + new CompletableFuture<>(); + servers.get(id) + .manager + .becomeLeaderOrFollower( + 0, + Collections.singletonList( + new NotifyLeaderAndIsrData( + DATA1_PHYSICAL_TABLE_PATH, BUCKET, REPLICAS, leaderAndIsr)), + result::complete); + assertThat(result.get(5, TimeUnit.SECONDS)) + .containsExactly(new NotifyLeaderAndIsrResultForBucket(BUCKET)); + } + + private CompletableFuture> append( + int id, List rows, long writerId, int acks) throws Exception { + CompletableFuture> result = new CompletableFuture<>(); + servers.get(id) + .manager + .appendRecordsToLog( + 10000, + acks, + Collections.singletonMap( + BUCKET, genMemoryLogRecordsWithWriterId(rows, writerId, 0, 0)), + null, + result::complete); + return result; + } + + private void deliver(int follower, int leader, long offset) throws Exception { + deliver(follower, leader, offset, false); + } + + private void deliver(int follower, int leader, long offset, boolean legacy) throws Exception { + ControlledRpcClient rpc = servers.get(follower).rpc; + PendingRequest pending = rpc.requests.poll(5, TimeUnit.SECONDS); + assertThat(pending).as("fetch from server %s", follower).isNotNull(); + assertThat(pending.destination).isEqualTo(leader); + assertThat(pending.request).isInstanceOf(FetchLogRequest.class); + FetchLogRequest request = (FetchLogRequest) pending.request; + if (offset >= 0) { + assertThat( + request.getTablesReqsList() + .get(0) + .getBucketsReqsList() + .get(0) + .getFetchOffset()) + .isEqualTo(offset); + } + FetchLogRequest serverRequest = new FetchLogRequest().copyFrom(request); + if (legacy) { + serverRequest + .getTablesReqsList() + .forEach( + table -> + table.getBucketsReqsList() + .forEach( + bucket -> { + bucket.clearCurrentLeaderEpoch(); + bucket.clearLastFetchedEpoch(); + })); + } + TestingLeaderEndpoint endpoint = + new TestingLeaderEndpoint( + servers.get(leader).conf, + servers.get(leader).manager, + new ServerNode( + follower, "localhost", 10000 + follower, ServerType.TABLET_SERVER)); + LeaderEndpoint.FetchData data = + endpoint.fetchLog( + new FetchLogContext( + Collections.singletonMap(DATA1_TABLE_ID, DATA1_TABLE_PATH), + serverRequest)) + .get(5, TimeUnit.SECONDS); + pending.result.complete(makeFetchLogResponse(data.getFetchLogResultMap())); + // The next request proves the actual background fetcher has consumed this response. + retry(Duration.ofSeconds(5), () -> assertThat(rpc.requests).isNotEmpty()); + } + + private List readWal(int id) throws Exception { + List records = new ArrayList<>(); + try (LogRecordReadContext context = + LogRecordReadContext.createArrowReadContext( + DATA1_ROW_TYPE, + DEFAULT_SCHEMA_ID, + new TestingSchemaGetter(DEFAULT_SCHEMA_ID, DATA1_SCHEMA))) { + for (LogRecordBatch batch : + replica(id) + .getLogTablet() + .read(0, Integer.MAX_VALUE, FetchIsolation.HIGH_WATERMARK, true) + .getRecords() + .batches()) { + batch.ensureValid(); + try (CloseableIterator iterator = batch.records(context)) { + while (iterator.hasNext()) { + LogRecord record = iterator.next(); + records.add( + record.logOffset() + + ":" + + record.getRow().getInt(0) + + ":" + + record.getRow().getString(1)); + } + } + } + } + return records; + } + + private static final class PendingRequest { + private final int destination; + private final ApiMessage request; + private final CompletableFuture result = new CompletableFuture<>(); + + private PendingRequest(int destination, ApiMessage request) { + this.destination = destination; + this.request = request; + } + } + + /** Holds RPC requests until the test delivers them to the real destination ReplicaManager. */ + private static final class ControlledRpcClient implements RpcClient { + private final BlockingQueue requests = new LinkedBlockingQueue<>(); + private boolean paused; + + @Override + public synchronized CompletableFuture sendRequest( + ServerNode node, ApiKeys apiKey, ApiMessage request) { + PendingRequest pending = new PendingRequest(node.id(), request); + if (paused) { + pending.result.completeExceptionally(new IOException("Connection paused")); + } else { + requests.add(pending); + } + return pending.result; + } + + synchronized void pause() { + paused = true; + PendingRequest request; + while ((request = requests.poll()) != null) { + request.result.completeExceptionally(new IOException("Connection paused")); + } + } + + synchronized void resume() { + paused = false; + } + + @Override + public boolean connect(ServerNode node) { + return true; + } + + @Override + public boolean isReady(String serverUid) { + return true; + } + + @Override + public CompletableFuture disconnect(String serverUid) { + pause(); + return CompletableFuture.completedFuture(null); + } + + @Override + public void close() { + pause(); + } + } + + private final class Server implements AutoCloseable { + private final Configuration conf = new Configuration(); + private final ControlledRpcClient rpc = new ControlledRpcClient(); + private final FlussScheduler scheduler = new FlussScheduler(2); + private final ExecutorService io = Executors.newSingleThreadExecutor(); + private final LocalDiskManager disk; + private final LogManager log; + private final KvManager kv; + private final ScannerManager scanner; + private final ReplicaManager manager; + + private Server(int id) throws Exception { + conf.set(ConfigOptions.TABLET_SERVER_ID, id); + conf.setString( + ConfigOptions.DATA_DIR, new File(tempDir, "server-" + id).getAbsolutePath()); + conf.set(ConfigOptions.REMOTE_DATA_DIR, new File(tempDir, "remote").getAbsolutePath()); + conf.set(ConfigOptions.SERVER_DATA_DISK_WRITE_LIMIT_RATIO, 1.0); + conf.set(ConfigOptions.LOG_REPLICA_FETCH_BACKOFF_INTERVAL, Duration.ofMillis(10)); + conf.set(ConfigOptions.LOG_REPLICA_FETCH_WAIT_MAX_TIME, Duration.ZERO); + scheduler.startup(); + disk = LocalDiskManager.create(conf); + log = + LogManager.create( + conf, + zkClient, + scheduler, + clock, + TestingMetricGroups.TABLET_SERVER_METRICS, + disk); + log.startup(); + kv = + KvManager.create( + conf, + zkClient, + log, + TestingMetricGroups.TABLET_SERVER_METRICS, + disk, + null, + clock); + kv.startup(); + scanner = new ScannerManager(conf, scheduler); + TabletServerMetadataCache metadata = + new TabletServerMetadataCache( + new MetadataManager( + zkClient, + conf, + new LakeCatalogDynamicLoader(conf, null, true))); + List members = new ArrayList<>(); + for (int member : REPLICAS) { + members.add( + new ServerInfo( + member, + null, + Endpoint.fromListenersString( + "FLUSS://localhost:" + (10000 + member)), + ServerType.TABLET_SERVER)); + } + metadata.updateClusterMetadata( + new ClusterMetadata( + new ServerInfo( + 0, + null, + Endpoint.fromListenersString("FLUSS://localhost:9999"), + ServerType.COORDINATOR), + new HashSet<>(members))); + manager = + new ReplicaManager( + conf, + scheduler, + log, + kv, + zkClient, + id, + metadata, + rpc, + new TestCoordinatorGateway(), + new TestingCompletedKvSnapshotCommitter(), + NOPErrorHandler.INSTANCE, + TestingMetricGroups.TABLET_SERVER_METRICS, + TestingMetricGroups.USER_METRICS, + scanner, + clock, + io, + disk, + null); + manager.startup(); + } + + @Override + public void close() throws Exception { + manager.shutdown(); + manager.getRemoteLogManager().close(); + scanner.close(); + kv.shutdown(); + log.shutdown(); + scheduler.shutdown(); + disk.close(); + io.shutdownNow(); + } + } +} diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherITCase.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherITCase.java index 36d3ef567ed..4d72281ec0d 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherITCase.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherITCase.java @@ -27,6 +27,7 @@ import org.apache.fluss.rpc.gateway.TabletServerGateway; import org.apache.fluss.rpc.messages.PbPutKvRespForBucket; import org.apache.fluss.rpc.messages.PutKvResponse; +import org.apache.fluss.rpc.messages.StopReplicaRequest; import org.apache.fluss.server.entity.FetchReqInfo; import org.apache.fluss.server.log.FetchParams; import org.apache.fluss.server.replica.Replica; @@ -66,6 +67,7 @@ import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newFetchLogRequest; import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newLookupRequest; import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newPutKvRequest; +import static org.apache.fluss.server.utils.ServerRpcMessageUtils.makeStopBucketReplica; import static org.apache.fluss.testutils.DataTestUtils.assertLogRecordsEquals; import static org.apache.fluss.testutils.DataTestUtils.assertLogRecordsEqualsWithRowKind; import static org.apache.fluss.testutils.DataTestUtils.genKvRecordBatch; @@ -269,9 +271,21 @@ void testFlushForPutKvNeedAck() throws Exception { .get() .id(); - int leaderEpoch = 0; - // stop the follower replica for the bucket - FLUSS_CLUSTER_EXTENSION.stopReplica(followerToStop, tb, leaderEpoch); + LeaderAndIsr currentLeaderAndIsr = zkClient.getLeaderAndIsr(tb).get(); + // Remove the local follower so it can be recreated in the same leader epoch. + FLUSS_CLUSTER_EXTENSION + .newTabletServerClientForNode(followerToStop) + .stopReplica( + new StopReplicaRequest() + .setCoordinatorEpoch(currentLeaderAndIsr.coordinatorEpoch()) + .addAllStopReplicasReqs( + Collections.singleton( + makeStopBucketReplica( + tb, + true, + false, + currentLeaderAndIsr.leaderEpoch())))) + .get(); // put kv record batch to the leader, // but as one server is killed, the put won't be ack @@ -310,20 +324,17 @@ void testFlushForPutKvNeedAck() throws Exception { // start the follower replica by notify leaderAndIsr, // then the kv should be flushed finally - LeaderAndIsr currentLeaderAndIsr = zkClient.getLeaderAndIsr(tb).get(); - LeaderAndIsr newLeaderAndIsr = - new LeaderAndIsr( - currentLeaderAndIsr.leader(), - currentLeaderAndIsr.leaderEpoch() + 1, - currentLeaderAndIsr.isr(), - Collections.emptyList(), - currentLeaderAndIsr.coordinatorEpoch(), - currentLeaderAndIsr.bucketEpoch()); FLUSS_CLUSTER_EXTENSION.notifyLeaderAndIsr( - followerToStop, DATA1_TABLE_PATH, tb, newLeaderAndIsr, Arrays.asList(0, 1, 2)); + followerToStop, + DATA1_TABLE_PATH_PK, + tb, + currentLeaderAndIsr, + Arrays.asList(0, 1, 2)); // wait until the put future is done - putResponse.get(); + for (PbPutKvRespForBucket result : putResponse.get().getBucketsRespsList()) { + assertThat(result.hasErrorCode()).isFalse(); + } // then we can check all the value for (Tuple2 keyValue : expectedKeyValues) { diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/TestingLeaderEndpoint.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/TestingLeaderEndpoint.java index ab6d8e5c43f..ce4681c038f 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/TestingLeaderEndpoint.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/TestingLeaderEndpoint.java @@ -213,11 +213,12 @@ private Map processResult( value.hasMinRetainOffset() ? value.getMinRetainOffset() : -1L; FetchLogResultForBucket memoryResult = FetchLogResultForBucket.records( - tb, - memRecords, - value.getHighWatermark(), - filteredEndOffset, - minRetainOffset); + tb, + memRecords, + value.getHighWatermark(), + filteredEndOffset, + minRetainOffset) + .withEpochInfo(value.epochInfo()); result.put(tb, memoryResult); } else { result.put(tb, value); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java b/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java index cff3f788c99..bccc0a3e8d8 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java @@ -137,6 +137,7 @@ public final class FlussClusterExtension private final Map tabletServers; private final Map tabletServerInfos; private final Configuration clusterConf; + private final Map tabletServerConfigs; private final Clock clock; private final String[] racks; private final List remoteDirNames; @@ -153,13 +154,17 @@ private FlussClusterExtension( Configuration clusterConf, Clock clock, String[] racks, - List remoteDirNames) { + List remoteDirNames, + Map tabletServerConfigs) { this.initialNumOfTabletServers = numOfTabletServers; this.tabletServers = new HashMap<>(numOfTabletServers); this.coordinatorServerListeners = coordinatorServerListeners; this.tabletServerListeners = tabletServerListeners; this.tabletServerInfos = new HashMap<>(); this.clusterConf = clusterConf; + this.tabletServerConfigs = new HashMap<>(); + tabletServerConfigs.forEach( + (id, config) -> this.tabletServerConfigs.put(id, new Configuration(config))); this.clock = clock; checkArgument( racks != null && racks.length == numOfTabletServers, @@ -339,7 +344,13 @@ private void startTabletServer(int serverId, @Nullable Configuration overwriteCo tabletServerConf.setString(ConfigOptions.BIND_LISTENERS, tabletServerListeners); tabletServerConf.setDouble(ConfigOptions.SERVER_DATA_DISK_WRITE_LIMIT_RATIO, 1.0); if (overwriteConfig != null) { - tabletServerConf.addAll(overwriteConfig); + tabletServerConfigs + .computeIfAbsent(serverId, id -> new Configuration()) + .addAll(overwriteConfig); + } + Configuration nodeConfig = tabletServerConfigs.get(serverId); + if (nodeConfig != null) { + tabletServerConf.addAll(nodeConfig); } setRemoteDataDir(tabletServerConf); @@ -1017,6 +1028,7 @@ public static class Builder { private List remoteDirNames = Collections.emptyList(); private final Configuration clusterConf = new Configuration(); + private final Map tabletServerConfigs = new HashMap<>(); public Builder() { // reduce testing resources @@ -1046,6 +1058,12 @@ public Builder setClusterConf(Configuration clusterConf) { return this; } + /** Sets persistent configuration overrides for one tablet server, including restarts. */ + public Builder setTabletServerConf(int serverId, Configuration config) { + tabletServerConfigs.put(serverId, new Configuration(config)); + return this; + } + /** Sets the listeners of tablet servers. */ public Builder setTabletServerListeners(String tabletServerListeners) { this.tabletServerListeners = tabletServerListeners; @@ -1091,7 +1109,8 @@ public FlussClusterExtension build() { clusterConf, clock, racks, - remoteDirNames); + remoteDirNames, + tabletServerConfigs); } } } From d0f598e1f90542fc9ac3d073f5baa799a9966951 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E9=B5=BA?= Date: Fri, 11 Sep 2026 10:00:41 +0800 Subject: [PATCH 2/4] [server] Preserve replica state when resuming leader epochs Preserve durable epoch boundaries on restart and follower progress on leader reelection. Correct the test endpoint leader identity and verify follower high watermark convergence. Regenerate Rust protobuf bindings and keep scanner requests free of replication-only epoch metadata. --- fluss-rust/crates/fluss/proto/FlussApi.proto | 13 ++++++++++- .../crates/fluss/src/client/table/scanner.rs | 14 +++++++++++ fluss-rust/crates/fluss/src/proto/fluss.rs | 23 +++++++++++++++++-- .../fluss/server/log/LeaderEpochHistory.java | 4 +++- .../apache/fluss/server/replica/Replica.java | 3 +-- .../server/log/LeaderEpochHistoryTest.java | 19 +++++++++++++++ .../fetcher/LeaderMigrationWalTest.java | 3 ++- .../fetcher/ReplicaFetcherThreadTest.java | 17 ++++++++++---- .../fetcher/TestingLeaderEndpoint.java | 9 ++++++-- 9 files changed, 92 insertions(+), 13 deletions(-) diff --git a/fluss-rust/crates/fluss/proto/FlussApi.proto b/fluss-rust/crates/fluss/proto/FlussApi.proto index b778db8c24a..71c0a2d2c1d 100644 --- a/fluss-rust/crates/fluss/proto/FlussApi.proto +++ b/fluss-rust/crates/fluss/proto/FlussApi.proto @@ -934,16 +934,23 @@ message PbFetchLogReqForTable { message PbFetchLogReqForBucket { optional int64 partition_id = 1; required int32 bucket_id = 2; - // TODO leader epoch required int64 fetch_offset = 3; required int32 max_fetch_bytes = 4; optional int32 routing_bucket_count = 5; + // Replication only. Both fields must be present to request epoch validation. + optional int32 current_leader_epoch = 6; + optional int32 last_fetched_epoch = 7; } message PbFetchLogRespForTable { required int64 table_id = 1; repeated PbFetchLogRespForBucket buckets_resp = 2; } +message PbLeaderEpochOffset { + required int32 epoch = 1; + required int64 offset = 2; +} + message PbFetchLogRespForBucket { optional int64 partition_id = 1; required int32 bucket_id = 2; @@ -960,6 +967,9 @@ message PbFetchLogRespForBucket { // The safe local log retention boundary confirmed by a committed KV snapshot. This is only // returned for KV follower fetches and is distinct from the physical log_start_offset. optional int64 min_retain_offset = 10; + optional int32 current_leader_epoch = 11; + optional PbLeaderEpochOffset diverging_epoch = 12; + repeated PbLeaderEpochOffset epoch_starts = 13; } message PbPutKvReqForBucket { @@ -1157,6 +1167,7 @@ message PbRemoteLogSegment { required int64 remote_log_end_offset = 3; required int32 segment_size_in_bytes = 4; optional int64 max_timestamp = 5; + repeated PbLeaderEpochOffset leader_epochs = 6; } message PbPartitionInfo { diff --git a/fluss-rust/crates/fluss/src/client/table/scanner.rs b/fluss-rust/crates/fluss/src/client/table/scanner.rs index a8720b5ef79..08f57f63afd 100644 --- a/fluss-rust/crates/fluss/src/client/table/scanner.rs +++ b/fluss-rust/crates/fluss/src/client/table/scanner.rs @@ -2358,6 +2358,8 @@ impl LogFetcher { fetch_offset: offset, max_fetch_bytes: self.fetch_max_bytes_for_bucket, routing_bucket_count: None, + current_leader_epoch: None, + last_fetched_epoch: None, }; fetch_log_req_for_buckets @@ -3000,6 +3002,9 @@ mod tests { records: None, filtered_end_offset, min_retain_offset: None, + current_leader_epoch: None, + diverging_epoch: None, + epoch_starts: Vec::new(), }], }], } @@ -3055,6 +3060,9 @@ mod tests { records: None, filtered_end_offset: None, min_retain_offset: None, + current_leader_epoch: None, + diverging_epoch: None, + epoch_starts: Vec::new(), }], }], }; @@ -3115,6 +3123,9 @@ mod tests { records: None, filtered_end_offset: None, min_retain_offset: None, + current_leader_epoch: None, + diverging_epoch: None, + epoch_starts: Vec::new(), }], }], }; @@ -3465,6 +3476,9 @@ mod tests { records: None, filtered_end_offset: None, min_retain_offset: None, + current_leader_epoch: None, + diverging_epoch: None, + epoch_starts: Vec::new(), }], }], }; diff --git a/fluss-rust/crates/fluss/src/proto/fluss.rs b/fluss-rust/crates/fluss/src/proto/fluss.rs index 4e4e06d4124..c890a4e96cd 100644 --- a/fluss-rust/crates/fluss/src/proto/fluss.rs +++ b/fluss-rust/crates/fluss/src/proto/fluss.rs @@ -1241,13 +1241,17 @@ pub struct PbFetchLogReqForBucket { pub partition_id: ::core::option::Option, #[prost(int32, required, tag = "2")] pub bucket_id: i32, - /// TODO leader epoch #[prost(int64, required, tag = "3")] pub fetch_offset: i64, #[prost(int32, required, tag = "4")] pub max_fetch_bytes: i32, #[prost(int32, optional, tag = "5")] pub routing_bucket_count: ::core::option::Option, + /// Replication only. Both fields must be present to request epoch validation. + #[prost(int32, optional, tag = "6")] + pub current_leader_epoch: ::core::option::Option, + #[prost(int32, optional, tag = "7")] + pub last_fetched_epoch: ::core::option::Option, } #[derive(Clone, PartialEq, ::prost::Message)] pub struct PbFetchLogRespForTable { @@ -1256,6 +1260,13 @@ pub struct PbFetchLogRespForTable { #[prost(message, repeated, tag = "2")] pub buckets_resp: ::prost::alloc::vec::Vec, } +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbLeaderEpochOffset { + #[prost(int32, required, tag = "1")] + pub epoch: i32, + #[prost(int64, required, tag = "2")] + pub offset: i64, +} #[derive(Clone, PartialEq, ::prost::Message)] pub struct PbFetchLogRespForBucket { #[prost(int64, optional, tag = "1")] @@ -1284,6 +1295,12 @@ pub struct PbFetchLogRespForBucket { /// returned for KV follower fetches and is distinct from the physical log_start_offset. #[prost(int64, optional, tag = "10")] pub min_retain_offset: ::core::option::Option, + #[prost(int32, optional, tag = "11")] + pub current_leader_epoch: ::core::option::Option, + #[prost(message, optional, tag = "12")] + pub diverging_epoch: ::core::option::Option, + #[prost(message, repeated, tag = "13")] + pub epoch_starts: ::prost::alloc::vec::Vec, } #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct PbPutKvReqForBucket { @@ -1570,7 +1587,7 @@ pub struct PbRemoteLogFetchInfo { #[prost(int32, optional, tag = "4")] pub first_start_pos: ::core::option::Option, } -#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct PbRemoteLogSegment { #[prost(string, required, tag = "1")] pub remote_log_segment_id: ::prost::alloc::string::String, @@ -1582,6 +1599,8 @@ pub struct PbRemoteLogSegment { pub segment_size_in_bytes: i32, #[prost(int64, optional, tag = "5")] pub max_timestamp: ::core::option::Option, + #[prost(message, repeated, tag = "6")] + pub leader_epochs: ::prost::alloc::vec::Vec, } #[derive(Clone, PartialEq, ::prost::Message)] pub struct PbPartitionInfo { diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LeaderEpochHistory.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LeaderEpochHistory.java index e4554ed4eb2..0a735668eb0 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/LeaderEpochHistory.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LeaderEpochHistory.java @@ -65,7 +65,9 @@ void assign(int epoch, long startOffset) throws IOException { checkArgument(epoch >= 0 && startOffset >= 0, "Negative epoch or offset."); Long existing = epochs.get(epoch); if (existing != null) { - checkArgument(existing == startOffset, "Epoch already starts at %s", existing); + // A restarted leader can resume the latest epoch at a later log end. + checkArgument(epoch == epochs.lastKey(), "Epoch must not decrease."); + checkArgument(startOffset >= existing, "Epoch already starts at %s", existing); return; } checkArgument(epochs.isEmpty() || epoch > epochs.lastKey(), "Epoch must increase."); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java index 0fbb601a20b..964416c4ed6 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java @@ -503,8 +503,7 @@ public void makeLeader(NotifyLeaderAndIsrData data) throws IOException { int requestLeaderEpoch = data.getLeaderEpoch(); if (requestLeaderEpoch > leaderEpoch) { - boolean resetFollowerOffsets = - logTablet.isLeaderEpochEnabled() || !isLeader(); + boolean resetFollowerOffsets = !isLeader(); leaderEpoch = requestLeaderEpoch; onBecomeNewLeader(); logTablet.assignLeaderEpoch(leaderEpoch); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/LeaderEpochHistoryTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/LeaderEpochHistoryTest.java index 86fcaaa3661..b5020d4c9b4 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/log/LeaderEpochHistoryTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/log/LeaderEpochHistoryTest.java @@ -54,6 +54,25 @@ void testUnknownPrefixAndSkippedEpochsSurviveRestart() throws Exception { assertThat(history.endOffsetFor(45, 200).get().offset()).isEqualTo(200); } + @Test + void testResumingLatestEpochPreservesItsStartAcrossRestart() throws Exception { + File file = new File(directory, "epochs"); + LeaderEpochHistory history = new LeaderEpochHistory(file); + history.assign(1, 0); + history.assign(2, 10); + history = new LeaderEpochHistory(file); + history.assign(2, 20); + history = new LeaderEpochHistory(file); + assertThat(history.epochForOffset(9, 20)).isEqualTo(1); + assertThat(history.epochForOffset(10, 20)).isEqualTo(2); + assertThat(history.endOffsetFor(1, 20).get().offset()).isEqualTo(10); + LeaderEpochHistory recovered = history; + assertThatThrownBy(() -> recovered.assign(1, 20)) + .isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> recovered.assign(2, 9)) + .isInstanceOf(IllegalArgumentException.class); + } + @Test void testEmptyEpochDoesNotChangeLastRecordEpoch() throws Exception { LeaderEpochHistory history = new LeaderEpochHistory(new File(directory, "epochs")); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/LeaderMigrationWalTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/LeaderMigrationWalTest.java index 3dcf25190ed..4190aaaf68e 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/LeaderMigrationWalTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/LeaderMigrationWalTest.java @@ -454,7 +454,8 @@ private void deliver(int follower, int leader, long offset, boolean legacy) thro servers.get(leader).conf, servers.get(leader).manager, new ServerNode( - follower, "localhost", 10000 + follower, ServerType.TABLET_SERVER)); + follower, "localhost", 10000 + follower, ServerType.TABLET_SERVER), + leader); LeaderEndpoint.FetchData data = endpoint.fetchLog( new FetchLogContext( diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThreadTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThreadTest.java index 8262d080f1f..9dfb856afa4 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThreadTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThreadTest.java @@ -144,7 +144,7 @@ public void setup() throws Exception { ServerNode follower = new ServerNode( followerServerId, "localhost", 10001, ServerType.TABLET_SERVER, "rack2"); - leaderEndpoint = new TestingLeaderEndpoint(conf, leaderRM, follower); + leaderEndpoint = new TestingLeaderEndpoint(conf, leaderRM, follower, leaderServerId); followerFetcher = new ReplicaFetcherThread("test-fetcher-thread", followerRM, leaderEndpoint, 1000); @@ -277,7 +277,7 @@ void testRestoreKvMinRetainOffsetFromDelayedEmptyFetchResponse() throws Exceptio } @Test - void testFollowerHighWatermarkHigherThanOrEqualToLeader() throws Exception { + void testFollowerHighWatermarkConvergesToLeader() throws Exception { Replica leaderReplica = leaderRM.getReplicaOrException(tb); Replica followerReplica = followerRM.getReplicaOrException(tb); @@ -310,7 +310,15 @@ void testFollowerHighWatermarkHigherThanOrEqualToLeader() throws Exception { assertThat(followerReplica.getLocalLogEndOffset()) .isEqualTo(baseOffset + 10L)); assertThat(followerReplica.getLogHighWatermark()) - .isGreaterThanOrEqualTo(leaderReplica.getLogHighWatermark()); + .isLessThanOrEqualTo(leaderReplica.getLogHighWatermark()) + .isLessThanOrEqualTo(followerReplica.getLocalLogEndOffset()); + retry( + Duration.ofSeconds(20), + () -> { + assertThat(leaderReplica.getLogHighWatermark()).isEqualTo(baseOffset + 10L); + assertThat(followerReplica.getLogHighWatermark()) + .isEqualTo(baseOffset + 10L); + }); } } @@ -479,7 +487,7 @@ void testFetchTimeoutReleasesPooledByteBuf() throws Exception { ServerType.TABLET_SERVER, "rack2"); TestingLeaderEndpoint testingEndpoint = - new TestingLeaderEndpoint(conf, leaderRM, followerNode); + new TestingLeaderEndpoint(conf, leaderRM, followerNode, leaderServerId); // Append records to leader so fetch responses carry actual data CompletableFuture> future = new CompletableFuture<>(); @@ -595,6 +603,7 @@ private void makeLeaderAndFollower( private LocalDiskManager createLocalDiskManager(int serverId) throws Exception { Configuration conf = new Configuration(); + conf.set(ConfigOptions.SERVER_DATA_DISK_WRITE_LIMIT_RATIO, 1.0); conf.set(ConfigOptions.TABLET_SERVER_ID, serverId); conf.setString(ConfigOptions.DATA_DIR, tempDir.getAbsolutePath() + "/server-" + serverId); return LocalDiskManager.create(conf); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/TestingLeaderEndpoint.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/TestingLeaderEndpoint.java index ce4681c038f..5c16e2b8d17 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/TestingLeaderEndpoint.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/TestingLeaderEndpoint.java @@ -49,6 +49,7 @@ public class TestingLeaderEndpoint implements LeaderEndpoint { private final ReplicaManager replicaManager; + private final int leaderServerId; private final ServerNode localNode; /** The max size for the fetch response. */ private final int maxFetchSize; @@ -72,8 +73,12 @@ public class TestingLeaderEndpoint implements LeaderEndpoint { new java.util.concurrent.CopyOnWriteArrayList<>(); public TestingLeaderEndpoint( - Configuration conf, ReplicaManager replicaManager, ServerNode localNode) { + Configuration conf, + ReplicaManager replicaManager, + ServerNode localNode, + int leaderServerId) { this.replicaManager = replicaManager; + this.leaderServerId = leaderServerId; this.localNode = localNode; this.maxFetchSize = (int) conf.get(ConfigOptions.LOG_REPLICA_FETCH_MAX_BYTES).getBytes(); this.maxFetchSizeForBucket = @@ -85,7 +90,7 @@ public TestingLeaderEndpoint( @Override public int leaderServerId() { - return localNode.id(); + return leaderServerId; } @Override From 01979bd60f8a98e8ac5824d134b750edd026d92b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E9=B5=BA?= Date: Fri, 11 Sep 2026 10:57:00 +0800 Subject: [PATCH 3/4] [test] Wait for KV high watermark after concurrent lookup inserts --- .../org/apache/fluss/server/replica/ReplicaManagerTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java index 5d95f19c9a5..a551f1a2a08 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java @@ -1266,6 +1266,8 @@ void testConcurrentLookupWithInsertIfNotExistsAutoIncrement() throws Exception { // Values should be 1, 2, 3 (in any order due to concurrency) assertThat(autoIncrementValues).containsExactlyInAnyOrder(1L, 2L, 3L); + // The insert response returns before the async KV flush publishes the high watermark. + waitUntilHighWatermark(tb, 3); // Verify exactly 3 changelog entries were written (one per unique key) FetchLogResultForBucket logResult = fetchLog(tb, 0L); // Only the first upsert for a given primary key generates changelog records. Subsequent From e47151506386c071202eb498c792d1d1dc51b681 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=99=BD=E9=B5=BA?= Date: Sat, 12 Sep 2026 21:19:28 +0800 Subject: [PATCH 4/4] [server] Preserve fatal WAL errors and retry deferred remote cleanup Propagate append IOExceptions to the replica fatal error handler after invalidating epoch history. Retry remote segment cleanup when the high watermark reaches the committed remote boundary. Exercise ISR recovery through a real follower restart without creating inconsistent leader epochs. --- .../org/apache/fluss/server/log/LogTablet.java | 11 +++++++++-- .../apache/fluss/server/log/LogTabletTest.java | 15 +++++++++++++++ .../fluss/server/replica/AdjustIsrITCase.java | 12 +----------- 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java index 14ad0e002c0..caf69113ed6 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LogTablet.java @@ -885,6 +885,14 @@ public void updateLakeMaxTimestamp(long lakeMaxTimestamp) { } private void onHighWatermarkUpdated(long previousHighWatermark, long currentHighWatermark) { + long cleanupToOffset = + remoteLogEndOffset == -1L + ? highestCopiedEndOffset + : Math.min(remoteLogEndOffset, highestCopiedEndOffset); + if (previousHighWatermark < cleanupToOffset && currentHighWatermark >= cleanupToOffset) { + // Remote offsets may arrive before a follower learns the committed high watermark. + deleteSegmentsAlreadyExistsInRemote(); + } if (!isDataLakeEnabled) { return; } @@ -1150,8 +1158,7 @@ private LogAppendInfo append( validRecords); } catch (IOException e) { leaderEpochHistory.markFailed(e); - throw new LogStorageException( - "Failed to append WAL for " + getTableBucket(), e); + throw e; } updateHighWatermarkWithLogEndOffset(); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/LogTabletTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/LogTabletTest.java index a74bb0e752e..48274293bb3 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/log/LogTabletTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/log/LogTabletTest.java @@ -230,6 +230,21 @@ void testRemoteRestoreInstallsWriterStateAndEpochHistory() throws Exception { assertThat(logTablet.localLogEndOffset()).isEqualTo(30); } + @Test + void testRemoteCleanupResumesWhenHighWatermarkCatchesUp() throws Exception { + for (int i = 0; i < 4; i++) { + logTablet.appendAsLeader(genMemoryLogRecordsByObject(DATA1)); + if (i < 3) { + logTablet.roll(Optional.empty()); + } + } + logTablet.updateRemoteLogOffsets(0L, 30L, 30L); + assertThat(logTablet.logSegments()).hasSize(4); + logTablet.updateHighWatermark(30L); + assertThat(logTablet.logSegments()).hasSize(1); + assertThat(logTablet.localLogStartOffset()).isEqualTo(30L); + } + @Test void testRemoteLogOffsetsCanResetAfterEmptyManifest() { logTablet.updateRemoteLogOffsets(0L, 10L, 10L); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrITCase.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrITCase.java index 846a1f5f56c..97f27ccfe49 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrITCase.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrITCase.java @@ -129,18 +129,8 @@ void testIsrShrinkAndExpand() throws Exception { .getHighWatermark()) .isEqualTo(10L)); - currentLeaderAndIsr = zkClient.getLeaderAndIsr(tb).get(); - LeaderAndIsr newLeaderAndIsr = - new LeaderAndIsr( - currentLeaderAndIsr.leader(), - currentLeaderAndIsr.leaderEpoch() + 1, - isr, - currentLeaderAndIsr.standbyReplicas(), - currentLeaderAndIsr.coordinatorEpoch(), - currentLeaderAndIsr.bucketEpoch()); isr.add(stopFollower); - FLUSS_CLUSTER_EXTENSION.notifyLeaderAndIsr( - stopFollower, DATA1_TABLE_PATH, tb, newLeaderAndIsr, isr); + FLUSS_CLUSTER_EXTENSION.restartTabletServer(stopFollower, new Configuration()); // retry until the stop follower add back to ISR. retry( Duration.ofMinutes(1),