levelCompressions = options.get(FILE_COMPRESSION_PER_LEVEL);
return levelCompressions.entrySet().stream()
diff --git a/paimon-core/src/main/java/org/apache/paimon/append/SortCompactCommitMessageRewriter.java b/paimon-core/src/main/java/org/apache/paimon/append/SortCompactCommitMessageRewriter.java
new file mode 100644
index 000000000000..25ec5cabd4ca
--- /dev/null
+++ b/paimon-core/src/main/java/org/apache/paimon/append/SortCompactCommitMessageRewriter.java
@@ -0,0 +1,534 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.append;
+
+import org.apache.paimon.Snapshot;
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.deletionvectors.append.AppendDeleteFileMaintainer;
+import org.apache.paimon.deletionvectors.append.BaseAppendDeleteFileMaintainer;
+import org.apache.paimon.index.IndexFileMeta;
+import org.apache.paimon.io.CompactIncrement;
+import org.apache.paimon.io.DataFileMeta;
+import org.apache.paimon.io.DataIncrement;
+import org.apache.paimon.manifest.FileKind;
+import org.apache.paimon.manifest.IndexManifestEntry;
+import org.apache.paimon.manifest.ManifestEntry;
+import org.apache.paimon.operation.FileStoreScan;
+import org.apache.paimon.table.BucketMode;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.sink.BatchWriteBuilder;
+import org.apache.paimon.table.sink.CommitMessage;
+import org.apache.paimon.table.sink.CommitMessageImpl;
+import org.apache.paimon.table.sink.TableCommit;
+import org.apache.paimon.table.source.DataSplit;
+
+import javax.annotation.Nullable;
+
+import java.io.FileNotFoundException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Rewrites the {@link CommitMessage}s produced by a sort compact write into compact commit
+ * messages, so that the commit is a {@link Snapshot.CommitKind#COMPACT} commit instead of an {@link
+ * Snapshot.CommitKind#OVERWRITE} commit.
+ *
+ * The sort compact write only creates new data files (the sorted output). The old files which
+ * are replaced by the sort compact are captured by the planned input {@link DataSplit}s. This
+ * helper rewrites them into compact changes: all planned old files become {@code compactBefore} for
+ * their original (partition, bucket), and all newly written files become {@code compactAfter} for
+ * their output (partition, bucket). This is important for hash-dynamic bucket sort compact, where
+ * output files may be assigned to different buckets from the planned input files. The result is a
+ * {@link CommitMessageImpl} with an empty {@link DataIncrement} and a populated {@link
+ * CompactIncrement}.
+ *
+ *
For deletion-vector enabled append tables, the deletion-vector index entries of the removed
+ * old files are cleaned up, mirroring {@link AppendCompactTask}.
+ */
+public class SortCompactCommitMessageRewriter {
+
+ private static final String ABORT_COMMIT_USER = "sort-compact-abort";
+
+ private final FileStoreTable table;
+ private final long baseSnapshotId;
+
+ /** Old files grouped by partition then bucket, captured from the planned input splits. */
+ private final Map>> compactBeforeFiles;
+
+ /**
+ * Total bucket counts grouped like {@link #compactBeforeFiles}, when carried by input splits.
+ */
+ private final Map> compactBeforeTotalBuckets;
+
+ /**
+ * Deletion-vector index entries captured from the base snapshot at planning time, grouped by
+ * partition.
+ */
+ private final Map> baseDeletionVectorEntries;
+
+ /**
+ * Hash index files captured from the base snapshot at planning time, grouped by partition then
+ * bucket.
+ */
+ private final Map> baseHashIndexes;
+
+ public SortCompactCommitMessageRewriter(
+ FileStoreTable table, long baseSnapshotId, List compactInputSplits) {
+ this(table, baseSnapshotId, compactInputSplits, null);
+ }
+
+ public SortCompactCommitMessageRewriter(
+ FileStoreTable table,
+ long baseSnapshotId,
+ List compactInputSplits,
+ @Nullable SortCompactPlanMetadata planMetadata) {
+ this.table = table;
+ this.baseSnapshotId = baseSnapshotId;
+ this.compactBeforeFiles = new HashMap<>();
+ this.compactBeforeTotalBuckets = new HashMap<>();
+ this.baseDeletionVectorEntries = new HashMap<>();
+ this.baseHashIndexes = new HashMap<>();
+ Set partitions = new HashSet<>();
+ for (DataSplit split : compactInputSplits) {
+ partitions.add(split.partition());
+ compactBeforeFiles
+ .computeIfAbsent(split.partition(), k -> new HashMap<>())
+ .computeIfAbsent(split.bucket(), k -> new ArrayList<>())
+ .addAll(split.dataFiles());
+ if (split.totalBuckets() != null) {
+ Integer previous =
+ compactBeforeTotalBuckets
+ .computeIfAbsent(split.partition(), k -> new HashMap<>())
+ .putIfAbsent(split.bucket(), split.totalBuckets());
+ if (previous != null && !previous.equals(split.totalBuckets())) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Conflicting total bucket counts for partition %s bucket %s: "
+ + "%s and %s.",
+ split.partition(),
+ split.bucket(),
+ previous,
+ split.totalBuckets()));
+ }
+ }
+ }
+ if (planMetadata != null) {
+ planMetadata.copyInto(baseDeletionVectorEntries, baseHashIndexes);
+ } else {
+ SortCompactPlanMetadata.captureInto(
+ table,
+ baseSnapshotId,
+ partitions,
+ compactInputSplits,
+ baseDeletionVectorEntries,
+ baseHashIndexes);
+ }
+ }
+
+ /**
+ * Rewrite the given written append commit messages into compact commit messages.
+ *
+ * Both the planned input splits and the written messages are grouped by (partition, bucket).
+ * Planned input groups are always emitted, even if the sort compact write produces no files for
+ * that group. Written output groups which were not present in the planned input are emitted as
+ * add-only compact messages.
+ *
+ * @param writtenMessages commit messages produced by the sort compact write stage (only new
+ * files in {@link DataIncrement})
+ * @return rewritten commit messages carrying {@link CompactIncrement}s
+ */
+ public List rewrite(List writtenMessages) {
+ validateWriteOnlyMessages(writtenMessages);
+
+ // group written messages by (partition, bucket)
+ Map>> grouped = new HashMap<>();
+ for (CommitMessage written : writtenMessages) {
+ CommitMessageImpl impl = (CommitMessageImpl) written;
+ grouped.computeIfAbsent(impl.partition(), k -> new HashMap<>())
+ .computeIfAbsent(impl.bucket(), k -> new ArrayList<>())
+ .add(impl);
+ }
+
+ List result = new ArrayList<>();
+ for (Map.Entry>> partitionEntry :
+ compactBeforeFiles.entrySet()) {
+ BinaryRow partition = partitionEntry.getKey();
+ Map> writtenInPartition = grouped.get(partition);
+ for (Integer bucket : partitionEntry.getValue().keySet()) {
+ List group = Collections.emptyList();
+ if (writtenInPartition != null) {
+ List writtenGroup = writtenInPartition.remove(bucket);
+ if (writtenGroup != null) {
+ group = writtenGroup;
+ }
+ }
+ result.add(rewriteGroup(partition, bucket, group));
+ }
+ if (writtenInPartition != null && writtenInPartition.isEmpty()) {
+ grouped.remove(partition);
+ }
+ }
+
+ for (Map.Entry>> partitionEntry :
+ grouped.entrySet()) {
+ BinaryRow partition = partitionEntry.getKey();
+ for (Map.Entry> bucketEntry :
+ partitionEntry.getValue().entrySet()) {
+ result.add(rewriteGroup(partition, bucketEntry.getKey(), bucketEntry.getValue()));
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Validate that the written messages only contain write-only append output. Sort compact must
+ * not run inline compaction in the write stage; otherwise compact output would be dropped and
+ * orphan files would be left on disk.
+ */
+ private void validateWriteOnlyMessages(List writtenMessages) {
+ for (CommitMessage written : writtenMessages) {
+ CommitMessageImpl impl = (CommitMessageImpl) written;
+ CompactIncrement compactIncrement = impl.compactIncrement();
+ if (!compactIncrement.compactBefore().isEmpty()
+ || !compactIncrement.compactAfter().isEmpty()) {
+ abortAndFail(
+ writtenMessages,
+ String.format(
+ "Sort compact write produced inline compaction changes for "
+ + "partition %s bucket %s (compactBefore = %s, "
+ + "compactAfter = %s). The write stage must run in "
+ + "write-only mode without waiting for compaction.",
+ impl.partition(),
+ impl.bucket(),
+ compactIncrement.compactBefore(),
+ compactIncrement.compactAfter()));
+ }
+ }
+ }
+
+ /** Abort newly written files when sort compact rewrite or commit fails. */
+ public void abortWrittenMessages(List writtenMessages) {
+ if (writtenMessages.isEmpty()) {
+ return;
+ }
+ try (TableCommit commit = table.newCommit(ABORT_COMMIT_USER)) {
+ commit.abort(writtenMessages);
+ } catch (Exception e) {
+ throw new IllegalStateException(
+ "Failed to clean up sort compact write output before aborting commit.", e);
+ }
+ }
+
+ private void abortAndFail(List writtenMessages, String message) {
+ abortWrittenMessages(writtenMessages);
+ throw new IllegalStateException(message);
+ }
+
+ private CommitMessage rewriteGroup(
+ BinaryRow partition, int bucket, List group) {
+ List compactBefore = compactBefore(partition, bucket);
+
+ // merge all newly written sorted files of this (partition, bucket) as compact output
+ List compactAfter = new ArrayList<>();
+ List newIndexFiles = new ArrayList<>();
+ List deletedIndexFiles = new ArrayList<>();
+ Integer totalBuckets = null;
+ for (CommitMessageImpl impl : group) {
+ compactAfter.addAll(toCompactAfter(impl.newFilesIncrement().newFiles()));
+ newIndexFiles.addAll(impl.compactIncrement().newIndexFiles());
+ newIndexFiles.addAll(impl.newFilesIncrement().newIndexFiles());
+ deletedIndexFiles.addAll(impl.compactIncrement().deletedIndexFiles());
+ deletedIndexFiles.addAll(impl.newFilesIncrement().deletedIndexFiles());
+ if (totalBuckets == null) {
+ totalBuckets = impl.totalBuckets();
+ }
+ }
+ if (totalBuckets == null) {
+ totalBuckets = compactBeforeTotalBuckets(partition, bucket);
+ }
+
+ // for deletion-vector append tables, clean up the DV index entries of removed old files
+ if (table.coreOptions().deletionVectorsEnabled()
+ && table.bucketMode() == BucketMode.BUCKET_UNAWARE
+ && !compactBefore.isEmpty()) {
+ AppendDeleteFileMaintainer dvMaintainer =
+ BaseAppendDeleteFileMaintainer.forUnawareAppend(
+ table.store().newIndexFileHandler(),
+ partition,
+ baseDeletionVectorEntries.getOrDefault(
+ partition, Collections.emptyList()));
+ for (DataFileMeta oldFile : compactBefore) {
+ dvMaintainer.notifyRemovedDeletionVector(oldFile.fileName());
+ }
+ for (IndexManifestEntry entry : dvMaintainer.persist()) {
+ if (entry.kind() == FileKind.ADD) {
+ newIndexFiles.add(entry.indexFile());
+ } else {
+ deletedIndexFiles.add(entry.indexFile());
+ }
+ }
+ }
+
+ // for hash-dynamic bucket tables, remove the old hash index of compacted buckets so that
+ // subsequent writes do not route keys back to stale buckets
+ if (table.bucketMode() == BucketMode.HASH_DYNAMIC && !compactBefore.isEmpty()) {
+ IndexFileMeta hashIndex =
+ baseHashIndexes.getOrDefault(partition, Collections.emptyMap()).get(bucket);
+ if (hashIndex != null) {
+ deletedIndexFiles.add(hashIndex);
+ }
+ }
+
+ CompactIncrement compactIncrement =
+ new CompactIncrement(
+ compactBefore,
+ compactAfter,
+ Collections.emptyList(),
+ newIndexFiles,
+ deletedIndexFiles);
+ return new CommitMessageImpl(
+ partition, bucket, totalBuckets, DataIncrement.emptyIncrement(), compactIncrement);
+ }
+
+ private List toCompactAfter(List newFiles) {
+ if (newFiles.isEmpty()) {
+ return newFiles;
+ }
+ List result = new ArrayList<>(newFiles.size());
+ for (DataFileMeta newFile : newFiles) {
+ if (newFile.fileSource().orElse(null)
+ == org.apache.paimon.manifest.FileSource.COMPACT) {
+ result.add(newFile);
+ continue;
+ }
+ // The sort compact writer preserves the sequence numbers in the records and computes
+ // the range for each output file. In particular, a hash-dynamic compact output bucket
+ // can contain records from several input buckets, so its range must not be inferred
+ // from the old files of the target bucket.
+ result.add(newFile.assignFileSource(org.apache.paimon.manifest.FileSource.COMPACT));
+ }
+ return result;
+ }
+
+ private List compactBefore(BinaryRow partition, int bucket) {
+ return compactBeforeFiles
+ .getOrDefault(partition, Collections.emptyMap())
+ .getOrDefault(bucket, Collections.emptyList());
+ }
+
+ @Nullable
+ private Integer compactBeforeTotalBuckets(BinaryRow partition, int bucket) {
+ return compactBeforeTotalBuckets
+ .getOrDefault(partition, Collections.emptyMap())
+ .get(bucket);
+ }
+
+ /** Latest snapshot id, or 0 when the table has no snapshot yet. */
+ public long latestSnapshotIdOrZero() {
+ Long latestId = table.snapshotManager().latestSnapshotId();
+ return latestId == null ? 0L : latestId;
+ }
+
+ /**
+ * Whether the given compact commit messages were already committed after {@code
+ * snapshotIdBeforeCommit}.
+ *
+ * Used to avoid aborting sort compact write output when {@link
+ * org.apache.paimon.table.sink.TableCommitImpl} fails after the snapshot is already visible.
+ * Matching is based on the compact output file set (or removed input files for delete-only
+ * commits), not on any unrelated batch COMPACT snapshot.
+ */
+ public boolean isBatchCompactCommitSucceeded(
+ long snapshotIdBeforeCommit, List compactMessages) {
+ Long latestId = table.snapshotManager().latestSnapshotId();
+ if (latestId == null || latestId <= snapshotIdBeforeCommit) {
+ return false;
+ }
+
+ CompactCommitFingerprint fingerprint = CompactCommitFingerprint.from(compactMessages);
+ Snapshot latestSnapshot = table.snapshotManager().snapshot(latestId);
+ if (matchesCompactCommit(latestSnapshot, fingerprint)) {
+ return true;
+ }
+
+ // Check older snapshots in reverse order. The compact commit is usually near latest, and
+ // scoped scans below avoid full-table manifest reads when probing history.
+ for (long id = latestId - 1; id > snapshotIdBeforeCommit; id--) {
+ Snapshot snapshot;
+ try {
+ snapshot = table.snapshotManager().tryGetSnapshot(id);
+ } catch (FileNotFoundException e) {
+ // Expired snapshots create gaps. Keep scanning older snapshots instead of treating
+ // the commit as failed.
+ continue;
+ }
+ if (matchesCompactCommit(snapshot, fingerprint)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean matchesCompactCommit(Snapshot snapshot, CompactCommitFingerprint fingerprint) {
+ if (!fingerprint.compactAfterFileNames.isEmpty()) {
+ // Output file names are unique. Match them in any surviving snapshot so that a
+ // successful compact is still detected after its COMPACT snapshot expires.
+ return snapshotContainsAllFiles(
+ snapshot, fingerprint, fingerprint.compactAfterFileNames);
+ }
+ if (snapshot.commitIdentifier() != BatchWriteBuilder.COMMIT_IDENTIFIER
+ || snapshot.commitKind() != Snapshot.CommitKind.COMPACT) {
+ return false;
+ }
+ if (!fingerprint.compactBeforeFileNames.isEmpty()) {
+ return !snapshotContainsAnyFile(
+ snapshot, fingerprint, fingerprint.compactBeforeFileNames);
+ }
+ return true;
+ }
+
+ private boolean snapshotContainsAllFiles(
+ Snapshot snapshot, CompactCommitFingerprint fingerprint, Set fileNames) {
+ if (fileNames.isEmpty()) {
+ return true;
+ }
+ Set found = new HashSet<>();
+ for (ManifestEntry entry :
+ createScopedScan(snapshot, fingerprint, fileNames).plan().files()) {
+ found.add(entry.file().fileName());
+ if (found.size() == fileNames.size()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean snapshotContainsAnyFile(
+ Snapshot snapshot, CompactCommitFingerprint fingerprint, Set fileNames) {
+ if (fileNames.isEmpty()) {
+ return false;
+ }
+ for (ManifestEntry entry :
+ createScopedScan(snapshot, fingerprint, fileNames).plan().files()) {
+ if (fileNames.contains(entry.file().fileName())) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private FileStoreScan createScopedScan(
+ Snapshot snapshot, CompactCommitFingerprint fingerprint, Set fileNames) {
+ FileStoreScan scan = table.store().newScan().withSnapshot(snapshot).dropStats();
+ if (!fingerprint.partitions.isEmpty()) {
+ scan = scan.withPartitionFilter(fingerprint.partitions);
+ }
+ if (!fingerprint.buckets.isEmpty()) {
+ scan = scan.withBucketFilter(fingerprint.buckets::contains);
+ }
+ if (!fileNames.isEmpty()) {
+ scan = scan.withDataFileNameFilter(fileNames::contains);
+ }
+ return scan;
+ }
+
+ private static final class CompactCommitFingerprint {
+ private final Set compactAfterFileNames;
+ private final Set compactBeforeFileNames;
+ private final List partitions;
+ private final Set buckets;
+
+ private CompactCommitFingerprint(
+ Set compactAfterFileNames,
+ Set compactBeforeFileNames,
+ List partitions,
+ Set buckets) {
+ this.compactAfterFileNames = compactAfterFileNames;
+ this.compactBeforeFileNames = compactBeforeFileNames;
+ this.partitions = partitions;
+ this.buckets = buckets;
+ }
+
+ private static CompactCommitFingerprint from(List compactMessages) {
+ Set compactAfterFileNames = new HashSet<>();
+ Set compactBeforeFileNames = new HashSet<>();
+ Set partitionSet = new HashSet<>();
+ Set buckets = new HashSet<>();
+ for (CommitMessage message : compactMessages) {
+ CommitMessageImpl impl = (CommitMessageImpl) message;
+ partitionSet.add(impl.partition());
+ buckets.add(impl.bucket());
+ for (DataFileMeta file : impl.compactIncrement().compactAfter()) {
+ compactAfterFileNames.add(file.fileName());
+ }
+ for (DataFileMeta file : impl.compactIncrement().compactBefore()) {
+ compactBeforeFileNames.add(file.fileName());
+ }
+ }
+ return new CompactCommitFingerprint(
+ compactAfterFileNames,
+ compactBeforeFileNames,
+ new ArrayList<>(partitionSet),
+ buckets);
+ }
+ }
+
+ /** Whether all planned compact-before files are already absent from the latest snapshot. */
+ public boolean isPlannedInputAlreadyCommitted() {
+ if (!hasInput()) {
+ return true;
+ }
+ Long latestId = table.snapshotManager().latestSnapshotId();
+ if (latestId == null) {
+ return false;
+ }
+ Snapshot snapshot = table.snapshotManager().snapshot(latestId);
+ Set beforeFileNames = new HashSet<>();
+ Set partitionSet = new HashSet<>();
+ Set buckets = new HashSet<>();
+ for (Map.Entry>> partitionEntry :
+ compactBeforeFiles.entrySet()) {
+ partitionSet.add(partitionEntry.getKey());
+ for (Map.Entry> bucketEntry :
+ partitionEntry.getValue().entrySet()) {
+ buckets.add(bucketEntry.getKey());
+ for (DataFileMeta file : bucketEntry.getValue()) {
+ beforeFileNames.add(file.fileName());
+ }
+ }
+ }
+ CompactCommitFingerprint fingerprint =
+ new CompactCommitFingerprint(
+ Collections.emptySet(),
+ beforeFileNames,
+ new ArrayList<>(partitionSet),
+ buckets);
+ return !snapshotContainsAnyFile(snapshot, fingerprint, beforeFileNames);
+ }
+
+ /** Whether this rewriter has captured any old files to compact. */
+ public boolean hasInput() {
+ return !compactBeforeFiles.isEmpty();
+ }
+}
diff --git a/paimon-core/src/main/java/org/apache/paimon/append/SortCompactPlanMetadata.java b/paimon-core/src/main/java/org/apache/paimon/append/SortCompactPlanMetadata.java
new file mode 100644
index 000000000000..6fd137cf448b
--- /dev/null
+++ b/paimon-core/src/main/java/org/apache/paimon/append/SortCompactPlanMetadata.java
@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.append;
+
+import org.apache.paimon.Snapshot;
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.index.IndexFileHandler;
+import org.apache.paimon.index.IndexFileMeta;
+import org.apache.paimon.index.IndexFileMetaSerializer;
+import org.apache.paimon.manifest.IndexManifestEntry;
+import org.apache.paimon.manifest.IndexManifestEntrySerializer;
+import org.apache.paimon.table.BucketMode;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.source.DataSplit;
+import org.apache.paimon.utils.Pair;
+
+import javax.annotation.Nullable;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.io.UncheckedIOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+import static org.apache.paimon.deletionvectors.DeletionVectorsIndexFile.DELETION_VECTORS_INDEX;
+import static org.apache.paimon.index.HashIndexFile.HASH_INDEX;
+import static org.apache.paimon.utils.SerializationUtils.deserializeBinaryRow;
+import static org.apache.paimon.utils.SerializationUtils.serializeBinaryRow;
+
+/**
+ * Base-snapshot index metadata captured at sort compact planning time.
+ *
+ * Flink carries this object in the job graph so commit recovery can still clean up deletion
+ * vectors and hash indexes even if the base snapshot has expired before the committer runs again.
+ * Index metadata is stored as Paimon byte arrays instead of non-{@link Serializable} POJOs.
+ */
+public final class SortCompactPlanMetadata implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ @Nullable private final byte[] serializedDeletionVectorEntries;
+ @Nullable private final List serializedHashIndexes;
+
+ private SortCompactPlanMetadata(
+ @Nullable byte[] serializedDeletionVectorEntries,
+ @Nullable List serializedHashIndexes) {
+ this.serializedDeletionVectorEntries = serializedDeletionVectorEntries;
+ this.serializedHashIndexes = serializedHashIndexes;
+ }
+
+ /** Capture index metadata from the base snapshot for the planned compact input. */
+ public static SortCompactPlanMetadata capture(
+ FileStoreTable table, long baseSnapshotId, List compactInputSplits) {
+ Set partitions = new HashSet<>();
+ for (DataSplit split : compactInputSplits) {
+ partitions.add(split.partition());
+ }
+
+ Map> baseDeletionVectorEntries = new HashMap<>();
+ Map> baseHashIndexes = new HashMap<>();
+ captureInto(
+ table,
+ baseSnapshotId,
+ partitions,
+ compactInputSplits,
+ baseDeletionVectorEntries,
+ baseHashIndexes);
+ return fromCapturedMaps(baseDeletionVectorEntries, baseHashIndexes);
+ }
+
+ static void captureInto(
+ FileStoreTable table,
+ long baseSnapshotId,
+ Set partitions,
+ List compactInputSplits,
+ Map> baseDeletionVectorEntries,
+ Map> baseHashIndexes) {
+ Snapshot snapshot = resolveBaseSnapshot(table, baseSnapshotId);
+ if (snapshot == null) {
+ return;
+ }
+
+ IndexFileHandler indexFileHandler = table.store().newIndexFileHandler();
+ if (table.coreOptions().deletionVectorsEnabled()
+ && table.bucketMode() == BucketMode.BUCKET_UNAWARE) {
+ for (IndexManifestEntry entry :
+ indexFileHandler.scan(snapshot, DELETION_VECTORS_INDEX)) {
+ if (partitions.contains(entry.partition())) {
+ baseDeletionVectorEntries
+ .computeIfAbsent(entry.partition(), k -> new ArrayList<>())
+ .add(entry);
+ }
+ }
+ }
+
+ if (table.bucketMode() == BucketMode.HASH_DYNAMIC) {
+ Set> buckets = new HashSet<>();
+ for (DataSplit split : compactInputSplits) {
+ buckets.add(Pair.of(split.partition(), split.bucket()));
+ }
+ Map, List> hashIndexes =
+ indexFileHandler.scanBuckets(snapshot, HASH_INDEX, buckets);
+ for (Map.Entry, List> entry :
+ hashIndexes.entrySet()) {
+ singleHashIndex(entry.getValue())
+ .ifPresent(
+ hashIndex ->
+ baseHashIndexes
+ .computeIfAbsent(
+ entry.getKey().getLeft(),
+ k -> new HashMap<>())
+ .put(entry.getKey().getRight(), hashIndex));
+ }
+ }
+ }
+
+ private static Optional singleHashIndex(
+ @Nullable List hashIndexes) {
+ if (hashIndexes == null || hashIndexes.isEmpty()) {
+ return Optional.empty();
+ }
+ if (hashIndexes.size() > 1) {
+ throw new IllegalArgumentException(
+ "Find multiple hash index files for one bucket: " + hashIndexes);
+ }
+ return Optional.of(hashIndexes.get(0));
+ }
+
+ void copyInto(
+ Map> baseDeletionVectorEntries,
+ Map> baseHashIndexes) {
+ if (serializedDeletionVectorEntries != null) {
+ IndexManifestEntrySerializer entrySerializer = new IndexManifestEntrySerializer();
+ try {
+ for (IndexManifestEntry entry :
+ entrySerializer.deserializeList(serializedDeletionVectorEntries)) {
+ baseDeletionVectorEntries
+ .computeIfAbsent(entry.partition(), k -> new ArrayList<>())
+ .add(entry);
+ }
+ } catch (IOException e) {
+ throw new UncheckedIOException(
+ "Failed to deserialize captured deletion vector metadata.", e);
+ }
+ }
+
+ if (serializedHashIndexes != null) {
+ IndexFileMetaSerializer metaSerializer = new IndexFileMetaSerializer();
+ for (SerializedHashIndex serializedHashIndex : serializedHashIndexes) {
+ try {
+ BinaryRow partition = deserializeBinaryRow(serializedHashIndex.partition);
+ IndexFileMeta hashIndex =
+ metaSerializer.deserializeFromBytes(serializedHashIndex.indexFileMeta);
+ baseHashIndexes
+ .computeIfAbsent(partition, k -> new HashMap<>())
+ .put(serializedHashIndex.bucket, hashIndex);
+ } catch (IOException e) {
+ throw new UncheckedIOException(
+ "Failed to deserialize captured hash index metadata.", e);
+ }
+ }
+ }
+ }
+
+ private static SortCompactPlanMetadata fromCapturedMaps(
+ Map> baseDeletionVectorEntries,
+ Map> baseHashIndexes) {
+ byte[] serializedDeletionVectorEntries = null;
+ if (!baseDeletionVectorEntries.isEmpty()) {
+ List entries = new ArrayList<>();
+ for (List partitionEntries : baseDeletionVectorEntries.values()) {
+ entries.addAll(partitionEntries);
+ }
+ IndexManifestEntrySerializer entrySerializer = new IndexManifestEntrySerializer();
+ try {
+ serializedDeletionVectorEntries = entrySerializer.serializeList(entries);
+ } catch (IOException e) {
+ throw new UncheckedIOException(
+ "Failed to serialize captured deletion vector metadata.", e);
+ }
+ }
+
+ List serializedHashIndexes = null;
+ if (!baseHashIndexes.isEmpty()) {
+ IndexFileMetaSerializer metaSerializer = new IndexFileMetaSerializer();
+ serializedHashIndexes = new ArrayList<>();
+ for (Map.Entry> partitionEntry :
+ baseHashIndexes.entrySet()) {
+ byte[] partitionBytes = serializeBinaryRow(partitionEntry.getKey());
+ for (Map.Entry bucketEntry :
+ partitionEntry.getValue().entrySet()) {
+ try {
+ serializedHashIndexes.add(
+ new SerializedHashIndex(
+ partitionBytes,
+ bucketEntry.getKey(),
+ metaSerializer.serializeToBytes(bucketEntry.getValue())));
+ } catch (IOException e) {
+ throw new UncheckedIOException(
+ "Failed to serialize captured hash index metadata.", e);
+ }
+ }
+ }
+ }
+
+ return new SortCompactPlanMetadata(serializedDeletionVectorEntries, serializedHashIndexes);
+ }
+
+ @Nullable
+ private static Snapshot resolveBaseSnapshot(FileStoreTable table, long snapshotId) {
+ if (snapshotId <= 0) {
+ return table.snapshotManager().latestSnapshot();
+ }
+ try {
+ return table.snapshotManager().tryGetSnapshot(snapshotId);
+ } catch (java.io.FileNotFoundException e) {
+ return null;
+ }
+ }
+
+ private static final class SerializedHashIndex implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private final byte[] partition;
+ private final int bucket;
+ private final byte[] indexFileMeta;
+
+ private SerializedHashIndex(byte[] partition, int bucket, byte[] indexFileMeta) {
+ this.partition = partition;
+ this.bucket = bucket;
+ this.indexFileMeta = indexFileMeta;
+ }
+ }
+}
diff --git a/paimon-core/src/main/java/org/apache/paimon/append/SortCompactSequenceUtils.java b/paimon-core/src/main/java/org/apache/paimon/append/SortCompactSequenceUtils.java
new file mode 100644
index 000000000000..7b46a90b2946
--- /dev/null
+++ b/paimon-core/src/main/java/org/apache/paimon/append/SortCompactSequenceUtils.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.append;
+
+import org.apache.paimon.KeyValue;
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.data.JoinedRow;
+import org.apache.paimon.io.DataFileMeta;
+import org.apache.paimon.manifest.FileSource;
+import org.apache.paimon.table.SpecialFields;
+import org.apache.paimon.table.sink.TableWriteImpl;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.OffsetRow;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/** Utilities for preserving key-value sequence numbers during sort compact. */
+public class SortCompactSequenceUtils {
+
+ private SortCompactSequenceUtils() {}
+
+ /**
+ * Row type with {@link SpecialFields#SEQUENCE_NUMBER} prepended for sort compact read/write.
+ */
+ public static RowType rowTypeWithKeyValueSequenceNumber(RowType rowType) {
+ List fields = new ArrayList<>(rowType.getFieldCount() + 1);
+ fields.add(SpecialFields.SEQUENCE_NUMBER);
+ fields.addAll(rowType.getFields());
+ return new RowType(fields);
+ }
+
+ public static boolean needsSequencePreservation(RowType logicalRowType, InternalRow row) {
+ return sequenceNumber(row, logicalRowType.getFieldCount()) != KeyValue.UNKNOWN_SEQUENCE;
+ }
+
+ public static long sequenceNumber(InternalRow row, int logicalFieldCount) {
+ if (row instanceof JoinedRow) {
+ JoinedRow joinedRow = (JoinedRow) row;
+ return joinedRow.row1().getLong(0);
+ }
+ if (row.getFieldCount() == logicalFieldCount + 1) {
+ return row.getLong(0);
+ }
+ return KeyValue.UNKNOWN_SEQUENCE;
+ }
+
+ public static InternalRow valueRow(InternalRow row, int logicalFieldCount) {
+ if (row instanceof JoinedRow) {
+ return ((JoinedRow) row).row2();
+ }
+ if (row.getFieldCount() == logicalFieldCount + 1) {
+ return new OffsetRow(logicalFieldCount, 1).replace(row);
+ }
+ return row;
+ }
+
+ public static TableWriteImpl.RecordExtractor sequencePreservingExtractor(
+ RowType logicalRowType, KeyValue reuse) {
+ int logicalFieldCount = logicalRowType.getFieldCount();
+ return (record, rowKind) -> {
+ InternalRow row = record.row();
+ long sequenceNumber = sequenceNumber(row, logicalFieldCount);
+ InternalRow valueRow = valueRow(row, logicalFieldCount);
+ return reuse.replace(record.primaryKey(), sequenceNumber, rowKind, valueRow);
+ };
+ }
+
+ public static DataFileMeta asCompactOutput(
+ DataFileMeta file, long minSequenceNumber, long maxSequenceNumber) {
+ return file.assignSequenceNumber(minSequenceNumber, maxSequenceNumber)
+ .assignFileSource(FileSource.COMPACT);
+ }
+
+ public static long minSequenceNumber(List files) {
+ return files.stream()
+ .mapToLong(DataFileMeta::minSequenceNumber)
+ .min()
+ .orElseThrow(
+ () ->
+ new IllegalStateException(
+ "Cannot get min sequence number from compact before files."));
+ }
+
+ public static long maxSequenceNumber(List files) {
+ return files.stream()
+ .mapToLong(DataFileMeta::maxSequenceNumber)
+ .max()
+ .orElseThrow(
+ () ->
+ new IllegalStateException(
+ "Cannot get max sequence number from compact before files."));
+ }
+}
diff --git a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/append/BaseAppendDeleteFileMaintainer.java b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/append/BaseAppendDeleteFileMaintainer.java
index b61985a17704..343b4900eb86 100644
--- a/paimon-core/src/main/java/org/apache/paimon/deletionvectors/append/BaseAppendDeleteFileMaintainer.java
+++ b/paimon-core/src/main/java/org/apache/paimon/deletionvectors/append/BaseAppendDeleteFileMaintainer.java
@@ -80,6 +80,13 @@ static AppendDeleteFileMaintainer forUnawareAppend(
indexFileHandler.scan(snapshot, DELETION_VECTORS_INDEX).stream()
.filter(e -> e.partition().equals(partition))
.collect(Collectors.toList());
+ return forUnawareAppend(indexFileHandler, partition, manifestEntries);
+ }
+
+ static AppendDeleteFileMaintainer forUnawareAppend(
+ IndexFileHandler indexFileHandler,
+ BinaryRow partition,
+ List manifestEntries) {
Map deletionFiles = new HashMap<>();
for (IndexManifestEntry file : manifestEntries) {
LinkedHashMap dvMetas = file.indexFile().dvRanges();
diff --git a/paimon-core/src/main/java/org/apache/paimon/io/DataFileMeta.java b/paimon-core/src/main/java/org/apache/paimon/io/DataFileMeta.java
index a8cdc031e134..e9649b6609e2 100644
--- a/paimon-core/src/main/java/org/apache/paimon/io/DataFileMeta.java
+++ b/paimon-core/src/main/java/org/apache/paimon/io/DataFileMeta.java
@@ -334,6 +334,17 @@ default Range nonNullRowIdRange() {
DataFileMeta assignSequenceNumber(long minSequenceNumber, long maxSequenceNumber);
+ default DataFileMeta assignFileSource(FileSource fileSource) {
+ Optional current = fileSource();
+ if (current.isPresent() && current.get() == fileSource) {
+ return this;
+ }
+ throw new UnsupportedOperationException(
+ String.format(
+ "Cannot assign file source %s to DataFileMeta '%s'.",
+ fileSource, fileName()));
+ }
+
DataFileMeta assignFirstRowId(long firstRowId);
DataFileMeta newFirstRowId(@Nullable Long newFirstRowId);
diff --git a/paimon-core/src/main/java/org/apache/paimon/io/PojoDataFileMeta.java b/paimon-core/src/main/java/org/apache/paimon/io/PojoDataFileMeta.java
index 9e845b26fe14..bb076f1497bb 100644
--- a/paimon-core/src/main/java/org/apache/paimon/io/PojoDataFileMeta.java
+++ b/paimon-core/src/main/java/org/apache/paimon/io/PojoDataFileMeta.java
@@ -354,6 +354,31 @@ public PojoDataFileMeta assignSequenceNumber(long minSequenceNumber, long maxSeq
writeCols);
}
+ @Override
+ public PojoDataFileMeta assignFileSource(FileSource fileSource) {
+ return new PojoDataFileMeta(
+ fileName,
+ fileSize,
+ rowCount,
+ minKey,
+ maxKey,
+ keyStats,
+ valueStats,
+ minSequenceNumber,
+ maxSequenceNumber,
+ schemaId,
+ level,
+ extraFiles,
+ creationTime,
+ deleteRowCount,
+ embeddedIndex,
+ fileSource,
+ valueStatsCols,
+ externalPath,
+ firstRowId,
+ writeCols);
+ }
+
@Override
public PojoDataFileMeta assignFirstRowId(long firstRowId) {
return new PojoDataFileMeta(
diff --git a/paimon-core/src/main/java/org/apache/paimon/mergetree/MergeTreeWriter.java b/paimon-core/src/main/java/org/apache/paimon/mergetree/MergeTreeWriter.java
index beb2651f1f5f..fce9c2814952 100644
--- a/paimon-core/src/main/java/org/apache/paimon/mergetree/MergeTreeWriter.java
+++ b/paimon-core/src/main/java/org/apache/paimon/mergetree/MergeTreeWriter.java
@@ -85,6 +85,9 @@ public class MergeTreeWriter implements RecordWriter, MemoryOwner {
private long newSequenceNumber;
private WriteBuffer writeBuffer;
+ private boolean preserveInputSequence;
+ private FileSource dataFileSource = FileSource.APPEND;
+
public MergeTreeWriter(
boolean writeBufferSpillable,
MemorySize maxDiskSize,
@@ -145,6 +148,14 @@ public CompactManager compactManager() {
return compactManager;
}
+ public void withPreserveInputSequence(boolean preserveInputSequence) {
+ this.preserveInputSequence = preserveInputSequence;
+ }
+
+ public void withDataFileSource(FileSource dataFileSource) {
+ this.dataFileSource = dataFileSource;
+ }
+
@Override
public void setMemoryPool(MemorySegmentPool memoryPool) {
this.writeBuffer =
@@ -162,7 +173,10 @@ public void setMemoryPool(MemorySegmentPool memoryPool) {
@Override
public void write(KeyValue kv) throws Exception {
- long sequenceNumber = newSequenceNumber();
+ long sequenceNumber =
+ preserveInputSequence && kv.sequenceNumber() != KeyValue.UNKNOWN_SEQUENCE
+ ? kv.sequenceNumber()
+ : newSequenceNumber();
boolean success = writeBuffer.put(sequenceNumber, kv.valueKind(), kv.key(), kv.value());
if (!success) {
flushWriteBuffer(false, false);
@@ -218,7 +232,7 @@ private void flushWriteBuffer(boolean waitForLatestCompaction, boolean forcedFul
? writerFactory.createRollingChangelogFileWriter(0)
: null;
final RollingFileWriter dataWriter =
- writerFactory.createRollingMergeTreeFileWriter(0, FileSource.APPEND);
+ writerFactory.createRollingMergeTreeFileWriter(0, dataFileSource);
try {
writeBuffer.forEach(
diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreWrite.java b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreWrite.java
index 5875840053ff..e1b1723be23d 100644
--- a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreWrite.java
+++ b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreWrite.java
@@ -59,6 +59,15 @@ default void withWriteType(RowType writeType) {
throw new UnsupportedOperationException();
}
+ /**
+ * Enable sort-compact write mode for primary-key tables with snapshot sequence ordering. Output
+ * files are written as {@code COMPACT} and preserve per-record sequence numbers from input
+ * rows.
+ */
+ default FileStoreWrite withSortCompactWrite(boolean sortCompactWrite) {
+ return this;
+ }
+
/**
* With memory pool factory for the current file store write.
*
diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/KeyValueFileStoreWrite.java b/paimon-core/src/main/java/org/apache/paimon/operation/KeyValueFileStoreWrite.java
index df2faf2dc800..5b6e87dfe387 100644
--- a/paimon-core/src/main/java/org/apache/paimon/operation/KeyValueFileStoreWrite.java
+++ b/paimon-core/src/main/java/org/apache/paimon/operation/KeyValueFileStoreWrite.java
@@ -36,6 +36,7 @@
import org.apache.paimon.io.KeyValueFileReaderFactory;
import org.apache.paimon.io.KeyValueFileWriterFactory;
import org.apache.paimon.io.RecordLevelExpire;
+import org.apache.paimon.manifest.FileSource;
import org.apache.paimon.mergetree.MergeTreeWriter;
import org.apache.paimon.mergetree.compact.KvCompactionManagerFactory;
import org.apache.paimon.mergetree.compact.LookupMergeFunction;
@@ -79,6 +80,8 @@ public class KeyValueFileStoreWrite extends MemoryFileStoreWrite {
private final String commitUser;
private final KvCompactionManagerFactory compactManagerFactory;
+ private boolean sortCompactWrite;
+
public KeyValueFileStoreWrite(
FileIO fileIO,
SchemaManager schemaManager,
@@ -180,6 +183,12 @@ protected boolean ignorePreviousFilesForWriter(
return ignorePreviousFiles;
}
+ @Override
+ public KeyValueFileStoreWrite withSortCompactWrite(boolean sortCompactWrite) {
+ this.sortCompactWrite = sortCompactWrite;
+ return this;
+ }
+
@Override
public KeyValueFileStoreWrite withIOManager(IOManager ioManager) {
super.withIOManager(ioManager);
@@ -227,21 +236,35 @@ protected MergeTreeWriter createWriter(
dvMaintainer,
ignorePreviousFiles);
- return new MergeTreeWriter(
- options.writeBufferSpillable(),
- options.writeBufferSpillDiskSize(),
- options.localSortMaxNumFileHandles(),
- options.spillCompressOptions(),
- ioManager,
- compactManager,
- restoredMaxSeqNumber,
- keyComparator,
- mfFactory.create(),
- writerFactory,
- options.commitForceCompact(),
- options.changelogProducer(),
- restoreIncrement,
- UserDefinedSeqComparator.create(valueType, options));
+ MergeTreeWriter writer =
+ new MergeTreeWriter(
+ options.writeBufferSpillable(),
+ options.writeBufferSpillDiskSize(),
+ options.localSortMaxNumFileHandles(),
+ options.spillCompressOptions(),
+ ioManager,
+ compactManager,
+ restoredMaxSeqNumber,
+ keyComparator,
+ mfFactory.create(),
+ writerFactory,
+ options.commitForceCompact(),
+ // Sort compact only rewrites (re-sorts) existing data, so it must not emit
+ // per-row input changelog. Those write-stage changelog files would be
+ // spurious (existing rows re-emitted as inserts) and are dropped by
+ // SortCompactCommitMessageRewriter, leaking on disk. Full-compaction
+ // changelog is produced by the compact manager, not the input changelog
+ // writer, so it is unaffected by forcing NONE here.
+ sortCompactWrite
+ ? CoreOptions.ChangelogProducer.NONE
+ : options.changelogProducer(),
+ restoreIncrement,
+ UserDefinedSeqComparator.create(valueType, options));
+ if (sortCompactWrite) {
+ writer.withPreserveInputSequence(true);
+ writer.withDataFileSource(FileSource.COMPACT);
+ }
+ return writer;
}
@Override
diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/MergeFileSplitRead.java b/paimon-core/src/main/java/org/apache/paimon/operation/MergeFileSplitRead.java
index 689117f1d22c..de9faf3c4d36 100644
--- a/paimon-core/src/main/java/org/apache/paimon/operation/MergeFileSplitRead.java
+++ b/paimon-core/src/main/java/org/apache/paimon/operation/MergeFileSplitRead.java
@@ -45,9 +45,11 @@
import org.apache.paimon.reader.RecordReader;
import org.apache.paimon.schema.TableSchema;
import org.apache.paimon.table.BucketMode;
+import org.apache.paimon.table.SpecialFields;
import org.apache.paimon.table.source.ChainSplit;
import org.apache.paimon.table.source.DataSplit;
import org.apache.paimon.table.source.DeletionFile;
+import org.apache.paimon.table.source.KeyValueTableRead;
import org.apache.paimon.table.source.Split;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.RowType;
@@ -85,6 +87,7 @@ public class MergeFileSplitRead implements SplitRead {
@Nullable private RowType readKeyType;
@Nullable private RowType outerReadType;
+ @Nullable private RowType pushedReadType;
@Nullable private List filtersForKeys;
@Nullable private List filtersForAll;
@@ -131,12 +134,13 @@ public MergeFileSplitRead withReadKeyType(RowType readKeyType) {
@Override
public MergeFileSplitRead withReadType(RowType readType) {
+ pushedReadType = readType;
RowType tableRowType = tableSchema.logicalRowType();
- RowType adjustedReadType = readType;
+ RowType adjustedReadType = stripKeyValueSequenceField(readType);
if (!sequenceFields.isEmpty()) {
// make sure actual readType contains sequence fields
- List readFieldNames = readType.getFieldNames();
+ List readFieldNames = adjustedReadType.getFieldNames();
List extraFields = new ArrayList<>();
for (String seqField : sequenceFields) {
if (!readFieldNames.contains(seqField)) {
@@ -144,7 +148,7 @@ public MergeFileSplitRead withReadType(RowType readType) {
}
}
if (!extraFields.isEmpty()) {
- List allFields = new ArrayList<>(readType.getFields());
+ List allFields = new ArrayList<>(adjustedReadType.getFields());
allFields.addAll(extraFields);
adjustedReadType = new RowType(allFields);
}
@@ -154,14 +158,32 @@ public MergeFileSplitRead withReadType(RowType readType) {
readerFactoryBuilder.withReadValueType(adjustedReadType);
mergeSorter.setProjectedValueType(adjustedReadType);
- // When finalReadType != readType, need to project the outer read type
- if (adjustedReadType != readType) {
- outerReadType = readType;
+ // KV _SEQUENCE_NUMBER is materialized at unwrap time, not from data files.
+ if (adjustedReadType != readType
+ && !isKeyValueSequenceOnlyDifference(readType, adjustedReadType)) {
+ outerReadType = stripKeyValueSequenceField(readType);
}
return this;
}
+ private static RowType stripKeyValueSequenceField(RowType readType) {
+ List fieldNames = readType.getFieldNames();
+ if (!fieldNames.contains(SpecialFields.SEQUENCE_NUMBER.name())) {
+ return readType;
+ }
+ return readType.project(
+ fieldNames.stream()
+ .filter(name -> !SpecialFields.SEQUENCE_NUMBER.name().equals(name))
+ .collect(Collectors.toList()));
+ }
+
+ private static boolean isKeyValueSequenceOnlyDifference(
+ RowType readType, RowType strippedReadType) {
+ return readType.getFieldCount() == strippedReadType.getFieldCount() + 1
+ && readType.getFieldNames().contains(SpecialFields.SEQUENCE_NUMBER.name());
+ }
+
@Override
public MergeFileSplitRead withIOManager(IOManager ioManager) {
this.mergeSorter.setIOManager(ioManager);
@@ -337,6 +359,16 @@ public RecordReader createNoMergeReader(
return projectOuter(ConcatRecordReader.create(suppliers));
}
+ @Nullable
+ public RowType outerReadType() {
+ return outerReadType;
+ }
+
+ public boolean keyValueSequenceNumberEnabled() {
+ return KeyValueTableRead.keyValueSequenceNumberEnabled(
+ tableSchema.options(), pushedReadType);
+ }
+
/**
* Returns the pushed read type if {@link #withReadType(RowType)} was called, else the default
* read type.
diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java b/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java
index a679b9d0914d..3b5bf3c7d906 100644
--- a/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java
+++ b/paimon-core/src/main/java/org/apache/paimon/operation/commit/ConflictDetection.java
@@ -66,6 +66,7 @@
import static org.apache.paimon.deletionvectors.DeletionVectorsIndexFile.DELETION_VECTORS_INDEX;
import static org.apache.paimon.format.blob.BlobFileFormat.isBlobFile;
+import static org.apache.paimon.index.HashIndexFile.HASH_INDEX;
import static org.apache.paimon.operation.commit.ManifestEntryChanges.changedPartitions;
import static org.apache.paimon.types.VectorType.isVectorStoreFile;
import static org.apache.paimon.utils.InternalRowPartitionComputer.partToSimpleString;
@@ -246,6 +247,18 @@ public Optional checkConflicts(
return exception;
}
+ exception =
+ checkHashIndexConflicts(
+ latestSnapshot,
+ baseEntries,
+ deltaEntries,
+ deltaIndexEntries,
+ baseCommitUser,
+ commitKind);
+ if (exception.isPresent()) {
+ return exception;
+ }
+
return checkForRowIdFromSnapshot(
latestSnapshot, deltaEntries, deltaIndexEntries, rowIdColumnConflictChecker);
}
@@ -687,6 +700,129 @@ private List globalIndexFileAdditions(
return result;
}
+ private Optional checkHashIndexConflicts(
+ Snapshot latestSnapshot,
+ List baseEntries,
+ List deltaEntries,
+ List deltaIndexEntries,
+ String baseCommitUser,
+ CommitKind commitKind) {
+ if (commitKind != CommitKind.COMPACT
+ || bucketMode != BucketMode.HASH_DYNAMIC
+ || indexFileHandler == null) {
+ return Optional.empty();
+ }
+
+ // Buckets whose hash index is being removed by this compact. For these buckets the compact
+ // rewrites an existing bucket, so the old hash index (captured from the base snapshot) is
+ // deleted and a fresh one is added. The DELETE check below guards these buckets; the ADD
+ // check must skip them to avoid false conflicts against the still-present old hash index.
+ Set> deletedHashIndexBuckets = new HashSet<>();
+ for (IndexManifestEntry entry : deltaIndexEntries) {
+ if (entry.kind() == FileKind.DELETE
+ && HASH_INDEX.equals(entry.indexFile().indexType())) {
+ deletedHashIndexBuckets.add(Pair.of(entry.partition(), entry.bucket()));
+ }
+ }
+
+ Set> bucketsToCheck = new HashSet<>();
+ for (IndexManifestEntry entry : deltaIndexEntries) {
+ if (!HASH_INDEX.equals(entry.indexFile().indexType())) {
+ continue;
+ }
+
+ Pair bucketKey = Pair.of(entry.partition(), entry.bucket());
+ if (entry.kind() == FileKind.DELETE) {
+ bucketsToCheck.add(bucketKey);
+ } else if (entry.kind() == FileKind.ADD
+ && !deletedHashIndexBuckets.contains(bucketKey)) {
+ bucketsToCheck.add(bucketKey);
+ }
+ }
+
+ Map, List> currentHashIndexes =
+ indexFileHandler.scanBuckets(latestSnapshot, HASH_INDEX, bucketsToCheck);
+
+ for (IndexManifestEntry entry : deltaIndexEntries) {
+ if (!HASH_INDEX.equals(entry.indexFile().indexType())) {
+ continue;
+ }
+
+ BinaryRow partition = entry.partition();
+ int bucket = entry.bucket();
+ Pair bucketKey = Pair.of(partition, bucket);
+ if (entry.kind() == FileKind.DELETE) {
+ // The compact rewrites a bucket which already existed at plan time. Its old hash
+ // index was captured from the base snapshot and is being removed. If a concurrent
+ // append has replaced it since the compact was planned, give up committing.
+ Optional currentHashIndex =
+ singleHashIndex(currentHashIndexes.get(bucketKey));
+ if (currentHashIndex.isPresent()
+ && !currentHashIndex
+ .get()
+ .fileName()
+ .equals(entry.indexFile().fileName())) {
+ return hashIndexConflict(
+ partition, bucket, baseCommitUser, baseEntries, deltaEntries);
+ }
+ } else if (entry.kind() == FileKind.ADD
+ && !deletedHashIndexBuckets.contains(bucketKey)) {
+ // The compact writes a fresh hash index for an output bucket which did not exist at
+ // plan time (no compact-before, so no DELETE). The BucketedCombiner replaces hash
+ // index entries by bucket, so if a concurrent append has already created this
+ // bucket and its hash index, this ADD would overwrite that index while leaving the
+ // append's data files visible, corrupting later upsert routing. Give up committing.
+ Optional currentHashIndex =
+ singleHashIndex(currentHashIndexes.get(bucketKey));
+ if (currentHashIndex.isPresent()
+ && !currentHashIndex
+ .get()
+ .fileName()
+ .equals(entry.indexFile().fileName())) {
+ return hashIndexConflict(
+ partition, bucket, baseCommitUser, baseEntries, deltaEntries);
+ }
+ }
+ }
+ return Optional.empty();
+ }
+
+ private static Optional singleHashIndex(
+ @Nullable List hashIndexes) {
+ if (hashIndexes == null || hashIndexes.isEmpty()) {
+ return Optional.empty();
+ }
+ if (hashIndexes.size() > 1) {
+ throw new IllegalArgumentException(
+ "Find multiple hash index files for one bucket: " + hashIndexes);
+ }
+ return Optional.of(hashIndexes.get(0));
+ }
+
+ private Optional hashIndexConflict(
+ BinaryRow partition,
+ int bucket,
+ String baseCommitUser,
+ List baseEntries,
+ List deltaEntries) {
+ String partitionInfo = partToSimpleString(partitionType, partition, "-", 200);
+ Pair conflictException =
+ createConflictException(
+ "Hash index conflict detected for sort compact! "
+ + "The hash index of partition "
+ + partitionInfo
+ + " bucket "
+ + bucket
+ + " has been updated by a concurrent append since the "
+ + "compact was planned. Give up committing.",
+ baseCommitUser,
+ baseEntries,
+ deltaEntries,
+ null);
+ LOG.warn("", conflictException.getLeft());
+ return Optional.of(conflictException.getRight());
+ }
+
Optional checkRowIdExistence(
List baseEntries,
List deltaEntries,
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/sink/SequencePreservingPartitionKeyExtractor.java b/paimon-core/src/main/java/org/apache/paimon/table/sink/SequencePreservingPartitionKeyExtractor.java
new file mode 100644
index 000000000000..e7e4eb417d30
--- /dev/null
+++ b/paimon-core/src/main/java/org/apache/paimon/table/sink/SequencePreservingPartitionKeyExtractor.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table.sink;
+
+import org.apache.paimon.append.SortCompactSequenceUtils;
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.schema.TableSchema;
+
+/**
+ * {@link PartitionKeyExtractor} that ignores a leading {@code _SEQUENCE_NUMBER} field when present.
+ */
+public class SequencePreservingPartitionKeyExtractor implements PartitionKeyExtractor {
+
+ private final int logicalFieldCount;
+ private final RowPartitionKeyExtractor partitionKeyExtractor;
+
+ public SequencePreservingPartitionKeyExtractor(TableSchema schema) {
+ this.logicalFieldCount = schema.logicalRowType().getFieldCount();
+ this.partitionKeyExtractor = new RowPartitionKeyExtractor(schema);
+ }
+
+ @Override
+ public BinaryRow partition(InternalRow record) {
+ return partitionKeyExtractor.partition(dataRow(record));
+ }
+
+ @Override
+ public BinaryRow trimmedPrimaryKey(InternalRow record) {
+ return partitionKeyExtractor.trimmedPrimaryKey(dataRow(record));
+ }
+
+ private InternalRow dataRow(InternalRow record) {
+ return SortCompactSequenceUtils.valueRow(record, logicalFieldCount);
+ }
+}
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/sink/SequencePreservingRowKeyExtractor.java b/paimon-core/src/main/java/org/apache/paimon/table/sink/SequencePreservingRowKeyExtractor.java
new file mode 100644
index 000000000000..3a2c9dd5d0c6
--- /dev/null
+++ b/paimon-core/src/main/java/org/apache/paimon/table/sink/SequencePreservingRowKeyExtractor.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table.sink;
+
+import org.apache.paimon.append.SortCompactSequenceUtils;
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.schema.TableSchema;
+
+/** {@link RowKeyExtractor} that ignores a leading {@code _SEQUENCE_NUMBER} field when present. */
+public class SequencePreservingRowKeyExtractor extends DynamicBucketRowKeyExtractor {
+
+ private final int logicalFieldCount;
+ private final RowPartitionKeyExtractor partitionKeyExtractor;
+
+ public SequencePreservingRowKeyExtractor(TableSchema schema) {
+ super(schema);
+ this.logicalFieldCount = schema.logicalRowType().getFieldCount();
+ this.partitionKeyExtractor = new RowPartitionKeyExtractor(schema);
+ }
+
+ @Override
+ public BinaryRow partition() {
+ return partitionKeyExtractor.partition(dataRow());
+ }
+
+ @Override
+ public BinaryRow trimmedPrimaryKey() {
+ return partitionKeyExtractor.trimmedPrimaryKey(dataRow());
+ }
+
+ private InternalRow dataRow() {
+ return SortCompactSequenceUtils.valueRow(record, logicalFieldCount);
+ }
+}
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java
index f5b8c7892bf8..2cead9ab9a1c 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java
@@ -306,6 +306,14 @@ public int filterAndCommitMultiple(List committables) {
return filterAndCommitMultiple(committables, true);
}
+ public List filterCommitted(List committables) {
+ List sortedCommittables =
+ committables.stream()
+ .sorted(Comparator.comparingLong(ManifestCommittable::identifier))
+ .collect(Collectors.toList());
+ return commit.filterCommitted(sortedCommittables);
+ }
+
public int filterAndCommitMultiple(
List committables, boolean checkAppendFiles) {
List sortedCommittables =
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/KeyValueTableRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/KeyValueTableRead.java
index fda7d70ffdf6..4e227f59581f 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/source/KeyValueTableRead.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/source/KeyValueTableRead.java
@@ -30,6 +30,7 @@
import org.apache.paimon.predicate.TopN;
import org.apache.paimon.reader.RecordReader;
import org.apache.paimon.schema.TableSchema;
+import org.apache.paimon.table.SpecialFields;
import org.apache.paimon.table.source.splitread.IncrementalChangelogReadProvider;
import org.apache.paimon.table.source.splitread.IncrementalDiffReadProvider;
import org.apache.paimon.table.source.splitread.MergeFileSplitReadProvider;
@@ -143,8 +144,10 @@ public TableRead withIOManager(IOManager ioManager) {
@Override
public RecordReader reader(Split split) throws IOException {
+ SplitReadProvider.Context context =
+ new SplitReadProvider.Context(forceKeepDelete, keyValueSequenceNumberEnabled());
for (SplitReadProvider readProvider : readProviders) {
- if (readProvider.match(split, new SplitReadProvider.Context(forceKeepDelete))) {
+ if (readProvider.match(split, context)) {
return readProvider.get().get().createReader(split);
}
}
@@ -152,19 +155,33 @@ public RecordReader reader(Split split) throws IOException {
throw new RuntimeException("Should not happen.");
}
+ private boolean keyValueSequenceNumberEnabled() {
+ return keyValueSequenceNumberEnabled(schema().options(), readType);
+ }
+
+ public static boolean keyValueSequenceNumberEnabled(
+ Map schemaOptions, @Nullable RowType readType) {
+ if (Boolean.parseBoolean(
+ schemaOptions.getOrDefault(
+ CoreOptions.KEY_VALUE_SEQUENCE_NUMBER_ENABLED.key(), "false"))) {
+ return true;
+ }
+ return readType != null
+ && readType.getFieldNames().contains(SpecialFields.SEQUENCE_NUMBER.name());
+ }
+
public static RecordReader unwrap(
RecordReader reader, Map schemaOptions) {
+ return unwrap(reader, keyValueSequenceNumberEnabled(schemaOptions, null));
+ }
+
+ public static RecordReader unwrap(
+ RecordReader reader, boolean keyValueSequenceNumberEnabled) {
return new RecordReader() {
@Nullable
@Override
public RecordIterator readBatch() throws IOException {
- boolean keyValueSequenceNumberEnabled =
- Boolean.parseBoolean(
- schemaOptions.getOrDefault(
- CoreOptions.KEY_VALUE_SEQUENCE_NUMBER_ENABLED.key(),
- "false"));
-
RecordIterator batch = reader.readBatch();
return batch == null
? null
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/splitread/IncrementalChangelogReadProvider.java b/paimon-core/src/main/java/org/apache/paimon/table/source/splitread/IncrementalChangelogReadProvider.java
index 56646dab845a..fe775dee7fb2 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/source/splitread/IncrementalChangelogReadProvider.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/source/splitread/IncrementalChangelogReadProvider.java
@@ -73,7 +73,7 @@ private SplitRead create(Supplier supplier) {
incrementalSplit.afterFiles(),
incrementalSplit.afterDeletionFiles(),
false));
- return unwrap(reader, read.tableSchema().options());
+ return unwrap(reader, read.keyValueSequenceNumberEnabled());
};
return SplitRead.convert(read, convertedFactory);
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/splitread/IncrementalDiffSplitRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/splitread/IncrementalDiffSplitRead.java
index c5032158fa8d..98ff9614d05d 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/source/splitread/IncrementalDiffSplitRead.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/source/splitread/IncrementalDiffSplitRead.java
@@ -29,6 +29,7 @@
import org.apache.paimon.operation.SplitRead;
import org.apache.paimon.predicate.Predicate;
import org.apache.paimon.reader.RecordReader;
+import org.apache.paimon.table.SpecialFields;
import org.apache.paimon.table.source.IncrementalSplit;
import org.apache.paimon.table.source.KeyValueTableRead;
import org.apache.paimon.table.source.Split;
@@ -106,11 +107,26 @@ public RecordReader createReader(Split s) throws IOException {
mergeRead.mergeSorter(),
forceKeepDelete);
if (readType != null) {
+ RowType projectedReadType = stripKeyValueSequenceField(readType);
ProjectedRow projectedRow =
- ProjectedRow.from(readType, mergeRead.tableSchema().logicalRowType());
+ ProjectedRow.from(projectedReadType, mergeRead.tableSchema().logicalRowType());
reader = reader.transform(kv -> kv.replaceValue(projectedRow.replaceRow(kv.value())));
}
- return KeyValueTableRead.unwrap(reader, mergeRead.tableSchema().options());
+ return KeyValueTableRead.unwrap(
+ reader,
+ KeyValueTableRead.keyValueSequenceNumberEnabled(
+ mergeRead.tableSchema().options(), readType));
+ }
+
+ private static RowType stripKeyValueSequenceField(RowType readType) {
+ List fieldNames = readType.getFieldNames();
+ if (!fieldNames.contains(SpecialFields.SEQUENCE_NUMBER.name())) {
+ return readType;
+ }
+ return readType.project(
+ fieldNames.stream()
+ .filter(name -> !SpecialFields.SEQUENCE_NUMBER.name().equals(name))
+ .collect(java.util.stream.Collectors.toList()));
}
private static RecordReader readDiff(
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/splitread/MergeFileSplitReadProvider.java b/paimon-core/src/main/java/org/apache/paimon/table/source/splitread/MergeFileSplitReadProvider.java
index 2e6c127c7fcc..adc6b569c8ab 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/source/splitread/MergeFileSplitReadProvider.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/source/splitread/MergeFileSplitReadProvider.java
@@ -50,7 +50,8 @@ public MergeFileSplitReadProvider(
private SplitRead create(Supplier supplier) {
final MergeFileSplitRead read = supplier.get().withReadKeyType(RowType.of());
return SplitRead.convert(
- read, split -> unwrap(read.createReader(split), read.tableSchema().options()));
+ read,
+ split -> unwrap(read.createReader(split), read.keyValueSequenceNumberEnabled()));
}
@Override
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/splitread/PrimaryKeyTableRawFileSplitReadProvider.java b/paimon-core/src/main/java/org/apache/paimon/table/source/splitread/PrimaryKeyTableRawFileSplitReadProvider.java
index 713977dc590d..972cab28746d 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/source/splitread/PrimaryKeyTableRawFileSplitReadProvider.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/source/splitread/PrimaryKeyTableRawFileSplitReadProvider.java
@@ -35,6 +35,9 @@ public PrimaryKeyTableRawFileSplitReadProvider(
@Override
public boolean match(Split split, Context context) {
+ if (context.keyValueSequenceNumberEnabled()) {
+ return false;
+ }
if (!(split instanceof DataSplit)) {
return false;
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/splitread/SplitReadProvider.java b/paimon-core/src/main/java/org/apache/paimon/table/source/splitread/SplitReadProvider.java
index abae60da81db..ff949afc2f2b 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/source/splitread/SplitReadProvider.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/source/splitread/SplitReadProvider.java
@@ -34,13 +34,23 @@ public interface SplitReadProvider {
class Context {
private final boolean forceKeepDelete;
+ private final boolean keyValueSequenceNumberEnabled;
public Context(boolean forceKeepDelete) {
+ this(forceKeepDelete, false);
+ }
+
+ public Context(boolean forceKeepDelete, boolean keyValueSequenceNumberEnabled) {
this.forceKeepDelete = forceKeepDelete;
+ this.keyValueSequenceNumberEnabled = keyValueSequenceNumberEnabled;
}
public boolean forceKeepDelete() {
return forceKeepDelete;
}
+
+ public boolean keyValueSequenceNumberEnabled() {
+ return keyValueSequenceNumberEnabled;
+ }
}
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/append/SortCompactCommitMessageRewriterTest.java b/paimon-core/src/test/java/org/apache/paimon/append/SortCompactCommitMessageRewriterTest.java
new file mode 100644
index 000000000000..b8d5bd3cfc09
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/append/SortCompactCommitMessageRewriterTest.java
@@ -0,0 +1,1044 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.append;
+
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.Snapshot;
+import org.apache.paimon.TestAppendFileStore;
+import org.apache.paimon.TestKeyValueGenerator;
+import org.apache.paimon.catalog.RenamingSnapshotCommit;
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.fs.FileIOFinder;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.fs.local.LocalFileIO;
+import org.apache.paimon.index.IndexFileHandler;
+import org.apache.paimon.index.IndexFileMeta;
+import org.apache.paimon.io.CompactIncrement;
+import org.apache.paimon.io.DataFileMeta;
+import org.apache.paimon.io.DataIncrement;
+import org.apache.paimon.manifest.FileKind;
+import org.apache.paimon.manifest.FileSource;
+import org.apache.paimon.manifest.IndexManifestEntry;
+import org.apache.paimon.manifest.IndexManifestFile;
+import org.apache.paimon.operation.Lock;
+import org.apache.paimon.schema.Schema;
+import org.apache.paimon.schema.SchemaManager;
+import org.apache.paimon.schema.SchemaUtils;
+import org.apache.paimon.schema.TableSchema;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.FileStoreTableFactory;
+import org.apache.paimon.table.sink.BatchTableCommit;
+import org.apache.paimon.table.sink.BatchWriteBuilder;
+import org.apache.paimon.table.sink.CommitMessage;
+import org.apache.paimon.table.sink.CommitMessageImpl;
+import org.apache.paimon.table.source.DataSplit;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.TraceableFileIO;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.paimon.io.DataFileTestUtils.newFile;
+import static org.apache.paimon.stats.SimpleStats.EMPTY_STATS;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/** Test for {@link SortCompactCommitMessageRewriter}. */
+public class SortCompactCommitMessageRewriterTest {
+
+ @TempDir java.nio.file.Path tempDir;
+
+ private static DataFileMeta asCompactAfter(DataFileMeta file) {
+ return file.assignFileSource(FileSource.COMPACT);
+ }
+
+ @Test
+ public void testRewriteToCompactMessages() throws Exception {
+ FileStoreTable table = createAppendTable(Collections.emptyMap());
+
+ DataFileMeta old0 = newFile("data-0.orc", 0, 0, 100, 100);
+ DataFileMeta old1 = newFile("data-1.orc", 0, 101, 200, 200);
+ DataFileMeta sorted = newFile("sorted-0.orc", 0, 0, 200, 200);
+
+ DataSplit split =
+ DataSplit.builder()
+ .withPartition(BinaryRow.EMPTY_ROW)
+ .withBucket(0)
+ .withBucketPath("bucket-0")
+ .withDataFiles(Arrays.asList(old0, old1))
+ .build();
+
+ CommitMessageImpl written =
+ new CommitMessageImpl(
+ BinaryRow.EMPTY_ROW,
+ 0,
+ table.coreOptions().bucket(),
+ new DataIncrement(
+ Collections.singletonList(sorted),
+ Collections.emptyList(),
+ Collections.emptyList()),
+ CompactIncrement.emptyIncrement());
+
+ List result =
+ new SortCompactCommitMessageRewriter(table, 0L, Collections.singletonList(split))
+ .rewrite(Collections.singletonList(written));
+
+ assertThat(result).hasSize(1);
+ CommitMessageImpl compact = (CommitMessageImpl) result.get(0);
+ assertThat(compact.newFilesIncrement().isEmpty()).isTrue();
+ CompactIncrement ci = compact.compactIncrement();
+ assertThat(ci.compactBefore()).containsExactly(old0, old1);
+ assertThat(ci.compactAfter()).containsExactly(asCompactAfter(sorted));
+ assertThat(ci.changelogFiles()).isEmpty();
+ }
+
+ @Test
+ public void testDetectBatchCompactCommitSucceeded() throws Exception {
+ TestAppendFileStore store = createAppendStore(tempDir, Collections.emptyMap());
+ FileStoreTable table =
+ FileStoreTableFactory.create(
+ store.fileIO(), store.options().path(), store.schema());
+ store.commit(
+ store.writeDataFiles(
+ BinaryRow.EMPTY_ROW, 0, Collections.singletonList("data-0.orc")));
+
+ long baseSnapshotId = table.snapshotManager().latestSnapshotId();
+ DataFileMeta old = newFile("data-0.orc", 0, 0, 100, 100);
+ CommitMessageImpl written =
+ store.writeDataFiles(
+ BinaryRow.EMPTY_ROW, 0, Collections.singletonList("sorted-0.orc"));
+ DataSplit split =
+ DataSplit.builder()
+ .withPartition(BinaryRow.EMPTY_ROW)
+ .withBucket(0)
+ .withBucketPath("bucket-0")
+ .withDataFiles(Collections.singletonList(old))
+ .build();
+
+ SortCompactCommitMessageRewriter rewriter =
+ new SortCompactCommitMessageRewriter(
+ table, baseSnapshotId, Collections.singletonList(split));
+ long snapshotIdBeforeCommit = rewriter.latestSnapshotIdOrZero();
+ List compactMessages = rewriter.rewrite(Collections.singletonList(written));
+ assertThat(rewriter.isBatchCompactCommitSucceeded(snapshotIdBeforeCommit, compactMessages))
+ .isFalse();
+
+ try (BatchTableCommit commit = table.newBatchWriteBuilder().newCommit()) {
+ commit.commit(compactMessages);
+ }
+
+ assertThat(rewriter.isBatchCompactCommitSucceeded(snapshotIdBeforeCommit, compactMessages))
+ .isTrue();
+ assertThat(table.snapshotManager().latestSnapshot().commitKind())
+ .isEqualTo(Snapshot.CommitKind.COMPACT);
+ assertThat(table.snapshotManager().latestSnapshot().commitIdentifier())
+ .isEqualTo(BatchWriteBuilder.COMMIT_IDENTIFIER);
+ }
+
+ @Test
+ public void testDetectBatchCompactCommitSucceededWhenNewerAppendExists() throws Exception {
+ TestAppendFileStore store = createAppendStore(tempDir, Collections.emptyMap());
+ FileStoreTable table =
+ FileStoreTableFactory.create(
+ store.fileIO(), store.options().path(), store.schema());
+ store.commit(
+ store.writeDataFiles(
+ BinaryRow.EMPTY_ROW, 0, Collections.singletonList("data-0.orc")));
+
+ long baseSnapshotId = table.snapshotManager().latestSnapshotId();
+ DataFileMeta old = newFile("data-0.orc", 0, 0, 100, 100);
+ CommitMessageImpl written =
+ store.writeDataFiles(
+ BinaryRow.EMPTY_ROW, 0, Collections.singletonList("sorted-0.orc"));
+ DataSplit split =
+ DataSplit.builder()
+ .withPartition(BinaryRow.EMPTY_ROW)
+ .withBucket(0)
+ .withBucketPath("bucket-0")
+ .withDataFiles(Collections.singletonList(old))
+ .build();
+
+ SortCompactCommitMessageRewriter rewriter =
+ new SortCompactCommitMessageRewriter(
+ table, baseSnapshotId, Collections.singletonList(split));
+ long snapshotIdBeforeCommit = rewriter.latestSnapshotIdOrZero();
+
+ List compactMessages = rewriter.rewrite(Collections.singletonList(written));
+ try (BatchTableCommit commit = table.newBatchWriteBuilder().newCommit()) {
+ commit.commit(compactMessages);
+ }
+ store.commit(
+ store.writeDataFiles(
+ BinaryRow.EMPTY_ROW, 0, Collections.singletonList("concurrent.orc")));
+
+ assertThat(table.snapshotManager().latestSnapshot().commitKind())
+ .isEqualTo(Snapshot.CommitKind.APPEND);
+ assertThat(rewriter.isBatchCompactCommitSucceeded(snapshotIdBeforeCommit, compactMessages))
+ .isTrue();
+ }
+
+ @Test
+ public void testDetectBatchCompactCommitSucceededAcrossExpiredSnapshotGap() throws Exception {
+ TestAppendFileStore store = createAppendStore(tempDir, Collections.emptyMap());
+ FileStoreTable table =
+ FileStoreTableFactory.create(
+ store.fileIO(), store.options().path(), store.schema());
+ store.commit(
+ store.writeDataFiles(
+ BinaryRow.EMPTY_ROW, 0, Collections.singletonList("data-0.orc")));
+
+ long baseSnapshotId = table.snapshotManager().latestSnapshotId();
+ DataFileMeta old = newFile("data-0.orc", 0, 0, 100, 100);
+ CommitMessageImpl written =
+ store.writeDataFiles(
+ BinaryRow.EMPTY_ROW, 0, Collections.singletonList("sorted-0.orc"));
+ DataSplit split =
+ DataSplit.builder()
+ .withPartition(BinaryRow.EMPTY_ROW)
+ .withBucket(0)
+ .withBucketPath("bucket-0")
+ .withDataFiles(Collections.singletonList(old))
+ .build();
+
+ SortCompactCommitMessageRewriter rewriter =
+ new SortCompactCommitMessageRewriter(
+ table, baseSnapshotId, Collections.singletonList(split));
+ long snapshotIdBeforeCommit = rewriter.latestSnapshotIdOrZero();
+ List compactMessages = rewriter.rewrite(Collections.singletonList(written));
+
+ store.commit(
+ store.writeDataFiles(
+ BinaryRow.EMPTY_ROW, 0, Collections.singletonList("filler.orc")));
+ long fillerSnapshotId = table.snapshotManager().latestSnapshotId();
+
+ try (BatchTableCommit commit = table.newBatchWriteBuilder().newCommit()) {
+ commit.commit(compactMessages);
+ }
+ table.snapshotManager().deleteSnapshot(fillerSnapshotId);
+
+ assertThat(rewriter.isBatchCompactCommitSucceeded(snapshotIdBeforeCommit, compactMessages))
+ .isTrue();
+ }
+
+ @Test
+ public void testDetectBatchCompactCommitSucceededWhenCompactSnapshotExpired() throws Exception {
+ TestAppendFileStore store = createAppendStore(tempDir, Collections.emptyMap());
+ FileStoreTable table =
+ FileStoreTableFactory.create(
+ store.fileIO(), store.options().path(), store.schema());
+ store.commit(
+ store.writeDataFiles(
+ BinaryRow.EMPTY_ROW, 0, Collections.singletonList("data-0.orc")));
+
+ long baseSnapshotId = table.snapshotManager().latestSnapshotId();
+ DataFileMeta old = newFile("data-0.orc", 0, 0, 100, 100);
+ CommitMessageImpl written =
+ store.writeDataFiles(
+ BinaryRow.EMPTY_ROW, 0, Collections.singletonList("sorted-0.orc"));
+ DataSplit split =
+ DataSplit.builder()
+ .withPartition(BinaryRow.EMPTY_ROW)
+ .withBucket(0)
+ .withBucketPath("bucket-0")
+ .withDataFiles(Collections.singletonList(old))
+ .build();
+
+ SortCompactCommitMessageRewriter rewriter =
+ new SortCompactCommitMessageRewriter(
+ table, baseSnapshotId, Collections.singletonList(split));
+ long snapshotIdBeforeCommit = rewriter.latestSnapshotIdOrZero();
+ List compactMessages = rewriter.rewrite(Collections.singletonList(written));
+
+ try (BatchTableCommit commit = table.newBatchWriteBuilder().newCommit()) {
+ commit.commit(compactMessages);
+ }
+ long compactSnapshotId = table.snapshotManager().latestSnapshotId();
+
+ store.commit(
+ store.writeDataFiles(
+ BinaryRow.EMPTY_ROW, 0, Collections.singletonList("concurrent.orc")));
+ table.snapshotManager().deleteSnapshot(compactSnapshotId);
+
+ assertThat(table.snapshotManager().latestSnapshot().commitKind())
+ .isEqualTo(Snapshot.CommitKind.APPEND);
+ assertThat(rewriter.isBatchCompactCommitSucceeded(snapshotIdBeforeCommit, compactMessages))
+ .isTrue();
+ }
+
+ @Test
+ public void testDetectBatchCompactCommitSucceededIgnoresUnrelatedCompact() throws Exception {
+ TestAppendFileStore store = createAppendStore(tempDir, Collections.emptyMap());
+ FileStoreTable table =
+ FileStoreTableFactory.create(
+ store.fileIO(), store.options().path(), store.schema());
+ store.commit(
+ store.writeDataFiles(
+ BinaryRow.EMPTY_ROW, 0, Arrays.asList("data-0.orc", "data-1.orc")));
+
+ long baseSnapshotId = table.snapshotManager().latestSnapshotId();
+ DataFileMeta old = newFile("data-0.orc", 0, 0, 100, 100);
+ CommitMessageImpl written =
+ store.writeDataFiles(
+ BinaryRow.EMPTY_ROW, 0, Collections.singletonList("sorted-0.orc"));
+ DataSplit split =
+ DataSplit.builder()
+ .withPartition(BinaryRow.EMPTY_ROW)
+ .withBucket(0)
+ .withBucketPath("bucket-0")
+ .withDataFiles(Collections.singletonList(old))
+ .build();
+
+ SortCompactCommitMessageRewriter rewriter =
+ new SortCompactCommitMessageRewriter(
+ table, baseSnapshotId, Collections.singletonList(split));
+ long snapshotIdBeforeCommit = rewriter.latestSnapshotIdOrZero();
+ List compactMessages = rewriter.rewrite(Collections.singletonList(written));
+
+ DataFileMeta otherOld = newFile("data-1.orc", 0, 101, 200, 200);
+ CommitMessageImpl otherWritten =
+ store.writeDataFiles(
+ BinaryRow.EMPTY_ROW, 0, Collections.singletonList("sorted-1.orc"));
+ DataSplit otherSplit =
+ DataSplit.builder()
+ .withPartition(BinaryRow.EMPTY_ROW)
+ .withBucket(0)
+ .withBucketPath("bucket-0")
+ .withDataFiles(Collections.singletonList(otherOld))
+ .build();
+ List otherCompactMessages =
+ new SortCompactCommitMessageRewriter(
+ table, baseSnapshotId, Collections.singletonList(otherSplit))
+ .rewrite(Collections.singletonList(otherWritten));
+ try (BatchTableCommit commit = table.newBatchWriteBuilder().newCommit()) {
+ commit.commit(otherCompactMessages);
+ }
+
+ assertThat(rewriter.isBatchCompactCommitSucceeded(snapshotIdBeforeCommit, compactMessages))
+ .isFalse();
+ }
+
+ @Test
+ public void testRewriteMultipleBuckets() throws Exception {
+ FileStoreTable table = createAppendTable(Collections.emptyMap());
+
+ DataFileMeta oldBucket0 = newFile("data-0.orc", 0, 0, 100, 100);
+ DataFileMeta oldBucket1 = newFile("data-1.orc", 0, 0, 100, 100);
+ DataFileMeta sortedBucket0 = newFile("sorted-0.orc", 0, 0, 100, 100);
+ DataFileMeta sortedBucket1 = newFile("sorted-1.orc", 0, 0, 100, 100);
+
+ DataSplit split0 =
+ DataSplit.builder()
+ .withPartition(BinaryRow.EMPTY_ROW)
+ .withBucket(0)
+ .withBucketPath("bucket-0")
+ .withDataFiles(Collections.singletonList(oldBucket0))
+ .build();
+ DataSplit split1 =
+ DataSplit.builder()
+ .withPartition(BinaryRow.EMPTY_ROW)
+ .withBucket(1)
+ .withBucketPath("bucket-1")
+ .withDataFiles(Collections.singletonList(oldBucket1))
+ .build();
+
+ CommitMessageImpl written0 =
+ new CommitMessageImpl(
+ BinaryRow.EMPTY_ROW,
+ 0,
+ table.coreOptions().bucket(),
+ new DataIncrement(
+ Collections.singletonList(sortedBucket0),
+ Collections.emptyList(),
+ Collections.emptyList()),
+ CompactIncrement.emptyIncrement());
+ CommitMessageImpl written1 =
+ new CommitMessageImpl(
+ BinaryRow.EMPTY_ROW,
+ 1,
+ table.coreOptions().bucket(),
+ new DataIncrement(
+ Collections.singletonList(sortedBucket1),
+ Collections.emptyList(),
+ Collections.emptyList()),
+ CompactIncrement.emptyIncrement());
+
+ List result =
+ new SortCompactCommitMessageRewriter(table, 0L, Arrays.asList(split0, split1))
+ .rewrite(Arrays.asList(written0, written1));
+
+ assertThat(result).hasSize(2);
+ for (CommitMessage message : result) {
+ CommitMessageImpl compact = (CommitMessageImpl) message;
+ assertThat(compact.newFilesIncrement().isEmpty()).isTrue();
+ assertThat(compact.compactIncrement().compactBefore()).hasSize(1);
+ assertThat(compact.compactIncrement().compactAfter()).hasSize(1);
+ }
+ CommitMessageImpl compact0 =
+ (CommitMessageImpl) result.stream().filter(m -> m.bucket() == 0).findFirst().get();
+ assertThat(compact0.compactIncrement().compactBefore()).containsExactly(oldBucket0);
+ assertThat(compact0.compactIncrement().compactAfter())
+ .containsExactly(asCompactAfter(sortedBucket0));
+ CommitMessageImpl compact1 =
+ (CommitMessageImpl) result.stream().filter(m -> m.bucket() == 1).findFirst().get();
+ assertThat(compact1.compactIncrement().compactBefore()).containsExactly(oldBucket1);
+ assertThat(compact1.compactIncrement().compactAfter())
+ .containsExactly(asCompactAfter(sortedBucket1));
+ }
+
+ @Test
+ public void testRewriteKeepsPlannedDeletesIndependentFromOutputBuckets() throws Exception {
+ FileStoreTable table = createAppendTable(Collections.emptyMap());
+
+ DataFileMeta oldBucket0 = newFile("data-0.orc", 0, 0, 100, 100);
+ DataFileMeta sortedBucket1 = newFile("sorted-1.orc", 0, 0, 100, 100);
+
+ DataSplit split =
+ DataSplit.builder()
+ .withPartition(BinaryRow.EMPTY_ROW)
+ .withBucket(0)
+ .withBucketPath("bucket-0")
+ .withTotalBuckets(2)
+ .withDataFiles(Collections.singletonList(oldBucket0))
+ .build();
+
+ CommitMessageImpl written =
+ new CommitMessageImpl(
+ BinaryRow.EMPTY_ROW,
+ 1,
+ 2,
+ new DataIncrement(
+ Collections.singletonList(sortedBucket1),
+ Collections.emptyList(),
+ Collections.emptyList()),
+ CompactIncrement.emptyIncrement());
+
+ List result =
+ new SortCompactCommitMessageRewriter(table, 0L, Collections.singletonList(split))
+ .rewrite(Collections.singletonList(written));
+
+ assertThat(result).hasSize(2);
+ CommitMessageImpl compactDelete =
+ (CommitMessageImpl) result.stream().filter(m -> m.bucket() == 0).findFirst().get();
+ assertThat(compactDelete.totalBuckets()).isEqualTo(2);
+ assertThat(compactDelete.compactIncrement().compactBefore()).containsExactly(oldBucket0);
+ assertThat(compactDelete.compactIncrement().compactAfter()).isEmpty();
+
+ CommitMessageImpl compactAdd =
+ (CommitMessageImpl) result.stream().filter(m -> m.bucket() == 1).findFirst().get();
+ assertThat(compactAdd.totalBuckets()).isEqualTo(2);
+ assertThat(compactAdd.compactIncrement().compactBefore()).isEmpty();
+ assertThat(compactAdd.compactIncrement().compactAfter())
+ .containsExactly(asCompactAfter(sortedBucket1));
+ }
+
+ @Test
+ public void testRewriteWithDeletionVectors() throws Exception {
+ TestAppendFileStore store =
+ createAppendStore(
+ tempDir,
+ Collections.singletonMap(
+ CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"));
+
+ // write deletion vectors for two old files
+ Map> dvs = new HashMap<>();
+ dvs.put("data-0.orc", Arrays.asList(1, 3, 5));
+ dvs.put("data-1.orc", Arrays.asList(2, 4, 6));
+ CommitMessageImpl dvMessage = store.writeDVIndexFiles(BinaryRow.EMPTY_ROW, 0, dvs);
+ store.commit(dvMessage);
+
+ FileStoreTable table =
+ FileStoreTableFactory.create(
+ store.fileIO(), store.options().path(), store.schema());
+ long baseSnapshotId = table.snapshotManager().latestSnapshotId();
+
+ DataFileMeta old0 = newFile("data-0.orc", 0, 0, 100, 100);
+ DataFileMeta old1 = newFile("data-1.orc", 0, 101, 200, 200);
+ DataFileMeta sorted = newFile("sorted-0.orc", 0, 0, 200, 200);
+
+ DataSplit split =
+ DataSplit.builder()
+ .withPartition(BinaryRow.EMPTY_ROW)
+ .withBucket(0)
+ .withBucketPath("bucket-0")
+ .withDataFiles(Arrays.asList(old0, old1))
+ .build();
+
+ CommitMessageImpl written =
+ new CommitMessageImpl(
+ BinaryRow.EMPTY_ROW,
+ 0,
+ table.coreOptions().bucket(),
+ new DataIncrement(
+ Collections.singletonList(sorted),
+ Collections.emptyList(),
+ Collections.emptyList()),
+ CompactIncrement.emptyIncrement());
+
+ List result =
+ new SortCompactCommitMessageRewriter(
+ table, baseSnapshotId, Collections.singletonList(split))
+ .rewrite(Collections.singletonList(written));
+
+ CommitMessageImpl compact = (CommitMessageImpl) result.get(0);
+ // all old files are removed, so their DV index entries must be cleaned up
+ assertThat(compact.compactIncrement().deletedIndexFiles()).isNotEmpty();
+ assertThat(compact.compactIncrement().newIndexFiles()).isEmpty();
+ assertThat(compact.compactIncrement().compactBefore()).containsExactly(old0, old1);
+ assertThat(compact.compactIncrement().compactAfter())
+ .containsExactly(asCompactAfter(sorted));
+ assertThat(compact.newFilesIncrement().isEmpty()).isTrue();
+ }
+
+ @Test
+ public void testRewriteInputOnlyGroupWithDeletionVectors() throws Exception {
+ TestAppendFileStore store =
+ createAppendStore(
+ tempDir,
+ Collections.singletonMap(
+ CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"));
+
+ Map> dvs = new HashMap<>();
+ dvs.put("data-0.orc", Arrays.asList(1, 3, 5));
+ CommitMessageImpl dvMessage = store.writeDVIndexFiles(BinaryRow.EMPTY_ROW, 0, dvs);
+ store.commit(dvMessage);
+
+ FileStoreTable table =
+ FileStoreTableFactory.create(
+ store.fileIO(), store.options().path(), store.schema());
+ long baseSnapshotId = table.snapshotManager().latestSnapshotId();
+
+ DataFileMeta old = newFile("data-0.orc", 0, 0, 100, 100);
+ DataSplit split =
+ DataSplit.builder()
+ .withPartition(BinaryRow.EMPTY_ROW)
+ .withBucket(0)
+ .withBucketPath("bucket-0")
+ .withDataFiles(Collections.singletonList(old))
+ .build();
+
+ List result =
+ new SortCompactCommitMessageRewriter(
+ table, baseSnapshotId, Collections.singletonList(split))
+ .rewrite(Collections.emptyList());
+
+ assertThat(result).hasSize(1);
+ CommitMessageImpl compact = (CommitMessageImpl) result.get(0);
+ assertThat(compact.newFilesIncrement().isEmpty()).isTrue();
+ assertThat(compact.compactIncrement().compactBefore()).containsExactly(old);
+ assertThat(compact.compactIncrement().compactAfter()).isEmpty();
+ assertThat(compact.compactIncrement().deletedIndexFiles()).isNotEmpty();
+ assertThat(compact.compactIncrement().newIndexFiles()).isEmpty();
+ }
+
+ @Test
+ public void testRewritePartialMessagesMustBeMerged() throws Exception {
+ FileStoreTable table = createAppendTable(Collections.emptyMap());
+
+ DataFileMeta oldBucket0 = newFile("data-0.orc", 0, 0, 100, 100);
+ DataFileMeta oldBucket1 = newFile("data-1.orc", 0, 0, 100, 100);
+ DataFileMeta sortedBucket0 = newFile("sorted-0.orc", 0, 0, 100, 100);
+ DataFileMeta sortedBucket1 = newFile("sorted-1.orc", 0, 0, 100, 100);
+
+ DataSplit split0 =
+ DataSplit.builder()
+ .withPartition(BinaryRow.EMPTY_ROW)
+ .withBucket(0)
+ .withBucketPath("bucket-0")
+ .withDataFiles(Collections.singletonList(oldBucket0))
+ .build();
+ DataSplit split1 =
+ DataSplit.builder()
+ .withPartition(BinaryRow.EMPTY_ROW)
+ .withBucket(1)
+ .withBucketPath("bucket-1")
+ .withDataFiles(Collections.singletonList(oldBucket1))
+ .build();
+
+ CommitMessageImpl writtenBucket0 =
+ new CommitMessageImpl(
+ BinaryRow.EMPTY_ROW,
+ 0,
+ table.coreOptions().bucket(),
+ new DataIncrement(
+ Collections.singletonList(sortedBucket0),
+ Collections.emptyList(),
+ Collections.emptyList()),
+ CompactIncrement.emptyIncrement());
+ CommitMessageImpl writtenBucket1 =
+ new CommitMessageImpl(
+ BinaryRow.EMPTY_ROW,
+ 1,
+ table.coreOptions().bucket(),
+ new DataIncrement(
+ Collections.singletonList(sortedBucket1),
+ Collections.emptyList(),
+ Collections.emptyList()),
+ CompactIncrement.emptyIncrement());
+
+ SortCompactCommitMessageRewriter rewriter =
+ new SortCompactCommitMessageRewriter(table, 0L, Arrays.asList(split0, split1));
+
+ // Rewriting partial outputs separately would duplicate compactBefore for every commit.
+ List partial0 = rewriter.rewrite(Collections.singletonList(writtenBucket0));
+ List partial1 = rewriter.rewrite(Collections.singletonList(writtenBucket1));
+ assertThat(partial0).hasSize(2);
+ assertThat(partial1).hasSize(2);
+
+ List merged =
+ rewriter.rewrite(Arrays.asList(writtenBucket0, writtenBucket1));
+ assertThat(merged).hasSize(2);
+ for (CommitMessage message : merged) {
+ CommitMessageImpl compact = (CommitMessageImpl) message;
+ assertThat(compact.newFilesIncrement().isEmpty()).isTrue();
+ assertThat(compact.compactIncrement().compactBefore()).hasSize(1);
+ assertThat(compact.compactIncrement().compactAfter()).hasSize(1);
+ }
+ }
+
+ @Test
+ public void testRewriteSnapshotOrderingKeepsWriterSequenceRange() throws Exception {
+ FileStoreTable table = createPrimaryKeyTable(Collections.emptyMap());
+
+ DataFileMeta oldBucket0 = newFile(1, 1);
+ DataFileMeta oldBucket1 = newFile(10, 20);
+ DataFileMeta sorted =
+ DataFileMeta.create(
+ "sorted-0.orc",
+ 100,
+ 2,
+ DataFileMeta.EMPTY_MIN_KEY,
+ DataFileMeta.EMPTY_MAX_KEY,
+ EMPTY_STATS,
+ EMPTY_STATS,
+ 1,
+ 20,
+ 0,
+ 0,
+ Collections.emptyList(),
+ 0L,
+ null,
+ FileSource.APPEND,
+ null,
+ null,
+ null,
+ null);
+
+ DataSplit split0 =
+ DataSplit.builder()
+ .withPartition(BinaryRow.EMPTY_ROW)
+ .withBucket(0)
+ .withBucketPath("bucket-0")
+ .withDataFiles(Collections.singletonList(oldBucket0))
+ .build();
+ DataSplit split1 =
+ DataSplit.builder()
+ .withPartition(BinaryRow.EMPTY_ROW)
+ .withBucket(1)
+ .withBucketPath("bucket-1")
+ .withDataFiles(Collections.singletonList(oldBucket1))
+ .build();
+
+ CommitMessageImpl written =
+ new CommitMessageImpl(
+ BinaryRow.EMPTY_ROW,
+ 1,
+ table.coreOptions().bucket(),
+ new DataIncrement(
+ Collections.singletonList(sorted),
+ Collections.emptyList(),
+ Collections.emptyList()),
+ CompactIncrement.emptyIncrement());
+
+ List result =
+ new SortCompactCommitMessageRewriter(table, 0L, Arrays.asList(split0, split1))
+ .rewrite(Collections.singletonList(written));
+
+ CommitMessageImpl compact =
+ (CommitMessageImpl) result.stream().filter(m -> m.bucket() == 1).findFirst().get();
+ DataFileMeta compactAfter = compact.compactIncrement().compactAfter().get(0);
+ assertThat(compactAfter.fileSource()).contains(FileSource.COMPACT);
+ assertThat(compactAfter.minSequenceNumber()).isEqualTo(1);
+ assertThat(compactAfter.maxSequenceNumber()).isEqualTo(20);
+ }
+
+ @Test
+ public void testRewriteUsesCapturedBaseSnapshotMetadata() throws Exception {
+ TestAppendFileStore store =
+ createAppendStore(
+ tempDir,
+ Collections.singletonMap(
+ CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"));
+
+ Map> dvs = new HashMap<>();
+ dvs.put("data-0.orc", Arrays.asList(1, 3, 5));
+ CommitMessageImpl dvMessage = store.writeDVIndexFiles(BinaryRow.EMPTY_ROW, 0, dvs);
+ store.commit(dvMessage);
+
+ FileStoreTable table =
+ FileStoreTableFactory.create(
+ store.fileIO(), store.options().path(), store.schema());
+ long baseSnapshotId = table.snapshotManager().latestSnapshotId();
+
+ DataFileMeta old = newFile("data-0.orc", 0, 0, 100, 100);
+ DataFileMeta sorted = newFile("sorted-0.orc", 0, 0, 100, 100);
+ DataSplit split =
+ DataSplit.builder()
+ .withPartition(BinaryRow.EMPTY_ROW)
+ .withBucket(0)
+ .withBucketPath("bucket-0")
+ .withDataFiles(Collections.singletonList(old))
+ .build();
+
+ CommitMessageImpl written =
+ new CommitMessageImpl(
+ BinaryRow.EMPTY_ROW,
+ 0,
+ table.coreOptions().bucket(),
+ new DataIncrement(
+ Collections.singletonList(sorted),
+ Collections.emptyList(),
+ Collections.emptyList()),
+ CompactIncrement.emptyIncrement());
+
+ SortCompactCommitMessageRewriter rewriter =
+ new SortCompactCommitMessageRewriter(
+ table, baseSnapshotId, Collections.singletonList(split));
+ table.snapshotManager().deleteSnapshot(baseSnapshotId);
+
+ List result = rewriter.rewrite(Collections.singletonList(written));
+
+ CommitMessageImpl compact = (CommitMessageImpl) result.get(0);
+ assertThat(compact.compactIncrement().deletedIndexFiles()).isNotEmpty();
+ assertThat(compact.compactIncrement().compactBefore()).containsExactly(old);
+ assertThat(compact.compactIncrement().compactAfter())
+ .containsExactly(asCompactAfter(sorted));
+ }
+
+ @Test
+ public void testPlanMetadataRoundTripSerialization() throws Exception {
+ TestAppendFileStore store =
+ createAppendStore(
+ tempDir,
+ Collections.singletonMap(
+ CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"));
+
+ Map> dvs = new HashMap<>();
+ dvs.put("data-0.orc", Arrays.asList(1, 3, 5));
+ CommitMessageImpl dvMessage = store.writeDVIndexFiles(BinaryRow.EMPTY_ROW, 0, dvs);
+ store.commit(dvMessage);
+
+ FileStoreTable table =
+ FileStoreTableFactory.create(
+ store.fileIO(), store.options().path(), store.schema());
+ long baseSnapshotId = table.snapshotManager().latestSnapshotId();
+ DataSplit split =
+ DataSplit.builder()
+ .withPartition(BinaryRow.EMPTY_ROW)
+ .withBucket(0)
+ .withBucketPath("bucket-0")
+ .withDataFiles(
+ Collections.singletonList(newFile("data-0.orc", 0, 0, 100, 100)))
+ .build();
+
+ SortCompactPlanMetadata captured =
+ SortCompactPlanMetadata.capture(
+ table, baseSnapshotId, Collections.singletonList(split));
+ SortCompactPlanMetadata restored;
+ try (ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ ObjectOutputStream oos = new ObjectOutputStream(baos)) {
+ oos.writeObject(captured);
+ try (ObjectInputStream ois =
+ new ObjectInputStream(new ByteArrayInputStream(baos.toByteArray()))) {
+ restored = (SortCompactPlanMetadata) ois.readObject();
+ }
+ }
+
+ Map> dvEntries = new HashMap<>();
+ Map> hashIndexes = new HashMap<>();
+ captured.copyInto(dvEntries, hashIndexes);
+ Map> restoredDvEntries = new HashMap<>();
+ Map> restoredHashIndexes = new HashMap<>();
+ restored.copyInto(restoredDvEntries, restoredHashIndexes);
+
+ assertThat(restoredDvEntries).isEqualTo(dvEntries);
+ assertThat(restoredHashIndexes).isEqualTo(hashIndexes);
+ }
+
+ @Test
+ public void testRewriteWithoutCapturedMetadataSkipsDvCleanupWhenBaseSnapshotExpired()
+ throws Exception {
+ TestAppendFileStore store =
+ createAppendStore(
+ tempDir,
+ Collections.singletonMap(
+ CoreOptions.DELETION_VECTORS_ENABLED.key(), "true"));
+
+ Map> dvs = new HashMap<>();
+ dvs.put("data-0.orc", Arrays.asList(1, 3, 5));
+ CommitMessageImpl dvMessage = store.writeDVIndexFiles(BinaryRow.EMPTY_ROW, 0, dvs);
+ store.commit(dvMessage);
+
+ FileStoreTable table =
+ FileStoreTableFactory.create(
+ store.fileIO(), store.options().path(), store.schema());
+ long baseSnapshotId = table.snapshotManager().latestSnapshotId();
+
+ DataFileMeta old = newFile("data-0.orc", 0, 0, 100, 100);
+ DataFileMeta sorted = newFile("sorted-0.orc", 0, 0, 100, 100);
+ DataSplit split =
+ DataSplit.builder()
+ .withPartition(BinaryRow.EMPTY_ROW)
+ .withBucket(0)
+ .withBucketPath("bucket-0")
+ .withDataFiles(Collections.singletonList(old))
+ .build();
+
+ CommitMessageImpl written =
+ new CommitMessageImpl(
+ BinaryRow.EMPTY_ROW,
+ 0,
+ table.coreOptions().bucket(),
+ new DataIncrement(
+ Collections.singletonList(sorted),
+ Collections.emptyList(),
+ Collections.emptyList()),
+ CompactIncrement.emptyIncrement());
+
+ SortCompactPlanMetadata planMetadata =
+ SortCompactPlanMetadata.capture(
+ table, baseSnapshotId, Collections.singletonList(split));
+ table.snapshotManager().deleteSnapshot(baseSnapshotId);
+
+ List withoutCapturedMetadata =
+ new SortCompactCommitMessageRewriter(
+ table, baseSnapshotId, Collections.singletonList(split))
+ .rewrite(Collections.singletonList(written));
+ assertThat(
+ ((CommitMessageImpl) withoutCapturedMetadata.get(0))
+ .compactIncrement()
+ .deletedIndexFiles())
+ .isEmpty();
+
+ List withCapturedMetadata =
+ new SortCompactCommitMessageRewriter(
+ table,
+ baseSnapshotId,
+ Collections.singletonList(split),
+ planMetadata)
+ .rewrite(Collections.singletonList(written));
+ assertThat(
+ ((CommitMessageImpl) withCapturedMetadata.get(0))
+ .compactIncrement()
+ .deletedIndexFiles())
+ .isNotEmpty();
+ }
+
+ @Test
+ public void testRewriteRejectsInlineCompactionOutput() throws Exception {
+ FileStoreTable table = createAppendTable(Collections.emptyMap());
+
+ DataFileMeta old = newFile("data-0.orc", 0, 0, 100, 100);
+ DataFileMeta l0File = newFile("l0-0.orc", 0, 0, 100, 100);
+ DataFileMeta compactedFile = newFile("compacted-0.orc", 0, 0, 100, 100);
+
+ DataSplit split =
+ DataSplit.builder()
+ .withPartition(BinaryRow.EMPTY_ROW)
+ .withBucket(0)
+ .withBucketPath("bucket-0")
+ .withDataFiles(Collections.singletonList(old))
+ .build();
+
+ CommitMessageImpl written =
+ new CommitMessageImpl(
+ BinaryRow.EMPTY_ROW,
+ 0,
+ table.coreOptions().bucket(),
+ new DataIncrement(
+ Collections.singletonList(l0File),
+ Collections.emptyList(),
+ Collections.emptyList()),
+ new CompactIncrement(
+ Collections.singletonList(old),
+ Collections.singletonList(compactedFile),
+ Collections.emptyList()));
+
+ SortCompactCommitMessageRewriter rewriter =
+ new SortCompactCommitMessageRewriter(table, 0L, Collections.singletonList(split));
+
+ assertThatThrownBy(() -> rewriter.rewrite(Collections.singletonList(written)))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessageContaining("inline compaction changes");
+ }
+
+ @Test
+ public void testRewriteCleansHashIndexForDynamicBucket() throws Exception {
+ FileStoreTable table = createPrimaryKeyTable(Collections.emptyMap());
+ IndexFileHandler indexFileHandler = table.store().newIndexFileHandler();
+ BinaryRow partition = BinaryRow.EMPTY_ROW;
+
+ IndexFileMeta hashIndex =
+ indexFileHandler.hashIndex(partition, 0).write(new int[] {1, 2, 5});
+ bootstrapHashIndexSnapshot(table, partition, 0, hashIndex);
+
+ long baseSnapshotId = table.snapshotManager().latestSnapshotId();
+ IndexFileMeta baseHashIndex =
+ indexFileHandler
+ .scanHashIndex(table.snapshotManager().latestSnapshot(), partition, 0)
+ .orElseThrow(() -> new IllegalStateException("Hash index should exist."));
+
+ DataFileMeta old = newFile("data-0.orc", 0, 0, 100, 100);
+ DataFileMeta sorted = newFile("sorted-0.orc", 0, 0, 100, 100);
+ DataSplit split =
+ DataSplit.builder()
+ .withPartition(partition)
+ .withBucket(0)
+ .withBucketPath("bucket-0")
+ .withDataFiles(Collections.singletonList(old))
+ .build();
+
+ CommitMessageImpl written =
+ new CommitMessageImpl(
+ partition,
+ 0,
+ -1,
+ new DataIncrement(
+ Collections.singletonList(sorted),
+ Collections.emptyList(),
+ Collections.emptyList()),
+ CompactIncrement.emptyIncrement());
+
+ List result =
+ new SortCompactCommitMessageRewriter(
+ table, baseSnapshotId, Collections.singletonList(split))
+ .rewrite(Collections.singletonList(written));
+
+ CommitMessageImpl compact = (CommitMessageImpl) result.get(0);
+ assertThat(compact.compactIncrement().deletedIndexFiles()).contains(baseHashIndex);
+ }
+
+ private void bootstrapHashIndexSnapshot(
+ FileStoreTable table, BinaryRow partition, int bucket, IndexFileMeta hashIndex)
+ throws Exception {
+ IndexManifestFile indexManifestFile = table.store().indexManifestFileFactory().create();
+ String indexManifest =
+ indexManifestFile.writeWithoutRolling(
+ Collections.singletonList(
+ new IndexManifestEntry(
+ FileKind.ADD, partition, bucket, hashIndex)));
+ Snapshot snapshot =
+ new Snapshot(
+ Snapshot.FIRST_SNAPSHOT_ID,
+ table.schema().id(),
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ indexManifest,
+ "test-user",
+ 0L,
+ Snapshot.CommitKind.APPEND,
+ System.currentTimeMillis(),
+ 0L,
+ 0L,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null);
+ new RenamingSnapshotCommit(table.snapshotManager(), Lock.empty())
+ .commit(snapshot, table.snapshotManager().branch(), Collections.emptyList());
+ }
+
+ private FileStoreTable createPrimaryKeyTable(Map dynamicOptions)
+ throws Exception {
+ String root = TraceableFileIO.SCHEME + "://" + tempDir.toString();
+ Path path = new Path(tempDir.toUri());
+ FileIO fileIO = FileIOFinder.find(new Path(root));
+ SchemaManager schemaManage = new SchemaManager(new LocalFileIO(), path);
+
+ Map options = new HashMap<>(dynamicOptions);
+ options.put(CoreOptions.PATH.key(), root);
+ options.put(CoreOptions.BUCKET.key(), "-1");
+ options.put(CoreOptions.WRITE_ONLY.key(), "true");
+ options.put(CoreOptions.SEQUENCE_SNAPSHOT_ORDERING.key(), "true");
+ TableSchema tableSchema =
+ SchemaUtils.forceCommit(
+ schemaManage,
+ new Schema(
+ Arrays.asList(
+ new org.apache.paimon.types.DataField(
+ 0, "k", org.apache.paimon.types.DataTypes.INT()),
+ new org.apache.paimon.types.DataField(
+ 1, "v", org.apache.paimon.types.DataTypes.INT())),
+ Collections.emptyList(),
+ Collections.singletonList("k"),
+ options,
+ null));
+ return FileStoreTableFactory.create(fileIO, new CoreOptions(options).path(), tableSchema);
+ }
+
+ private FileStoreTable createAppendTable(Map dynamicOptions) throws Exception {
+ TestAppendFileStore store = createAppendStore(tempDir, dynamicOptions);
+ return FileStoreTableFactory.create(store.fileIO(), store.options().path(), store.schema());
+ }
+
+ private TestAppendFileStore createAppendStore(
+ java.nio.file.Path tempDir, Map dynamicOptions) throws Exception {
+ String root = TraceableFileIO.SCHEME + "://" + tempDir.toString();
+ Path path = new Path(tempDir.toUri());
+ FileIO fileIO = FileIOFinder.find(new Path(root));
+ SchemaManager schemaManage = new SchemaManager(new LocalFileIO(), path);
+
+ Map options = new HashMap<>(dynamicOptions);
+ options.put(CoreOptions.PATH.key(), root);
+ TableSchema tableSchema =
+ SchemaUtils.forceCommit(
+ schemaManage,
+ new Schema(
+ TestKeyValueGenerator.DEFAULT_ROW_TYPE.getFields(),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ options,
+ null));
+ return new TestAppendFileStore(
+ fileIO,
+ schemaManage,
+ new CoreOptions(options),
+ tableSchema,
+ RowType.of(),
+ RowType.of(),
+ TestKeyValueGenerator.DEFAULT_ROW_TYPE,
+ (new Path(root)).getName());
+ }
+}
diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java
index 5be15f0dcc1e..533d9e432477 100644
--- a/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/operation/commit/ConflictDetectionTest.java
@@ -22,6 +22,7 @@
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.index.DeletionVectorMeta;
import org.apache.paimon.index.GlobalIndexMeta;
+import org.apache.paimon.index.IndexFileHandler;
import org.apache.paimon.index.IndexFileMeta;
import org.apache.paimon.manifest.FileEntry;
import org.apache.paimon.manifest.FileKind;
@@ -30,6 +31,7 @@
import org.apache.paimon.manifest.SimpleFileEntryWithDV;
import org.apache.paimon.table.BucketMode;
import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.Pair;
import org.junit.jupiter.api.Test;
@@ -45,11 +47,14 @@
import static org.apache.paimon.data.BinaryRow.EMPTY_ROW;
import static org.apache.paimon.deletionvectors.DeletionVectorsIndexFile.DELETION_VECTORS_INDEX;
+import static org.apache.paimon.index.HashIndexFile.HASH_INDEX;
import static org.apache.paimon.manifest.FileKind.ADD;
import static org.apache.paimon.manifest.FileKind.DELETE;
import static org.apache.paimon.operation.commit.ConflictDetection.buildBaseEntriesWithDV;
import static org.apache.paimon.operation.commit.ConflictDetection.buildDeltaEntriesWithDV;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
class ConflictDetectionTest {
@@ -761,6 +766,84 @@ void testCheckGlobalIndexRowIdExistenceSkipsDeleteIndexEntry() {
assertThat(exception).isNotPresent();
}
+ @Test
+ void testCheckHashIndexConflictForSortCompact() {
+ IndexFileHandler indexFileHandler = mock(IndexFileHandler.class);
+ Snapshot latestSnapshot = snapshot(2);
+ IndexFileMeta currentHashIndex = createHashIndexFile("hash-v2");
+ when(indexFileHandler.scanBuckets(
+ latestSnapshot, HASH_INDEX, Collections.singleton(Pair.of(EMPTY_ROW, 0))))
+ .thenReturn(
+ Collections.singletonMap(
+ Pair.of(EMPTY_ROW, 0),
+ Collections.singletonList(currentHashIndex)));
+
+ ConflictDetection detection = createHashDynamicConflictDetection(indexFileHandler);
+ Optional exception =
+ detection.checkConflicts(
+ latestSnapshot,
+ Collections.emptyList(),
+ Collections.emptyList(),
+ Collections.singletonList(
+ createHashIndexEntry("hash-v1", DELETE, EMPTY_ROW, 0)),
+ null,
+ Snapshot.CommitKind.COMPACT);
+
+ assertThat(exception).isPresent();
+ assertThat(exception.get()).hasMessageContaining("Hash index conflict");
+ }
+
+ @Test
+ void testCheckHashIndexNoConflictWhenUnchanged() {
+ IndexFileHandler indexFileHandler = mock(IndexFileHandler.class);
+ Snapshot latestSnapshot = snapshot(2);
+ IndexFileMeta hashIndex = createHashIndexFile("hash-v1");
+ when(indexFileHandler.scanBuckets(
+ latestSnapshot, HASH_INDEX, Collections.singleton(Pair.of(EMPTY_ROW, 0))))
+ .thenReturn(
+ Collections.singletonMap(
+ Pair.of(EMPTY_ROW, 0), Collections.singletonList(hashIndex)));
+
+ ConflictDetection detection = createHashDynamicConflictDetection(indexFileHandler);
+ Optional exception =
+ detection.checkConflicts(
+ latestSnapshot,
+ Collections.emptyList(),
+ Collections.emptyList(),
+ Collections.singletonList(
+ createHashIndexEntry("hash-v1", DELETE, EMPTY_ROW, 0)),
+ null,
+ Snapshot.CommitKind.COMPACT);
+
+ assertThat(exception).isNotPresent();
+ }
+
+ private ConflictDetection createHashDynamicConflictDetection(
+ IndexFileHandler indexFileHandler) {
+ return new ConflictDetection(
+ "test-table",
+ "test-user",
+ RowType.of(),
+ null,
+ null,
+ BucketMode.HASH_DYNAMIC,
+ false,
+ false,
+ false,
+ indexFileHandler,
+ null,
+ null);
+ }
+
+ private IndexManifestEntry createHashIndexEntry(
+ String fileName, FileKind kind, BinaryRow partition, int bucket) {
+ return new IndexManifestEntry(kind, partition, bucket, createHashIndexFile(fileName));
+ }
+
+ private IndexFileMeta createHashIndexFile(String fileName) {
+ return new IndexFileMeta(HASH_INDEX, fileName, 1, 1, (GlobalIndexMeta) null, null);
+ }
+
private ConflictDetection createConflictDetection() {
return new ConflictDetection(
"test-table",
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/KeyValueTableReadSequenceTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/KeyValueTableReadSequenceTest.java
new file mode 100644
index 000000000000..c6e7095c2fa5
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/table/source/KeyValueTableReadSequenceTest.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table.source;
+
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.append.SortCompactSequenceUtils;
+import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.data.JoinedRow;
+import org.apache.paimon.schema.Schema;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.Table;
+import org.apache.paimon.table.TableTestBase;
+import org.apache.paimon.table.sink.BatchTableCommit;
+import org.apache.paimon.table.sink.BatchTableWrite;
+import org.apache.paimon.table.sink.BatchWriteBuilder;
+import org.apache.paimon.table.source.snapshot.SnapshotReader;
+import org.apache.paimon.types.DataTypes;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Test for {@link KeyValueTableRead} with key-value sequence number on raw-convertible splits. */
+public class KeyValueTableReadSequenceTest extends TableTestBase {
+
+ @Test
+ public void testReadKeyValueSequenceOnRawConvertibleSplit() throws Exception {
+ Map options = new HashMap<>();
+ options.put(CoreOptions.BUCKET.key(), "1");
+ options.put(CoreOptions.SEQUENCE_SNAPSHOT_ORDERING.key(), "true");
+ options.put(CoreOptions.WRITE_ONLY.key(), "true");
+ options.put(CoreOptions.NUM_SORTED_RUNS_COMPACTION_TRIGGER.key(), "1");
+
+ Schema schema =
+ Schema.newBuilder()
+ .column("pk", DataTypes.BIGINT())
+ .column("val", DataTypes.BIGINT())
+ .primaryKey("pk")
+ .options(options)
+ .build();
+ catalog.createTable(identifier(), schema, false);
+ FileStoreTable table = (FileStoreTable) catalog.getTable(identifier());
+
+ writeRow(table, GenericRow.of(1L, 100L));
+ writeRow(table, GenericRow.of(1L, 200L));
+ writeRow(table, GenericRow.of(2L, 300L));
+
+ FileStoreTable compactTable =
+ table.copy(Collections.singletonMap(CoreOptions.WRITE_ONLY.key(), "false"));
+ compact(compactTable, BinaryRow.EMPTY_ROW, 0);
+
+ SnapshotReader snapshotReader = table.newSnapshotReader();
+ List splits = snapshotReader.read().dataSplits();
+ assertThat(splits).anyMatch(DataSplit::rawConvertible);
+
+ Map readOptions = new HashMap<>();
+ readOptions.put(CoreOptions.KEY_VALUE_SEQUENCE_NUMBER_ENABLED.key(), "true");
+ FileStoreTable readTable = table.copy(readOptions);
+ TableRead read =
+ readTable
+ .newReadBuilder()
+ .withReadType(
+ SortCompactSequenceUtils.rowTypeWithKeyValueSequenceNumber(
+ readTable.rowType()))
+ .newRead();
+
+ AtomicBoolean foundPk1 = new AtomicBoolean(false);
+ for (DataSplit split : splits) {
+ if (!split.rawConvertible()) {
+ continue;
+ }
+ read.createReader(split)
+ .forEachRemaining(
+ row -> {
+ assertThat(row).isInstanceOf(JoinedRow.class);
+ long sequence =
+ SortCompactSequenceUtils.sequenceNumber(
+ row, readTable.rowType().getFieldCount());
+ assertThat(sequence).isGreaterThanOrEqualTo(0L);
+ InternalRow valueRow =
+ SortCompactSequenceUtils.valueRow(
+ row, readTable.rowType().getFieldCount());
+ if (valueRow.getLong(0) == 1L) {
+ assertThat(valueRow.getLong(1)).isEqualTo(200L);
+ assertThat(sequence).isEqualTo(2L);
+ foundPk1.set(true);
+ }
+ });
+ }
+ assertThat(foundPk1).isTrue();
+ }
+
+ @Test
+ public void testReadKeyValueSequenceWithCustomReadTypeOnly() throws Exception {
+ Map options = new HashMap<>();
+ options.put(CoreOptions.BUCKET.key(), "1");
+ options.put(CoreOptions.SEQUENCE_SNAPSHOT_ORDERING.key(), "true");
+ options.put(CoreOptions.WRITE_ONLY.key(), "true");
+ options.put(CoreOptions.NUM_SORTED_RUNS_COMPACTION_TRIGGER.key(), "1");
+
+ Schema schema =
+ Schema.newBuilder()
+ .column("pk", DataTypes.BIGINT())
+ .column("val", DataTypes.BIGINT())
+ .primaryKey("pk")
+ .options(options)
+ .build();
+ catalog.createTable(identifier(), schema, false);
+ FileStoreTable table = (FileStoreTable) catalog.getTable(identifier());
+
+ writeRow(table, GenericRow.of(1L, 100L));
+ writeRow(table, GenericRow.of(1L, 200L));
+ writeRow(table, GenericRow.of(2L, 300L));
+
+ FileStoreTable compactTable =
+ table.copy(Collections.singletonMap(CoreOptions.WRITE_ONLY.key(), "false"));
+ compact(compactTable, BinaryRow.EMPTY_ROW, 0);
+
+ SnapshotReader snapshotReader = table.newSnapshotReader();
+ List splits = snapshotReader.read().dataSplits();
+ assertThat(splits).anyMatch(DataSplit::rawConvertible);
+
+ TableRead read =
+ table.newReadBuilder()
+ .withReadType(
+ SortCompactSequenceUtils.rowTypeWithKeyValueSequenceNumber(
+ table.rowType()))
+ .newRead();
+
+ AtomicBoolean foundPk1 = new AtomicBoolean(false);
+ for (DataSplit split : splits) {
+ if (!split.rawConvertible()) {
+ continue;
+ }
+ read.createReader(split)
+ .forEachRemaining(
+ row -> {
+ assertThat(row).isInstanceOf(JoinedRow.class);
+ long sequence =
+ SortCompactSequenceUtils.sequenceNumber(
+ row, table.rowType().getFieldCount());
+ assertThat(sequence).isGreaterThanOrEqualTo(0L);
+ InternalRow valueRow =
+ SortCompactSequenceUtils.valueRow(
+ row, table.rowType().getFieldCount());
+ if (valueRow.getLong(0) == 1L) {
+ assertThat(valueRow.getLong(1)).isEqualTo(200L);
+ assertThat(sequence).isEqualTo(2L);
+ foundPk1.set(true);
+ }
+ });
+ }
+ assertThat(foundPk1).isTrue();
+ }
+
+ private void writeRow(Table table, GenericRow row) throws Exception {
+ BatchWriteBuilder builder = table.newBatchWriteBuilder();
+ try (BatchTableWrite write = builder.newWrite();
+ BatchTableCommit commit = builder.newCommit()) {
+ write.write(row);
+ commit.commit(write.prepareCommit());
+ }
+ }
+}
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/splitread/IncrementalDiffSplitReadTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/splitread/IncrementalDiffSplitReadTest.java
new file mode 100644
index 000000000000..c834947719f8
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/table/source/splitread/IncrementalDiffSplitReadTest.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table.source.splitread;
+
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.Snapshot;
+import org.apache.paimon.append.SortCompactSequenceUtils;
+import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.data.JoinedRow;
+import org.apache.paimon.schema.Schema;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.Table;
+import org.apache.paimon.table.TableTestBase;
+import org.apache.paimon.table.sink.BatchTableCommit;
+import org.apache.paimon.table.sink.BatchTableWrite;
+import org.apache.paimon.table.sink.BatchWriteBuilder;
+import org.apache.paimon.table.source.IncrementalSplit;
+import org.apache.paimon.table.source.Split;
+import org.apache.paimon.table.source.TableRead;
+import org.apache.paimon.table.source.snapshot.SnapshotReader;
+import org.apache.paimon.types.DataTypes;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Test for {@link IncrementalDiffSplitRead}. */
+public class IncrementalDiffSplitReadTest extends TableTestBase {
+
+ @Test
+ public void testReadKeyValueSequenceOnIncrementalSplit() throws Exception {
+ Map options = new HashMap<>();
+ options.put(CoreOptions.BUCKET.key(), "1");
+ options.put(CoreOptions.SEQUENCE_SNAPSHOT_ORDERING.key(), "true");
+ options.put(CoreOptions.WRITE_ONLY.key(), "true");
+
+ Schema schema =
+ Schema.newBuilder()
+ .column("pk", DataTypes.BIGINT())
+ .column("val", DataTypes.BIGINT())
+ .primaryKey("pk")
+ .options(options)
+ .build();
+ catalog.createTable(identifier(), schema, false);
+ FileStoreTable table = (FileStoreTable) catalog.getTable(identifier());
+
+ writeRow(table, GenericRow.of(1L, 100L));
+ Snapshot beforeSnapshot = table.snapshotManager().latestSnapshot();
+ writeRow(table, GenericRow.of(1L, 200L));
+ writeRow(table, GenericRow.of(2L, 300L));
+
+ SnapshotReader snapshotReader = table.newSnapshotReader();
+ List splits =
+ snapshotReader
+ .withSnapshot(table.snapshotManager().latestSnapshot())
+ .readIncrementalDiff(beforeSnapshot)
+ .splits();
+ assertThat(splits).isNotEmpty();
+ assertThat(splits.get(0)).isInstanceOf(IncrementalSplit.class);
+ assertThat(((IncrementalSplit) splits.get(0)).isStreaming()).isFalse();
+
+ TableRead read =
+ table.newReadBuilder()
+ .withReadType(
+ SortCompactSequenceUtils.rowTypeWithKeyValueSequenceNumber(
+ table.rowType()))
+ .newRead();
+
+ AtomicBoolean foundPk1 = new AtomicBoolean(false);
+ for (Split split : splits) {
+ read.createReader(split)
+ .forEachRemaining(
+ row -> {
+ assertThat(row).isInstanceOf(JoinedRow.class);
+ assertThat(row.getFieldCount())
+ .isEqualTo(table.rowType().getFieldCount() + 1);
+ long sequence =
+ SortCompactSequenceUtils.sequenceNumber(
+ row, table.rowType().getFieldCount());
+ assertThat(sequence).isGreaterThanOrEqualTo(0L);
+ InternalRow valueRow =
+ SortCompactSequenceUtils.valueRow(
+ row, table.rowType().getFieldCount());
+ if (valueRow.getLong(0) == 1L) {
+ assertThat(valueRow.getLong(1)).isEqualTo(200L);
+ assertThat(sequence).isEqualTo(2L);
+ foundPk1.set(true);
+ }
+ });
+ }
+ assertThat(foundPk1).isTrue();
+ }
+
+ private void writeRow(Table table, GenericRow row) throws Exception {
+ BatchWriteBuilder builder = table.newBatchWriteBuilder();
+ try (BatchTableWrite write = builder.newWrite();
+ BatchTableCommit commit = builder.newCommit()) {
+ write.write(row);
+ commit.commit(write.prepareCommit());
+ }
+ }
+}
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/splitread/PrimaryKeyTableRawFileSplitReadProviderTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/splitread/PrimaryKeyTableRawFileSplitReadProviderTest.java
new file mode 100644
index 000000000000..58e9b7a461d9
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/table/source/splitread/PrimaryKeyTableRawFileSplitReadProviderTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.table.source.splitread;
+
+import org.apache.paimon.io.DataFileMeta;
+import org.apache.paimon.table.source.DataSplit;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.Collections;
+
+import static org.apache.paimon.data.BinaryRow.EMPTY_ROW;
+import static org.apache.paimon.io.DataFileTestUtils.newFile;
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Test for {@link PrimaryKeyTableRawFileSplitReadProvider}. */
+public class PrimaryKeyTableRawFileSplitReadProviderTest {
+
+ @Test
+ public void testBypassRawReadWhenKeyValueSequenceNumberEnabled() {
+ PrimaryKeyTableRawFileSplitReadProvider provider =
+ new PrimaryKeyTableRawFileSplitReadProvider(() -> null, read -> {});
+
+ DataFileMeta file = newFile("data-0.orc", 1, 0, 100, 100, 0L);
+ DataSplit split =
+ DataSplit.builder()
+ .withPartition(EMPTY_ROW)
+ .withBucket(0)
+ .withBucketPath("bucket-0")
+ .withDataFiles(Collections.singletonList(file))
+ .rawConvertible(true)
+ .build();
+
+ assertThat(provider.match(split, new SplitReadProvider.Context(false))).isTrue();
+ assertThat(provider.match(split, new SplitReadProvider.Context(false, true))).isFalse();
+ }
+}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactAction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactAction.java
index cbdcf825f2e2..dbfa5ac3cbcd 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactAction.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactAction.java
@@ -132,7 +132,10 @@ public CompactAction withFullCompaction(Boolean fullCompaction) {
@Override
public void build() throws Exception {
- buildImpl();
+ if (!buildImpl()) {
+ // empty input: no-op topology so procedure execution succeeds without commit
+ env.fromSequence(0, 0).name("Nothing to Sort Compact").sinkTo(new DiscardingSink<>());
+ }
}
protected boolean buildImpl() throws Exception {
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/SortCompactAction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/SortCompactAction.java
index 289802e2ba6b..4c9f341fcd0f 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/SortCompactAction.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/SortCompactAction.java
@@ -20,13 +20,18 @@
import org.apache.paimon.CoreOptions;
import org.apache.paimon.CoreOptions.OrderType;
+import org.apache.paimon.append.SortCompactSequenceUtils;
import org.apache.paimon.flink.FlinkConnectorOptions;
import org.apache.paimon.flink.sink.SortCompactSinkBuilder;
import org.apache.paimon.flink.sorter.TableSortInfo;
import org.apache.paimon.flink.sorter.TableSorter;
import org.apache.paimon.flink.source.FlinkSourceBuilder;
+import org.apache.paimon.partition.PartitionPredicate;
import org.apache.paimon.table.BucketMode;
import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.source.DataSplit;
+import org.apache.paimon.table.source.snapshot.SnapshotReader;
+import org.apache.paimon.types.RowType;
import org.apache.flink.api.common.RuntimeExecutionMode;
import org.apache.flink.configuration.ExecutionOptions;
@@ -38,6 +43,7 @@
import java.util.Arrays;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@@ -61,12 +67,13 @@ public SortCompactAction(
@Override
public void run() throws Exception {
- build();
- execute("Sort Compact Job");
+ if (buildImpl()) {
+ execute("Sort Compact Job");
+ }
}
@Override
- public void build() throws Exception {
+ protected boolean buildImpl() throws Exception {
// only support batch sort yet
if (env.getConfiguration().get(ExecutionOptions.RUNTIME_MODE)
!= RuntimeExecutionMode.BATCH) {
@@ -80,21 +87,59 @@ public void build() throws Exception {
throw new UnsupportedOperationException("Data Evolution table cannot be sorted!");
}
+ if (fileStoreTable.coreOptions().rowTrackingEnabled()) {
+ throw new UnsupportedOperationException(
+ "Sort compact is unsupported for row tracking tables.");
+ }
+
if (fileStoreTable.bucketMode() != BucketMode.BUCKET_UNAWARE
&& fileStoreTable.bucketMode() != BucketMode.HASH_DYNAMIC) {
throw new IllegalArgumentException("Sort Compact only supports bucket=-1 yet.");
}
+
+ // Capture the base snapshot and the planned input splits. The old files in these splits
+ // become compactBefore of the compact commit, so that sort compact is committed as a
+ // normal COMPACT commit (instead of OVERWRITE) and does not drop data appended
+ // concurrently since the base snapshot.
+ PartitionPredicate partitionPredicate = getPartitionPredicate();
+ SnapshotReader snapshotReader = fileStoreTable.newSnapshotReader();
+ if (partitionPredicate != null) {
+ snapshotReader.withPartitionFilter(partitionPredicate);
+ }
+ SnapshotReader.Plan plan = snapshotReader.read();
+ Long baseSnapshotId = plan.snapshotId();
+ List dataSplits = plan.dataSplits();
+ if (dataSplits.isEmpty()) {
+ // empty table or no matching partitions: no-op job with no commit
+ return false;
+ }
+
+ // Pin the source to the captured base snapshot so that the sort compact reads exactly
+ // the planned data, while the sink stays write-only.
+ Map sourceOptions = new HashMap<>();
+ sourceOptions.put(
+ CoreOptions.SCAN_SNAPSHOT_ID.key(),
+ String.valueOf(baseSnapshotId == null ? 0L : baseSnapshotId));
+ RowType sortRowType = fileStoreTable.rowType();
+ if (fileStoreTable.coreOptions().snapshotSequenceOrdering()
+ && !fileStoreTable.primaryKeys().isEmpty()) {
+ sourceOptions.put(CoreOptions.KEY_VALUE_SEQUENCE_NUMBER_ENABLED.key(), "true");
+ sortRowType = SortCompactSequenceUtils.rowTypeWithKeyValueSequenceNumber(sortRowType);
+ }
+ FileStoreTable sourceTable = fileStoreTable.copy(sourceOptions);
+
Map tableConfig = fileStoreTable.options();
FlinkSourceBuilder sourceBuilder =
- new FlinkSourceBuilder(fileStoreTable)
+ new FlinkSourceBuilder(sourceTable)
.sourceName(
ObjectIdentifier.of(
catalogName,
identifier.getDatabaseName(),
identifier.getObjectName())
- .asSummaryString());
+ .asSummaryString())
+ .readType(sortRowType);
- sourceBuilder.partitionPredicate(getPartitionPredicate());
+ sourceBuilder.partitionPredicate(partitionPredicate);
String scanParallelism = tableConfig.get(FlinkConnectorOptions.SCAN_PARALLELISM.key());
if (scanParallelism != null) {
@@ -129,17 +174,14 @@ public void build() throws Exception {
TableSorter sorter =
TableSorter.getSorter(
- env,
- source,
- fileStoreTable.coreOptions(),
- fileStoreTable.rowType(),
- sortInfo);
+ env, source, fileStoreTable.coreOptions(), sortRowType, sortInfo);
new SortCompactSinkBuilder(fileStoreTable)
.forCompact(true)
+ .withSortCompactInput(baseSnapshotId == null ? 0L : baseSnapshotId, dataSplits)
.forRowData(sorter.sort())
- .overwrite()
.build();
+ return true;
}
public SortCompactAction withOrderStrategy(String sortStrategy) {
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupCompactDiffRead.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupCompactDiffRead.java
index e4870de58336..6632cce35c5a 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupCompactDiffRead.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/lookup/LookupCompactDiffRead.java
@@ -49,7 +49,8 @@ public LookupCompactDiffRead(MergeFileSplitRead mergeRead, TableSchema schema) {
mergeRead,
split ->
KeyValueTableRead.unwrap(
- mergeRead.createReader(split), schema.options()));
+ mergeRead.createReader(split),
+ mergeRead.keyValueSequenceNumberEnabled()));
}
@Override
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkSink.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkSink.java
index 7405ae4894e3..ff87b0695854 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkSink.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkSink.java
@@ -87,6 +87,11 @@ public FlinkSink(FileStoreTable table, boolean ignorePreviousFiles) {
this.ignorePreviousFiles = ignorePreviousFiles;
}
+ @Nullable
+ protected StoreSinkWrite.Provider writeProviderOverride() {
+ return null;
+ }
+
public DataStreamSink sinkFrom(DataStream input) {
// This commitUser is valid only for new jobs.
// After the job starts, this commitUser will be recorded into the states of write and
@@ -130,18 +135,21 @@ public DataStream doWrite(
boolean isStreaming = isStreaming(input);
boolean writeOnly = table.coreOptions().writeOnly();
+ StoreSinkWrite.Provider writeProvider = writeProviderOverride();
+ if (writeProvider == null) {
+ writeProvider =
+ StoreSinkWrite.createWriteProvider(
+ table,
+ env.getCheckpointConfig(),
+ isStreaming,
+ ignorePreviousFiles,
+ hasSinkMaterializer(input));
+ }
SingleOutputStreamOperator written =
input.transform(
(writeOnly ? WRITER_WRITE_ONLY_NAME : WRITER_NAME) + " : " + table.name(),
new CommittableTypeInfo(),
- createWriteOperatorFactory(
- StoreSinkWrite.createWriteProvider(
- table,
- env.getCheckpointConfig(),
- isStreaming,
- ignorePreviousFiles,
- hasSinkMaterializer(input)),
- commitUser));
+ createWriteOperatorFactory(writeProvider, commitUser));
if (parallelism == null) {
forwardParallelism(written, input);
} else {
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkSinkBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkSinkBuilder.java
index c3818647477e..6a7ccc8952b2 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkSinkBuilder.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/FlinkSinkBuilder.java
@@ -88,13 +88,18 @@ public class FlinkSinkBuilder {
private DataStream input;
@Nullable protected Map overwritePartition;
- @Nullable private Integer parallelism;
+ @Nullable protected Integer parallelism;
@Nullable private TableSortInfo tableSortInfo;
// ============== for extension ==============
protected boolean compactSink = false;
+ /** Row type used to convert sink input {@link RowData} to {@link InternalRow}. */
+ protected org.apache.paimon.types.RowType sinkInputRowType() {
+ return table.rowType();
+ }
+
public FlinkSinkBuilder(Table table) {
if (!(table instanceof FileStoreTable)) {
throw new UnsupportedOperationException("Unsupported table type: " + table);
@@ -221,11 +226,33 @@ public DataStreamSink build() {
DataStream input =
mapToInternalRow(
this.input,
- table.rowType(),
+ sinkInputRowType(),
contextForDescriptor,
table.coreOptions().blobWriteNullOnMissingFile(),
table.coreOptions().blobWriteNullOnFetchFailure());
- if (table.coreOptions().localMergeEnabled() && table.schema().primaryKeys().size() > 0) {
+ boolean localMergeSupported = sinkInputRowType().equals(table.schema().logicalRowType());
+ // LocalMergeOperator is built from table.schema(), whose logical row type does not include
+ // the prepended _SEQUENCE_NUMBER column used by the sort compact sequence-ordering path
+ // (see SortCompactSinkBuilder.sinkInputRowType). Its key projection uses unshifted field
+ // positions and would treat the sequence column as a user column, corrupting the local
+ // merge. Local merge is therefore unsupported for that path; warn loudly and skip it
+ // rather than silently dropping the user-configured local-merge-buffer-size.
+ if (table.coreOptions().localMergeEnabled()
+ && table.schema().primaryKeys().size() > 0
+ && !localMergeSupported) {
+ LOG.warn(
+ "Local merge ("
+ + CoreOptions.LOCAL_MERGE_BUFFER_SIZE.key()
+ + ") is enabled but is not supported for sort compact with snapshot "
+ + "sequence ordering, because the local merge operator is not aware "
+ + "of the prepended _SEQUENCE_NUMBER column. Skipping local merge for "
+ + "this job. To avoid this warning, unset "
+ + CoreOptions.LOCAL_MERGE_BUFFER_SIZE.key()
+ + " for this table before running sort compact.");
+ }
+ if (table.coreOptions().localMergeEnabled()
+ && table.schema().primaryKeys().size() > 0
+ && localMergeSupported) {
SingleOutputStreamOperator newInput =
input.forward()
.transform(
@@ -352,7 +379,7 @@ private DataStreamSink buildPostponeBucketSink(DataStream input)
}
}
- private DataStreamSink buildUnawareBucketSink(DataStream input) {
+ protected DataStreamSink buildUnawareBucketSink(DataStream input) {
checkArgument(
table.primaryKeys().isEmpty(),
"Unaware bucket mode only works with append-only table for now.");
@@ -370,7 +397,12 @@ private DataStreamSink buildUnawareBucketSink(DataStream input)
}
}
- return new RowAppendTableSink(table, overwritePartition, parallelism).sinkFrom(input);
+ return createAppendTableSink().sinkFrom(input);
+ }
+
+ /** Create the {@link RowAppendTableSink} for the unaware bucket mode. */
+ protected RowAppendTableSink createAppendTableSink() {
+ return new RowAppendTableSink(table, overwritePartition, parallelism);
}
private DataStream applyDynamicPartitionShuffle(DataStream input) {
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/SortCompactAppendSinkWrite.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/SortCompactAppendSinkWrite.java
new file mode 100644
index 000000000000..7e47bd1d7124
--- /dev/null
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/SortCompactAppendSinkWrite.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.flink.sink;
+
+import org.apache.paimon.memory.MemoryPoolFactory;
+import org.apache.paimon.table.FileStoreTable;
+
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.runtime.io.disk.iomanager.IOManager;
+
+import javax.annotation.Nullable;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * {@link StoreSinkWrite} for sort compact on bucket-unaware append tables.
+ *
+ * Always uses {@code waitCompaction=false} so sorted output lands in {@code newFilesIncrement}.
+ * The committer rewrites those files into {@code compactAfter}.
+ */
+public class SortCompactAppendSinkWrite extends StoreSinkWriteImpl {
+
+ public SortCompactAppendSinkWrite(
+ FileStoreTable table,
+ String commitUser,
+ StoreSinkWriteState state,
+ IOManager ioManager,
+ boolean ignorePreviousFiles,
+ boolean isStreamingMode,
+ MemoryPoolFactory memoryPoolFactory,
+ @Nullable MetricGroup metricGroup) {
+ super(
+ table,
+ commitUser,
+ state,
+ ioManager,
+ ignorePreviousFiles,
+ false,
+ isStreamingMode,
+ memoryPoolFactory,
+ metricGroup);
+ }
+
+ @Override
+ public List prepareCommit(boolean waitCompaction, long checkpointId)
+ throws IOException {
+ // Batch endInput always passes waitCompaction=true, but sort compact write must not wait
+ // for inline compaction. The committer rewrites append-style newFiles into compactAfter.
+ return super.prepareCommit(false, checkpointId);
+ }
+
+ public static StoreSinkWrite.Provider provider() {
+ return (table, commitUser, state, ioManager, memoryPoolFactory, metricGroup) ->
+ new SortCompactAppendSinkWrite(
+ table,
+ commitUser,
+ state,
+ ioManager,
+ true,
+ false,
+ memoryPoolFactory,
+ metricGroup);
+ }
+}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/SortCompactAppendTableSink.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/SortCompactAppendTableSink.java
new file mode 100644
index 000000000000..e65ce683bc15
--- /dev/null
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/SortCompactAppendTableSink.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.flink.sink;
+
+import org.apache.paimon.append.SortCompactCommitMessageRewriter;
+import org.apache.paimon.append.SortCompactPlanMetadata;
+import org.apache.paimon.flink.sink.StoreSinkWrite.Provider;
+import org.apache.paimon.manifest.ManifestCommittable;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.source.DataSplit;
+
+import javax.annotation.Nullable;
+
+import java.util.List;
+
+/**
+ * A {@link RowAppendTableSink} for the sort compact topology of bucket-unaware append tables. It
+ * produces a {@link SortCompactCommitter} so that the written append files are committed as a
+ * {@code COMPACT} snapshot.
+ *
+ * The compact-before files (captured by the sort compact plan) are carried as {@link DataSplit}s
+ * (which are serializable) into the job graph and used at commit time to build the compact commit
+ * messages.
+ *
+ *
Scale note: see {@link SortCompactSinkBuilder#withSortCompactInput(long, List)}.
+ */
+public class SortCompactAppendTableSink extends RowAppendTableSink {
+
+ private static final long serialVersionUID = 1L;
+
+ private final long baseSnapshotId;
+ private final List compactInputSplits;
+ private final SortCompactPlanMetadata planMetadata;
+
+ public SortCompactAppendTableSink(
+ FileStoreTable table,
+ @Nullable Integer parallelism,
+ long baseSnapshotId,
+ List compactInputSplits) {
+ super(table, null, parallelism);
+ this.baseSnapshotId = baseSnapshotId;
+ this.compactInputSplits = compactInputSplits;
+ this.planMetadata =
+ SortCompactPlanMetadata.capture(table, baseSnapshotId, compactInputSplits);
+ }
+
+ @Override
+ protected Provider writeProviderOverride() {
+ if (table.coreOptions().snapshotSequenceOrdering() && !table.primaryKeys().isEmpty()) {
+ return SortCompactSinkWrite.provider();
+ }
+ return SortCompactAppendSinkWrite.provider();
+ }
+
+ @Override
+ protected Committer.Factory createCommitterFactory() {
+ // If checkpoint is enabled for streaming job, we have to commit new files list even if
+ // they're empty. Otherwise we can't tell if the commit is successful after a restart.
+ return context ->
+ new SortCompactCommitter(
+ table,
+ table.newCommit(context.commitUser())
+ .ignoreEmptyCommit(!context.streamingCheckpointEnabled()),
+ context,
+ new SortCompactCommitMessageRewriter(
+ table, baseSnapshotId, compactInputSplits, planMetadata));
+ }
+}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/SortCompactCommitter.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/SortCompactCommitter.java
new file mode 100644
index 000000000000..0f8d1e994350
--- /dev/null
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/SortCompactCommitter.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.flink.sink;
+
+import org.apache.paimon.append.SortCompactCommitMessageRewriter;
+import org.apache.paimon.manifest.ManifestCommittable;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.sink.CommitMessage;
+import org.apache.paimon.table.sink.CommitMessageImpl;
+import org.apache.paimon.table.sink.TableCommit;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * A {@link StoreCommitter} for the sort compact topology. It rewrites the append {@link
+ * CommitMessage}s produced by the sort compact write stage into compact commit messages (via {@link
+ * SortCompactCommitMessageRewriter}) before delegating to the normal {@link StoreCommitter} commit,
+ * so that the resulting snapshot is a {@code COMPACT} snapshot instead of an {@code OVERWRITE}
+ * snapshot.
+ */
+public class SortCompactCommitter extends StoreCommitter {
+
+ private final SortCompactCommitMessageRewriter rewriter;
+
+ public SortCompactCommitter(
+ FileStoreTable table,
+ TableCommit commit,
+ Context context,
+ SortCompactCommitMessageRewriter rewriter) {
+ super(table, commit, context);
+ this.rewriter = rewriter;
+ }
+
+ @Override
+ protected long additionalBytesOut(CommitMessageImpl impl) {
+ return calcTotalFileSize(impl.compactIncrement().compactAfter());
+ }
+
+ @Override
+ protected long additionalRecordsOut(CommitMessageImpl impl) {
+ return calcTotalFileRowCount(impl.compactIncrement().compactAfter());
+ }
+
+ @Override
+ public void commit(List committables)
+ throws java.io.IOException, InterruptedException {
+ super.commit(rewriteAll(committables));
+ }
+
+ @Override
+ public int filterAndCommit(
+ List globalCommittables,
+ boolean checkAppendFiles,
+ boolean partitionMarkDoneRecoverFromState) {
+ List sortedCommittables =
+ globalCommittables.stream()
+ .sorted(Comparator.comparingLong(ManifestCommittable::identifier))
+ .collect(Collectors.toList());
+ List retryCommittables = commit.filterCommitted(sortedCommittables);
+ if (retryCommittables.isEmpty()) {
+ // Delete-only compact commits are only valid at job end (filterAndCommit with
+ // checkAppendFiles=false, e.g. CommitterOperator#endInput). Recovery paths call
+ // filterAndCommit with checkAppendFiles=true and an empty restored list; treating
+ // that as delete-only would remove all planned input files before writers run.
+ if (!checkAppendFiles
+ && sortedCommittables.isEmpty()
+ && rewriter.hasInput()
+ && !rewriter.isPlannedInputAlreadyCommitted()) {
+ return filterAndCommitDeleteOnly(
+ globalCommittables, checkAppendFiles, partitionMarkDoneRecoverFromState);
+ }
+ commitListeners.notifyCommittable(
+ globalCommittables, partitionMarkDoneRecoverFromState);
+ return 0;
+ }
+
+ List rewritten = rewriteAll(retryCommittables);
+ int committed = commit.filterAndCommitMultiple(rewritten, checkAppendFiles);
+ calcNumBytesAndRecordsOut(rewritten);
+ commitListeners.notifyCommittable(globalCommittables, partitionMarkDoneRecoverFromState);
+ return committed;
+ }
+
+ private int filterAndCommitDeleteOnly(
+ List globalCommittables,
+ boolean checkAppendFiles,
+ boolean partitionMarkDoneRecoverFromState) {
+ List rewritten = rewriteAll(Collections.emptyList());
+ if (rewritten.isEmpty()) {
+ return 0;
+ }
+ int committed = commit.filterAndCommitMultiple(rewritten, checkAppendFiles);
+ calcNumBytesAndRecordsOut(rewritten);
+ commitListeners.notifyCommittable(globalCommittables, partitionMarkDoneRecoverFromState);
+ return committed;
+ }
+
+ /**
+ * Merge all partial write outputs from multiple committables and rewrite once. Each individual
+ * rewrite would attach the full planned {@code compactBefore}, so committing partial outputs
+ * separately would delete all old files too early.
+ *
+ * This also collapses multiple committables into a single one with the maximum identifier,
+ * which changes the per-identifier deduplication semantics used during job recovery. Sort
+ * compact is a batch job and does not rely on that recovery path, so this is acceptable here.
+ */
+ private List rewriteAll(List committables) {
+ if (committables.isEmpty()) {
+ if (!rewriter.hasInput()) {
+ return committables;
+ }
+
+ // A sort compact is a batch job. Even when all input rows are filtered out (for
+ // example, by deletion vectors), its planned input files still need to be removed.
+ return Collections.singletonList(
+ new ManifestCommittable(
+ Long.MAX_VALUE,
+ null,
+ rewriter.rewrite(Collections.emptyList()),
+ Collections.emptyMap()));
+ }
+
+ List allWrittenMessages = new ArrayList<>();
+ long identifier = Long.MIN_VALUE;
+ Long watermark = null;
+ Map properties = new HashMap<>();
+
+ for (ManifestCommittable committable : committables) {
+ allWrittenMessages.addAll(committable.fileCommittables());
+ identifier = Math.max(identifier, committable.identifier());
+ if (committable.watermark() != null) {
+ watermark =
+ watermark == null
+ ? committable.watermark()
+ : Math.max(watermark, committable.watermark());
+ }
+ properties.putAll(committable.properties());
+ }
+
+ List compactMessages = rewriter.rewrite(allWrittenMessages);
+ return Collections.singletonList(
+ new ManifestCommittable(identifier, watermark, compactMessages, properties));
+ }
+}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/SortCompactDynamicBucketSink.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/SortCompactDynamicBucketSink.java
new file mode 100644
index 000000000000..f8230800d8b9
--- /dev/null
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/SortCompactDynamicBucketSink.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.flink.sink;
+
+import org.apache.paimon.append.SortCompactCommitMessageRewriter;
+import org.apache.paimon.append.SortCompactPlanMetadata;
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.flink.sink.StoreSinkWrite.Provider;
+import org.apache.paimon.manifest.ManifestCommittable;
+import org.apache.paimon.schema.TableSchema;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.sink.PartitionKeyExtractor;
+import org.apache.paimon.table.sink.SequencePreservingPartitionKeyExtractor;
+import org.apache.paimon.table.source.DataSplit;
+import org.apache.paimon.utils.SerializableFunction;
+
+import java.util.List;
+
+/**
+ * A {@link DynamicBucketCompactSink} for the sort compact topology of hash-dynamic bucket tables.
+ * It produces a {@link SortCompactCommitter} so that the written append files are committed as a
+ * {@code COMPACT} snapshot.
+ *
+ * The compact-before files (captured by the sort compact plan) are carried as {@link DataSplit}s
+ * (which are serializable) into the job graph and used at commit time to build the compact commit
+ * messages.
+ *
+ *
Scale note: each planned {@link DataSplit} embeds full {@link
+ * org.apache.paimon.io.DataFileMeta} objects and is serialized into the committer factory closure.
+ * For tables with very large numbers of input files (hundreds of thousands or more), this can
+ * significantly inflate the Flink job graph and RPC payload. Consider compacting in smaller
+ * partition batches when approaching that scale.
+ */
+public class SortCompactDynamicBucketSink extends DynamicBucketCompactSink {
+
+ private static final long serialVersionUID = 1L;
+
+ private final long baseSnapshotId;
+ private final List compactInputSplits;
+ private final SortCompactPlanMetadata planMetadata;
+
+ public SortCompactDynamicBucketSink(
+ FileStoreTable table, long baseSnapshotId, List compactInputSplits) {
+ super(table, null);
+ this.baseSnapshotId = baseSnapshotId;
+ this.compactInputSplits = compactInputSplits;
+ this.planMetadata =
+ SortCompactPlanMetadata.capture(table, baseSnapshotId, compactInputSplits);
+ }
+
+ @Override
+ protected Provider writeProviderOverride() {
+ return SortCompactSinkWrite.provider();
+ }
+
+ @Override
+ protected SerializableFunction>
+ extractorFunction() {
+ if (table.coreOptions().snapshotSequenceOrdering() && !table.primaryKeys().isEmpty()) {
+ return SequencePreservingPartitionKeyExtractor::new;
+ }
+ return super.extractorFunction();
+ }
+
+ @Override
+ protected CommittableStateManager createCommittableStateManager() {
+ // Batch sort compact is a one-shot job; use restore-only semantics like append sort
+ // compact instead of restore-and-fail, which intentionally fails after recovery commit.
+ return FlinkWriteSink.createRestoreOnlyCommittableStateManager(table);
+ }
+
+ @Override
+ protected Committer.Factory createCommitterFactory() {
+ // If checkpoint is enabled for streaming job, we have to commit new files list even if
+ // they're empty. Otherwise we can't tell if the commit is successful after a restart.
+ return context ->
+ new SortCompactCommitter(
+ table,
+ table.newCommit(context.commitUser())
+ .ignoreEmptyCommit(!context.streamingCheckpointEnabled()),
+ context,
+ new SortCompactCommitMessageRewriter(
+ table, baseSnapshotId, compactInputSplits, planMetadata));
+ }
+}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/SortCompactSinkBuilder.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/SortCompactSinkBuilder.java
index c30ebc824b85..7b05e514f724 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/SortCompactSinkBuilder.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/SortCompactSinkBuilder.java
@@ -18,17 +18,113 @@
package org.apache.paimon.flink.sink;
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.append.SortCompactSequenceUtils;
+import org.apache.paimon.data.InternalRow;
import org.apache.paimon.table.Table;
+import org.apache.paimon.table.source.DataSplit;
+import org.apache.paimon.types.RowType;
+
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.DataStreamSink;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+
+import java.util.List;
/** A special version {@link FlinkSinkBuilder} for sort compact. */
public class SortCompactSinkBuilder extends FlinkSinkBuilder {
+ private static final Logger LOG = LoggerFactory.getLogger(SortCompactSinkBuilder.class);
+
+ private long baseSnapshotId;
+ @Nullable private List compactInputSplits;
+ private boolean sortCompactInputSet = false;
+
public SortCompactSinkBuilder(Table table) {
super(table);
}
- public FlinkSinkBuilder forCompact(boolean compactSink) {
+ public SortCompactSinkBuilder forCompact(boolean compactSink) {
this.compactSink = compactSink;
return this;
}
+
+ /**
+ * Capture the base snapshot id and the planned compact input splits of the sort compact. The
+ * splits (serializable) are carried into the job graph and used at commit time to rewrite the
+ * written append files into a compact commit.
+ *
+ * Scale note: each {@link DataSplit} embeds full file metadata and is serialized into
+ * the committer factory closure. For tables with very large numbers of input files, this can
+ * significantly inflate the Flink job graph. Consider compacting in smaller partition batches
+ * when approaching hundreds of thousands of files.
+ */
+ public SortCompactSinkBuilder withSortCompactInput(
+ long baseSnapshotId, List compactInputSplits) {
+ validateSortCompactInput(compactInputSplits);
+ this.baseSnapshotId = baseSnapshotId;
+ this.compactInputSplits = compactInputSplits;
+ this.sortCompactInputSet = true;
+ return this;
+ }
+
+ private void validateSortCompactInput(List compactInputSplits) {
+ long inputFileCount =
+ compactInputSplits.stream().mapToLong(split -> split.dataFiles().size()).sum();
+ int warnThreshold = table.coreOptions().sortCompactionWarnInputFiles();
+ int maxThreshold = table.coreOptions().sortCompactionMaxInputFiles();
+ if (inputFileCount > warnThreshold) {
+ LOG.warn(
+ "Sort compact plan contains {} input files across {} splits, which exceeds the "
+ + "warn threshold {}. Each input file is serialized into the Flink job "
+ + "graph and may significantly inflate job submission time and RPC "
+ + "payload. Consider compacting in smaller partition batches or raising "
+ + "'{}' if this is expected.",
+ inputFileCount,
+ compactInputSplits.size(),
+ warnThreshold,
+ CoreOptions.SORT_COMPACTION_WARN_INPUT_FILES.key());
+ }
+ if (inputFileCount > maxThreshold) {
+ throw new IllegalArgumentException(
+ String.format(
+ "Sort compact plan contains %d input files across %d splits, which "
+ + "exceeds the limit %d. Compact in smaller partition batches "
+ + "or raise '%s'.",
+ inputFileCount,
+ compactInputSplits.size(),
+ maxThreshold,
+ CoreOptions.SORT_COMPACTION_MAX_INPUT_FILES.key()));
+ }
+ }
+
+ @Override
+ protected RowType sinkInputRowType() {
+ if (table.coreOptions().snapshotSequenceOrdering() && !table.primaryKeys().isEmpty()) {
+ return SortCompactSequenceUtils.rowTypeWithKeyValueSequenceNumber(table.rowType());
+ }
+ return table.rowType();
+ }
+
+ @Override
+ protected RowAppendTableSink createAppendTableSink() {
+ if (sortCompactInputSet) {
+ return new SortCompactAppendTableSink(
+ table, parallelism, baseSnapshotId, compactInputSplits);
+ }
+ return super.createAppendTableSink();
+ }
+
+ @Override
+ protected DataStreamSink buildDynamicBucketSink(
+ DataStream input, boolean globalIndex) {
+ if (sortCompactInputSet && compactSink && !globalIndex) {
+ return new SortCompactDynamicBucketSink(table, baseSnapshotId, compactInputSplits)
+ .build(input, parallelism);
+ }
+ return super.buildDynamicBucketSink(input, globalIndex);
+ }
}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/SortCompactSinkWrite.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/SortCompactSinkWrite.java
new file mode 100644
index 000000000000..38cc205d6419
--- /dev/null
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/SortCompactSinkWrite.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.flink.sink;
+
+import org.apache.paimon.KeyValue;
+import org.apache.paimon.KeyValueFileStore;
+import org.apache.paimon.append.SortCompactSequenceUtils;
+import org.apache.paimon.flink.metrics.FlinkMetricRegistry;
+import org.apache.paimon.memory.MemoryPoolFactory;
+import org.apache.paimon.operation.FileStoreWrite;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.sink.RowKindGenerator;
+import org.apache.paimon.table.sink.SequencePreservingRowKeyExtractor;
+import org.apache.paimon.table.sink.TableWriteImpl;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.RowKindFilter;
+
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.flink.runtime.io.disk.iomanager.IOManager;
+
+import javax.annotation.Nullable;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * {@link StoreSinkWrite} for sort compact on primary-key tables.
+ *
+ * Always uses {@code withSortCompactWrite(true)}, {@code ignorePreviousFiles=true}, and {@code
+ * waitCompaction=false} so sorted output lands in {@code newFilesIncrement} and write-stage
+ * changelog is disabled. Snapshot-ordering tables additionally preserve {@code _SEQUENCE_NUMBER}.
+ */
+public class SortCompactSinkWrite extends StoreSinkWriteImpl {
+
+ public SortCompactSinkWrite(
+ FileStoreTable table,
+ String commitUser,
+ StoreSinkWriteState state,
+ IOManager ioManager,
+ boolean ignorePreviousFiles,
+ boolean waitCompaction,
+ boolean isStreamingMode,
+ MemoryPoolFactory memoryPoolFactory,
+ @Nullable MetricGroup metricGroup) {
+ super(
+ table,
+ commitUser,
+ state,
+ ioManager,
+ ignorePreviousFiles,
+ waitCompaction,
+ isStreamingMode,
+ memoryPoolFactory,
+ metricGroup);
+ }
+
+ @Override
+ protected TableWriteImpl newTableWrite(FileStoreTable table) {
+ // Use table.rowType() directly: super() calls this method before subclass fields are
+ // initialized.
+ FileStoreWrite storeWrite =
+ ((KeyValueFileStore) table.store())
+ .newWrite(commitUser, state.getSubtaskId())
+ .withSortCompactWrite(true);
+ TableWriteImpl tableWrite;
+ if (table.coreOptions().snapshotSequenceOrdering() && !table.primaryKeys().isEmpty()) {
+ RowType logicalRowType = table.rowType();
+ KeyValue reuse = new KeyValue();
+ tableWrite =
+ new TableWriteImpl<>(
+ SortCompactSequenceUtils.rowTypeWithKeyValueSequenceNumber(
+ logicalRowType),
+ storeWrite,
+ new SequencePreservingRowKeyExtractor(table.schema()),
+ SortCompactSequenceUtils.sequencePreservingExtractor(
+ logicalRowType, reuse),
+ null,
+ RowKindFilter.of(table.coreOptions()));
+ } else {
+ KeyValue reuse = new KeyValue();
+ tableWrite =
+ new TableWriteImpl<>(
+ table.rowType(),
+ storeWrite,
+ table.createRowKeyExtractor(),
+ (record, rowKind) ->
+ reuse.replace(
+ record.primaryKey(),
+ KeyValue.UNKNOWN_SEQUENCE,
+ rowKind,
+ record.row()),
+ RowKindGenerator.create(table.schema(), table.store().options()),
+ RowKindFilter.of(table.coreOptions()));
+ }
+ tableWrite
+ .withIOManager(paimonIOManager)
+ .withIgnorePreviousFiles(ignorePreviousFiles)
+ .withMemoryPoolFactory(memoryPoolFactory);
+ if (metricGroup != null) {
+ tableWrite.withMetricRegistry(new FlinkMetricRegistry(metricGroup));
+ }
+ return tableWrite;
+ }
+
+ @Override
+ public List prepareCommit(boolean waitCompaction, long checkpointId)
+ throws IOException {
+ // Batch endInput always passes waitCompaction=true, but sort compact write must not wait
+ // for inline compaction. The committer rewrites append-style newFiles into compactAfter.
+ return super.prepareCommit(false, checkpointId);
+ }
+
+ public static StoreSinkWrite.Provider provider() {
+ return (table, commitUser, state, ioManager, memoryPoolFactory, metricGroup) ->
+ new SortCompactSinkWrite(
+ table,
+ commitUser,
+ state,
+ ioManager,
+ true,
+ false,
+ false,
+ memoryPoolFactory,
+ metricGroup);
+ }
+}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreCommitter.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreCommitter.java
index 4c353517c3f0..e8786278aaf4 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreCommitter.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/StoreCommitter.java
@@ -44,9 +44,9 @@
/** {@link Committer} for dynamic store. */
public class StoreCommitter implements Committer {
- private final TableCommitImpl commit;
+ protected final TableCommitImpl commit;
@Nullable private final CommitterMetrics committerMetrics;
- private final CommitListeners commitListeners;
+ protected final CommitListeners commitListeners;
@Nullable private final IOManager commitIOManager;
private final boolean allowLogOffsetDuplicate;
@@ -150,7 +150,7 @@ public boolean allowLogOffsetDuplicate() {
return allowLogOffsetDuplicate;
}
- private void calcNumBytesAndRecordsOut(List committables) {
+ protected void calcNumBytesAndRecordsOut(List committables) {
if (committerMetrics == null) {
return;
}
@@ -160,25 +160,32 @@ private void calcNumBytesAndRecordsOut(List committables) {
for (ManifestCommittable committable : committables) {
List