Skip to content

[core] Reuse sidecar block cache for unfiltered scans #9924

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JingsongLi merged 2 commits into from
Sep 17, 2026
Merged

[core] Reuse sidecar block cache for unfiltered scans #9924

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
00fbe9d
[core] Reuse sidecar block cache for unfiltered scans
JingsongLi Sep 17, 2026
d3e178e
[core] Preserve metrics for manifest block cache
JingsongLi Sep 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions docs/docs/concepts/spec/manifest.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,9 +77,11 @@ writers and scans use sidecars when `manifest.sidecar.enabled` is true; when uns
the completed output manifest and publish its `_EXTRA_FILES` reference only after both files
close successfully. Failed writes and aborted writers clean up their own manifest/sidecar pairs.
Scans with partition, row-ID or bucket filters select blocks before reading manifest entries.
Normal entry filtering and ADD/DELETE reconciliation still apply. Missing or unusable sidecars
fall back to normal manifest reads; disabled sidecars and scans without these filters do not
perform sidecar I/O. Sidecar caching is controlled by the catalog option
Unfiltered scans also select all sidecar blocks when a manifest cache is configured, so unfiltered
prefetches and later filtered reads share the same block cache. Without a manifest cache, unfiltered
scans keep the normal whole-manifest read path. Normal entry filtering and ADD/DELETE reconciliation
still apply. Missing or unusable sidecars fall back to normal manifest reads; disabled sidecars do
not perform sidecar I/O. Sidecar caching is controlled by the catalog option
`cache.manifest-sidecar.max-memory` (64 MiB by default). A positive value supplies an
additional budget independent of the manifest content cache. When set to 0, sidecars reuse
the manifest content cache, or remain uncached if that cache is disabled. Sidecar caching
Expand Down
50 changes: 35 additions & 15 deletions paimon-core/src/main/java/org/apache/paimon/manifest/ManifestFile.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ public class ManifestFile extends ObjectsFile<ManifestEntry> {
private final long suggestedFileSize;
private final CoreOptions options;
@Nullable private final SegmentsCache<Path> sidecarCache;
@Nullable private CacheMetrics cacheMetrics;

private ManifestFile(
FileIO fileIO,
Expand Down Expand Up @@ -116,6 +117,7 @@ protected ManifestEntryCache createCache(
@Override
public ManifestFile withCacheMetrics(@Nullable CacheMetrics cacheMetrics) {
super.withCacheMetrics(cacheMetrics);
this.cacheMetrics = cacheMetrics;
return this;
}

Expand Down Expand Up @@ -177,16 +179,32 @@ public <T> List<T> read(
return cache.read(path, fileSize, filters, convertor);
}

CloseableIterator<InternalRow> iterator =
createManifestIterator(
fileIO,
path,
ManifestEntry.MANIFEST_ROW_TYPE,
partitionFilter,
bucketFilter,
selected,
cache == null ? null : cache.segmentsCache());
return readFromIterator(iterator, serializer, readFilter, readTFilter, convertor);
CacheMetrics metrics = cacheMetrics;
ManifestSidecar.CacheStatus cacheStatus =
selected != null && cache != null && metrics != null
? new ManifestSidecar.CacheStatus()
: null;
try {
CloseableIterator<InternalRow> iterator =
createManifestIterator(
fileIO,
path,
ManifestEntry.MANIFEST_ROW_TYPE,
partitionFilter,
bucketFilter,
selected,
cache == null ? null : cache.segmentsCache(),
cacheStatus);
return readFromIterator(iterator, serializer, readFilter, readTFilter, convertor);
} finally {
if (cacheStatus != null) {
if (cacheStatus.hit()) {
metrics.increaseHitObject();
} else {
metrics.increaseMissedObject();
}
}
}
} catch (IOException e) {
throw new UncheckedIOException(e);
}
Expand Down Expand Up @@ -240,7 +258,7 @@ private static CloseableIterator<InternalRow> createManifestIterator(
@Nullable BucketFilter bucketFilter)
throws IOException {
return createManifestIterator(
fileIO, path, projectedType, partitionFilter, bucketFilter, null, null);
fileIO, path, projectedType, partitionFilter, bucketFilter, null, null, null);
}

private static CloseableIterator<InternalRow> createManifestIterator(
Expand All @@ -250,12 +268,14 @@ private static CloseableIterator<InternalRow> createManifestIterator(
@Nullable PartitionPredicate partitionFilter,
@Nullable BucketFilter bucketFilter,
@Nullable ManifestSidecar.Selection selected,
@Nullable SegmentsCache<Object> cache)
@Nullable SegmentsCache<Object> cache,
@Nullable ManifestSidecar.CacheStatus cacheStatus)
throws IOException {
try {
ManifestAvroReader reader =
new ManifestAvroReader(
ManifestSidecar.openManifest(fileIO, path, selected, cache));
ManifestSidecar.openManifest(
fileIO, path, selected, cache, cacheStatus));
return reader.read(projectedType, partitionFilter, bucketFilter);
} catch (IOException e) {
FileUtils.checkExists(fileIO, path);
Expand Down Expand Up @@ -402,8 +422,8 @@ public ManifestSidecar.Selection selectBlocks(
@Nullable RowRangeIndex query,
@Nullable PartitionPredicate partitionFilter,
@Nullable BucketFilter bucketFilter) {
return !options.manifestSidecarEnabled()
|| (query == null && partitionFilter == null && bucketFilter == null)
boolean hasFilter = query != null || partitionFilter != null || bucketFilter != null;
return !options.manifestSidecarEnabled() || (!hasFilter && cache == null)

@leaves12138 leaves12138 Sep 17, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could we preserve the cache hit/miss metrics when routing unfiltered scans through the block cache?

Previously, an unfiltered scan used ObjectsCache.read(), which updates the CacheMetrics attached by withCacheMetrics(). With this change, a usable sidecar produces a non-null selection, so ManifestFile.read() bypasses that method. SelectedBlockInput accesses SegmentsCache directly and never updates these counters.

I reproduced this with one sidecar-backed manifest and an enabled manifest cache: perform one cold unfiltered read and then one warm unfiltered read. The second read performs no file I/O, but both manifestHitCache and manifestMissedCache remain 0, rather than recording one hit and one miss. The regression test fails on this commit and passes when only the previous selectBlocks() condition is restored.

This does not change query results, but it makes the existing cache observability silently stop working for unfiltered sidecar-backed scans as well. Please carry the metrics into the block-cache path and preserve the per-manifest accounting, with a regression test covering cold and warm reads.

@JingsongLi JingsongLi Sep 17, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in d3e178e. The selected-block reader now tracks actual block-cache accesses and reports exactly one manifest-level hit or miss after each read: any uncached selected block is a miss, while an all-block cache hit is a hit. The regression test covers one cold unfiltered read followed by one warm unfiltered read and verifies both I/O and the hit/miss counters. Five focused ManifestFile tests and the paimon-core compile with Checkstyle/Spotless pass.

? null
: ManifestSidecar.read(
fileIO,
Expand Down
50 changes: 47 additions & 3 deletions paimon-core/src/main/java/org/apache/paimon/manifest/ManifestSidecar.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -692,7 +692,7 @@ private static byte[] readBytes(FileIO io, Path path) throws IOException {

static InputStream openManifest(FileIO io, Path path, @Nullable Selection selected)
throws IOException {
return openManifest(io, path, selected, null);
return openManifest(io, path, selected, null, null);
}

static InputStream openManifest(
Expand All @@ -701,9 +701,38 @@ static InputStream openManifest(
@Nullable Selection selected,
@Nullable SegmentsCache<Object> cache)
throws IOException {
return openManifest(io, path, selected, cache, null);
}

static InputStream openManifest(
FileIO io,
Path path,
@Nullable Selection selected,
@Nullable SegmentsCache<Object> cache,
@Nullable CacheStatus cacheStatus)
throws IOException {
return selected == null
? io.newInputStream(path)
: new SelectedBlockInput(io, path, selected, cache);
: new SelectedBlockInput(io, path, selected, cache, cacheStatus);
}

/** Per-manifest status for reporting whether every selected block was served by the cache. */
static final class CacheStatus {
private boolean accessed;
private boolean missed;

private void hitBlock() {
accessed = true;
}

private void miss() {
accessed = true;
missed = true;
}

boolean hit() {
return accessed && !missed;
}
}

/** Separates physical byte ranges from whole-file cache keys. */
Expand Down Expand Up @@ -753,6 +782,7 @@ private static final class SelectedBlockInput extends InputStream {
private final Path path;
private final Selection selected;
@Nullable private final SegmentsCache<Object> cache;
@Nullable private final CacheStatus cacheStatus;
@Nullable private SeekableInputStream input;
private boolean closed;
private int headerPosition;
Expand All @@ -763,11 +793,16 @@ private static final class SelectedBlockInput extends InputStream {
private int bufferLimit;

private SelectedBlockInput(
FileIO io, Path path, Selection selected, @Nullable SegmentsCache<Object> cache) {
FileIO io,
Path path,
Selection selected,
@Nullable SegmentsCache<Object> cache,
@Nullable CacheStatus cacheStatus) {
this.io = io;
this.path = path;
this.selected = selected;
this.cache = cache;
this.cacheStatus = cacheStatus;
}

@Override
Expand Down Expand Up @@ -822,6 +857,9 @@ private boolean fillBuffer() throws IOException {
> cache.maxElementSize())) {
end += selected.blocks.get(blockPosition++).length;
}
if (cacheStatus != null) {
cacheStatus.miss();
}
seekInput(block.offset);
remaining = end - block.offset;
}
Expand All @@ -841,9 +879,15 @@ private boolean fillBuffer() throws IOException {
private void readCachedBlocks(Block first) throws IOException {
byte[] cached = cachedBlock(first);
if (cached != null) {
if (cacheStatus != null) {
cacheStatus.hitBlock();
}
blockPosition++;
buffer = cached;
} else {
if (cacheStatus != null) {
cacheStatus.miss();
}
int firstPosition = blockPosition++;
long end = first.offset + first.length;
while (blockPosition < selected.blocks.size()) {
Expand Down
107 changes: 104 additions & 3 deletions paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import org.apache.paimon.io.DataFileMetaWriteColsLegacySerializer;
import org.apache.paimon.operation.AppendOnlyFileStoreScan;
import org.apache.paimon.operation.ManifestsReader;
import org.apache.paimon.operation.metrics.CacheMetrics;
import org.apache.paimon.options.MemorySize;
import org.apache.paimon.options.Options;
import org.apache.paimon.partition.PartitionPredicate;
Expand Down Expand Up @@ -1856,15 +1857,16 @@ void testPartitionOnlyPlanningUsesBlocksWithoutRowIds() {
}

@Test
void testUnknownRowIdKeepsPartitionIndexAndNoQueryDoesNotReadSidecar() {
void testUnfilteredReadWithoutCacheSkipsSidecar() {
Options options = new Options();
options.set(CoreOptions.DATA_EVOLUTION_ENABLED, true);
options.set(CoreOptions.MANIFEST_SIDECAR_ENABLED, true);
RecordingFileIO fileIO = new RecordingFileIO();
ManifestFile manifests =
createManifestFileFactory(tempDir.toString(), Long.MAX_VALUE, options, fileIO)
.create();
ManifestFileMeta meta = manifests.write(Collections.singletonList(gen.next())).get(0);
ManifestEntry entry = gen.next();
ManifestFileMeta meta = manifests.write(Collections.singletonList(entry)).get(0);
assertThat(ManifestSidecar.fileName(meta)).isNotNull();
assertThat(
java.nio.file.Files.exists(
Expand All @@ -1874,7 +1876,106 @@ void testUnknownRowIdKeepsPartitionIndexAndNoQueryDoesNotReadSidecar() {

fileIO.reset();
assertThat(manifests.selectBlocks(meta, null)).isNull();
assertThat(fileIO.opened).isEmpty();
assertThat(manifests.read(meta.fileName())).containsExactly(entry);
assertThat(fileIO.opened)
.containsExactly(new Path(tempDir.toString(), "manifest/" + meta.fileName()));
}

@Test
void testUnfilteredReadWithCacheAndWithoutSidecarUsesWholeManifestCache() {
Options options = new Options();
options.set(CoreOptions.MANIFEST_SIDECAR_ENABLED, true);
RecordingFileIO io = new RecordingFileIO();
SegmentsCache<Path> cache =
new SegmentsCache<>(1024, MemorySize.ofMebiBytes(16), Long.MAX_VALUE);
ManifestFile manifests =
createManifestFileFactory(tempDir.toString(), Long.MAX_VALUE, options, io, cache)
.create();
ManifestEntry entry = gen.next();
ManifestFileMeta written = manifests.write(Collections.singletonList(entry)).get(0);
ManifestFileMeta unindexed = withExtraFiles(written, null);
Path manifestPath = new Path(tempDir.toString(), "manifest/" + written.fileName());

io.reset();
ManifestSidecar.Selection selected = manifests.selectBlocks(unindexed, null);
assertThat(selected).isNull();
assertThat(
manifests.read(
unindexed.fileName(),
unindexed.fileSize(),
null,
null,
row -> true,
manifestEntry -> true,
java.util.function.Function.identity(),
selected))
.containsExactly(entry);
assertThat(io.opened).containsExactly(manifestPath);
assertThat(cache.getIfPresents(manifestPath)).isNotNull();

io.reset();
assertThat(manifests.read(unindexed.fileName())).containsExactly(entry);
assertThat(io.opened).isEmpty();
}

@Test
void testUnfilteredReadWarmsBlockCacheForFilteredRead() throws Exception {
Options options = new Options();
options.set(CoreOptions.BUCKET, 4);
options.set(CoreOptions.MANIFEST_SIDECAR_ENABLED, true);
RecordingFileIO io = new RecordingFileIO();
SegmentsCache<Path> cache =
new SegmentsCache<>(1024, MemorySize.ofMebiBytes(16), Long.MAX_VALUE);
ManifestFile.Factory factory =
createManifestFileFactory(tempDir.toString(), Long.MAX_VALUE, options, io, cache);
List<ManifestEntry> entries = new ArrayList<>();
for (int i = 0; i < 4000; i++) {
ManifestEntry entry = gen.next();
entries.add(
ManifestEntry.create(
FileKind.ADD, entry.partition(), i / 1000, 4, entry.file()));
}
ManifestFileMeta meta = factory.create().write(entries).get(0);
CacheMetrics metrics = new CacheMetrics();
ManifestFile manifests = factory.create().withCacheMetrics(metrics);
Path manifestPath = new Path(tempDir.toString(), "manifest/" + meta.fileName());
Path sidecarPath = ManifestSidecar.path(manifestPath);

io.reset();
ManifestSidecar.Selection allBlocks = manifests.selectBlocks(meta, null);
assertThat(readSelectedEntries(manifests, meta, allBlocks))
.containsExactlyElementsOf(entries);
assertThat(io.opened).containsExactly(sidecarPath, manifestPath);
assertThat(cache.getIfPresents(manifestPath)).isNull();
assertThat(metrics.getMissedObject()).hasValue(1);
assertThat(metrics.getHitObject()).hasValue(0);

io.reset();
ManifestSidecar.Selection cachedBlocks = manifests.selectBlocks(meta, null);
assertThat(readSelectedEntries(manifests, meta, cachedBlocks))
.containsExactlyElementsOf(entries);
assertThat(io.opened).isEmpty();
assertThat(metrics.getMissedObject()).hasValue(1);
assertThat(metrics.getHitObject()).hasValue(1);

BucketFilter bucketFilter = new BucketFilter(false, 1, null, null);
io.reset();
ManifestSidecar.Selection selected = manifests.selectBlocks(meta, null, null, bucketFilter);
assertThat(
manifests.read(
meta.fileName(),
meta.fileSize(),
null,
bucketFilter,
row -> true,
entry -> true,
java.util.function.Function.identity(),
selected))
.containsExactlyElementsOf(entries.subList(1000, 2000));
assertThat(io.opened).isEmpty();
assertThat(io.bytes.get()).isZero();
assertThat(metrics.getMissedObject()).hasValue(1);
assertThat(metrics.getHitObject()).hasValue(2);
}

@Test
Expand Down
Loading