From dd904fab96b57a4f8e7c44f506c752bc254b91a1 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Tue, 20 Oct 2026 08:44:10 +0200 Subject: [PATCH 1/4] fix(reader)!: ArrayStats mirrors Rust's array stats table ArrayStats.fromFbs dropped is_sorted, is_strict_sorted, is_constant, uncompressed_size_in_bytes and nan_count, and the record carried a trueCount the wire has no field for (Rust has no TrueCount stat: a bool column's true count is its sum). Rust writes is_sorted/is_strict_sorted into flat array stats, so Java read them as unknown. The record now has one component per field of the FlatBuffer ArrayStats table (Rust's Stat enum), in schema order; the 7-arg constructor is gone. min_precision/max_precision are still not read: an inexact bound is still a valid bound, so pruning stays correct. Cross-check: vortex-jni writes an ascending id column, and every flat data segment's peeked stats report isSorted/isStrictSorted true. Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 3 + .../dfa1/vortex/inspect/InspectorTree.java | 2 +- .../dfa1/vortex/inspect/VortexInspector.java | 2 +- .../dfa1/vortex/inspect/HtmlReportTest.java | 10 +-- .../vortex/inspect/VortexInspectorTest.java | 4 +- .../VortexInspectorIntegrationTest.java | 41 ++++++++++++ .../github/dfa1/vortex/reader/ArrayStats.java | 66 +++++++++---------- .../dfa1/vortex/reader/ScanIterator.java | 8 +-- .../dfa1/vortex/reader/VortexReader.java | 7 +- .../dfa1/vortex/reader/ArrayStatsTest.java | 35 +++++++++- 10 files changed, 125 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4817da462..115bb60e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed +- **Breaking:** `ArrayStats` has one component per field of Rust's array stats, so `isSorted`, `isStrictSorted`, `isConstant`, `uncompressedSizeInBytes` and `nanCount` are read from the file and `trueCount` is gone (a bool column's true count is its `sum`). ([#581](https://github.com/dfa1/vortex-java/pull/581)) + ## [0.18.0] — 2026-10-19 ### Added diff --git a/inspector/src/main/java/io/github/dfa1/vortex/inspect/InspectorTree.java b/inspector/src/main/java/io/github/dfa1/vortex/inspect/InspectorTree.java index df0074c4d..5707b378d 100644 --- a/inspector/src/main/java/io/github/dfa1/vortex/inspect/InspectorTree.java +++ b/inspector/src/main/java/io/github/dfa1/vortex/inspect/InspectorTree.java @@ -289,7 +289,7 @@ private static ArrayStats fold(List zones) { if (min == null && max == null) { return ArrayStats.empty(); } - return new ArrayStats(min, max, null, null, null, null, null); + return new ArrayStats(min, max, null, null, null, null, null, null, null); } /// The chunks of a column: the children of the first chunked node on its data path, or the diff --git a/inspector/src/main/java/io/github/dfa1/vortex/inspect/VortexInspector.java b/inspector/src/main/java/io/github/dfa1/vortex/inspect/VortexInspector.java index 8adff6da8..ed54bee75 100644 --- a/inspector/src/main/java/io/github/dfa1/vortex/inspect/VortexInspector.java +++ b/inspector/src/main/java/io/github/dfa1/vortex/inspect/VortexInspector.java @@ -103,7 +103,7 @@ static ArrayStats aggregateStats(InspectorTree.Node node) { if (min == null && max == null) { return ArrayStats.empty(); } - return new ArrayStats(min, max, null, null, null, null, null); + return new ArrayStats(min, max, null, null, null, null, null, null, null); } @SuppressWarnings({"unchecked", "rawtypes"}) diff --git a/inspector/src/test/java/io/github/dfa1/vortex/inspect/HtmlReportTest.java b/inspector/src/test/java/io/github/dfa1/vortex/inspect/HtmlReportTest.java index 9c506d65d..597efb1f1 100644 --- a/inspector/src/test/java/io/github/dfa1/vortex/inspect/HtmlReportTest.java +++ b/inspector/src/test/java/io/github/dfa1/vortex/inspect/HtmlReportTest.java @@ -101,7 +101,7 @@ void render_escapesUntrustedColumnNames() { @Test void render_escapesUntrustedStatValues() { // Given — min/max are decoded scalars from the file, so a string column can carry markup - InspectorTree sut = singleColumnWithStats(new ArrayStats("", "\"&\"", null, null, 3L, null, null)); + InspectorTree sut = singleColumnWithStats(new ArrayStats("", "\"&\"", null, null, null, null, 3L, null, null)); // When String result = HtmlReport.render(sut, "evil.vortex"); @@ -540,7 +540,7 @@ void render_timestampColumn_showsBoundsAsInstantsNotEpochIntegers() { Layout leaf = new Layout(LayoutId.parse("vortex.flat"), 2, null, List.of(), List.of(0)); Layout root = new Layout(LayoutId.parse("vortex.struct"), 2, null, List.of(leaf), List.of()); InspectorTree.Node leafNode = new InspectorTree.Node(leaf, Optional.of("ts"), Set.of(), - new ArrayStats(1_704_067_200_000L, 1_706_867_193_000L, null, null, null, null, null), + new ArrayStats(1_704_067_200_000L, 1_706_867_193_000L, null, null, null, null, null, null, null), List.of()); InspectorTree.Node rootNode = new InspectorTree.Node(root, Optional.empty(), Set.of(), ArrayStats.empty(), List.of(leafNode)); @@ -590,7 +590,7 @@ private static InspectorTree twoColumnTree() { Layout root = new Layout(LayoutId.parse("vortex.struct"), 1000, null, List.of(idLeaf, valLeaf), List.of()); InspectorTree.Node idNode = new InspectorTree.Node(idLeaf, Optional.of("id"), - Set.of("fastlanes.bitpacked"), new ArrayStats(1L, 999L, null, null, 0L, null, null), List.of()); + Set.of("fastlanes.bitpacked"), new ArrayStats(1L, 999L, null, null, null, null, 0L, null, null), List.of()); InspectorTree.Node valNode = new InspectorTree.Node(valLeaf, Optional.of("value"), Set.of("vortex.constant"), ArrayStats.empty(), List.of()); InspectorTree.Node rootNode = new InspectorTree.Node(root, Optional.empty(), @@ -612,9 +612,9 @@ private static InspectorTree chunkedTree() { Layout root = new Layout(LayoutId.parse("vortex.struct"), 1000, null, List.of(chunked), List.of()); InspectorTree.Node n0 = new InspectorTree.Node(c0, Optional.empty(), Set.of(), - new ArrayStats(0L, 499L, null, null, null, null, null), List.of()); + new ArrayStats(0L, 499L, null, null, null, null, null, null, null), List.of()); InspectorTree.Node n1 = new InspectorTree.Node(c1, Optional.empty(), Set.of(), - new ArrayStats(500L, 999L, null, null, null, null, null), List.of()); + new ArrayStats(500L, 999L, null, null, null, null, null, null, null), List.of()); InspectorTree.Node col = new InspectorTree.Node(chunked, Optional.of("id"), Set.of(), ArrayStats.empty(), List.of(n0, n1)); InspectorTree.Node rootNode = new InspectorTree.Node(root, Optional.empty(), Set.of(), diff --git a/inspector/src/test/java/io/github/dfa1/vortex/inspect/VortexInspectorTest.java b/inspector/src/test/java/io/github/dfa1/vortex/inspect/VortexInspectorTest.java index 7f6712b9e..0204e8680 100644 --- a/inspector/src/test/java/io/github/dfa1/vortex/inspect/VortexInspectorTest.java +++ b/inspector/src/test/java/io/github/dfa1/vortex/inspect/VortexInspectorTest.java @@ -135,9 +135,9 @@ void render_aggregatesMinMaxAcrossChunks() { Layout structLayout = new Layout(LayoutId.parse("vortex.struct"), 1000, null, List.of(chunked), List.of()); InspectorTree.Node c1 = new InspectorTree.Node(chunk1, Optional.empty(), Set.of(), - new ArrayStats(10L, 50L, null, null, null, null, null), List.of()); + new ArrayStats(10L, 50L, null, null, null, null, null, null, null), List.of()); InspectorTree.Node c2 = new InspectorTree.Node(chunk2, Optional.empty(), Set.of(), - new ArrayStats(5L, 100L, null, null, null, null, null), List.of()); + new ArrayStats(5L, 100L, null, null, null, null, null, null, null), List.of()); InspectorTree.Node chunkedN = new InspectorTree.Node(chunked, Optional.of("id"), Set.of("vortex.flat"), ArrayStats.empty(), List.of(c1, c2)); InspectorTree.Node rootN = new InspectorTree.Node(structLayout, Optional.empty(), diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/VortexInspectorIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/VortexInspectorIntegrationTest.java index 1b0095d2f..5b92f3320 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/VortexInspectorIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/VortexInspectorIntegrationTest.java @@ -6,6 +6,7 @@ import dev.vortex.jni.NativeLoader; import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.reader.ArrayStats; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.writer.WriteOptions; import io.github.dfa1.vortex.inspect.HtmlReport; @@ -31,7 +32,9 @@ import java.nio.file.StandardOpenOption; import java.util.Map; import java.nio.file.Path; +import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.Random; import static org.assertj.core.api.Assertions.assertThat; @@ -162,6 +165,44 @@ void htmlReport_globalDictColumn_chunkMinMaxAreValuesNotCodes(@TempDir Path tmp) .contains("cherrydate"); } + @Test + void peek_readsSortednessRustRecordsInFlatArrayStats(@TempDir Path tmp) throws IOException { + // Given — Rust records is_sorted/is_strict_sorted in a flat segment's array stats for an + // ascending integer column. Java's ArrayStats used to drop both (and carried a trueCount + // the wire has no field for), so these read as unknown. + Path file = tmp.resolve("sorted.vtx"); + writeJni(file, 50_000); + + // When + List result = new ArrayList<>(); + try (VortexReader vf = VortexReader.open(file, ReadRegistry.loadAll())) { + InspectorTree tree = InspectorTree.buildShallow(vf); + InspectorTree.Node id = tree.root().children().stream() + .filter(c -> c.fieldName().equals(Optional.of("id"))).findFirst().orElseThrow(); + for (InspectorTree.Node flat : dataFlats(id)) { + result.add(InspectorTree.peek(flat, vf).stats()); + } + } + + // Then + assertThat(result).isNotEmpty().allSatisfy(stats -> { + assertThat(stats.isSorted()).isTrue(); + assertThat(stats.isStrictSorted()).isTrue(); + }); + } + + /// The flat segments holding a column's values: every chunk of a chunked node, else the first + /// child (a zoned node's second child is its zone-map table, not data). + private static List dataFlats(InspectorTree.Node node) { + if (node.layout().isFlat()) { + return List.of(node); + } + if (node.layout().isChunked()) { + return node.children().stream().flatMap(c -> dataFlats(c).stream()).toList(); + } + return dataFlats(node.children().getFirst()); + } + private static String[] repeat(int rows, String a, String b) { String[] out = new String[rows]; for (int i = 0; i < rows; i++) { diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/ArrayStats.java b/reader/src/main/java/io/github/dfa1/vortex/reader/ArrayStats.java index e8c0dca3e..3cfa769b4 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/ArrayStats.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/ArrayStats.java @@ -7,41 +7,34 @@ import java.lang.foreign.MemorySegment; import java.nio.charset.StandardCharsets; -/// Per-array statistics embedded in the encoding tree. +/// Per-array statistics embedded in the encoding tree: one component per statistic of the wire +/// `ArrayStats` table (Rust's `Stat` enum). A bool column's true count is its `sum`, as in Rust. +/// The table's `min_precision`/`max_precision` are not read, so `min`/`max` may be inexact +/// bounds: Rust's flat writer truncates a string bound longer than 64 bytes. /// -/// @param min minimum value in the array, or `null` if unknown -/// @param max maximum value in the array, or `null` if unknown -/// @param sum sum of values (`Long` for integer columns, `Double` for floats), or `null` if unknown -/// @param trueCount number of `true` values for bool columns, or `null` if unknown -/// @param nullCount number of null values, or `null` if unknown -/// @param isSorted `true` if the array is sorted in ascending order, or `null` if unknown -/// @param isStrictSorted `true` if the array is strictly sorted (no duplicates), or `null` if unknown -/// @param nanCount number of `NaN` values in a float column (`min` and `max` skip them), or `null` if unknown +/// The counts are `u64` on the wire, read as a signed `Long`. +/// +/// @param min minimum value in the array, or `null` if unknown +/// @param max maximum value in the array, or `null` if unknown +/// @param sum sum of values (`Long` for integer and bool columns, `Double` for floats), or `null` if unknown +/// @param isSorted `true` if the values are sorted ascending with nulls first (Rust orders null lowest), or `null` if unknown +/// @param isStrictSorted `true` if also no two neighbors are equal, two nulls included, or `null` if unknown +/// @param isConstant `true` if every value is null, or none is and all are equal, or `null` if unknown +/// @param nullCount number of null values, or `null` if unknown +/// @param uncompressedSizeInBytes the array's uncompressed size in bytes, or `null` if unknown +/// @param nanCount number of `NaN` values in a float column (`min` and `max` skip them), or `null` if unknown public record ArrayStats( Object min, Object max, Object sum, - Long trueCount, - Long nullCount, Boolean isSorted, Boolean isStrictSorted, + Boolean isConstant, + Long nullCount, + Long uncompressedSizeInBytes, Long nanCount ) { - private static final ArrayStats EMPTY = new ArrayStats(null, null, null, null, null, null, null, null); - - /// Creates stats with no `NaN` count. - /// - /// @param min minimum value in the array, or `null` if unknown - /// @param max maximum value in the array, or `null` if unknown - /// @param sum sum of values, or `null` if unknown - /// @param trueCount number of `true` values for bool columns, or `null` if unknown - /// @param nullCount number of null values, or `null` if unknown - /// @param isSorted `true` if the array is sorted in ascending order, or `null` if unknown - /// @param isStrictSorted `true` if the array is strictly sorted, or `null` if unknown - public ArrayStats(Object min, Object max, Object sum, Long trueCount, Long nullCount, - Boolean isSorted, Boolean isStrictSorted) { - this(min, max, sum, trueCount, nullCount, isSorted, isStrictSorted, null); - } + private static final ArrayStats EMPTY = new ArrayStats(null, null, null, null, null, null, null, null, null); /// Returns an empty stats instance with all fields set to `null`. /// @@ -51,7 +44,7 @@ public static ArrayStats empty() { } /// Parses stats from a FlatBuffers [io.github.dfa1.vortex.core.fbs.FbsArrayStats] table. - /// Returns an empty instance when `fbs` is `null` or carries no min/max and no null count. + /// Returns an empty instance when `fbs` is `null` or carries no statistic. /// /// @param fbs the FlatBuffers stats table, or `null` /// @return parsed stats, or an empty instance if no usable data is present @@ -59,14 +52,17 @@ public static ArrayStats fromFbs(io.github.dfa1.vortex.core.fbs.FbsArrayStats fb if (fbs == null) { return EMPTY; } - Object min = decodeScalar(fbs.minAsSegment()); - Object max = decodeScalar(fbs.maxAsSegment()); - Object sum = decodeScalar(fbs.sumAsSegment()); - Long nullCount = fbs.hasNullCount() ? fbs.nullCount() : null; - if (min == null && max == null && sum == null && nullCount == null) { - return EMPTY; - } - return new ArrayStats(min, max, sum, null, nullCount, null, null); + ArrayStats stats = new ArrayStats( + decodeScalar(fbs.minAsSegment()), + decodeScalar(fbs.maxAsSegment()), + decodeScalar(fbs.sumAsSegment()), + fbs.hasIsSorted() ? fbs.isSorted() : null, + fbs.hasIsStrictSorted() ? fbs.isStrictSorted() : null, + fbs.hasIsConstant() ? fbs.isConstant() : null, + fbs.hasNullCount() ? fbs.nullCount() : null, + fbs.hasUncompressedSizeInBytes() ? fbs.uncompressedSizeInBytes() : null, + fbs.hasNanCount() ? fbs.nanCount() : null); + return stats.equals(EMPTY) ? EMPTY : stats; } private static Object decodeScalar(MemorySegment seg) { diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java index 0868b4057..72998ce73 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java @@ -455,8 +455,8 @@ public long[] chunkRowCounts() { /// table — min/max/sum/null count per zone — decoded once from the small stats segment /// without touching any data segment. This is the source Rust populates for `SUM`, so the /// values match files written by either implementation. When no zone map is present the - /// list falls back to each chunk's embedded `ArrayStats` (min/max/null count; `sum` is - /// `null`, since the flat writer does not retain it). Either way a column that is absent + /// list falls back to each chunk's embedded `ArrayStats` (whatever the flat writer retained; + /// `sum` is `null`, since it does not retain it). Either way a column that is absent /// or carries no stats yields [ArrayStats#empty()] per zone. /// /// Zone granularity is the layout's, not the scan's. The fallback path is one entry per scan @@ -590,9 +590,9 @@ private List decodeZoneTable(ColumnName column, boolean includeBound minA != null ? boxedScalar(minA, i) : boxedScalar(boundedMinA, i), maxA != null ? boxedScalar(maxA, i) : boundedMaxScalar(boundedMaxA, i), boxedScalar(sumA, i), - null, + null, null, null, nullCount == null ? null : ((Number) nullCount).longValue(), - null, null, + null, nanCount == null ? null : ((Number) nanCount).longValue())); } return out; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/VortexReader.java b/reader/src/main/java/io/github/dfa1/vortex/reader/VortexReader.java index 704a58b76..2bb17693c 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/VortexReader.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/VortexReader.java @@ -243,8 +243,9 @@ public Map columnStats() { // A string zone map holds only bounds (Rust's bounded_max/bounded_min), which // are not exact: take min/max from the chunks' own array stats instead. ArrayStats exact = aggregateStats(iter.chunkStats(name), column); - stats = new ArrayStats(exact.min(), exact.max(), stats.sum(), stats.trueCount(), - stats.nullCount(), stats.isSorted(), stats.isStrictSorted()); + // aggregateStats folds only min/max/null count, the rest is always null. + stats = new ArrayStats(exact.min(), exact.max(), null, null, null, null, + stats.nullCount(), null, null); } result.put(name, stats); } @@ -278,7 +279,7 @@ private ArrayStats aggregateStats(List zones, DType column) { } // Sum is left null at the file level: it lives in the per-zone stats table, surfaced by // ScanIterator.columnZoneStats rather than folded here. - return new ArrayStats(globalMin, globalMax, null, null, nullCount, null, null); + return new ArrayStats(globalMin, globalMax, null, null, null, null, nullCount, null, null); } @Override diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayStatsTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayStatsTest.java index 57e336115..dd11c7042 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayStatsTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayStatsTest.java @@ -29,10 +29,13 @@ void hasAllNullComponents() { // Then assertThat(sut.min()).isNull(); assertThat(sut.max()).isNull(); - assertThat(sut.trueCount()).isNull(); - assertThat(sut.nullCount()).isNull(); + assertThat(sut.sum()).isNull(); assertThat(sut.isSorted()).isNull(); assertThat(sut.isStrictSorted()).isNull(); + assertThat(sut.isConstant()).isNull(); + assertThat(sut.nullCount()).isNull(); + assertThat(sut.uncompressedSizeInBytes()).isNull(); + assertThat(sut.nanCount()).isNull(); } } @@ -192,6 +195,34 @@ void emptyByteVector_treatedAsAbsent() { assertThat(sut).isSameAs(ArrayStats.empty()); } + @Test + void everyNonScalarStat_decodes() { + // Given — every stat besides min/max (covered above), with false/0 values on purpose: + // they equal the FlatBuffer scalar defaults, so a reader that skips a field or tests the + // value instead of its presence reads them as unknown. A lost nan_count = 0 reads as + // "may hold NaN", which caps a float chunk's max at NaN so it never prunes. + FbsBuilder builder = new FbsBuilder(128); + int sumOff = io.github.dfa1.vortex.core.fbs.FbsArrayStats.createSumVector( + builder, ProtoScalarValue.ofInt64Value(12L).encode()); + builder.forceDefaults(true); + io.github.dfa1.vortex.core.fbs.FbsArrayStats.startFbsArrayStats(builder); + io.github.dfa1.vortex.core.fbs.FbsArrayStats.addSum(builder, sumOff); + io.github.dfa1.vortex.core.fbs.FbsArrayStats.addIsSorted(builder, true); + io.github.dfa1.vortex.core.fbs.FbsArrayStats.addIsStrictSorted(builder, false); + io.github.dfa1.vortex.core.fbs.FbsArrayStats.addIsConstant(builder, false); + io.github.dfa1.vortex.core.fbs.FbsArrayStats.addNullCount(builder, 0L); + io.github.dfa1.vortex.core.fbs.FbsArrayStats.addUncompressedSizeInBytes(builder, 800L); + io.github.dfa1.vortex.core.fbs.FbsArrayStats.addNanCount(builder, 0L); + builder.finish(io.github.dfa1.vortex.core.fbs.FbsArrayStats.endFbsArrayStats(builder)); + var fbs = io.github.dfa1.vortex.core.fbs.FbsArrayStats.getRootAsFbsArrayStats(builder.dataSegment()); + + // When + ArrayStats result = ArrayStats.fromFbs(fbs); + + // Then + assertThat(result).isEqualTo(new ArrayStats(null, null, 12L, true, false, false, 0L, 800L, 0L)); + } + @Test void malformedScalarBytes_throwsVortexException() { // Given — varint tag with no continuation byte: ProtoReader hits EOF inside readVarint From 63875b000c0a5db4289625c9a5f4b614133595ca Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Tue, 20 Oct 2026 08:49:51 +0200 Subject: [PATCH 2/4] fix(reader): treat a u64 nan_count past 2^63 as nonzero when pruning nan_count is u64 on the wire and surfaces as a signed Long, so `> 0` read a count >= 2^63 as "no NaN" and let a float chunk's max prune a filter that a NaN row matches. Rust sees a huge count and keeps the chunk; `!= 0` matches it. Co-Authored-By: Claude Sonnet 5 --- .../main/java/io/github/dfa1/vortex/reader/ScanIterator.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java index 72998ce73..c72ddb96e 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java @@ -1255,9 +1255,10 @@ private static boolean canPrune(Predicate predicate, ArrayStats stats, long rowC Object max = stats.max(); Long nullCount = stats.nullCount(); if (ct instanceof DType.Primitive p && p.ptype().isFloating() - && (stats.nanCount() == null || stats.nanCount() > 0)) { + && (stats.nanCount() == null || stats.nanCount() != 0)) { // Float min/max skip NaN, but filters order floats totally (Rust), where NaN is the - // maximum: a chunk that may hold one (nan_count absent or positive) has a NaN upper bound. + // maximum: a chunk that may hold one (nan_count absent or nonzero) has a NaN upper bound. + // nan_count is u64: a count past 2^63 reads as a negative Long, still not zero. max = Double.NaN; } return switch (predicate) { From c83b91e253ecfa1786216e44ad23395dabbb5d76 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Tue, 20 Oct 2026 09:03:59 +0200 Subject: [PATCH 3/4] fix(writer): mark flat array min/max as exact buildArrayFlatBuffer never set min_precision/max_precision, so they took the FlatBuffer default Inexact and Rust read every Java-written bound as inexact. The encoders' min/max are the exact extremes (Java never truncates flat stats), so write Precision::Exact as Rust's flat writer does for an untruncated stat. Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 3 +++ .../main/java/io/github/dfa1/vortex/writer/VortexWriter.java | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 115bb60e4..734d0ec66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - **Breaking:** `ArrayStats` has one component per field of Rust's array stats, so `isSorted`, `isStrictSorted`, `isConstant`, `uncompressedSizeInBytes` and `nanCount` are read from the file and `trueCount` is gone (a bool column's true count is its `sum`). ([#581](https://github.com/dfa1/vortex-java/pull/581)) +### Fixed +- Files written by vortex-java mark their array min/max as exact, which Rust read as inexact bounds. ([#581](https://github.com/dfa1/vortex-java/pull/581)) + ## [0.18.0] — 2026-10-19 ### Added diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java index a7e755efe..a0ae26496 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java @@ -1058,8 +1058,12 @@ private ByteBuffer buildArrayFlatBuffer(EncodeResult result, int[] paddings, lon fbb.forceDefaults(true); io.github.dfa1.vortex.core.fbs.FbsArrayStats.startFbsArrayStats(fbb); if (result.hasStats()) { + // The encoder's min/max are the exact extremes, never truncated. An absent precision + // is the FlatBuffer default Inexact, which is what Rust read every Java bound as. io.github.dfa1.vortex.core.fbs.FbsArrayStats.addMin(fbb, minVec); + io.github.dfa1.vortex.core.fbs.FbsArrayStats.addMinPrecision(fbb, io.github.dfa1.vortex.core.fbs.FbsPrecision.Exact); io.github.dfa1.vortex.core.fbs.FbsArrayStats.addMax(fbb, maxVec); + io.github.dfa1.vortex.core.fbs.FbsArrayStats.addMaxPrecision(fbb, io.github.dfa1.vortex.core.fbs.FbsPrecision.Exact); } io.github.dfa1.vortex.core.fbs.FbsArrayStats.addNullCount(fbb, nullCount); int statsOff = io.github.dfa1.vortex.core.fbs.FbsArrayStats.endFbsArrayStats(fbb); From b4cf93b56d7333d6a21c47ea4a1d9f3f7007dcbc Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Tue, 20 Oct 2026 09:04:15 +0200 Subject: [PATCH 4/4] fix(reader): honor min/max precision in array and zone stats ArrayStats ignored min_precision/max_precision, so a truncated bound (Rust's flat writer cuts a string min/max past 64 bytes, a legacy vortex.stats table flags it in *_is_truncated) read as the exact extreme. VortexReader.columnStats fell back to those flat stats for string columns, and Calcite answered SQL MIN/MAX from them. - ArrayStats gains minExact/maxExact. fromFbs reads precision as Rust's from_flatbuffer does: absent is Inexact, a value other than 0/1 is a corrupted field (VortexException), read only when its bound is present. - Zone tables: vortex.zoned MIN/MAX are exact, legacy MIN/MAX inexact where *_is_truncated is set, bounded_min/bounded_max never exact. - columnStats reports only exact min/max (null otherwise); Calcite's zone fold answers MIN/MAX only from exact stats. - An inexact bound still brackets the values, so range pruning keeps using it; only the min == max == v cases (Neq prune, Calcite Eq IN / Neq OUT) now require both bounds exact. vortex-jni writes no flat min/max on its default path (probed), so no Rust file produces an inexact flat bound to cross-check against; the cross-check is the Java-written >64-byte key column that vortex-jni filters, whose columnStats must be the full keys. Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 3 +- .../dfa1/vortex/calcite/VortexTable.java | 19 ++++-- docs/reference.md | 2 +- .../dfa1/vortex/inspect/InspectorTree.java | 3 +- .../dfa1/vortex/inspect/VortexInspector.java | 3 +- .../dfa1/vortex/inspect/HtmlReportTest.java | 10 +-- .../vortex/inspect/VortexInspectorTest.java | 4 +- .../JavaWritesRustReadsIntegrationTest.java | 9 ++- .../github/dfa1/vortex/reader/ArrayStats.java | 34 +++++++++-- .../dfa1/vortex/reader/ScanIterator.java | 15 ++++- .../dfa1/vortex/reader/VortexReader.java | 25 ++++++-- .../dfa1/vortex/reader/ArrayStatsTest.java | 61 ++++++++++++++++++- 12 files changed, 154 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 734d0ec66..0a0b0b2a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Changed -- **Breaking:** `ArrayStats` has one component per field of Rust's array stats, so `isSorted`, `isStrictSorted`, `isConstant`, `uncompressedSizeInBytes` and `nanCount` are read from the file and `trueCount` is gone (a bool column's true count is its `sum`). ([#581](https://github.com/dfa1/vortex-java/pull/581)) +- **Breaking:** `ArrayStats` has one component per field of Rust's array stats, so `minExact`, `maxExact`, `isSorted`, `isStrictSorted`, `isConstant`, `uncompressedSizeInBytes` and `nanCount` are read from the file and `trueCount` is gone (a bool column's true count is its `sum`). ([#581](https://github.com/dfa1/vortex-java/pull/581)) ### Fixed - Files written by vortex-java mark their array min/max as exact, which Rust read as inexact bounds. ([#581](https://github.com/dfa1/vortex-java/pull/581)) +- `VortexReader.columnStats()` and SQL `MIN`/`MAX` no longer report a truncated (inexact) bound as a column's minimum or maximum. ([#581](https://github.com/dfa1/vortex-java/pull/581)) ## [0.18.0] — 2026-10-19 diff --git a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexTable.java b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexTable.java index d30fedba8..60b849c82 100644 --- a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexTable.java +++ b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexTable.java @@ -346,12 +346,12 @@ public Optional filteredFold(RowFilter filter, ColumnName aggColum fold.keptRows += zone.rowCount(); if (aggColumn != null) { ArrayStats stats = zoneStats.get(aggColumn).get(z); - // a zone with values but no min/max stat (a string zone holds only bounds) - // leaves the extreme unknown, rather than silently ignored + // a zone with values but no exact min/max stat (a string zone holds only + // bounds) leaves the extreme unknown, rather than silently ignored boolean hasValues = stats.nullCount() == null || stats.nullCount() < zone.rowCount(); fold.addSum(stats.sum()); - fold.addMin(stats.min(), hasValues); - fold.addMax(stats.max(), hasValues); + fold.addMin(stats.minExact() ? stats.min() : null, hasValues); + fold.addMax(stats.maxExact() ? stats.max() : null, hasValues); fold.addNulls(stats.nullCount()); } } @@ -1169,7 +1169,8 @@ private static Match classifyColumnRaw(Predicate predicate, ArrayStats s, long r if (compareStat(value, s.min()) < 0 || compareStat(value, s.max()) > 0) { yield Match.OUT; // the value lies outside the zone's range } - if (compareStat(s.min(), s.max()) == 0 && compareStat(s.min(), value) == 0 && noNulls(s)) { + if (exactBounds(s) && compareStat(s.min(), s.max()) == 0 && compareStat(s.min(), value) == 0 + && noNulls(s)) { yield Match.IN; // every row equals the value } yield Match.BOUNDARY; @@ -1178,7 +1179,7 @@ private static Match classifyColumnRaw(Predicate predicate, ArrayStats s, long r if (uncomparable(s, value)) { yield Match.BOUNDARY; } - if (compareStat(s.min(), s.max()) == 0 && compareStat(s.min(), value) == 0) { + if (exactBounds(s) && compareStat(s.min(), s.max()) == 0 && compareStat(s.min(), value) == 0) { yield Match.OUT; // every row equals the value, so none differ } if ((compareStat(value, s.min()) < 0 || compareStat(value, s.max()) > 0) && noNulls(s)) { @@ -1280,6 +1281,12 @@ private static boolean uncomparable(ArrayStats s, Object value) { || value instanceof Number != s.min() instanceof Number; } + /// Whether the zone's min and max are its exact extremes: an inexact bound only brackets the + /// values, which is enough to rule a range in or out but not to prove `min == max`. + private static boolean exactBounds(ArrayStats s) { + return s.minExact() && s.maxExact(); + } + /// Whether the zone provably carries no nulls (its null count is recorded and zero). private static boolean noNulls(ArrayStats s) { return s.nullCount() != null && s.nullCount() == 0; diff --git a/docs/reference.md b/docs/reference.md index b43724b30..ae82dab10 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -105,7 +105,7 @@ all `Array` buffers obtained during scans become invalid. | `version()` | `int` | File format version | | `fileSize()` | `long` | File size in bytes | | `scan(ScanOptions)` | `ScanIterator` | Open a scan | -| `columnStats()` | `Map` | Aggregated min/max per column | +| `columnStats()` | `Map` | Exact min/max per column (`null` when only a bound is known) | | `slice(offset, length)` | `MemorySegment` | Zero-copy slice of mmap region | | `close()` | — | Releases mmap | diff --git a/inspector/src/main/java/io/github/dfa1/vortex/inspect/InspectorTree.java b/inspector/src/main/java/io/github/dfa1/vortex/inspect/InspectorTree.java index 5707b378d..bb62f82fd 100644 --- a/inspector/src/main/java/io/github/dfa1/vortex/inspect/InspectorTree.java +++ b/inspector/src/main/java/io/github/dfa1/vortex/inspect/InspectorTree.java @@ -289,7 +289,8 @@ private static ArrayStats fold(List zones) { if (min == null && max == null) { return ArrayStats.empty(); } - return new ArrayStats(min, max, null, null, null, null, null, null, null); + // display-only fold: precision is not tracked, so the bounds are not claimed exact + return new ArrayStats(min, false, max, false, null, null, null, null, null, null, null); } /// The chunks of a column: the children of the first chunked node on its data path, or the diff --git a/inspector/src/main/java/io/github/dfa1/vortex/inspect/VortexInspector.java b/inspector/src/main/java/io/github/dfa1/vortex/inspect/VortexInspector.java index ed54bee75..fc9e9bf42 100644 --- a/inspector/src/main/java/io/github/dfa1/vortex/inspect/VortexInspector.java +++ b/inspector/src/main/java/io/github/dfa1/vortex/inspect/VortexInspector.java @@ -103,7 +103,8 @@ static ArrayStats aggregateStats(InspectorTree.Node node) { if (min == null && max == null) { return ArrayStats.empty(); } - return new ArrayStats(min, max, null, null, null, null, null, null, null); + // display-only fold: precision is not tracked, so the bounds are not claimed exact + return new ArrayStats(min, false, max, false, null, null, null, null, null, null, null); } @SuppressWarnings({"unchecked", "rawtypes"}) diff --git a/inspector/src/test/java/io/github/dfa1/vortex/inspect/HtmlReportTest.java b/inspector/src/test/java/io/github/dfa1/vortex/inspect/HtmlReportTest.java index 597efb1f1..69fc7e799 100644 --- a/inspector/src/test/java/io/github/dfa1/vortex/inspect/HtmlReportTest.java +++ b/inspector/src/test/java/io/github/dfa1/vortex/inspect/HtmlReportTest.java @@ -101,7 +101,7 @@ void render_escapesUntrustedColumnNames() { @Test void render_escapesUntrustedStatValues() { // Given — min/max are decoded scalars from the file, so a string column can carry markup - InspectorTree sut = singleColumnWithStats(new ArrayStats("", "\"&\"", null, null, null, null, 3L, null, null)); + InspectorTree sut = singleColumnWithStats(new ArrayStats("", true, "\"&\"", true, null, null, null, null, 3L, null, null)); // When String result = HtmlReport.render(sut, "evil.vortex"); @@ -540,7 +540,7 @@ void render_timestampColumn_showsBoundsAsInstantsNotEpochIntegers() { Layout leaf = new Layout(LayoutId.parse("vortex.flat"), 2, null, List.of(), List.of(0)); Layout root = new Layout(LayoutId.parse("vortex.struct"), 2, null, List.of(leaf), List.of()); InspectorTree.Node leafNode = new InspectorTree.Node(leaf, Optional.of("ts"), Set.of(), - new ArrayStats(1_704_067_200_000L, 1_706_867_193_000L, null, null, null, null, null, null, null), + new ArrayStats(1_704_067_200_000L, true, 1_706_867_193_000L, true, null, null, null, null, null, null, null), List.of()); InspectorTree.Node rootNode = new InspectorTree.Node(root, Optional.empty(), Set.of(), ArrayStats.empty(), List.of(leafNode)); @@ -590,7 +590,7 @@ private static InspectorTree twoColumnTree() { Layout root = new Layout(LayoutId.parse("vortex.struct"), 1000, null, List.of(idLeaf, valLeaf), List.of()); InspectorTree.Node idNode = new InspectorTree.Node(idLeaf, Optional.of("id"), - Set.of("fastlanes.bitpacked"), new ArrayStats(1L, 999L, null, null, null, null, 0L, null, null), List.of()); + Set.of("fastlanes.bitpacked"), new ArrayStats(1L, true, 999L, true, null, null, null, null, 0L, null, null), List.of()); InspectorTree.Node valNode = new InspectorTree.Node(valLeaf, Optional.of("value"), Set.of("vortex.constant"), ArrayStats.empty(), List.of()); InspectorTree.Node rootNode = new InspectorTree.Node(root, Optional.empty(), @@ -612,9 +612,9 @@ private static InspectorTree chunkedTree() { Layout root = new Layout(LayoutId.parse("vortex.struct"), 1000, null, List.of(chunked), List.of()); InspectorTree.Node n0 = new InspectorTree.Node(c0, Optional.empty(), Set.of(), - new ArrayStats(0L, 499L, null, null, null, null, null, null, null), List.of()); + new ArrayStats(0L, true, 499L, true, null, null, null, null, null, null, null), List.of()); InspectorTree.Node n1 = new InspectorTree.Node(c1, Optional.empty(), Set.of(), - new ArrayStats(500L, 999L, null, null, null, null, null, null, null), List.of()); + new ArrayStats(500L, true, 999L, true, null, null, null, null, null, null, null), List.of()); InspectorTree.Node col = new InspectorTree.Node(chunked, Optional.of("id"), Set.of(), ArrayStats.empty(), List.of(n0, n1)); InspectorTree.Node rootNode = new InspectorTree.Node(root, Optional.empty(), Set.of(), diff --git a/inspector/src/test/java/io/github/dfa1/vortex/inspect/VortexInspectorTest.java b/inspector/src/test/java/io/github/dfa1/vortex/inspect/VortexInspectorTest.java index 0204e8680..1b108f878 100644 --- a/inspector/src/test/java/io/github/dfa1/vortex/inspect/VortexInspectorTest.java +++ b/inspector/src/test/java/io/github/dfa1/vortex/inspect/VortexInspectorTest.java @@ -135,9 +135,9 @@ void render_aggregatesMinMaxAcrossChunks() { Layout structLayout = new Layout(LayoutId.parse("vortex.struct"), 1000, null, List.of(chunked), List.of()); InspectorTree.Node c1 = new InspectorTree.Node(chunk1, Optional.empty(), Set.of(), - new ArrayStats(10L, 50L, null, null, null, null, null, null, null), List.of()); + new ArrayStats(10L, true, 50L, true, null, null, null, null, null, null, null), List.of()); InspectorTree.Node c2 = new InspectorTree.Node(chunk2, Optional.empty(), Set.of(), - new ArrayStats(5L, 100L, null, null, null, null, null, null, null), List.of()); + new ArrayStats(5L, true, 100L, true, null, null, null, null, null, null, null), List.of()); InspectorTree.Node chunkedN = new InspectorTree.Node(chunked, Optional.of("id"), Set.of("vortex.flat"), ArrayStats.empty(), List.of(c1, c2)); InspectorTree.Node rootN = new InspectorTree.Node(structLayout, Optional.empty(), diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java index 233d56a8d..1e71bc47e 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java @@ -979,9 +979,16 @@ void javaWriter_jniReader_filteredUtf8_prunesOnTruncatedBounds(@TempDir Path tmp // When List result = readColumnFiltered(file, "k", Expression.binary(Expression.BinaryOp.EQ, Expression.column("k"), Expression.literal(target))); + io.github.dfa1.vortex.reader.ArrayStats stats; + try (var reader = io.github.dfa1.vortex.reader.VortexReader.open(file)) { + stats = reader.columnStats().get(ColumnName.of("k")); + } - // Then + // Then — vortex-jni accepts the flat stats' precision fields, and the column's min/max are + // the full keys: only the zone map truncates, so SQL MIN/MAX may answer from these stats. assertThat(result).extracting(Object::toString).containsExactly(target); + assertThat(stats.min()).isEqualTo(prefix + String.format("%08d", 0)); + assertThat(stats.max()).isEqualTo(prefix + String.format("%08d", 3 * batch - 1)); } @Test diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/ArrayStats.java b/reader/src/main/java/io/github/dfa1/vortex/reader/ArrayStats.java index 3cfa769b4..6e17089e8 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/ArrayStats.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/ArrayStats.java @@ -7,15 +7,20 @@ import java.lang.foreign.MemorySegment; import java.nio.charset.StandardCharsets; -/// Per-array statistics embedded in the encoding tree: one component per statistic of the wire +/// Per-array statistics embedded in the encoding tree: one component per field of the wire /// `ArrayStats` table (Rust's `Stat` enum). A bool column's true count is its `sum`, as in Rust. -/// The table's `min_precision`/`max_precision` are not read, so `min`/`max` may be inexact -/// bounds: Rust's flat writer truncates a string bound longer than 64 bytes. +/// +/// `min`/`max` may be inexact (Rust's `Precision::Inexact`): still a valid bound, so pruning on a +/// range may use it, but not the true extreme (Rust's flat writer truncates a string bound longer +/// than 64 bytes). Anything that reports the value, or needs `min == max`, requires +/// `minExact`/`maxExact`. /// /// The counts are `u64` on the wire, read as a signed `Long`. /// /// @param min minimum value in the array, or `null` if unknown +/// @param minExact `true` if `min` is the exact minimum, not only a lower bound /// @param max maximum value in the array, or `null` if unknown +/// @param maxExact `true` if `max` is the exact maximum, not only an upper bound /// @param sum sum of values (`Long` for integer and bool columns, `Double` for floats), or `null` if unknown /// @param isSorted `true` if the values are sorted ascending with nulls first (Rust orders null lowest), or `null` if unknown /// @param isStrictSorted `true` if also no two neighbors are equal, two nulls included, or `null` if unknown @@ -25,7 +30,9 @@ /// @param nanCount number of `NaN` values in a float column (`min` and `max` skip them), or `null` if unknown public record ArrayStats( Object min, + boolean minExact, Object max, + boolean maxExact, Object sum, Boolean isSorted, Boolean isStrictSorted, @@ -34,7 +41,8 @@ public record ArrayStats( Long uncompressedSizeInBytes, Long nanCount ) { - private static final ArrayStats EMPTY = new ArrayStats(null, null, null, null, null, null, null, null, null); + private static final ArrayStats EMPTY = + new ArrayStats(null, false, null, false, null, null, null, null, null, null, null); /// Returns an empty stats instance with all fields set to `null`. /// @@ -52,9 +60,13 @@ public static ArrayStats fromFbs(io.github.dfa1.vortex.core.fbs.FbsArrayStats fb if (fbs == null) { return EMPTY; } + Object min = decodeScalar(fbs.minAsSegment()); + Object max = decodeScalar(fbs.maxAsSegment()); ArrayStats stats = new ArrayStats( - decodeScalar(fbs.minAsSegment()), - decodeScalar(fbs.maxAsSegment()), + min, + min != null && isExact(fbs.minPrecision(), "min_precision"), + max, + max != null && isExact(fbs.maxPrecision(), "max_precision"), decodeScalar(fbs.sumAsSegment()), fbs.hasIsSorted() ? fbs.isSorted() : null, fbs.hasIsStrictSorted() ? fbs.isStrictSorted() : null, @@ -65,6 +77,16 @@ public static ArrayStats fromFbs(io.github.dfa1.vortex.core.fbs.FbsArrayStats fb return stats.equals(EMPTY) ? EMPTY : stats; } + /// Rust's `from_flatbuffer`: an absent precision is the FlatBuffer default `Inexact`, and any + /// value other than `Inexact`/`Exact` is a corrupted field. Read only when its bound is present. + private static boolean isExact(int precision, String field) { + return switch (precision) { + case io.github.dfa1.vortex.core.fbs.FbsPrecision.Inexact -> false; + case io.github.dfa1.vortex.core.fbs.FbsPrecision.Exact -> true; + default -> throw new VortexException("corrupted " + field + " in array stats: " + precision); + }; + } + private static Object decodeScalar(MemorySegment seg) { if (seg == null || seg.byteSize() == 0) { return null; diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java index c72ddb96e..359f2045c 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java @@ -578,6 +578,10 @@ private List decodeZoneTable(ColumnName column, boolean includeBound Array sumA = fieldOrNull(table, ZonedStatsSchema.Stat.SUM.fieldName()); Array nullCountA = fieldOrNull(table, ZonedStatsSchema.Stat.NULL_COUNT.fieldName()); Array nanCountA = fieldOrNull(table, ZonedStatsSchema.Stat.NAN_COUNT.fieldName()); + // A legacy vortex.stats MIN/MAX is truncated (inexact) where its flag says so; a + // vortex.zoned MIN/MAX is exact, and a bounded_min/bounded_max only ever a bound. + Array minTruncatedA = fieldOrNull(table, ZonedStatsSchema.Stat.MIN_IS_TRUNCATED); + Array maxTruncatedA = fieldOrNull(table, ZonedStatsSchema.Stat.MAX_IS_TRUNCATED); // Not pre-sized from nZones: it is bounded above only by Integer.MAX_VALUE (see the // guard above), and a single ArrayList allocation at that scale is itself an // OutOfMemoryError vector the security contract forbids. Growing incrementally keeps @@ -586,9 +590,13 @@ private List decodeZoneTable(ColumnName column, boolean includeBound for (long i = 0; i < nZones; i++) { Object nullCount = boxedScalar(nullCountA, i); Object nanCount = boxedScalar(nanCountA, i); + Object min = minA != null ? boxedScalar(minA, i) : boxedScalar(boundedMinA, i); + Object max = maxA != null ? boxedScalar(maxA, i) : boundedMaxScalar(boundedMaxA, i); out.add(new ArrayStats( - minA != null ? boxedScalar(minA, i) : boxedScalar(boundedMinA, i), - maxA != null ? boxedScalar(maxA, i) : boundedMaxScalar(boundedMaxA, i), + min, + min != null && minA != null && !Boolean.TRUE.equals(boxedScalar(minTruncatedA, i)), + max, + max != null && maxA != null && !Boolean.TRUE.equals(boxedScalar(maxTruncatedA, i)), boxedScalar(sumA, i), null, null, null, nullCount == null ? null : ((Number) nullCount).longValue(), @@ -1267,7 +1275,8 @@ private static boolean canPrune(Predicate predicate, ArrayStats stats, long rowC && (Compare.values(v, min, ct) < 0 || Compare.values(v, max, ct) > 0); // Every row equals val (min == max == val) and no nulls → no row differs. A null row is // neither = nor != val under three-valued logic, so require a provably null-free chunk. - case Predicate.Neq(var v) -> min != null && max != null + // Inexact bounds only bracket the values, so min == max proves nothing without both exact. + case Predicate.Neq(var v) -> min != null && max != null && stats.minExact() && stats.maxExact() && nullCount != null && nullCount == 0 && Compare.values(v, min, ct) == 0 && Compare.values(v, max, ct) == 0; // max <= val → no row is > val. diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/VortexReader.java b/reader/src/main/java/io/github/dfa1/vortex/reader/VortexReader.java index 2bb17693c..60f228809 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/VortexReader.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/VortexReader.java @@ -227,6 +227,9 @@ private void validateColumns(List columns) { /// Returns an empty map if the root layout is not a struct. /// Columns with no embedded stats return [ArrayStats#empty()]. /// + /// Min/max are the exact extremes or `null`: a column whose stats only bound its values (an + /// inexact `min`/`max`) reports no min/max rather than a bound. + /// /// Folds the column's per-zone statistics ([ScanIterator#columnZoneStats(ColumnName)]), which for a /// global-dictionary column is the only place its values' min/max live — the flat nodes under /// the zone map hold dictionary codes, whose own min/max describe the codes, not the values. @@ -239,12 +242,15 @@ public Map columnStats() { for (ColumnName name : schema.fieldNames()) { DType column = schema.fieldTypes().get(schema.fieldNames().indexOf(name)); ArrayStats stats = aggregateStats(iter.columnZoneStats(name), column); - if (stats.min() == null && stats.max() == null) { - // A string zone map holds only bounds (Rust's bounded_max/bounded_min), which - // are not exact: take min/max from the chunks' own array stats instead. - ArrayStats exact = aggregateStats(iter.chunkStats(name), column); + if (!stats.minExact() || !stats.maxExact()) { + // A zone map may hold only bounds (a string's bounded_max/bounded_min, or a + // legacy truncated min/max): take the extreme from the chunks' own array stats + // instead, and report none when those are not exact either. + ArrayStats chunks = aggregateStats(iter.chunkStats(name), column); + Object min = stats.minExact() ? stats.min() : chunks.minExact() ? chunks.min() : null; + Object max = stats.maxExact() ? stats.max() : chunks.maxExact() ? chunks.max() : null; // aggregateStats folds only min/max/null count, the rest is always null. - stats = new ArrayStats(exact.min(), exact.max(), null, null, null, null, + stats = new ArrayStats(min, min != null, max, max != null, null, null, null, null, stats.nullCount(), null, null); } result.put(name, stats); @@ -256,6 +262,10 @@ public Map columnStats() { private ArrayStats aggregateStats(List zones, DType column) { Object globalMin = null; Object globalMax = null; + // The fold is exact only when every bound it took is: one inexact zone may hide its true + // extreme past the others'. + boolean minExact = true; + boolean maxExact = true; // Null count is meaningful only when every chunk carries it; one missing makes the column // total unknown (null), so don't report a partial count. long totalNullCount = 0L; @@ -263,9 +273,11 @@ private ArrayStats aggregateStats(List zones, DType column) { for (ArrayStats s : zones) { if (s.min() != null) { globalMin = globalMin == null ? s.min() : minOf(globalMin, s.min(), column); + minExact &= s.minExact(); } if (s.max() != null) { globalMax = globalMax == null ? s.max() : maxOf(globalMax, s.max(), column); + maxExact &= s.maxExact(); } if (s.nullCount() != null) { totalNullCount += s.nullCount(); @@ -279,7 +291,8 @@ private ArrayStats aggregateStats(List zones, DType column) { } // Sum is left null at the file level: it lives in the per-zone stats table, surfaced by // ScanIterator.columnZoneStats rather than folded here. - return new ArrayStats(globalMin, globalMax, null, null, null, null, nullCount, null, null); + return new ArrayStats(globalMin, globalMin != null && minExact, globalMax, globalMax != null && maxExact, + null, null, null, null, nullCount, null, null); } @Override diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayStatsTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayStatsTest.java index dd11c7042..486a77640 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayStatsTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayStatsTest.java @@ -220,7 +220,56 @@ void everyNonScalarStat_decodes() { ArrayStats result = ArrayStats.fromFbs(fbs); // Then - assertThat(result).isEqualTo(new ArrayStats(null, null, 12L, true, false, false, 0L, 800L, 0L)); + assertThat(result).isEqualTo( + new ArrayStats(null, false, null, false, 12L, true, false, false, 0L, 800L, 0L)); + } + + @Test + void exactPrecision_marksBoundsExact() { + // Given + io.github.dfa1.vortex.core.fbs.FbsArrayStats fbs = buildFbs( + ProtoScalarValue.ofStringValue("alpha").encode(), + ProtoScalarValue.ofStringValue("omega").encode(), + (int) io.github.dfa1.vortex.core.fbs.FbsPrecision.Exact); + + // When + ArrayStats result = ArrayStats.fromFbs(fbs); + + // Then + assertThat(result.minExact()).isTrue(); + assertThat(result.maxExact()).isTrue(); + } + + @Test + void absentPrecision_readsInexact() { + // Given — no precision written: the FlatBuffer default is Inexact, as Rust reads it. A + // truncated 64-byte string bound must not be reported as the column's real MAX. + io.github.dfa1.vortex.core.fbs.FbsArrayStats fbs = buildFbs( + ProtoScalarValue.ofStringValue("alpha").encode(), + ProtoScalarValue.ofStringValue("omega").encode()); + + // When + ArrayStats result = ArrayStats.fromFbs(fbs); + + // Then + assertThat(result.min()).isEqualTo("alpha"); + assertThat(result.minExact()).isFalse(); + assertThat(result.max()).isEqualTo("omega"); + assertThat(result.maxExact()).isFalse(); + } + + @Test + void corruptedPrecision_throwsVortexException() { + // Given — only Inexact (0) and Exact (1) exist; Rust bails on anything else + io.github.dfa1.vortex.core.fbs.FbsArrayStats fbs = buildFbs( + ProtoScalarValue.ofInt64Value(1L).encode(), + ProtoScalarValue.ofInt64Value(2L).encode(), + 2); + + // When / Then + assertThatThrownBy(() -> ArrayStats.fromFbs(fbs)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("corrupted min_precision"); } @Test @@ -236,6 +285,12 @@ void malformedScalarBytes_throwsVortexException() { } private static io.github.dfa1.vortex.core.fbs.FbsArrayStats buildFbs(byte[] minBytes, byte[] maxBytes) { + return buildFbs(minBytes, maxBytes, null); + } + + /// `precision` is written to both `min_precision` and `max_precision`, or left absent when `null`. + private static io.github.dfa1.vortex.core.fbs.FbsArrayStats buildFbs(byte[] minBytes, byte[] maxBytes, + Integer precision) { FbsBuilder b = new FbsBuilder(64); int minOff = minBytes == null ? 0 : io.github.dfa1.vortex.core.fbs.FbsArrayStats.createMinVector(b, minBytes); int maxOff = maxBytes == null ? 0 : io.github.dfa1.vortex.core.fbs.FbsArrayStats.createMaxVector(b, maxBytes); @@ -246,6 +301,10 @@ private static io.github.dfa1.vortex.core.fbs.FbsArrayStats buildFbs(byte[] minB if (maxBytes != null) { io.github.dfa1.vortex.core.fbs.FbsArrayStats.addMax(b, maxOff); } + if (precision != null) { + io.github.dfa1.vortex.core.fbs.FbsArrayStats.addMinPrecision(b, precision); + io.github.dfa1.vortex.core.fbs.FbsArrayStats.addMaxPrecision(b, precision); + } int root = io.github.dfa1.vortex.core.fbs.FbsArrayStats.endFbsArrayStats(b); b.finish(root); return io.github.dfa1.vortex.core.fbs.FbsArrayStats.getRootAsFbsArrayStats(b.dataSegment());