diff --git a/CHANGELOG.md b/CHANGELOG.md index 4817da462..0a0b0b2a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed +- **Breaking:** `ArrayStats` has one component per field of Rust's array stats, so `minExact`, `maxExact`, `isSorted`, `isStrictSorted`, `isConstant`, `uncompressedSizeInBytes` and `nanCount` are read from the file and `trueCount` is gone (a bool column's true count is its `sum`). ([#581](https://github.com/dfa1/vortex-java/pull/581)) + +### Fixed +- Files written by vortex-java mark their array min/max as exact, which Rust read as inexact bounds. ([#581](https://github.com/dfa1/vortex-java/pull/581)) +- `VortexReader.columnStats()` and SQL `MIN`/`MAX` no longer report a truncated (inexact) bound as a column's minimum or maximum. ([#581](https://github.com/dfa1/vortex-java/pull/581)) + ## [0.18.0] — 2026-10-19 ### Added diff --git a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexTable.java b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexTable.java index d30fedba8..60b849c82 100644 --- a/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexTable.java +++ b/calcite/src/main/java/io/github/dfa1/vortex/calcite/VortexTable.java @@ -346,12 +346,12 @@ public Optional filteredFold(RowFilter filter, ColumnName aggColum fold.keptRows += zone.rowCount(); if (aggColumn != null) { ArrayStats stats = zoneStats.get(aggColumn).get(z); - // a zone with values but no min/max stat (a string zone holds only bounds) - // leaves the extreme unknown, rather than silently ignored + // a zone with values but no exact min/max stat (a string zone holds only + // bounds) leaves the extreme unknown, rather than silently ignored boolean hasValues = stats.nullCount() == null || stats.nullCount() < zone.rowCount(); fold.addSum(stats.sum()); - fold.addMin(stats.min(), hasValues); - fold.addMax(stats.max(), hasValues); + fold.addMin(stats.minExact() ? stats.min() : null, hasValues); + fold.addMax(stats.maxExact() ? stats.max() : null, hasValues); fold.addNulls(stats.nullCount()); } } @@ -1169,7 +1169,8 @@ private static Match classifyColumnRaw(Predicate predicate, ArrayStats s, long r if (compareStat(value, s.min()) < 0 || compareStat(value, s.max()) > 0) { yield Match.OUT; // the value lies outside the zone's range } - if (compareStat(s.min(), s.max()) == 0 && compareStat(s.min(), value) == 0 && noNulls(s)) { + if (exactBounds(s) && compareStat(s.min(), s.max()) == 0 && compareStat(s.min(), value) == 0 + && noNulls(s)) { yield Match.IN; // every row equals the value } yield Match.BOUNDARY; @@ -1178,7 +1179,7 @@ private static Match classifyColumnRaw(Predicate predicate, ArrayStats s, long r if (uncomparable(s, value)) { yield Match.BOUNDARY; } - if (compareStat(s.min(), s.max()) == 0 && compareStat(s.min(), value) == 0) { + if (exactBounds(s) && compareStat(s.min(), s.max()) == 0 && compareStat(s.min(), value) == 0) { yield Match.OUT; // every row equals the value, so none differ } if ((compareStat(value, s.min()) < 0 || compareStat(value, s.max()) > 0) && noNulls(s)) { @@ -1280,6 +1281,12 @@ private static boolean uncomparable(ArrayStats s, Object value) { || value instanceof Number != s.min() instanceof Number; } + /// Whether the zone's min and max are its exact extremes: an inexact bound only brackets the + /// values, which is enough to rule a range in or out but not to prove `min == max`. + private static boolean exactBounds(ArrayStats s) { + return s.minExact() && s.maxExact(); + } + /// Whether the zone provably carries no nulls (its null count is recorded and zero). private static boolean noNulls(ArrayStats s) { return s.nullCount() != null && s.nullCount() == 0; diff --git a/docs/reference.md b/docs/reference.md index b43724b30..ae82dab10 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -105,7 +105,7 @@ all `Array` buffers obtained during scans become invalid. | `version()` | `int` | File format version | | `fileSize()` | `long` | File size in bytes | | `scan(ScanOptions)` | `ScanIterator` | Open a scan | -| `columnStats()` | `Map` | Aggregated min/max per column | +| `columnStats()` | `Map` | Exact min/max per column (`null` when only a bound is known) | | `slice(offset, length)` | `MemorySegment` | Zero-copy slice of mmap region | | `close()` | — | Releases mmap | diff --git a/inspector/src/main/java/io/github/dfa1/vortex/inspect/InspectorTree.java b/inspector/src/main/java/io/github/dfa1/vortex/inspect/InspectorTree.java index df0074c4d..bb62f82fd 100644 --- a/inspector/src/main/java/io/github/dfa1/vortex/inspect/InspectorTree.java +++ b/inspector/src/main/java/io/github/dfa1/vortex/inspect/InspectorTree.java @@ -289,7 +289,8 @@ private static ArrayStats fold(List zones) { if (min == null && max == null) { return ArrayStats.empty(); } - return new ArrayStats(min, max, null, null, null, null, null); + // display-only fold: precision is not tracked, so the bounds are not claimed exact + return new ArrayStats(min, false, max, false, null, null, null, null, null, null, null); } /// The chunks of a column: the children of the first chunked node on its data path, or the diff --git a/inspector/src/main/java/io/github/dfa1/vortex/inspect/VortexInspector.java b/inspector/src/main/java/io/github/dfa1/vortex/inspect/VortexInspector.java index 8adff6da8..fc9e9bf42 100644 --- a/inspector/src/main/java/io/github/dfa1/vortex/inspect/VortexInspector.java +++ b/inspector/src/main/java/io/github/dfa1/vortex/inspect/VortexInspector.java @@ -103,7 +103,8 @@ static ArrayStats aggregateStats(InspectorTree.Node node) { if (min == null && max == null) { return ArrayStats.empty(); } - return new ArrayStats(min, max, null, null, null, null, null); + // display-only fold: precision is not tracked, so the bounds are not claimed exact + return new ArrayStats(min, false, max, false, null, null, null, null, null, null, null); } @SuppressWarnings({"unchecked", "rawtypes"}) diff --git a/inspector/src/test/java/io/github/dfa1/vortex/inspect/HtmlReportTest.java b/inspector/src/test/java/io/github/dfa1/vortex/inspect/HtmlReportTest.java index 9c506d65d..69fc7e799 100644 --- a/inspector/src/test/java/io/github/dfa1/vortex/inspect/HtmlReportTest.java +++ b/inspector/src/test/java/io/github/dfa1/vortex/inspect/HtmlReportTest.java @@ -101,7 +101,7 @@ void render_escapesUntrustedColumnNames() { @Test void render_escapesUntrustedStatValues() { // Given — min/max are decoded scalars from the file, so a string column can carry markup - InspectorTree sut = singleColumnWithStats(new ArrayStats("", "\"&\"", null, null, 3L, null, null)); + InspectorTree sut = singleColumnWithStats(new ArrayStats("", true, "\"&\"", true, null, null, null, null, 3L, null, null)); // When String result = HtmlReport.render(sut, "evil.vortex"); @@ -540,7 +540,7 @@ void render_timestampColumn_showsBoundsAsInstantsNotEpochIntegers() { Layout leaf = new Layout(LayoutId.parse("vortex.flat"), 2, null, List.of(), List.of(0)); Layout root = new Layout(LayoutId.parse("vortex.struct"), 2, null, List.of(leaf), List.of()); InspectorTree.Node leafNode = new InspectorTree.Node(leaf, Optional.of("ts"), Set.of(), - new ArrayStats(1_704_067_200_000L, 1_706_867_193_000L, null, null, null, null, null), + new ArrayStats(1_704_067_200_000L, true, 1_706_867_193_000L, true, null, null, null, null, null, null, null), List.of()); InspectorTree.Node rootNode = new InspectorTree.Node(root, Optional.empty(), Set.of(), ArrayStats.empty(), List.of(leafNode)); @@ -590,7 +590,7 @@ private static InspectorTree twoColumnTree() { Layout root = new Layout(LayoutId.parse("vortex.struct"), 1000, null, List.of(idLeaf, valLeaf), List.of()); InspectorTree.Node idNode = new InspectorTree.Node(idLeaf, Optional.of("id"), - Set.of("fastlanes.bitpacked"), new ArrayStats(1L, 999L, null, null, 0L, null, null), List.of()); + Set.of("fastlanes.bitpacked"), new ArrayStats(1L, true, 999L, true, null, null, null, null, 0L, null, null), List.of()); InspectorTree.Node valNode = new InspectorTree.Node(valLeaf, Optional.of("value"), Set.of("vortex.constant"), ArrayStats.empty(), List.of()); InspectorTree.Node rootNode = new InspectorTree.Node(root, Optional.empty(), @@ -612,9 +612,9 @@ private static InspectorTree chunkedTree() { Layout root = new Layout(LayoutId.parse("vortex.struct"), 1000, null, List.of(chunked), List.of()); InspectorTree.Node n0 = new InspectorTree.Node(c0, Optional.empty(), Set.of(), - new ArrayStats(0L, 499L, null, null, null, null, null), List.of()); + new ArrayStats(0L, true, 499L, true, null, null, null, null, null, null, null), List.of()); InspectorTree.Node n1 = new InspectorTree.Node(c1, Optional.empty(), Set.of(), - new ArrayStats(500L, 999L, null, null, null, null, null), List.of()); + new ArrayStats(500L, true, 999L, true, null, null, null, null, null, null, null), List.of()); InspectorTree.Node col = new InspectorTree.Node(chunked, Optional.of("id"), Set.of(), ArrayStats.empty(), List.of(n0, n1)); InspectorTree.Node rootNode = new InspectorTree.Node(root, Optional.empty(), Set.of(), diff --git a/inspector/src/test/java/io/github/dfa1/vortex/inspect/VortexInspectorTest.java b/inspector/src/test/java/io/github/dfa1/vortex/inspect/VortexInspectorTest.java index 7f6712b9e..1b108f878 100644 --- a/inspector/src/test/java/io/github/dfa1/vortex/inspect/VortexInspectorTest.java +++ b/inspector/src/test/java/io/github/dfa1/vortex/inspect/VortexInspectorTest.java @@ -135,9 +135,9 @@ void render_aggregatesMinMaxAcrossChunks() { Layout structLayout = new Layout(LayoutId.parse("vortex.struct"), 1000, null, List.of(chunked), List.of()); InspectorTree.Node c1 = new InspectorTree.Node(chunk1, Optional.empty(), Set.of(), - new ArrayStats(10L, 50L, null, null, null, null, null), List.of()); + new ArrayStats(10L, true, 50L, true, null, null, null, null, null, null, null), List.of()); InspectorTree.Node c2 = new InspectorTree.Node(chunk2, Optional.empty(), Set.of(), - new ArrayStats(5L, 100L, null, null, null, null, null), List.of()); + new ArrayStats(5L, true, 100L, true, null, null, null, null, null, null, null), List.of()); InspectorTree.Node chunkedN = new InspectorTree.Node(chunked, Optional.of("id"), Set.of("vortex.flat"), ArrayStats.empty(), List.of(c1, c2)); InspectorTree.Node rootN = new InspectorTree.Node(structLayout, Optional.empty(), diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java index 233d56a8d..1e71bc47e 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/JavaWritesRustReadsIntegrationTest.java @@ -979,9 +979,16 @@ void javaWriter_jniReader_filteredUtf8_prunesOnTruncatedBounds(@TempDir Path tmp // When List result = readColumnFiltered(file, "k", Expression.binary(Expression.BinaryOp.EQ, Expression.column("k"), Expression.literal(target))); + io.github.dfa1.vortex.reader.ArrayStats stats; + try (var reader = io.github.dfa1.vortex.reader.VortexReader.open(file)) { + stats = reader.columnStats().get(ColumnName.of("k")); + } - // Then + // Then — vortex-jni accepts the flat stats' precision fields, and the column's min/max are + // the full keys: only the zone map truncates, so SQL MIN/MAX may answer from these stats. assertThat(result).extracting(Object::toString).containsExactly(target); + assertThat(stats.min()).isEqualTo(prefix + String.format("%08d", 0)); + assertThat(stats.max()).isEqualTo(prefix + String.format("%08d", 3 * batch - 1)); } @Test diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/VortexInspectorIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/VortexInspectorIntegrationTest.java index 1b0095d2f..5b92f3320 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/VortexInspectorIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/VortexInspectorIntegrationTest.java @@ -6,6 +6,7 @@ import dev.vortex.jni.NativeLoader; import io.github.dfa1.vortex.core.model.ColumnName; import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.reader.ArrayStats; import io.github.dfa1.vortex.reader.ReadRegistry; import io.github.dfa1.vortex.writer.WriteOptions; import io.github.dfa1.vortex.inspect.HtmlReport; @@ -31,7 +32,9 @@ import java.nio.file.StandardOpenOption; import java.util.Map; import java.nio.file.Path; +import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.Random; import static org.assertj.core.api.Assertions.assertThat; @@ -162,6 +165,44 @@ void htmlReport_globalDictColumn_chunkMinMaxAreValuesNotCodes(@TempDir Path tmp) .contains("cherrydate"); } + @Test + void peek_readsSortednessRustRecordsInFlatArrayStats(@TempDir Path tmp) throws IOException { + // Given — Rust records is_sorted/is_strict_sorted in a flat segment's array stats for an + // ascending integer column. Java's ArrayStats used to drop both (and carried a trueCount + // the wire has no field for), so these read as unknown. + Path file = tmp.resolve("sorted.vtx"); + writeJni(file, 50_000); + + // When + List result = new ArrayList<>(); + try (VortexReader vf = VortexReader.open(file, ReadRegistry.loadAll())) { + InspectorTree tree = InspectorTree.buildShallow(vf); + InspectorTree.Node id = tree.root().children().stream() + .filter(c -> c.fieldName().equals(Optional.of("id"))).findFirst().orElseThrow(); + for (InspectorTree.Node flat : dataFlats(id)) { + result.add(InspectorTree.peek(flat, vf).stats()); + } + } + + // Then + assertThat(result).isNotEmpty().allSatisfy(stats -> { + assertThat(stats.isSorted()).isTrue(); + assertThat(stats.isStrictSorted()).isTrue(); + }); + } + + /// The flat segments holding a column's values: every chunk of a chunked node, else the first + /// child (a zoned node's second child is its zone-map table, not data). + private static List dataFlats(InspectorTree.Node node) { + if (node.layout().isFlat()) { + return List.of(node); + } + if (node.layout().isChunked()) { + return node.children().stream().flatMap(c -> dataFlats(c).stream()).toList(); + } + return dataFlats(node.children().getFirst()); + } + private static String[] repeat(int rows, String a, String b) { String[] out = new String[rows]; for (int i = 0; i < rows; i++) { diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/ArrayStats.java b/reader/src/main/java/io/github/dfa1/vortex/reader/ArrayStats.java index e8c0dca3e..6e17089e8 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/ArrayStats.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/ArrayStats.java @@ -7,41 +7,42 @@ import java.lang.foreign.MemorySegment; import java.nio.charset.StandardCharsets; -/// Per-array statistics embedded in the encoding tree. +/// Per-array statistics embedded in the encoding tree: one component per field of the wire +/// `ArrayStats` table (Rust's `Stat` enum). A bool column's true count is its `sum`, as in Rust. /// -/// @param min minimum value in the array, or `null` if unknown -/// @param max maximum value in the array, or `null` if unknown -/// @param sum sum of values (`Long` for integer columns, `Double` for floats), or `null` if unknown -/// @param trueCount number of `true` values for bool columns, or `null` if unknown -/// @param nullCount number of null values, or `null` if unknown -/// @param isSorted `true` if the array is sorted in ascending order, or `null` if unknown -/// @param isStrictSorted `true` if the array is strictly sorted (no duplicates), or `null` if unknown -/// @param nanCount number of `NaN` values in a float column (`min` and `max` skip them), or `null` if unknown +/// `min`/`max` may be inexact (Rust's `Precision::Inexact`): still a valid bound, so pruning on a +/// range may use it, but not the true extreme (Rust's flat writer truncates a string bound longer +/// than 64 bytes). Anything that reports the value, or needs `min == max`, requires +/// `minExact`/`maxExact`. +/// +/// The counts are `u64` on the wire, read as a signed `Long`. +/// +/// @param min minimum value in the array, or `null` if unknown +/// @param minExact `true` if `min` is the exact minimum, not only a lower bound +/// @param max maximum value in the array, or `null` if unknown +/// @param maxExact `true` if `max` is the exact maximum, not only an upper bound +/// @param sum sum of values (`Long` for integer and bool columns, `Double` for floats), or `null` if unknown +/// @param isSorted `true` if the values are sorted ascending with nulls first (Rust orders null lowest), or `null` if unknown +/// @param isStrictSorted `true` if also no two neighbors are equal, two nulls included, or `null` if unknown +/// @param isConstant `true` if every value is null, or none is and all are equal, or `null` if unknown +/// @param nullCount number of null values, or `null` if unknown +/// @param uncompressedSizeInBytes the array's uncompressed size in bytes, or `null` if unknown +/// @param nanCount number of `NaN` values in a float column (`min` and `max` skip them), or `null` if unknown public record ArrayStats( Object min, + boolean minExact, Object max, + boolean maxExact, Object sum, - Long trueCount, - Long nullCount, Boolean isSorted, Boolean isStrictSorted, + Boolean isConstant, + Long nullCount, + Long uncompressedSizeInBytes, Long nanCount ) { - private static final ArrayStats EMPTY = new ArrayStats(null, null, null, null, null, null, null, null); - - /// Creates stats with no `NaN` count. - /// - /// @param min minimum value in the array, or `null` if unknown - /// @param max maximum value in the array, or `null` if unknown - /// @param sum sum of values, or `null` if unknown - /// @param trueCount number of `true` values for bool columns, or `null` if unknown - /// @param nullCount number of null values, or `null` if unknown - /// @param isSorted `true` if the array is sorted in ascending order, or `null` if unknown - /// @param isStrictSorted `true` if the array is strictly sorted, or `null` if unknown - public ArrayStats(Object min, Object max, Object sum, Long trueCount, Long nullCount, - Boolean isSorted, Boolean isStrictSorted) { - this(min, max, sum, trueCount, nullCount, isSorted, isStrictSorted, null); - } + private static final ArrayStats EMPTY = + new ArrayStats(null, false, null, false, null, null, null, null, null, null, null); /// Returns an empty stats instance with all fields set to `null`. /// @@ -51,7 +52,7 @@ public static ArrayStats empty() { } /// Parses stats from a FlatBuffers [io.github.dfa1.vortex.core.fbs.FbsArrayStats] table. - /// Returns an empty instance when `fbs` is `null` or carries no min/max and no null count. + /// Returns an empty instance when `fbs` is `null` or carries no statistic. /// /// @param fbs the FlatBuffers stats table, or `null` /// @return parsed stats, or an empty instance if no usable data is present @@ -61,12 +62,29 @@ public static ArrayStats fromFbs(io.github.dfa1.vortex.core.fbs.FbsArrayStats fb } Object min = decodeScalar(fbs.minAsSegment()); Object max = decodeScalar(fbs.maxAsSegment()); - Object sum = decodeScalar(fbs.sumAsSegment()); - Long nullCount = fbs.hasNullCount() ? fbs.nullCount() : null; - if (min == null && max == null && sum == null && nullCount == null) { - return EMPTY; - } - return new ArrayStats(min, max, sum, null, nullCount, null, null); + ArrayStats stats = new ArrayStats( + min, + min != null && isExact(fbs.minPrecision(), "min_precision"), + max, + max != null && isExact(fbs.maxPrecision(), "max_precision"), + decodeScalar(fbs.sumAsSegment()), + fbs.hasIsSorted() ? fbs.isSorted() : null, + fbs.hasIsStrictSorted() ? fbs.isStrictSorted() : null, + fbs.hasIsConstant() ? fbs.isConstant() : null, + fbs.hasNullCount() ? fbs.nullCount() : null, + fbs.hasUncompressedSizeInBytes() ? fbs.uncompressedSizeInBytes() : null, + fbs.hasNanCount() ? fbs.nanCount() : null); + return stats.equals(EMPTY) ? EMPTY : stats; + } + + /// Rust's `from_flatbuffer`: an absent precision is the FlatBuffer default `Inexact`, and any + /// value other than `Inexact`/`Exact` is a corrupted field. Read only when its bound is present. + private static boolean isExact(int precision, String field) { + return switch (precision) { + case io.github.dfa1.vortex.core.fbs.FbsPrecision.Inexact -> false; + case io.github.dfa1.vortex.core.fbs.FbsPrecision.Exact -> true; + default -> throw new VortexException("corrupted " + field + " in array stats: " + precision); + }; } private static Object decodeScalar(MemorySegment seg) { diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java index 0868b4057..359f2045c 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java @@ -455,8 +455,8 @@ public long[] chunkRowCounts() { /// table — min/max/sum/null count per zone — decoded once from the small stats segment /// without touching any data segment. This is the source Rust populates for `SUM`, so the /// values match files written by either implementation. When no zone map is present the - /// list falls back to each chunk's embedded `ArrayStats` (min/max/null count; `sum` is - /// `null`, since the flat writer does not retain it). Either way a column that is absent + /// list falls back to each chunk's embedded `ArrayStats` (whatever the flat writer retained; + /// `sum` is `null`, since it does not retain it). Either way a column that is absent /// or carries no stats yields [ArrayStats#empty()] per zone. /// /// Zone granularity is the layout's, not the scan's. The fallback path is one entry per scan @@ -578,6 +578,10 @@ private List decodeZoneTable(ColumnName column, boolean includeBound Array sumA = fieldOrNull(table, ZonedStatsSchema.Stat.SUM.fieldName()); Array nullCountA = fieldOrNull(table, ZonedStatsSchema.Stat.NULL_COUNT.fieldName()); Array nanCountA = fieldOrNull(table, ZonedStatsSchema.Stat.NAN_COUNT.fieldName()); + // A legacy vortex.stats MIN/MAX is truncated (inexact) where its flag says so; a + // vortex.zoned MIN/MAX is exact, and a bounded_min/bounded_max only ever a bound. + Array minTruncatedA = fieldOrNull(table, ZonedStatsSchema.Stat.MIN_IS_TRUNCATED); + Array maxTruncatedA = fieldOrNull(table, ZonedStatsSchema.Stat.MAX_IS_TRUNCATED); // Not pre-sized from nZones: it is bounded above only by Integer.MAX_VALUE (see the // guard above), and a single ArrayList allocation at that scale is itself an // OutOfMemoryError vector the security contract forbids. Growing incrementally keeps @@ -586,13 +590,17 @@ private List decodeZoneTable(ColumnName column, boolean includeBound for (long i = 0; i < nZones; i++) { Object nullCount = boxedScalar(nullCountA, i); Object nanCount = boxedScalar(nanCountA, i); + Object min = minA != null ? boxedScalar(minA, i) : boxedScalar(boundedMinA, i); + Object max = maxA != null ? boxedScalar(maxA, i) : boundedMaxScalar(boundedMaxA, i); out.add(new ArrayStats( - minA != null ? boxedScalar(minA, i) : boxedScalar(boundedMinA, i), - maxA != null ? boxedScalar(maxA, i) : boundedMaxScalar(boundedMaxA, i), + min, + min != null && minA != null && !Boolean.TRUE.equals(boxedScalar(minTruncatedA, i)), + max, + max != null && maxA != null && !Boolean.TRUE.equals(boxedScalar(maxTruncatedA, i)), boxedScalar(sumA, i), - null, + null, null, null, nullCount == null ? null : ((Number) nullCount).longValue(), - null, null, + null, nanCount == null ? null : ((Number) nanCount).longValue())); } return out; @@ -1255,9 +1263,10 @@ private static boolean canPrune(Predicate predicate, ArrayStats stats, long rowC Object max = stats.max(); Long nullCount = stats.nullCount(); if (ct instanceof DType.Primitive p && p.ptype().isFloating() - && (stats.nanCount() == null || stats.nanCount() > 0)) { + && (stats.nanCount() == null || stats.nanCount() != 0)) { // Float min/max skip NaN, but filters order floats totally (Rust), where NaN is the - // maximum: a chunk that may hold one (nan_count absent or positive) has a NaN upper bound. + // maximum: a chunk that may hold one (nan_count absent or nonzero) has a NaN upper bound. + // nan_count is u64: a count past 2^63 reads as a negative Long, still not zero. max = Double.NaN; } return switch (predicate) { @@ -1266,7 +1275,8 @@ private static boolean canPrune(Predicate predicate, ArrayStats stats, long rowC && (Compare.values(v, min, ct) < 0 || Compare.values(v, max, ct) > 0); // Every row equals val (min == max == val) and no nulls → no row differs. A null row is // neither = nor != val under three-valued logic, so require a provably null-free chunk. - case Predicate.Neq(var v) -> min != null && max != null + // Inexact bounds only bracket the values, so min == max proves nothing without both exact. + case Predicate.Neq(var v) -> min != null && max != null && stats.minExact() && stats.maxExact() && nullCount != null && nullCount == 0 && Compare.values(v, min, ct) == 0 && Compare.values(v, max, ct) == 0; // max <= val → no row is > val. diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/VortexReader.java b/reader/src/main/java/io/github/dfa1/vortex/reader/VortexReader.java index 704a58b76..60f228809 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/VortexReader.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/VortexReader.java @@ -227,6 +227,9 @@ private void validateColumns(List columns) { /// Returns an empty map if the root layout is not a struct. /// Columns with no embedded stats return [ArrayStats#empty()]. /// + /// Min/max are the exact extremes or `null`: a column whose stats only bound its values (an + /// inexact `min`/`max`) reports no min/max rather than a bound. + /// /// Folds the column's per-zone statistics ([ScanIterator#columnZoneStats(ColumnName)]), which for a /// global-dictionary column is the only place its values' min/max live — the flat nodes under /// the zone map hold dictionary codes, whose own min/max describe the codes, not the values. @@ -239,12 +242,16 @@ public Map columnStats() { for (ColumnName name : schema.fieldNames()) { DType column = schema.fieldTypes().get(schema.fieldNames().indexOf(name)); ArrayStats stats = aggregateStats(iter.columnZoneStats(name), column); - if (stats.min() == null && stats.max() == null) { - // A string zone map holds only bounds (Rust's bounded_max/bounded_min), which - // are not exact: take min/max from the chunks' own array stats instead. - ArrayStats exact = aggregateStats(iter.chunkStats(name), column); - stats = new ArrayStats(exact.min(), exact.max(), stats.sum(), stats.trueCount(), - stats.nullCount(), stats.isSorted(), stats.isStrictSorted()); + if (!stats.minExact() || !stats.maxExact()) { + // A zone map may hold only bounds (a string's bounded_max/bounded_min, or a + // legacy truncated min/max): take the extreme from the chunks' own array stats + // instead, and report none when those are not exact either. + ArrayStats chunks = aggregateStats(iter.chunkStats(name), column); + Object min = stats.minExact() ? stats.min() : chunks.minExact() ? chunks.min() : null; + Object max = stats.maxExact() ? stats.max() : chunks.maxExact() ? chunks.max() : null; + // aggregateStats folds only min/max/null count, the rest is always null. + stats = new ArrayStats(min, min != null, max, max != null, null, null, null, null, + stats.nullCount(), null, null); } result.put(name, stats); } @@ -255,6 +262,10 @@ public Map columnStats() { private ArrayStats aggregateStats(List zones, DType column) { Object globalMin = null; Object globalMax = null; + // The fold is exact only when every bound it took is: one inexact zone may hide its true + // extreme past the others'. + boolean minExact = true; + boolean maxExact = true; // Null count is meaningful only when every chunk carries it; one missing makes the column // total unknown (null), so don't report a partial count. long totalNullCount = 0L; @@ -262,9 +273,11 @@ private ArrayStats aggregateStats(List zones, DType column) { for (ArrayStats s : zones) { if (s.min() != null) { globalMin = globalMin == null ? s.min() : minOf(globalMin, s.min(), column); + minExact &= s.minExact(); } if (s.max() != null) { globalMax = globalMax == null ? s.max() : maxOf(globalMax, s.max(), column); + maxExact &= s.maxExact(); } if (s.nullCount() != null) { totalNullCount += s.nullCount(); @@ -278,7 +291,8 @@ private ArrayStats aggregateStats(List zones, DType column) { } // Sum is left null at the file level: it lives in the per-zone stats table, surfaced by // ScanIterator.columnZoneStats rather than folded here. - return new ArrayStats(globalMin, globalMax, null, null, nullCount, null, null); + return new ArrayStats(globalMin, globalMin != null && minExact, globalMax, globalMax != null && maxExact, + null, null, null, null, nullCount, null, null); } @Override diff --git a/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayStatsTest.java b/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayStatsTest.java index 57e336115..486a77640 100644 --- a/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayStatsTest.java +++ b/reader/src/test/java/io/github/dfa1/vortex/reader/ArrayStatsTest.java @@ -29,10 +29,13 @@ void hasAllNullComponents() { // Then assertThat(sut.min()).isNull(); assertThat(sut.max()).isNull(); - assertThat(sut.trueCount()).isNull(); - assertThat(sut.nullCount()).isNull(); + assertThat(sut.sum()).isNull(); assertThat(sut.isSorted()).isNull(); assertThat(sut.isStrictSorted()).isNull(); + assertThat(sut.isConstant()).isNull(); + assertThat(sut.nullCount()).isNull(); + assertThat(sut.uncompressedSizeInBytes()).isNull(); + assertThat(sut.nanCount()).isNull(); } } @@ -192,6 +195,83 @@ void emptyByteVector_treatedAsAbsent() { assertThat(sut).isSameAs(ArrayStats.empty()); } + @Test + void everyNonScalarStat_decodes() { + // Given — every stat besides min/max (covered above), with false/0 values on purpose: + // they equal the FlatBuffer scalar defaults, so a reader that skips a field or tests the + // value instead of its presence reads them as unknown. A lost nan_count = 0 reads as + // "may hold NaN", which caps a float chunk's max at NaN so it never prunes. + FbsBuilder builder = new FbsBuilder(128); + int sumOff = io.github.dfa1.vortex.core.fbs.FbsArrayStats.createSumVector( + builder, ProtoScalarValue.ofInt64Value(12L).encode()); + builder.forceDefaults(true); + io.github.dfa1.vortex.core.fbs.FbsArrayStats.startFbsArrayStats(builder); + io.github.dfa1.vortex.core.fbs.FbsArrayStats.addSum(builder, sumOff); + io.github.dfa1.vortex.core.fbs.FbsArrayStats.addIsSorted(builder, true); + io.github.dfa1.vortex.core.fbs.FbsArrayStats.addIsStrictSorted(builder, false); + io.github.dfa1.vortex.core.fbs.FbsArrayStats.addIsConstant(builder, false); + io.github.dfa1.vortex.core.fbs.FbsArrayStats.addNullCount(builder, 0L); + io.github.dfa1.vortex.core.fbs.FbsArrayStats.addUncompressedSizeInBytes(builder, 800L); + io.github.dfa1.vortex.core.fbs.FbsArrayStats.addNanCount(builder, 0L); + builder.finish(io.github.dfa1.vortex.core.fbs.FbsArrayStats.endFbsArrayStats(builder)); + var fbs = io.github.dfa1.vortex.core.fbs.FbsArrayStats.getRootAsFbsArrayStats(builder.dataSegment()); + + // When + ArrayStats result = ArrayStats.fromFbs(fbs); + + // Then + assertThat(result).isEqualTo( + new ArrayStats(null, false, null, false, 12L, true, false, false, 0L, 800L, 0L)); + } + + @Test + void exactPrecision_marksBoundsExact() { + // Given + io.github.dfa1.vortex.core.fbs.FbsArrayStats fbs = buildFbs( + ProtoScalarValue.ofStringValue("alpha").encode(), + ProtoScalarValue.ofStringValue("omega").encode(), + (int) io.github.dfa1.vortex.core.fbs.FbsPrecision.Exact); + + // When + ArrayStats result = ArrayStats.fromFbs(fbs); + + // Then + assertThat(result.minExact()).isTrue(); + assertThat(result.maxExact()).isTrue(); + } + + @Test + void absentPrecision_readsInexact() { + // Given — no precision written: the FlatBuffer default is Inexact, as Rust reads it. A + // truncated 64-byte string bound must not be reported as the column's real MAX. + io.github.dfa1.vortex.core.fbs.FbsArrayStats fbs = buildFbs( + ProtoScalarValue.ofStringValue("alpha").encode(), + ProtoScalarValue.ofStringValue("omega").encode()); + + // When + ArrayStats result = ArrayStats.fromFbs(fbs); + + // Then + assertThat(result.min()).isEqualTo("alpha"); + assertThat(result.minExact()).isFalse(); + assertThat(result.max()).isEqualTo("omega"); + assertThat(result.maxExact()).isFalse(); + } + + @Test + void corruptedPrecision_throwsVortexException() { + // Given — only Inexact (0) and Exact (1) exist; Rust bails on anything else + io.github.dfa1.vortex.core.fbs.FbsArrayStats fbs = buildFbs( + ProtoScalarValue.ofInt64Value(1L).encode(), + ProtoScalarValue.ofInt64Value(2L).encode(), + 2); + + // When / Then + assertThatThrownBy(() -> ArrayStats.fromFbs(fbs)) + .isInstanceOf(VortexException.class) + .hasMessageContaining("corrupted min_precision"); + } + @Test void malformedScalarBytes_throwsVortexException() { // Given — varint tag with no continuation byte: ProtoReader hits EOF inside readVarint @@ -205,6 +285,12 @@ void malformedScalarBytes_throwsVortexException() { } private static io.github.dfa1.vortex.core.fbs.FbsArrayStats buildFbs(byte[] minBytes, byte[] maxBytes) { + return buildFbs(minBytes, maxBytes, null); + } + + /// `precision` is written to both `min_precision` and `max_precision`, or left absent when `null`. + private static io.github.dfa1.vortex.core.fbs.FbsArrayStats buildFbs(byte[] minBytes, byte[] maxBytes, + Integer precision) { FbsBuilder b = new FbsBuilder(64); int minOff = minBytes == null ? 0 : io.github.dfa1.vortex.core.fbs.FbsArrayStats.createMinVector(b, minBytes); int maxOff = maxBytes == null ? 0 : io.github.dfa1.vortex.core.fbs.FbsArrayStats.createMaxVector(b, maxBytes); @@ -215,6 +301,10 @@ private static io.github.dfa1.vortex.core.fbs.FbsArrayStats buildFbs(byte[] minB if (maxBytes != null) { io.github.dfa1.vortex.core.fbs.FbsArrayStats.addMax(b, maxOff); } + if (precision != null) { + io.github.dfa1.vortex.core.fbs.FbsArrayStats.addMinPrecision(b, precision); + io.github.dfa1.vortex.core.fbs.FbsArrayStats.addMaxPrecision(b, precision); + } int root = io.github.dfa1.vortex.core.fbs.FbsArrayStats.endFbsArrayStats(b); b.finish(root); return io.github.dfa1.vortex.core.fbs.FbsArrayStats.getRootAsFbsArrayStats(b.dataSegment()); diff --git a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java index a7e755efe..a0ae26496 100644 --- a/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java +++ b/writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java @@ -1058,8 +1058,12 @@ private ByteBuffer buildArrayFlatBuffer(EncodeResult result, int[] paddings, lon fbb.forceDefaults(true); io.github.dfa1.vortex.core.fbs.FbsArrayStats.startFbsArrayStats(fbb); if (result.hasStats()) { + // The encoder's min/max are the exact extremes, never truncated. An absent precision + // is the FlatBuffer default Inexact, which is what Rust read every Java bound as. io.github.dfa1.vortex.core.fbs.FbsArrayStats.addMin(fbb, minVec); + io.github.dfa1.vortex.core.fbs.FbsArrayStats.addMinPrecision(fbb, io.github.dfa1.vortex.core.fbs.FbsPrecision.Exact); io.github.dfa1.vortex.core.fbs.FbsArrayStats.addMax(fbb, maxVec); + io.github.dfa1.vortex.core.fbs.FbsArrayStats.addMaxPrecision(fbb, io.github.dfa1.vortex.core.fbs.FbsPrecision.Exact); } io.github.dfa1.vortex.core.fbs.FbsArrayStats.addNullCount(fbb, nullCount); int statsOff = io.github.dfa1.vortex.core.fbs.FbsArrayStats.endFbsArrayStats(fbb);