From 92417156cbd99ebdad417a52e083d0887398bbec Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sun, 18 Oct 2026 11:03:33 +0200 Subject: [PATCH] perf(calcite): push a prefix LIKE into the scan as a string range Calcite matched LIKE row by row, and its interpreter compiled the pattern to a regex for every row (Like.sqlToRegexLike + Pattern.compile were 30% of a 2.5M-row title LIKE 'Star Wars%'). RexFilterTranslator now translates col LIKE 'prefix%' on a string column into col >= prefix AND col < successor(prefix), where the successor raises the prefix's last code point by one (skipping the surrogate block; a trailing U+10FFFF is dropped first). Code-point order is the UTF-8 byte order RowFilter compares strings in, so the range holds exactly the strings starting with the prefix: it prunes zones from their string bounds and the scan enforces it, so Calcite stops re-checking the row. A pattern with another wildcard or an ESCAPE clause is left to Calcite; LIKE '%' keeps every non-null row. Raincloud imdb title (2.53M rows), best of 3: count(*) where title like 'Star Wars%' 1049 -> 206 ms Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01WtuVGQ5h5Lk6Zn4JnkxF1K --- CHANGELOG.md | 1 + .../vortex/calcite/RexFilterTranslator.java | 49 ++++++++++- .../dfa1/vortex/calcite/LikePushDownTest.java | 88 +++++++++++++++++++ docs/reference.md | 5 +- .../CalciteInteropIntegrationTest.java | 18 ++++ 5 files changed, 158 insertions(+), 3 deletions(-) create mode 100644 calcite/src/test/java/io/github/dfa1/vortex/calcite/LikePushDownTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 375a93acf..8df59519c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Custom extension types: register an `ExtensionDecoder` on `ReadRegistry` and an `ExtensionEncoder` on `WriteRegistry`, with a `validate` hook on both sides, and `Chunk.as` decodes any registered type. ([#530](https://github.com/dfa1/vortex-java/issues/530)) ### Changed +- SQL `LIKE 'prefix%'` on a string column is about 5 times faster: it is pushed into the scan as a string range that prunes chunks, instead of being matched by Calcite row by row. ([#558](https://github.com/dfa1/vortex-java/pull/558)) - SQL `ORDER BY ... LIMIT n` is up to 9 times faster: the first rows are kept in a bounded heap during the scan instead of sorting the whole table. ([#557](https://github.com/dfa1/vortex-java/pull/557)) - SQL `GROUP BY` on a dictionary-encoded string column is 10 to 30 times faster: rows are counted and summed per dictionary code instead of being built one by one for Calcite. ([#555](https://github.com/dfa1/vortex-java/pull/555)) - A filtered SQL `COUNT`/`MIN`/`MAX` answered from zone maps decodes the zones its filter cuts once per query instead of up to six times. ([#555](https://github.com/dfa1/vortex-java/pull/555)) diff --git a/calcite/src/main/java/io/github/dfa1/vortex/calcite/RexFilterTranslator.java b/calcite/src/main/java/io/github/dfa1/vortex/calcite/RexFilterTranslator.java index ff83817f5..d28d9d5ea 100644 --- a/calcite/src/main/java/io/github/dfa1/vortex/calcite/RexFilterTranslator.java +++ b/calcite/src/main/java/io/github/dfa1/vortex/calcite/RexFilterTranslator.java @@ -35,7 +35,7 @@ private RexFilterTranslator() { /// Translates the Calcite predicates into a zone-map [RowFilter], keeping only what we can prune /// on: `=`, `<>`, `<`, `<=`, `>`, `>=` between a column and a literal, `IS [NOT] NULL`, a - /// single-column `OR` (such as an `IN` list), and `AND`. Predicates we don't understand are + /// prefix `LIKE 'abc%'`, a single-column `OR` (such as an `IN` list), and `AND`. Predicates we don't understand are /// simply not pushed — Calcite still applies them. /// /// @param filters the predicates Calcite pushed onto the scan @@ -172,6 +172,7 @@ private static Optional comparison(RexNode node, List nam case EQUALS, NOT_EQUALS, LESS_THAN, LESS_THAN_OR_EQUAL, GREATER_THAN, GREATER_THAN_OR_EQUAL -> binary(call, names, types); case IS_NULL, IS_NOT_NULL -> nullCheck(call, names); + case LIKE -> like(call, names, types); default -> Optional.empty(); }; } @@ -213,6 +214,52 @@ private static Optional nullCheck(RexCall call, List name : RowFilter.isNotNull(col)); } + /// Translates `col LIKE 'prefix%'` on a string column into the range `col >= prefix AND col < + /// successor(prefix)`, which holds exactly the strings starting with `prefix`: the successor + /// raises the prefix's last code point by one, and code-point order is the UTF-8 byte order + /// [RowFilter] compares strings in. A pattern with any other wildcard (`_`, an inner `%`) or an + /// `ESCAPE` clause is not translated; Calcite evaluates it. `LIKE '%'` keeps every non-null row. + private static Optional like(RexCall call, List names, List types) { + List ops = call.getOperands(); + if (ops.size() != 2 || !(ops.get(0) instanceof RexInputRef ref) || !(ops.get(1) instanceof RexLiteral lit) + || !(types.get(ref.getIndex()) instanceof DType.Utf8)) { + return Optional.empty(); + } + String pattern = lit.getValueAs(String.class); + if (pattern == null || !pattern.endsWith("%")) { + return Optional.empty(); + } + String prefix = pattern.substring(0, pattern.length() - 1); + if (prefix.indexOf('%') >= 0 || prefix.indexOf('_') >= 0) { + return Optional.empty(); + } + ColumnName col = names.get(ref.getIndex()); + if (prefix.isEmpty()) { + return Optional.of(RowFilter.isNotNull(col)); + } + return Optional.of(successor(prefix) + .map(upper -> RowFilter.and(RowFilter.gte(col, prefix), RowFilter.lt(col, upper))) + .orElseGet(() -> RowFilter.gte(col, prefix))); + } + + /// The least string greater than every string starting with `prefix`: its last code point raised + /// by one, skipping the surrogate block; a trailing U+10FFFF has no successor and is dropped + /// first. Empty when every code point is U+10FFFF. + static Optional successor(String prefix) { + int[] codePoints = prefix.codePoints().toArray(); + for (int end = codePoints.length - 1; end >= 0; end--) { + int next = codePoints[end] + 1; + if (next == Character.MIN_SURROGATE) { + next = Character.MAX_SURROGATE + 1; + } + if (next <= Character.MAX_CODE_POINT) { + codePoints[end] = next; + return Optional.of(new String(codePoints, 0, end + 1)); + } + } + return Optional.empty(); + } + private static Optional binary(RexCall call, List names, List types) { List ops = call.getOperands(); if (ops.size() != 2) { diff --git a/calcite/src/test/java/io/github/dfa1/vortex/calcite/LikePushDownTest.java b/calcite/src/test/java/io/github/dfa1/vortex/calcite/LikePushDownTest.java new file mode 100644 index 000000000..12fb93b82 --- /dev/null +++ b/calcite/src/test/java/io/github/dfa1/vortex/calcite/LikePushDownTest.java @@ -0,0 +1,88 @@ +package io.github.dfa1.vortex.calcite; + +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.writer.VortexWriter; +import io.github.dfa1.vortex.writer.WriteOptions; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +/// A prefix `LIKE 'abc%'` is pushed into the scan as a string range, which prunes zones and spares +/// Calcite compiling the pattern per row. Every query must return the rows of the same query over +/// `s || ''`, an expression the translation leaves to Calcite. +class LikePushDownTest { + + @TempDir + static Path tmp; + private static Path file; + + /// Two chunks of strings around the edges of a prefix range: the prefix itself, a string just + /// past its successor, non-Latin-1 code points, and a code point outside the BMP. + @BeforeAll + static void writeFile() throws Exception { + file = tmp.resolve("like.vortex"); + DType.Struct schema = DType.structBuilder() + .field(ColumnName.of("id"), DType.I64) + .field(ColumnName.of("s"), new DType.Utf8(true)) + .build(); + try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + var writer = VortexWriter.create(ch, schema, WriteOptions.defaults())) { + writer.writeChunk(Map.of(ColumnName.of("id"), new long[]{1, 2, 3, 4, 5, 6}, + ColumnName.of("s"), new String[]{"abc", "abcd", "abd", "ab", "Abc", null})); + writer.writeChunk(Map.of(ColumnName.of("id"), new long[]{7, 8, 9, 10, 11, 12}, + ColumnName.of("s"), new String[]{"Ωmega", "ÿz", "Ā", "a_c", "a%c", "😀x"})); + } + } + + @ParameterizedTest + @ValueSource(strings = { + // the successor of ÿ (U+00FF) is Ā (U+0100), outside Latin-1: it bounds the range but + // never appears in SQL, which only takes Latin-1 literals + "ab%", "abc%", "abd%", "A%", "ÿ%", "zzz%", "%", + // not a prefix pattern: left to Calcite, same answer + "a_c%", "%c", "abc"}) + void like_returnsTheRowsCalciteReturns(String pattern) throws Exception { + // Given / When + List result = ids("select id from vtx.t where s like '" + pattern + "' order by id"); + + // Then + assertThat(result).isEqualTo(ids("select id from vtx.t where s || '' like '" + pattern + "' order by id")); + } + + @Test + void successor_raisesTheLastCodePointSkippingSurrogatesAndMaxima() { + // Given / When / Then + assertThat(RexFilterTranslator.successor("abc")).contains("abd"); + assertThat(RexFilterTranslator.successor("a퟿")).contains("a"); + assertThat(RexFilterTranslator.successor("a" + Character.toString(Character.MAX_CODE_POINT))).contains("b"); + assertThat(RexFilterTranslator.successor(Character.toString(Character.MAX_CODE_POINT))).isEmpty(); + } + + private static List ids(String sql) throws Exception { + List out = new ArrayList<>(); + try (Connection conn = VortexCalcite.connect("vtx", Map.of("t", file)); + Statement st = conn.createStatement(); + ResultSet rs = st.executeQuery(sql)) { + while (rs.next()) { + out.add(rs.getLong(1)); + } + } + return out; + } +} diff --git a/docs/reference.md b/docs/reference.md index 928f1d7af..4cbe05e2b 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -612,7 +612,8 @@ preserved) and the **Babel** SQL parser, so a column named after a reserved word keywords that open a typed literal (`date`, `time`, `timestamp`, `interval`), which stay reserved even under Babel and need back-ticks: `` select `date` from vtx.ohlc ``. -A `WHERE` of those predicates (an `OR` only within one column) prunes chunks from zone maps and is +A `WHERE` of column-vs-literal comparisons, `IN` lists, `IS [NOT] NULL` and prefix `LIKE 'abc%'` +patterns, joined by `AND` (an `OR` only within one column), prunes chunks from zone maps and is enforced by the scan, so Calcite does not re-check it; anything else is left to Calcite. Column types map as follows. Unsigned integers widen to the next signed type (`U64` stays `BIGINT` @@ -632,7 +633,7 @@ and fails on values past 2^63); `F16` reads as `REAL`. `count`/`min`/`max`/`sum` (and `avg`, reduced to `sum`/`count`) are answered from zone-map statistics via `VortexAggregatePushDownRule` where possible, with or without a `WHERE` of -column-vs-literal comparisons, `IN` lists and `IS [NOT] NULL` joined by `AND`: zones the filter selects entirely fold from their statistics, and only +column-vs-literal comparisons, `IN` lists, `IS [NOT] NULL` and prefix `LIKE` joined by `AND`: zones the filter selects entirely fold from their statistics, and only the rows of zones it cuts are decoded. `sum` needs a per-zone sum, which only the legacy `vortex.stats` zone map records (writes targeting an edition before `core2026.08.0`); with Rust's `vortex.zoned` — the default since #447, and every Rust-written file — `sum` is answered by a scan. diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/CalciteInteropIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/CalciteInteropIntegrationTest.java index 8c49da49c..39d351cfb 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/CalciteInteropIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/CalciteInteropIntegrationTest.java @@ -310,6 +310,24 @@ void jniWriter_orderByLimit_returnsTheRowsCalciteSortReturns(String tail, String } } + @ParameterizedTest(name = "like {0}") + @org.junit.jupiter.params.provider.ValueSource(strings = {"A%", "caf%", "MS%", "NVDB%", "%", "_APL%"}) + void jniWriter_likePrefix_returnsTheRowsCalciteReturns(String pattern, @TempDir Path tmp) throws Exception { + // Given — a prefix LIKE is pushed into the scan as a string range; "s || ''" keeps it out, + // so Calcite evaluates the pattern over the same Rust-written rows itself + Path file = tmp.resolve("jni_like.vortex"); + writeGroupsWithJni(file, new String[]{"AAPL", "MSFT", "NVDA", "caf\u00E9", "AMZN", null}); + + try (Connection conn = VortexCalcite.connect("vtx", Map.of("t", file))) { + // When + List result = sqlRows(conn, "select id, s from vtx.t where s like '" + pattern + "' order by id"); + + // Then + assertThat(result).isEqualTo( + sqlRows(conn, "select id, s from vtx.t where s || '' like '" + pattern + "' order by id")); + } + } + private static List sqlRows(Connection conn, String sql) throws Exception { List out = new java.util.ArrayList<>(); try (Statement st = conn.createStatement(); ResultSet rs = st.executeQuery(sql)) {