perf: CSV import ~1.5x faster (pipelined, byte-level parse, UTF-8 bytes into the writer)

CHANGELOG.md

@@ -7,9 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+- `VortexWriter` accepts a `Utf8` column as its UTF-8 bytes (`byte[][]`), skipping the per-string encoding; malformed UTF-8 is rejected. ([#590](https://github.com/dfa1/vortex-java/pull/590))
+
 ### Changed
 - **Breaking:** SQL identifiers are quoted with double quotes, as IntelliJ and DBeaver generate them, instead of back-ticks, so opening a table or sorting its grid works; write `"date"` where you wrote `` `date` ``. ([#588](https://github.com/dfa1/vortex-java/pull/588))
 - **Breaking:** the `ScanIterator` constructor and the reader's Pco helpers (`PcoTansDecoder`, `PcoBin`, `LeBitReader`) are no longer public, and the built-in extension codecs are reached only through their `INSTANCE`. ([4ba6ea12](https://github.com/dfa1/vortex-java/commit/4ba6ea12))
+- CSV import is about 40% faster: it parses bytes instead of `String`s and overlaps parsing with writing; the delimiter must be an ASCII character. ([#590](https://github.com/dfa1/vortex-java/pull/590))
 - SQL `ORDER BY ... LIMIT` on a string column is up to 3 times faster: rows that cannot make the top are ranked by their bytes without building a `String`, and a chunk stored in the opposite order is read backward. ([#575](https://github.com/dfa1/vortex-java/pull/575))
 
 ## [0.18.0] — 2026-10-20

CLAUDE.md

@@ -38,7 +38,7 @@ core    — everything lives under `io.github.dfa1.vortex.core.*`:
                         EditionId, Edition, EditionFamily, Editions, MemorySize
           core.io       IoBounds, PTypeIO, VortexFormat
           core.error    VortexException
-          core.compute  FastLanes, PrimitiveArrays, Utf8Order
+          core.compute  FastLanes, PrimitiveArrays, Utf8Order, Utf8Validation
           core.simd     SimdOperations (kernels shared by reader+writer, all eleven ptypes),
                         AutoVectorizedSimdOperations (C2-shaped loops, the Rust-parity default),
                         VectorApiSimdOperations (opt-in via --add-modules jdk.incubator.vector; Rust-parity results,
@@ -366,6 +366,9 @@ procedure):
   `forEachChunk`; use `Optional`, records, sealed types, pattern switches, virtual threads, FFM).
   New APIs should feel like JDK APIs.
 - Always braces for `if`/`else`/`for`/`while`, even one-liners (`if (c) { return a; }`).
+- **No ternary that wraps or binds.** A `?:` that spans lines, or whose condition binds a pattern
+  (`x instanceof Foo f && … ? f.a() : x`), is an `if`: assign the default, then override it.
+  A short one-line ternary (`n == 0 ? 1 : n`) is fine.
 - **Enums: never call `values()` on a call path.** `values()` allocates a fresh array on every call (64 B for
   `PType`), which the JIT does not remove. Look a constant up by wire tag, ordinal or number with a `switch`
   expression: no allocation, and about 4x faster than `values()[i]` when the ordinals repeat, as in real files

core/src/main/java/io/github/dfa1/vortex/core/compute/Utf8Validation.java

@@ -0,0 +1,61 @@
+package io.github.dfa1.vortex.core.compute;
+
+/// Checks that bytes are well-formed UTF-8, as a Utf8 array's bytes must be.
+///
+/// Rust validates a Utf8 array's bytes when it builds the array, so a writer that takes UTF-8
+/// bytes from its caller must reject what Rust would. Well-formed is RFC 3629: no overlong forms,
+/// no surrogates (U+D800..U+DFFF), nothing above U+10FFFF.
+public final class Utf8Validation {
+
+    private Utf8Validation() {
+    }
+
+    /// Whether `bytes[from, to)` is well-formed UTF-8.
+    ///
+    /// @param bytes the bytes
+    /// @param from  the first byte, inclusive
+    /// @param to    the last byte, exclusive
+    /// @return `true` if the range is well-formed UTF-8
+    public static boolean isWellFormed(byte[] bytes, int from, int to) {
+        int i = from;
+        while (i < to) {
+            int lead = bytes[i] & 0xFF;
+            if (lead < 0x80) {
+                i++;
+                continue;
+            }
+            int length;
+            if (lead >= 0xC2 && lead <= 0xDF) {
+                length = 2;
+            } else if (lead >= 0xE0 && lead <= 0xEF) {
+                length = 3;
+            } else if (lead >= 0xF0 && lead <= 0xF4) {
+                length = 4;
+            } else {
+                return false;
+            }
+            if (to - i < length || !secondByteFits(lead, bytes[i + 1] & 0xFF)) {
+                return false;
+            }
+            for (int k = 2; k < length; k++) {
+                if ((bytes[i + k] & 0xC0) != 0x80) {
+                    return false;
+                }
+            }
+            i += length;
+        }
+        return true;
+    }
+
+    /// The second byte's range narrows after the leads that would otherwise allow an overlong
+    /// form (E0, F0), a surrogate (ED) or a code point above U+10FFFF (F4): RFC 3629 table 3.7.
+    private static boolean secondByteFits(int lead, int second) {
+        return switch (lead) {
+            case 0xE0 -> second >= 0xA0 && second <= 0xBF;
+            case 0xED -> second >= 0x80 && second <= 0x9F;
+            case 0xF0 -> second >= 0x90 && second <= 0xBF;
+            case 0xF4 -> second >= 0x80 && second <= 0x8F;
+            default -> second >= 0x80 && second <= 0xBF;
+        };
+    }
+}

core/src/test/java/io/github/dfa1/vortex/core/compute/Utf8ValidationTest.java

@@ -0,0 +1,95 @@
+package io.github.dfa1.vortex.core.compute;
+
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.CodingErrorAction;
+import java.nio.charset.StandardCharsets;
+import java.util.HexFormat;
+import java.util.Random;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+class Utf8ValidationTest {
+
+    @ParameterizedTest
+    @CsvSource({
+            "'', true",
+            "61, true",                   // a
+            "c3a9, true",                 // é
+            "e282ac, true",               // €
+            "f09f9880, true",             // U+1F600
+            "f48fbfbf, true",             // U+10FFFF, the last code point
+            "efbfbd, true",               // U+FFFD
+            "c080, false",                // overlong NUL
+            "e08080, false",              // overlong 3-byte
+            "f0808080, false",            // overlong 4-byte
+            "eda080, false",              // U+D800, a surrogate
+            "f4908080, false",            // above U+10FFFF
+            "f5808080, false",            // lead byte past F4
+            "80, false",                  // a stray continuation byte
+            "c3, false",                  // truncated
+            "e282, false",                // truncated
+            "c328, false",                // second byte is not a continuation
+            "e28228, false",              // third byte is not a continuation
+    })
+    void isWellFormed_namedCases(String hex, boolean expected) {
+        // Given
+        byte[] bytes = HexFormat.of().parseHex(hex);
+
+        // When
+        boolean result = Utf8Validation.isWellFormed(bytes, 0, bytes.length);
+
+        // Then
+        assertThat(result).isEqualTo(expected);
+    }
+
+    @ParameterizedTest
+    @ValueSource(longs = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10})
+    void isWellFormed_agreesWithTheJdkDecoder(long seed) {
+        // Given: short random byte runs biased toward lead and continuation bytes, so most are
+        // near-misses rather than plain ASCII; the JDK's strict decoder is the oracle
+        Random random = new Random(seed);
+        int[] palette = {0x41, 0x80, 0x8f, 0x90, 0x9f, 0xa0, 0xbf, 0xc2, 0xdf, 0xe0, 0xed, 0xef, 0xf0, 0xf4, 0xf5, 0xff};
+        for (int run = 0; run < 2_000; run++) {
+            byte[] bytes = new byte[random.nextInt(6)];
+            for (int i = 0; i < bytes.length; i++) {
+                bytes[i] = (byte) palette[random.nextInt(palette.length)];
+            }
+
+            // When
+            boolean result = Utf8Validation.isWellFormed(bytes, 0, bytes.length);
+
+            // Then
+            assertThat(result).as(HexFormat.of().formatHex(bytes)).isEqualTo(jdkAccepts(bytes));
+        }
+    }
+
+    @ParameterizedTest
+    @CsvSource({"0, 1, true", "1, 3, true", "0, 2, false", "2, 3, false"})
+    void isWellFormed_checksOnlyTheRange(int from, int to, boolean expected) {
+        // Given: 'a' then 'é' (c3 a9): a range that cuts the 2-byte character is malformed
+        byte[] bytes = {0x61, (byte) 0xc3, (byte) 0xa9};
+
+        // When
+        boolean result = Utf8Validation.isWellFormed(bytes, from, to);
+
+        // Then
+        assertThat(result).isEqualTo(expected);
+    }
+
+    private static boolean jdkAccepts(byte[] bytes) {
+        try {
+            StandardCharsets.UTF_8.newDecoder()
+                    .onMalformedInput(CodingErrorAction.REPORT)
+                    .onUnmappableCharacter(CodingErrorAction.REPORT)
+                    .decode(ByteBuffer.wrap(bytes));
+            return true;
+        } catch (CharacterCodingException _) {
+            return false;
+        }
+    }
+}

csv/src/main/java/io/github/dfa1/vortex/csv/ColumnChunk.java

@@ -0,0 +1,111 @@
+package io.github.dfa1.vortex.csv;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+
+/// One chunk of CSV fields kept column by column as raw bytes: each column is one growing byte
+/// buffer plus the end offset of each row's field, so parsing allocates nothing per field and each
+/// column's conversion reads one contiguous buffer.
+final class ColumnChunk {
+
+    private static final int INITIAL_COLUMN_BYTES = 1 << 12;
+
+    private byte[][] bytes = new byte[0][];
+    private int[][] ends = new int[0][];
+    private int[] used = new int[0];
+    private final int capacity;
+    private int rows;
+
+    /// @param capacity the most rows the chunk holds
+    ColumnChunk(int capacity) {
+        this.capacity = capacity;
+    }
+
+    /// Appends `src[from, to)` to field `column` of the row being read; a field may arrive in
+    /// several pieces (a quoted field around each escaped quote).
+    void append(int column, byte[] src, int from, int to) {
+        if (column >= used.length) {
+            addColumns(column + 1);
+        }
+        int length = to - from;
+        byte[] buffer = bytes[column];
+        int at = used[column];
+        if (buffer.length - at < length) {
+            buffer = Arrays.copyOf(buffer, Math.max(buffer.length * 2, at + length));
+            bytes[column] = buffer;
+        }
+        System.arraycopy(src, from, buffer, at, length);
+        used[column] = at + length;
+    }
+
+    /// Ends field `column` of the row being read.
+    void endField(int column) {
+        if (column >= used.length) {
+            addColumns(column + 1);
+        }
+        ends[column][rows] = used[column];
+    }
+
+    /// Ends the row being read.
+    void endRow() {
+        rows++;
+    }
+
+    /// Drops the fields of the row being read, which the input ended in the middle of.
+    void abortRow() {
+        for (int c = 0; c < used.length; c++) {
+            used[c] = rows == 0 ? 0 : ends[c][rows - 1];
+        }
+    }
+
+    /// The number of complete rows.
+    int rows() {
+        return rows;
+    }
+
+    /// Whether the chunk holds its capacity of rows.
+    boolean isFull() {
+        return rows == capacity;
+    }
+
+    /// The number of columns any row has had.
+    int columns() {
+        return used.length;
+    }
+
+    /// Column `column`'s bytes, or `null` for a column no row has.
+    byte[] bytes(int column) {
+        return column < bytes.length ? bytes[column] : null;
+    }
+
+    /// Where row `row`'s field starts in [#bytes(int)].
+    int start(int column, int row) {
+        return row == 0 ? 0 : ends[column][row - 1];
+    }
+
+    /// Where row `row`'s field ends in [#bytes(int)].
+    int end(int column, int row) {
+        return ends[column][row];
+    }
+
+    /// Row `row`'s field in column `column` as text, `""` for a column the row lacks; malformed
+    /// UTF-8 decodes to U+FFFD as the JDK decoder does.
+    String text(int column, int row) {
+        if (column >= bytes.length) {
+            return "";
+        }
+        int from = start(column, row);
+        return new String(bytes[column], from, end(column, row) - from, StandardCharsets.UTF_8);
+    }
+
+    private void addColumns(int count) {
+        int old = used.length;
+        bytes = Arrays.copyOf(bytes, count);
+        ends = Arrays.copyOf(ends, count);
+        used = Arrays.copyOf(used, count);
+        for (int c = old; c < count; c++) {
+            bytes[c] = new byte[INITIAL_COLUMN_BYTES];
+            ends[c] = new int[capacity];
+        }
+    }
+}