From abdb9f6e1c7d536d5136e24a14bcdb7a54ad5896 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sun, 25 Oct 2026 08:32:11 +0100 Subject: [PATCH 1/3] fix(inspector): print sizes past 1 GiB in GB, TB, PB or EB ByteSize stopped at MB, so a 2.3 GB file read as "2179.1 MB" in inspect, tui, import, export and explore output. Rather than add one more tier, derive the unit from the count's highest set bit (each unit is 10 more bits), with units up to EB: a long tops out just under 8 EiB, so no byte count can outgrow the list again. Units stay binary, like the KB and MB ones: that file now prints as 2.1 GB, and Long.MAX_VALUE as 8.0 EB. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01SwhK4W5cgj7y7s4ZPCif5x --- CHANGELOG.md | 1 + .../github/dfa1/vortex/inspect/ByteSize.java | 23 ++++++++++--------- .../dfa1/vortex/inspect/ByteSizeTest.java | 11 ++++++++- 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 635e45d86..5b2dce382 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - A non-null integer or float column gets a dict layout only when the compressor picks a dictionary for its first 8192 rows, as Rust's writer decides, so a column that bit-packs better is no longer dictionary-encoded. ([#599](https://github.com/dfa1/vortex-java/pull/599)) - The writer cuts string and nullable columns into the same chunks as vortex-jni 0.89, which measures its 8192-row blocks by the bytes their rows reference. ([#596](https://github.com/dfa1/vortex-java/pull/596)) - `filter` and `select --where` on the command line work on a nullable column, where they failed; a null row matches no comparison, as in SQL. ([#593](https://github.com/dfa1/vortex-java/pull/593)) +- The command line prints sizes past 1 GiB in GB, TB, PB or EB, where a 2.3 GB file read as `2179.1 MB`. ([#608](https://github.com/dfa1/vortex-java/pull/608)) ## [0.18.1] — 2026-10-22 diff --git a/inspector/src/main/java/io/github/dfa1/vortex/inspect/ByteSize.java b/inspector/src/main/java/io/github/dfa1/vortex/inspect/ByteSize.java index 513d0b04b..82bf02a3a 100644 --- a/inspector/src/main/java/io/github/dfa1/vortex/inspect/ByteSize.java +++ b/inspector/src/main/java/io/github/dfa1/vortex/inspect/ByteSize.java @@ -1,29 +1,30 @@ package io.github.dfa1.vortex.inspect; +import java.util.List; import java.util.Locale; /// Formats a byte count as a short human-readable string using binary (1024-based) units. public final class ByteSize { - private static final long KIB = 1024L; - private static final long MIB = KIB * KIB; + /// Every unit a `long` can reach: its maximum is just under 8 EiB, so no byte count runs past + /// the end of the list. + private static final List UNITS = List.of("KB", "MB", "GB", "TB", "PB", "EB"); private ByteSize() { } - /// Formats `bytes` as `B` below 1 KiB, `KB` below 1 MiB, otherwise `MB`, with one decimal - /// place for the KB/MB forms. Uses [Locale#ROOT] so the decimal separator is stable regardless - /// of the default locale. + /// Formats `bytes` as `B` below 1 KiB, else in the largest unit (`KB` up to `EB`) it is at least + /// one of, with one decimal place. Uses [Locale#ROOT] so the decimal separator is stable + /// regardless of the default locale. /// /// @param bytes the byte count - /// @return a human-readable size such as `512 B`, `1.5 KB`, or `2.0 MB` + /// @return a human-readable size such as `512 B`, `1.5 KB`, `2.0 MB` or `2.1 GB` public static String format(long bytes) { - if (bytes < KIB) { + if (bytes < 1024) { return bytes + " B"; } - if (bytes < MIB) { - return String.format(Locale.ROOT, "%.1f KB", bytes / (double) KIB); - } - return String.format(Locale.ROOT, "%.1f MB", bytes / (double) MIB); + // Each unit is 10 more bits: the highest set bit picks the unit, with no table of bounds. + int unit = (Long.SIZE - 1 - Long.numberOfLeadingZeros(bytes)) / 10; + return String.format(Locale.ROOT, "%.1f %s", bytes / (double) (1L << (10 * unit)), UNITS.get(unit - 1)); } } diff --git a/inspector/src/test/java/io/github/dfa1/vortex/inspect/ByteSizeTest.java b/inspector/src/test/java/io/github/dfa1/vortex/inspect/ByteSizeTest.java index 12b79bc48..25f7c5bfd 100644 --- a/inspector/src/test/java/io/github/dfa1/vortex/inspect/ByteSizeTest.java +++ b/inspector/src/test/java/io/github/dfa1/vortex/inspect/ByteSizeTest.java @@ -19,6 +19,15 @@ class ByteSizeTest { "1048576, 1.0 MB", // first MB "1572864, 1.5 MB", "5242880, 5.0 MB", + "1073741823, 1024.0 MB", // last value below the GB boundary + "1073741824, 1.0 GB", // first GB + "2284919630, 2.1 GB", // a 100M-row file read as 2179.1 MB before GB existed + "1099511627775, 1024.0 GB", // last value below the TB boundary + "1099511627776, 1.0 TB", + "1125899906842624, 1.0 PB", + "1152921504606846975, 1024.0 PB", // last value below the EB boundary + "1152921504606846976, 1.0 EB", + "9223372036854775807, 8.0 EB", // Long.MAX_VALUE: the largest count there is, still in a unit }) void format_rendersBinaryUnits(long bytes, String expected) { // Given a byte count @@ -26,7 +35,7 @@ void format_rendersBinaryUnits(long bytes, String expected) { // When String result = ByteSize.format(bytes); - // Then it picks B / KB / MB with one decimal for the scaled forms + // Then it picks B, or KB up to EB with one decimal assertThat(result).isEqualTo(expected); } } From 337a5c32cd2f777c74a64f511146ba98f2546270 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sun, 25 Oct 2026 08:32:11 +0100 Subject: [PATCH 2/3] feat(reader): expose ScanIterator.chunkStats Each chunk's embedded array stats carry what a vortex.zoned zone map does not: exact string min/max (the zone map only bounds them) and, where the writer records it, a sum. Tooling outside the reader (the explore statistics table) needs them, so the method becomes public. A dictionary chunk now reports only its values pool's min/max. The pool's sum and null count describe the dictionary, not the chunk's rows; no caller read them before, but a public method must not hand them out as the chunk's. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01SwhK4W5cgj7y7s4ZPCif5x --- CHANGELOG.md | 1 + .../io/github/dfa1/vortex/reader/ScanIterator.java | 14 +++++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b2dce382..d66b8e9fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - A CSV import schema can declare any integer type from `I8` to `U64`, range-checked per value, and `F32`; an unsupported type fails in `withSchema` instead of at the first chunk. ([#593](https://github.com/dfa1/vortex-java/pull/593)) +- `ScanIterator.chunkStats` is public: each chunk's own array stats, which carry a string column's exact min/max where the zone map only bounds it. ([#608](https://github.com/dfa1/vortex-java/pull/608)) ### Changed - **Breaking:** `RowFilter` is renamed `ScanFilter`, matching `ScanOptions`. ([#598](https://github.com/dfa1/vortex-java/pull/598)) diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java index a3fc3a196..2df51c79e 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java @@ -590,11 +590,15 @@ public List columnZoneStats(ColumnName name) { /// Each scan window's exact embedded array stats for `column`, read from the covering chunk's /// data segment rather than the zone map: what [VortexReader#columnStats()] falls back on when /// the zone map only bounds a string column's min/max (Rust's `bounded_max`/`bounded_min`). - /// A dictionary chunk reports its values pool's stats, which bound exactly the values it holds. + /// A dictionary chunk reports only the min/max of its values pool, which bound exactly the + /// values it holds; the pool's other stats (its sum, its null count) describe the dictionary, + /// not the chunk's rows, so they are left absent. + /// + /// Like [#columnZoneStats(ColumnName)], filter pruning and `ScanOptions.limit()` are not applied. /// /// @param name the column name /// @return per-window stats, positionally aligned with [#chunkRowCounts()] - List chunkStats(ColumnName name) { + public List chunkStats(ColumnName name) { if (chunks == null) { initialize(); } @@ -603,7 +607,11 @@ List chunkStats(ColumnName name) { Layout layout = spec.layoutFor(name); if (layout != null && layout.isDict() && !layout.children().isEmpty()) { // child[0] = values, child[1] = codes — the order DictLayoutDecoder decodes. - layout = layout.children().getFirst(); + Layout values = layout.children().getFirst(); + ArrayStats pool = values.isFlat() ? readFlatStats(values) : ArrayStats.empty(); + out.add(new ArrayStats(pool.min(), pool.minExact(), pool.max(), pool.maxExact(), + null, null, null, null, null, null, null)); + continue; } out.add(layout == null || !layout.isFlat() ? ArrayStats.empty() : readFlatStats(layout)); } From b456d5ec33372446d99014163a98c20a693e108d Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sun, 25 Oct 2026 08:32:11 +0100 Subject: [PATCH 3/3] feat(cli): explore, a data-first view of a file for business users `inspect` answers how a file is stored; `explore` answers what is in it. explore [--serve] serves a page on 127.0.0.1, under a random path (as Jupyter does), and opens it in the browser. It leads with one row per column, folded from the statistics the file already records, so nothing is decoded and a 2.3 GB, 100M-row file opens in well under a second: null %, exact min/max, mean where the file records a sum, and a sparkline of each zone's min-max band over the rows (sorted, clustered or random at a glance). Analyze on a column reads its values once for the count, null %, total, mean, distinct count, string byte lengths, and its spread: every value with its share up to 30 distinct values, else a 30-bin histogram for a numeric, date or timestamp column (bins from the recorded min/max, or a min/max pass when the file has none), else the 10 most frequent values, and nothing past 10,000 distinct. Totals reuse the fused aggregate behind Calcite's SUM push-down and value counts its GROUP BY fold, both working off dictionary codes. Columns are analyzed in parallel. A reader's arena is confined to its thread, so the file is opened once per core, each on its own IoWorker, and the server answers on virtual threads. Analyze all on that 100M-row file takes 2.2 s on 10 cores against 9-11 s one column at a time. explore --html out.html analyzes every column up front and writes the page as one self-contained file to publish: 1.5 s and 111 KB for the January 2024 NYC taxi file (3M rows, 19 columns). The data page lives in ExplorePage, apart from the storage-first HtmlReport, which is unchanged (inspect --html keeps its output). vortex-jni cross-checks: a Rust-written file's profile, total and histogram match the data written, and a dictionary column's value counts are exact. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01SwhK4W5cgj7y7s4ZPCif5x --- CHANGELOG.md | 1 + .../dfa1/vortex/cli/ExploreCommand.java | 274 +++++++++++++ .../io/github/dfa1/vortex/cli/VortexCli.java | 4 + .../dfa1/vortex/cli/ExploreCommandTest.java | 227 +++++++++++ docs/how-to.md | 45 ++ docs/reference.md | 1 + .../vortex/inspect/ColumnDistribution.java | 383 ++++++++++++++++++ .../dfa1/vortex/inspect/ColumnProfile.java | 153 +++++++ .../dfa1/vortex/inspect/ExplorePage.java | 316 +++++++++++++++ .../dfa1/vortex/inspect/HtmlReport.java | 41 +- .../inspect/ColumnDistributionTest.java | 97 +++++ .../vortex/inspect/ColumnProfileTest.java | 121 ++++++ .../dfa1/vortex/inspect/ExplorePageTest.java | 240 +++++++++++ .../dfa1/vortex/inspect/HtmlReportTest.java | 12 + .../VortexInspectorIntegrationTest.java | 125 ++++++ 15 files changed, 2029 insertions(+), 11 deletions(-) create mode 100644 cli/src/main/java/io/github/dfa1/vortex/cli/ExploreCommand.java create mode 100644 cli/src/test/java/io/github/dfa1/vortex/cli/ExploreCommandTest.java create mode 100644 inspector/src/main/java/io/github/dfa1/vortex/inspect/ColumnDistribution.java create mode 100644 inspector/src/main/java/io/github/dfa1/vortex/inspect/ColumnProfile.java create mode 100644 inspector/src/main/java/io/github/dfa1/vortex/inspect/ExplorePage.java create mode 100644 inspector/src/test/java/io/github/dfa1/vortex/inspect/ColumnDistributionTest.java create mode 100644 inspector/src/test/java/io/github/dfa1/vortex/inspect/ColumnProfileTest.java create mode 100644 inspector/src/test/java/io/github/dfa1/vortex/inspect/ExplorePageTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index d66b8e9fe..e5f44378f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - A CSV import schema can declare any integer type from `I8` to `U64`, range-checked per value, and `F32`; an unsupported type fails in `withSchema` instead of at the first chunk. ([#593](https://github.com/dfa1/vortex-java/pull/593)) - `ScanIterator.chunkStats` is public: each chunk's own array stats, which carry a string column's exact min/max where the zone map only bounds it. ([#608](https://github.com/dfa1/vortex-java/pull/608)) +- `vortex explore` shows what is in a file: null %, min/max and a zone sparkline per column at once, then each column's total, mean, distinct count and value counts or histogram, served to the browser or written to one HTML file with `--html`. ([#608](https://github.com/dfa1/vortex-java/pull/608)) ### Changed - **Breaking:** `RowFilter` is renamed `ScanFilter`, matching `ScanOptions`. ([#598](https://github.com/dfa1/vortex-java/pull/598)) diff --git a/cli/src/main/java/io/github/dfa1/vortex/cli/ExploreCommand.java b/cli/src/main/java/io/github/dfa1/vortex/cli/ExploreCommand.java new file mode 100644 index 000000000..adf1217f7 --- /dev/null +++ b/cli/src/main/java/io/github/dfa1/vortex/cli/ExploreCommand.java @@ -0,0 +1,274 @@ +package io.github.dfa1.vortex.cli; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import io.github.dfa1.vortex.cli.tui.IoWorker; +import io.github.dfa1.vortex.inspect.ByteSize; +import io.github.dfa1.vortex.inspect.ColumnDistribution; +import io.github.dfa1.vortex.inspect.ColumnProfile; +import io.github.dfa1.vortex.inspect.ExplorePage; +import io.github.dfa1.vortex.reader.VortexHandle; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.HexFormat; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.IntFunction; +import java.util.function.Supplier; + +/// `explore [--serve | --html ] `: the data-first view of a file, per-column +/// statistics and distributions (`inspect --html` is the storage-first one). +/// +/// - `--serve` (the default) serves the page on a local port, opens it in the browser and computes +/// a column's distribution when the page asks for it. Runs until interrupted. +/// - `--html` analyzes every column up front and writes the page, distributions included, to one +/// self-contained file to publish. +/// +/// The file is open once per core, each on its own [IoWorker]: a reader's arena is confined to the +/// thread that opened it, so columns are analyzed in parallel by separate readers rather than by +/// threads sharing one. +@SuppressWarnings("java:S106") // CLI command: stdout is the intended output channel +final class ExploreCommand { + + private static final String USAGE = "usage: explore [--serve | --html ] "; + + private ExploreCommand() { + } + + static int run(String[] args) { + String target; + Path html = null; + if (args.length == 2 && !args[1].startsWith("--")) { + target = args[1]; + } else if (args.length == 3 && "--serve".equals(args[1])) { + target = args[2]; + } else if (args.length == 4 && "--html".equals(args[1])) { + html = Path.of(args[2]); + target = args[3]; + } else { + System.err.println(USAGE); + return ExitStatus.USAGE_ERROR; + } + IoWorker worker = new IoWorker("explore-io-0"); + try { + Optional opened = CliHandles.openOnWorker(worker, target); + if (opened.isEmpty()) { + worker.close(); + return ExitStatus.FILE_NOT_FOUND; + } + VortexHandle handle = opened.get(); + List profiles = onWorker(worker, () -> ColumnProfile.of(handle)); + String title = target.substring(target.lastIndexOf('/') + 1); + BlockingQueue readers = readers(new Reader(worker, handle), target, profiles.size()); + IntFunction analyze = index -> { + Reader reader = take(readers); + try { + return onWorker(reader.worker(), () -> ExplorePage.distribution( + profiles.get(index), ColumnDistribution.of(reader.handle(), profiles.get(index)))); + } finally { + readers.add(reader); + } + }; + if (html != null) { + List analyses = analyzeAll(profiles.size(), analyze); + Files.writeString(html, ExplorePage.render(title, handle.fileSize(), + handle.layout().rowCount(), profiles, analyses)); + readers.forEach(reader -> CliHandles.closeOnWorker(reader.worker(), reader.handle())); + readers.forEach(reader -> reader.worker().close()); + System.out.printf("written: %s (%d columns analyzed, %s)%n", html, profiles.size(), + ByteSize.format(Files.size(html))); + return ExitStatus.OK; + } + String page = ExplorePage.render(title, handle.fileSize(), handle.layout().rowCount(), profiles); + byte[] token = new byte[16]; + new SecureRandom().nextBytes(token); + String path = "/" + HexFormat.of().formatHex(token) + "/"; + HttpServer server = serve(page, path, profiles.size(), analyze); + URI url = URI.create("http://127.0.0.1:" + server.getAddress().getPort() + path); + System.out.println("serving " + url + " - press Ctrl-C to stop"); + openBrowser(url); + Thread.currentThread().join(); + return ExitStatus.OK; + } catch (IOException | RuntimeException e) { + System.err.println("error: " + CliHandles.describe(e)); + worker.close(); + return ExitStatus.ERROR; + } catch (InterruptedException _) { + Thread.currentThread().interrupt(); + return ExitStatus.OK; + } + } + + /// One open file on the thread that owns it. + private record Reader(IoWorker worker, VortexHandle handle) { + } + + /// One reader per core, up to one per column, so as many columns are analyzed at once. Each + /// opens the file on its own worker thread: a reader cannot be shared across threads, but + /// mapping a file again is cheap and the operating system shares its pages between the mappings. + private static BlockingQueue readers(Reader first, String target, int columns) + throws IOException, InterruptedException { + int count = Math.clamp(Runtime.getRuntime().availableProcessors(), 1, Math.max(1, columns)); + BlockingQueue readers = new ArrayBlockingQueue<>(count); + readers.add(first); + for (int i = 1; i < count; i++) { + IoWorker worker = new IoWorker("explore-io-" + i); + VortexHandle handle = CliHandles.openOnWorker(worker, target) + .orElseThrow(() -> new IOException("file vanished: " + target)); + readers.add(new Reader(worker, handle)); + } + return readers; + } + + /// Every column's analysis, as many at once as there are readers, in column order. + private static List analyzeAll(int columns, IntFunction analyze) throws InterruptedException { + try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + List> futures = new ArrayList<>(columns); + for (int i = 0; i < columns; i++) { + int index = i; + futures.add(executor.submit(() -> analyze.apply(index))); + } + List analyses = new ArrayList<>(columns); + for (Future future : futures) { + try { + analyses.add(future.get()); + } catch (ExecutionException e) { + // Rethrow the column's own failure (a malformed segment, say), not the wrapper. + if (e.getCause() instanceof RuntimeException cause) { + throw cause; + } + throw new IllegalStateException(e.getCause()); + } + } + return analyses; + } + } + + private static Reader take(BlockingQueue readers) { + try { + return readers.take(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("interrupted", e); + } + } + + /// Starts a server on a free port of `127.0.0.1` answering two paths: `path` with `page`, and + /// `path` + `column/` with `analyze`'s fragment for that column. Every other path is a + /// 404, and a failed analysis is a 500 carrying the error as plain text. + /// + /// The page shows file contents, so it is reachable only from this machine and only under + /// `path`, a random one in [#run(String[])], as Jupyter does: another local user, or a web page + /// probing localhost ports, cannot guess it. + /// + /// @param page the HTML document to serve + /// @param path the page's path, e.g. `/3f2a.../` + /// @param columns how many columns `analyze` accepts, indexed from `0` + /// @param analyze renders one column's distribution as an HTML fragment + /// @return the started server + /// @throws IOException if the server cannot bind + static HttpServer serve(String page, String path, int columns, IntFunction analyze) throws IOException { + byte[] body = page.getBytes(StandardCharsets.UTF_8); + String columnPrefix = path + "column/"; + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/", exchange -> { + try (exchange) { + String requested = exchange.getRequestURI().getPath(); + if (path.equals(requested)) { + respond(exchange, 200, "text/html", body); + return; + } + int index = -1; + if (requested.startsWith(columnPrefix)) { + index = columnIndex(requested.substring(columnPrefix.length())); + } + if (index < 0 || index >= columns) { + exchange.sendResponseHeaders(404, -1); + return; + } + try { + respond(exchange, 200, "text/html", analyze.apply(index).getBytes(StandardCharsets.UTF_8)); + } catch (RuntimeException e) { + respond(exchange, 500, "text/plain", CliHandles.describe(e).getBytes(StandardCharsets.UTF_8)); + } + } + }); + // A thread per request: `analyze` blocks until a reader is free, and the default executor's + // single thread would serialize the columns the readers could analyze at once. + server.setExecutor(Executors.newVirtualThreadPerTaskExecutor()); + server.start(); + return server; + } + + private static int columnIndex(String digits) { + if (digits.isEmpty() || digits.length() > 9 || !digits.chars().allMatch(Character::isDigit)) { + return -1; + } + return Integer.parseInt(digits); + } + + private static void respond(HttpExchange exchange, int status, String type, byte[] body) throws IOException { + exchange.getResponseHeaders().set("Content-Type", type + "; charset=utf-8"); + exchange.sendResponseHeaders(status, body.length); + try (OutputStream out = exchange.getResponseBody()) { + out.write(body); + } + } + + /// Runs `task` on the thread that owns the file and returns its result, rethrowing its failure. + private static T onWorker(IoWorker worker, Supplier task) { + AtomicReference result = new AtomicReference<>(); + AtomicReference failure = new AtomicReference<>(); + try { + worker.runAndAwait(() -> { + try { + result.set(task.get()); + } catch (RuntimeException e) { + failure.set(e); + } + }); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("interrupted", e); + } + if (failure.get() != null) { + throw failure.get(); + } + return result.get(); + } + + /// Best effort: the URL is printed either way, so a machine with no desktop (a server over SSH) + /// still gets a link to forward. + private static void openBrowser(URI url) { + String os = System.getProperty("os.name", "").toLowerCase(Locale.ROOT); + String[] command; + if (os.contains("mac")) { + command = new String[] {"open", url.toString()}; + } else if (os.contains("win")) { + command = new String[] {"rundll32", "url.dll,FileProtocolHandler", url.toString()}; + } else { + command = new String[] {"xdg-open", url.toString()}; + } + try { + new ProcessBuilder(command).redirectErrorStream(true).redirectOutput(ProcessBuilder.Redirect.DISCARD).start(); + } catch (IOException _) { + // no browser launcher: the printed URL is the fallback + } + } +} diff --git a/cli/src/main/java/io/github/dfa1/vortex/cli/VortexCli.java b/cli/src/main/java/io/github/dfa1/vortex/cli/VortexCli.java index 00fdd4115..0d1bc867d 100644 --- a/cli/src/main/java/io/github/dfa1/vortex/cli/VortexCli.java +++ b/cli/src/main/java/io/github/dfa1/vortex/cli/VortexCli.java @@ -58,6 +58,7 @@ private static int dispatch(String[] args) { } return switch (args[0]) { case "inspect" -> InspectCommand.run(args); + case "explore" -> ExploreCommand.run(args); case "tui" -> TuiCommand.run(args); case "view" -> ViewCommand.run(args); case "export" -> ExportCommand.run(args); @@ -78,6 +79,9 @@ private static int dispatch(String[] args) { static void printUsage(PrintStream out) { out.println("Usage: java -jar vortex-cli--all.jar [args] [--timing] [--allow-unknown]"); out.println(" inspect [--html] print file structure; --html writes an HTML report"); + out.println(" explore [--serve | --html ] "); + out.println(" per-column statistics: serve them to the browser (default),"); + out.println(" or write one HTML file with every column analyzed"); out.println(" tui open interactive inspector; url is http(s)://"); out.println(" view open scrollable data grid; url is http(s)://"); out.println(" export [out.csv|out.parquet|-] write CSV or Parquet; default is .csv, `-` for stdout"); diff --git a/cli/src/test/java/io/github/dfa1/vortex/cli/ExploreCommandTest.java b/cli/src/test/java/io/github/dfa1/vortex/cli/ExploreCommandTest.java new file mode 100644 index 000000000..e6cfb2979 --- /dev/null +++ b/cli/src/test/java/io/github/dfa1/vortex/cli/ExploreCommandTest.java @@ -0,0 +1,227 @@ +package io.github.dfa1.vortex.cli; + +import com.sun.net.httpserver.HttpServer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static io.github.dfa1.vortex.cli.CliTestSupport.capture; +import static io.github.dfa1.vortex.cli.CliTestSupport.writeTypedVortex; +import static org.assertj.core.api.Assertions.assertThat; + +class ExploreCommandTest { + + @Test + void wrongArity_returnsUsageError() { + // Given / When + CliTestSupport.Captured result = capture(() -> ExploreCommand.run(new String[]{"explore"})); + + // Then + assertThat(result.status()).isEqualTo(ExitStatus.USAGE_ERROR); + assertThat(result.stderr()).contains("usage: explore"); + } + + @Test + void missingFile_returnsFileNotFound(@TempDir Path tmp) { + // Given + Path missing = tmp.resolve("nope.vortex"); + + // When + CliTestSupport.Captured result = capture(() -> ExploreCommand.run(new String[]{"explore", missing.toString()})); + + // Then — fails before any server starts + assertThat(result.status()).isEqualTo(ExitStatus.FILE_NOT_FOUND); + assertThat(result.stderr()).contains("file not found"); + } + + @ParameterizedTest + @ValueSource(strings = {"--html", "--bogus"}) + void flagWithoutItsArguments_returnsUsageError(String flag) { + // Given / When — `explore --html data.vortex` would otherwise write the page over data.vortex + CliTestSupport.Captured result = capture(() -> ExploreCommand.run(new String[]{"explore", flag, "data.vortex"})); + + // Then + assertThat(result.status()).isEqualTo(ExitStatus.USAGE_ERROR); + assertThat(result.stderr()).contains("usage: explore"); + } + + @Test + void html_writesEveryColumnAlreadyAnalyzed(@TempDir Path tmp) throws IOException { + // Given — a 3-row file of id (I64), qty (I32), price (F64), name (Utf8) + Path file = writeTypedVortex(tmp, "typed.vortex"); + Path out = tmp.resolve("report.html"); + + // When + CliTestSupport.Captured result = capture(() -> + ExploreCommand.run(new String[]{"explore", "--html", out.toString(), file.toString()})); + + // Then — one file to publish: no server to ask, so no Analyze button, every column's + // distribution in place, each behind a working Hide toggle + assertThat(result.status()).isEqualTo(ExitStatus.OK); + assertThat(result.stdout()).contains("written: ").contains("4 columns analyzed"); + String page = Files.readString(out); + assertThat(page) + .startsWith("") + .doesNotContain(">Analyze") + .doesNotContain("id=\"analyze-all\"") + .contains("total 6") // id: 1 + 2 + 3 + .contains("total 60") // qty: 10 + 20 + 30 + .contains("carol"); // name: each of 3 values counted + assertThat(page.split("data-state=\"done\">Hide", -1)).hasSize(5); + } + + @Test + void html_missingFile_returnsFileNotFoundAndWritesNothing(@TempDir Path tmp) { + // Given + Path out = tmp.resolve("report.html"); + + // When + CliTestSupport.Captured result = capture(() -> ExploreCommand.run( + new String[]{"explore", "--html", out.toString(), tmp.resolve("nope.vortex").toString()})); + + // Then + assertThat(result.status()).isEqualTo(ExitStatus.FILE_NOT_FOUND); + assertThat(out).doesNotExist(); + } + + @Test + void serve_answersOnlyTheSecretPath() throws IOException, InterruptedException { + // Given — the page shows file contents: a local process or a web page probing localhost + // ports must not reach it without the random path + HttpServer sut = ExploreCommand.serve("

hello

", "/s3cret/", 2, index -> "column " + index); + try (HttpClient client = HttpClient.newHttpClient()) { + // When + HttpResponse result = get(client, sut, "/s3cret/"); + HttpResponse root = get(client, sut, "/"); + HttpResponse guess = get(client, sut, "/s3cret/x"); + HttpResponse columnWithoutSecret = get(client, sut, "/column/0"); + + // Then + assertThat(result.statusCode()).isEqualTo(200); + assertThat(result.body()).contains("hello"); + assertThat(result.headers().firstValue("Content-Type")).hasValue("text/html; charset=utf-8"); + assertThat(root.statusCode()).isEqualTo(404); + assertThat(guess.statusCode()).isEqualTo(404); + assertThat(columnWithoutSecret.statusCode()).isEqualTo(404); + } finally { + sut.stop(0); + } + } + + @Test + void serve_columnPathReturnsThatColumnsAnalysis() throws IOException, InterruptedException { + // Given + HttpServer sut = ExploreCommand.serve("", "/s/", 2, index -> "

column " + index + "

"); + try (HttpClient client = HttpClient.newHttpClient()) { + // When + HttpResponse result = get(client, sut, "/s/column/1"); + + // Then + assertThat(result.statusCode()).isEqualTo(200); + assertThat(result.body()).isEqualTo("

column 1

"); + } finally { + sut.stop(0); + } + } + + @ParameterizedTest + @ValueSource(strings = {"/s/column/2", "/s/column/-1", "/s/column/", "/s/column/x", "/s/column/99999999999"}) + void serve_columnOutOfRangeOrMalformed_is404(String path) throws IOException, InterruptedException { + // Given — the index comes from the request: anything but a column the file has must not + // reach the analysis (an out-of-range index would be an IndexOutOfBoundsException there) + HttpServer sut = ExploreCommand.serve("", "/s/", 2, index -> "column " + index); + try (HttpClient client = HttpClient.newHttpClient()) { + // When + HttpResponse result = get(client, sut, path); + + // Then + assertThat(result.statusCode()).isEqualTo(404); + } finally { + sut.stop(0); + } + } + + @Test + void serve_analyzesColumnsConcurrently() throws IOException, InterruptedException { + // Given — each analysis waits until both are running: on the server's default single + // thread the first would wait forever for the second, and "Analyze all" would run one + // column at a time however many readers there are + CountDownLatch bothRunning = new CountDownLatch(2); + HttpServer sut = ExploreCommand.serve("", "/s/", 2, index -> { + bothRunning.countDown(); + try { + return bothRunning.await(10, TimeUnit.SECONDS) ? "together" : "alone"; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return "interrupted"; + } + }); + try (HttpClient client = HttpClient.newHttpClient()) { + // When + CompletableFuture> first = client.sendAsync(request(sut, "/s/column/0"), + HttpResponse.BodyHandlers.ofString()); + CompletableFuture> second = client.sendAsync(request(sut, "/s/column/1"), + HttpResponse.BodyHandlers.ofString()); + List result = List.of(first.join().body(), second.join().body()); + + // Then + assertThat(result).containsExactly("together", "together"); + } finally { + sut.stop(0); + } + } + + @Test + void serve_failedAnalysisIs500WithTheReason() throws IOException, InterruptedException { + // Given — a malformed segment surfaces when the column is decoded, long after the page loaded + HttpServer sut = ExploreCommand.serve("", "/s/", 1, index -> { + throw new IllegalStateException("bad segment"); + }); + try (HttpClient client = HttpClient.newHttpClient()) { + // When + HttpResponse result = get(client, sut, "/s/column/0"); + + // Then — the server keeps serving; the page shows the reason in place of the chart + assertThat(result.statusCode()).isEqualTo(500); + assertThat(result.body()).contains("bad segment"); + assertThat(result.headers().firstValue("Content-Type")).hasValue("text/plain; charset=utf-8"); + } finally { + sut.stop(0); + } + } + + @Test + void serve_bindsLoopbackOnly() throws IOException { + // Given / When + HttpServer result = ExploreCommand.serve("", "/x/", 0, index -> ""); + + // Then — never reachable from another machine on the network + try { + assertThat(result.getAddress().getAddress().isLoopbackAddress()).isTrue(); + } finally { + result.stop(0); + } + } + + private static HttpResponse get(HttpClient client, HttpServer server, String path) + throws IOException, InterruptedException { + return client.send(request(server, path), HttpResponse.BodyHandlers.ofString()); + } + + private static HttpRequest request(HttpServer server, String path) { + return HttpRequest.newBuilder(URI.create("http://127.0.0.1:" + server.getAddress().getPort() + path)).build(); + } +} diff --git a/docs/how-to.md b/docs/how-to.md index 54a64bb53..d650e056c 100644 --- a/docs/how-to.md +++ b/docs/how-to.md @@ -66,6 +66,51 @@ java -jar cli/target/vortex-cli-*-all.jar stats data.vortex --- +## Explore in the browser + +`explore` is the data-first view, for asking what is in a file rather than how it is stored. By +default (`--serve`) it serves a page on a local port and opens it in your browser, and runs until +you press Ctrl-C. + +```bash +java -jar cli/target/vortex-cli-*-all.jar explore data.vortex +# serving http://127.0.0.1:49605/b3a1f2.../ - press Ctrl-C to stop +``` + +To publish or send the page instead, `--html` analyzes every column up front and writes one +self-contained file, nothing to fetch and no server needed: + +```bash +java -jar cli/target/vortex-cli-*-all.jar explore --html report.html data.vortex +# written: report.html (19 columns analyzed, 111.4 KB) +``` + +The page lists every column with its share of nulls, its min and max, and a sparkline of each +zone's min-max band from the first row to the last. The sparkline shows whether a column is sorted +(a rising band), clustered or random, which is also how well a filter on it prunes zones. Everything +comes from the statistics the file already records and no data is decoded, so a 2.3 GB, +100-million-row file opens in well under a second. A mean appears only for files that record a sum: Rust's zone map and array +stats carry none. + +**Analyze** on a column (or **Analyze all**) reads its values once and adds, under its row, the +non-null count, the total and mean of a numeric column, the number of distinct values, a string +column's byte lengths, and how the values are spread: + +- a column with at most 30 distinct values (a status, a country, a flag) shows each value with its + count and share +- a numeric, date or timestamp column with more shows a 30-bin histogram between its min and max +- a string column with more shows its 10 most frequent values, and past 10,000 distinct values none + +This one decodes the column: on a 2.3 GB, 100-million-row file each column takes one to two seconds. +Columns are analyzed in parallel, one per core, so **Analyze all** on that file's nine columns takes +about as long as the slowest one: 2.2 s on 10 cores against 9-11 s one after another. + +The page is reachable only from this machine (`127.0.0.1`) and only under the random path printed +at start-up. The file can also be an `http(s)://` URL, as for `inspect`; analyzing a column then +downloads it. + +--- + ## Inspect as a web page `inspect --html` writes a single self-contained HTML page to stdout — no scripts, no CDN, diff --git a/docs/reference.md b/docs/reference.md index 30ac2caba..afb3849e4 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -746,6 +746,7 @@ java -jar cli/target/vortex-cli-*-all.jar [args] | Subcommand | Syntax | Description | |------------|------------------------------------------------|--------------------------------------------------| | `inspect` | `inspect [--html] ` | Layout tree, encodings, row counts, buffer sizes; `--html` emits a self-contained HTML report on stdout | +| `explore` | `explore [--serve \| --html ] ` | Per-column nulls, min/max and zone sparklines from the recorded statistics. `--serve` (default) serves them on `127.0.0.1` under a random path and opens the page in the browser, where **Analyze** reads a column for its total, mean, distinct count and value counts or histogram; `--html` writes one self-contained page with every column already analyzed; see [Explore in the browser](how-to.md#explore-in-the-browser) | | `tui` | `tui ` | Interactive layout-tree browser (lazy stats + data) | | `view` | `view ` | Interactive spreadsheet-grid browser over the row data | | `schema` | `schema ` | Column names and types | diff --git a/inspector/src/main/java/io/github/dfa1/vortex/inspect/ColumnDistribution.java b/inspector/src/main/java/io/github/dfa1/vortex/inspect/ColumnDistribution.java new file mode 100644 index 000000000..79c95d839 --- /dev/null +++ b/inspector/src/main/java/io/github/dfa1/vortex/inspect/ColumnDistribution.java @@ -0,0 +1,383 @@ +package io.github.dfa1.vortex.inspect; + +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.reader.Chunk; +import io.github.dfa1.vortex.reader.ScanFilter; +import io.github.dfa1.vortex.reader.ScanIterator; +import io.github.dfa1.vortex.reader.ScanOptions; +import io.github.dfa1.vortex.reader.VortexHandle; +import io.github.dfa1.vortex.reader.array.Array; +import io.github.dfa1.vortex.reader.array.BoolArray; +import io.github.dfa1.vortex.reader.array.ByteArray; +import io.github.dfa1.vortex.reader.array.DoubleArray; +import io.github.dfa1.vortex.reader.array.FloatArray; +import io.github.dfa1.vortex.reader.array.IntArray; +import io.github.dfa1.vortex.reader.array.LongArray; +import io.github.dfa1.vortex.reader.array.MaskedArray; +import io.github.dfa1.vortex.reader.array.NullArray; +import io.github.dfa1.vortex.reader.array.ShortArray; +import io.github.dfa1.vortex.reader.array.VarBinArray; +import io.github.dfa1.vortex.reader.compute.Compute; +import io.github.dfa1.vortex.reader.compute.FilteredAggregate; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.LongToDoubleFunction; + +/// What one full pass over a column's values tells that its recorded statistics cannot: its total, +/// how its values are spread, and for a string column how long they are. Unlike [ColumnProfile] it +/// decodes the column, so it is computed on demand, one column at a time. +/// +/// The spread is a value count when the column has few distinct values (a status, a country, a +/// flag), else a histogram for a numeric column, else the most frequent values. A column with more +/// than [#MAX_DISTINCT] distinct values is not counted at all: past that, a count says nothing a +/// histogram does not, and holding it costs memory per value. +/// +/// @param rows rows scanned +/// @param nulls null values among them +/// @param sum the total of a numeric column (`Long` for integers, `Double` for floats), or `null` +/// for a non-numeric column or an integer total past the `long` range +/// @param distinct distinct non-null values, or `null` when there are more than [#MAX_DISTINCT] +/// @param bars the spread: value counts or histogram bins, in display order +/// @param histogram `true` when `bars` are numeric bins rather than individual values +/// @param lengths byte lengths of a string column's non-null values, or `null` for other columns +public record ColumnDistribution( + long rows, + long nulls, + Number sum, + Long distinct, + List bars, + boolean histogram, + Lengths lengths) { + + /// Distinct values counted before a column counts as high-cardinality. + public static final int MAX_DISTINCT = 10_000; + + /// A column with at most this many distinct values shows every one of them; above it a numeric + /// column shows a histogram and any other the [#TOP_VALUES] most frequent. + static final int MAX_CATEGORIES = 30; + + /// Most frequent values shown for a column with too many distinct values to list. + static final int TOP_VALUES = 10; + + /// Bins in a histogram. + static final int BINS = 30; + + /// Matches every row: the aggregate kernels take a filter, and an empty `AND` holds for all. + private static final ScanFilter EVERY_ROW = new ScanFilter.And(List.of()); + + /// One bar of the spread. + /// + /// @param value the value counted, or a histogram bin's lower bound (a `Double`) + /// @param upper a histogram bin's upper bound (a `Double`), or `null` for a value count + /// @param count rows holding the value, or falling in the bin + public record Bar(Object value, Object upper, long count) { + } + + /// Byte lengths of a string column's values. + /// + /// @param min shortest value + /// @param max longest value + /// @param total bytes over all values, for the average + public record Lengths(long min, long max, long total) { + } + + /// The mean of the non-null values. + /// + /// @return `sum / (rows - nulls)`, or `null` without a sum or with no non-null value + public Double mean() { + long present = rows - nulls; + return sum == null || present <= 0 ? null : sum.doubleValue() / present; + } + + /// Scans `profile`'s column once. A histogram's bins span the profile's min and max; a numeric + /// column whose file records no exact min/max (an older write of a timestamp, say) is scanned + /// once more first, to find them. + /// + /// @param handle an open file; must be used from the thread that opened it + /// @param profile the column's recorded statistics, from [ColumnProfile#of(VortexHandle)] + /// @return the column's distribution + public static ColumnDistribution of(VortexHandle handle, ColumnProfile profile) { + ColumnName name = ColumnName.of(profile.name()); + ColumnProfile bounded = profile; + if (isNumeric(profile.type()) && !(profile.min() instanceof Number && profile.max() instanceof Number)) { + bounded = withScannedRange(handle, profile, name); + } + Accumulator acc = new Accumulator(bounded); + try (ScanIterator scan = handle.scan(ScanOptions.columns(name))) { + while (scan.hasNext()) { + try (Chunk chunk = scan.next()) { + acc.add(chunk, name); + } + } + } + return acc.result(); + } + + private static boolean isNumeric(DType type) { + return type instanceof DType.Primitive + || type instanceof DType.Extension ext && ext.storageDType() instanceof DType.Primitive; + } + + /// The profile with its min/max replaced by the column's scanned extremes, or unchanged when + /// the column holds no non-null, non-`NaN` value. + private static ColumnProfile withScannedRange(VortexHandle handle, ColumnProfile profile, ColumnName name) { + double[] range = {Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY}; + try (ScanIterator scan = handle.scan(ScanOptions.columns(name))) { + while (scan.hasNext()) { + try (Chunk chunk = scan.next()) { + MaskedArray.Unwrapped split = MaskedArray.unwrap(chunk.column(name)); + LongToDoubleFunction value = Accumulator.numericReader(split.inner()); + BoolArray validity = split.validity(); + for (long i = 0; value != null && i < chunk.rowCount(); i++) { + double v = value.applyAsDouble(i); + // NaN would poison min/max; the histogram leaves it out too. + if ((validity == null || validity.getBoolean(i)) && v == v) { + range[0] = Math.min(range[0], v); + range[1] = Math.max(range[1], v); + } + } + } + } + } + if (!(range[0] <= range[1])) { + return profile; + } + return new ColumnProfile(profile.name(), profile.type(), profile.rows(), profile.nullCount(), + profile.nanCount(), range[0], range[1], profile.sum(), profile.zones()); + } + + /// Picks the bars to show from what the scan gathered. + /// + /// @param counts rows per distinct non-null value, or `null` when there were too many to count + /// @param bins rows per histogram bin, or `null` when the column has no histogram + /// @param lo the first bin's lower bound + /// @param hi the last bin's upper bound + /// @return the bars, and whether they are histogram bins + static Map.Entry, Boolean> bars(Map counts, long[] bins, double lo, double hi) { + if (counts != null && counts.size() <= MAX_CATEGORIES) { + List bars = new ArrayList<>(counts.size()); + counts.forEach((value, count) -> bars.add(new Bar(value, null, count))); + // Numbers, dates and flags read in their own order; names read most frequent first. + if (counts.keySet().stream().allMatch(value -> value instanceof Number || value instanceof Boolean)) { + bars.sort(Comparator.comparing(Bar::value, ColumnDistribution::compareKeys)); + } else { + bars.sort(Comparator.comparingLong(Bar::count).reversed()); + } + return Map.entry(List.copyOf(bars), false); + } + if (bins != null) { + List bars = new ArrayList<>(bins.length); + double width = (hi - lo) / bins.length; + for (int i = 0; i < bins.length; i++) { + bars.add(new Bar(lo + i * width, i == bins.length - 1 ? hi : lo + (i + 1) * width, bins[i])); + } + return Map.entry(List.copyOf(bars), true); + } + if (counts != null) { + List top = counts.entrySet().stream() + .sorted(Map.Entry.comparingByValue().reversed()) + .limit(TOP_VALUES) + .map(entry -> new Bar(entry.getKey(), null, entry.getValue())) + .toList(); + return Map.entry(top, false); + } + return Map.entry(List.of(), false); + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private static int compareKeys(Object a, Object b) { + if (a instanceof Number x && b instanceof Number y) { + return Double.compare(x.doubleValue(), y.doubleValue()); + } + return ((Comparable) a).compareTo(b); + } + + /// The running state of one scan. + private static final class Accumulator { + + private final DType type; + private final boolean summable; + private Map counts = new HashMap<>(); + private final long[] bins; + private final double lo; + private final double hi; + private final double scale; + private long rows; + private long nulls; + private Number sum; + private boolean summed; + private long[] lengths; + + Accumulator(ColumnProfile profile) { + this.type = profile.type(); + this.summable = type instanceof DType.Primitive; + if (isNumeric(type) && profile.min() instanceof Number min && profile.max() instanceof Number max) { + this.bins = new long[BINS]; + this.lo = min.doubleValue(); + this.hi = max.doubleValue(); + // A constant column puts everything in the first bin rather than dividing by zero. + this.scale = hi > lo ? BINS / (hi - lo) : 0; + } else { + this.bins = null; + this.lo = 0; + this.hi = 0; + this.scale = 0; + } + } + + void add(Chunk chunk, ColumnName name) { + long n = chunk.rowCount(); + rows += n; + Array column = chunk.column(name); + if (column instanceof NullArray) { + nulls += n; + return; + } + MaskedArray.Unwrapped split = MaskedArray.unwrap(column); + BoolArray validity = split.validity(); + nulls += validity == null ? 0 : n - trueCount(validity, n); + if (summable) { + Number part = Compute.filteredAggregate(chunk, EVERY_ROW, name).sum(); + sum = summed ? addSum(sum, part) : part; + summed = true; + } + if (counts != null) { + count(chunk, name, split.inner(), validity); + } + if (bins != null) { + LongToDoubleFunction value = numericReader(split.inner()); + if (value != null) { + histogram(value, validity, n); + } + } + if (split.inner() instanceof VarBinArray strings) { + lengths(strings, validity, n); + } + } + + ColumnDistribution result() { + Map.Entry, Boolean> bars = bars(counts, bins, lo, hi); + Lengths stringLengths = null; + if (lengths != null && lengths[0] != Long.MAX_VALUE) { + stringLengths = new Lengths(lengths[0], lengths[1], lengths[2]); + } + return new ColumnDistribution(rows, nulls, sum, counts == null ? null : (long) counts.size(), + bars.getKey(), bars.getValue(), stringLengths); + } + + /// Counts through the grouping kernel, which reads dictionary codes rather than decoding + /// every row; a plain (non-dictionary) string chunk is counted here instead. A float column + /// is not counted: its values are a histogram's job. + private void count(Chunk chunk, ColumnName name, Array data, BoolArray validity) { + Optional> groups = Compute.groupedAggregate(chunk, name, null, null); + if (groups.isPresent()) { + groups.get().forEach((value, group) -> { + if (value != null) { + counts.merge(value, group.selectedRows(), Long::sum); + } + }); + } else if (data instanceof VarBinArray strings) { + for (long i = 0; i < strings.length() && counts.size() <= MAX_DISTINCT; i++) { + if (validity == null || validity.getBoolean(i)) { + counts.merge(strings.getString(i), 1L, Long::sum); + } + } + } else { + counts = null; + return; + } + if (counts.size() > MAX_DISTINCT) { + counts = null; + } + } + + /// Branch-split on validity so the common no-null chunk runs a uniform loop. + private void histogram(LongToDoubleFunction value, BoolArray validity, long n) { + if (validity == null) { + for (long i = 0; i < n; i++) { + bin(value.applyAsDouble(i)); + } + } else { + for (long i = 0; i < n; i++) { + if (validity.getBoolean(i)) { + bin(value.applyAsDouble(i)); + } + } + } + } + + private void bin(double v) { + // NaN sits outside every bin; min/max skip it too. + if (v == v) { + bins[Math.clamp((long) ((v - lo) * scale), 0, BINS - 1)]++; + } + } + + private void lengths(VarBinArray strings, BoolArray validity, long n) { + if (lengths == null) { + lengths = new long[] {Long.MAX_VALUE, 0, 0}; + } + for (long i = 0; i < n; i++) { + if (validity == null || validity.getBoolean(i)) { + int length = strings.getByteLength(i); + lengths[0] = Math.min(lengths[0], length); + lengths[1] = Math.max(lengths[1], length); + lengths[2] += length; + } + } + } + + private static long trueCount(BoolArray validity, long n) { + long valid = 0; + for (long i = 0; i < n; i++) { + if (validity.getBoolean(i)) { + valid++; + } + } + return valid; + } + + /// Resolves the typed accessor once per chunk, so the histogram loop calls one monomorphic + /// function rather than switching on the array type per row. + private static LongToDoubleFunction numericReader(Array data) { + boolean unsigned = data.dtype().isUnsigned(); + return switch (data) { + case LongArray a when unsigned -> i -> unsignedToDouble(a.getLong(i)); + case LongArray a -> i -> a.getLong(i); + case IntArray a when unsigned -> i -> Integer.toUnsignedLong(a.getInt(i)); + case IntArray a -> i -> a.getInt(i); + case ShortArray a when unsigned -> i -> Short.toUnsignedInt(a.getShort(i)); + case ShortArray a -> i -> a.getShort(i); + case ByteArray a when unsigned -> i -> Byte.toUnsignedInt(a.getByte(i)); + case ByteArray a -> i -> a.getByte(i); + case DoubleArray a -> a::getDouble; + case FloatArray a -> a::getFloat; + default -> null; + }; + } + + private static double unsignedToDouble(long v) { + return v >= 0 ? v : (v >>> 1) * 2.0 + (v & 1); + } + + private static Number addSum(Number total, Number part) { + if (total == null || part == null) { + return null; + } + if (total instanceof Long a && part instanceof Long b) { + try { + return Math.addExact(a, b); + } catch (ArithmeticException _) { + return null; + } + } + return total.doubleValue() + part.doubleValue(); + } + } +} diff --git a/inspector/src/main/java/io/github/dfa1/vortex/inspect/ColumnProfile.java b/inspector/src/main/java/io/github/dfa1/vortex/inspect/ColumnProfile.java new file mode 100644 index 000000000..ee1465829 --- /dev/null +++ b/inspector/src/main/java/io/github/dfa1/vortex/inspect/ColumnProfile.java @@ -0,0 +1,153 @@ +package io.github.dfa1.vortex.inspect; + +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.reader.ArrayStats; +import io.github.dfa1.vortex.reader.ScanIterator; +import io.github.dfa1.vortex.reader.ScanOptions; +import io.github.dfa1.vortex.reader.VortexHandle; +import io.github.dfa1.vortex.reader.Zone; + +import java.util.ArrayList; +import java.util.List; + +/// One top-level column's statistics, folded from what the file already records - its zone map, +/// or each chunk's embedded stats when there is none. No data segment is decoded, so a profile +/// costs the same on a 10 GB file as on a 10 KB one, and over HTTP only the footer and the zone +/// tables are fetched. +/// +/// Every aggregate is either exact or `null`: a column-wide value is reported only when every zone +/// that holds a value carries it, and min/max only when each zone's bound is exact (Rust records a +/// string's zone bounds truncated, so a utf8 column usually has no min/max here). +/// +/// @param name column name +/// @param type column dtype +/// @param rows rows in the file +/// @param nullCount null values in the column, or `null` if a zone does not record it +/// @param nanCount `NaN` values in a float column, or `null` if unknown +/// @param min the column's exact minimum, or `null` if unknown +/// @param max the column's exact maximum, or `null` if unknown +/// @param sum the column's sum (`Long` for integer and bool columns, `Double` for floats), or `null` if unknown +/// @param zones the zone map in row order, empty when the column has none +public record ColumnProfile( + String name, + DType type, + long rows, + Long nullCount, + Long nanCount, + Object min, + Object max, + Object sum, + List zones) { + + /// Profiles every top-level column of the file. + /// + /// @param handle an open file + /// @return one profile per schema field in order; empty when the root dtype is not a struct + public static List of(VortexHandle handle) { + if (!(handle.dtype() instanceof DType.Struct schema)) { + return List.of(); + } + long rows = handle.layout().rowCount(); + List profiles = new ArrayList<>(schema.fieldNames().size()); + try (ScanIterator iter = handle.scan(ScanOptions.all())) { + for (int i = 0; i < schema.fieldNames().size(); i++) { + ColumnName name = schema.fieldNames().get(i); + ColumnProfile profile = fold(name.value(), schema.fieldTypes().get(i), rows, + iter.columnZoneStats(name), iter.columnZones(name)); + if (profile.min() == null || profile.max() == null || profile.sum() == null) { + profile = profile.orChunks(fold(name.value(), profile.type(), rows, iter.chunkStats(name), List.of())); + } + profiles.add(profile); + } + } + return List.copyOf(profiles); + } + + /// The mean of the non-null, non-`NaN` values: `sum / count`. For a bool column that is the + /// share of `true`. + /// + /// @return the mean, or `null` when the sum or the null count is unknown, or no value is present + public Double mean() { + if (!(sum instanceof Number total) || nullCount == null) { + return null; + } + long present = rows - nullCount - (nanCount == null ? 0 : nanCount); + return present <= 0 ? null : total.doubleValue() / present; + } + + /// Fills what the zone map left unknown from the chunks' own array stats: a `vortex.zoned` map + /// records no sum (Rust's aggregate set), and only truncated bounds for a string, while each + /// chunk's array stats carry both exactly. Null counts stay the zone map's. + private ColumnProfile orChunks(ColumnProfile chunks) { + return new ColumnProfile(name, type, rows, nullCount, nanCount, + min != null ? min : chunks.min(), + max != null ? max : chunks.max(), + sum != null ? sum : chunks.sum(), + zones); + } + + static ColumnProfile fold(String name, DType type, long rows, List stats, List zones) { + Long nulls = 0L; + Long nans = 0L; + Object min = null; + Object max = null; + Object sum = null; + boolean minKnown = true; + boolean maxKnown = true; + boolean sumKnown = true; + for (ArrayStats zone : stats) { + nulls = add(nulls, zone.nullCount()); + nans = add(nans, zone.nanCount()); + // Rust leaves min/max/sum absent on an all-null zone; that says nothing about the rest. + if (zone.min() == null && zone.max() == null && zone.sum() == null && zone.nullCount() != null + && zone.nullCount() > 0) { + continue; + } + minKnown &= zone.min() != null && zone.minExact(); + maxKnown &= zone.max() != null && zone.maxExact(); + sumKnown &= zone.sum() != null; + if (minKnown) { + min = VortexInspector.pickMin(min, zone.min()); + } + if (maxKnown) { + max = VortexInspector.pickMax(max, zone.max()); + } + if (sumKnown) { + sum = addSum(sum, zone.sum()); + sumKnown = sum != null; + } + } + boolean any = !stats.isEmpty(); + return new ColumnProfile(name, type, rows, + any ? nulls : null, + any ? nans : null, + any && minKnown ? min : null, + any && maxKnown ? max : null, + any && sumKnown ? sum : null, + zones); + } + + private static Long add(Long total, Long part) { + return total == null || part == null ? null : total + part; + } + + /// Integer sums stay exact and give up on overflow rather than wrap; anything else (float sums, + /// mixed types) adds as `double`. + private static Object addSum(Object total, Object part) { + if (total == null) { + return part; + } + if (total instanceof Long a && part instanceof Long b) { + try { + return Math.addExact(a, b); + } catch (ArithmeticException _) { + return null; + } + } + if (total instanceof Number a && part instanceof Number b) { + return a.doubleValue() + b.doubleValue(); + } + return null; + } +} diff --git a/inspector/src/main/java/io/github/dfa1/vortex/inspect/ExplorePage.java b/inspector/src/main/java/io/github/dfa1/vortex/inspect/ExplorePage.java new file mode 100644 index 000000000..4e868dc5d --- /dev/null +++ b/inspector/src/main/java/io/github/dfa1/vortex/inspect/ExplorePage.java @@ -0,0 +1,316 @@ +package io.github.dfa1.vortex.inspect; + +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.reader.ArrayStats; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +import static io.github.dfa1.vortex.inspect.HtmlReport.appendHead; +import static io.github.dfa1.vortex.inspect.HtmlReport.bar; +import static io.github.dfa1.vortex.inspect.HtmlReport.count; +import static io.github.dfa1.vortex.inspect.HtmlReport.escape; +import static io.github.dfa1.vortex.inspect.HtmlReport.fill; +import static io.github.dfa1.vortex.inspect.HtmlReport.percent; +import static io.github.dfa1.vortex.inspect.HtmlReport.readable; +import static io.github.dfa1.vortex.inspect.HtmlReport.stat; +import static io.github.dfa1.vortex.inspect.HtmlReport.value; + +/// The data-first page, for people who ask what is in a file rather than how it is stored: +/// per-column statistics and nothing about encodings, segments or bytes. [HtmlReport] is the +/// storage-first one. +/// +/// The page opens on the statistics the file records ([ColumnProfile]); each column's +/// distribution ([ColumnDistribution]) needs a pass over its values. Served live +/// ([#render(String, long, long, List)]), the page fetches it on demand from `column/` +/// relative to its own URL; published ([#render(String, long, long, List, List)]), every column's is +/// already in the page. Either way a distribution is the HTML fragment +/// [#distribution(ColumnProfile, ColumnDistribution)] renders, and every value in it is escaped here, +/// as on the page. The page's only script fetches fragments and shows or hides them. +public final class ExplorePage { + + private ExplorePage() { + } + + /// Renders the live page, which asks its server for each column's distribution on demand. + /// + /// @param title file name shown in the header; escaped before it reaches the page + /// @param fileSize size of the file in bytes + /// @param rows rows in the file + /// @param profiles per-column statistics, from [ColumnProfile#of(io.github.dfa1.vortex.reader.VortexHandle)] + /// @return a complete HTML document + public static String render(String title, long fileSize, long rows, List profiles) { + return render(title, fileSize, rows, profiles, List.of()); + } + + /// Renders the page with every column's distribution already in it: one file that needs no + /// server, to publish or send. + /// + /// @param title file name shown in the header; escaped before it reaches the page + /// @param fileSize size of the file in bytes + /// @param rows rows in the file + /// @param profiles per-column statistics, from [ColumnProfile#of(io.github.dfa1.vortex.reader.VortexHandle)] + /// @param analyses each column's fragment from [#distribution(ColumnProfile, ColumnDistribution)], in + /// column order; empty for the live page + /// @return a complete HTML document + public static String render(String title, long fileSize, long rows, List profiles, + List analyses) { + if (!analyses.isEmpty() && analyses.size() != profiles.size()) { + throw new IllegalArgumentException(analyses.size() + " analyses for " + profiles.size() + " columns"); + } + StringBuilder sb = new StringBuilder(16 * 1024); + appendHead(sb, title + " - Vortex explorer"); + sb.append("
\n

").append(escape(title)).append("

\n
\n"); + stat(sb, "Rows", count(rows)); + stat(sb, "Columns", count(profiles.size())); + stat(sb, "Size", ByteSize.format(fileSize)); + sb.append("
\n
\n"); + appendProfiles(sb, profiles, analyses); + sb.append("\n\n\n\n"); + return sb.toString(); + } + + /// Renders one column's distribution as the fragment the page inserts under the column's row. + /// + /// @param profile the column's recorded statistics + /// @param distribution the column's distribution, from [ColumnDistribution#of(io.github.dfa1.vortex.reader.VortexHandle, ColumnProfile)] + /// @return an HTML fragment + public static String distribution(ColumnProfile profile, ColumnDistribution distribution) { + StringBuilder sb = new StringBuilder(4 * 1024); + DType type = profile.type(); + List facts = new ArrayList<>(); + facts.add(count(distribution.rows() - distribution.nulls()) + " values"); + facts.add(count(distribution.nulls()) + " null (" + percent(distribution.nulls(), distribution.rows()) + ")"); + if (distribution.sum() != null) { + facts.add("total " + number(distribution.sum().doubleValue()) + ""); + } + if (distribution.mean() != null) { + facts.add("mean " + number(distribution.mean()) + ""); + } + if (distribution.distinct() == null) { + facts.add("more than " + count(ColumnDistribution.MAX_DISTINCT) + " distinct"); + } else { + facts.add(count(distribution.distinct()) + " distinct"); + } + ColumnDistribution.Lengths lengths = distribution.lengths(); + if (lengths != null) { + long present = distribution.rows() - distribution.nulls(); + facts.add(count(lengths.min()) + "-" + count(lengths.max()) + " bytes long, " + + number((double) lengths.total() / present) + " on average"); + } + sb.append("
\n

").append(String.join(" · ", facts)).append("

\n"); + List bars = distribution.bars(); + if (distribution.histogram()) { + appendHistogram(sb, bars, type); + } else if (!bars.isEmpty()) { + appendCounts(sb, bars, type, distribution.rows() - distribution.nulls(), + distribution.distinct() != null && distribution.distinct() > bars.size()); + } + sb.append("
\n"); + return sb.toString(); + } + + // ---------------------------------------------------------------- statistics table + + /// What a reader of the data asks first - how complete is each column, what range does it span, + /// where does it sit on average - answered from the file's recorded statistics alone. + private static void appendProfiles(StringBuilder sb, List profiles, List analyses) { + // Only files that record a sum have a mean (Rust's zone map and array stats carry none), + // so the column is left out rather than shown as a row of dashes. + boolean withMean = profiles.stream().anyMatch(profile -> profile.mean() != null); + boolean analyzed = !analyses.isEmpty(); + sb.append("
\n
Columns"); + if (analyzed) { + sb.append("every column analyzed
\n"); + } else { + sb.append("statistics recorded in the file, nothing decoded") + .append("\n"); + } + sb.append("
\n\n") + .append("") + .append(withMean ? "" : "") + .append("") + .append("\n\n"); + for (int i = 0; i < profiles.size(); i++) { + ColumnProfile profile = profiles.get(i); + sb.append(""); + if (withMean) { + sb.append(""); + } + sb.append(""); + if (analyzed) { + // Same markup the live page's script leaves after an analysis, so its toggle works here too. + sb.append("\n") + .append("\n"); + } else { + sb.append("\n"); + } + } + sb.append("\n
columnnull %minmaxmeanrange over rows
") + .append(escape(profile.name())).append(" ") + .append(escape(VortexInspector.formatDType(profile.type()))).append(""); + if (profile.nullCount() == null) { + sb.append("-"); + } else { + bar(sb, profile.nullCount(), profile.rows(), "var(--meta)"); + sb.append(percent(profile.nullCount(), profile.rows())); + } + sb.append("").append(value(profile.min(), profile.type())) + .append("").append(value(profile.max(), profile.type())).append("").append(mean(profile)).append("").append(sparkline(profile, fill(i))).append("
") + .append(analyses.get(i)).append("
\n
\n
\n"); + } + + /// The mean, only where it reads as a number: a bool column's is its share of `true`; an + /// extension's (a date, a timestamp) is an epoch average nobody asked for, so it is left out. + private static String mean(ColumnProfile profile) { + Double mean = profile.mean(); + if (mean == null) { + return "-"; + } + return switch (profile.type()) { + case DType.Bool _ -> String.format(Locale.ROOT, "%.1f%% true", 100 * mean); + case DType.Primitive _ -> "" + number(mean) + ""; + default -> "-"; + }; + } + + /// Upper bound on points in a sparkline; consecutive zones merge (min of mins, max of maxes). + private static final int MAX_SPARK_POINTS = 120; + + /// Each zone's min-max band across the rows, as an inline SVG. Shows at a glance whether a + /// column is sorted (a rising band), clustered (narrow bands) or random (every band spans the + /// whole range) - which is also how well a filter on it prunes zones. + private static String sparkline(ColumnProfile profile, String fill) { + List points = new ArrayList<>(); + int stride = Math.max(1, (profile.zones().size() + MAX_SPARK_POINTS - 1) / MAX_SPARK_POINTS); + for (int i = 0; i < profile.zones().size(); i += stride) { + double lo = Double.POSITIVE_INFINITY; + double hi = Double.NEGATIVE_INFINITY; + for (int j = i; j < Math.min(i + stride, profile.zones().size()); j++) { + ArrayStats stats = profile.zones().get(j).stats(); + if (stats.min() instanceof Number min && stats.max() instanceof Number max) { + lo = Math.min(lo, min.doubleValue()); + hi = Math.max(hi, max.doubleValue()); + } + } + if (lo <= hi) { + points.add(new double[] {lo, hi}); + } + } + if (points.size() < 2) { + return "-"; + } + double lo = points.stream().mapToDouble(p -> p[0]).min().orElse(0); + double hi = points.stream().mapToDouble(p -> p[1]).max().orElse(0); + double span = hi > lo ? hi - lo : 1; + StringBuilder top = new StringBuilder(); + StringBuilder bottom = new StringBuilder(); + for (int i = 0; i < points.size(); i++) { + double x = 100.0 * i / (points.size() - 1); + top.append(String.format(Locale.ROOT, "%.2f,%.2f ", x, 22 - 20 * (points.get(i)[1] - lo) / span)); + bottom.insert(0, String.format(Locale.ROOT, "%.2f,%.2f ", x, 22 - 20 * (points.get(i)[0] - lo) / span)); + } + return "" + + "zone min-max over " + count(profile.zones().size()) + " zones" + + ""; + } + + // ---------------------------------------------------------------- distribution + + /// A column chart over the bins, its range labeled underneath; hovering a bar names its bin. + private static void appendHistogram(StringBuilder sb, List bars, DType type) { + long tallest = bars.stream().mapToLong(ColumnDistribution.Bar::count).max().orElse(1L); + double width = 300.0 / bars.size(); + sb.append("") + .append("histogram of ").append(bars.size()).append(" equal-width bins"); + for (int i = 0; i < bars.size(); i++) { + ColumnDistribution.Bar bin = bars.get(i); + double height = tallest == 0 ? 0 : 78.0 * bin.count() / tallest; + sb.append(String.format(Locale.ROOT, "", + i * width + 0.5, 80 - height, width - 1, height)) + .append("").append(escape(bound(bin.value(), type))).append(" to ") + .append(escape(bound(bin.upper(), type))).append(": ").append(count(bin.count())) + .append(" rows"); + } + sb.append("\n
").append(escape(bound(bars.getFirst().value(), type))) + .append("").append(escape(bound(bars.getLast().upper(), type))).append("
\n"); + } + + /// One bar per value, longest first or in value order, each with its count and share. + private static void appendCounts(StringBuilder sb, List bars, DType type, long present, + boolean top) { + if (top) { + sb.append("

most frequent ").append(bars.size()).append(":

\n"); + } + long widest = bars.stream().mapToLong(ColumnDistribution.Bar::count).max().orElse(1L); + sb.append("\n\n"); + for (ColumnDistribution.Bar bar : bars) { + sb.append("\n"); + } + sb.append("\n
").append(value(bar.value(), type)).append(""); + bar(sb, bar.count(), widest, "var(--series-1)"); + sb.append("").append(count(bar.count())).append("") + .append(percent(bar.count(), present)).append("
\n"); + } + + /// A histogram bound as the column reads: a date or timestamp bin edge shown as a date, not an + /// epoch number. + private static String bound(Object raw, DType type) { + if (type instanceof DType.Extension && raw instanceof Double d) { + return String.valueOf(readable(Math.round(d), type)); + } + return raw instanceof Double d ? number(d) : String.valueOf(raw); + } + + /// A whole number without decimals (a total, an integer bin edge), else two decimals, or four + /// significant digits below one. + private static String number(double value) { + if (value == Math.rint(value) && Math.abs(value) < 1e15) { + return String.format(Locale.ROOT, "%,.0f", value); + } + return String.format(Locale.ROOT, Math.abs(value) >= 1 ? "%,.2f" : "%.4g", value); + } + + /// Fetches a column's distribution into a row under it, then turns the button into a Hide/Show + /// toggle for that row; "Analyze all" asks for every column not analyzed yet at once, and the + /// server analyzes as many at a time as it has readers. + private static final String SCRIPT = """ + async function analyze(button) { + button.dataset.state = 'running'; + button.disabled = true; + button.textContent = 'Analyzing...'; + const row = button.closest('tr'); + const response = await fetch('column/' + button.dataset.column); + const detail = document.createElement('tr'); + detail.className = 'detail'; + const cell = document.createElement('td'); + cell.colSpan = row.children.length; + if (response.ok) { + cell.innerHTML = await response.text(); + } else { + cell.textContent = 'failed: ' + response.status + ' ' + await response.text(); + } + detail.append(cell); + row.after(detail); + button.dataset.state = 'done'; + button.disabled = false; + button.textContent = 'Hide'; + } + function toggle(button) { + const detail = button.closest('tr').nextElementSibling; + detail.hidden = !detail.hidden; + button.textContent = detail.hidden ? 'Show' : 'Hide'; + } + document.addEventListener('click', event => { + const button = event.target.closest('button[data-column]'); + if (!button) { return; } + if (button.dataset.state === 'done') { toggle(button); } else if (!button.dataset.state) { analyze(button); } + }); + document.getElementById('analyze-all')?.addEventListener('click', event => { + event.target.disabled = true; + document.querySelectorAll('button[data-column]:not([data-state])').forEach(analyze); + }); + """; +} diff --git a/inspector/src/main/java/io/github/dfa1/vortex/inspect/HtmlReport.java b/inspector/src/main/java/io/github/dfa1/vortex/inspect/HtmlReport.java index 8e075603a..ba64e3688 100644 --- a/inspector/src/main/java/io/github/dfa1/vortex/inspect/HtmlReport.java +++ b/inspector/src/main/java/io/github/dfa1/vortex/inspect/HtmlReport.java @@ -62,10 +62,7 @@ private HtmlReport() { public static String render(InspectorTree tree, String title) { List columns = columns(tree); StringBuilder sb = new StringBuilder(64 * 1024); - sb.append("\n\n\n\n") - .append("\n") - .append("").append(escape(title)).append(" - Vortex inspector\n") - .append("\n\n\n
\n"); + appendHead(sb, title + " - Vortex inspector"); appendHeader(sb, tree, columns, title); appendStrip(sb, tree, columns); sb.append("
\n"); @@ -77,6 +74,13 @@ public static String render(InspectorTree tree, String title) { return sb.toString(); } + static void appendHead(StringBuilder sb, String title) { + sb.append("\n\n\n\n") + .append("\n") + .append("").append(escape(title)).append("\n") + .append("\n\n\n
\n"); + } + // ---------------------------------------------------------------- sections private static void appendHeader(StringBuilder sb, InspectorTree tree, List columns, String title) { @@ -501,7 +505,7 @@ private static long spanBytes(List columns, RowSpan span) { // ---------------------------------------------------------------- fragments - private static void stat(StringBuilder sb, String label, String value) { + static void stat(StringBuilder sb, String label, String value) { sb.append("
").append(label).append("
").append(value).append("
\n"); } @@ -516,7 +520,7 @@ private static void block(StringBuilder sb, long fileSize, long offset, long len .append(";background:").append(fill).append("\" title=\"").append(escape(tip)).append("\">\n"); } - private static void bar(StringBuilder sb, long value, long max, String fill) { + static void bar(StringBuilder sb, long value, long max, String fill) { sb.append(""); } @@ -531,7 +535,7 @@ private static String ratio(ColumnView column) { return factor < 1.05 ? "" : String.format(Locale.ROOT, "%.1fx", factor); } - private static String fill(int column) { + static String fill(int column) { return column < 0 || column >= LIGHT_SERIES.size() ? "var(--other)" : "var(--series-" + (column + 1) + ")"; @@ -542,11 +546,11 @@ private static String width(long value, long total) { return String.format(Locale.ROOT, "%.4f%%", pct); } - private static String percent(long value, long total) { + static String percent(long value, long total) { return total <= 0 ? "0%" : String.format(Locale.ROOT, "%.1f%%", 100.0 * value / total); } - private static String count(long n) { + static String count(long n) { return String.format(Locale.ROOT, "%,d", n); } @@ -561,7 +565,7 @@ private static String rowRange(long firstRow, long rows) { return count(firstRow) + "-" + count(firstRow + rows - 1); } - private static String value(Object raw, DType type) { + static String value(Object raw, DType type) { if (raw == null) { return "-"; } @@ -578,7 +582,7 @@ private static String value(Object raw, DType type) { /// utf8 showed `2024-01-01T00:00:00` would make the better-typed file the harder one to read. /// Anything unrecognized, and any metadata this cannot parse - the bytes come from an /// untrusted file - falls back to the raw value rather than failing the render. - private static Object readable(Object raw, DType type) { + static Object readable(Object raw, DType type) { if (!(type instanceof DType.Extension ext) || !(raw instanceof Long epoch)) { return raw; } @@ -725,5 +729,20 @@ private static List vars(List series) { .num { font-variant-numeric: tabular-nums; } .sized { text-align: right; white-space: nowrap; } .sized .bar { width: 46px; display: inline-flex; margin: 0 8px 0 0; vertical-align: 1px; } + .scroll { overflow-x: auto; } + .profiles td { white-space: nowrap; } + .spark { width: 120px; height: 24px; display: block; } + .spark polygon { fill-opacity: 0.35; stroke-width: 1; vector-effect: non-scaling-stroke; stroke-linejoin: round; } + button { font: inherit; font-size: 12px; color: var(--text-secondary); background: var(--surface-0); border: 1px solid var(--border); border-radius: 6px; padding: 2px 10px; cursor: pointer; } + button:hover:not(:disabled) { color: var(--text-primary); border-color: var(--text-muted); } + button:disabled { cursor: default; color: var(--text-muted); } + .profiles tr.detail > td { white-space: normal; background: var(--surface-0); padding: 10px 14px 14px; } + .dist { max-width: 720px; } + .hist { width: 100%%; height: 90px; display: block; } + .hist rect { fill: var(--series-1); } + .hist rect:hover { fill-opacity: 0.7; } + .axis { display: flex; justify-content: space-between; font-size: 11px; color: var(--text-muted); margin-top: 2px; } + .bars td { border-bottom: 0; padding: 2px 10px 2px 0; } + .bars .track { width: 50%%; } """; } diff --git a/inspector/src/test/java/io/github/dfa1/vortex/inspect/ColumnDistributionTest.java b/inspector/src/test/java/io/github/dfa1/vortex/inspect/ColumnDistributionTest.java new file mode 100644 index 000000000..b48091c86 --- /dev/null +++ b/inspector/src/test/java/io/github/dfa1/vortex/inspect/ColumnDistributionTest.java @@ -0,0 +1,97 @@ +package io.github.dfa1.vortex.inspect; + +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +class ColumnDistributionTest { + + @Test + void bars_fewNumbersListEachValueInValueOrder() { + // Given — a rating column: 5 bars in order beat a 30-bin histogram with 25 empty bins + Map counts = Map.of(3L, 10L, 1L, 2L, 5L, 7L); + + // When + Map.Entry, Boolean> result = + ColumnDistribution.bars(counts, new long[ColumnDistribution.BINS], 1, 5); + + // Then + assertThat(result.getValue()).isFalse(); + assertThat(result.getKey()).extracting(ColumnDistribution.Bar::value).containsExactly(1L, 3L, 5L); + } + + @Test + void bars_fewNamesListMostFrequentFirst() { + // Given + Map counts = Map.of("CASH", 3L, "STOCK", 9L, "BOND", 1L); + + // When + Map.Entry, Boolean> result = ColumnDistribution.bars(counts, null, 0, 0); + + // Then + assertThat(result.getKey()).extracting(ColumnDistribution.Bar::value).containsExactly("STOCK", "CASH", "BOND"); + } + + @Test + void bars_manyNumbersBecomeAHistogramSpanningMinToMax() { + // Given — more distinct values than are worth listing + Map counts = new HashMap<>(); + for (long v = 0; v <= ColumnDistribution.MAX_CATEGORIES; v++) { + counts.put(v, 1L); + } + long[] bins = new long[ColumnDistribution.BINS]; + bins[0] = 4; + + // When + Map.Entry, Boolean> result = ColumnDistribution.bars(counts, bins, 0, 30); + + // Then — the last bin ends exactly at max, not at a rounded lo + 30 * width + assertThat(result.getValue()).isTrue(); + assertThat(result.getKey()).hasSize(ColumnDistribution.BINS); + assertThat(result.getKey().getFirst()).isEqualTo(new ColumnDistribution.Bar(0.0, 1.0, 4)); + assertThat(result.getKey().getLast().upper()).isEqualTo(30.0); + } + + @Test + void bars_manyNamesShowTheMostFrequent() { + // Given + Map counts = new HashMap<>(); + for (int i = 0; i < 100; i++) { + counts.put("v" + i, (long) i); + } + + // When + Map.Entry, Boolean> result = ColumnDistribution.bars(counts, null, 0, 0); + + // Then + assertThat(result.getKey()).hasSize(ColumnDistribution.TOP_VALUES); + assertThat(result.getKey().getFirst().value()).isEqualTo("v99"); + } + + @Test + void bars_tooManyToCountAndNoHistogramShowsNothing() { + // Given — a high-cardinality string column: no honest picture to draw + + // When + Map.Entry, Boolean> result = ColumnDistribution.bars(null, null, 0, 0); + + // Then + assertThat(result.getKey()).isEmpty(); + } + + @Test + void mean_dividesByPresentValuesOnly() { + // Given — 10 rows, 2 null + ColumnDistribution sut = new ColumnDistribution(10, 2, 40L, null, List.of(), false, null); + + // When + Double result = sut.mean(); + + // Then + assertThat(result).isEqualTo(5.0); + } +} diff --git a/inspector/src/test/java/io/github/dfa1/vortex/inspect/ColumnProfileTest.java b/inspector/src/test/java/io/github/dfa1/vortex/inspect/ColumnProfileTest.java new file mode 100644 index 000000000..df0922d5d --- /dev/null +++ b/inspector/src/test/java/io/github/dfa1/vortex/inspect/ColumnProfileTest.java @@ -0,0 +1,121 @@ +package io.github.dfa1.vortex.inspect; + +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.reader.ArrayStats; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class ColumnProfileTest { + + @Test + void fold_combinesZonesIntoColumnTotals() { + // Given — two zones of a 10-row column, 1 null between them + List zones = List.of(stats(3L, 9L, 30L, 0L), stats(1L, 5L, 15L, 1L)); + + // When + ColumnProfile result = ColumnProfile.fold("x", DType.I64, 10, zones, List.of()); + + // Then + assertThat(result.min()).isEqualTo(1L); + assertThat(result.max()).isEqualTo(9L); + assertThat(result.sum()).isEqualTo(45L); + assertThat(result.nullCount()).isEqualTo(1L); + assertThat(result.mean()).isEqualTo(5.0); + } + + @Test + void fold_skipsAllNullZoneInsteadOfLosingMinMax() { + // Given — Rust leaves min/max/sum absent on a zone holding only nulls; that zone says nothing + // about the column's range, so it must not make the whole column's min/max unknown + ArrayStats allNull = new ArrayStats(null, false, null, false, null, null, null, null, 4L, null, null); + List zones = List.of(stats(2L, 7L, 9L, 0L), allNull); + + // When + ColumnProfile result = ColumnProfile.fold("x", DType.I64, 6, zones, List.of()); + + // Then + assertThat(result.min()).isEqualTo(2L); + assertThat(result.max()).isEqualTo(7L); + assertThat(result.nullCount()).isEqualTo(4L); + } + + @Test + void fold_inexactBoundIsNotReportedAsTheMinimum() { + // Given — a truncated string bound brackets the values but is not one of them + ArrayStats bounded = new ArrayStats("ab", false, "zz", true, null, null, null, null, 0L, null, null); + + // When + ColumnProfile result = ColumnProfile.fold("s", DType.UTF8, 3, List.of(bounded), List.of()); + + // Then + assertThat(result.min()).isNull(); + assertThat(result.max()).isEqualTo("zz"); + } + + @Test + void fold_oneZoneMissingSumLeavesColumnSumUnknown() { + // Given — a partial sum would print a wrong mean, not an approximate one + List zones = List.of(stats(1L, 2L, 3L, 0L), stats(1L, 2L, null, 0L)); + + // When + ColumnProfile result = ColumnProfile.fold("x", DType.I64, 4, zones, List.of()); + + // Then + assertThat(result.sum()).isNull(); + assertThat(result.mean()).isNull(); + } + + @Test + void fold_overflowingIntegerSumIsUnknownNotWrapped() { + // Given + List zones = List.of(stats(0L, 1L, Long.MAX_VALUE, 0L), stats(0L, 1L, 1L, 0L)); + + // When + ColumnProfile result = ColumnProfile.fold("x", DType.I64, 2, zones, List.of()); + + // Then + assertThat(result.sum()).isNull(); + } + + @Test + void fold_noStatsAtAllMeansEveryAggregateUnknown() { + // Given / When + ColumnProfile result = ColumnProfile.fold("x", DType.I64, 0, List.of(), List.of()); + + // Then — not "0 nulls": nothing was recorded + assertThat(result.nullCount()).isNull(); + assertThat(result.min()).isNull(); + assertThat(result.sum()).isNull(); + } + + @Test + void mean_excludesNullsAndNans() { + // Given — 10 rows, 2 null, 3 NaN: the sum covers the other 5 + ColumnProfile sut = new ColumnProfile("f", DType.F64, 10, 2L, 3L, 1.0, 4.0, 10.0, List.of()); + + // When + Double result = sut.mean(); + + // Then + assertThat(result).isEqualTo(2.0); + } + + @Test + void mean_allNullColumnHasNoMean() { + // Given + ColumnProfile sut = new ColumnProfile("f", DType.F64, 3, 3L, 0L, null, null, 0.0, List.of()); + + // When + Double result = sut.mean(); + + // Then — not a division by zero, and not NaN + assertThat(result).isNull(); + } + + private static ArrayStats stats(Long min, Long max, Long sum, Long nulls) { + return new ArrayStats(min, true, max, true, sum, null, null, null, nulls, null, null); + } +} diff --git a/inspector/src/test/java/io/github/dfa1/vortex/inspect/ExplorePageTest.java b/inspector/src/test/java/io/github/dfa1/vortex/inspect/ExplorePageTest.java new file mode 100644 index 000000000..28a2d3b22 --- /dev/null +++ b/inspector/src/test/java/io/github/dfa1/vortex/inspect/ExplorePageTest.java @@ -0,0 +1,240 @@ +package io.github.dfa1.vortex.inspect; + +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.reader.ArrayStats; +import io.github.dfa1.vortex.reader.Zone; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class ExplorePageTest { + + @Nested + class Render { + + @Test + void isSelfContainedAndSaysNothingAboutStorage() { + // Given + ColumnProfile id = new ColumnProfile("id", DType.I64, 1000, 0L, 0L, 1L, 999L, null, List.of()); + + // When + String result = ExplorePage.render("data.vortex", 4096L, 1000L, List.of(id)); + + // Then — a business user sees rows and columns, not segments and encodings; the one + // script is inline, nothing is fetched from elsewhere + assertThat(result) + .startsWith("") + .endsWith("\n") + .contains("data.vortex - Vortex explorer") + .contains("1,000") + .doesNotContain("