diff --git a/CHANGELOG.md b/CHANGELOG.md index 635e45d86..e5f44378f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - A CSV import schema can declare any integer type from `I8` to `U64`, range-checked per value, and `F32`; an unsupported type fails in `withSchema` instead of at the first chunk. ([#593](https://github.com/dfa1/vortex-java/pull/593)) +- `ScanIterator.chunkStats` is public: each chunk's own array stats, which carry a string column's exact min/max where the zone map only bounds it. ([#608](https://github.com/dfa1/vortex-java/pull/608)) +- `vortex explore` shows what is in a file: null %, min/max and a zone sparkline per column at once, then each column's total, mean, distinct count and value counts or histogram, served to the browser or written to one HTML file with `--html`. ([#608](https://github.com/dfa1/vortex-java/pull/608)) ### Changed - **Breaking:** `RowFilter` is renamed `ScanFilter`, matching `ScanOptions`. ([#598](https://github.com/dfa1/vortex-java/pull/598)) @@ -29,6 +31,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - A non-null integer or float column gets a dict layout only when the compressor picks a dictionary for its first 8192 rows, as Rust's writer decides, so a column that bit-packs better is no longer dictionary-encoded. ([#599](https://github.com/dfa1/vortex-java/pull/599)) - The writer cuts string and nullable columns into the same chunks as vortex-jni 0.89, which measures its 8192-row blocks by the bytes their rows reference. ([#596](https://github.com/dfa1/vortex-java/pull/596)) - `filter` and `select --where` on the command line work on a nullable column, where they failed; a null row matches no comparison, as in SQL. ([#593](https://github.com/dfa1/vortex-java/pull/593)) +- The command line prints sizes past 1 GiB in GB, TB, PB or EB, where a 2.3 GB file read as `2179.1 MB`. ([#608](https://github.com/dfa1/vortex-java/pull/608)) ## [0.18.1] — 2026-10-22 diff --git a/cli/src/main/java/io/github/dfa1/vortex/cli/ExploreCommand.java b/cli/src/main/java/io/github/dfa1/vortex/cli/ExploreCommand.java new file mode 100644 index 000000000..adf1217f7 --- /dev/null +++ b/cli/src/main/java/io/github/dfa1/vortex/cli/ExploreCommand.java @@ -0,0 +1,274 @@ +package io.github.dfa1.vortex.cli; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import io.github.dfa1.vortex.cli.tui.IoWorker; +import io.github.dfa1.vortex.inspect.ByteSize; +import io.github.dfa1.vortex.inspect.ColumnDistribution; +import io.github.dfa1.vortex.inspect.ColumnProfile; +import io.github.dfa1.vortex.inspect.ExplorePage; +import io.github.dfa1.vortex.reader.VortexHandle; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.HexFormat; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.IntFunction; +import java.util.function.Supplier; + +/// `explore [--serve | --html ] `: the data-first view of a file, per-column +/// statistics and distributions (`inspect --html` is the storage-first one). +/// +/// - `--serve` (the default) serves the page on a local port, opens it in the browser and computes +/// a column's distribution when the page asks for it. Runs until interrupted. +/// - `--html` analyzes every column up front and writes the page, distributions included, to one +/// self-contained file to publish. +/// +/// The file is open once per core, each on its own [IoWorker]: a reader's arena is confined to the +/// thread that opened it, so columns are analyzed in parallel by separate readers rather than by +/// threads sharing one. +@SuppressWarnings("java:S106") // CLI command: stdout is the intended output channel +final class ExploreCommand { + + private static final String USAGE = "usage: explore [--serve | --html ] "; + + private ExploreCommand() { + } + + static int run(String[] args) { + String target; + Path html = null; + if (args.length == 2 && !args[1].startsWith("--")) { + target = args[1]; + } else if (args.length == 3 && "--serve".equals(args[1])) { + target = args[2]; + } else if (args.length == 4 && "--html".equals(args[1])) { + html = Path.of(args[2]); + target = args[3]; + } else { + System.err.println(USAGE); + return ExitStatus.USAGE_ERROR; + } + IoWorker worker = new IoWorker("explore-io-0"); + try { + Optional opened = CliHandles.openOnWorker(worker, target); + if (opened.isEmpty()) { + worker.close(); + return ExitStatus.FILE_NOT_FOUND; + } + VortexHandle handle = opened.get(); + List profiles = onWorker(worker, () -> ColumnProfile.of(handle)); + String title = target.substring(target.lastIndexOf('/') + 1); + BlockingQueue readers = readers(new Reader(worker, handle), target, profiles.size()); + IntFunction analyze = index -> { + Reader reader = take(readers); + try { + return onWorker(reader.worker(), () -> ExplorePage.distribution( + profiles.get(index), ColumnDistribution.of(reader.handle(), profiles.get(index)))); + } finally { + readers.add(reader); + } + }; + if (html != null) { + List analyses = analyzeAll(profiles.size(), analyze); + Files.writeString(html, ExplorePage.render(title, handle.fileSize(), + handle.layout().rowCount(), profiles, analyses)); + readers.forEach(reader -> CliHandles.closeOnWorker(reader.worker(), reader.handle())); + readers.forEach(reader -> reader.worker().close()); + System.out.printf("written: %s (%d columns analyzed, %s)%n", html, profiles.size(), + ByteSize.format(Files.size(html))); + return ExitStatus.OK; + } + String page = ExplorePage.render(title, handle.fileSize(), handle.layout().rowCount(), profiles); + byte[] token = new byte[16]; + new SecureRandom().nextBytes(token); + String path = "/" + HexFormat.of().formatHex(token) + "/"; + HttpServer server = serve(page, path, profiles.size(), analyze); + URI url = URI.create("http://127.0.0.1:" + server.getAddress().getPort() + path); + System.out.println("serving " + url + " - press Ctrl-C to stop"); + openBrowser(url); + Thread.currentThread().join(); + return ExitStatus.OK; + } catch (IOException | RuntimeException e) { + System.err.println("error: " + CliHandles.describe(e)); + worker.close(); + return ExitStatus.ERROR; + } catch (InterruptedException _) { + Thread.currentThread().interrupt(); + return ExitStatus.OK; + } + } + + /// One open file on the thread that owns it. + private record Reader(IoWorker worker, VortexHandle handle) { + } + + /// One reader per core, up to one per column, so as many columns are analyzed at once. Each + /// opens the file on its own worker thread: a reader cannot be shared across threads, but + /// mapping a file again is cheap and the operating system shares its pages between the mappings. + private static BlockingQueue readers(Reader first, String target, int columns) + throws IOException, InterruptedException { + int count = Math.clamp(Runtime.getRuntime().availableProcessors(), 1, Math.max(1, columns)); + BlockingQueue readers = new ArrayBlockingQueue<>(count); + readers.add(first); + for (int i = 1; i < count; i++) { + IoWorker worker = new IoWorker("explore-io-" + i); + VortexHandle handle = CliHandles.openOnWorker(worker, target) + .orElseThrow(() -> new IOException("file vanished: " + target)); + readers.add(new Reader(worker, handle)); + } + return readers; + } + + /// Every column's analysis, as many at once as there are readers, in column order. + private static List analyzeAll(int columns, IntFunction analyze) throws InterruptedException { + try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + List> futures = new ArrayList<>(columns); + for (int i = 0; i < columns; i++) { + int index = i; + futures.add(executor.submit(() -> analyze.apply(index))); + } + List analyses = new ArrayList<>(columns); + for (Future future : futures) { + try { + analyses.add(future.get()); + } catch (ExecutionException e) { + // Rethrow the column's own failure (a malformed segment, say), not the wrapper. + if (e.getCause() instanceof RuntimeException cause) { + throw cause; + } + throw new IllegalStateException(e.getCause()); + } + } + return analyses; + } + } + + private static Reader take(BlockingQueue readers) { + try { + return readers.take(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("interrupted", e); + } + } + + /// Starts a server on a free port of `127.0.0.1` answering two paths: `path` with `page`, and + /// `path` + `column/` with `analyze`'s fragment for that column. Every other path is a + /// 404, and a failed analysis is a 500 carrying the error as plain text. + /// + /// The page shows file contents, so it is reachable only from this machine and only under + /// `path`, a random one in [#run(String[])], as Jupyter does: another local user, or a web page + /// probing localhost ports, cannot guess it. + /// + /// @param page the HTML document to serve + /// @param path the page's path, e.g. `/3f2a.../` + /// @param columns how many columns `analyze` accepts, indexed from `0` + /// @param analyze renders one column's distribution as an HTML fragment + /// @return the started server + /// @throws IOException if the server cannot bind + static HttpServer serve(String page, String path, int columns, IntFunction analyze) throws IOException { + byte[] body = page.getBytes(StandardCharsets.UTF_8); + String columnPrefix = path + "column/"; + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/", exchange -> { + try (exchange) { + String requested = exchange.getRequestURI().getPath(); + if (path.equals(requested)) { + respond(exchange, 200, "text/html", body); + return; + } + int index = -1; + if (requested.startsWith(columnPrefix)) { + index = columnIndex(requested.substring(columnPrefix.length())); + } + if (index < 0 || index >= columns) { + exchange.sendResponseHeaders(404, -1); + return; + } + try { + respond(exchange, 200, "text/html", analyze.apply(index).getBytes(StandardCharsets.UTF_8)); + } catch (RuntimeException e) { + respond(exchange, 500, "text/plain", CliHandles.describe(e).getBytes(StandardCharsets.UTF_8)); + } + } + }); + // A thread per request: `analyze` blocks until a reader is free, and the default executor's + // single thread would serialize the columns the readers could analyze at once. + server.setExecutor(Executors.newVirtualThreadPerTaskExecutor()); + server.start(); + return server; + } + + private static int columnIndex(String digits) { + if (digits.isEmpty() || digits.length() > 9 || !digits.chars().allMatch(Character::isDigit)) { + return -1; + } + return Integer.parseInt(digits); + } + + private static void respond(HttpExchange exchange, int status, String type, byte[] body) throws IOException { + exchange.getResponseHeaders().set("Content-Type", type + "; charset=utf-8"); + exchange.sendResponseHeaders(status, body.length); + try (OutputStream out = exchange.getResponseBody()) { + out.write(body); + } + } + + /// Runs `task` on the thread that owns the file and returns its result, rethrowing its failure. + private static T onWorker(IoWorker worker, Supplier task) { + AtomicReference result = new AtomicReference<>(); + AtomicReference failure = new AtomicReference<>(); + try { + worker.runAndAwait(() -> { + try { + result.set(task.get()); + } catch (RuntimeException e) { + failure.set(e); + } + }); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("interrupted", e); + } + if (failure.get() != null) { + throw failure.get(); + } + return result.get(); + } + + /// Best effort: the URL is printed either way, so a machine with no desktop (a server over SSH) + /// still gets a link to forward. + private static void openBrowser(URI url) { + String os = System.getProperty("os.name", "").toLowerCase(Locale.ROOT); + String[] command; + if (os.contains("mac")) { + command = new String[] {"open", url.toString()}; + } else if (os.contains("win")) { + command = new String[] {"rundll32", "url.dll,FileProtocolHandler", url.toString()}; + } else { + command = new String[] {"xdg-open", url.toString()}; + } + try { + new ProcessBuilder(command).redirectErrorStream(true).redirectOutput(ProcessBuilder.Redirect.DISCARD).start(); + } catch (IOException _) { + // no browser launcher: the printed URL is the fallback + } + } +} diff --git a/cli/src/main/java/io/github/dfa1/vortex/cli/VortexCli.java b/cli/src/main/java/io/github/dfa1/vortex/cli/VortexCli.java index 00fdd4115..0d1bc867d 100644 --- a/cli/src/main/java/io/github/dfa1/vortex/cli/VortexCli.java +++ b/cli/src/main/java/io/github/dfa1/vortex/cli/VortexCli.java @@ -58,6 +58,7 @@ private static int dispatch(String[] args) { } return switch (args[0]) { case "inspect" -> InspectCommand.run(args); + case "explore" -> ExploreCommand.run(args); case "tui" -> TuiCommand.run(args); case "view" -> ViewCommand.run(args); case "export" -> ExportCommand.run(args); @@ -78,6 +79,9 @@ private static int dispatch(String[] args) { static void printUsage(PrintStream out) { out.println("Usage: java -jar vortex-cli--all.jar [args] [--timing] [--allow-unknown]"); out.println(" inspect [--html] print file structure; --html writes an HTML report"); + out.println(" explore [--serve | --html ] "); + out.println(" per-column statistics: serve them to the browser (default),"); + out.println(" or write one HTML file with every column analyzed"); out.println(" tui open interactive inspector; url is http(s)://"); out.println(" view open scrollable data grid; url is http(s)://"); out.println(" export [out.csv|out.parquet|-] write CSV or Parquet; default is .csv, `-` for stdout"); diff --git a/cli/src/test/java/io/github/dfa1/vortex/cli/ExploreCommandTest.java b/cli/src/test/java/io/github/dfa1/vortex/cli/ExploreCommandTest.java new file mode 100644 index 000000000..e6cfb2979 --- /dev/null +++ b/cli/src/test/java/io/github/dfa1/vortex/cli/ExploreCommandTest.java @@ -0,0 +1,227 @@ +package io.github.dfa1.vortex.cli; + +import com.sun.net.httpserver.HttpServer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static io.github.dfa1.vortex.cli.CliTestSupport.capture; +import static io.github.dfa1.vortex.cli.CliTestSupport.writeTypedVortex; +import static org.assertj.core.api.Assertions.assertThat; + +class ExploreCommandTest { + + @Test + void wrongArity_returnsUsageError() { + // Given / When + CliTestSupport.Captured result = capture(() -> ExploreCommand.run(new String[]{"explore"})); + + // Then + assertThat(result.status()).isEqualTo(ExitStatus.USAGE_ERROR); + assertThat(result.stderr()).contains("usage: explore"); + } + + @Test + void missingFile_returnsFileNotFound(@TempDir Path tmp) { + // Given + Path missing = tmp.resolve("nope.vortex"); + + // When + CliTestSupport.Captured result = capture(() -> ExploreCommand.run(new String[]{"explore", missing.toString()})); + + // Then — fails before any server starts + assertThat(result.status()).isEqualTo(ExitStatus.FILE_NOT_FOUND); + assertThat(result.stderr()).contains("file not found"); + } + + @ParameterizedTest + @ValueSource(strings = {"--html", "--bogus"}) + void flagWithoutItsArguments_returnsUsageError(String flag) { + // Given / When — `explore --html data.vortex` would otherwise write the page over data.vortex + CliTestSupport.Captured result = capture(() -> ExploreCommand.run(new String[]{"explore", flag, "data.vortex"})); + + // Then + assertThat(result.status()).isEqualTo(ExitStatus.USAGE_ERROR); + assertThat(result.stderr()).contains("usage: explore"); + } + + @Test + void html_writesEveryColumnAlreadyAnalyzed(@TempDir Path tmp) throws IOException { + // Given — a 3-row file of id (I64), qty (I32), price (F64), name (Utf8) + Path file = writeTypedVortex(tmp, "typed.vortex"); + Path out = tmp.resolve("report.html"); + + // When + CliTestSupport.Captured result = capture(() -> + ExploreCommand.run(new String[]{"explore", "--html", out.toString(), file.toString()})); + + // Then — one file to publish: no server to ask, so no Analyze button, every column's + // distribution in place, each behind a working Hide toggle + assertThat(result.status()).isEqualTo(ExitStatus.OK); + assertThat(result.stdout()).contains("written: ").contains("4 columns analyzed"); + String page = Files.readString(out); + assertThat(page) + .startsWith("") + .doesNotContain(">Analyze") + .doesNotContain("id=\"analyze-all\"") + .contains("total 6") // id: 1 + 2 + 3 + .contains("total 60") // qty: 10 + 20 + 30 + .contains("carol"); // name: each of 3 values counted + assertThat(page.split("data-state=\"done\">Hide", -1)).hasSize(5); + } + + @Test + void html_missingFile_returnsFileNotFoundAndWritesNothing(@TempDir Path tmp) { + // Given + Path out = tmp.resolve("report.html"); + + // When + CliTestSupport.Captured result = capture(() -> ExploreCommand.run( + new String[]{"explore", "--html", out.toString(), tmp.resolve("nope.vortex").toString()})); + + // Then + assertThat(result.status()).isEqualTo(ExitStatus.FILE_NOT_FOUND); + assertThat(out).doesNotExist(); + } + + @Test + void serve_answersOnlyTheSecretPath() throws IOException, InterruptedException { + // Given — the page shows file contents: a local process or a web page probing localhost + // ports must not reach it without the random path + HttpServer sut = ExploreCommand.serve("

hello

", "/s3cret/", 2, index -> "column " + index); + try (HttpClient client = HttpClient.newHttpClient()) { + // When + HttpResponse result = get(client, sut, "/s3cret/"); + HttpResponse root = get(client, sut, "/"); + HttpResponse guess = get(client, sut, "/s3cret/x"); + HttpResponse columnWithoutSecret = get(client, sut, "/column/0"); + + // Then + assertThat(result.statusCode()).isEqualTo(200); + assertThat(result.body()).contains("hello"); + assertThat(result.headers().firstValue("Content-Type")).hasValue("text/html; charset=utf-8"); + assertThat(root.statusCode()).isEqualTo(404); + assertThat(guess.statusCode()).isEqualTo(404); + assertThat(columnWithoutSecret.statusCode()).isEqualTo(404); + } finally { + sut.stop(0); + } + } + + @Test + void serve_columnPathReturnsThatColumnsAnalysis() throws IOException, InterruptedException { + // Given + HttpServer sut = ExploreCommand.serve("", "/s/", 2, index -> "

column " + index + "

"); + try (HttpClient client = HttpClient.newHttpClient()) { + // When + HttpResponse result = get(client, sut, "/s/column/1"); + + // Then + assertThat(result.statusCode()).isEqualTo(200); + assertThat(result.body()).isEqualTo("

column 1

"); + } finally { + sut.stop(0); + } + } + + @ParameterizedTest + @ValueSource(strings = {"/s/column/2", "/s/column/-1", "/s/column/", "/s/column/x", "/s/column/99999999999"}) + void serve_columnOutOfRangeOrMalformed_is404(String path) throws IOException, InterruptedException { + // Given — the index comes from the request: anything but a column the file has must not + // reach the analysis (an out-of-range index would be an IndexOutOfBoundsException there) + HttpServer sut = ExploreCommand.serve("", "/s/", 2, index -> "column " + index); + try (HttpClient client = HttpClient.newHttpClient()) { + // When + HttpResponse result = get(client, sut, path); + + // Then + assertThat(result.statusCode()).isEqualTo(404); + } finally { + sut.stop(0); + } + } + + @Test + void serve_analyzesColumnsConcurrently() throws IOException, InterruptedException { + // Given — each analysis waits until both are running: on the server's default single + // thread the first would wait forever for the second, and "Analyze all" would run one + // column at a time however many readers there are + CountDownLatch bothRunning = new CountDownLatch(2); + HttpServer sut = ExploreCommand.serve("", "/s/", 2, index -> { + bothRunning.countDown(); + try { + return bothRunning.await(10, TimeUnit.SECONDS) ? "together" : "alone"; + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return "interrupted"; + } + }); + try (HttpClient client = HttpClient.newHttpClient()) { + // When + CompletableFuture> first = client.sendAsync(request(sut, "/s/column/0"), + HttpResponse.BodyHandlers.ofString()); + CompletableFuture> second = client.sendAsync(request(sut, "/s/column/1"), + HttpResponse.BodyHandlers.ofString()); + List result = List.of(first.join().body(), second.join().body()); + + // Then + assertThat(result).containsExactly("together", "together"); + } finally { + sut.stop(0); + } + } + + @Test + void serve_failedAnalysisIs500WithTheReason() throws IOException, InterruptedException { + // Given — a malformed segment surfaces when the column is decoded, long after the page loaded + HttpServer sut = ExploreCommand.serve("", "/s/", 1, index -> { + throw new IllegalStateException("bad segment"); + }); + try (HttpClient client = HttpClient.newHttpClient()) { + // When + HttpResponse result = get(client, sut, "/s/column/0"); + + // Then — the server keeps serving; the page shows the reason in place of the chart + assertThat(result.statusCode()).isEqualTo(500); + assertThat(result.body()).contains("bad segment"); + assertThat(result.headers().firstValue("Content-Type")).hasValue("text/plain; charset=utf-8"); + } finally { + sut.stop(0); + } + } + + @Test + void serve_bindsLoopbackOnly() throws IOException { + // Given / When + HttpServer result = ExploreCommand.serve("", "/x/", 0, index -> ""); + + // Then — never reachable from another machine on the network + try { + assertThat(result.getAddress().getAddress().isLoopbackAddress()).isTrue(); + } finally { + result.stop(0); + } + } + + private static HttpResponse get(HttpClient client, HttpServer server, String path) + throws IOException, InterruptedException { + return client.send(request(server, path), HttpResponse.BodyHandlers.ofString()); + } + + private static HttpRequest request(HttpServer server, String path) { + return HttpRequest.newBuilder(URI.create("http://127.0.0.1:" + server.getAddress().getPort() + path)).build(); + } +} diff --git a/docs/how-to.md b/docs/how-to.md index 54a64bb53..d650e056c 100644 --- a/docs/how-to.md +++ b/docs/how-to.md @@ -66,6 +66,51 @@ java -jar cli/target/vortex-cli-*-all.jar stats data.vortex --- +## Explore in the browser + +`explore` is the data-first view, for asking what is in a file rather than how it is stored. By +default (`--serve`) it serves a page on a local port and opens it in your browser, and runs until +you press Ctrl-C. + +```bash +java -jar cli/target/vortex-cli-*-all.jar explore data.vortex +# serving http://127.0.0.1:49605/b3a1f2.../ - press Ctrl-C to stop +``` + +To publish or send the page instead, `--html` analyzes every column up front and writes one +self-contained file, nothing to fetch and no server needed: + +```bash +java -jar cli/target/vortex-cli-*-all.jar explore --html report.html data.vortex +# written: report.html (19 columns analyzed, 111.4 KB) +``` + +The page lists every column with its share of nulls, its min and max, and a sparkline of each +zone's min-max band from the first row to the last. The sparkline shows whether a column is sorted +(a rising band), clustered or random, which is also how well a filter on it prunes zones. Everything +comes from the statistics the file already records and no data is decoded, so a 2.3 GB, +100-million-row file opens in well under a second. A mean appears only for files that record a sum: Rust's zone map and array +stats carry none. + +**Analyze** on a column (or **Analyze all**) reads its values once and adds, under its row, the +non-null count, the total and mean of a numeric column, the number of distinct values, a string +column's byte lengths, and how the values are spread: + +- a column with at most 30 distinct values (a status, a country, a flag) shows each value with its + count and share +- a numeric, date or timestamp column with more shows a 30-bin histogram between its min and max +- a string column with more shows its 10 most frequent values, and past 10,000 distinct values none + +This one decodes the column: on a 2.3 GB, 100-million-row file each column takes one to two seconds. +Columns are analyzed in parallel, one per core, so **Analyze all** on that file's nine columns takes +about as long as the slowest one: 2.2 s on 10 cores against 9-11 s one after another. + +The page is reachable only from this machine (`127.0.0.1`) and only under the random path printed +at start-up. The file can also be an `http(s)://` URL, as for `inspect`; analyzing a column then +downloads it. + +--- + ## Inspect as a web page `inspect --html` writes a single self-contained HTML page to stdout — no scripts, no CDN, diff --git a/docs/reference.md b/docs/reference.md index 30ac2caba..afb3849e4 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -746,6 +746,7 @@ java -jar cli/target/vortex-cli-*-all.jar [args] | Subcommand | Syntax | Description | |------------|------------------------------------------------|--------------------------------------------------| | `inspect` | `inspect [--html] ` | Layout tree, encodings, row counts, buffer sizes; `--html` emits a self-contained HTML report on stdout | +| `explore` | `explore [--serve \| --html ] ` | Per-column nulls, min/max and zone sparklines from the recorded statistics. `--serve` (default) serves them on `127.0.0.1` under a random path and opens the page in the browser, where **Analyze** reads a column for its total, mean, distinct count and value counts or histogram; `--html` writes one self-contained page with every column already analyzed; see [Explore in the browser](how-to.md#explore-in-the-browser) | | `tui` | `tui ` | Interactive layout-tree browser (lazy stats + data) | | `view` | `view ` | Interactive spreadsheet-grid browser over the row data | | `schema` | `schema ` | Column names and types | diff --git a/inspector/src/main/java/io/github/dfa1/vortex/inspect/ByteSize.java b/inspector/src/main/java/io/github/dfa1/vortex/inspect/ByteSize.java index 513d0b04b..82bf02a3a 100644 --- a/inspector/src/main/java/io/github/dfa1/vortex/inspect/ByteSize.java +++ b/inspector/src/main/java/io/github/dfa1/vortex/inspect/ByteSize.java @@ -1,29 +1,30 @@ package io.github.dfa1.vortex.inspect; +import java.util.List; import java.util.Locale; /// Formats a byte count as a short human-readable string using binary (1024-based) units. public final class ByteSize { - private static final long KIB = 1024L; - private static final long MIB = KIB * KIB; + /// Every unit a `long` can reach: its maximum is just under 8 EiB, so no byte count runs past + /// the end of the list. + private static final List UNITS = List.of("KB", "MB", "GB", "TB", "PB", "EB"); private ByteSize() { } - /// Formats `bytes` as `B` below 1 KiB, `KB` below 1 MiB, otherwise `MB`, with one decimal - /// place for the KB/MB forms. Uses [Locale#ROOT] so the decimal separator is stable regardless - /// of the default locale. + /// Formats `bytes` as `B` below 1 KiB, else in the largest unit (`KB` up to `EB`) it is at least + /// one of, with one decimal place. Uses [Locale#ROOT] so the decimal separator is stable + /// regardless of the default locale. /// /// @param bytes the byte count - /// @return a human-readable size such as `512 B`, `1.5 KB`, or `2.0 MB` + /// @return a human-readable size such as `512 B`, `1.5 KB`, `2.0 MB` or `2.1 GB` public static String format(long bytes) { - if (bytes < KIB) { + if (bytes < 1024) { return bytes + " B"; } - if (bytes < MIB) { - return String.format(Locale.ROOT, "%.1f KB", bytes / (double) KIB); - } - return String.format(Locale.ROOT, "%.1f MB", bytes / (double) MIB); + // Each unit is 10 more bits: the highest set bit picks the unit, with no table of bounds. + int unit = (Long.SIZE - 1 - Long.numberOfLeadingZeros(bytes)) / 10; + return String.format(Locale.ROOT, "%.1f %s", bytes / (double) (1L << (10 * unit)), UNITS.get(unit - 1)); } } diff --git a/inspector/src/main/java/io/github/dfa1/vortex/inspect/ColumnDistribution.java b/inspector/src/main/java/io/github/dfa1/vortex/inspect/ColumnDistribution.java new file mode 100644 index 000000000..79c95d839 --- /dev/null +++ b/inspector/src/main/java/io/github/dfa1/vortex/inspect/ColumnDistribution.java @@ -0,0 +1,383 @@ +package io.github.dfa1.vortex.inspect; + +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.reader.Chunk; +import io.github.dfa1.vortex.reader.ScanFilter; +import io.github.dfa1.vortex.reader.ScanIterator; +import io.github.dfa1.vortex.reader.ScanOptions; +import io.github.dfa1.vortex.reader.VortexHandle; +import io.github.dfa1.vortex.reader.array.Array; +import io.github.dfa1.vortex.reader.array.BoolArray; +import io.github.dfa1.vortex.reader.array.ByteArray; +import io.github.dfa1.vortex.reader.array.DoubleArray; +import io.github.dfa1.vortex.reader.array.FloatArray; +import io.github.dfa1.vortex.reader.array.IntArray; +import io.github.dfa1.vortex.reader.array.LongArray; +import io.github.dfa1.vortex.reader.array.MaskedArray; +import io.github.dfa1.vortex.reader.array.NullArray; +import io.github.dfa1.vortex.reader.array.ShortArray; +import io.github.dfa1.vortex.reader.array.VarBinArray; +import io.github.dfa1.vortex.reader.compute.Compute; +import io.github.dfa1.vortex.reader.compute.FilteredAggregate; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.LongToDoubleFunction; + +/// What one full pass over a column's values tells that its recorded statistics cannot: its total, +/// how its values are spread, and for a string column how long they are. Unlike [ColumnProfile] it +/// decodes the column, so it is computed on demand, one column at a time. +/// +/// The spread is a value count when the column has few distinct values (a status, a country, a +/// flag), else a histogram for a numeric column, else the most frequent values. A column with more +/// than [#MAX_DISTINCT] distinct values is not counted at all: past that, a count says nothing a +/// histogram does not, and holding it costs memory per value. +/// +/// @param rows rows scanned +/// @param nulls null values among them +/// @param sum the total of a numeric column (`Long` for integers, `Double` for floats), or `null` +/// for a non-numeric column or an integer total past the `long` range +/// @param distinct distinct non-null values, or `null` when there are more than [#MAX_DISTINCT] +/// @param bars the spread: value counts or histogram bins, in display order +/// @param histogram `true` when `bars` are numeric bins rather than individual values +/// @param lengths byte lengths of a string column's non-null values, or `null` for other columns +public record ColumnDistribution( + long rows, + long nulls, + Number sum, + Long distinct, + List bars, + boolean histogram, + Lengths lengths) { + + /// Distinct values counted before a column counts as high-cardinality. + public static final int MAX_DISTINCT = 10_000; + + /// A column with at most this many distinct values shows every one of them; above it a numeric + /// column shows a histogram and any other the [#TOP_VALUES] most frequent. + static final int MAX_CATEGORIES = 30; + + /// Most frequent values shown for a column with too many distinct values to list. + static final int TOP_VALUES = 10; + + /// Bins in a histogram. + static final int BINS = 30; + + /// Matches every row: the aggregate kernels take a filter, and an empty `AND` holds for all. + private static final ScanFilter EVERY_ROW = new ScanFilter.And(List.of()); + + /// One bar of the spread. + /// + /// @param value the value counted, or a histogram bin's lower bound (a `Double`) + /// @param upper a histogram bin's upper bound (a `Double`), or `null` for a value count + /// @param count rows holding the value, or falling in the bin + public record Bar(Object value, Object upper, long count) { + } + + /// Byte lengths of a string column's values. + /// + /// @param min shortest value + /// @param max longest value + /// @param total bytes over all values, for the average + public record Lengths(long min, long max, long total) { + } + + /// The mean of the non-null values. + /// + /// @return `sum / (rows - nulls)`, or `null` without a sum or with no non-null value + public Double mean() { + long present = rows - nulls; + return sum == null || present <= 0 ? null : sum.doubleValue() / present; + } + + /// Scans `profile`'s column once. A histogram's bins span the profile's min and max; a numeric + /// column whose file records no exact min/max (an older write of a timestamp, say) is scanned + /// once more first, to find them. + /// + /// @param handle an open file; must be used from the thread that opened it + /// @param profile the column's recorded statistics, from [ColumnProfile#of(VortexHandle)] + /// @return the column's distribution + public static ColumnDistribution of(VortexHandle handle, ColumnProfile profile) { + ColumnName name = ColumnName.of(profile.name()); + ColumnProfile bounded = profile; + if (isNumeric(profile.type()) && !(profile.min() instanceof Number && profile.max() instanceof Number)) { + bounded = withScannedRange(handle, profile, name); + } + Accumulator acc = new Accumulator(bounded); + try (ScanIterator scan = handle.scan(ScanOptions.columns(name))) { + while (scan.hasNext()) { + try (Chunk chunk = scan.next()) { + acc.add(chunk, name); + } + } + } + return acc.result(); + } + + private static boolean isNumeric(DType type) { + return type instanceof DType.Primitive + || type instanceof DType.Extension ext && ext.storageDType() instanceof DType.Primitive; + } + + /// The profile with its min/max replaced by the column's scanned extremes, or unchanged when + /// the column holds no non-null, non-`NaN` value. + private static ColumnProfile withScannedRange(VortexHandle handle, ColumnProfile profile, ColumnName name) { + double[] range = {Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY}; + try (ScanIterator scan = handle.scan(ScanOptions.columns(name))) { + while (scan.hasNext()) { + try (Chunk chunk = scan.next()) { + MaskedArray.Unwrapped split = MaskedArray.unwrap(chunk.column(name)); + LongToDoubleFunction value = Accumulator.numericReader(split.inner()); + BoolArray validity = split.validity(); + for (long i = 0; value != null && i < chunk.rowCount(); i++) { + double v = value.applyAsDouble(i); + // NaN would poison min/max; the histogram leaves it out too. + if ((validity == null || validity.getBoolean(i)) && v == v) { + range[0] = Math.min(range[0], v); + range[1] = Math.max(range[1], v); + } + } + } + } + } + if (!(range[0] <= range[1])) { + return profile; + } + return new ColumnProfile(profile.name(), profile.type(), profile.rows(), profile.nullCount(), + profile.nanCount(), range[0], range[1], profile.sum(), profile.zones()); + } + + /// Picks the bars to show from what the scan gathered. + /// + /// @param counts rows per distinct non-null value, or `null` when there were too many to count + /// @param bins rows per histogram bin, or `null` when the column has no histogram + /// @param lo the first bin's lower bound + /// @param hi the last bin's upper bound + /// @return the bars, and whether they are histogram bins + static Map.Entry, Boolean> bars(Map counts, long[] bins, double lo, double hi) { + if (counts != null && counts.size() <= MAX_CATEGORIES) { + List bars = new ArrayList<>(counts.size()); + counts.forEach((value, count) -> bars.add(new Bar(value, null, count))); + // Numbers, dates and flags read in their own order; names read most frequent first. + if (counts.keySet().stream().allMatch(value -> value instanceof Number || value instanceof Boolean)) { + bars.sort(Comparator.comparing(Bar::value, ColumnDistribution::compareKeys)); + } else { + bars.sort(Comparator.comparingLong(Bar::count).reversed()); + } + return Map.entry(List.copyOf(bars), false); + } + if (bins != null) { + List bars = new ArrayList<>(bins.length); + double width = (hi - lo) / bins.length; + for (int i = 0; i < bins.length; i++) { + bars.add(new Bar(lo + i * width, i == bins.length - 1 ? hi : lo + (i + 1) * width, bins[i])); + } + return Map.entry(List.copyOf(bars), true); + } + if (counts != null) { + List top = counts.entrySet().stream() + .sorted(Map.Entry.comparingByValue().reversed()) + .limit(TOP_VALUES) + .map(entry -> new Bar(entry.getKey(), null, entry.getValue())) + .toList(); + return Map.entry(top, false); + } + return Map.entry(List.of(), false); + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private static int compareKeys(Object a, Object b) { + if (a instanceof Number x && b instanceof Number y) { + return Double.compare(x.doubleValue(), y.doubleValue()); + } + return ((Comparable) a).compareTo(b); + } + + /// The running state of one scan. + private static final class Accumulator { + + private final DType type; + private final boolean summable; + private Map counts = new HashMap<>(); + private final long[] bins; + private final double lo; + private final double hi; + private final double scale; + private long rows; + private long nulls; + private Number sum; + private boolean summed; + private long[] lengths; + + Accumulator(ColumnProfile profile) { + this.type = profile.type(); + this.summable = type instanceof DType.Primitive; + if (isNumeric(type) && profile.min() instanceof Number min && profile.max() instanceof Number max) { + this.bins = new long[BINS]; + this.lo = min.doubleValue(); + this.hi = max.doubleValue(); + // A constant column puts everything in the first bin rather than dividing by zero. + this.scale = hi > lo ? BINS / (hi - lo) : 0; + } else { + this.bins = null; + this.lo = 0; + this.hi = 0; + this.scale = 0; + } + } + + void add(Chunk chunk, ColumnName name) { + long n = chunk.rowCount(); + rows += n; + Array column = chunk.column(name); + if (column instanceof NullArray) { + nulls += n; + return; + } + MaskedArray.Unwrapped split = MaskedArray.unwrap(column); + BoolArray validity = split.validity(); + nulls += validity == null ? 0 : n - trueCount(validity, n); + if (summable) { + Number part = Compute.filteredAggregate(chunk, EVERY_ROW, name).sum(); + sum = summed ? addSum(sum, part) : part; + summed = true; + } + if (counts != null) { + count(chunk, name, split.inner(), validity); + } + if (bins != null) { + LongToDoubleFunction value = numericReader(split.inner()); + if (value != null) { + histogram(value, validity, n); + } + } + if (split.inner() instanceof VarBinArray strings) { + lengths(strings, validity, n); + } + } + + ColumnDistribution result() { + Map.Entry, Boolean> bars = bars(counts, bins, lo, hi); + Lengths stringLengths = null; + if (lengths != null && lengths[0] != Long.MAX_VALUE) { + stringLengths = new Lengths(lengths[0], lengths[1], lengths[2]); + } + return new ColumnDistribution(rows, nulls, sum, counts == null ? null : (long) counts.size(), + bars.getKey(), bars.getValue(), stringLengths); + } + + /// Counts through the grouping kernel, which reads dictionary codes rather than decoding + /// every row; a plain (non-dictionary) string chunk is counted here instead. A float column + /// is not counted: its values are a histogram's job. + private void count(Chunk chunk, ColumnName name, Array data, BoolArray validity) { + Optional> groups = Compute.groupedAggregate(chunk, name, null, null); + if (groups.isPresent()) { + groups.get().forEach((value, group) -> { + if (value != null) { + counts.merge(value, group.selectedRows(), Long::sum); + } + }); + } else if (data instanceof VarBinArray strings) { + for (long i = 0; i < strings.length() && counts.size() <= MAX_DISTINCT; i++) { + if (validity == null || validity.getBoolean(i)) { + counts.merge(strings.getString(i), 1L, Long::sum); + } + } + } else { + counts = null; + return; + } + if (counts.size() > MAX_DISTINCT) { + counts = null; + } + } + + /// Branch-split on validity so the common no-null chunk runs a uniform loop. + private void histogram(LongToDoubleFunction value, BoolArray validity, long n) { + if (validity == null) { + for (long i = 0; i < n; i++) { + bin(value.applyAsDouble(i)); + } + } else { + for (long i = 0; i < n; i++) { + if (validity.getBoolean(i)) { + bin(value.applyAsDouble(i)); + } + } + } + } + + private void bin(double v) { + // NaN sits outside every bin; min/max skip it too. + if (v == v) { + bins[Math.clamp((long) ((v - lo) * scale), 0, BINS - 1)]++; + } + } + + private void lengths(VarBinArray strings, BoolArray validity, long n) { + if (lengths == null) { + lengths = new long[] {Long.MAX_VALUE, 0, 0}; + } + for (long i = 0; i < n; i++) { + if (validity == null || validity.getBoolean(i)) { + int length = strings.getByteLength(i); + lengths[0] = Math.min(lengths[0], length); + lengths[1] = Math.max(lengths[1], length); + lengths[2] += length; + } + } + } + + private static long trueCount(BoolArray validity, long n) { + long valid = 0; + for (long i = 0; i < n; i++) { + if (validity.getBoolean(i)) { + valid++; + } + } + return valid; + } + + /// Resolves the typed accessor once per chunk, so the histogram loop calls one monomorphic + /// function rather than switching on the array type per row. + private static LongToDoubleFunction numericReader(Array data) { + boolean unsigned = data.dtype().isUnsigned(); + return switch (data) { + case LongArray a when unsigned -> i -> unsignedToDouble(a.getLong(i)); + case LongArray a -> i -> a.getLong(i); + case IntArray a when unsigned -> i -> Integer.toUnsignedLong(a.getInt(i)); + case IntArray a -> i -> a.getInt(i); + case ShortArray a when unsigned -> i -> Short.toUnsignedInt(a.getShort(i)); + case ShortArray a -> i -> a.getShort(i); + case ByteArray a when unsigned -> i -> Byte.toUnsignedInt(a.getByte(i)); + case ByteArray a -> i -> a.getByte(i); + case DoubleArray a -> a::getDouble; + case FloatArray a -> a::getFloat; + default -> null; + }; + } + + private static double unsignedToDouble(long v) { + return v >= 0 ? v : (v >>> 1) * 2.0 + (v & 1); + } + + private static Number addSum(Number total, Number part) { + if (total == null || part == null) { + return null; + } + if (total instanceof Long a && part instanceof Long b) { + try { + return Math.addExact(a, b); + } catch (ArithmeticException _) { + return null; + } + } + return total.doubleValue() + part.doubleValue(); + } + } +} diff --git a/inspector/src/main/java/io/github/dfa1/vortex/inspect/ColumnProfile.java b/inspector/src/main/java/io/github/dfa1/vortex/inspect/ColumnProfile.java new file mode 100644 index 000000000..ee1465829 --- /dev/null +++ b/inspector/src/main/java/io/github/dfa1/vortex/inspect/ColumnProfile.java @@ -0,0 +1,153 @@ +package io.github.dfa1.vortex.inspect; + +import io.github.dfa1.vortex.core.model.ColumnName; +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.reader.ArrayStats; +import io.github.dfa1.vortex.reader.ScanIterator; +import io.github.dfa1.vortex.reader.ScanOptions; +import io.github.dfa1.vortex.reader.VortexHandle; +import io.github.dfa1.vortex.reader.Zone; + +import java.util.ArrayList; +import java.util.List; + +/// One top-level column's statistics, folded from what the file already records - its zone map, +/// or each chunk's embedded stats when there is none. No data segment is decoded, so a profile +/// costs the same on a 10 GB file as on a 10 KB one, and over HTTP only the footer and the zone +/// tables are fetched. +/// +/// Every aggregate is either exact or `null`: a column-wide value is reported only when every zone +/// that holds a value carries it, and min/max only when each zone's bound is exact (Rust records a +/// string's zone bounds truncated, so a utf8 column usually has no min/max here). +/// +/// @param name column name +/// @param type column dtype +/// @param rows rows in the file +/// @param nullCount null values in the column, or `null` if a zone does not record it +/// @param nanCount `NaN` values in a float column, or `null` if unknown +/// @param min the column's exact minimum, or `null` if unknown +/// @param max the column's exact maximum, or `null` if unknown +/// @param sum the column's sum (`Long` for integer and bool columns, `Double` for floats), or `null` if unknown +/// @param zones the zone map in row order, empty when the column has none +public record ColumnProfile( + String name, + DType type, + long rows, + Long nullCount, + Long nanCount, + Object min, + Object max, + Object sum, + List zones) { + + /// Profiles every top-level column of the file. + /// + /// @param handle an open file + /// @return one profile per schema field in order; empty when the root dtype is not a struct + public static List of(VortexHandle handle) { + if (!(handle.dtype() instanceof DType.Struct schema)) { + return List.of(); + } + long rows = handle.layout().rowCount(); + List profiles = new ArrayList<>(schema.fieldNames().size()); + try (ScanIterator iter = handle.scan(ScanOptions.all())) { + for (int i = 0; i < schema.fieldNames().size(); i++) { + ColumnName name = schema.fieldNames().get(i); + ColumnProfile profile = fold(name.value(), schema.fieldTypes().get(i), rows, + iter.columnZoneStats(name), iter.columnZones(name)); + if (profile.min() == null || profile.max() == null || profile.sum() == null) { + profile = profile.orChunks(fold(name.value(), profile.type(), rows, iter.chunkStats(name), List.of())); + } + profiles.add(profile); + } + } + return List.copyOf(profiles); + } + + /// The mean of the non-null, non-`NaN` values: `sum / count`. For a bool column that is the + /// share of `true`. + /// + /// @return the mean, or `null` when the sum or the null count is unknown, or no value is present + public Double mean() { + if (!(sum instanceof Number total) || nullCount == null) { + return null; + } + long present = rows - nullCount - (nanCount == null ? 0 : nanCount); + return present <= 0 ? null : total.doubleValue() / present; + } + + /// Fills what the zone map left unknown from the chunks' own array stats: a `vortex.zoned` map + /// records no sum (Rust's aggregate set), and only truncated bounds for a string, while each + /// chunk's array stats carry both exactly. Null counts stay the zone map's. + private ColumnProfile orChunks(ColumnProfile chunks) { + return new ColumnProfile(name, type, rows, nullCount, nanCount, + min != null ? min : chunks.min(), + max != null ? max : chunks.max(), + sum != null ? sum : chunks.sum(), + zones); + } + + static ColumnProfile fold(String name, DType type, long rows, List stats, List zones) { + Long nulls = 0L; + Long nans = 0L; + Object min = null; + Object max = null; + Object sum = null; + boolean minKnown = true; + boolean maxKnown = true; + boolean sumKnown = true; + for (ArrayStats zone : stats) { + nulls = add(nulls, zone.nullCount()); + nans = add(nans, zone.nanCount()); + // Rust leaves min/max/sum absent on an all-null zone; that says nothing about the rest. + if (zone.min() == null && zone.max() == null && zone.sum() == null && zone.nullCount() != null + && zone.nullCount() > 0) { + continue; + } + minKnown &= zone.min() != null && zone.minExact(); + maxKnown &= zone.max() != null && zone.maxExact(); + sumKnown &= zone.sum() != null; + if (minKnown) { + min = VortexInspector.pickMin(min, zone.min()); + } + if (maxKnown) { + max = VortexInspector.pickMax(max, zone.max()); + } + if (sumKnown) { + sum = addSum(sum, zone.sum()); + sumKnown = sum != null; + } + } + boolean any = !stats.isEmpty(); + return new ColumnProfile(name, type, rows, + any ? nulls : null, + any ? nans : null, + any && minKnown ? min : null, + any && maxKnown ? max : null, + any && sumKnown ? sum : null, + zones); + } + + private static Long add(Long total, Long part) { + return total == null || part == null ? null : total + part; + } + + /// Integer sums stay exact and give up on overflow rather than wrap; anything else (float sums, + /// mixed types) adds as `double`. + private static Object addSum(Object total, Object part) { + if (total == null) { + return part; + } + if (total instanceof Long a && part instanceof Long b) { + try { + return Math.addExact(a, b); + } catch (ArithmeticException _) { + return null; + } + } + if (total instanceof Number a && part instanceof Number b) { + return a.doubleValue() + b.doubleValue(); + } + return null; + } +} diff --git a/inspector/src/main/java/io/github/dfa1/vortex/inspect/ExplorePage.java b/inspector/src/main/java/io/github/dfa1/vortex/inspect/ExplorePage.java new file mode 100644 index 000000000..4e868dc5d --- /dev/null +++ b/inspector/src/main/java/io/github/dfa1/vortex/inspect/ExplorePage.java @@ -0,0 +1,316 @@ +package io.github.dfa1.vortex.inspect; + +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.reader.ArrayStats; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +import static io.github.dfa1.vortex.inspect.HtmlReport.appendHead; +import static io.github.dfa1.vortex.inspect.HtmlReport.bar; +import static io.github.dfa1.vortex.inspect.HtmlReport.count; +import static io.github.dfa1.vortex.inspect.HtmlReport.escape; +import static io.github.dfa1.vortex.inspect.HtmlReport.fill; +import static io.github.dfa1.vortex.inspect.HtmlReport.percent; +import static io.github.dfa1.vortex.inspect.HtmlReport.readable; +import static io.github.dfa1.vortex.inspect.HtmlReport.stat; +import static io.github.dfa1.vortex.inspect.HtmlReport.value; + +/// The data-first page, for people who ask what is in a file rather than how it is stored: +/// per-column statistics and nothing about encodings, segments or bytes. [HtmlReport] is the +/// storage-first one. +/// +/// The page opens on the statistics the file records ([ColumnProfile]); each column's +/// distribution ([ColumnDistribution]) needs a pass over its values. Served live +/// ([#render(String, long, long, List)]), the page fetches it on demand from `column/` +/// relative to its own URL; published ([#render(String, long, long, List, List)]), every column's is +/// already in the page. Either way a distribution is the HTML fragment +/// [#distribution(ColumnProfile, ColumnDistribution)] renders, and every value in it is escaped here, +/// as on the page. The page's only script fetches fragments and shows or hides them. +public final class ExplorePage { + + private ExplorePage() { + } + + /// Renders the live page, which asks its server for each column's distribution on demand. + /// + /// @param title file name shown in the header; escaped before it reaches the page + /// @param fileSize size of the file in bytes + /// @param rows rows in the file + /// @param profiles per-column statistics, from [ColumnProfile#of(io.github.dfa1.vortex.reader.VortexHandle)] + /// @return a complete HTML document + public static String render(String title, long fileSize, long rows, List profiles) { + return render(title, fileSize, rows, profiles, List.of()); + } + + /// Renders the page with every column's distribution already in it: one file that needs no + /// server, to publish or send. + /// + /// @param title file name shown in the header; escaped before it reaches the page + /// @param fileSize size of the file in bytes + /// @param rows rows in the file + /// @param profiles per-column statistics, from [ColumnProfile#of(io.github.dfa1.vortex.reader.VortexHandle)] + /// @param analyses each column's fragment from [#distribution(ColumnProfile, ColumnDistribution)], in + /// column order; empty for the live page + /// @return a complete HTML document + public static String render(String title, long fileSize, long rows, List profiles, + List analyses) { + if (!analyses.isEmpty() && analyses.size() != profiles.size()) { + throw new IllegalArgumentException(analyses.size() + " analyses for " + profiles.size() + " columns"); + } + StringBuilder sb = new StringBuilder(16 * 1024); + appendHead(sb, title + " - Vortex explorer"); + sb.append("
\n

").append(escape(title)).append("

\n
\n"); + stat(sb, "Rows", count(rows)); + stat(sb, "Columns", count(profiles.size())); + stat(sb, "Size", ByteSize.format(fileSize)); + sb.append("
\n
\n"); + appendProfiles(sb, profiles, analyses); + sb.append("\n\n\n\n"); + return sb.toString(); + } + + /// Renders one column's distribution as the fragment the page inserts under the column's row. + /// + /// @param profile the column's recorded statistics + /// @param distribution the column's distribution, from [ColumnDistribution#of(io.github.dfa1.vortex.reader.VortexHandle, ColumnProfile)] + /// @return an HTML fragment + public static String distribution(ColumnProfile profile, ColumnDistribution distribution) { + StringBuilder sb = new StringBuilder(4 * 1024); + DType type = profile.type(); + List facts = new ArrayList<>(); + facts.add(count(distribution.rows() - distribution.nulls()) + " values"); + facts.add(count(distribution.nulls()) + " null (" + percent(distribution.nulls(), distribution.rows()) + ")"); + if (distribution.sum() != null) { + facts.add("total " + number(distribution.sum().doubleValue()) + ""); + } + if (distribution.mean() != null) { + facts.add("mean " + number(distribution.mean()) + ""); + } + if (distribution.distinct() == null) { + facts.add("more than " + count(ColumnDistribution.MAX_DISTINCT) + " distinct"); + } else { + facts.add(count(distribution.distinct()) + " distinct"); + } + ColumnDistribution.Lengths lengths = distribution.lengths(); + if (lengths != null) { + long present = distribution.rows() - distribution.nulls(); + facts.add(count(lengths.min()) + "-" + count(lengths.max()) + " bytes long, " + + number((double) lengths.total() / present) + " on average"); + } + sb.append("
\n

").append(String.join(" · ", facts)).append("

\n"); + List bars = distribution.bars(); + if (distribution.histogram()) { + appendHistogram(sb, bars, type); + } else if (!bars.isEmpty()) { + appendCounts(sb, bars, type, distribution.rows() - distribution.nulls(), + distribution.distinct() != null && distribution.distinct() > bars.size()); + } + sb.append("
\n"); + return sb.toString(); + } + + // ---------------------------------------------------------------- statistics table + + /// What a reader of the data asks first - how complete is each column, what range does it span, + /// where does it sit on average - answered from the file's recorded statistics alone. + private static void appendProfiles(StringBuilder sb, List profiles, List analyses) { + // Only files that record a sum have a mean (Rust's zone map and array stats carry none), + // so the column is left out rather than shown as a row of dashes. + boolean withMean = profiles.stream().anyMatch(profile -> profile.mean() != null); + boolean analyzed = !analyses.isEmpty(); + sb.append("
\n
Columns"); + if (analyzed) { + sb.append("every column analyzed
\n"); + } else { + sb.append("statistics recorded in the file, nothing decoded") + .append("\n"); + } + sb.append("
\n\n") + .append("") + .append(withMean ? "" : "") + .append("") + .append("\n\n"); + for (int i = 0; i < profiles.size(); i++) { + ColumnProfile profile = profiles.get(i); + sb.append(""); + if (withMean) { + sb.append(""); + } + sb.append(""); + if (analyzed) { + // Same markup the live page's script leaves after an analysis, so its toggle works here too. + sb.append("\n") + .append("\n"); + } else { + sb.append("\n"); + } + } + sb.append("\n
columnnull %minmaxmeanrange over rows
") + .append(escape(profile.name())).append(" ") + .append(escape(VortexInspector.formatDType(profile.type()))).append(""); + if (profile.nullCount() == null) { + sb.append("-"); + } else { + bar(sb, profile.nullCount(), profile.rows(), "var(--meta)"); + sb.append(percent(profile.nullCount(), profile.rows())); + } + sb.append("").append(value(profile.min(), profile.type())) + .append("").append(value(profile.max(), profile.type())).append("").append(mean(profile)).append("").append(sparkline(profile, fill(i))).append("
") + .append(analyses.get(i)).append("
\n
\n
\n"); + } + + /// The mean, only where it reads as a number: a bool column's is its share of `true`; an + /// extension's (a date, a timestamp) is an epoch average nobody asked for, so it is left out. + private static String mean(ColumnProfile profile) { + Double mean = profile.mean(); + if (mean == null) { + return "-"; + } + return switch (profile.type()) { + case DType.Bool _ -> String.format(Locale.ROOT, "%.1f%% true", 100 * mean); + case DType.Primitive _ -> "" + number(mean) + ""; + default -> "-"; + }; + } + + /// Upper bound on points in a sparkline; consecutive zones merge (min of mins, max of maxes). + private static final int MAX_SPARK_POINTS = 120; + + /// Each zone's min-max band across the rows, as an inline SVG. Shows at a glance whether a + /// column is sorted (a rising band), clustered (narrow bands) or random (every band spans the + /// whole range) - which is also how well a filter on it prunes zones. + private static String sparkline(ColumnProfile profile, String fill) { + List points = new ArrayList<>(); + int stride = Math.max(1, (profile.zones().size() + MAX_SPARK_POINTS - 1) / MAX_SPARK_POINTS); + for (int i = 0; i < profile.zones().size(); i += stride) { + double lo = Double.POSITIVE_INFINITY; + double hi = Double.NEGATIVE_INFINITY; + for (int j = i; j < Math.min(i + stride, profile.zones().size()); j++) { + ArrayStats stats = profile.zones().get(j).stats(); + if (stats.min() instanceof Number min && stats.max() instanceof Number max) { + lo = Math.min(lo, min.doubleValue()); + hi = Math.max(hi, max.doubleValue()); + } + } + if (lo <= hi) { + points.add(new double[] {lo, hi}); + } + } + if (points.size() < 2) { + return "-"; + } + double lo = points.stream().mapToDouble(p -> p[0]).min().orElse(0); + double hi = points.stream().mapToDouble(p -> p[1]).max().orElse(0); + double span = hi > lo ? hi - lo : 1; + StringBuilder top = new StringBuilder(); + StringBuilder bottom = new StringBuilder(); + for (int i = 0; i < points.size(); i++) { + double x = 100.0 * i / (points.size() - 1); + top.append(String.format(Locale.ROOT, "%.2f,%.2f ", x, 22 - 20 * (points.get(i)[1] - lo) / span)); + bottom.insert(0, String.format(Locale.ROOT, "%.2f,%.2f ", x, 22 - 20 * (points.get(i)[0] - lo) / span)); + } + return "" + + "zone min-max over " + count(profile.zones().size()) + " zones" + + ""; + } + + // ---------------------------------------------------------------- distribution + + /// A column chart over the bins, its range labeled underneath; hovering a bar names its bin. + private static void appendHistogram(StringBuilder sb, List bars, DType type) { + long tallest = bars.stream().mapToLong(ColumnDistribution.Bar::count).max().orElse(1L); + double width = 300.0 / bars.size(); + sb.append("") + .append("histogram of ").append(bars.size()).append(" equal-width bins"); + for (int i = 0; i < bars.size(); i++) { + ColumnDistribution.Bar bin = bars.get(i); + double height = tallest == 0 ? 0 : 78.0 * bin.count() / tallest; + sb.append(String.format(Locale.ROOT, "", + i * width + 0.5, 80 - height, width - 1, height)) + .append("").append(escape(bound(bin.value(), type))).append(" to ") + .append(escape(bound(bin.upper(), type))).append(": ").append(count(bin.count())) + .append(" rows"); + } + sb.append("\n
").append(escape(bound(bars.getFirst().value(), type))) + .append("").append(escape(bound(bars.getLast().upper(), type))).append("
\n"); + } + + /// One bar per value, longest first or in value order, each with its count and share. + private static void appendCounts(StringBuilder sb, List bars, DType type, long present, + boolean top) { + if (top) { + sb.append("

most frequent ").append(bars.size()).append(":

\n"); + } + long widest = bars.stream().mapToLong(ColumnDistribution.Bar::count).max().orElse(1L); + sb.append("\n\n"); + for (ColumnDistribution.Bar bar : bars) { + sb.append("\n"); + } + sb.append("\n
").append(value(bar.value(), type)).append(""); + bar(sb, bar.count(), widest, "var(--series-1)"); + sb.append("").append(count(bar.count())).append("") + .append(percent(bar.count(), present)).append("
\n"); + } + + /// A histogram bound as the column reads: a date or timestamp bin edge shown as a date, not an + /// epoch number. + private static String bound(Object raw, DType type) { + if (type instanceof DType.Extension && raw instanceof Double d) { + return String.valueOf(readable(Math.round(d), type)); + } + return raw instanceof Double d ? number(d) : String.valueOf(raw); + } + + /// A whole number without decimals (a total, an integer bin edge), else two decimals, or four + /// significant digits below one. + private static String number(double value) { + if (value == Math.rint(value) && Math.abs(value) < 1e15) { + return String.format(Locale.ROOT, "%,.0f", value); + } + return String.format(Locale.ROOT, Math.abs(value) >= 1 ? "%,.2f" : "%.4g", value); + } + + /// Fetches a column's distribution into a row under it, then turns the button into a Hide/Show + /// toggle for that row; "Analyze all" asks for every column not analyzed yet at once, and the + /// server analyzes as many at a time as it has readers. + private static final String SCRIPT = """ + async function analyze(button) { + button.dataset.state = 'running'; + button.disabled = true; + button.textContent = 'Analyzing...'; + const row = button.closest('tr'); + const response = await fetch('column/' + button.dataset.column); + const detail = document.createElement('tr'); + detail.className = 'detail'; + const cell = document.createElement('td'); + cell.colSpan = row.children.length; + if (response.ok) { + cell.innerHTML = await response.text(); + } else { + cell.textContent = 'failed: ' + response.status + ' ' + await response.text(); + } + detail.append(cell); + row.after(detail); + button.dataset.state = 'done'; + button.disabled = false; + button.textContent = 'Hide'; + } + function toggle(button) { + const detail = button.closest('tr').nextElementSibling; + detail.hidden = !detail.hidden; + button.textContent = detail.hidden ? 'Show' : 'Hide'; + } + document.addEventListener('click', event => { + const button = event.target.closest('button[data-column]'); + if (!button) { return; } + if (button.dataset.state === 'done') { toggle(button); } else if (!button.dataset.state) { analyze(button); } + }); + document.getElementById('analyze-all')?.addEventListener('click', event => { + event.target.disabled = true; + document.querySelectorAll('button[data-column]:not([data-state])').forEach(analyze); + }); + """; +} diff --git a/inspector/src/main/java/io/github/dfa1/vortex/inspect/HtmlReport.java b/inspector/src/main/java/io/github/dfa1/vortex/inspect/HtmlReport.java index 8e075603a..ba64e3688 100644 --- a/inspector/src/main/java/io/github/dfa1/vortex/inspect/HtmlReport.java +++ b/inspector/src/main/java/io/github/dfa1/vortex/inspect/HtmlReport.java @@ -62,10 +62,7 @@ private HtmlReport() { public static String render(InspectorTree tree, String title) { List columns = columns(tree); StringBuilder sb = new StringBuilder(64 * 1024); - sb.append("\n\n\n\n") - .append("\n") - .append("").append(escape(title)).append(" - Vortex inspector\n") - .append("\n\n\n
\n"); + appendHead(sb, title + " - Vortex inspector"); appendHeader(sb, tree, columns, title); appendStrip(sb, tree, columns); sb.append("
\n"); @@ -77,6 +74,13 @@ public static String render(InspectorTree tree, String title) { return sb.toString(); } + static void appendHead(StringBuilder sb, String title) { + sb.append("\n\n\n\n") + .append("\n") + .append("").append(escape(title)).append("\n") + .append("\n\n\n
\n"); + } + // ---------------------------------------------------------------- sections private static void appendHeader(StringBuilder sb, InspectorTree tree, List columns, String title) { @@ -501,7 +505,7 @@ private static long spanBytes(List columns, RowSpan span) { // ---------------------------------------------------------------- fragments - private static void stat(StringBuilder sb, String label, String value) { + static void stat(StringBuilder sb, String label, String value) { sb.append("
").append(label).append("
").append(value).append("
\n"); } @@ -516,7 +520,7 @@ private static void block(StringBuilder sb, long fileSize, long offset, long len .append(";background:").append(fill).append("\" title=\"").append(escape(tip)).append("\">\n"); } - private static void bar(StringBuilder sb, long value, long max, String fill) { + static void bar(StringBuilder sb, long value, long max, String fill) { sb.append(""); } @@ -531,7 +535,7 @@ private static String ratio(ColumnView column) { return factor < 1.05 ? "" : String.format(Locale.ROOT, "%.1fx", factor); } - private static String fill(int column) { + static String fill(int column) { return column < 0 || column >= LIGHT_SERIES.size() ? "var(--other)" : "var(--series-" + (column + 1) + ")"; @@ -542,11 +546,11 @@ private static String width(long value, long total) { return String.format(Locale.ROOT, "%.4f%%", pct); } - private static String percent(long value, long total) { + static String percent(long value, long total) { return total <= 0 ? "0%" : String.format(Locale.ROOT, "%.1f%%", 100.0 * value / total); } - private static String count(long n) { + static String count(long n) { return String.format(Locale.ROOT, "%,d", n); } @@ -561,7 +565,7 @@ private static String rowRange(long firstRow, long rows) { return count(firstRow) + "-" + count(firstRow + rows - 1); } - private static String value(Object raw, DType type) { + static String value(Object raw, DType type) { if (raw == null) { return "-"; } @@ -578,7 +582,7 @@ private static String value(Object raw, DType type) { /// utf8 showed `2024-01-01T00:00:00` would make the better-typed file the harder one to read. /// Anything unrecognized, and any metadata this cannot parse - the bytes come from an /// untrusted file - falls back to the raw value rather than failing the render. - private static Object readable(Object raw, DType type) { + static Object readable(Object raw, DType type) { if (!(type instanceof DType.Extension ext) || !(raw instanceof Long epoch)) { return raw; } @@ -725,5 +729,20 @@ private static List vars(List series) { .num { font-variant-numeric: tabular-nums; } .sized { text-align: right; white-space: nowrap; } .sized .bar { width: 46px; display: inline-flex; margin: 0 8px 0 0; vertical-align: 1px; } + .scroll { overflow-x: auto; } + .profiles td { white-space: nowrap; } + .spark { width: 120px; height: 24px; display: block; } + .spark polygon { fill-opacity: 0.35; stroke-width: 1; vector-effect: non-scaling-stroke; stroke-linejoin: round; } + button { font: inherit; font-size: 12px; color: var(--text-secondary); background: var(--surface-0); border: 1px solid var(--border); border-radius: 6px; padding: 2px 10px; cursor: pointer; } + button:hover:not(:disabled) { color: var(--text-primary); border-color: var(--text-muted); } + button:disabled { cursor: default; color: var(--text-muted); } + .profiles tr.detail > td { white-space: normal; background: var(--surface-0); padding: 10px 14px 14px; } + .dist { max-width: 720px; } + .hist { width: 100%%; height: 90px; display: block; } + .hist rect { fill: var(--series-1); } + .hist rect:hover { fill-opacity: 0.7; } + .axis { display: flex; justify-content: space-between; font-size: 11px; color: var(--text-muted); margin-top: 2px; } + .bars td { border-bottom: 0; padding: 2px 10px 2px 0; } + .bars .track { width: 50%%; } """; } diff --git a/inspector/src/test/java/io/github/dfa1/vortex/inspect/ByteSizeTest.java b/inspector/src/test/java/io/github/dfa1/vortex/inspect/ByteSizeTest.java index 12b79bc48..25f7c5bfd 100644 --- a/inspector/src/test/java/io/github/dfa1/vortex/inspect/ByteSizeTest.java +++ b/inspector/src/test/java/io/github/dfa1/vortex/inspect/ByteSizeTest.java @@ -19,6 +19,15 @@ class ByteSizeTest { "1048576, 1.0 MB", // first MB "1572864, 1.5 MB", "5242880, 5.0 MB", + "1073741823, 1024.0 MB", // last value below the GB boundary + "1073741824, 1.0 GB", // first GB + "2284919630, 2.1 GB", // a 100M-row file read as 2179.1 MB before GB existed + "1099511627775, 1024.0 GB", // last value below the TB boundary + "1099511627776, 1.0 TB", + "1125899906842624, 1.0 PB", + "1152921504606846975, 1024.0 PB", // last value below the EB boundary + "1152921504606846976, 1.0 EB", + "9223372036854775807, 8.0 EB", // Long.MAX_VALUE: the largest count there is, still in a unit }) void format_rendersBinaryUnits(long bytes, String expected) { // Given a byte count @@ -26,7 +35,7 @@ void format_rendersBinaryUnits(long bytes, String expected) { // When String result = ByteSize.format(bytes); - // Then it picks B / KB / MB with one decimal for the scaled forms + // Then it picks B, or KB up to EB with one decimal assertThat(result).isEqualTo(expected); } } diff --git a/inspector/src/test/java/io/github/dfa1/vortex/inspect/ColumnDistributionTest.java b/inspector/src/test/java/io/github/dfa1/vortex/inspect/ColumnDistributionTest.java new file mode 100644 index 000000000..b48091c86 --- /dev/null +++ b/inspector/src/test/java/io/github/dfa1/vortex/inspect/ColumnDistributionTest.java @@ -0,0 +1,97 @@ +package io.github.dfa1.vortex.inspect; + +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +class ColumnDistributionTest { + + @Test + void bars_fewNumbersListEachValueInValueOrder() { + // Given — a rating column: 5 bars in order beat a 30-bin histogram with 25 empty bins + Map counts = Map.of(3L, 10L, 1L, 2L, 5L, 7L); + + // When + Map.Entry, Boolean> result = + ColumnDistribution.bars(counts, new long[ColumnDistribution.BINS], 1, 5); + + // Then + assertThat(result.getValue()).isFalse(); + assertThat(result.getKey()).extracting(ColumnDistribution.Bar::value).containsExactly(1L, 3L, 5L); + } + + @Test + void bars_fewNamesListMostFrequentFirst() { + // Given + Map counts = Map.of("CASH", 3L, "STOCK", 9L, "BOND", 1L); + + // When + Map.Entry, Boolean> result = ColumnDistribution.bars(counts, null, 0, 0); + + // Then + assertThat(result.getKey()).extracting(ColumnDistribution.Bar::value).containsExactly("STOCK", "CASH", "BOND"); + } + + @Test + void bars_manyNumbersBecomeAHistogramSpanningMinToMax() { + // Given — more distinct values than are worth listing + Map counts = new HashMap<>(); + for (long v = 0; v <= ColumnDistribution.MAX_CATEGORIES; v++) { + counts.put(v, 1L); + } + long[] bins = new long[ColumnDistribution.BINS]; + bins[0] = 4; + + // When + Map.Entry, Boolean> result = ColumnDistribution.bars(counts, bins, 0, 30); + + // Then — the last bin ends exactly at max, not at a rounded lo + 30 * width + assertThat(result.getValue()).isTrue(); + assertThat(result.getKey()).hasSize(ColumnDistribution.BINS); + assertThat(result.getKey().getFirst()).isEqualTo(new ColumnDistribution.Bar(0.0, 1.0, 4)); + assertThat(result.getKey().getLast().upper()).isEqualTo(30.0); + } + + @Test + void bars_manyNamesShowTheMostFrequent() { + // Given + Map counts = new HashMap<>(); + for (int i = 0; i < 100; i++) { + counts.put("v" + i, (long) i); + } + + // When + Map.Entry, Boolean> result = ColumnDistribution.bars(counts, null, 0, 0); + + // Then + assertThat(result.getKey()).hasSize(ColumnDistribution.TOP_VALUES); + assertThat(result.getKey().getFirst().value()).isEqualTo("v99"); + } + + @Test + void bars_tooManyToCountAndNoHistogramShowsNothing() { + // Given — a high-cardinality string column: no honest picture to draw + + // When + Map.Entry, Boolean> result = ColumnDistribution.bars(null, null, 0, 0); + + // Then + assertThat(result.getKey()).isEmpty(); + } + + @Test + void mean_dividesByPresentValuesOnly() { + // Given — 10 rows, 2 null + ColumnDistribution sut = new ColumnDistribution(10, 2, 40L, null, List.of(), false, null); + + // When + Double result = sut.mean(); + + // Then + assertThat(result).isEqualTo(5.0); + } +} diff --git a/inspector/src/test/java/io/github/dfa1/vortex/inspect/ColumnProfileTest.java b/inspector/src/test/java/io/github/dfa1/vortex/inspect/ColumnProfileTest.java new file mode 100644 index 000000000..df0922d5d --- /dev/null +++ b/inspector/src/test/java/io/github/dfa1/vortex/inspect/ColumnProfileTest.java @@ -0,0 +1,121 @@ +package io.github.dfa1.vortex.inspect; + +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.reader.ArrayStats; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class ColumnProfileTest { + + @Test + void fold_combinesZonesIntoColumnTotals() { + // Given — two zones of a 10-row column, 1 null between them + List zones = List.of(stats(3L, 9L, 30L, 0L), stats(1L, 5L, 15L, 1L)); + + // When + ColumnProfile result = ColumnProfile.fold("x", DType.I64, 10, zones, List.of()); + + // Then + assertThat(result.min()).isEqualTo(1L); + assertThat(result.max()).isEqualTo(9L); + assertThat(result.sum()).isEqualTo(45L); + assertThat(result.nullCount()).isEqualTo(1L); + assertThat(result.mean()).isEqualTo(5.0); + } + + @Test + void fold_skipsAllNullZoneInsteadOfLosingMinMax() { + // Given — Rust leaves min/max/sum absent on a zone holding only nulls; that zone says nothing + // about the column's range, so it must not make the whole column's min/max unknown + ArrayStats allNull = new ArrayStats(null, false, null, false, null, null, null, null, 4L, null, null); + List zones = List.of(stats(2L, 7L, 9L, 0L), allNull); + + // When + ColumnProfile result = ColumnProfile.fold("x", DType.I64, 6, zones, List.of()); + + // Then + assertThat(result.min()).isEqualTo(2L); + assertThat(result.max()).isEqualTo(7L); + assertThat(result.nullCount()).isEqualTo(4L); + } + + @Test + void fold_inexactBoundIsNotReportedAsTheMinimum() { + // Given — a truncated string bound brackets the values but is not one of them + ArrayStats bounded = new ArrayStats("ab", false, "zz", true, null, null, null, null, 0L, null, null); + + // When + ColumnProfile result = ColumnProfile.fold("s", DType.UTF8, 3, List.of(bounded), List.of()); + + // Then + assertThat(result.min()).isNull(); + assertThat(result.max()).isEqualTo("zz"); + } + + @Test + void fold_oneZoneMissingSumLeavesColumnSumUnknown() { + // Given — a partial sum would print a wrong mean, not an approximate one + List zones = List.of(stats(1L, 2L, 3L, 0L), stats(1L, 2L, null, 0L)); + + // When + ColumnProfile result = ColumnProfile.fold("x", DType.I64, 4, zones, List.of()); + + // Then + assertThat(result.sum()).isNull(); + assertThat(result.mean()).isNull(); + } + + @Test + void fold_overflowingIntegerSumIsUnknownNotWrapped() { + // Given + List zones = List.of(stats(0L, 1L, Long.MAX_VALUE, 0L), stats(0L, 1L, 1L, 0L)); + + // When + ColumnProfile result = ColumnProfile.fold("x", DType.I64, 2, zones, List.of()); + + // Then + assertThat(result.sum()).isNull(); + } + + @Test + void fold_noStatsAtAllMeansEveryAggregateUnknown() { + // Given / When + ColumnProfile result = ColumnProfile.fold("x", DType.I64, 0, List.of(), List.of()); + + // Then — not "0 nulls": nothing was recorded + assertThat(result.nullCount()).isNull(); + assertThat(result.min()).isNull(); + assertThat(result.sum()).isNull(); + } + + @Test + void mean_excludesNullsAndNans() { + // Given — 10 rows, 2 null, 3 NaN: the sum covers the other 5 + ColumnProfile sut = new ColumnProfile("f", DType.F64, 10, 2L, 3L, 1.0, 4.0, 10.0, List.of()); + + // When + Double result = sut.mean(); + + // Then + assertThat(result).isEqualTo(2.0); + } + + @Test + void mean_allNullColumnHasNoMean() { + // Given + ColumnProfile sut = new ColumnProfile("f", DType.F64, 3, 3L, 0L, null, null, 0.0, List.of()); + + // When + Double result = sut.mean(); + + // Then — not a division by zero, and not NaN + assertThat(result).isNull(); + } + + private static ArrayStats stats(Long min, Long max, Long sum, Long nulls) { + return new ArrayStats(min, true, max, true, sum, null, null, null, nulls, null, null); + } +} diff --git a/inspector/src/test/java/io/github/dfa1/vortex/inspect/ExplorePageTest.java b/inspector/src/test/java/io/github/dfa1/vortex/inspect/ExplorePageTest.java new file mode 100644 index 000000000..28a2d3b22 --- /dev/null +++ b/inspector/src/test/java/io/github/dfa1/vortex/inspect/ExplorePageTest.java @@ -0,0 +1,240 @@ +package io.github.dfa1.vortex.inspect; + +import io.github.dfa1.vortex.core.model.DType; +import io.github.dfa1.vortex.reader.ArrayStats; +import io.github.dfa1.vortex.reader.Zone; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class ExplorePageTest { + + @Nested + class Render { + + @Test + void isSelfContainedAndSaysNothingAboutStorage() { + // Given + ColumnProfile id = new ColumnProfile("id", DType.I64, 1000, 0L, 0L, 1L, 999L, null, List.of()); + + // When + String result = ExplorePage.render("data.vortex", 4096L, 1000L, List.of(id)); + + // Then — a business user sees rows and columns, not segments and encodings; the one + // script is inline, nothing is fetched from elsewhere + assertThat(result) + .startsWith("") + .endsWith("\n") + .contains("data.vortex - Vortex explorer") + .contains("1,000") + .doesNotContain("