From e570be42d8268633c1174a422bd04d69f4663fd2 Mon Sep 17 00:00:00 2001 From: dentinyhao Date: Mon, 3 Aug 2026 13:14:05 -0700 Subject: [PATCH 1/3] IO optimization for loading dataset --- backend/app.py | 103 ++++++++++++++++++++--------------- backend/tests/test_api.py | 29 ++++++++++ web/vanilla/app.js | 111 +++++++++++++++++++------------------- 3 files changed, 146 insertions(+), 97 deletions(-) diff --git a/backend/app.py b/backend/app.py index 03e354a..1d7a2c4 100644 --- a/backend/app.py +++ b/backend/app.py @@ -65,6 +65,42 @@ def get_lance_connection(): raise HTTPException(status_code=500, detail="Data path not found") return lancedb.connect(str(DATA_PATH)) + +def describe_schema(schema): + """Build schema and column metadata in one pass.""" + fields = [] + columns = [] + for field in schema: + is_vector = ( + (pa.types.is_list(field.type) or pa.types.is_fixed_size_list(field.type)) + and pa.types.is_floating(field.type.value_type) + ) + field_info = { + "name": field.name, + "type": str(field.type), + "nullable": field.nullable, + } + if is_vector: + field_info["vector_dim"] = None + fields.append(field_info) + + column_info = { + "name": field.name, + "type": str(field.type), + "nullable": field.nullable, + "is_vector": is_vector, + } + if is_vector: + column_info["dim"] = None + columns.append(column_info) + + return { + "fields": fields, + "metadata": schema.metadata or {}, + "columns": columns, + } + + def serialize_arrow_value(value): try: # Stop immediately if the Arrow scalar is null @@ -171,7 +207,7 @@ async def health_check(): return {"ok": False, "error": str(e)} @app.get("/datasets") -async def list_datasets(): +def list_datasets(): try: db = get_lance_connection() if hasattr(db, "list_tables"): @@ -186,73 +222,56 @@ async def list_datasets(): logger.error(f"Error listing datasets: {e}") raise HTTPException(status_code=500, detail="Failed to list datasets") -@app.get("/datasets/{dataset_name}/schema") -async def get_dataset_schema(dataset_name: str): + +@app.get("/datasets/{dataset_name}/metadata") +def get_dataset_metadata(dataset_name: str): if not validate_dataset_name(dataset_name): raise HTTPException(status_code=400, detail="Invalid dataset name") try: db = get_lance_connection() table = db.open_table(dataset_name) - schema = table.schema - - schema_dict = { - "fields": [], - "metadata": schema.metadata or {} - } - - for field in schema: - field_info = { - "name": field.name, - "type": str(field.type), - "nullable": field.nullable - } + return describe_schema(table.schema) + except Exception as e: + logger.error(f"Error getting metadata for {dataset_name}: {e}") + raise HTTPException(status_code=500, detail="Failed to get dataset metadata") - if (pa.types.is_list(field.type) or pa.types.is_fixed_size_list(field.type)) and pa.types.is_floating(field.type.value_type): - field_info["vector_dim"] = None - schema_dict["fields"].append(field_info) +@app.get("/datasets/{dataset_name}/schema") +def get_dataset_schema(dataset_name: str): + if not validate_dataset_name(dataset_name): + raise HTTPException(status_code=400, detail="Invalid dataset name") - return schema_dict + try: + db = get_lance_connection() + table = db.open_table(dataset_name) + description = describe_schema(table.schema) + return { + "fields": description["fields"], + "metadata": description["metadata"], + } except Exception as e: logger.error(f"Error getting schema for {dataset_name}: {e}") raise HTTPException(status_code=500, detail="Failed to get dataset schema") @app.get("/datasets/{dataset_name}/columns") -async def get_dataset_columns(dataset_name: str): +def get_dataset_columns(dataset_name: str): if not validate_dataset_name(dataset_name): raise HTTPException(status_code=400, detail="Invalid dataset name") try: db = get_lance_connection() table = db.open_table(dataset_name) - schema = table.schema - - columns = [] - for field in schema: - col_info = { - "name": field.name, - "type": str(field.type), - "nullable": field.nullable - } - - if (pa.types.is_list(field.type) or pa.types.is_fixed_size_list(field.type)) and pa.types.is_floating(field.type.value_type): - col_info["is_vector"] = True - col_info["dim"] = None - else: - col_info["is_vector"] = False - - columns.append(col_info) - - return {"columns": columns} + description = describe_schema(table.schema) + return {"columns": description["columns"]} except Exception as e: logger.error(f"Error getting columns for {dataset_name}: {e}") raise HTTPException(status_code=500, detail="Failed to get dataset columns") @app.get("/datasets/{dataset_name}/rows") -async def get_dataset_rows( +def get_dataset_rows( dataset_name: str, limit: int = Query(default=50, ge=1, le=MAX_LIMIT), offset: int = Query(default=0, ge=0), @@ -353,7 +372,7 @@ async def get_dataset_rows( raise HTTPException(status_code=500, detail="Failed to get dataset rows") @app.get("/datasets/{dataset_name}/vector/preview") -async def get_vector_preview( +def get_vector_preview( dataset_name: str, column: str, limit: int = Query(default=100, le=MAX_LIMIT) diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 1e253a9..c19cc30 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -6,6 +6,7 @@ """ import base64 +import inspect import lancedb import pytest @@ -43,6 +44,34 @@ def test_datasets_lists_created_tables(client): assert "broken" in names +# /datasets/{name}/metadata + +def test_metadata_combines_schema_and_columns(client): + response = client.get("/datasets/sample/metadata") + assert response.status_code == 200 + body = response.json() + assert {field["name"] for field in body["fields"]} == { + "id", "text", "score", "blob", "vec", "embedding" + } + columns = {column["name"]: column for column in body["columns"]} + assert columns["vec"]["is_vector"] is True + assert columns["id"]["is_vector"] is False + + +def test_dataset_io_handlers_are_synchronous(): + import app as app_module + + handlers = ( + app_module.list_datasets, + app_module.get_dataset_metadata, + app_module.get_dataset_schema, + app_module.get_dataset_columns, + app_module.get_dataset_rows, + app_module.get_vector_preview, + ) + assert all(not inspect.iscoroutinefunction(handler) for handler in handlers) + + # /datasets/{name}/schema def test_schema_fields(client): diff --git a/web/vanilla/app.js b/web/vanilla/app.js index d2ef982..89a9274 100644 --- a/web/vanilla/app.js +++ b/web/vanilla/app.js @@ -129,78 +129,77 @@ class LanceViewer { this.currentDataset = datasetName; this.currentPage = 0; + this.allColumns = []; + this.selectedColumns = []; this.elements.datasetTitle.textContent = datasetName; this.elements.datasetHeader.style.display = 'block'; - await this.loadSchema(); - await this.loadColumns(); - await this.loadData(); + await Promise.all([ + this.loadMetadata(), + this.loadData() + ]); } - async loadSchema() { + async loadMetadata() { try { - const response = await fetch(`${this.apiBase}/datasets/${this.currentDataset}/schema`); + const response = await fetch(`${this.apiBase}/datasets/${this.currentDataset}/metadata`); if (!response.ok) { throw new Error(`API error: ${response.status} ${response.statusText}`); } - const schema = await response.json(); - - this.elements.schemaDisplay.innerHTML = ''; - schema.fields.forEach(field => { - const fieldDiv = document.createElement('div'); - const isVector = field.type.includes('list') || field.type.includes('fixed_size_list'); - fieldDiv.className = isVector ? 'schema-field vector' : 'schema-field'; - - let typeDisplay; - if (isVector) { - // Check if this is a CLIP vector - if (field.type.includes('[512]')) { - typeDisplay = `${field.name}: CLIP vector (512-dim float)`; - } else { - typeDisplay = `${field.name}: vector (${field.type})`; - } + const metadata = await response.json(); + this.renderSchema(metadata.fields); + this.renderColumns(metadata.columns); + return true; + } catch (error) { + this.showError('Failed to load metadata'); + return false; + } + } + + renderSchema(fields) { + this.elements.schemaDisplay.innerHTML = ''; + fields.forEach(field => { + const fieldDiv = document.createElement('div'); + const isVector = field.type.includes('list') || field.type.includes('fixed_size_list'); + fieldDiv.className = isVector ? 'schema-field vector' : 'schema-field'; + + let typeDisplay; + if (isVector) { + // Check if this is a CLIP vector + if (field.type.includes('[512]')) { + typeDisplay = `${field.name}: CLIP vector (512-dim float)`; } else { - typeDisplay = `${field.name}: ${field.type}`; + typeDisplay = `${field.name}: vector (${field.type})`; } + } else { + typeDisplay = `${field.name}: ${field.type}`; + } - fieldDiv.textContent = typeDisplay; - this.elements.schemaDisplay.appendChild(fieldDiv); - }); + fieldDiv.textContent = typeDisplay; + this.elements.schemaDisplay.appendChild(fieldDiv); + }); - this.elements.schemaSection.style.display = 'block'; - } catch (error) { - this.showError('Failed to load schema'); - } + this.elements.schemaSection.style.display = 'block'; } - async loadColumns() { - try { - const response = await fetch(`${this.apiBase}/datasets/${this.currentDataset}/columns`); - if (!response.ok) { - throw new Error(`API error: ${response.status} ${response.statusText}`); - } - const data = await response.json(); + renderColumns(columns) { + this.allColumns = columns; + this.selectedColumns = columns.map(col => col.name); - this.allColumns = data.columns; - this.selectedColumns = data.columns.map(col => col.name); - - this.elements.columnSelect.innerHTML = ''; - data.columns.forEach(column => { - const option = document.createElement('option'); - option.value = column.name; - option.textContent = column.is_vector - ? `${column.name} (vector)` - : column.name; - option.selected = true; - this.elements.columnSelect.appendChild(option); - }); + this.elements.columnSelect.innerHTML = ''; + columns.forEach(column => { + const option = document.createElement('option'); + option.value = column.name; + option.textContent = column.is_vector + ? `${column.name} (vector)` + : column.name; + option.selected = true; + this.elements.columnSelect.appendChild(option); + }); - this.elements.columnSelect.style.display = 'block'; - this.elements.columnSelect.parentElement.querySelector('.column-controls').style.display = 'flex'; - this.elements.columnSection.style.display = 'block'; - } catch (error) { - this.showError('Failed to load columns'); - } + this.elements.columnSelect.style.display = 'block'; + this.elements.columnSelect.parentElement.querySelector('.column-controls').style.display = 'flex'; + this.elements.columnSection.style.display = 'block'; } selectAllColumns() { @@ -246,10 +245,12 @@ class LanceViewer { this.renderTable(data.rows); this.updatePagination(); this.hideLoading(); + return true; } catch (error) { this.hideLoading(); this.showError('Failed to load data'); + return false; } } From 6ba4aab3f46e70d7e89a8782474c5fcd5dc7998a Mon Sep 17 00:00:00 2001 From: dentinyhao Date: Wed, 5 Aug 2026 11:34:37 -0700 Subject: [PATCH 2/3] address comment: serialization coverage, changelog --- CHANGELOG.md | 1 + backend/app.py | 16 ++++++++++- backend/tests/test_api.py | 29 ++++++++++++++++++-- docs/spec.md | 58 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 100 insertions(+), 4 deletions(-) create mode 100644 docs/spec.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 882a6ef..d24d025 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - GitHub Actions pinned to the majors that run on Node 24, before Node 20 leaves the runners (#60). +- Dataset loading now fetches combined schema and column metadata in parallel with row data, reducing four sequential I/O operations to two concurrent requests (#75). ### Fixed - `docker build` no longer fails with "the destination must be a directory and end with a /". `COPY backend/*.py .` needs a trailing slash when it copies more than one file. The classic builder rejected it, the BuildKit builder did not (#62). diff --git a/backend/app.py b/backend/app.py index 1d7a2c4..c5453ef 100644 --- a/backend/app.py +++ b/backend/app.py @@ -66,6 +66,20 @@ def get_lance_connection(): return lancedb.connect(str(DATA_PATH)) +def serialize_schema_metadata(metadata): + """Convert Arrow schema metadata into a JSON-safe dictionary. + + PyArrow exposes schema metadata as bytes keys and values, but JSON requires + string keys and values. ``serialize_value`` decodes valid UTF-8 bytes and + base64-encodes bytes that cannot be decoded, preventing FastAPI response + serialization from raising ``UnicodeDecodeError``. + """ + return { + serialize_value(key): serialize_value(value) + for key, value in (metadata or {}).items() + } + + def describe_schema(schema): """Build schema and column metadata in one pass.""" fields = [] @@ -96,7 +110,7 @@ def describe_schema(schema): return { "fields": fields, - "metadata": schema.metadata or {}, + "metadata": serialize_schema_metadata(schema.metadata), "columns": columns, } diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index c19cc30..b81ed6c 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -1,14 +1,16 @@ """API endpoint tests, based on docs/spec.md. -Covers /healthz, /datasets, /schema, /columns, /rows (pagination, column -filtering, serialization), /vector/preview, and the graceful-degradation -path for unreadable datasets. +Covers /healthz, /datasets, /metadata, /schema, /columns, /rows (pagination, +column filtering, serialization), /vector/preview, and the +graceful-degradation path for unreadable datasets. """ import base64 import inspect +from types import SimpleNamespace import lancedb +import pyarrow as pa import pytest from packaging.version import parse as parse_version @@ -58,6 +60,27 @@ def test_metadata_combines_schema_and_columns(client): assert columns["id"]["is_vector"] is False +def test_metadata_serializes_utf8_and_binary_schema_metadata(client, monkeypatch): + import app as app_module + + schema = pa.schema( + [pa.field("id", pa.int64())], + metadata={ + "café".encode(): "naïve".encode(), + b"binary": b"\xff\xfe\x01\x02", + }, + ) + table = SimpleNamespace(schema=schema) + db = SimpleNamespace(open_table=lambda _name: table) + monkeypatch.setattr(app_module, "get_lance_connection", lambda: db) + + response = client.get("/datasets/sample/metadata") + assert response.status_code == 200 + metadata = response.json()["metadata"] + assert metadata["café"] == "naïve" + assert metadata["binary"] == base64.b64encode(b"\xff\xfe\x01\x02").decode() + + def test_dataset_io_handlers_are_synchronous(): import app as app_module diff --git a/docs/spec.md b/docs/spec.md new file mode 100644 index 0000000..be9f08a --- /dev/null +++ b/docs/spec.md @@ -0,0 +1,58 @@ +# Behavioral specification + +This document defines the observable API and dataset-loading behavior of the +vanilla web viewer. + +## Dataset selection + +When a user selects a dataset, the frontend: + +1. clears the previous column selection; +2. requests `GET /datasets/{dataset_name}/metadata` and + `GET /datasets/{dataset_name}/rows` concurrently; +3. renders the schema and column controls from the metadata response; and +4. renders the first page from the rows response. + +Metadata and row failures are reported independently. The frontend does not +make separate schema and columns requests during normal dataset loading. + +The LanceDB APIs used by the dataset handlers are synchronous. Those FastAPI +handlers must therefore be regular `def` functions so FastAPI executes them in +its thread pool instead of blocking the event loop. + +## Combined metadata endpoint + +`GET /datasets/{dataset_name}/metadata` opens the dataset once and returns: + +```json +{ + "fields": [ + {"name": "id", "type": "int64", "nullable": true} + ], + "metadata": { + "description": "Café vectors", + "binary": "//4BAg==" + }, + "columns": [ + { + "name": "id", + "type": "int64", + "nullable": true, + "is_vector": false + } + ] +} +``` + +- `fields` follows the existing `/schema` field representation. Vector fields + additionally contain `"vector_dim": null`. +- `columns` follows the existing `/columns` representation. Vector columns + additionally contain `"dim": null`. +- Arrow schema metadata keys and values are bytes. UTF-8 byte sequences are + decoded as JSON strings, including non-ASCII text. Values that are not valid + UTF-8 are returned as base64 strings. +- Invalid dataset names return 400. Datasets that cannot be opened return 500. + +`GET /datasets/{dataset_name}/schema` and +`GET /datasets/{dataset_name}/columns` remain available for API compatibility +and use the same metadata description and serialization rules. From 3265e4e3dc52da5cf5f7a3aa20b2d1905df5bbca Mon Sep 17 00:00:00 2001 From: Gordon Murray Date: Wed, 5 Aug 2026 20:26:43 +0100 Subject: [PATCH 3/3] docs: drop docs/spec.md The file is not part of this repo. Asking for it was a maintainer error. --- docs/spec.md | 58 ---------------------------------------------------- 1 file changed, 58 deletions(-) delete mode 100644 docs/spec.md diff --git a/docs/spec.md b/docs/spec.md deleted file mode 100644 index be9f08a..0000000 --- a/docs/spec.md +++ /dev/null @@ -1,58 +0,0 @@ -# Behavioral specification - -This document defines the observable API and dataset-loading behavior of the -vanilla web viewer. - -## Dataset selection - -When a user selects a dataset, the frontend: - -1. clears the previous column selection; -2. requests `GET /datasets/{dataset_name}/metadata` and - `GET /datasets/{dataset_name}/rows` concurrently; -3. renders the schema and column controls from the metadata response; and -4. renders the first page from the rows response. - -Metadata and row failures are reported independently. The frontend does not -make separate schema and columns requests during normal dataset loading. - -The LanceDB APIs used by the dataset handlers are synchronous. Those FastAPI -handlers must therefore be regular `def` functions so FastAPI executes them in -its thread pool instead of blocking the event loop. - -## Combined metadata endpoint - -`GET /datasets/{dataset_name}/metadata` opens the dataset once and returns: - -```json -{ - "fields": [ - {"name": "id", "type": "int64", "nullable": true} - ], - "metadata": { - "description": "Café vectors", - "binary": "//4BAg==" - }, - "columns": [ - { - "name": "id", - "type": "int64", - "nullable": true, - "is_vector": false - } - ] -} -``` - -- `fields` follows the existing `/schema` field representation. Vector fields - additionally contain `"vector_dim": null`. -- `columns` follows the existing `/columns` representation. Vector columns - additionally contain `"dim": null`. -- Arrow schema metadata keys and values are bytes. UTF-8 byte sequences are - decoded as JSON strings, including non-ASCII text. Values that are not valid - UTF-8 are returned as base64 strings. -- Invalid dataset names return 400. Datasets that cannot be opened return 500. - -`GET /datasets/{dataset_name}/schema` and -`GET /datasets/{dataset_name}/columns` remain available for API compatibility -and use the same metadata description and serialization rules.