diff --git a/CHANGELOG.md b/CHANGELOG.md index ce78fe7..4a460d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - GitHub Actions pinned to the majors that run on Node 24, before Node 20 leaves the runners (#60). - FastAPI startup logging moved from the deprecated `on_event` decorator to a lifespan context manager (#77). +- Dataset loading now fetches combined schema and column metadata in parallel with row data, reducing four sequential I/O operations to two concurrent requests (#75). ### Fixed - `docker build` no longer fails with "the destination must be a directory and end with a /". `COPY backend/*.py .` needs a trailing slash when it copies more than one file. The classic builder rejected it, the BuildKit builder did not (#62). diff --git a/backend/app.py b/backend/app.py index 9a721d7..226702e 100644 --- a/backend/app.py +++ b/backend/app.py @@ -70,6 +70,56 @@ def get_lance_connection(): raise HTTPException(status_code=500, detail="Data path not found") return lancedb.connect(str(DATA_PATH)) + +def serialize_schema_metadata(metadata): + """Convert Arrow schema metadata into a JSON-safe dictionary. + + PyArrow exposes schema metadata as bytes keys and values, but JSON requires + string keys and values. ``serialize_value`` decodes valid UTF-8 bytes and + base64-encodes bytes that cannot be decoded, preventing FastAPI response + serialization from raising ``UnicodeDecodeError``. + """ + return { + serialize_value(key): serialize_value(value) + for key, value in (metadata or {}).items() + } + + +def describe_schema(schema): + """Build schema and column metadata in one pass.""" + fields = [] + columns = [] + for field in schema: + is_vector = ( + (pa.types.is_list(field.type) or pa.types.is_fixed_size_list(field.type)) + and pa.types.is_floating(field.type.value_type) + ) + field_info = { + "name": field.name, + "type": str(field.type), + "nullable": field.nullable, + } + if is_vector: + field_info["vector_dim"] = None + fields.append(field_info) + + column_info = { + "name": field.name, + "type": str(field.type), + "nullable": field.nullable, + "is_vector": is_vector, + } + if is_vector: + column_info["dim"] = None + columns.append(column_info) + + return { + "fields": fields, + "metadata": serialize_schema_metadata(schema.metadata), + "columns": columns, + } + + def serialize_arrow_value(value): try: # Stop immediately if the Arrow scalar is null @@ -176,7 +226,7 @@ async def health_check(): return {"ok": False, "error": str(e)} @app.get("/datasets") -async def list_datasets(): +def list_datasets(): try: db = get_lance_connection() if hasattr(db, "list_tables"): @@ -191,73 +241,56 @@ async def list_datasets(): logger.error(f"Error listing datasets: {e}") raise HTTPException(status_code=500, detail="Failed to list datasets") -@app.get("/datasets/{dataset_name}/schema") -async def get_dataset_schema(dataset_name: str): + +@app.get("/datasets/{dataset_name}/metadata") +def get_dataset_metadata(dataset_name: str): if not validate_dataset_name(dataset_name): raise HTTPException(status_code=400, detail="Invalid dataset name") try: db = get_lance_connection() table = db.open_table(dataset_name) - schema = table.schema - - schema_dict = { - "fields": [], - "metadata": schema.metadata or {} - } - - for field in schema: - field_info = { - "name": field.name, - "type": str(field.type), - "nullable": field.nullable - } + return describe_schema(table.schema) + except Exception as e: + logger.error(f"Error getting metadata for {dataset_name}: {e}") + raise HTTPException(status_code=500, detail="Failed to get dataset metadata") - if (pa.types.is_list(field.type) or pa.types.is_fixed_size_list(field.type)) and pa.types.is_floating(field.type.value_type): - field_info["vector_dim"] = None - schema_dict["fields"].append(field_info) +@app.get("/datasets/{dataset_name}/schema") +def get_dataset_schema(dataset_name: str): + if not validate_dataset_name(dataset_name): + raise HTTPException(status_code=400, detail="Invalid dataset name") - return schema_dict + try: + db = get_lance_connection() + table = db.open_table(dataset_name) + description = describe_schema(table.schema) + return { + "fields": description["fields"], + "metadata": description["metadata"], + } except Exception as e: logger.error(f"Error getting schema for {dataset_name}: {e}") raise HTTPException(status_code=500, detail="Failed to get dataset schema") @app.get("/datasets/{dataset_name}/columns") -async def get_dataset_columns(dataset_name: str): +def get_dataset_columns(dataset_name: str): if not validate_dataset_name(dataset_name): raise HTTPException(status_code=400, detail="Invalid dataset name") try: db = get_lance_connection() table = db.open_table(dataset_name) - schema = table.schema - - columns = [] - for field in schema: - col_info = { - "name": field.name, - "type": str(field.type), - "nullable": field.nullable - } - - if (pa.types.is_list(field.type) or pa.types.is_fixed_size_list(field.type)) and pa.types.is_floating(field.type.value_type): - col_info["is_vector"] = True - col_info["dim"] = None - else: - col_info["is_vector"] = False - - columns.append(col_info) - - return {"columns": columns} + description = describe_schema(table.schema) + return {"columns": description["columns"]} except Exception as e: logger.error(f"Error getting columns for {dataset_name}: {e}") raise HTTPException(status_code=500, detail="Failed to get dataset columns") @app.get("/datasets/{dataset_name}/rows") -async def get_dataset_rows( +def get_dataset_rows( dataset_name: str, limit: int = Query(default=50, ge=1, le=MAX_LIMIT), offset: int = Query(default=0, ge=0), @@ -358,7 +391,7 @@ async def get_dataset_rows( raise HTTPException(status_code=500, detail="Failed to get dataset rows") @app.get("/datasets/{dataset_name}/vector/preview") -async def get_vector_preview( +def get_vector_preview( dataset_name: str, column: str, limit: int = Query(default=100, le=MAX_LIMIT) diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 1e253a9..b81ed6c 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -1,13 +1,16 @@ """API endpoint tests, based on docs/spec.md. -Covers /healthz, /datasets, /schema, /columns, /rows (pagination, column -filtering, serialization), /vector/preview, and the graceful-degradation -path for unreadable datasets. +Covers /healthz, /datasets, /metadata, /schema, /columns, /rows (pagination, +column filtering, serialization), /vector/preview, and the +graceful-degradation path for unreadable datasets. """ import base64 +import inspect +from types import SimpleNamespace import lancedb +import pyarrow as pa import pytest from packaging.version import parse as parse_version @@ -43,6 +46,55 @@ def test_datasets_lists_created_tables(client): assert "broken" in names +# /datasets/{name}/metadata + +def test_metadata_combines_schema_and_columns(client): + response = client.get("/datasets/sample/metadata") + assert response.status_code == 200 + body = response.json() + assert {field["name"] for field in body["fields"]} == { + "id", "text", "score", "blob", "vec", "embedding" + } + columns = {column["name"]: column for column in body["columns"]} + assert columns["vec"]["is_vector"] is True + assert columns["id"]["is_vector"] is False + + +def test_metadata_serializes_utf8_and_binary_schema_metadata(client, monkeypatch): + import app as app_module + + schema = pa.schema( + [pa.field("id", pa.int64())], + metadata={ + "café".encode(): "naïve".encode(), + b"binary": b"\xff\xfe\x01\x02", + }, + ) + table = SimpleNamespace(schema=schema) + db = SimpleNamespace(open_table=lambda _name: table) + monkeypatch.setattr(app_module, "get_lance_connection", lambda: db) + + response = client.get("/datasets/sample/metadata") + assert response.status_code == 200 + metadata = response.json()["metadata"] + assert metadata["café"] == "naïve" + assert metadata["binary"] == base64.b64encode(b"\xff\xfe\x01\x02").decode() + + +def test_dataset_io_handlers_are_synchronous(): + import app as app_module + + handlers = ( + app_module.list_datasets, + app_module.get_dataset_metadata, + app_module.get_dataset_schema, + app_module.get_dataset_columns, + app_module.get_dataset_rows, + app_module.get_vector_preview, + ) + assert all(not inspect.iscoroutinefunction(handler) for handler in handlers) + + # /datasets/{name}/schema def test_schema_fields(client): diff --git a/web/vanilla/app.js b/web/vanilla/app.js index d2ef982..89a9274 100644 --- a/web/vanilla/app.js +++ b/web/vanilla/app.js @@ -129,78 +129,77 @@ class LanceViewer { this.currentDataset = datasetName; this.currentPage = 0; + this.allColumns = []; + this.selectedColumns = []; this.elements.datasetTitle.textContent = datasetName; this.elements.datasetHeader.style.display = 'block'; - await this.loadSchema(); - await this.loadColumns(); - await this.loadData(); + await Promise.all([ + this.loadMetadata(), + this.loadData() + ]); } - async loadSchema() { + async loadMetadata() { try { - const response = await fetch(`${this.apiBase}/datasets/${this.currentDataset}/schema`); + const response = await fetch(`${this.apiBase}/datasets/${this.currentDataset}/metadata`); if (!response.ok) { throw new Error(`API error: ${response.status} ${response.statusText}`); } - const schema = await response.json(); - - this.elements.schemaDisplay.innerHTML = ''; - schema.fields.forEach(field => { - const fieldDiv = document.createElement('div'); - const isVector = field.type.includes('list') || field.type.includes('fixed_size_list'); - fieldDiv.className = isVector ? 'schema-field vector' : 'schema-field'; - - let typeDisplay; - if (isVector) { - // Check if this is a CLIP vector - if (field.type.includes('[512]')) { - typeDisplay = `${field.name}: CLIP vector (512-dim float)`; - } else { - typeDisplay = `${field.name}: vector (${field.type})`; - } + const metadata = await response.json(); + this.renderSchema(metadata.fields); + this.renderColumns(metadata.columns); + return true; + } catch (error) { + this.showError('Failed to load metadata'); + return false; + } + } + + renderSchema(fields) { + this.elements.schemaDisplay.innerHTML = ''; + fields.forEach(field => { + const fieldDiv = document.createElement('div'); + const isVector = field.type.includes('list') || field.type.includes('fixed_size_list'); + fieldDiv.className = isVector ? 'schema-field vector' : 'schema-field'; + + let typeDisplay; + if (isVector) { + // Check if this is a CLIP vector + if (field.type.includes('[512]')) { + typeDisplay = `${field.name}: CLIP vector (512-dim float)`; } else { - typeDisplay = `${field.name}: ${field.type}`; + typeDisplay = `${field.name}: vector (${field.type})`; } + } else { + typeDisplay = `${field.name}: ${field.type}`; + } - fieldDiv.textContent = typeDisplay; - this.elements.schemaDisplay.appendChild(fieldDiv); - }); + fieldDiv.textContent = typeDisplay; + this.elements.schemaDisplay.appendChild(fieldDiv); + }); - this.elements.schemaSection.style.display = 'block'; - } catch (error) { - this.showError('Failed to load schema'); - } + this.elements.schemaSection.style.display = 'block'; } - async loadColumns() { - try { - const response = await fetch(`${this.apiBase}/datasets/${this.currentDataset}/columns`); - if (!response.ok) { - throw new Error(`API error: ${response.status} ${response.statusText}`); - } - const data = await response.json(); + renderColumns(columns) { + this.allColumns = columns; + this.selectedColumns = columns.map(col => col.name); - this.allColumns = data.columns; - this.selectedColumns = data.columns.map(col => col.name); - - this.elements.columnSelect.innerHTML = ''; - data.columns.forEach(column => { - const option = document.createElement('option'); - option.value = column.name; - option.textContent = column.is_vector - ? `${column.name} (vector)` - : column.name; - option.selected = true; - this.elements.columnSelect.appendChild(option); - }); + this.elements.columnSelect.innerHTML = ''; + columns.forEach(column => { + const option = document.createElement('option'); + option.value = column.name; + option.textContent = column.is_vector + ? `${column.name} (vector)` + : column.name; + option.selected = true; + this.elements.columnSelect.appendChild(option); + }); - this.elements.columnSelect.style.display = 'block'; - this.elements.columnSelect.parentElement.querySelector('.column-controls').style.display = 'flex'; - this.elements.columnSection.style.display = 'block'; - } catch (error) { - this.showError('Failed to load columns'); - } + this.elements.columnSelect.style.display = 'block'; + this.elements.columnSelect.parentElement.querySelector('.column-controls').style.display = 'flex'; + this.elements.columnSection.style.display = 'block'; } selectAllColumns() { @@ -246,10 +245,12 @@ class LanceViewer { this.renderTable(data.rows); this.updatePagination(); this.hideLoading(); + return true; } catch (error) { this.hideLoading(); this.showError('Failed to load data'); + return false; } }