diff --git a/backend/serialize_value.py b/backend/serialize_value.py index d27fd99..3c9fcc3 100644 --- a/backend/serialize_value.py +++ b/backend/serialize_value.py @@ -5,6 +5,125 @@ import pyarrow as pa +# Media larger than this is described but not sent. Base64 adds a third to +# the size, so a page of 50 rows stays near 4 MB in the worst case. +MEDIA_INLINE_MAX_BYTES = 64 * 1024 + +# Sizes of the DIB header that follows the 14-byte BMP file header. Every +# valid BMP uses one of these, so it tells a real bitmap apart from text. +_BMP_DIB_HEADER_SIZES = frozenset({12, 40, 52, 56, 64, 108, 124}) + + +def _is_bmp(raw: bytes) -> bool: + """Check the BMP magic and the DIB header behind it. + + "BM" on its own is two printable characters, so text such as "BM25" would + otherwise be read as a bitmap. + """ + if len(raw) < 18 or not raw.startswith(b"BM"): + return False + return int.from_bytes(raw[14:18], "little") in _BMP_DIB_HEADER_SIZES + + +def _is_id3(raw: bytes) -> bool: + """Check the ID3v2 magic, version, and synchsafe size. + + "ID3" is also three printable characters, so text such as "ID3 tags" would + otherwise be read as an MP3. + """ + if len(raw) < 10 or not raw.startswith(b"ID3"): + return False + if raw[3] not in (2, 3, 4) or raw[4] == 0xFF: + return False + return all(byte < 0x80 for byte in raw[6:10]) + + +def _is_ftyp(raw: bytes) -> bool: + """Check the ISO base media magic and the box size in front of it. + + "ftyp" starts four bytes into the file, so binary that holds those + characters at that offset would otherwise be read as video. A real box + is at least 16 bytes and holds a whole number of 4-byte fields, so the + size tells the two apart. + """ + if len(raw) < 16 or raw[4:8] != b"ftyp": + return False + box_size = int.from_bytes(raw[0:4], "big") + return 16 <= box_size <= len(raw) and box_size % 4 == 0 + + +def detect_media_type(raw: bytes): + """Return (media category, MIME type) from common file signatures.""" + if raw.startswith(b"\x89PNG\r\n\x1a\n"): + return "image", "image/png" + if raw.startswith(b"\xff\xd8\xff"): + return "image", "image/jpeg" + if raw.startswith((b"GIF87a", b"GIF89a")): + return "image", "image/gif" + if _is_bmp(raw): + return "image", "image/bmp" + if raw.startswith((b"II*\x00", b"MM\x00*")): + return "image", "image/tiff" + + if len(raw) >= 12 and raw.startswith(b"RIFF"): + container = raw[8:12] + if container == b"WEBP": + return "image", "image/webp" + if container == b"WAVE": + return "audio", "audio/wav" + if container == b"AVI ": + return "video", "video/x-msvideo" + + if raw.startswith(b"fLaC"): + return "audio", "audio/flac" + if raw.startswith(b"OggS"): + return "audio", "audio/ogg" + if _is_id3(raw): + return "audio", "audio/mpeg" + + if _is_ftyp(raw): + brands = raw[8:32] + if any(brand in brands for brand in (b"avif", b"avis")): + return "image", "image/avif" + if any(brand in brands for brand in (b"heic", b"heix")): + return "image", "image/heic" + if any(brand in brands for brand in (b"M4A ", b"M4B ")): + return "audio", "audio/mp4" + return "video", "video/mp4" + if raw.startswith(b"\x1aE\xdf\xa3"): + return "video", "video/webm" + if raw.startswith((b"\x00\x00\x01\xba", b"\x00\x00\x01\xb3")): + return "video", "video/mpeg" + + return None + + +def _serialize_binary(raw): + if raw is None: + return None + if isinstance(raw, str): + return raw + + media = detect_media_type(raw) + if media: + media_type, mime_type = media + value = { + "type": "media", + "media_type": media_type, + "mime_type": mime_type, + "size": len(raw), + "inline": len(raw) <= MEDIA_INLINE_MAX_BYTES, + } + if value["inline"]: + value["base64"] = base64.b64encode(raw).decode("ascii") + return value + + try: + return raw.decode("utf-8") + except UnicodeDecodeError: + return base64.b64encode(raw).decode("ascii") + + def _serialize_temporal(obj): """Convert temporal types to string representation.""" if obj is None: @@ -22,15 +141,7 @@ def _serialize_pyarrow_scalar(obj): return None if pa.types.is_binary(obj.type) or pa.types.is_large_binary(obj.type): - raw = obj.as_py() - if raw is None: - return None - if isinstance(raw, str): - return raw - try: - return raw.decode("utf-8") - except UnicodeDecodeError: - return base64.b64encode(raw).decode("utf-8") + return _serialize_binary(obj.as_py()) if pa.types.is_temporal(obj.type): return _serialize_temporal(obj.as_py()) @@ -71,20 +182,9 @@ def _serialize_container(obj): def _serialize_basic_types(obj): """Convert basic Python types to JSON-serializable format.""" if isinstance(obj, bytes): - try: - return obj.decode("utf-8") - except UnicodeDecodeError: - return base64.b64encode(obj).decode("utf-8") + return _serialize_binary(obj) if isinstance(obj, pa.BinaryScalar): - raw = obj.as_py() - if raw is None: - return None - if isinstance(raw, str): - return raw - try: - return raw.decode("utf-8") - except UnicodeDecodeError: - return base64.b64encode(raw).decode("utf-8") + return _serialize_binary(obj.as_py()) if isinstance(obj, (datetime, date, time)): return obj.isoformat() if isinstance(obj, timedelta): diff --git a/backend/tests/test_media_serialization.py b/backend/tests/test_media_serialization.py new file mode 100644 index 0000000..4d3721d --- /dev/null +++ b/backend/tests/test_media_serialization.py @@ -0,0 +1,154 @@ +import base64 + +import pyarrow as pa +import pytest + +from serialize_value import ( + MEDIA_INLINE_MAX_BYTES, + detect_media_type, + serialize_value, +) + + +# A 4x1 24-bit bitmap: the 14-byte file header, a 40-byte DIB header, and +# one row of pixel data. +BMP_IMAGE = ( + b"BM" + + (66).to_bytes(4, "little") + + b"\x00\x00\x00\x00" + + (54).to_bytes(4, "little") + + (40).to_bytes(4, "little") + + (4).to_bytes(4, "little") + + (1).to_bytes(4, "little") + + (1).to_bytes(2, "little") + + (24).to_bytes(2, "little") + + b"\x00" * 24 +) + +# A 24-byte ISO base media box: the size, the "ftyp" marker, the major +# brand, the minor version, and two compatible brands. +MP4_HEADER = ( + (24).to_bytes(4, "big") + + b"ftyp" + + b"isom" + + b"\x00\x00\x02\x00" + + b"isom" + + b"iso2" +) + + +@pytest.mark.parametrize( + ("payload", "media_type", "mime_type"), + [ + (b"\x89PNG\r\n\x1a\npayload", "image", "image/png"), + (b"\xff\xd8\xff\xe0payload", "image", "image/jpeg"), + (b"RIFF\x00\x00\x00\x00WAVEpayload", "audio", "audio/wav"), + (b"ID3\x04\x00\x00payload", "audio", "audio/mpeg"), + (MP4_HEADER, "video", "video/mp4"), + (b"\x1aE\xdf\xa3payload", "video", "video/webm"), + (BMP_IMAGE, "image", "image/bmp"), + ], +) +def test_detect_media_type(payload, media_type, mime_type): + assert detect_media_type(payload) == (media_type, mime_type) + + +@pytest.mark.parametrize( + "payload", + [ + b"BMW is a great car", + b"BM25 scoring is used for full-text search", + b"ID3 tag documentation", + b"ID3v2 notes", + ], +) +def test_text_that_starts_with_a_signature_stays_text(payload): + """"BM" and "ID3" are printable, so plain text can start with them.""" + assert detect_media_type(payload) is None + assert serialize_value(payload) == payload.decode("utf-8") + + +@pytest.mark.parametrize( + "payload", + [ + # UTF-16 text starts with the byte order mark FF FE, which is also a + # valid MPEG-1 Layer I frame header. + "A note held in a binary column, long enough to be a frame.".encode( + "utf-16" + ), + # Any binary at all can open with those two bytes. + b"\xff\xe0" + b"\x00" * 200, + ], +) +def test_binary_that_opens_like_a_frame_is_not_audio(payload): + """A bare frame header is 11 bits, too few to name a value as audio.""" + assert detect_media_type(payload) is None + + +@pytest.mark.parametrize( + "payload", + [ + # The box size is smaller than the header it counts. + (8).to_bytes(4, "big") + MP4_HEADER[4:], + # The box size is longer than the value that holds it. + (4096).to_bytes(4, "big") + MP4_HEADER[4:], + # A box holds whole 4-byte fields, so 26 cannot be a size. + (26).to_bytes(4, "big") + MP4_HEADER[4:] + b"\x00" * 8, + # Text that carries the marker at the offset a real box uses. + b"the ftyp box names the brand of an MP4 file", + ], +) +def test_ftyp_without_a_valid_box_size_is_not_video(payload): + """"ftyp" is four printable characters four bytes into the value.""" + assert detect_media_type(payload) is None + + +def test_mp3_needs_an_id3_tag_to_be_detected(): + """Dropping the bare frame header costs us the tagless MP3. + + Such a value falls back to base64, which is what it did before media + detection existed. + """ + tagged = b"ID3\x03\x00\x00\x00\x00\x00\x00" + b"\xff\xfb" + b"\x00" * 128 + assert detect_media_type(tagged) == ("audio", "audio/mpeg") + assert detect_media_type(b"\xff\xfb" + b"\x00" * 128) is None + + +def test_media_binary_serialization(): + payload = b"\x89PNG\r\n\x1a\npayload" + result = serialize_value(pa.scalar(payload, type=pa.large_binary())) + assert result == { + "type": "media", + "media_type": "image", + "mime_type": "image/png", + "size": len(payload), + "inline": True, + "base64": base64.b64encode(payload).decode("ascii"), + } + + +def test_media_at_the_size_limit_is_still_inline(): + payload = b"\x89PNG\r\n\x1a\n" + b"\x00" * (MEDIA_INLINE_MAX_BYTES - 8) + result = serialize_value(payload) + assert len(payload) == MEDIA_INLINE_MAX_BYTES + assert result["inline"] is True + assert result["base64"] == base64.b64encode(payload).decode("ascii") + + +def test_media_over_the_size_limit_carries_no_payload(): + payload = b"\x89PNG\r\n\x1a\n" + b"\x00" * MEDIA_INLINE_MAX_BYTES + result = serialize_value(payload) + assert result == { + "type": "media", + "media_type": "image", + "mime_type": "image/png", + "size": len(payload), + "inline": False, + } + assert "base64" not in result + + +def test_non_media_binary_serialization_is_unchanged(): + assert serialize_value(b"hello") == "hello" + payload = b"\xff\xfe\x01\x02" + assert serialize_value(payload) == base64.b64encode(payload).decode("ascii") diff --git a/web/vanilla/app.js b/web/vanilla/app.js index d2ef982..0e48ca5 100644 --- a/web/vanilla/app.js +++ b/web/vanilla/app.js @@ -280,6 +280,8 @@ class LanceViewer { if (value && typeof value === 'object') { if (value.type === 'vector') { this.renderVectorCell(td, value, column); + } else if (value.type === 'media') { + this.renderMediaCell(td, value); } else { // Pass complex objects to our new recursive UI builder this.renderComplexObject(td, value, column); @@ -296,6 +298,56 @@ class LanceViewer { this.elements.dataSection.style.display = 'block'; } + renderMediaCell(cell, mediaData) { + cell.className = 'media-cell'; + const wrapper = document.createElement('div'); + wrapper.className = 'media-preview'; + + if (!mediaData.base64) { + const summary = document.createElement('div'); + summary.className = 'media-info media-too-large'; + summary.textContent = `${mediaData.mime_type} • ${this.formatBytes(mediaData.size)} • too large to preview`; + wrapper.appendChild(summary); + cell.appendChild(wrapper); + return; + } + + const source = `data:${mediaData.mime_type};base64,${mediaData.base64}`; + + let mediaElement; + if (mediaData.media_type === 'image') { + mediaElement = document.createElement('img'); + mediaElement.alt = mediaData.mime_type; + mediaElement.loading = 'lazy'; + } else if (mediaData.media_type === 'audio') { + mediaElement = document.createElement('audio'); + mediaElement.controls = true; + mediaElement.preload = 'metadata'; + } else if (mediaData.media_type === 'video') { + mediaElement = document.createElement('video'); + mediaElement.controls = true; + mediaElement.preload = 'metadata'; + } else { + cell.textContent = `Unsupported media type: ${mediaData.mime_type}`; + return; + } + + mediaElement.src = source; + const info = document.createElement('div'); + info.className = 'media-info'; + info.textContent = `${mediaData.mime_type} • ${this.formatBytes(mediaData.size)}`; + + wrapper.appendChild(mediaElement); + wrapper.appendChild(info); + cell.appendChild(wrapper); + } + + formatBytes(size) { + if (size < 1024) return `${size} B`; + if (size < 1024 * 1024) return `${(size / 1024).toFixed(1)} KB`; + return `${(size / (1024 * 1024)).toFixed(1)} MB`; + } + renderVectorCell(cell, vectorData, columnName) { cell.className = 'vector-cell'; @@ -498,7 +550,15 @@ class LanceViewer { return; } - // 4. Handle Arrays + // 4. Handle Nested Media + if (obj.type === 'media') { + const mediaWrap = document.createElement('div'); + this.renderMediaCell(mediaWrap, obj); + parent.appendChild(mediaWrap); + return; + } + + // 5. Handle Arrays if (Array.isArray(obj)) { const list = document.createElement('div'); list.className = 'co-list'; diff --git a/web/vanilla/styles.css b/web/vanilla/styles.css index 3e2aaca..ab41c53 100644 --- a/web/vanilla/styles.css +++ b/web/vanilla/styles.css @@ -275,6 +275,44 @@ table tr:hover { background-color: #f8f9fa; } +.media-cell { + min-width: 240px; +} + +.media-preview { + display: flex; + flex-direction: column; + gap: 6px; + align-items: flex-start; +} + +.media-preview img, +.media-preview video { + width: 240px; + max-height: 180px; + object-fit: contain; + border-radius: 4px; + background: #f8f9fa; +} + +.media-preview audio { + width: 280px; + max-width: 100%; +} + +.media-info { + color: #6c757d; + font-family: 'SF Mono', 'Monaco', 'Consolas', monospace; + font-size: 0.75rem; +} + +.media-too-large { + background: #f8f9fa; + border: 1px dashed #ced4da; + border-radius: 4px; + padding: 12px; +} + .vector-cell { max-width: 200px; position: relative;