Support blob type visualization

backend/serialize_value.py

@@ -5,6 +5,125 @@
 import pyarrow as pa
 
 
+# Media larger than this is described but not sent. Base64 adds a third to
+# the size, so a page of 50 rows stays near 4 MB in the worst case.
+MEDIA_INLINE_MAX_BYTES = 64 * 1024
+
+# Sizes of the DIB header that follows the 14-byte BMP file header. Every
+# valid BMP uses one of these, so it tells a real bitmap apart from text.
+_BMP_DIB_HEADER_SIZES = frozenset({12, 40, 52, 56, 64, 108, 124})
+
+
+def _is_bmp(raw: bytes) -> bool:
+    """Check the BMP magic and the DIB header behind it.
+
+    "BM" on its own is two printable characters, so text such as "BM25" would
+    otherwise be read as a bitmap.
+    """
+    if len(raw) < 18 or not raw.startswith(b"BM"):
+        return False
+    return int.from_bytes(raw[14:18], "little") in _BMP_DIB_HEADER_SIZES
+
+
+def _is_id3(raw: bytes) -> bool:
+    """Check the ID3v2 magic, version, and synchsafe size.
+
+    "ID3" is also three printable characters, so text such as "ID3 tags" would
+    otherwise be read as an MP3.
+    """
+    if len(raw) < 10 or not raw.startswith(b"ID3"):
+        return False
+    if raw[3] not in (2, 3, 4) or raw[4] == 0xFF:
+        return False
+    return all(byte < 0x80 for byte in raw[6:10])
+
+
+def _is_ftyp(raw: bytes) -> bool:
+    """Check the ISO base media magic and the box size in front of it.
+
+    "ftyp" starts four bytes into the file, so binary that holds those
+    characters at that offset would otherwise be read as video. A real box
+    is at least 16 bytes and holds a whole number of 4-byte fields, so the
+    size tells the two apart.
+    """
+    if len(raw) < 16 or raw[4:8] != b"ftyp":
+        return False
+    box_size = int.from_bytes(raw[0:4], "big")
+    return 16 <= box_size <= len(raw) and box_size % 4 == 0
+
+
+def detect_media_type(raw: bytes):
+    """Return (media category, MIME type) from common file signatures."""
+    if raw.startswith(b"\x89PNG\r\n\x1a\n"):
+        return "image", "image/png"
+    if raw.startswith(b"\xff\xd8\xff"):
+        return "image", "image/jpeg"
+    if raw.startswith((b"GIF87a", b"GIF89a")):
+        return "image", "image/gif"
+    if _is_bmp(raw):
+        return "image", "image/bmp"
+    if raw.startswith((b"II*\x00", b"MM\x00*")):
+        return "image", "image/tiff"
+
+    if len(raw) >= 12 and raw.startswith(b"RIFF"):
+        container = raw[8:12]
+        if container == b"WEBP":
+            return "image", "image/webp"
+        if container == b"WAVE":
+            return "audio", "audio/wav"
+        if container == b"AVI ":
+            return "video", "video/x-msvideo"
+
+    if raw.startswith(b"fLaC"):
+        return "audio", "audio/flac"
+    if raw.startswith(b"OggS"):
+        return "audio", "audio/ogg"
+    if _is_id3(raw):
+        return "audio", "audio/mpeg"
+
+    if _is_ftyp(raw):
+        brands = raw[8:32]
+        if any(brand in brands for brand in (b"avif", b"avis")):
+            return "image", "image/avif"
+        if any(brand in brands for brand in (b"heic", b"heix")):
+            return "image", "image/heic"
+        if any(brand in brands for brand in (b"M4A ", b"M4B ")):
+            return "audio", "audio/mp4"
+        return "video", "video/mp4"
+    if raw.startswith(b"\x1aE\xdf\xa3"):
+        return "video", "video/webm"
+    if raw.startswith((b"\x00\x00\x01\xba", b"\x00\x00\x01\xb3")):
+        return "video", "video/mpeg"
+
+    return None
+
+
+def _serialize_binary(raw):
+    if raw is None:
+        return None
+    if isinstance(raw, str):
+        return raw
+
+    media = detect_media_type(raw)
+    if media:
+        media_type, mime_type = media
+        value = {
+            "type": "media",
+            "media_type": media_type,
+            "mime_type": mime_type,
+            "size": len(raw),
+            "inline": len(raw) <= MEDIA_INLINE_MAX_BYTES,
+        }
+        if value["inline"]:
+            value["base64"] = base64.b64encode(raw).decode("ascii")
+        return value
+
+    try:
+        return raw.decode("utf-8")
+    except UnicodeDecodeError:
+        return base64.b64encode(raw).decode("ascii")
+
+
 def _serialize_temporal(obj):
     """Convert temporal types to string representation."""
     if obj is None:
@@ -22,15 +141,7 @@ def _serialize_pyarrow_scalar(obj):
         return None
 
     if pa.types.is_binary(obj.type) or pa.types.is_large_binary(obj.type):
-        raw = obj.as_py()
-        if raw is None:
-            return None
-        if isinstance(raw, str):
-            return raw
-        try:
-            return raw.decode("utf-8")
-        except UnicodeDecodeError:
-            return base64.b64encode(raw).decode("utf-8")
+        return _serialize_binary(obj.as_py())
 
     if pa.types.is_temporal(obj.type):
         return _serialize_temporal(obj.as_py())
@@ -71,20 +182,9 @@ def _serialize_container(obj):
 def _serialize_basic_types(obj):
     """Convert basic Python types to JSON-serializable format."""
     if isinstance(obj, bytes):
-        try:
-            return obj.decode("utf-8")
-        except UnicodeDecodeError:
-            return base64.b64encode(obj).decode("utf-8")
+        return _serialize_binary(obj)
     if isinstance(obj, pa.BinaryScalar):
-        raw = obj.as_py()
-        if raw is None:
-            return None
-        if isinstance(raw, str):
-            return raw
-        try:
-            return raw.decode("utf-8")
-        except UnicodeDecodeError:
-            return base64.b64encode(raw).decode("utf-8")
+        return _serialize_binary(obj.as_py())
     if isinstance(obj, (datetime, date, time)):
         return obj.isoformat()
     if isinstance(obj, timedelta):

backend/tests/test_media_serialization.py

@@ -0,0 +1,154 @@
+import base64
+
+import pyarrow as pa
+import pytest
+
+from serialize_value import (
+    MEDIA_INLINE_MAX_BYTES,
+    detect_media_type,
+    serialize_value,
+)
+
+
+# A 4x1 24-bit bitmap: the 14-byte file header, a 40-byte DIB header, and
+# one row of pixel data.
+BMP_IMAGE = (
+    b"BM"
+    + (66).to_bytes(4, "little")
+    + b"\x00\x00\x00\x00"
+    + (54).to_bytes(4, "little")
+    + (40).to_bytes(4, "little")
+    + (4).to_bytes(4, "little")
+    + (1).to_bytes(4, "little")
+    + (1).to_bytes(2, "little")
+    + (24).to_bytes(2, "little")
+    + b"\x00" * 24
+)
+
+# A 24-byte ISO base media box: the size, the "ftyp" marker, the major
+# brand, the minor version, and two compatible brands.
+MP4_HEADER = (
+    (24).to_bytes(4, "big")
+    + b"ftyp"
+    + b"isom"
+    + b"\x00\x00\x02\x00"
+    + b"isom"
+    + b"iso2"
+)
+
+
+@pytest.mark.parametrize(
+    ("payload", "media_type", "mime_type"),
+    [
+        (b"\x89PNG\r\n\x1a\npayload", "image", "image/png"),
+        (b"\xff\xd8\xff\xe0payload", "image", "image/jpeg"),
+        (b"RIFF\x00\x00\x00\x00WAVEpayload", "audio", "audio/wav"),
+        (b"ID3\x04\x00\x00payload", "audio", "audio/mpeg"),
+        (MP4_HEADER, "video", "video/mp4"),
+        (b"\x1aE\xdf\xa3payload", "video", "video/webm"),
+        (BMP_IMAGE, "image", "image/bmp"),
+    ],
+)
+def test_detect_media_type(payload, media_type, mime_type):
+    assert detect_media_type(payload) == (media_type, mime_type)
+
+
+@pytest.mark.parametrize(
+    "payload",
+    [
+        b"BMW is a great car",
+        b"BM25 scoring is used for full-text search",
+        b"ID3 tag documentation",
+        b"ID3v2 notes",
+    ],
+)
+def test_text_that_starts_with_a_signature_stays_text(payload):
+    """"BM" and "ID3" are printable, so plain text can start with them."""
+    assert detect_media_type(payload) is None
+    assert serialize_value(payload) == payload.decode("utf-8")
+
+
+@pytest.mark.parametrize(
+    "payload",
+    [
+        # UTF-16 text starts with the byte order mark FF FE, which is also a
+        # valid MPEG-1 Layer I frame header.
+        "A note held in a binary column, long enough to be a frame.".encode(
+            "utf-16"
+        ),
+        # Any binary at all can open with those two bytes.
+        b"\xff\xe0" + b"\x00" * 200,
+    ],
+)
+def test_binary_that_opens_like_a_frame_is_not_audio(payload):
+    """A bare frame header is 11 bits, too few to name a value as audio."""
+    assert detect_media_type(payload) is None
+
+
+@pytest.mark.parametrize(
+    "payload",
+    [
+        # The box size is smaller than the header it counts.
+        (8).to_bytes(4, "big") + MP4_HEADER[4:],
+        # The box size is longer than the value that holds it.
+        (4096).to_bytes(4, "big") + MP4_HEADER[4:],
+        # A box holds whole 4-byte fields, so 26 cannot be a size.
+        (26).to_bytes(4, "big") + MP4_HEADER[4:] + b"\x00" * 8,
+        # Text that carries the marker at the offset a real box uses.
+        b"the ftyp box names the brand of an MP4 file",
+    ],
+)
+def test_ftyp_without_a_valid_box_size_is_not_video(payload):
+    """"ftyp" is four printable characters four bytes into the value."""
+    assert detect_media_type(payload) is None
+
+
+def test_mp3_needs_an_id3_tag_to_be_detected():
+    """Dropping the bare frame header costs us the tagless MP3.
+
+    Such a value falls back to base64, which is what it did before media
+    detection existed.
+    """
+    tagged = b"ID3\x03\x00\x00\x00\x00\x00\x00" + b"\xff\xfb" + b"\x00" * 128
+    assert detect_media_type(tagged) == ("audio", "audio/mpeg")
+    assert detect_media_type(b"\xff\xfb" + b"\x00" * 128) is None
+
+
+def test_media_binary_serialization():
+    payload = b"\x89PNG\r\n\x1a\npayload"
+    result = serialize_value(pa.scalar(payload, type=pa.large_binary()))
+    assert result == {
+        "type": "media",
+        "media_type": "image",
+        "mime_type": "image/png",
+        "size": len(payload),
+        "inline": True,
+        "base64": base64.b64encode(payload).decode("ascii"),
+    }
+
+
+def test_media_at_the_size_limit_is_still_inline():
+    payload = b"\x89PNG\r\n\x1a\n" + b"\x00" * (MEDIA_INLINE_MAX_BYTES - 8)
+    result = serialize_value(payload)
+    assert len(payload) == MEDIA_INLINE_MAX_BYTES
+    assert result["inline"] is True
+    assert result["base64"] == base64.b64encode(payload).decode("ascii")
+
+
+def test_media_over_the_size_limit_carries_no_payload():
+    payload = b"\x89PNG\r\n\x1a\n" + b"\x00" * MEDIA_INLINE_MAX_BYTES
+    result = serialize_value(payload)
+    assert result == {
+        "type": "media",
+        "media_type": "image",
+        "mime_type": "image/png",
+        "size": len(payload),
+        "inline": False,
+    }
+    assert "base64" not in result
+
+
+def test_non_media_binary_serialization_is_unchanged():
+    assert serialize_value(b"hello") == "hello"
+    payload = b"\xff\xfe\x01\x02"
+    assert serialize_value(payload) == base64.b64encode(payload).decode("ascii")