fix(html): extract definition lists instead of discarding them

CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.27.10
+
+### Fixes
+
+- **Extract definition lists instead of discarding them**: `<dl>`, `<dt>` and `<dd>` were mapped to `RemovedBlock`, so `partition_html()` dropped every glossary and every Sphinx-generated API reference (each documented function with its parameters and return value) without an error. `<dl>` is now a list container, each `<dd>` definition a `ListItem` and each `<dt>` term an ordinary text block. The v2 (ontology) parser already kept definition lists.
+
 ## 0.27.9
 
 ### Fixes

test_unstructured/partition/html/test_partition.py

@@ -1633,3 +1633,53 @@ def test_partition_html_leaves_page_number_None_when_not_present():
     html_text = "<html><body><p>No page markup.</p></body></html>"
     elements = partition_html(text=html_text)
     assert all(e.metadata.page_number is None for e in elements)
+
+
+# -- definition lists (`<dl>`) --------------------------------------------------------------------
+
+
+def test_partition_html_extracts_definition_list_terms_and_definitions():
+    """`<dl>`, `<dt>` and `<dd>` were `RemovedBlock`, so a glossary vanished without an error."""
+    html_text = (
+        "<html><body><h2>Glossary</h2><dl>"
+        "<dt>Shard</dt><dd>A horizontal partition of a table stored on its own server.</dd>"
+        "<dt>Replica</dt><dd>A read-only copy of a shard.</dd>"
+        "</dl></body></html>"
+    )
+
+    elements = partition_html(text=html_text)
+
+    assert [(type(e).__name__, e.text, e.metadata.category_depth) for e in elements] == [
+        ("Title", "Glossary", 1),
+        ("Text", "Shard", None),
+        ("ListItem", "A horizontal partition of a table stored on its own server.", 1),
+        ("Text", "Replica", None),
+        ("ListItem", "A read-only copy of a shard.", 1),
+    ]
+
+
+def test_partition_html_extracts_a_sphinx_style_api_reference():
+    """Sphinx renders each documented object as a `<dl>`, so a whole API reference was dropped."""
+    html_text = (
+        '<html><body><h1>API reference</h1><dl class="py function">'
+        '<dt class="sig sig-object py"><span class="sig-name">connect</span>'
+        "(<em>host</em>, <em>port=5432</em>)</dt>"
+        "<dd><p>Open a connection to the database server.</p>"
+        '<dl class="field-list simple"><dt class="field-odd">Parameters</dt>'
+        '<dd class="field-odd"><ul><li><p><strong>host</strong>: server hostname.</p></li>'
+        "<li><p><strong>port</strong>: TCP port.</p></li></ul></dd>"
+        '<dt class="field-even">Returns</dt>'
+        '<dd class="field-even"><p>An open connection object.</p></dd></dl>'
+        "</dd></dl></body></html>"
+    )
+
+    assert [e.text for e in partition_html(text=html_text)] == [
+        "API reference",
+        "connect(host, port=5432)",
+        "Open a connection to the database server.",
+        "Parameters",
+        "host: server hostname.",
+        "port: TCP port.",
+        "Returns",
+        "An open connection object.",
+    ]

unstructured/__version__.py

@@ -1 +1 @@
-__version__ = "0.27.9"  # pragma: no cover
+__version__ = "0.27.10"  # pragma: no cover

unstructured/partition/html/parser.py

@@ -1031,8 +1031,11 @@ def derive_element_type_from_text(text: str) -> type[Text] | None:
         "p": BlockItem,
         "pre": Pre,
         # -- list blocks --
+        "dl": ListBlock,
         "ol": ListBlock,
         "ul": ListBlock,
+        "dd": ListItemBlock,
+        "dt": BlockItem,
         "li": ListItemBlock,
         # -- image --
         "img": ImageBlock,
@@ -1075,9 +1078,6 @@ def derive_element_type_from_text(text: str) -> type[Text] | None:
         "label": RemovedPhrasing,
         # -- removed block --
         "details": RemovedBlock,  # -- likely boilerplate --
-        "dl": RemovedBlock,
-        "dd": RemovedBlock,
-        "dt": RemovedBlock,
         "figure": RemovedBlock,
         "hr": RemovedBlock,
         "nav": RemovedBlock,