diff --git a/CHANGELOG.md b/CHANGELOG.md index ad71b4196f..d2c032e5ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 0.27.10 + +### Fixes + +- **Extract definition lists instead of discarding them**: `
`, `
` and `
` were mapped to `RemovedBlock`, so `partition_html()` dropped every glossary and every Sphinx-generated API reference (each documented function with its parameters and return value) without an error. `
` is now a list container, each `
` definition a `ListItem` and each `
` term an ordinary text block. The v2 (ontology) parser already kept definition lists. + ## 0.27.9 ### Fixes diff --git a/test_unstructured/partition/html/test_partition.py b/test_unstructured/partition/html/test_partition.py index a3f0d7740d..80054133ea 100644 --- a/test_unstructured/partition/html/test_partition.py +++ b/test_unstructured/partition/html/test_partition.py @@ -1633,3 +1633,53 @@ def test_partition_html_leaves_page_number_None_when_not_present(): html_text = "

No page markup.

" elements = partition_html(text=html_text) assert all(e.metadata.page_number is None for e in elements) + + +# -- definition lists (`
`) -------------------------------------------------------------------- + + +def test_partition_html_extracts_definition_list_terms_and_definitions(): + """`
`, `
` and `
` were `RemovedBlock`, so a glossary vanished without an error.""" + html_text = ( + "

Glossary

" + "
Shard
A horizontal partition of a table stored on its own server.
" + "
Replica
A read-only copy of a shard.
" + "
" + ) + + elements = partition_html(text=html_text) + + assert [(type(e).__name__, e.text, e.metadata.category_depth) for e in elements] == [ + ("Title", "Glossary", 1), + ("Text", "Shard", None), + ("ListItem", "A horizontal partition of a table stored on its own server.", 1), + ("Text", "Replica", None), + ("ListItem", "A read-only copy of a shard.", 1), + ] + + +def test_partition_html_extracts_a_sphinx_style_api_reference(): + """Sphinx renders each documented object as a `
`, so a whole API reference was dropped.""" + html_text = ( + '

API reference

' + '
connect' + "(host, port=5432)
" + "

Open a connection to the database server.

" + '
Parameters
' + '
  • host: server hostname.

  • ' + "
  • port: TCP port.

" + '
Returns
' + '

An open connection object.

' + "
" + ) + + assert [e.text for e in partition_html(text=html_text)] == [ + "API reference", + "connect(host, port=5432)", + "Open a connection to the database server.", + "Parameters", + "host: server hostname.", + "port: TCP port.", + "Returns", + "An open connection object.", + ] diff --git a/unstructured/__version__.py b/unstructured/__version__.py index 11757d2a5f..654374eb31 100644 --- a/unstructured/__version__.py +++ b/unstructured/__version__.py @@ -1 +1 @@ -__version__ = "0.27.9" # pragma: no cover +__version__ = "0.27.10" # pragma: no cover diff --git a/unstructured/partition/html/parser.py b/unstructured/partition/html/parser.py index 61c568f945..5bb5c481cb 100644 --- a/unstructured/partition/html/parser.py +++ b/unstructured/partition/html/parser.py @@ -1031,8 +1031,11 @@ def derive_element_type_from_text(text: str) -> type[Text] | None: "p": BlockItem, "pre": Pre, # -- list blocks -- + "dl": ListBlock, "ol": ListBlock, "ul": ListBlock, + "dd": ListItemBlock, + "dt": BlockItem, "li": ListItemBlock, # -- image -- "img": ImageBlock, @@ -1075,9 +1078,6 @@ def derive_element_type_from_text(text: str) -> type[Text] | None: "label": RemovedPhrasing, # -- removed block -- "details": RemovedBlock, # -- likely boilerplate -- - "dl": RemovedBlock, - "dd": RemovedBlock, - "dt": RemovedBlock, "figure": RemovedBlock, "hr": RemovedBlock, "nav": RemovedBlock,