diff --git a/CHANGELOG.md b/CHANGELOG.md index 408f913c2f..0fa497e0c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 0.27.19 + +### Fixes + +- **Extract `
`/`` content instead of discarding it**: both tags were mapped to `RemovedBlock`, so `partition_html()` silently dropped every disclosure widget and everything inside it. FAQ and documentation pages built from accordions lost all of their questions and answers, and the caller saw a shorter element list rather than an error. `
` is now `Flow` (an ordinary block container) and the `` label is emitted as a `Title`, including when it follows an expand/collapse icon or is wrapped in a `

` or `

` -- so `chunk_by_title()` opens a new section per entry and a question stays attached to its own answer. The label's `category_depth` is one level below `h6`, plus one per enclosing `
`, so it gets the heading before it as its `parent_id`, a nested `
` label nests under its parent's label, and the next heading is not nested under the label. Resolves #3919. + ## 0.27.18 ### Fixes diff --git a/test_unstructured/partition/html/test_partition.py b/test_unstructured/partition/html/test_partition.py index 80054133ea..5c345a4db9 100644 --- a/test_unstructured/partition/html/test_partition.py +++ b/test_unstructured/partition/html/test_partition.py @@ -1683,3 +1683,213 @@ def test_partition_html_extracts_a_sphinx_style_api_reference(): "Returns", "An open connection object.", ] + + +# -- disclosure widgets (`
`/``) ------------------------------------------------- + + +def test_partition_html_extracts_details_and_summary_content(): + """`
` and `` were mapped to `RemovedBlock`, so an accordion vanished. + + FAQ pages are commonly built this way, and every question and answer was dropped silently -- + the caller got a shorter element list, not an error. See #3919. + """ + html_text = ( + "" + "

Support FAQ

" + "
Which Bluetooth profiles are supported?" + "

Your device supports A2DP and HFP profiles.

" + "
How do I pair my hearing aid?" + "

Open settings and select Bluetooth.

" + "" + ) + + elements = partition_html(text=html_text) + + assert [(type(e).__name__, e.text) for e in elements] == [ + ("Title", "Support FAQ"), + ("Title", "Which Bluetooth profiles are supported?"), + ("NarrativeText", "Your device supports A2DP and HFP profiles."), + ("Title", "How do I pair my hearing aid?"), + ("NarrativeText", "Open settings and select Bluetooth."), + ] + + +def test_partition_html_makes_a_summary_start_its_own_chunk_section(): + """The reason the element type matters: `chunk_by_title` opens a new section at a `Title`. + + Two unrelated Q&A pairs land in separate chunks instead of one blended chunk. Small sections + are still merged under the default `combine_text_under_n_chars`, so that is disabled here to + isolate the sectioning behaviour rather than the merging. + """ + html_text = ( + "" + "
First question?

First answer.

" + "
Second question?

Second answer.

" + "" + ) + + chunks = chunk_by_title( + partition_html(text=html_text), max_characters=200, combine_text_under_n_chars=0 + ) + + assert [c.text for c in chunks] == [ + "First question?\n\nFirst answer.", + "Second question?\n\nSecond answer.", + ] + + +def test_partition_html_extracts_a_details_block_that_has_no_summary(): + html_text = "

Just the body.

" + + assert [e.text for e in partition_html(text=html_text)] == ["Just the body."] + + +def test_partition_html_extracts_nested_details_blocks(): + html_text = ( + "
Outer" + "
Inner

Innermost body.

" + "
" + ) + + assert [(type(e).__name__, e.text) for e in partition_html(text=html_text)] == [ + ("Title", "Outer"), + ("Title", "Inner"), + ("Text", "Innermost body."), + ] + + +def test_partition_html_keeps_a_summary_a_title_after_a_leading_image(): + """An expand/collapse icon before the label must not demote the label to `Text`. + + Otherwise `chunk_by_title()` sees no boundary and merges the second entry into the first. + """ + html_text = ( + "" + "
First question?

First answer.

" + '
Second question?' + "

Second answer.

" + "" + ) + + elements = partition_html(text=html_text) + + assert [(type(e).__name__, e.text) for e in elements] == [ + ("Title", "First question?"), + ("Text", "First answer."), + ("Image", ""), + ("Title", "Second question?"), + ("Text", "Second answer."), + ] + chunks = chunk_by_title(elements, max_characters=200, combine_text_under_n_chars=0) + assert [c.text for c in chunks] == [ + "First question?\n\nFirst answer.", + "Second question?\n\nSecond answer.", + ] + + +@pytest.mark.parametrize("wrapper", ["p", "div"]) +def test_partition_html_keeps_a_block_wrapped_summary_a_title(wrapper: str): + html_text = ( + f"
<{wrapper}>Wrapped question?" + "

Answer.

" + ) + + assert [(type(e).__name__, e.text) for e in partition_html(text=html_text)] == [ + ("Title", "Wrapped question?"), + ("Text", "Answer."), + ] + + +def test_partition_html_does_not_repeat_a_heading_nested_in_a_summary(): + html_text = ( + "

Heading question?

" + "

Answer.

" + ) + + assert [(type(e).__name__, e.text) for e in partition_html(text=html_text)] == [ + ("Title", "Heading question?"), + ("Text", "Answer."), + ] + + +def test_partition_html_keeps_text_after_a_summary_out_of_its_title(): + html_text = ( + "
LabelLoose body text that follows" + "

Body paragraph.

" + ) + + assert [(type(e).__name__, e.text) for e in partition_html(text=html_text)] == [ + ("Title", "Label"), + ("NarrativeText", "Loose body text that follows"), + ("Text", "Body paragraph."), + ] + + +def test_partition_html_keeps_annotations_and_page_number_on_a_summary_title(): + html_text = ( + '
' + '
How do I reset my password?
' + "

Use the link.

" + ) + + title = next(e for e in partition_html(text=html_text) if e.text.startswith("How do I")) + + assert isinstance(title, Title) + assert title.text == "How do I reset my password?" + assert title.metadata.emphasized_text_contents == ["reset"] + assert title.metadata.emphasized_text_tags == ["b"] + assert title.metadata.link_texts == ["password"] + assert title.metadata.link_urls == ["/reset"] + assert title.metadata.page_number == 3 + + +def test_partition_html_nests_a_summary_under_the_heading_before_it(): + html_text = ( + "

FAQ

" + "
Which profiles are supported?" + "

Your device supports A2DP and HFP profiles.

" + "

Shipping

Orders ship within two business days.

" + ) + + faq, question, answer, shipping, shipping_text = partition_html(text=html_text) + + assert question.text == "Which profiles are supported?" + assert question.metadata.parent_id == faq.id + assert answer.metadata.parent_id == question.id + # -- the next heading of the same level is not swallowed by the summary -- + assert shipping.metadata.parent_id is None + assert shipping_text.metadata.parent_id == shipping.id + assert [e.metadata.category_depth for e in (faq, question, shipping)] == [1, 6, 1] + + +def test_partition_html_nests_a_nested_details_summary_under_its_parent_summary(): + html_text = ( + "
Outer question?" + "

The outer answer goes here.

" + "
Inner question?

The inner answer goes here.

" + "
" + ) + + outer, outer_answer, inner, inner_answer = partition_html(text=html_text) + + assert outer.metadata.parent_id is None + assert outer_answer.metadata.parent_id == outer.id + assert inner.metadata.parent_id == outer.id + assert inner_answer.metadata.parent_id == inner.id + assert [outer.metadata.category_depth, inner.metadata.category_depth] == [6, 7] + + +def test_partition_html_places_a_wrapped_summary_label_like_a_plain_one(): + html_text = ( + "

Billing

" + "

Can I get a refund?

" + "

Refunds are issued within thirty days.

" + ) + + billing, question, answer = partition_html(text=html_text) + + assert question.text == "Can I get a refund?" + assert question.metadata.category_depth == 6 + assert question.metadata.parent_id == billing.id + assert answer.metadata.parent_id == question.id diff --git a/unstructured/__version__.py b/unstructured/__version__.py index 1c8487b647..31850ea4b7 100644 --- a/unstructured/__version__.py +++ b/unstructured/__version__.py @@ -1 +1 @@ -__version__ = "0.27.18" # pragma: no cover +__version__ = "0.27.19" # pragma: no cover diff --git a/unstructured/partition/common/metadata.py b/unstructured/partition/common/metadata.py index 706e0bd98d..5217b9f4af 100644 --- a/unstructured/partition/common/metadata.py +++ b/unstructured/partition/common/metadata.py @@ -91,7 +91,10 @@ def get_last_modified_date(filename: str) -> str | None: def category_depth_from_html_tag( - ElementCls: type[Element], tag: str | None, list_ancestor_count: int = 0 + ElementCls: type[Element], + tag: str | None, + list_ancestor_count: int = 0, + details_ancestor_count: int = 0, ) -> int | None: """Compute `category_depth` from an element's HTML heading level (not DOM-nesting depth). @@ -101,6 +104,10 @@ def category_depth_from_html_tag( - `Title` (which includes ontology Title/Subtitle/Heading, i.e. ``

``-``

``): the heading level, zero-indexed -- ``h1`` -> 0, ``h2`` -> 1, ... ``h6`` -> 5. A `Title` whose tag is not a heading (e.g. a styled paragraph) is treated as a top-level heading (0). + - `Title` from a ```` label: one level below ``h6``, plus one for each ``
`` + it is nested in beyond its own (``details_ancestor_count``, passed in by the caller). The + label has no heading level of its own, so this places it under whatever heading precedes it + and keeps the next heading of any level from nesting under it. - `ListItem`: the number of enclosing list containers (``ol``/``ul``/``dl``), passed in by the caller (the v1 HTML parser, which computes it from list nesting). The v2 converter serializes a whole ``ol``/``ul``/``dl`` as one element and never emits a standalone ``ListItem``, so it @@ -113,6 +120,8 @@ def category_depth_from_html_tag( return list_ancestor_count if ElementCls is Title: + if tag == "summary": + return len(_HEADING_DEPTH) + max(details_ancestor_count - 1, 0) return _HEADING_DEPTH.get(tag, 0) return None diff --git a/unstructured/partition/html/parser.py b/unstructured/partition/html/parser.py index 5bb5c481cb..ee404fb1e6 100644 --- a/unstructured/partition/html/parser.py +++ b/unstructured/partition/html/parser.py @@ -281,13 +281,29 @@ def _category_depth(self, ElementCls: type[Element]) -> int | None: Delegates to the shared `category_depth_from_html_tag` helper so the v1 and v2 (ontology) HTML parsers compute `category_depth` identically. """ + tag = self._element.tag list_ancestor_count = ( len([e for e in self._element.iterancestors() if e.tag in ("dl", "ol", "ul")]) - if self._element.tag in ("li", "dd") + if tag in ("li", "dd") else 0 ) + details_ancestor_count = 0 + if ElementCls is Title: + # -- text anywhere in a `` label, such as inside a wrapping `

`, is placed by + # -- the `

` nesting of that label rather than by its own tag -- + summary = ( + self._element + if tag == "summary" + else next(self._element.iterancestors("summary"), None) + ) + if summary is not None: + tag = "summary" + details_ancestor_count = sum(1 for _ in summary.iterancestors("details")) return category_depth_from_html_tag( - ElementCls, self._element.tag, list_ancestor_count=list_ancestor_count + ElementCls, + tag, + list_ancestor_count=list_ancestor_count, + details_ancestor_count=details_ancestor_count, ) @property @@ -458,6 +474,31 @@ class Heading(Flow): _ElementCls = Title +class Summary(Heading): + """A `` element, the visible label of a `
` disclosure widget. + + The label is a `Title` even when it follows an image (such as an expand/collapse icon) or is + wrapped in an ordinary block like `

` or `

`, so each widget still starts a new section + when chunking by title. + """ + + def iter_elements(self) -> Iterator[Element]: + yield from self._iter_label_elements(self) + + def _iter_label_elements(self, block: Flow) -> Iterator[Element]: + """Generate the elements in `block`, making every run of its text a `Title`.""" + q: deque[Flow | Phrasing] = deque(block) + yield from block._element_from_text_or_tail(block.text or "", q, Title) + while q: + child = cast(Flow, q.popleft()) + # -- descend into a plain wrapper; an image, heading, table, etc. keeps its own type -- + if type(child) in (Flow, BlockItem): + yield from self._iter_label_elements(child) + else: + yield from child.iter_elements() + yield from block._element_from_text_or_tail(child.tail or "", q, Title) + + class ListBlock(Flow): """Either a `
    ` or `
      ` element, maybe a `
      ` element at some point. @@ -1076,8 +1117,10 @@ def derive_element_type_from_text(text: str) -> type[Text] | None: # -- removed phrasing -- "button": RemovedPhrasing, "label": RemovedPhrasing, + # -- disclosure widget: `` is its heading, the rest is ordinary flow content -- + "details": Flow, + "summary": Summary, # -- removed block -- - "details": RemovedBlock, # -- likely boilerplate -- "figure": RemovedBlock, "hr": RemovedBlock, "nav": RemovedBlock, @@ -1085,6 +1128,5 @@ def derive_element_type_from_text(text: str) -> type[Text] | None: # -- removed form-related -- "form": RemovedBlock, "input": RemovedBlock, - "summary": RemovedBlock, # -- child of `details` } )