fix(html): extract <details>/<summary> content instead of discarding it

CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.27.19
+
+### Fixes
+
+- **Extract `<details>`/`<summary>` content instead of discarding it**: both tags were mapped to `RemovedBlock`, so `partition_html()` silently dropped every disclosure widget and everything inside it. FAQ and documentation pages built from accordions lost all of their questions and answers, and the caller saw a shorter element list rather than an error. `<details>` is now `Flow` (an ordinary block container) and the `<summary>` label is emitted as a `Title`, including when it follows an expand/collapse icon or is wrapped in a `<p>` or `<div>` -- so `chunk_by_title()` opens a new section per entry and a question stays attached to its own answer. The label's `category_depth` is one level below `h6`, plus one per enclosing `<details>`, so it gets the heading before it as its `parent_id`, a nested `<details>` label nests under its parent's label, and the next heading is not nested under the label. Resolves #3919.
+
 ## 0.27.18
 
 ### Fixes

test_unstructured/partition/html/test_partition.py

@@ -1683,3 +1683,213 @@ def test_partition_html_extracts_a_sphinx_style_api_reference():
         "Returns",
         "An open connection object.",
     ]
+
+
+# -- disclosure widgets (`<details>`/`<summary>`) -------------------------------------------------
+
+
+def test_partition_html_extracts_details_and_summary_content():
+    """`<details>` and `<summary>` were mapped to `RemovedBlock`, so an accordion vanished.
+
+    FAQ pages are commonly built this way, and every question and answer was dropped silently --
+    the caller got a shorter element list, not an error. See #3919.
+    """
+    html_text = (
+        "<html><body>"
+        "<h1>Support FAQ</h1>"
+        "<details><summary>Which Bluetooth profiles are supported?</summary>"
+        "<p>Your device supports A2DP and HFP profiles.</p></details>"
+        "<details open><summary>How do I pair my hearing aid?</summary>"
+        "<p>Open settings and select Bluetooth.</p></details>"
+        "</body></html>"
+    )
+
+    elements = partition_html(text=html_text)
+
+    assert [(type(e).__name__, e.text) for e in elements] == [
+        ("Title", "Support FAQ"),
+        ("Title", "Which Bluetooth profiles are supported?"),
+        ("NarrativeText", "Your device supports A2DP and HFP profiles."),
+        ("Title", "How do I pair my hearing aid?"),
+        ("NarrativeText", "Open settings and select Bluetooth."),
+    ]
+
+
+def test_partition_html_makes_a_summary_start_its_own_chunk_section():
+    """The reason the element type matters: `chunk_by_title` opens a new section at a `Title`.
+
+    Two unrelated Q&A pairs land in separate chunks instead of one blended chunk. Small sections
+    are still merged under the default `combine_text_under_n_chars`, so that is disabled here to
+    isolate the sectioning behaviour rather than the merging.
+    """
+    html_text = (
+        "<html><body>"
+        "<details><summary>First question?</summary><p>First answer.</p></details>"
+        "<details><summary>Second question?</summary><p>Second answer.</p></details>"
+        "</body></html>"
+    )
+
+    chunks = chunk_by_title(
+        partition_html(text=html_text), max_characters=200, combine_text_under_n_chars=0
+    )
+
+    assert [c.text for c in chunks] == [
+        "First question?\n\nFirst answer.",
+        "Second question?\n\nSecond answer.",
+    ]
+
+
+def test_partition_html_extracts_a_details_block_that_has_no_summary():
+    html_text = "<html><body><details><p>Just the body.</p></details></body></html>"
+
+    assert [e.text for e in partition_html(text=html_text)] == ["Just the body."]
+
+
+def test_partition_html_extracts_nested_details_blocks():
+    html_text = (
+        "<html><body><details><summary>Outer</summary>"
+        "<details><summary>Inner</summary><p>Innermost body.</p></details>"
+        "</details></body></html>"
+    )
+
+    assert [(type(e).__name__, e.text) for e in partition_html(text=html_text)] == [
+        ("Title", "Outer"),
+        ("Title", "Inner"),
+        ("Text", "Innermost body."),
+    ]
+
+
+def test_partition_html_keeps_a_summary_a_title_after_a_leading_image():
+    """An expand/collapse icon before the label must not demote the label to `Text`.
+
+    Otherwise `chunk_by_title()` sees no boundary and merges the second entry into the first.
+    """
+    html_text = (
+        "<html><body>"
+        "<details><summary>First question?</summary><p>First answer.</p></details>"
+        '<details><summary><img src="/chevron.svg" alt="">Second question?</summary>'
+        "<p>Second answer.</p></details>"
+        "</body></html>"
+    )
+
+    elements = partition_html(text=html_text)
+
+    assert [(type(e).__name__, e.text) for e in elements] == [
+        ("Title", "First question?"),
+        ("Text", "First answer."),
+        ("Image", ""),
+        ("Title", "Second question?"),
+        ("Text", "Second answer."),
+    ]
+    chunks = chunk_by_title(elements, max_characters=200, combine_text_under_n_chars=0)
+    assert [c.text for c in chunks] == [
+        "First question?\n\nFirst answer.",
+        "Second question?\n\nSecond answer.",
+    ]
+
+
+@pytest.mark.parametrize("wrapper", ["p", "div"])
+def test_partition_html_keeps_a_block_wrapped_summary_a_title(wrapper: str):
+    html_text = (
+        f"<html><body><details><summary><{wrapper}>Wrapped question?</{wrapper}></summary>"
+        "<p>Answer.</p></details></body></html>"
+    )
+
+    assert [(type(e).__name__, e.text) for e in partition_html(text=html_text)] == [
+        ("Title", "Wrapped question?"),
+        ("Text", "Answer."),
+    ]
+
+
+def test_partition_html_does_not_repeat_a_heading_nested_in_a_summary():
+    html_text = (
+        "<html><body><details><summary><h3>Heading question?</h3></summary>"
+        "<p>Answer.</p></details></body></html>"
+    )
+
+    assert [(type(e).__name__, e.text) for e in partition_html(text=html_text)] == [
+        ("Title", "Heading question?"),
+        ("Text", "Answer."),
+    ]
+
+
+def test_partition_html_keeps_text_after_a_summary_out_of_its_title():
+    html_text = (
+        "<html><body><details><summary>Label</summary>Loose body text that follows"
+        "<p>Body paragraph.</p></details></body></html>"
+    )
+
+    assert [(type(e).__name__, e.text) for e in partition_html(text=html_text)] == [
+        ("Title", "Label"),
+        ("NarrativeText", "Loose body text that follows"),
+        ("Text", "Body paragraph."),
+    ]
+
+
+def test_partition_html_keeps_annotations_and_page_number_on_a_summary_title():
+    html_text = (
+        '<html><body><details data-page-number="3"><summary><img src="/i.svg" alt="">'
+        '<div>How do I <b>reset</b> my <a href="/reset">password</a>?</div></summary>'
+        "<p>Use the link.</p></details></body></html>"
+    )
+
+    title = next(e for e in partition_html(text=html_text) if e.text.startswith("How do I"))
+
+    assert isinstance(title, Title)
+    assert title.text == "How do I reset my password?"
+    assert title.metadata.emphasized_text_contents == ["reset"]
+    assert title.metadata.emphasized_text_tags == ["b"]
+    assert title.metadata.link_texts == ["password"]
+    assert title.metadata.link_urls == ["/reset"]
+    assert title.metadata.page_number == 3
+
+
+def test_partition_html_nests_a_summary_under_the_heading_before_it():
+    html_text = (
+        "<html><body><h2>FAQ</h2>"
+        "<details><summary>Which profiles are supported?</summary>"
+        "<p>Your device supports A2DP and HFP profiles.</p></details>"
+        "<h2>Shipping</h2><p>Orders ship within two business days.</p></body></html>"
+    )
+
+    faq, question, answer, shipping, shipping_text = partition_html(text=html_text)
+
+    assert question.text == "Which profiles are supported?"
+    assert question.metadata.parent_id == faq.id
+    assert answer.metadata.parent_id == question.id
+    # -- the next heading of the same level is not swallowed by the summary --
+    assert shipping.metadata.parent_id is None
+    assert shipping_text.metadata.parent_id == shipping.id
+    assert [e.metadata.category_depth for e in (faq, question, shipping)] == [1, 6, 1]
+
+
+def test_partition_html_nests_a_nested_details_summary_under_its_parent_summary():
+    html_text = (
+        "<html><body><details><summary>Outer question?</summary>"
+        "<p>The outer answer goes here.</p>"
+        "<details><summary>Inner question?</summary><p>The inner answer goes here.</p></details>"
+        "</details></body></html>"
+    )
+
+    outer, outer_answer, inner, inner_answer = partition_html(text=html_text)
+
+    assert outer.metadata.parent_id is None
+    assert outer_answer.metadata.parent_id == outer.id
+    assert inner.metadata.parent_id == outer.id
+    assert inner_answer.metadata.parent_id == inner.id
+    assert [outer.metadata.category_depth, inner.metadata.category_depth] == [6, 7]
+
+
+def test_partition_html_places_a_wrapped_summary_label_like_a_plain_one():
+    html_text = (
+        "<html><body><h3>Billing</h3>"
+        "<details><summary><p>Can I get a refund?</p></summary>"
+        "<p>Refunds are issued within thirty days.</p></details></body></html>"
+    )
+
+    billing, question, answer = partition_html(text=html_text)
+
+    assert question.text == "Can I get a refund?"
+    assert question.metadata.category_depth == 6
+    assert question.metadata.parent_id == billing.id
+    assert answer.metadata.parent_id == question.id

unstructured/__version__.py

@@ -1 +1 @@
-__version__ = "0.27.18"  # pragma: no cover
+__version__ = "0.27.19"  # pragma: no cover

unstructured/partition/common/metadata.py

@@ -91,7 +91,10 @@ def get_last_modified_date(filename: str) -> str | None:
 
 
 def category_depth_from_html_tag(
-    ElementCls: type[Element], tag: str | None, list_ancestor_count: int = 0
+    ElementCls: type[Element],
+    tag: str | None,
+    list_ancestor_count: int = 0,
+    details_ancestor_count: int = 0,
 ) -> int | None:
     """Compute `category_depth` from an element's HTML heading level (not DOM-nesting depth).
 
@@ -101,6 +104,10 @@ def category_depth_from_html_tag(
     - `Title` (which includes ontology Title/Subtitle/Heading, i.e. ``<h1>``-``<h6>``): the heading
       level, zero-indexed -- ``h1`` -> 0, ``h2`` -> 1, ... ``h6`` -> 5. A `Title` whose tag is not a
       heading (e.g. a styled paragraph) is treated as a top-level heading (0).
+    - `Title` from a ``<summary>`` label: one level below ``h6``, plus one for each ``<details>``
+      it is nested in beyond its own (``details_ancestor_count``, passed in by the caller). The
+      label has no heading level of its own, so this places it under whatever heading precedes it
+      and keeps the next heading of any level from nesting under it.
     - `ListItem`: the number of enclosing list containers (``ol``/``ul``/``dl``), passed in by the
       caller (the v1 HTML parser, which computes it from list nesting). The v2 converter serializes
       a whole ``ol``/``ul``/``dl`` as one element and never emits a standalone ``ListItem``, so it
@@ -113,6 +120,8 @@ def category_depth_from_html_tag(
         return list_ancestor_count
 
     if ElementCls is Title:
+        if tag == "summary":
+            return len(_HEADING_DEPTH) + max(details_ancestor_count - 1, 0)
         return _HEADING_DEPTH.get(tag, 0)
 
     return None

unstructured/partition/html/parser.py

@@ -281,13 +281,29 @@ def _category_depth(self, ElementCls: type[Element]) -> int | None:
         Delegates to the shared `category_depth_from_html_tag` helper so the v1 and v2 (ontology)
         HTML parsers compute `category_depth` identically.
         """
+        tag = self._element.tag
         list_ancestor_count = (
             len([e for e in self._element.iterancestors() if e.tag in ("dl", "ol", "ul")])
-            if self._element.tag in ("li", "dd")
+            if tag in ("li", "dd")
             else 0
         )
+        details_ancestor_count = 0
+        if ElementCls is Title:
+            # -- text anywhere in a `<summary>` label, such as inside a wrapping `<p>`, is placed by
+            # -- the `<details>` nesting of that label rather than by its own tag --
+            summary = (
+                self._element
+                if tag == "summary"
+                else next(self._element.iterancestors("summary"), None)
+            )
+            if summary is not None:
+                tag = "summary"
+                details_ancestor_count = sum(1 for _ in summary.iterancestors("details"))
         return category_depth_from_html_tag(
-            ElementCls, self._element.tag, list_ancestor_count=list_ancestor_count
+            ElementCls,
+            tag,
+            list_ancestor_count=list_ancestor_count,
+            details_ancestor_count=details_ancestor_count,
         )
 
     @property
@@ -458,6 +474,31 @@ class Heading(Flow):
     _ElementCls = Title
 
 
+class Summary(Heading):
+    """A `<summary>` element, the visible label of a `<details>` disclosure widget.
+
+    The label is a `Title` even when it follows an image (such as an expand/collapse icon) or is
+    wrapped in an ordinary block like `<p>` or `<div>`, so each widget still starts a new section
+    when chunking by title.
+    """
+
+    def iter_elements(self) -> Iterator[Element]:
+        yield from self._iter_label_elements(self)
+
+    def _iter_label_elements(self, block: Flow) -> Iterator[Element]:
+        """Generate the elements in `block`, making every run of its text a `Title`."""
+        q: deque[Flow | Phrasing] = deque(block)
+        yield from block._element_from_text_or_tail(block.text or "", q, Title)
+        while q:
+            child = cast(Flow, q.popleft())
+            # -- descend into a plain wrapper; an image, heading, table, etc. keeps its own type --
+            if type(child) in (Flow, BlockItem):
+                yield from self._iter_label_elements(child)
+            else:
+                yield from child.iter_elements()
+            yield from block._element_from_text_or_tail(child.tail or "", q, Title)
+
+
 class ListBlock(Flow):
     """Either a `<ul>` or `<ol>` element, maybe a `<dl>` element at some point.
 
@@ -1076,15 +1117,16 @@ def derive_element_type_from_text(text: str) -> type[Text] | None:
         # -- removed phrasing --
         "button": RemovedPhrasing,
         "label": RemovedPhrasing,
+        # -- disclosure widget: `<summary>` is its heading, the rest is ordinary flow content --
+        "details": Flow,
+        "summary": Summary,
         # -- removed block --
-        "details": RemovedBlock,  # -- likely boilerplate --
         "figure": RemovedBlock,
         "hr": RemovedBlock,
         "nav": RemovedBlock,
         "template": RemovedBlock,
         # -- removed form-related --
         "form": RemovedBlock,
         "input": RemovedBlock,
-        "summary": RemovedBlock,  # -- child of `details`
     }
 )