fix(html): keep text inside unrecognized elements

CHANGELOG.md

@@ -1,9 +1,13 @@
-## 0.27.28-dev0
+## 0.27.28-dev1
 
 ### Enhancements
 
 - **Optionally use `langdetect-rs` for partition language detection.** Set `UNSTRUCTURED_USE_LANGDETECT_RS=true` before importing `unstructured` and install `langdetect-rs==0.1.0` to detect `metadata.languages` with this faster Rust port of `langdetect`, which matched its seeded results on the tested inputs. If the package is missing, a warning is logged and `langdetect` is used. Without the variable, nothing changes.
 
+### Fixes
+
+- **Keep the text inside HTML elements the parser does not recognize.** Since 0.15.0, `partition_html()` (and the `.md`, `.eml`, `.msg`, `.epub`, `.rst` and `.org` partitioners that route through it) kept only the text *after* an element with no registered element class and dropped its content: `<font>` text in email bodies, custom elements, and the Inline XBRL facts and notes of SEC filings (`example-10k.html` lost about a third of its text, including Notes 1-14). An unrecognized element is now transparent, like a `<span>`: its text joins the surrounding paragraph and any block items it contains become their own elements. Elements whose content is not rendered as text (`<head>`, `<title>`, `<svg>`, `<iframe>`, `<object>`, `<audio>`, `<video>`, `<canvas>`, `<noembed>`, `<noframes>`, `<select>`, `<datalist>`, `<textarea>`, `<dialog>`, and legacy `<xml>` data islands) and the hidden Inline XBRL `<ix:header>` are still skipped. Resolves #4496 and #3842.
+
 ## 0.27.27
 
 ### Fixes

test_unstructured/partition/html/test_parser.py

@@ -23,6 +23,7 @@
     Annotation,
     DefaultElement,
     Flow,
+    Math,
     Phrasing,
     RemovedPhrasing,
     TextSegment,
@@ -557,6 +558,20 @@ def it_falls_back_to_outer_page_number_when_inner_is_non_numeric(self):
         p = etree.fromstring(html, html_parser).xpath(".//p")[0]
         assert p._page_number == 1
 
+    def it_finds_page_number_of_a_block_nested_in_hundreds_of_unrecognized_elements(self):
+        """Unrecognized elements are Flow, so each one is an ancestor searched for a page number.
+
+        That search happens while phrasing traversal is already one or more stack frames deep for
+        each of those elements, so it must not add a stack frame per ancestor.
+        """
+        html = f'<div data-page-number="7">{"<foobar>" * 250}<p>text</p></div>'
+        div = etree.fromstring(html, html_parser).xpath(".//div")[0]
+
+        (element,) = div.iter_elements()
+
+        assert element == Text("text")
+        assert element.metadata.page_number == 7
+
     # -- ._element_from_text_or_tail() ------------------------------------
 
     def it_assembles_text_and_tail_document_elements_to_help(self):
@@ -1511,6 +1526,24 @@ def it_adds_a_newline_in_its_place(self):
         assert _normalize_text("".join(texts)) == "spaceships of the Vogon Constructor Fleet"
 
 
+class DescribeMath:
+    """Isolated unit-test suite for `unstructured.partition.html.parser.Math`.
+
+    Used for `<math>` elements. Its text extraction is covered by the MathML tests in
+    `test_partition.py`.
+    """
+
+    @pytest.mark.parametrize("wrapper", ["{}", "<font size='2'>{}</font>", "<my-card>{}</my-card>"])
+    def it_skips_mathml_that_has_no_text_representation_but_not_its_tail(self, wrapper: str):
+        html = "<div>Before {} after.</div>".format(
+            wrapper.format("<math><mi>x</mi><mo>=</mo><mn>2</mn></math>")
+        )
+        div = etree.fromstring(html, html_parser).xpath(".//div")[0]
+
+        assert isinstance(div.xpath(".//math")[0], Math)
+        assert [e.text for e in div.iter_elements()] == ["Before after."]
+
+
 class DescribeRemovedPhrasing:
     """Isolated unit-test suite for `unstructured.partition.html.parser.RemovedPhrasing`.
 
@@ -1533,6 +1566,41 @@ def it_behaves_like_an_empty_element(self):
         assert label.is_phrasing is True
         assert text_segment.text == "\n  Like vastly, hugely big.\n"
 
+    @pytest.mark.parametrize(
+        "html_text",
+        [
+            "<audio src='a.mp3'>Your browser does not support audio.</audio>",
+            "<canvas>Your browser does not support canvas.</canvas>",
+            "<datalist><option value='Chrome'>Chrome</option></datalist>",
+            "<dialog>Subscribe to our newsletter</dialog>",
+            "<dialog open>Subscribe to our newsletter</dialog>",
+            "<iframe src='a.html'>Your browser does not support iframes.</iframe>",
+            "<ix:header><ix:hidden><ix:nonNumeric name='dei:AmendmentFlag'>false</ix:nonNumeric>"
+            "</ix:hidden></ix:header>",
+            "<noembed>Your browser does not support plugins.</noembed>",
+            "<noframes>Your browser does not support frames.</noframes>",
+            "<object data='a.pdf'><p>Download the PDF instead.</p></object>",
+            "<select><option>Alpha</option><option>Beta</option></select>",
+            "<svg><title>Revenue chart</title><text>2021</text></svg>",
+            "<textarea>Type your comment here</textarea>",
+            "<title>Page title</title>",
+            "<video src='a.mp4'>Your browser does not support video.</video>",
+            "<xml><o:DocumentProperties><o:Author>Jane</o:Author></o:DocumentProperties></xml>",
+        ],
+    )
+    def it_is_used_for_elements_whose_content_is_not_rendered_as_text(self, html_text: str):
+        html = f"<div>Before {html_text} after.</div>"
+        div = etree.fromstring(html, html_parser).xpath(".//div")[0]
+
+        assert isinstance(div[0], RemovedPhrasing)
+        assert [e.text for e in div.iter_elements()] == ["Before after."]
+
+    def and_it_is_used_for_head_which_is_only_reached_when_there_is_no_body(self):
+        root = etree.fromstring("<html><head><foobar>Metadata</foobar></head></html>", html_parser)
+
+        assert isinstance(root.find("head"), RemovedPhrasing)
+        assert list(root.iter_elements()) == []
+
 
 # -- DEFAULT ELEMENT -----------------------------------------------------------------------------
 
@@ -1543,9 +1611,9 @@ class DescribeDefaultElement:
     Used for any element we haven't assigned a custom element-class too. This prominently includes
     any non-HTML elements that can be embedded in the HTML.
 
-    It identifies as a block item but it can behave as either a block-item or phrasing. Its behavior
-    is a combination of RemovedBlock and RemovedPhrasing. Namely, it iterates zero elements and only
-    iterates a text-segment for its tail.
+    It identifies as phrasing but it can behave as either a block-item or phrasing. Either way it is
+    transparent; its text, its children, and its tail are processed as though its tags were not
+    there.
     """
 
     # -- .is_phrasing -----------------------------------------------------
@@ -1558,36 +1626,43 @@ def it_identifies_as_a_phrasing_element(self):
 
     # -- .iter_elements() -------------------------------------------------
 
-    def it_generates_zero_elements_as_a_block_item(self):
-        """Should never be called but belts and suspenders."""
+    def it_generates_the_elements_of_its_contents_as_a_block_item(self):
+        """Only happens when it is the root, like an `<html>` element that has no `<body>`."""
         foobar = etree.fromstring(
             "<foobar>Space<p>is big</p>, <b>mind-bogglingly</b> big.</foobar>",
             html_parser,
         ).xpath(".//foobar")[0]
 
-        elements = foobar.iter_elements()
+        texts = [e.text for e in foobar.iter_elements()]
 
-        with pytest.raises(StopIteration):
-            next(elements)
+        assert texts == ["Space", "is big", ", mind-bogglingly big."]
 
     # -- .iter_text_segments() --------------------------------------------
 
-    def it_generates_its_tail_but_no_inner_text_segments_when_called_like_phrasing(self):
+    def it_generates_text_segments_for_its_text_and_children_and_tail_when_called_like_phrasing(
+        self,
+    ):
         foobar = etree.fromstring(
             "<div>\n"
             "  O Deep Thought computer, he said,\n"
-            "  <foobar>Vogon Constructor Fleet</foobar>\n"
+            "  <foobar>Vogon <b>Constructor</b> Fleet</foobar>\n"
             "  The task we have designed you to perform is this.\n"
             "  <p>We want you to tell us.... he paused,</p>\n"
             "</div>",
             html_parser,
         ).xpath(".//foobar")[0]
 
-        texts = [ts.text for ts in foobar.iter_text_segments()]
-
-        assert texts == ["\n  The task we have designed you to perform is this.\n  "]
+        assert list(foobar.iter_text_segments()) == [
+            TextSegment("Vogon ", {}),
+            TextSegment(
+                "Constructor",
+                {"emphasized_text_contents": "Constructor", "emphasized_text_tags": "b"},
+            ),
+            TextSegment(" Fleet", {}),
+            TextSegment("\n  The task we have designed you to perform is this.\n  ", {}),
+        ]
 
-    def and_it_behaves_like_an_empty_phrasing_element_inside_a_block_element(self):
+    def and_its_text_is_part_of_the_paragraph_it_appears_in(self):
         div = etree.fromstring(
             "<div>\n"
             "  O Deep Thought computer, he said,\n"
@@ -1601,6 +1676,67 @@ def and_it_behaves_like_an_empty_phrasing_element_inside_a_block_element(self):
         texts = [e.text for e in div.iter_elements()]
 
         assert texts == [
-            "O Deep Thought computer, he said, The task we have designed you to perform is this.",
+            "O Deep Thought computer, he said, Vogon Constructor Fleet The task we have designed"
+            " you to perform is this.",
             "We want you to tell us.... he paused,",
         ]
+
+    @pytest.mark.parametrize(
+        "wrapper",
+        [
+            "<foobar>{}</foobar>",
+            "<ix:nonNumeric name='us-gaap:InventoryDisclosureTextBlock'>{}</ix:nonNumeric>",
+            # -- nested unrecognized elements are each transparent --
+            "<my-card><font size='2'>{}</font></my-card>",
+        ],
+    )
+    @pytest.mark.parametrize(
+        "content",
+        [
+            "<p>Revenue increased in 2021.</p><p>Costs were flat.</p>",
+            "<h2>NOTE 4. INVENTORY</h2><p>Inventory consisted of the following:</p>",
+            "<div>Inventory</div><table><tr><td>Finished goods</td><td>356,928</td></tr></table>",
+            "<ul><li>Patents</li><li>Trademarks</li></ul>",
+        ],
+    )
+    def and_block_items_inside_it_produce_the_same_elements_as_without_it(
+        self, wrapper: str, content: str
+    ):
+        """So it can wrap whole sections, like an Inline XBRL note, as well as a few words."""
+        wrapped = etree.fromstring(wrapper.format(content), html_parser).find("body")
+        bare = etree.fromstring(content, html_parser).find("body")
+
+        actual = list(wrapped.iter_elements())
+        expected = list(bare.iter_elements())
+
+        assert len(actual) == 2
+        assert actual == expected
+        assert [e.metadata.to_dict() for e in actual] == [e.metadata.to_dict() for e in expected]
+
+    def and_text_around_a_block_item_inside_it_is_handled_as_inside_a_span(self):
+        wrapped = etree.fromstring(
+            "<div><foobar>Before<p>Inside</p>after</foobar></div>", html_parser
+        )
+        spanned = etree.fromstring("<div><span>Before<p>Inside</p>after</span></div>", html_parser)
+
+        actual = list(wrapped.find("body").iter_elements())
+
+        assert [e.text for e in actual] == ["Before", "Inside", "after"]
+        assert actual == list(spanned.find("body").iter_elements())
+
+    def and_it_neither_adds_nor_drops_annotations_of_its_contents(self):
+        p = etree.fromstring(
+            "<p><i>Read <foobar>the <b>bold</b> print</foobar></i> and"
+            ' <foobar><a href="https://example.com">the <foobar>link</foobar></a></foobar>.</p>',
+            html_parser,
+        ).xpath(".//p")[0]
+
+        (element,) = p.iter_elements()
+
+        assert element.text == "Read the bold print and the link."
+        assert element.metadata.to_dict() == {
+            "emphasized_text_contents": ["Read", "the", "bold", "print"],
+            "emphasized_text_tags": ["i", "i", "bi", "i"],
+            "link_texts": ["the link"],
+            "link_urls": ["https://example.com"],
+        }

test_unstructured/partition/html/test_partition.py

@@ -773,6 +773,126 @@ def test_partition_html_tag_tail_parsing():
     assert elements == [Text("Head"), Text("Nested"), Text("Tail")]
 
 
+def test_partition_html_keeps_the_text_of_font_elements():
+    """`<font>` is not a registered element but is still common, in email bodies especially."""
+    html_text = (
+        "<div>\n"
+        '  <p><font face="Arial"><b>From:</b> Maria Lopez</font> (Procurement)</p>\n'
+        '  <font size="2">\n'
+        "    <p>The vendor confirmed the delivery date.</p>\n"
+        "    <p>Please sign the revised purchase order by Friday.</p>\n"
+        "  </font>\n"
+        "</div>\n"
+    )
+
+    elements = partition_html(text=html_text)
+
+    assert [e.text for e in elements] == [
+        "From: Maria Lopez (Procurement)",
+        "The vendor confirmed the delivery date.",
+        "Please sign the revised purchase order by Friday.",
+    ]
+    assert elements[0].metadata.emphasized_text_contents == ["From:"]
+
+
+def test_partition_html_keeps_inline_xbrl_facts_but_not_the_hidden_xbrl_header():
+    """SEC filings are Inline XBRL; a fact can wrap a number or a whole note, tables included."""
+    html_text = (
+        "<body>\n"
+        '<div style="display:none">\n'
+        "  <ix:header>\n"
+        "    <ix:hidden>\n"
+        '      <ix:nonNumeric name="dei:AmendmentFlag" contextRef="c-1">false</ix:nonNumeric>\n'
+        "    </ix:hidden>\n"
+        "    <ix:resources>\n"
+        '      <xbrli:context id="c-1"><xbrli:entity><xbrli:identifier scheme="http://www.sec.gov/'
+        'CIK">0000013156</xbrli:identifier></xbrli:entity></xbrli:context>\n'
+        "    </ix:resources>\n"
+        "  </ix:header>\n"
+        "</div>\n"
+        '<p>Revenue was $<ix:nonFraction name="us-gaap:Revenues" contextRef="c-1" unitRef="usd"'
+        ' decimals="0">19,984,378</ix:nonFraction> for the year.</p>\n'
+        '<ix:nonNumeric name="us-gaap:InventoryDisclosureTextBlock" contextRef="c-1">\n'
+        "  <p>NOTE 4. INVENTORY</p>\n"
+        "  <p>Inventory consisted of the following as of December 31, 2021:</p>\n"
+        "  <table><tr><td>Finished goods</td><td>$ 356,928</td></tr></table>\n"
+        "</ix:nonNumeric>\n"
+        "</body>\n"
+    )
+
+    elements = partition_html(text=html_text)
+
+    assert [e.text for e in elements] == [
+        "Revenue was $19,984,378 for the year.",
+        "NOTE 4. INVENTORY",
+        "Inventory consisted of the following as of December 31, 2021:",
+        "Finished goods $ 356,928",
+    ]
+    assert isinstance(elements[3], Table)
+
+
+@pytest.mark.parametrize("wrapper", ["<font size='2'>{}</font>", "<my-card>{}</my-card>"])
+def test_partition_html_keeps_each_formula_once_inside_an_unrecognized_element(wrapper: str):
+    html = "<div>{}</div>".format(
+        wrapper.format(
+            "The error rate is"
+            " <math><semantics><mrow><mn>10</mn></mrow>"
+            '<annotation encoding="application/x-tex">10^{-4}</annotation></semantics></math>'
+            " and the loss is"
+            ' <span class="katex">'
+            '<span class="katex-mathml"><math alttext="x^2"><semantics>'
+            "<msup><mi>x</mi><mn>2</mn></msup>"
+            '<annotation encoding="application/x-tex">x^2</annotation>'
+            "</semantics></math></span>"
+            '<span class="katex-html" aria-hidden="true">'
+            '<span class="base"><span>x</span><span>2</span></span>'
+            "</span></span>"
+            " per token."
+        )
+    )
+
+    elements = partition_html(text=html)
+
+    assert [e.text for e in elements] == [
+        "The error rate is 10^{-4} and the loss is x^2 per token."
+    ]
+
+
+@pytest.mark.parametrize("wrapper", ["<font size='2'>{}</font>", "<my-card>{}</my-card>"])
+def test_partition_html_keeps_katex_html_of_unusable_mathml_inside_an_unrecognized_element(
+    wrapper: str,
+):
+    html = "<div>{}</div>".format(
+        wrapper.format(
+            "Before"
+            ' <span class="katex">'
+            '<span class="katex-mathml"><math><semantics><mi>x</mi></semantics></math></span>'
+            '<span class="katex-html" aria-hidden="true">'
+            '<span class="base"><span>x</span><span>2</span></span>'
+            "</span></span>"
+            " After"
+        )
+    )
+
+    elements = partition_html(text=html)
+
+    assert [e.text for e in elements] == ["Before x2 After"]
+
+
+@pytest.mark.parametrize("wrapper", ["<font size='2'>{}</font>", "<my-card>{}</my-card>"])
+def test_partition_html_keeps_display_math_boundaries_inside_an_unrecognized_element(
+    wrapper: str,
+):
+    html = '<div data-page-number="7">{}</div>'.format(
+        wrapper.format('Before<math display="block" alttext="x=1"></math>After')
+    )
+
+    elements = partition_html(text=html)
+
+    assert [e.text for e in elements] == ["Before", "x=1", "After"]
+    assert [e.metadata.page_number for e in elements] == [7, 7, 7]
+
+
 # -- parsing edge cases --------------------------------------------------------------------------
 
 

unstructured/__version__.py

@@ -1 +1 @@
-__version__ = "0.27.28-dev0"  # pragma: no cover
+__version__ = "0.27.28-dev1"  # pragma: no cover