diff --git a/CHANGELOG.md b/CHANGELOG.md index 4271297c3c..066ad5a40c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,13 @@ -## 0.27.28-dev0 +## 0.27.28-dev1 ### Enhancements - **Optionally use `langdetect-rs` for partition language detection.** Set `UNSTRUCTURED_USE_LANGDETECT_RS=true` before importing `unstructured` and install `langdetect-rs==0.1.0` to detect `metadata.languages` with this faster Rust port of `langdetect`, which matched its seeded results on the tested inputs. If the package is missing, a warning is logged and `langdetect` is used. Without the variable, nothing changes. +### Fixes + +- **Keep the text inside HTML elements the parser does not recognize.** Since 0.15.0, `partition_html()` (and the `.md`, `.eml`, `.msg`, `.epub`, `.rst` and `.org` partitioners that route through it) kept only the text *after* an element with no registered element class and dropped its content: `` text in email bodies, custom elements, and the Inline XBRL facts and notes of SEC filings (`example-10k.html` lost about a third of its text, including Notes 1-14). An unrecognized element is now transparent, like a ``: its text joins the surrounding paragraph and any block items it contains become their own elements. Elements whose content is not rendered as text (``, ``, `<svg>`, `<iframe>`, `<object>`, `<audio>`, `<video>`, `<canvas>`, `<noembed>`, `<noframes>`, `<select>`, `<datalist>`, `<textarea>`, `<dialog>`, and legacy `<xml>` data islands) and the hidden Inline XBRL `<ix:header>` are still skipped. Resolves #4496 and #3842. + ## 0.27.27 ### Fixes diff --git a/test_unstructured/partition/html/test_parser.py b/test_unstructured/partition/html/test_parser.py index 4fe3e8f4c2..37014f18f3 100644 --- a/test_unstructured/partition/html/test_parser.py +++ b/test_unstructured/partition/html/test_parser.py @@ -23,6 +23,7 @@ Annotation, DefaultElement, Flow, + Math, Phrasing, RemovedPhrasing, TextSegment, @@ -557,6 +558,20 @@ def it_falls_back_to_outer_page_number_when_inner_is_non_numeric(self): p = etree.fromstring(html, html_parser).xpath(".//p")[0] assert p._page_number == 1 + def it_finds_page_number_of_a_block_nested_in_hundreds_of_unrecognized_elements(self): + """Unrecognized elements are Flow, so each one is an ancestor searched for a page number. + + That search happens while phrasing traversal is already one or more stack frames deep for + each of those elements, so it must not add a stack frame per ancestor. + """ + html = f'<div data-page-number="7">{"<foobar>" * 250}<p>text</p></div>' + div = etree.fromstring(html, html_parser).xpath(".//div")[0] + + (element,) = div.iter_elements() + + assert element == Text("text") + assert element.metadata.page_number == 7 + # -- ._element_from_text_or_tail() ------------------------------------ def it_assembles_text_and_tail_document_elements_to_help(self): @@ -1511,6 +1526,24 @@ def it_adds_a_newline_in_its_place(self): assert _normalize_text("".join(texts)) == "spaceships of the Vogon Constructor Fleet" +class DescribeMath: + """Isolated unit-test suite for `unstructured.partition.html.parser.Math`. + + Used for `<math>` elements. Its text extraction is covered by the MathML tests in + `test_partition.py`. + """ + + @pytest.mark.parametrize("wrapper", ["{}", "<font size='2'>{}</font>", "<my-card>{}</my-card>"]) + def it_skips_mathml_that_has_no_text_representation_but_not_its_tail(self, wrapper: str): + html = "<div>Before {} after.</div>".format( + wrapper.format("<math><mi>x</mi><mo>=</mo><mn>2</mn></math>") + ) + div = etree.fromstring(html, html_parser).xpath(".//div")[0] + + assert isinstance(div.xpath(".//math")[0], Math) + assert [e.text for e in div.iter_elements()] == ["Before after."] + + class DescribeRemovedPhrasing: """Isolated unit-test suite for `unstructured.partition.html.parser.RemovedPhrasing`. @@ -1533,6 +1566,41 @@ def it_behaves_like_an_empty_element(self): assert label.is_phrasing is True assert text_segment.text == "\n Like vastly, hugely big.\n" + @pytest.mark.parametrize( + "html_text", + [ + "<audio src='a.mp3'>Your browser does not support audio.</audio>", + "<canvas>Your browser does not support canvas.</canvas>", + "<datalist><option value='Chrome'>Chrome</option></datalist>", + "<dialog>Subscribe to our newsletter</dialog>", + "<dialog open>Subscribe to our newsletter</dialog>", + "<iframe src='a.html'>Your browser does not support iframes.</iframe>", + "<ix:header><ix:hidden><ix:nonNumeric name='dei:AmendmentFlag'>false</ix:nonNumeric>" + "</ix:hidden></ix:header>", + "<noembed>Your browser does not support plugins.</noembed>", + "<noframes>Your browser does not support frames.</noframes>", + "<object data='a.pdf'><p>Download the PDF instead.</p></object>", + "<select><option>Alpha</option><option>Beta</option></select>", + "<svg><title>Revenue chart2021", + "", + "Page title", + "", + "Jane", + ], + ) + def it_is_used_for_elements_whose_content_is_not_rendered_as_text(self, html_text: str): + html = f"
Before {html_text} after.
" + div = etree.fromstring(html, html_parser).xpath(".//div")[0] + + assert isinstance(div[0], RemovedPhrasing) + assert [e.text for e in div.iter_elements()] == ["Before after."] + + def and_it_is_used_for_head_which_is_only_reached_when_there_is_no_body(self): + root = etree.fromstring("Metadata", html_parser) + + assert isinstance(root.find("head"), RemovedPhrasing) + assert list(root.iter_elements()) == [] + # -- DEFAULT ELEMENT ----------------------------------------------------------------------------- @@ -1543,9 +1611,9 @@ class DescribeDefaultElement: Used for any element we haven't assigned a custom element-class too. This prominently includes any non-HTML elements that can be embedded in the HTML. - It identifies as a block item but it can behave as either a block-item or phrasing. Its behavior - is a combination of RemovedBlock and RemovedPhrasing. Namely, it iterates zero elements and only - iterates a text-segment for its tail. + It identifies as phrasing but it can behave as either a block-item or phrasing. Either way it is + transparent; its text, its children, and its tail are processed as though its tags were not + there. """ # -- .is_phrasing ----------------------------------------------------- @@ -1558,36 +1626,43 @@ def it_identifies_as_a_phrasing_element(self): # -- .iter_elements() ------------------------------------------------- - def it_generates_zero_elements_as_a_block_item(self): - """Should never be called but belts and suspenders.""" + def it_generates_the_elements_of_its_contents_as_a_block_item(self): + """Only happens when it is the root, like an `` element that has no ``.""" foobar = etree.fromstring( "Space

is big

, mind-bogglingly big.", html_parser, ).xpath(".//foobar")[0] - elements = foobar.iter_elements() + texts = [e.text for e in foobar.iter_elements()] - with pytest.raises(StopIteration): - next(elements) + assert texts == ["Space", "is big", ", mind-bogglingly big."] # -- .iter_text_segments() -------------------------------------------- - def it_generates_its_tail_but_no_inner_text_segments_when_called_like_phrasing(self): + def it_generates_text_segments_for_its_text_and_children_and_tail_when_called_like_phrasing( + self, + ): foobar = etree.fromstring( "
\n" " O Deep Thought computer, he said,\n" - " Vogon Constructor Fleet\n" + " Vogon Constructor Fleet\n" " The task we have designed you to perform is this.\n" "

We want you to tell us.... he paused,

\n" "
", html_parser, ).xpath(".//foobar")[0] - texts = [ts.text for ts in foobar.iter_text_segments()] - - assert texts == ["\n The task we have designed you to perform is this.\n "] + assert list(foobar.iter_text_segments()) == [ + TextSegment("Vogon ", {}), + TextSegment( + "Constructor", + {"emphasized_text_contents": "Constructor", "emphasized_text_tags": "b"}, + ), + TextSegment(" Fleet", {}), + TextSegment("\n The task we have designed you to perform is this.\n ", {}), + ] - def and_it_behaves_like_an_empty_phrasing_element_inside_a_block_element(self): + def and_its_text_is_part_of_the_paragraph_it_appears_in(self): div = etree.fromstring( "
\n" " O Deep Thought computer, he said,\n" @@ -1601,6 +1676,67 @@ def and_it_behaves_like_an_empty_phrasing_element_inside_a_block_element(self): texts = [e.text for e in div.iter_elements()] assert texts == [ - "O Deep Thought computer, he said, The task we have designed you to perform is this.", + "O Deep Thought computer, he said, Vogon Constructor Fleet The task we have designed" + " you to perform is this.", "We want you to tell us.... he paused,", ] + + @pytest.mark.parametrize( + "wrapper", + [ + "{}", + "{}", + # -- nested unrecognized elements are each transparent -- + "{}", + ], + ) + @pytest.mark.parametrize( + "content", + [ + "

Revenue increased in 2021.

Costs were flat.

", + "

NOTE 4. INVENTORY

Inventory consisted of the following:

", + "
Inventory
Finished goods356,928
", + "
  • Patents
  • Trademarks
", + ], + ) + def and_block_items_inside_it_produce_the_same_elements_as_without_it( + self, wrapper: str, content: str + ): + """So it can wrap whole sections, like an Inline XBRL note, as well as a few words.""" + wrapped = etree.fromstring(wrapper.format(content), html_parser).find("body") + bare = etree.fromstring(content, html_parser).find("body") + + actual = list(wrapped.iter_elements()) + expected = list(bare.iter_elements()) + + assert len(actual) == 2 + assert actual == expected + assert [e.metadata.to_dict() for e in actual] == [e.metadata.to_dict() for e in expected] + + def and_text_around_a_block_item_inside_it_is_handled_as_inside_a_span(self): + wrapped = etree.fromstring( + "
Before

Inside

after
", html_parser + ) + spanned = etree.fromstring("
Before

Inside

after
", html_parser) + + actual = list(wrapped.find("body").iter_elements()) + + assert [e.text for e in actual] == ["Before", "Inside", "after"] + assert actual == list(spanned.find("body").iter_elements()) + + def and_it_neither_adds_nor_drops_annotations_of_its_contents(self): + p = etree.fromstring( + "

Read the bold print and" + ' the link.

', + html_parser, + ).xpath(".//p")[0] + + (element,) = p.iter_elements() + + assert element.text == "Read the bold print and the link." + assert element.metadata.to_dict() == { + "emphasized_text_contents": ["Read", "the", "bold", "print"], + "emphasized_text_tags": ["i", "i", "bi", "i"], + "link_texts": ["the link"], + "link_urls": ["https://example.com"], + } diff --git a/test_unstructured/partition/html/test_partition.py b/test_unstructured/partition/html/test_partition.py index c281555c1d..3acac0b168 100644 --- a/test_unstructured/partition/html/test_partition.py +++ b/test_unstructured/partition/html/test_partition.py @@ -773,6 +773,126 @@ def test_partition_html_tag_tail_parsing(): assert elements == [Text("Head"), Text("Nested"), Text("Tail")] +def test_partition_html_keeps_the_text_of_font_elements(): + """`` is not a registered element but is still common, in email bodies especially.""" + html_text = ( + "
\n" + '

From: Maria Lopez (Procurement)

\n' + ' \n' + "

The vendor confirmed the delivery date.

\n" + "

Please sign the revised purchase order by Friday.

\n" + " \n" + "
\n" + ) + + elements = partition_html(text=html_text) + + assert [e.text for e in elements] == [ + "From: Maria Lopez (Procurement)", + "The vendor confirmed the delivery date.", + "Please sign the revised purchase order by Friday.", + ] + assert elements[0].metadata.emphasized_text_contents == ["From:"] + + +def test_partition_html_keeps_inline_xbrl_facts_but_not_the_hidden_xbrl_header(): + """SEC filings are Inline XBRL; a fact can wrap a number or a whole note, tables included.""" + html_text = ( + "\n" + '
\n' + " \n" + " \n" + ' false\n' + " \n" + " \n" + ' 0000013156\n' + " \n" + " \n" + "
\n" + '

Revenue was $19,984,378 for the year.

\n' + '\n' + "

NOTE 4. INVENTORY

\n" + "

Inventory consisted of the following as of December 31, 2021:

\n" + "
Finished goods$ 356,928
\n" + "\n" + "\n" + ) + + elements = partition_html(text=html_text) + + assert [e.text for e in elements] == [ + "Revenue was $19,984,378 for the year.", + "NOTE 4. INVENTORY", + "Inventory consisted of the following as of December 31, 2021:", + "Finished goods $ 356,928", + ] + assert isinstance(elements[3], Table) + + +@pytest.mark.parametrize("wrapper", ["{}", "{}"]) +def test_partition_html_keeps_each_formula_once_inside_an_unrecognized_element(wrapper: str): + html = "
{}
".format( + wrapper.format( + "The error rate is" + " 10" + '10^{-4}' + " and the loss is" + ' ' + '' + "x2" + 'x^2' + "" + '" + " per token." + ) + ) + + elements = partition_html(text=html) + + assert [e.text for e in elements] == [ + "The error rate is 10^{-4} and the loss is x^2 per token." + ] + + +@pytest.mark.parametrize("wrapper", ["{}", "{}"]) +def test_partition_html_keeps_katex_html_of_unusable_mathml_inside_an_unrecognized_element( + wrapper: str, +): + html = "
{}
".format( + wrapper.format( + "Before" + ' ' + 'x' + '" + " After" + ) + ) + + elements = partition_html(text=html) + + assert [e.text for e in elements] == ["Before x2 After"] + + +@pytest.mark.parametrize("wrapper", ["{}", "{}"]) +def test_partition_html_keeps_display_math_boundaries_inside_an_unrecognized_element( + wrapper: str, +): + html = '
{}
'.format( + wrapper.format('BeforeAfter') + ) + + elements = partition_html(text=html) + + assert [e.text for e in elements] == ["Before", "x=1", "After"] + assert [e.metadata.page_number for e in elements] == [7, 7, 7] + + # -- parsing edge cases -------------------------------------------------------------------------- diff --git a/unstructured/__version__.py b/unstructured/__version__.py index 9fdb32c95e..ae8b423c2a 100644 --- a/unstructured/__version__.py +++ b/unstructured/__version__.py @@ -1 +1 @@ -__version__ = "0.27.28-dev0" # pragma: no cover +__version__ = "0.27.28-dev1" # pragma: no cover diff --git a/unstructured/partition/html/parser.py b/unstructured/partition/html/parser.py index 6f4baccaf5..ebc8a08023 100644 --- a/unstructured/partition/html/parser.py +++ b/unstructured/partition/html/parser.py @@ -364,15 +364,17 @@ def is_phrasing(self) -> bool: @cached_property def _page_number(self) -> int | None: """Page number from nearest ancestor (or self) with a valid `data-page-number` attribute.""" - page_attr = self.get("data-page-number") - if page_attr is not None: - try: - return int(page_attr) - except (ValueError, TypeError): - pass - parent = self.getparent() - if parent is not None and isinstance(parent, Flow): - return parent._page_number + # -- iterative, not recursive, because this can be called from deep inside nested + # -- unrecognized elements (which are Flow) where the call stack is already deep -- + element: etree._Element | None = self + while isinstance(element, Flow): + page_attr = element.get("data-page-number") + if page_attr is not None: + try: + return int(page_attr) + except (ValueError, TypeError): + pass + element = element.getparent() return None def iter_elements(self) -> Iterator[Element]: @@ -1053,12 +1055,18 @@ def iter_text_segments(self, enclosing_emphasis: str = "") -> Iterator[TextSegme class DefaultElement(Flow, Phrasing): """Custom element-class used for any element without an assigned custom element class. - An unrecognized element is given both Flow (block) and Phrasing (inline) behaviors. It behaves - like a Flow element When nested in a Flow element like a Phrasing element when nested in a - Phrasing element. + This includes custom elements like ``, obsolete elements like ``, and elements + from other vocabularies embedded in the HTML, like Inline XBRL ``. + + An unrecognized element is _transparent_. Its text, children, and tail are processed as though + its tags were not there. It is given both Flow (block) and Phrasing (inline) behaviors. It + identifies as phrasing, so its text is part of the paragraph it appears in, while any block + items it contains, like `

` or ``, each still produce their own document-elements, + just as they would inside a ``. Its Flow `.iter_elements()` is only called when it is the + root of the parsed document, like an `` element that has no ``. - The contents of the element is skipped in either case, but its tail is not when it behaves as a - Phrasing element. The tail is processed by its parent when that is a Flow element. + An element whose content should not appear in the document, like ``; not displayed -- + "dialog": RemovedPhrasing, # -- popup UI; hidden until opened, likely boilerplate -- + "head": RemovedPhrasing, # -- document metadata; only reached when there's no `` -- + "iframe": RemovedPhrasing, # -- displays another document; its own content is not shown -- + # -- Inline XBRL hidden facts, contexts, and units. The HTML parser has no namespaces so + # -- this matches the conventional `ix` prefix only, not another prefix bound to the same + # -- Inline XBRL namespace. + "ix:header": RemovedPhrasing, + "noembed": RemovedPhrasing, # -- obsolete; fallback content for `` -- + "noframes": RemovedPhrasing, # -- obsolete; fallback content for frames -- + "object": RemovedPhrasing, # -- content is fallback for when the object can't be shown -- + "select": RemovedPhrasing, # -- form control; its option list is not document text -- + "svg": RemovedPhrasing, # -- graphics; `` and `<desc>` inside it are metadata -- + "textarea": RemovedPhrasing, # -- form control; its content is the editable value -- + "title": RemovedPhrasing, # -- document metadata; lands in `<body>` in malformed HTML -- + "video": RemovedPhrasing, # -- content is fallback for browsers without video support -- + "xml": RemovedPhrasing, # -- legacy "data island", like Office document-properties -- # -- removed block -- "details": RemovedBlock, # -- likely boilerplate -- "figure": RemovedBlock,