From 7a0efd03d99da390f5610f045d82b093934bdb19 Mon Sep 17 00:00:00 2001 From: huuufu <61640849+huuufu@users.noreply.github.com> Date: Thu, 24 Sep 2026 13:38:43 +0900 Subject: [PATCH 1/2] fix(html): keep text inside unrecognized elements The v1 HTML parser gives every element without a registered class a DefaultElement, which emitted only the element's tail and silently dropped its own text and everything inside it: text in email bodies, custom elements, and the Inline XBRL facts and notes of SEC filings (example-10k.html lost about a third of its text, including Notes 1-14). This regressed in the parser rewrite (#3218, 0.15.0). An unrecognized element is now transparent. It still identifies as phrasing, like the `display: inline` a browser gives an unknown element, and any block items inside it become their own elements, just as inside a . So the same behavior fits a wrapped number and a whole iXBRL note with tables. Elements whose content is not rendered as text stay skipped. They are registered as RemovedPhrasing, which behaves exactly as DefaultElement did: head, title, svg, math, iframe, object, audio, video, canvas, noembed, noframes, select, datalist, textarea, dialog, legacy xml data islands, and the hidden ix:header (matched by its conventional `ix` prefix, since the HTML parser has no namespaces). Flow._page_number now walks ancestors in a loop instead of recursing: a block can be nested in hundreds of unrecognized (Flow) elements while phrasing traversal is already that deep, which raised RecursionError. Fixes #4496. Resolves #3842. --- CHANGELOG.md | 6 + .../partition/html/test_parser.py | 148 ++++++++++++++++-- .../partition/html/test_partition.py | 58 +++++++ unstructured/__version__.py | 2 +- unstructured/partition/html/parser.py | 81 +++++----- 5 files changed, 242 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac7833f0c5..3c717db4b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 0.27.9-dev0 + +### Fixes + +- **Keep the text inside HTML elements the parser does not recognize.** Since 0.15.0, `partition_html()` (and the `.md`, `.eml`, `.msg`, `.epub`, `.rst` and `.org` partitioners that route through it) kept only the text *after* an element with no registered element class and dropped its content: `` text in email bodies, custom elements, and the Inline XBRL facts and notes of SEC filings (`example-10k.html` lost about a third of its text, including Notes 1-14). An unrecognized element is now transparent, like a ``: its text joins the surrounding paragraph and any block items it contains become their own elements. Elements whose content is not rendered as text (``, ``, `<svg>`, `<math>`, `<iframe>`, `<object>`, `<audio>`, `<video>`, `<canvas>`, `<noembed>`, `<noframes>`, `<select>`, `<datalist>`, `<textarea>`, `<dialog>`, and legacy `<xml>` data islands) and the hidden Inline XBRL `<ix:header>` are still skipped. Resolves #4496 and #3842. + ## 0.27.8 ### Enhancements diff --git a/test_unstructured/partition/html/test_parser.py b/test_unstructured/partition/html/test_parser.py index 4fe3e8f4c2..56a2f1d134 100644 --- a/test_unstructured/partition/html/test_parser.py +++ b/test_unstructured/partition/html/test_parser.py @@ -557,6 +557,20 @@ def it_falls_back_to_outer_page_number_when_inner_is_non_numeric(self): p = etree.fromstring(html, html_parser).xpath(".//p")[0] assert p._page_number == 1 + def it_finds_page_number_of_a_block_nested_in_hundreds_of_unrecognized_elements(self): + """Unrecognized elements are Flow, so each one is an ancestor searched for a page number. + + That search happens while phrasing traversal is already one or more stack frames deep for + each of those elements, so it must not add a stack frame per ancestor. + """ + html = f'<div data-page-number="7">{"<foobar>" * 250}<p>text</p></div>' + div = etree.fromstring(html, html_parser).xpath(".//div")[0] + + (element,) = div.iter_elements() + + assert element == Text("text") + assert element.metadata.page_number == 7 + # -- ._element_from_text_or_tail() ------------------------------------ def it_assembles_text_and_tail_document_elements_to_help(self): @@ -1533,6 +1547,42 @@ def it_behaves_like_an_empty_element(self): assert label.is_phrasing is True assert text_segment.text == "\n Like vastly, hugely big.\n" + @pytest.mark.parametrize( + "html_text", + [ + "<audio src='a.mp3'>Your browser does not support audio.</audio>", + "<canvas>Your browser does not support canvas.</canvas>", + "<datalist><option value='Chrome'>Chrome</option></datalist>", + "<dialog>Subscribe to our newsletter</dialog>", + "<dialog open>Subscribe to our newsletter</dialog>", + "<iframe src='a.html'>Your browser does not support iframes.</iframe>", + "<ix:header><ix:hidden><ix:nonNumeric name='dei:AmendmentFlag'>false</ix:nonNumeric>" + "</ix:hidden></ix:header>", + "<math><mi>x</mi><mo>=</mo><mn>2</mn></math>", + "<noembed>Your browser does not support plugins.</noembed>", + "<noframes>Your browser does not support frames.</noframes>", + "<object data='a.pdf'><p>Download the PDF instead.</p></object>", + "<select><option>Alpha</option><option>Beta</option></select>", + "<svg><title>Revenue chart2021", + "", + "Page title", + "", + "Jane", + ], + ) + def it_is_used_for_elements_whose_content_is_not_rendered_as_text(self, html_text: str): + html = f"
Before {html_text} after.
" + div = etree.fromstring(html, html_parser).xpath(".//div")[0] + + assert isinstance(div[0], RemovedPhrasing) + assert [e.text for e in div.iter_elements()] == ["Before after."] + + def and_it_is_used_for_head_which_is_only_reached_when_there_is_no_body(self): + root = etree.fromstring("Metadata", html_parser) + + assert isinstance(root.find("head"), RemovedPhrasing) + assert list(root.iter_elements()) == [] + # -- DEFAULT ELEMENT ----------------------------------------------------------------------------- @@ -1543,9 +1593,9 @@ class DescribeDefaultElement: Used for any element we haven't assigned a custom element-class too. This prominently includes any non-HTML elements that can be embedded in the HTML. - It identifies as a block item but it can behave as either a block-item or phrasing. Its behavior - is a combination of RemovedBlock and RemovedPhrasing. Namely, it iterates zero elements and only - iterates a text-segment for its tail. + It identifies as phrasing but it can behave as either a block-item or phrasing. Either way it is + transparent; its text, its children, and its tail are processed as though its tags were not + there. """ # -- .is_phrasing ----------------------------------------------------- @@ -1558,36 +1608,43 @@ def it_identifies_as_a_phrasing_element(self): # -- .iter_elements() ------------------------------------------------- - def it_generates_zero_elements_as_a_block_item(self): - """Should never be called but belts and suspenders.""" + def it_generates_the_elements_of_its_contents_as_a_block_item(self): + """Only happens when it is the root, like an `` element that has no ``.""" foobar = etree.fromstring( "Space

is big

, mind-bogglingly big.", html_parser, ).xpath(".//foobar")[0] - elements = foobar.iter_elements() + texts = [e.text for e in foobar.iter_elements()] - with pytest.raises(StopIteration): - next(elements) + assert texts == ["Space", "is big", ", mind-bogglingly big."] # -- .iter_text_segments() -------------------------------------------- - def it_generates_its_tail_but_no_inner_text_segments_when_called_like_phrasing(self): + def it_generates_text_segments_for_its_text_and_children_and_tail_when_called_like_phrasing( + self, + ): foobar = etree.fromstring( "
\n" " O Deep Thought computer, he said,\n" - " Vogon Constructor Fleet\n" + " Vogon Constructor Fleet\n" " The task we have designed you to perform is this.\n" "

We want you to tell us.... he paused,

\n" "
", html_parser, ).xpath(".//foobar")[0] - texts = [ts.text for ts in foobar.iter_text_segments()] - - assert texts == ["\n The task we have designed you to perform is this.\n "] + assert list(foobar.iter_text_segments()) == [ + TextSegment("Vogon ", {}), + TextSegment( + "Constructor", + {"emphasized_text_contents": "Constructor", "emphasized_text_tags": "b"}, + ), + TextSegment(" Fleet", {}), + TextSegment("\n The task we have designed you to perform is this.\n ", {}), + ] - def and_it_behaves_like_an_empty_phrasing_element_inside_a_block_element(self): + def and_its_text_is_part_of_the_paragraph_it_appears_in(self): div = etree.fromstring( "
\n" " O Deep Thought computer, he said,\n" @@ -1601,6 +1658,67 @@ def and_it_behaves_like_an_empty_phrasing_element_inside_a_block_element(self): texts = [e.text for e in div.iter_elements()] assert texts == [ - "O Deep Thought computer, he said, The task we have designed you to perform is this.", + "O Deep Thought computer, he said, Vogon Constructor Fleet The task we have designed" + " you to perform is this.", "We want you to tell us.... he paused,", ] + + @pytest.mark.parametrize( + "wrapper", + [ + "{}", + "{}", + # -- nested unrecognized elements are each transparent -- + "{}", + ], + ) + @pytest.mark.parametrize( + "content", + [ + "

Revenue increased in 2021.

Costs were flat.

", + "

NOTE 4. INVENTORY

Inventory consisted of the following:

", + "
Inventory
Finished goods356,928
", + "
  • Patents
  • Trademarks
", + ], + ) + def and_block_items_inside_it_produce_the_same_elements_as_without_it( + self, wrapper: str, content: str + ): + """So it can wrap whole sections, like an Inline XBRL note, as well as a few words.""" + wrapped = etree.fromstring(wrapper.format(content), html_parser).find("body") + bare = etree.fromstring(content, html_parser).find("body") + + actual = list(wrapped.iter_elements()) + expected = list(bare.iter_elements()) + + assert len(actual) == 2 + assert actual == expected + assert [e.metadata.to_dict() for e in actual] == [e.metadata.to_dict() for e in expected] + + def and_text_around_a_block_item_inside_it_is_handled_as_inside_a_span(self): + wrapped = etree.fromstring( + "
Before

Inside

after
", html_parser + ) + spanned = etree.fromstring("
Before

Inside

after
", html_parser) + + actual = list(wrapped.find("body").iter_elements()) + + assert [e.text for e in actual] == ["Before", "Inside", "after"] + assert actual == list(spanned.find("body").iter_elements()) + + def and_it_neither_adds_nor_drops_annotations_of_its_contents(self): + p = etree.fromstring( + "

Read the bold print and" + ' the link.

', + html_parser, + ).xpath(".//p")[0] + + (element,) = p.iter_elements() + + assert element.text == "Read the bold print and the link." + assert element.metadata.to_dict() == { + "emphasized_text_contents": ["Read", "the", "bold", "print"], + "emphasized_text_tags": ["i", "i", "bi", "i"], + "link_texts": ["the link"], + "link_urls": ["https://example.com"], + } diff --git a/test_unstructured/partition/html/test_partition.py b/test_unstructured/partition/html/test_partition.py index a3f0d7740d..d888a9c366 100644 --- a/test_unstructured/partition/html/test_partition.py +++ b/test_unstructured/partition/html/test_partition.py @@ -772,6 +772,64 @@ def test_partition_html_tag_tail_parsing(): assert elements == [Text("Head"), Text("Nested"), Text("Tail")] +def test_partition_html_keeps_the_text_of_font_elements(): + """`` is not a registered element but is still common, in email bodies especially.""" + html_text = ( + "
\n" + '

From: Maria Lopez (Procurement)

\n' + ' \n' + "

The vendor confirmed the delivery date.

\n" + "

Please sign the revised purchase order by Friday.

\n" + " \n" + "
\n" + ) + + elements = partition_html(text=html_text) + + assert [e.text for e in elements] == [ + "From: Maria Lopez (Procurement)", + "The vendor confirmed the delivery date.", + "Please sign the revised purchase order by Friday.", + ] + assert elements[0].metadata.emphasized_text_contents == ["From:"] + + +def test_partition_html_keeps_inline_xbrl_facts_but_not_the_hidden_xbrl_header(): + """SEC filings are Inline XBRL; a fact can wrap a number or a whole note, tables included.""" + html_text = ( + "\n" + '
\n' + " \n" + " \n" + ' false\n' + " \n" + " \n" + ' 0000013156\n' + " \n" + " \n" + "
\n" + '

Revenue was $19,984,378 for the year.

\n' + '\n' + "

NOTE 4. INVENTORY

\n" + "

Inventory consisted of the following as of December 31, 2021:

\n" + "
Finished goods$ 356,928
\n" + "\n" + "\n" + ) + + elements = partition_html(text=html_text) + + assert [e.text for e in elements] == [ + "Revenue was $19,984,378 for the year.", + "NOTE 4. INVENTORY", + "Inventory consisted of the following as of December 31, 2021:", + "Finished goods $ 356,928", + ] + assert isinstance(elements[3], Table) + + # -- parsing edge cases -------------------------------------------------------------------------- diff --git a/unstructured/__version__.py b/unstructured/__version__.py index fd75f2ac73..552da117b8 100644 --- a/unstructured/__version__.py +++ b/unstructured/__version__.py @@ -1 +1 @@ -__version__ = "0.27.8" # pragma: no cover +__version__ = "0.27.9-dev0" # pragma: no cover diff --git a/unstructured/partition/html/parser.py b/unstructured/partition/html/parser.py index 61c568f945..ddae90a43d 100644 --- a/unstructured/partition/html/parser.py +++ b/unstructured/partition/html/parser.py @@ -351,15 +351,17 @@ def is_phrasing(self) -> bool: @cached_property def _page_number(self) -> int | None: """Page number from nearest ancestor (or self) with a valid `data-page-number` attribute.""" - page_attr = self.get("data-page-number") - if page_attr is not None: - try: - return int(page_attr) - except (ValueError, TypeError): - pass - parent = self.getparent() - if parent is not None and isinstance(parent, Flow): - return parent._page_number + # -- iterative, not recursive, because this can be called from deep inside nested + # -- unrecognized elements (which are Flow) where the call stack is already deep -- + element: etree._Element | None = self + while isinstance(element, Flow): + page_attr = element.get("data-page-number") + if page_attr is not None: + try: + return int(page_attr) + except (ValueError, TypeError): + pass + element = element.getparent() return None def iter_elements(self) -> Iterator[Element]: @@ -922,12 +924,18 @@ def iter_text_segments(self, enclosing_emphasis: str = "") -> Iterator[TextSegme class DefaultElement(Flow, Phrasing): """Custom element-class used for any element without an assigned custom element class. - An unrecognized element is given both Flow (block) and Phrasing (inline) behaviors. It behaves - like a Flow element When nested in a Flow element like a Phrasing element when nested in a - Phrasing element. + This includes custom elements like ``, obsolete elements like ``, and elements + from other vocabularies embedded in the HTML, like Inline XBRL ``. - The contents of the element is skipped in either case, but its tail is not when it behaves as a - Phrasing element. The tail is processed by its parent when that is a Flow element. + An unrecognized element is _transparent_. Its text, children, and tail are processed as though + its tags were not there. It is given both Flow (block) and Phrasing (inline) behaviors. It + identifies as phrasing, so its text is part of the paragraph it appears in, while any block + items it contains, like `

` or ``, each still produce their own document-elements, + just as they would inside a ``. Its Flow `.iter_elements()` is only called when it is the + root of the parsed document, like an `` element that has no ``. + + An element whose content should not appear in the document, like ``; not displayed -- + "dialog": RemovedPhrasing, # -- popup UI; hidden until opened, likely boilerplate -- + "head": RemovedPhrasing, # -- document metadata; only reached when there's no `` -- + "iframe": RemovedPhrasing, # -- displays another document; its own content is not shown -- + # -- Inline XBRL hidden facts, contexts, and units. The HTML parser has no namespaces so + # -- this matches the conventional `ix` prefix only, not another prefix bound to the same + # -- Inline XBRL namespace. + "ix:header": RemovedPhrasing, + "math": RemovedPhrasing, # -- MathML; no plain-text representation yet -- + "noembed": RemovedPhrasing, # -- obsolete; fallback content for `` -- + "noframes": RemovedPhrasing, # -- obsolete; fallback content for frames -- + "object": RemovedPhrasing, # -- content is fallback for when the object can't be shown -- + "select": RemovedPhrasing, # -- form control; its option list is not document text -- + "svg": RemovedPhrasing, # -- graphics; `` and `<desc>` inside it are metadata -- + "textarea": RemovedPhrasing, # -- form control; its content is the editable value -- + "title": RemovedPhrasing, # -- document metadata; lands in `<body>` in malformed HTML -- + "video": RemovedPhrasing, # -- content is fallback for browsers without video support -- + "xml": RemovedPhrasing, # -- legacy "data island", like Office document-properties -- # -- removed block -- "details": RemovedBlock, # -- likely boilerplate -- "dl": RemovedBlock, From 9e54d3098e05fce256ded70a5db071120b533d45 Mon Sep 17 00:00:00 2001 From: huuufu <61640849+huuufu@users.noreply.github.com> Date: Thu, 15 Oct 2026 10:00:39 +0900 Subject: [PATCH 2/2] fix(html): keep the MathML handler from #4478 for <math> #4478 registered `"math": Math`. This branch had registered `"math": RemovedPhrasing` later in the same lookup dict, so after the merge the later entry won: formulas were silently dropped, and for paired KaTeX output outside a table `Math._normalize_katex()` read `._latex` on a RemovedPhrasing instance and raised AttributeError, failing the whole partition_html() call. Remove this branch's mapping so `<math>` keeps the single Math registration, and drop `<math>` from the RemovedPhrasing test cases and the CHANGELOG list. Math still skips MathML that has no TeX annotation or alttext while keeping its tail; that is now covered under Math, bare and inside unrecognized wrappers. Add combined coverage inside a newly transparent `<font>` or custom element: each formula appears once (TeX annotation and paired KaTeX), KaTeX HTML is kept when its MathML is unusable, and display math keeps its element boundaries and page number. --- CHANGELOG.md | 2 +- .../partition/html/test_parser.py | 20 +++++- .../partition/html/test_partition.py | 62 +++++++++++++++++++ unstructured/partition/html/parser.py | 1 - 4 files changed, 82 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 69eee6f1e5..e302802fa7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ### Fixes -- **Keep the text inside HTML elements the parser does not recognize.** Since 0.15.0, `partition_html()` (and the `.md`, `.eml`, `.msg`, `.epub`, `.rst` and `.org` partitioners that route through it) kept only the text *after* an element with no registered element class and dropped its content: `<font>` text in email bodies, custom elements, and the Inline XBRL facts and notes of SEC filings (`example-10k.html` lost about a third of its text, including Notes 1-14). An unrecognized element is now transparent, like a `<span>`: its text joins the surrounding paragraph and any block items it contains become their own elements. Elements whose content is not rendered as text (`<head>`, `<title>`, `<svg>`, `<math>`, `<iframe>`, `<object>`, `<audio>`, `<video>`, `<canvas>`, `<noembed>`, `<noframes>`, `<select>`, `<datalist>`, `<textarea>`, `<dialog>`, and legacy `<xml>` data islands) and the hidden Inline XBRL `<ix:header>` are still skipped. Resolves #4496 and #3842. +- **Keep the text inside HTML elements the parser does not recognize.** Since 0.15.0, `partition_html()` (and the `.md`, `.eml`, `.msg`, `.epub`, `.rst` and `.org` partitioners that route through it) kept only the text *after* an element with no registered element class and dropped its content: `<font>` text in email bodies, custom elements, and the Inline XBRL facts and notes of SEC filings (`example-10k.html` lost about a third of its text, including Notes 1-14). An unrecognized element is now transparent, like a `<span>`: its text joins the surrounding paragraph and any block items it contains become their own elements. Elements whose content is not rendered as text (`<head>`, `<title>`, `<svg>`, `<iframe>`, `<object>`, `<audio>`, `<video>`, `<canvas>`, `<noembed>`, `<noframes>`, `<select>`, `<datalist>`, `<textarea>`, `<dialog>`, and legacy `<xml>` data islands) and the hidden Inline XBRL `<ix:header>` are still skipped. Resolves #4496 and #3842. ## 0.27.27 diff --git a/test_unstructured/partition/html/test_parser.py b/test_unstructured/partition/html/test_parser.py index 56a2f1d134..37014f18f3 100644 --- a/test_unstructured/partition/html/test_parser.py +++ b/test_unstructured/partition/html/test_parser.py @@ -23,6 +23,7 @@ Annotation, DefaultElement, Flow, + Math, Phrasing, RemovedPhrasing, TextSegment, @@ -1525,6 +1526,24 @@ def it_adds_a_newline_in_its_place(self): assert _normalize_text("".join(texts)) == "spaceships of the Vogon Constructor Fleet" +class DescribeMath: + """Isolated unit-test suite for `unstructured.partition.html.parser.Math`. + + Used for `<math>` elements. Its text extraction is covered by the MathML tests in + `test_partition.py`. + """ + + @pytest.mark.parametrize("wrapper", ["{}", "<font size='2'>{}</font>", "<my-card>{}</my-card>"]) + def it_skips_mathml_that_has_no_text_representation_but_not_its_tail(self, wrapper: str): + html = "<div>Before {} after.</div>".format( + wrapper.format("<math><mi>x</mi><mo>=</mo><mn>2</mn></math>") + ) + div = etree.fromstring(html, html_parser).xpath(".//div")[0] + + assert isinstance(div.xpath(".//math")[0], Math) + assert [e.text for e in div.iter_elements()] == ["Before after."] + + class DescribeRemovedPhrasing: """Isolated unit-test suite for `unstructured.partition.html.parser.RemovedPhrasing`. @@ -1558,7 +1577,6 @@ def it_behaves_like_an_empty_element(self): "<iframe src='a.html'>Your browser does not support iframes.</iframe>", "<ix:header><ix:hidden><ix:nonNumeric name='dei:AmendmentFlag'>false</ix:nonNumeric>" "</ix:hidden></ix:header>", - "<math><mi>x</mi><mo>=</mo><mn>2</mn></math>", "<noembed>Your browser does not support plugins.</noembed>", "<noframes>Your browser does not support frames.</noframes>", "<object data='a.pdf'><p>Download the PDF instead.</p></object>", diff --git a/test_unstructured/partition/html/test_partition.py b/test_unstructured/partition/html/test_partition.py index 4e9f1f6f81..3acac0b168 100644 --- a/test_unstructured/partition/html/test_partition.py +++ b/test_unstructured/partition/html/test_partition.py @@ -831,6 +831,68 @@ def test_partition_html_keeps_inline_xbrl_facts_but_not_the_hidden_xbrl_header() assert isinstance(elements[3], Table) +@pytest.mark.parametrize("wrapper", ["<font size='2'>{}</font>", "<my-card>{}</my-card>"]) +def test_partition_html_keeps_each_formula_once_inside_an_unrecognized_element(wrapper: str): + html = "<div>{}</div>".format( + wrapper.format( + "The error rate is" + " <math><semantics><mrow><mn>10</mn></mrow>" + '<annotation encoding="application/x-tex">10^{-4}</annotation></semantics></math>' + " and the loss is" + ' <span class="katex">' + '<span class="katex-mathml"><math alttext="x^2"><semantics>' + "<msup><mi>x</mi><mn>2</mn></msup>" + '<annotation encoding="application/x-tex">x^2</annotation>' + "</semantics></math></span>" + '<span class="katex-html" aria-hidden="true">' + '<span class="base"><span>x</span><span>2</span></span>' + "</span></span>" + " per token." + ) + ) + + elements = partition_html(text=html) + + assert [e.text for e in elements] == [ + "The error rate is 10^{-4} and the loss is x^2 per token." + ] + + +@pytest.mark.parametrize("wrapper", ["<font size='2'>{}</font>", "<my-card>{}</my-card>"]) +def test_partition_html_keeps_katex_html_of_unusable_mathml_inside_an_unrecognized_element( + wrapper: str, +): + html = "<div>{}</div>".format( + wrapper.format( + "Before" + ' <span class="katex">' + '<span class="katex-mathml"><math><semantics><mi>x</mi></semantics></math></span>' + '<span class="katex-html" aria-hidden="true">' + '<span class="base"><span>x</span><span>2</span></span>' + "</span></span>" + " After" + ) + ) + + elements = partition_html(text=html) + + assert [e.text for e in elements] == ["Before x2 After"] + + +@pytest.mark.parametrize("wrapper", ["<font size='2'>{}</font>", "<my-card>{}</my-card>"]) +def test_partition_html_keeps_display_math_boundaries_inside_an_unrecognized_element( + wrapper: str, +): + html = '<div data-page-number="7">{}</div>'.format( + wrapper.format('Before<math display="block" alttext="x=1"></math>After') + ) + + elements = partition_html(text=html) + + assert [e.text for e in elements] == ["Before", "x=1", "After"] + assert [e.metadata.page_number for e in elements] == [7, 7, 7] + + # -- parsing edge cases -------------------------------------------------------------------------- diff --git a/unstructured/partition/html/parser.py b/unstructured/partition/html/parser.py index 745c0b3884..ebc8a08023 100644 --- a/unstructured/partition/html/parser.py +++ b/unstructured/partition/html/parser.py @@ -1206,7 +1206,6 @@ def derive_element_type_from_text(text: str) -> type[Text] | None: # -- this matches the conventional `ix` prefix only, not another prefix bound to the same # -- Inline XBRL namespace. "ix:header": RemovedPhrasing, - "math": RemovedPhrasing, # -- MathML; no plain-text representation yet -- "noembed": RemovedPhrasing, # -- obsolete; fallback content for `<embed>` -- "noframes": RemovedPhrasing, # -- obsolete; fallback content for frames -- "object": RemovedPhrasing, # -- content is fallback for when the object can't be shown --