From 333fb213c477772ec2be886e5d2f159bdef17089 Mon Sep 17 00:00:00 2001 From: L4XB Date: Thu, 24 Sep 2026 22:15:37 +0200 Subject: [PATCH] fix(docx): read content controls and tracked revisions instead of skipping them python-docx reads the w:p and w:tbl children of the body, a cell, a header or a footer, and the runs and hyperlinks directly in a paragraph. Content one level deeper was skipped by partition_docx(): - paragraphs, tables, rows and runs held in a content control (w:sdt), which forms, templates and Word's document-property fields use; - text inserted or moved with track changes on (w:ins, w:moveTo); - runs in custom XML, smart tags, simple fields and bidirectional spans. When the document is loaded, each of these wrappers in the main document, header and footer parts is replaced by its content, in one pass over the tree. Every reader then sees that content without further changes: tables, headers, footers, links, emphasis and page breaks. Deleted and moved-away text stays where python-docx skips it. The automatic table of contents stays unread, and a control still showing its placeholder text is dropped. Resolves #3553 and #1821. --- CHANGELOG.md | 6 ++ test_unstructured/partition/test_docx.py | 126 +++++++++++++++++++++++ unstructured/__version__.py | 2 +- unstructured/partition/docx.py | 73 ++++++++++++- 4 files changed, 204 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac7833f0c5..57cd3aef51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 0.27.9-dev0 + +### Fixes + +- **DOCX content in content controls and tracked revisions is no longer skipped.** `partition_docx()` read only what python-docx finds directly under the body, a table cell, a header or a footer, and directly in a paragraph. So it dropped the paragraphs, tables, rows and runs held in a content control (`w:sdt`, used by forms, templates and Word's document-property fields), text inserted or moved with track changes on and not yet accepted (`w:ins`, `w:moveTo`), and runs in custom XML markup, smart tags, simple fields and bidirectional spans. These wrappers are now replaced by their content before partitioning. Deleted and moved-away text still stays out, the automatic table of contents is still skipped, and a content control's placeholder text is not emitted. Resolves #3553 and #1821. + ## 0.27.8 ### Enhancements diff --git a/test_unstructured/partition/test_docx.py b/test_unstructured/partition/test_docx.py index 02214c3619..37c7967173 100644 --- a/test_unstructured/partition/test_docx.py +++ b/test_unstructured/partition/test_docx.py @@ -15,6 +15,7 @@ import pytest from docx.document import Document from docx.enum.section import WD_SECTION +from docx.oxml import parse_xml from docx.section import Section from docx.text.paragraph import Paragraph from pytest_mock import MockFixture @@ -279,6 +280,131 @@ def test_partition_docx_merged_cell_table_chunks_without_corrupting_rowspan_geom ) +_W_NS = 'xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"' +_REVISION = 'w:author="a" w:date="2024-01-01T00:00:00Z"' + + +def _w_run(text: str) -> str: + return f'{text}' + + +def _w_control(content: str, properties: str = "") -> str: + return ( + f"{properties}" + f"{content}" + ) + + +def _append_to_body(document: Document, xml: str) -> None: + """Insert raw WordprocessingML ahead of the body's closing `w:sectPr`.""" + body = document.element.body + body.insert(len(body) - 1, parse_xml(xml)) + + +def test_partition_docx_reads_content_nested_in_content_controls_and_revisions(tmp_path): + """python-docx skips content one level below the body or a paragraph; none of it is lost.""" + document = docx.Document() + _append_to_body(document, _w_control(f"{_w_run('In a content control')}")) + _append_to_body( + document, + f'' + f"{_w_run('In custom XML')}", + ) + for content in ( + _w_run("Signed by ") + + f"{_w_run('Jane Doe')}", + _w_run("Dose: ") + + f'{_w_run("20 mg")}' + + f'10 mg', + f'{_w_run("Moved away")}' + f'{_w_run("Moved here")}', + _w_run("Updated ") + f'{_w_run("2024-05-01")}', + f'{_w_run("Reutlingen")}', + f'{_w_run("Custom inline")}', + f'{_w_run("Embedded")}' + f'{_w_run(" override")}', + f'' + f"{_w_run('Inserted paragraph')}", + ): + _append_to_body(document, f"{content}") + docx_path = tmp_path / "nested-content.docx" + document.save(str(docx_path)) + + elements = partition_docx(str(docx_path)) + + assert [e.text for e in elements] == [ + "In a content control", + "In custom XML", + "Signed by Jane Doe", + "Dose: 20 mg", + "Moved here", + "Updated 2024-05-01", + "Reutlingen", + "Custom inline", + "Embedded override", + "Inserted paragraph", + ] + + +def test_partition_docx_reads_content_controls_in_tables_headers_and_footers(tmp_path): + document = docx.Document() + table = document.add_table(rows=1, cols=2) + table.cell(0, 0).text = "Name" + table.cell(0, 1)._tc.p_lst[0].append(parse_xml(_w_control(_w_run("Jane Doe")))) + # -- a repeating-section item: the row itself is wrapped in a content control -- + table._tbl.append( + parse_xml( + _w_control( + f"{_w_run('Role')}" + f"{_w_run('Reviewer')}" + ) + ) + ) + section = document.sections[0] + section.header.paragraphs[0]._p.append(parse_xml(_w_control(_w_run("Quarterly report")))) + section.footer.paragraphs[0]._p.append(parse_xml(_w_control(_w_run("Confidential")))) + docx_path = tmp_path / "controls-in-tables-and-headers.docx" + document.save(str(docx_path)) + + elements = partition_docx(str(docx_path), infer_table_structure=True) + + table_element = next(e for e in elements if isinstance(e, Table)) + assert table_element.text == "Name Jane Doe Role Reviewer" + assert table_element.metadata.text_as_html == ( + "" + "
NameJane Doe
RoleReviewer
" + ) + assert [e.text for e in elements if isinstance(e, Header)] == ["Quarterly report"] + assert [e.text for e in elements if isinstance(e, Footer)] == ["Confidential"] + + +def test_partition_docx_leaves_out_the_table_of_contents_and_placeholder_text(tmp_path): + """The automatic table of contents stays unread, and so does a control's placeholder text.""" + toc = '' + document = docx.Document() + document.add_paragraph("Before") + _append_to_body( + document, + _w_control( + f"{_w_run('Contents')}{_w_run('Introduction')}" + f"{_w_run('1')}", + properties=toc, + ), + ) + _append_to_body( + document, + f"{_w_run('Due date: ')}" + f"{_w_run('Click or tap to enter a date.')}", + ) + document.add_paragraph("After") + docx_path = tmp_path / "toc-and-placeholder.docx" + document.save(str(docx_path)) + + elements = partition_docx(str(docx_path)) + + assert [e.text for e in elements] == ["Before", "Due date: ", "After"] + + def test_partition_docx_grabs_header_and_footer(): elements = partition_docx(example_doc_path("handbook-1p.docx")) diff --git a/unstructured/__version__.py b/unstructured/__version__.py index fd75f2ac73..552da117b8 100644 --- a/unstructured/__version__.py +++ b/unstructured/__version__.py @@ -1 +1 @@ -__version__ = "0.27.8" # pragma: no cover +__version__ = "0.27.9-dev0" # pragma: no cover diff --git a/unstructured/partition/docx.py b/unstructured/partition/docx.py index 7fba4759c3..6c8d865492 100644 --- a/unstructured/partition/docx.py +++ b/unstructured/partition/docx.py @@ -16,6 +16,9 @@ from docx.enum.section import WD_SECTION_START from docx.oxml.table import CT_Tbl from docx.oxml.text.paragraph import CT_P +from docx.oxml.xmlchemy import BaseOxmlElement +from docx.parts.document import DocumentPart +from docx.parts.hdrftr import FooterPart, HeaderPart from docx.section import Section, _Footer, _Header from docx.table import Table as DocxTable from docx.table import _Cell, _Row @@ -231,8 +234,16 @@ def register_picture_partitioner(cls, picture_partitioner: PicturePartitionerT): @cached_property def document(self) -> Document: - """The python-docx `Document` object loaded from file or filename.""" - return docx.Document(self._docx_file) + """The python-docx `Document` object loaded from file or filename. + + Content that python-docx would skip because it is nested in a content control, a tracked + revision or a similar wrapper is lifted into place first; see `_unwrap_nested_content()`. + """ + document = docx.Document(self._docx_file) + for part in document.part.package.iter_parts(): + if isinstance(part, (DocumentPart, HeaderPart, FooterPart)): + _unwrap_nested_content(part.element) + return document @cached_property def include_page_breaks(self) -> bool: @@ -1017,6 +1028,64 @@ def _table_emphasis(self, table: DocxTable) -> tuple[list[str], list[str]]: return ([e["text"] for e in iter_tbl_emph], [e["tag"] for e in iter_tbl_emph_2]) +# ================================================================================================ +# NESTED CONTENT +# ================================================================================================ + +_W = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}" +# -- wrappers whose content is ordinary content: the content is kept, the wrapper removed -- +_UNWRAPPED_TAGS = frozenset( + _W + tag for tag in ("ins", "moveTo", "customXml", "smartTag", "fldSimple", "dir", "bdo") +) + + +def _unwrap_nested_content(root: BaseOxmlElement) -> None: + """Lift content python-docx would skip into the place it occupies, in the tree under `root`. + + python-docx reads the `w:p` and `w:tbl` children of the body, a cell, a header or a footer, + and the runs and hyperlinks directly in a paragraph. Content one level deeper is skipped: + + - a content control (`w:sdt`) keeps its paragraphs, tables, rows, cells or runs in + `w:sdtContent`; + - an insertion or a move made with track changes on (`w:ins`, `w:moveTo`) wraps its runs; + - custom XML markup, a smart tag, a simple field and a bidirectional span (`w:customXml`, + `w:smartTag`, `w:fldSimple`, `w:dir`, `w:bdo`) wrap theirs. + + Each wrapper is replaced by its content. Deleted and moved-away content (`w:del`, + `w:moveFrom`) stays where it is, where python-docx skips it, so what is read is the text Word + shows once all revisions are accepted. + + Two content controls keep their current treatment: the automatic table of contents stays + unread, and a control still showing its placeholder text is dropped, since that text only + tells the reader what to type there. An empty `w:ins` marking a revision in run, paragraph or + row properties holds no content, so removing it changes nothing that is read. + """ + for wrapper in list(root.iter(_W + "sdt", *_UNWRAPPED_TAGS)): + parent = wrapper.getparent() + if parent is None: + continue + children = ( + _content_control_children(wrapper) if wrapper.tag == _W + "sdt" else list(wrapper) + ) + if children is None: + continue + index = parent.index(wrapper) + parent[index : index + 1] = children + + +def _content_control_children(sdt: BaseOxmlElement) -> list[BaseOxmlElement] | None: + """The elements to put in place of content control `sdt`, or None to leave it as it is.""" + properties = sdt.find(_W + "sdtPr") + if properties is not None: + if properties.find(_W + "showingPlcHdr") is not None: + return [] + gallery = properties.find(f"{_W}docPartObj/{_W}docPartGallery") + if gallery is not None and gallery.get(_W + "val") == "Table of Contents": + return None + content = sdt.find(_W + "sdtContent") + return [] if content is None else list(content) + + # ================================================================================================ # SUB-PARTITIONERS # ================================================================================================