diff --git a/CHANGELOG.md b/CHANGELOG.md index 408f913c2f..e8bbe81cb4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 0.27.19-dev0 + +### Fixes + +- **DOCX content in content controls and tracked revisions is no longer skipped.** `partition_docx()` read only what python-docx finds directly under the body, a table cell, a header or a footer, and directly in a paragraph. So it dropped the paragraphs, tables, rows and runs held in a content control (`w:sdt`, used by forms, templates and Word's document-property fields), text inserted or moved with track changes on and not yet accepted (`w:ins`, `w:moveTo`), and runs in custom XML markup, smart tags, simple fields and bidirectional spans. These wrappers are now replaced by their content before partitioning. Deleted and moved-away text still stays out, the automatic table of contents is still skipped, and a content control's placeholder text is not emitted. Resolves #3553 and #1821. + ## 0.27.18 ### Fixes diff --git a/test_unstructured/partition/test_docx.py b/test_unstructured/partition/test_docx.py index 7b29c73de9..e449be9112 100644 --- a/test_unstructured/partition/test_docx.py +++ b/test_unstructured/partition/test_docx.py @@ -281,6 +281,131 @@ def test_partition_docx_merged_cell_table_chunks_without_corrupting_rowspan_geom ) +_W_NS = 'xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"' +_REVISION = 'w:author="a" w:date="2024-01-01T00:00:00Z"' + + +def _w_run(text: str) -> str: + return f'{text}' + + +def _w_control(content: str, properties: str = "") -> str: + return ( + f"{properties}" + f"{content}" + ) + + +def _append_to_body(document: Document, xml: str) -> None: + """Insert raw WordprocessingML ahead of the body's closing `w:sectPr`.""" + body = document.element.body + body.insert(len(body) - 1, parse_xml(xml)) + + +def test_partition_docx_reads_content_nested_in_content_controls_and_revisions(tmp_path): + """python-docx skips content one level below the body or a paragraph; none of it is lost.""" + document = docx.Document() + _append_to_body(document, _w_control(f"{_w_run('In a content control')}")) + _append_to_body( + document, + f'' + f"{_w_run('In custom XML')}", + ) + for content in ( + _w_run("Signed by ") + + f"{_w_run('Jane Doe')}", + _w_run("Dose: ") + + f'{_w_run("20 mg")}' + + f'10 mg', + f'{_w_run("Moved away")}' + f'{_w_run("Moved here")}', + _w_run("Updated ") + f'{_w_run("2024-05-01")}', + f'{_w_run("Reutlingen")}', + f'{_w_run("Custom inline")}', + f'{_w_run("Embedded")}' + f'{_w_run(" override")}', + f'' + f"{_w_run('Inserted paragraph')}", + ): + _append_to_body(document, f"{content}") + docx_path = tmp_path / "nested-content.docx" + document.save(str(docx_path)) + + elements = partition_docx(str(docx_path)) + + assert [e.text for e in elements] == [ + "In a content control", + "In custom XML", + "Signed by Jane Doe", + "Dose: 20 mg", + "Moved here", + "Updated 2024-05-01", + "Reutlingen", + "Custom inline", + "Embedded override", + "Inserted paragraph", + ] + + +def test_partition_docx_reads_content_controls_in_tables_headers_and_footers(tmp_path): + document = docx.Document() + table = document.add_table(rows=1, cols=2) + table.cell(0, 0).text = "Name" + table.cell(0, 1)._tc.p_lst[0].append(parse_xml(_w_control(_w_run("Jane Doe")))) + # -- a repeating-section item: the row itself is wrapped in a content control -- + table._tbl.append( + parse_xml( + _w_control( + f"{_w_run('Role')}" + f"{_w_run('Reviewer')}" + ) + ) + ) + section = document.sections[0] + section.header.paragraphs[0]._p.append(parse_xml(_w_control(_w_run("Quarterly report")))) + section.footer.paragraphs[0]._p.append(parse_xml(_w_control(_w_run("Confidential")))) + docx_path = tmp_path / "controls-in-tables-and-headers.docx" + document.save(str(docx_path)) + + elements = partition_docx(str(docx_path), infer_table_structure=True) + + table_element = next(e for e in elements if isinstance(e, Table)) + assert table_element.text == "Name Jane Doe Role Reviewer" + assert table_element.metadata.text_as_html == ( + "" + "
NameJane Doe
RoleReviewer
" + ) + assert [e.text for e in elements if isinstance(e, Header)] == ["Quarterly report"] + assert [e.text for e in elements if isinstance(e, Footer)] == ["Confidential"] + + +def test_partition_docx_leaves_out_the_table_of_contents_and_placeholder_text(tmp_path): + """The automatic table of contents stays unread, and so does a control's placeholder text.""" + toc = '' + document = docx.Document() + document.add_paragraph("Before") + _append_to_body( + document, + _w_control( + f"{_w_run('Contents')}{_w_run('Introduction')}" + f"{_w_run('1')}", + properties=toc, + ), + ) + _append_to_body( + document, + f"{_w_run('Due date: ')}" + f"{_w_run('Click or tap to enter a date.')}", + ) + document.add_paragraph("After") + docx_path = tmp_path / "toc-and-placeholder.docx" + document.save(str(docx_path)) + + elements = partition_docx(str(docx_path)) + + assert [e.text for e in elements] == ["Before", "Due date: ", "After"] + + def test_partition_docx_grabs_header_and_footer(): elements = partition_docx(example_doc_path("handbook-1p.docx")) diff --git a/unstructured/__version__.py b/unstructured/__version__.py index 1c8487b647..26a99944d7 100644 --- a/unstructured/__version__.py +++ b/unstructured/__version__.py @@ -1 +1 @@ -__version__ = "0.27.18" # pragma: no cover +__version__ = "0.27.19-dev0" # pragma: no cover diff --git a/unstructured/partition/docx.py b/unstructured/partition/docx.py index 75aa8d39f9..19331ab93d 100644 --- a/unstructured/partition/docx.py +++ b/unstructured/partition/docx.py @@ -18,6 +18,9 @@ from docx.oxml.ns import nsmap, qn from docx.oxml.table import CT_Tbl from docx.oxml.text.paragraph import CT_P +from docx.oxml.xmlchemy import BaseOxmlElement +from docx.parts.document import DocumentPart +from docx.parts.hdrftr import FooterPart, HeaderPart from docx.section import Section, _Footer, _Header from docx.table import Table as DocxTable from docx.table import _Cell, _Row @@ -235,8 +238,16 @@ def register_picture_partitioner(cls, picture_partitioner: PicturePartitionerT): @cached_property def document(self) -> Document: - """The python-docx `Document` object loaded from file or filename.""" - return docx.Document(self._docx_file) + """The python-docx `Document` object loaded from file or filename. + + Content that python-docx would skip because it is nested in a content control, a tracked + revision or a similar wrapper is lifted into place first; see `_unwrap_nested_content()`. + """ + document = docx.Document(self._docx_file) + for part in document.part.package.iter_parts(): + if isinstance(part, (DocumentPart, HeaderPart, FooterPart)): + _unwrap_nested_content(part.element) + return document @cached_property def include_page_breaks(self) -> bool: @@ -1112,6 +1123,64 @@ def _row_grid_width(row: _Row) -> int: return sum(max(count, 0) for count in counts) +# ================================================================================================ +# NESTED CONTENT +# ================================================================================================ + +_W = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}" +# -- wrappers whose content is ordinary content: the content is kept, the wrapper removed -- +_UNWRAPPED_TAGS = frozenset( + _W + tag for tag in ("ins", "moveTo", "customXml", "smartTag", "fldSimple", "dir", "bdo") +) + + +def _unwrap_nested_content(root: BaseOxmlElement) -> None: + """Lift content python-docx would skip into the place it occupies, in the tree under `root`. + + python-docx reads the `w:p` and `w:tbl` children of the body, a cell, a header or a footer, + and the runs and hyperlinks directly in a paragraph. Content one level deeper is skipped: + + - a content control (`w:sdt`) keeps its paragraphs, tables, rows, cells or runs in + `w:sdtContent`; + - an insertion or a move made with track changes on (`w:ins`, `w:moveTo`) wraps its runs; + - custom XML markup, a smart tag, a simple field and a bidirectional span (`w:customXml`, + `w:smartTag`, `w:fldSimple`, `w:dir`, `w:bdo`) wrap theirs. + + Each wrapper is replaced by its content. Deleted and moved-away content (`w:del`, + `w:moveFrom`) stays where it is, where python-docx skips it, so what is read is the text Word + shows once all revisions are accepted. + + Two content controls keep their current treatment: the automatic table of contents stays + unread, and a control still showing its placeholder text is dropped, since that text only + tells the reader what to type there. An empty `w:ins` marking a revision in run, paragraph or + row properties holds no content, so removing it changes nothing that is read. + """ + for wrapper in list(root.iter(_W + "sdt", *_UNWRAPPED_TAGS)): + parent = wrapper.getparent() + if parent is None: + continue + children = ( + _content_control_children(wrapper) if wrapper.tag == _W + "sdt" else list(wrapper) + ) + if children is None: + continue + index = parent.index(wrapper) + parent[index : index + 1] = children + + +def _content_control_children(sdt: BaseOxmlElement) -> list[BaseOxmlElement] | None: + """The elements to put in place of content control `sdt`, or None to leave it as it is.""" + properties = sdt.find(_W + "sdtPr") + if properties is not None: + if properties.find(_W + "showingPlcHdr") is not None: + return [] + gallery = properties.find(f"{_W}docPartObj/{_W}docPartGallery") + if gallery is not None and gallery.get(_W + "val") == "Table of Contents": + return None + content = sdt.find(_W + "sdtContent") + return [] if content is None else list(content) + + # ================================================================================================ # SUB-PARTITIONERS # ================================================================================================