fix(docx): read content controls and tracked revisions instead of skipping them

CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.27.19-dev0
+
+### Fixes
+
+- **DOCX content in content controls and tracked revisions is no longer skipped.** `partition_docx()` read only what python-docx finds directly under the body, a table cell, a header or a footer, and directly in a paragraph. So it dropped the paragraphs, tables, rows and runs held in a content control (`w:sdt`, used by forms, templates and Word's document-property fields), text inserted or moved with track changes on and not yet accepted (`w:ins`, `w:moveTo`), and runs in custom XML markup, smart tags, simple fields and bidirectional spans. These wrappers are now replaced by their content before partitioning. Deleted and moved-away text still stays out, the automatic table of contents is still skipped, and a content control's placeholder text is not emitted. Resolves #3553 and #1821.
+
 ## 0.27.18
 
 ### Fixes

test_unstructured/partition/test_docx.py

@@ -281,6 +281,131 @@ def test_partition_docx_merged_cell_table_chunks_without_corrupting_rowspan_geom
     )
 
 
+_W_NS = 'xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"'
+_REVISION = 'w:author="a" w:date="2024-01-01T00:00:00Z"'
+
+
+def _w_run(text: str) -> str:
+    return f'<w:r><w:t xml:space="preserve">{text}</w:t></w:r>'
+
+
+def _w_control(content: str, properties: str = "") -> str:
+    return (
+        f"<w:sdt {_W_NS}><w:sdtPr>{properties}</w:sdtPr>"
+        f"<w:sdtContent>{content}</w:sdtContent></w:sdt>"
+    )
+
+
+def _append_to_body(document: Document, xml: str) -> None:
+    """Insert raw WordprocessingML ahead of the body's closing `w:sectPr`."""
+    body = document.element.body
+    body.insert(len(body) - 1, parse_xml(xml))
+
+
+def test_partition_docx_reads_content_nested_in_content_controls_and_revisions(tmp_path):
+    """python-docx skips content one level below the body or a paragraph; none of it is lost."""
+    document = docx.Document()
+    _append_to_body(document, _w_control(f"<w:p>{_w_run('In a content control')}</w:p>"))
+    _append_to_body(
+        document,
+        f'<w:customXml {_W_NS} w:element="clause">'
+        f"<w:p>{_w_run('In custom XML')}</w:p></w:customXml>",
+    )
+    for content in (
+        _w_run("Signed by ")
+        + f"<w:sdt><w:sdtPr/><w:sdtContent>{_w_run('Jane Doe')}</w:sdtContent></w:sdt>",
+        _w_run("Dose: ")
+        + f'<w:ins w:id="1" {_REVISION}>{_w_run("20 mg")}</w:ins>'
+        + f'<w:del w:id="2" {_REVISION}><w:r><w:tab/><w:delText>10 mg</w:delText></w:r></w:del>',
+        f'<w:moveFrom w:id="3" {_REVISION}>{_w_run("Moved away")}</w:moveFrom>'
+        f'<w:moveTo w:id="4" {_REVISION}>{_w_run("Moved here")}</w:moveTo>',
+        _w_run("Updated ") + f'<w:fldSimple w:instr=" DATE ">{_w_run("2024-05-01")}</w:fldSimple>',
+        f'<w:smartTag w:uri="urn:x" w:element="place">{_w_run("Reutlingen")}</w:smartTag>',
+        f'<w:customXml w:element="name">{_w_run("Custom inline")}</w:customXml>',
+        f'<w:dir w:val="rtl">{_w_run("Embedded")}</w:dir>'
+        f'<w:bdo w:val="rtl">{_w_run(" override")}</w:bdo>',
+        f'<w:pPr><w:rPr><w:ins w:id="5" {_REVISION}/></w:rPr></w:pPr>'
+        f"{_w_run('Inserted paragraph')}",
+    ):
+        _append_to_body(document, f"<w:p {_W_NS}>{content}</w:p>")
+    docx_path = tmp_path / "nested-content.docx"
+    document.save(str(docx_path))
+
+    elements = partition_docx(str(docx_path))
+
+    assert [e.text for e in elements] == [
+        "In a content control",
+        "In custom XML",
+        "Signed by Jane Doe",
+        "Dose: 20 mg",
+        "Moved here",
+        "Updated 2024-05-01",
+        "Reutlingen",
+        "Custom inline",
+        "Embedded override",
+        "Inserted paragraph",
+    ]
+
+
+def test_partition_docx_reads_content_controls_in_tables_headers_and_footers(tmp_path):
+    document = docx.Document()
+    table = document.add_table(rows=1, cols=2)
+    table.cell(0, 0).text = "Name"
+    table.cell(0, 1)._tc.p_lst[0].append(parse_xml(_w_control(_w_run("Jane Doe"))))
+    # -- a repeating-section item: the row itself is wrapped in a content control --
+    table._tbl.append(
+        parse_xml(
+            _w_control(
+                f"<w:tr><w:tc><w:p>{_w_run('Role')}</w:p></w:tc>"
+                f"<w:tc><w:p>{_w_run('Reviewer')}</w:p></w:tc></w:tr>"
+            )
+        )
+    )
+    section = document.sections[0]
+    section.header.paragraphs[0]._p.append(parse_xml(_w_control(_w_run("Quarterly report"))))
+    section.footer.paragraphs[0]._p.append(parse_xml(_w_control(_w_run("Confidential"))))
+    docx_path = tmp_path / "controls-in-tables-and-headers.docx"
+    document.save(str(docx_path))
+
+    elements = partition_docx(str(docx_path), infer_table_structure=True)
+
+    table_element = next(e for e in elements if isinstance(e, Table))
+    assert table_element.text == "Name Jane Doe Role Reviewer"
+    assert table_element.metadata.text_as_html == (
+        "<table><tr><td>Name</td><td>Jane Doe</td></tr>"
+        "<tr><td>Role</td><td>Reviewer</td></tr></table>"
+    )
+    assert [e.text for e in elements if isinstance(e, Header)] == ["Quarterly report"]
+    assert [e.text for e in elements if isinstance(e, Footer)] == ["Confidential"]
+
+
+def test_partition_docx_leaves_out_the_table_of_contents_and_placeholder_text(tmp_path):
+    """The automatic table of contents stays unread, and so does a control's placeholder text."""
+    toc = '<w:docPartObj><w:docPartGallery w:val="Table of Contents"/></w:docPartObj>'
+    document = docx.Document()
+    document.add_paragraph("Before")
+    _append_to_body(
+        document,
+        _w_control(
+            f"<w:p>{_w_run('Contents')}</w:p><w:p>{_w_run('Introduction')}<w:r><w:tab/></w:r>"
+            f"{_w_run('1')}</w:p>",
+            properties=toc,
+        ),
+    )
+    _append_to_body(
+        document,
+        f"<w:p {_W_NS}>{_w_run('Due date: ')}<w:sdt><w:sdtPr><w:showingPlcHdr/></w:sdtPr>"
+        f"<w:sdtContent>{_w_run('Click or tap to enter a date.')}</w:sdtContent></w:sdt></w:p>",
+    )
+    document.add_paragraph("After")
+    docx_path = tmp_path / "toc-and-placeholder.docx"
+    document.save(str(docx_path))
+
+    elements = partition_docx(str(docx_path))
+
+    assert [e.text for e in elements] == ["Before", "Due date: ", "After"]
+
+
 def test_partition_docx_grabs_header_and_footer():
     elements = partition_docx(example_doc_path("handbook-1p.docx"))
 

unstructured/__version__.py

@@ -1 +1 @@
-__version__ = "0.27.18"  # pragma: no cover
+__version__ = "0.27.19-dev0"  # pragma: no cover

unstructured/partition/docx.py

@@ -18,6 +18,9 @@
 from docx.oxml.ns import nsmap, qn
 from docx.oxml.table import CT_Tbl
 from docx.oxml.text.paragraph import CT_P
+from docx.oxml.xmlchemy import BaseOxmlElement
+from docx.parts.document import DocumentPart
+from docx.parts.hdrftr import FooterPart, HeaderPart
 from docx.section import Section, _Footer, _Header
 from docx.table import Table as DocxTable
 from docx.table import _Cell, _Row
@@ -235,8 +238,16 @@ def register_picture_partitioner(cls, picture_partitioner: PicturePartitionerT):
 
     @cached_property
     def document(self) -> Document:
-        """The python-docx `Document` object loaded from file or filename."""
-        return docx.Document(self._docx_file)
+        """The python-docx `Document` object loaded from file or filename.
+
+        Content that python-docx would skip because it is nested in a content control, a tracked
+        revision or a similar wrapper is lifted into place first; see `_unwrap_nested_content()`.
+        """
+        document = docx.Document(self._docx_file)
+        for part in document.part.package.iter_parts():
+            if isinstance(part, (DocumentPart, HeaderPart, FooterPart)):
+                _unwrap_nested_content(part.element)
+        return document
 
     @cached_property
     def include_page_breaks(self) -> bool:
@@ -1112,6 +1123,64 @@ def _row_grid_width(row: _Row) -> int:
     return sum(max(count, 0) for count in counts)
 
 
+# ================================================================================================
+# NESTED CONTENT
+# ================================================================================================
+
+_W = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"
+# -- wrappers whose content is ordinary content: the content is kept, the wrapper removed --
+_UNWRAPPED_TAGS = frozenset(
+    _W + tag for tag in ("ins", "moveTo", "customXml", "smartTag", "fldSimple", "dir", "bdo")
+)
+
+
+def _unwrap_nested_content(root: BaseOxmlElement) -> None:
+    """Lift content python-docx would skip into the place it occupies, in the tree under `root`.
+
+    python-docx reads the `w:p` and `w:tbl` children of the body, a cell, a header or a footer,
+    and the runs and hyperlinks directly in a paragraph. Content one level deeper is skipped:
+
+    - a content control (`w:sdt`) keeps its paragraphs, tables, rows, cells or runs in
+      `w:sdtContent`;
+    - an insertion or a move made with track changes on (`w:ins`, `w:moveTo`) wraps its runs;
+    - custom XML markup, a smart tag, a simple field and a bidirectional span (`w:customXml`,
+      `w:smartTag`, `w:fldSimple`, `w:dir`, `w:bdo`) wrap theirs.
+
+    Each wrapper is replaced by its content. Deleted and moved-away content (`w:del`,
+    `w:moveFrom`) stays where it is, where python-docx skips it, so what is read is the text Word
+    shows once all revisions are accepted.
+
+    Two content controls keep their current treatment: the automatic table of contents stays
+    unread, and a control still showing its placeholder text is dropped, since that text only
+    tells the reader what to type there. An empty `w:ins` marking a revision in run, paragraph or
+    row properties holds no content, so removing it changes nothing that is read.
+    """
+    for wrapper in list(root.iter(_W + "sdt", *_UNWRAPPED_TAGS)):
+        parent = wrapper.getparent()
+        if parent is None:
+            continue
+        children = (
+            _content_control_children(wrapper) if wrapper.tag == _W + "sdt" else list(wrapper)
+        )
+        if children is None:
+            continue
+        index = parent.index(wrapper)
+        parent[index : index + 1] = children
+
+
+def _content_control_children(sdt: BaseOxmlElement) -> list[BaseOxmlElement] | None:
+    """The elements to put in place of content control `sdt`, or None to leave it as it is."""
+    properties = sdt.find(_W + "sdtPr")
+    if properties is not None:
+        if properties.find(_W + "showingPlcHdr") is not None:
+            return []
+        gallery = properties.find(f"{_W}docPartObj/{_W}docPartGallery")
+        if gallery is not None and gallery.get(_W + "val") == "Table of Contents":
+            return None
+    content = sdt.find(_W + "sdtContent")
+    return [] if content is None else list(content)
+
+
 # ================================================================================================
 # SUB-PARTITIONERS
 # ================================================================================================