diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b2c256a..d946dd4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,5 +17,12 @@ jobs: with: node-version: 22 cache: npm + - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + with: + python-version: '3.13' + cache: pip + cache-dependency-path: plugins/Hylouis233/document-skills/tests/requirements-fixtures.txt + - run: sudo apt-get update && sudo apt-get install -y --no-install-recommends libreoffice-writer + - run: python -m pip install -r plugins/Hylouis233/document-skills/tests/requirements-fixtures.txt - run: npm ci - run: npm run check diff --git a/plugins/Hylouis233/document-skills/LICENSE b/plugins/Hylouis233/document-skills/LICENSE new file mode 100644 index 0000000..125be1b --- /dev/null +++ b/plugins/Hylouis233/document-skills/LICENSE @@ -0,0 +1,192 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + Copyright 2026 MCode Plugins contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/Hylouis233/document-skills/README.md b/plugins/Hylouis233/document-skills/README.md new file mode 100644 index 0000000..dbd2265 --- /dev/null +++ b/plugins/Hylouis233/document-skills/README.md @@ -0,0 +1,110 @@ +# Document Skills + +## The problem + +Users keep asking coding agents for document deliverables — a Word report, a PDF handout, a slide +deck, a data workbook — and get unreliable results: ZIP files rewritten by hand and corrupted, +spreadsheets where formulas were typed as text, slide decks that no viewer opens, PDFs that are a +stack of screenshots instead of text. Office and PDF documents are container formats with strict +internal contracts, and an agent without format knowledge breaks them silently. + +This Plugin installs four workbench Skills — `docx`, `pdf`, `pptx`, `xlsx` — that route each +task to the right standard tool (python-docx, python-pptx, openpyxl, pypdf, ReportLab, PyMuPDF), +enforce the container contracts (valid ZIP, correct content types, calculated dimensions), and +verify output before it is handed back. + +## Try it + +```text +Use the document-skills plugin: open sales-2024.xlsx, add a sheet "Summary" with per-region +totals computed by formula, a bar chart of the top 5 products, and currency formatting. +``` + +Expected result: the agent loads the workbook with openpyxl, inspects sheet names, headers, and +dtypes, writes `SUMIF`/`COUNTIF` formulas (not pasted values), adds a native `BarChart` anchored +to the new sheet, applies number formats, and reports the cell ranges it changed. + +```text +Use the document-skills plugin: create a PDF one-pager "Q3 launch checklist" from this outline +with a title block and a two-column checklist that fits exactly one A4 page. +``` + +Expected result: the agent generates the page with ReportLab flowables on an A4 canvas, measures +the checklist blocks, and confirms with pypdf that the output is exactly 1 page with extractable +text — not a screenshot. + +## What the Skills do + +Shared spine (all four Skills follow it): + +1. Classify the request as **create**, **read**, **edit**, or **review**. +2. Check tool availability first and report missing dependencies instead of improvising. +3. Follow format-specific rules (packages below). +4. Run the post-generation verification checklist; fix and re-verify until it passes. +5. Report the output path, the page/sheet/slide inventory, and any remaining caveats. + +Per format: + +- **docx** — create with python-docx from a heading outline; use python-docx first for routine + structural edits (paragraphs, tables, images, styles, and text runs), and reserve direct OOXML + surgery for fields, tracked changes, or package features python-docx cannot express; extract + text with python-docx or `pandoc -t markdown`; postcheck with python-docx re-open and + `soffice --headless --convert-to` PDF smoke test when LibreOffice is present. Depth + references: CJK typography (east-asian font slots, 字号 table, char-based indents, GB/T 9704 + page geometry) and scene patterns (academic paper, resume, official document, contract). +- **xlsx** — openpyxl for reading, editing, styling, and native charts; formulas as formulas, + never as pasted results; `data_only=True` only for reading cached values; date/number formats + applied explicitly; recalculation contract documented (openpyxl writes formulas, the Skills + set `fullCalcOnLoad` so even manual-calc workbooks recalculate when a viewer opens them). Depth references: conditional formatting rules, structured tables, and honest + pivot-style aggregation (openpyxl cannot create pivot tables; the reference gives the + formula-sheet, frozen-values, and user-template routes). +- **pptx** — python-pptx to build decks (7 common slide patterns: title, agenda, bullet, two + image+text, table, chart, quote/closing); edit only named, existing shapes, never blind + rewriting of the whole XML; text measured against shape width with font-size reduction rules; + presentation-level verification via `python-pptx` re-open plus a rendered PDF smoke test when + LibreOffice is available. +- **pdf** — creation prefers ReportLab (real, selectable text with flowable structure) over + HTML-to-print paths; extraction (text, coordinates, tables, images) and rasterization use + PyMuPDF, while pypdf is reserved for page-level transforms such as split, merge, rotate, + watermark, encryption, and forms; an explicit one-tool-per-job table prevents accidental API + mixing. Note: ReportLab output is not tagged PDF/UA — when the user needs an accessible + (screen-reader-ready) PDF, the pdf Skill says to report that limitation honestly instead of + claiming accessibility. + +## Verification-first output + +Every Skill ends with the same rule: do not hand back a file you have not re-opened. The checklists +are specific (re-open the archive, confirm the sheet count and formula presence, confirm the slide +count, confirm page count and text extraction) and the Skills require reporting what was verified +versus what was assumed. + +The [`tests/`](tests/) directory ships one runnable fixture script per format covering the +snippets with the worst silent-failure modes (PDF AcroForm clone-and-fill, watermark write, +encrypted extraction, CMYK conversion, soft masks; PPTX run-preserving edits, table-cell locating, +actual content extraction, grouped shapes, script-aware per-master themes; XLSX dialect sniffing, +array formulas, streamed extension checks, structural-reference audits; DOCX bounded package review, +content controls, per-run glyph checks, guarded replacement, and a LibreOffice-rendered numbering +restart). Each script is self-contained and exits non-zero on failure. `npm run check` executes +all four through `tests/fixtures.test.mjs`; install the pinned Python environment with +`python -m pip install -r tests/requirements-fixtures.txt` and make `soffice` available first. +The repository CI installs those dependencies and LibreOffice Writer before running the same gate. + +## Requirements + +- Python 3.9+ with `python-docx`, `python-pptx`, `openpyxl`, `pypdf`, `reportlab`, + `pymupdf` (`pip install python-docx python-pptx openpyxl pypdf reportlab pymupdf`). +- Optional: LibreOffice (`soffice`) for PDF smoke tests of DOCX/PPTX output; `pandoc` for + markdown extraction from DOCX. +- Works on Windows, macOS, and Linux. All commands are given in cross-platform form; the Skills + say how to resolve the skill directory path on each platform. + +## Data and network + +- No network access. All processing is local file conversion and generation. +- No credentials required. +- The Skills only read and write document files the user points at; temporary files go to the + system temp directory and are cleaned up. + +## License + +Apache-2.0. See [LICENSE](LICENSE). diff --git a/plugins/Hylouis233/document-skills/plugin.json b/plugins/Hylouis233/document-skills/plugin.json new file mode 100644 index 0000000..7fe7217 --- /dev/null +++ b/plugins/Hylouis233/document-skills/plugin.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json", + "name": "document-skills", + "version": "0.1.0", + "description": "Workbench Skills for creating, reading, editing, and checking DOCX, PDF, PPTX, and XLSX documents with standard open-source tooling.", + "author": { + "name": "Hylouis233", + "url": "https://github.com/Hylouis233" + }, + "license": "Apache-2.0", + "keywords": [ + "minimax-code", + "plugin", + "documents", + "docx", + "pdf", + "pptx", + "xlsx", + "office", + "ooxml" + ] +} diff --git a/plugins/Hylouis233/document-skills/skills/docx/SKILL.md b/plugins/Hylouis233/document-skills/skills/docx/SKILL.md new file mode 100644 index 0000000..08b26dc --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/docx/SKILL.md @@ -0,0 +1,83 @@ +--- +name: docx +description: Create, read, edit, or review Microsoft Word .docx documents. Use this Skill whenever the task involves a .docx file - generating reports, letters, or structured documents from scratch; extracting or reorganizing content; applying consistent formatting (headings, tables, styles, headers/footers, page numbers); editing an existing document while preserving its layout; or checking that a produced document actually opens and renders. +--- + +# DOCX workbench + +A `.docx` file is a ZIP archive of XML parts around one `word/document.xml`. Treat the format +with respect: silent corruption is the default failure mode when agents edit the XML blind. + +## Step 0 — Resolve paths and check the toolchain + +Resolve this Skill's directory once (`SKILL_DIR` = the folder containing this SKILL.md), then +check what is installed before promising anything: + +```bash +python -c "import docx; print(docx.__version__ if hasattr(docx,'__version__') else 'ok')" +``` + +- `python-docx` present -> full create/read/edit support. +- `python-docx` missing but `pandoc` present -> read/extract requests can still + be served by the pandoc route in [references/read.md](references/read.md); + report that structural reads, edits, and creation are unavailable + (`pip install python-docx`) and stop only for those routes. Never hand-write + OOXML as a substitute. +- Both missing -> report the missing tools and stop. +- `pandoc` present -> prefer it for text/markdown extraction (fast, faithful). +- `soffice` present -> enables the PDF smoke test in postcheck. + +## Step 1 — Classify the task + +| Request looks like | Route | Go to | +|---|---|---| +| "Write / create / generate a document" (no input file) | Create | [references/create.md](references/create.md) | +| "Edit / change / add to this document" (input file given) | Edit | [references/edit.md](references/edit.md) | +| "Read / extract / summarize / convert" | Read | [references/read.md](references/read.md) | +| "Check / fix / why is it broken" | Review | [references/review.md](references/review.md) | + +Two depth references slot in before create/edit: [references/cjk.md](references/cjk.md) +whenever the document contains CJK text (east-asian fonts, 字号, char-based indents, 公文 +geometry), and [references/scenes.md](references/scenes.md) when the request matches a known +scene - academic paper, resume, official document (公文), or contract. + +Two routes can chain (read -> edit, create -> review). Never skip classification; it decides +between python-docx generation and raw XML surgery, which have opposite safety profiles. + +## Step 2 — Shared rules that always apply + +1. **Never edit `word/document.xml` with find/replace on rendered text.** Run-level splits mean + the visible word "contract" may be spread across three `` nodes. Use the edit route's + paragraph-index addressing instead. +2. **Prefer modifying an existing document with python-docx when the change is structural** + (add paragraphs, tables, images, styles); drop to raw XML only for things python-docx cannot + express (field codes, exotic properties, tracked-change surgery). +3. **Always write to a new output path first.** Only overwrite the source when the user explicitly + asked for in-place modification, and then only after the postcheck passes. +4. **Keep styles semantic.** Use `Heading 1..4` styles instead of "bold 16pt text" so + navigation, TOC fields, and accessibility keep working. +5. **Units**: lengths in the XML are twentieths of a point (twips); python-docx accepts + `.inches`/`.cm`/`.Pt` helpers - use the helpers. +6. **Non-ASCII**: write files as UTF-8 without BOM; declare encoding when opening text side + files. + +## Step 3 — Execute the route + +Follow the referenced file for concrete code patterns, then continue to postcheck. Do not +improvise container-level operations (re-zipping, renaming parts) outside the patterns in +[references/edit.md](references/edit.md). + +## Step 4 — Postcheck (mandatory before handing back the file) + +Run every applicable item and report results explicitly: + +1. Re-open the output with python-docx (`Document(path)`); confirm it parses and count + paragraphs/tables. +2. If `soffice` exists: `soffice --headless --convert-to pdf --outdir ` + must exit 0; the PDF is a smoke test, keep or delete it per the user's ask. +3. Verify requested features landed: search the re-opened document for the expected heading + texts, table row counts, or inserted image count. +4. Report: output path, paragraph/table counts, what was verified, what could not be verified + locally (e.g. exact pagination in MS Word), and any font substitutions expected. + +If a check fails, fix and re-run; do not deliver with a known-broken archive. diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md b/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md new file mode 100644 index 0000000..242466a --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md @@ -0,0 +1,252 @@ +# CJK typography in DOCX (Chinese, Japanese, Korean) + +python-docx is Latin-first: `run.font.name` writes only the `w:ascii` and `w:hAnsi` slots. +Chinese text renders from the `w:eastAsia` slot, so a "font is wrong in Word but fine in +LibreOffice" report almost always means the east-asian face was never set. + +## Set the CJK face explicitly + +```python +from docx.oxml.ns import qn + +LATIN, EAST_ASIAN = "Times New Roman", "宋体" # define both names once, up front + +def set_fonts(run, latin=LATIN, east_asian=EAST_ASIAN): + run.font.name = latin # writes w:ascii + w:hAnsi + rPr = run._element.get_or_add_rPr() + rFonts = rPr.get_or_add_rFonts() + rFonts.set(qn("w:eastAsia"), east_asian) # python-docx has no helper for this slot + +# Style level: do the same on the style so body text inherits it +style = doc.styles["Normal"] +style.font.name = LATIN +style.element.get_or_add_rPr().get_or_add_rFonts().set(qn("w:eastAsia"), EAST_ASIAN) +``` + +Convention for mixed-script body text: CJK glyphs from the east-asian face, digits and Latin +from a Latin face (Times New Roman or Arial). Both slots set = deterministic rendering. + +## Chinese font-size table (字号) + +Word's Chinese UI names map to point sizes; scripts must use the points: + +| 字号 | pt | typical use | 字号 | pt | +|---|---|---|---|---| +| 初号 | 42 | big covers | 小三 | 15 | +| 小初 | 36 | covers | 四号 | 14 | +| 一号 | 26 | 公文标题辅助 | 小四 | 12 | +| 小一 | 24 | | 五号 | 10.5 | +| 二号 | 22 | 公文标题 | 小五 | 9 | +| 小二 | 18 | section covers | 六号 | 7.5 | +| 三号 | 16 | 公文正文 | 七号 | 5.5 | +| | | | 八号 | 5 | + +Common pairings: 正文宋体小四 (reports), 仿宋三号 (official documents), 黑体 for headings at +one size step above the body. + +## Paragraph properties CJK actually needs + +```python +from docx.shared import Pt + +pPr = p._p.get_or_add_pPr() + +# First-line indent of exactly two characters - use the char-based attribute, not twips, +# so the indent survives font-size changes +ind = pPr.get_or_add_ind() +ind.set(qn("w:firstLineChars"), "200") # units are 1/100 of a character + +# Fixed line spacing (公文 practice: 28-30 pt fixed; pick one value and stay consistent) +p.paragraph_format.line_spacing = Pt(28) # writes w:line=560, w:lineRule="exact" +``` + +- Word's CJK defaults (kinsoku line-breaking, auto space between CJK and Latin) live in + document defaults and need no action unless the source file disabled them. +- Do not fake a two-char indent with spaces or full-width spaces; `firstLineChars` is the + durable mechanism and survives re-flow. +- Justify body text (`WD_ALIGN_PARAGRAPH.JUSTIFY`); CJK justification is the expected look. + +## Official-document page geometry (GB/T 9704 family) + +Public-standard values for 党政机关公文 style documents on A4 - always confirm the user's +edition before promising compliance: + +```python +from docx.shared import Cm + +section = doc.sections[0] +section.top_margin, section.bottom_margin = Cm(3.7), Cm(3.5) +section.left_margin, section.right_margin = Cm(2.8), Cm(2.6) +# conventional target density with 三号仿宋 body: ~22 lines per page, ~28 chars per line +``` + +## Font availability is a delivery risk + +- `宋体`/`黑体`/`楷体` exist on Chinese Windows; `仿宋_GB2312` may not. Non-Chinese systems + substitute silently. Report the faces you referenced and expected substitutions in the + postcheck summary. +- python-docx cannot embed fonts. If the recipient machine is unknown and the layout must be + exact, say so and suggest embedding from Word (File > Options > Save > Embed fonts), or + deliver a PDF alongside. + +## Postcheck additions for CJK documents + +1. Re-open the output and assert the east-asian slot is set on body runs and styles + (`.get(qn("w:eastAsia"))` is not None) - not just `run.font.name`. +2. Tofu (missing-glyph boxes) cannot be detected by text extraction: the rendered PDF's text + layer keeps the original codepoint while the glyph is a box, so "extracted text matches" + proves nothing about rendering. Check glyph coverage directly instead - with fontTools + (`pip install fonttools`) when the referenced faces can be located: + + ```python + from docx import Document + from docx.enum.style import WD_STYLE_TYPE + from fontTools.ttLib import TTFont + from docx.oxml.ns import qn + from docx.text.paragraph import Paragraph + from docx.text.run import Run + + doc = Document("output.docx") + normal_style = doc.styles["Normal"] + def paragraph_runs(paragraph): + def walk(element): + for child in element.iterchildren(): + if child.tag == qn("w:r"): + yield Run(child, paragraph) + elif child.tag != qn("w:p"): + yield from walk(child) + yield from walk(paragraph._p) + + def xml_runs(element, parent): + # Covers paragraphs in tables/block controls and runs in inline controls/hyperlinks. + for paragraph_element in element.iter(qn("w:p")): + yield from paragraph_runs(Paragraph(paragraph_element, parent)) + + emitted_runs = list(xml_runs(doc.element.body, doc)) + for section in doc.sections: + emitted_runs.extend(xml_runs(section.header._element, section.header)) + emitted_runs.extend(xml_runs(section.footer._element, section.footer)) + + def face_from_rpr(rpr, slot): + if rpr is None: + return None + rfonts = rpr.find(qn("w:rFonts")) + if rfonts is None: + return None + theme_attribute = { + "ascii": "asciiTheme", "hAnsi": "hAnsiTheme", + "eastAsia": "eastAsiaTheme", "cs": "cstheme", + }[slot] + theme_token = rfonts.get(qn("w:" + theme_attribute)) + if theme_token is not None: + # A direct theme declaration must not fall through to an inherited literal face. + # Resolve the document theme/fontScheme (including themeFontLang/script mapping) + # before auditing this run. Both a literal and theme attribute for one slot are + # ambiguous across consumers, so fail closed in that case too. + literal = rfonts.get(qn("w:" + slot)) + raise LookupError( + f"unresolved direct {theme_attribute}={theme_token!r}" + + (f" alongside {slot}={literal!r}" if literal is not None else "") + ) + return rfonts.get(qn("w:" + slot)) + + def style_faces(style, slot): + while style is not None: + face = face_from_rpr(style.element.find(qn("w:rPr")), slot) + if face: + yield face + style = style.base_style + + def table_style_faces(run, slot): + """Find table/conditional style faces that python-docx does not cascade for runs.""" + element = run._r + while element is not None and element.tag != qn("w:tbl"): + element = element.getparent() + if element is None: + return [] + table_properties = element.find(qn("w:tblPr")) + table_style = None if table_properties is None else table_properties.find(qn("w:tblStyle")) + style_id = None if table_style is None else table_style.get(qn("w:val")) + style = None if not style_id else doc.styles.get_by_id(style_id, WD_STYLE_TYPE.TABLE) + faces = [] + while style is not None: + rprs = [style.element.find(qn("w:rPr"))] + rprs.extend( + region.find(qn("w:rPr")) + for region in style.element.findall(qn("w:tblStylePr")) + ) + faces.extend(face for rpr in rprs if (face := face_from_rpr(rpr, slot))) + style = style.base_style + return list(dict.fromkeys(faces)) + + def effective_face(run, slot): + direct = face_from_rpr(run._r.find(qn("w:rPr")), slot) + if direct: + return direct + if faces := table_style_faces(run, slot): + raise LookupError( + f"table style can override {slot} with {faces}; resolve the applicable " + "tblStylePr region or require rendered inspection" + ) + # Header/footer runs belong to a HeaderPart/FooterPart, which has no .document. + # Resolve the owning document's Normal style once rather than via run.part. + for style in (run.style, run._parent.style, normal_style): + if face := next(style_faces(style, slot), None): + return face + raise LookupError(f"no resolved {slot} face for run {run.text!r}; resolve theme defaults") + + def font_slot(character): + # This is the ordinary non-cs/rtl mapping. Resolve effective w:cs/w:rtl and special + # w:rFonts@w:hint overrides separately before relying on a slot for those runs. + codepoint = ord(character) + uses_east_asian_slot = ( + 0x1100 <= codepoint <= 0x11FF # Hangul Jamo + or 0x2E80 <= codepoint <= 0x9FFF # CJK radicals, kana, bopomofo, ideographs + or 0x3130 <= codepoint <= 0x318F # Hangul compatibility Jamo + or 0xA000 <= codepoint <= 0xA4CF # Yi syllables and radicals + or 0xA960 <= codepoint <= 0xA97F # Hangul Jamo Extended-A + or 0xAC00 <= codepoint <= 0xD7FF # Hangul syllables and Extended-B + or 0xF900 <= codepoint <= 0xFAFF # CJK compatibility ideographs + or 0xFE30 <= codepoint <= 0xFE6F # CJK compatibility and small forms + or 0xFF00 <= codepoint <= 0xFFEF # fullwidth and halfwidth forms + or 0x20000 <= codepoint <= 0x3134F # CJK unified ideograph extensions B-G + or 0x31350 <= codepoint <= 0x33479 # Unicode 17 extensions H-J + ) + return "eastAsia" if uses_east_asian_slot else ( + "ascii" if codepoint < 128 else "hAnsi" + ) + + # Resolve installed files by exact face name first; do not pool their cmaps. + # TTC collections need the face's fontNumber; ordinary TTF files use -1. + font_files_by_face = { + "宋体": ("path/to/simsun.ttc", 0), + "Times New Roman": ("path/to/times.ttf", -1), + } + cmaps = {} + missing = [] + unresolved = [] + for run_index, run in enumerate(emitted_runs): + for character in run.text: + if character.isspace(): + continue + face = effective_face(run, font_slot(character)) + font_spec = font_files_by_face.get(face) + if not font_spec: + unresolved.append((run_index, character, face)) + continue + if font_spec not in cmaps: + font_path, font_number = font_spec + cmaps[font_spec] = TTFont(font_path, fontNumber=font_number).getBestCmap() + cmap = cmaps[font_spec] + if ord(character) not in cmap: + missing.append((run_index, character, face)) + # These are mandatory audit gates, so raise explicitly: `python -O` removes asserts. + if unresolved: + raise ValueError(f"font files not resolved per run: {unresolved}") + if missing: + raise ValueError(f"glyph missing from the run's effective font: {missing}") + ``` + + When font files cannot be located, rasterize the rendered pages with PyMuPDF and inspect + the images visually, and say in the report that glyph rendering was not machine-verified. +3. Confirm `firstLineChars` survived on body paragraphs if the indent was requested. diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/create.md b/plugins/Hylouis233/document-skills/skills/docx/references/create.md new file mode 100644 index 0000000..d195ccc --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/docx/references/create.md @@ -0,0 +1,132 @@ +# Create a new DOCX + +Use python-docx from an outline. Never assemble the ZIP by hand. + +## Pattern + +```python +from docx import Document +from docx.shared import Pt, Cm, RGBColor +from docx.enum.text import WD_ALIGN_PARAGRAPH + +doc = Document() + +# 1. Page geometry (A4 default is US Letter in python-docx - set explicitly) +for section in doc.sections: + section.page_width, section.page_height = Cm(21.0), Cm(29.7) + section.left_margin = section.right_margin = Cm(2.5) + section.top_margin = section.bottom_margin = Cm(2.5) + +# 2. Semantic outline: title + headings drive navigation and TOC +doc.add_heading("Quarterly Reliability Report", level=0) # uses Title style +doc.add_heading("Summary", level=1) +doc.add_heading("Incident review", level=2) + +# 3. Body paragraphs; set spacing once via style, not per paragraph +style = doc.styles["Normal"] +style.font.name = "Calibri" +style.font.size = Pt(11) +style.paragraph_format.space_after = Pt(6) +style.paragraph_format.line_spacing = 1.15 + +p = doc.add_paragraph("Text with ") +p.add_run("bold segment").bold = True +p.add_run(" and regular text.") + +# 4. Tables with header row emphasis +table = doc.add_table(rows=1, cols=3) +table.style = "Light Grid Accent 1" +hdr = table.rows[0].cells +for cell, text in zip(hdr, ["Region", "Incidents", "MTTR (h)"]): + cell.text = text + for par in cell.paragraphs: + for run in par.runs: + run.font.bold = True +row = table.add_row().cells +row[0].text, row[1].text, row[2].text = "eu-1", "3", "0.8" + +# 5. Optional footer with page number field +from docx.oxml.ns import qn +from docx.oxml import OxmlElement +footer = doc.sections[0].footer +fp = footer.paragraphs[0] +fp.alignment = WD_ALIGN_PARAGRAPH.CENTER +fld = OxmlElement("w:fldSimple") +fld.set(qn("w:instr"), "PAGE") +fp._p.append(fld) + +doc.save("report.docx") +``` + +## Rules + +- Build the heading outline **before** writing prose; a wrong outline is the most expensive + late fix. +- One table style per document. Cell-level font overrides are for exceptions only. +- Bullet lists: `add_paragraph(text, style="List Bullet")`. Numbered lists: `style="List + Number"` - but reusing that style makes every list **continue** the same sequence, because + all paragraphs share one numbering definition. When a second list must restart at 1, clone + the numbering definition and point the new list's paragraphs at the clone: + + ```python + import copy + from docx.oxml.ns import qn + from docx.oxml import OxmlElement + from docx.opc.constants import RELATIONSHIP_TYPE as RT + + def new_restart_num_id(doc, base_num_id): + """Clone base_num_id with a startOverride so the next list restarts at 1.""" + numbering = doc.part.part_related_by(RT.NUMBERING).element + source = next( + n for n in numbering.findall(qn("w:num")) + if n.get(qn("w:numId")) == str(base_num_id) + ) + clone = copy.deepcopy(source) + new_id = max(int(n.get(qn("w:numId"))) for n in numbering.findall(qn("w:num"))) + 1 + clone.set(qn("w:numId"), str(new_id)) + level_zero_overrides = [ + item for item in clone.findall(qn("w:lvlOverride")) + if item.get(qn("w:ilvl")) == "0" + ] + if len(level_zero_overrides) > 1: + raise ValueError("base numbering has duplicate level-zero overrides") + if level_zero_overrides: + override = level_zero_overrides[0] + for old_start in override.findall(qn("w:startOverride")): + override.remove(old_start) + else: + override = OxmlElement("w:lvlOverride") + override.set(qn("w:ilvl"), "0") + clone.append(override) + start = OxmlElement("w:startOverride") + start.set(qn("w:val"), "1") + override.insert(0, start) # startOverride precedes an optional embedded w:lvl + numbering.append(clone) + return new_id + + def numbered_paragraph(doc, text, num_id): + p = doc.add_paragraph(text, style="List Number") + pPr = p._p.get_or_add_pPr() + numPr = OxmlElement("w:numPr") + ilvl = OxmlElement("w:ilvl"); ilvl.set(qn("w:val"), "0") + numId = OxmlElement("w:numId"); numId.set(qn("w:val"), str(num_id)) + numPr.append(ilvl); numPr.append(numId) + pPr.append(numPr) + return p + + # Find the numId behind "List Number" in numbering.xml (inspect it once, then hard-code), + # then give each independent list its own cloned definition. + ``` +- Images: `doc.add_picture(path, width=Cm(14))` - always set width so oversized images do not + overflow the text column. Keep aspect ratio by setting only one dimension. +- A real TOC is a **field**, it renders after the user opens the file and updates fields + (Word prompts, or Ctrl+A then F9). If a TOC is requested, insert the field and tell the user + it needs one field refresh; optionally pre-populate static entries from your outline. +- Do not attempt pixel-exact page design (complex multi-column covers, inline floating + wrap). If the user wants that fidelity, produce the PDF route instead. + +## Quality bar + +- Heading levels never skip (no Heading 1 straight to Heading 3). +- No empty sections: every heading is followed by content or removed. +- Numbers in tables right-aligned; units declared in the header. diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/edit.md b/plugins/Hylouis233/document-skills/skills/docx/references/edit.md new file mode 100644 index 0000000..4e90b56 --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/docx/references/edit.md @@ -0,0 +1,304 @@ +# Edit an existing DOCX + +Editing has two tiers. Pick the lowest tier that can express the change. + +## Tier 1 - python-docx structural edits (preferred) + +python-docx opens the real package and rewrites it safely. Use it for: adding/removing +paragraphs, tables, images; changing styles; editing text of a specific run; find-replace over +paragraph text. + +```python +from docx import Document +from docx.oxml.ns import qn + +SAFE_RUN_CHILDREN = { + qn("w:rPr"), qn("w:t"), qn("w:tab"), qn("w:cr"), +} +MODELED_PARAGRAPH_CHILDREN = {qn("w:pPr"), qn("w:r")} + +def unsafe_run_content(run): + unsafe = [] + for child in run._r: + # Assigning run.text can reconstruct text, tabs, and ordinary line breaks only. + ordinary_break = child.tag == qn("w:br") and dict(child.attrib) in ( + {}, {qn("w:type"): "textWrapping"}, + ) + if child.tag not in SAFE_RUN_CHILDREN and not ordinary_break: + unsafe.append(child.tag.rsplit("}", 1)[-1]) + return unsafe + +def replace_across_runs(paragraph, old, new): + """Replace text-only matches; reject drawings, fields, and other lossy run content.""" + if not old: + raise ValueError("old must not be empty") + + unmodeled = [ + child.tag.rsplit("}", 1)[-1] + for child in paragraph._p + if child.tag not in MODELED_PARAGRAPH_CHILDREN + ] + if unmodeled: + raise ValueError(f"paragraph contains unmodeled inline containers: {unmodeled}") + + runs = list(paragraph.runs) + text = "".join(run.text for run in runs) + starts = [] + position = 0 + while (start := text.find(old, position)) != -1: + starts.append(start) + position = start + len(old) + + # Map each non-empty run to its character range in the original paragraph text. + spans = [] + position = 0 + for index, run in enumerate(runs): + end = position + len(run.text) + if end > position: + spans.append((index, position, end)) + position = end + + matches = [] + for start in starts: + end = start + len(old) + first, first_start, _ = next(s for s in spans if s[1] <= start < s[2]) + last, last_start, _ = next(s for s in spans if s[1] < end <= s[2]) + matches.append((start, end, first, first_start, last, last_start)) + + # Validate every affected run before mutating any of them. Assigning run.text replaces the + # run XML and would otherwise silently delete an inline drawing, field, footnote reference, + # or a page/column break. + affected_indexes = { + index + for _, _, first, _, last, _ in matches + for index in range(first, last + 1) + } + unsafe = {} + for index in affected_indexes: + if children := unsafe_run_content(runs[index]): + unsafe[index] = children + if unsafe: + raise ValueError(f"matched runs contain non-text content: {unsafe}") + + # Work backwards so changing a later match cannot move an earlier match. + for start, end, first, first_start, last, last_start in reversed(matches): + prefix = runs[first].text[:start - first_start] + suffix = runs[last].text[end - last_start:] + + if first == last: + runs[first].text = prefix + new + suffix + else: + runs[first].text = prefix + new + for index in range(first + 1, last): + runs[index].text = "" + runs[last].text = suffix + + return len(starts) + +doc = Document("input.docx") + +# Address paragraphs by index over doc.paragraphs (body level). Tables' cells hold their own +# paragraphs: table.rows[i].cells[j].paragraphs +for par in doc.paragraphs: + replace_across_runs(par, "TBD", "To be decided") + +# Append content at a specific position: manipulate the XML tree +target = doc.paragraphs[7]._p +new_par = doc.add_paragraph("Inserted after the target.") +target.addnext(new_par._p) + +doc.save("input.edited.docx") +``` + +The replacement text inherits the first matched run's formatting. Unmatched text before and +after it stays in its original runs, so its formatting is preserved. The routine fails before +making changes if the paragraph contains an inline container that `paragraph.runs` does not expose, +or if any matched run contains a drawing, field, reference, typed page/column break, or a +text-wrapping break with `w:clear` that `run.text` would destroy. Use raw OOXML for those cases. + +## Tier 2 - raw OOXML surgery (only when Tier 1 cannot express it) + +For field codes, sectPr surgery, tracked changes, or parts python-docx does not model. +Rules that keep the archive valid: + +1. Operate on a **copy** of the file. +2. Run the bounded package validator below **before extracting anything**, then extract into a + **new empty temporary directory for every input**. Never reuse a fixed `work/` directory: + members absent from the next DOCX would remain there and be repacked as stale or confidential + content. +3. Parse XML with `lxml`/`xml.etree` - never string replace. Text lives in `w:t` inside runs + (`w:r`) inside paragraphs (`w:p`); a logical sentence can span several runs. +4. Repack with `[Content_Types].xml` first and stored/deflated entries only: + +```python +import os +import unicodedata +from pathlib import Path, PurePosixPath +from tempfile import TemporaryDirectory +from zipfile import ZIP_DEFLATED, ZipFile +from lxml import etree + +MAX_ARCHIVE_BYTES = 200 * 1024 * 1024 +MAX_MEMBERS = 10_000 +MAX_XML_PART = 20 * 1024 * 1024 +MAX_ENTRY = 100 * 1024 * 1024 +MAX_TOTAL_UNCOMPRESSED = 500 * 1024 * 1024 +MAX_COMPRESSION_RATIO = 200 +MAX_MEMBER_COMPONENT_BYTES = 255 +MAX_MEMBER_COMPONENT_UTF16_UNITS = 255 +MAX_MEMBER_PATH_BYTES = 1024 +MAX_MEMBER_PATH_UTF16_UNITS = 240 +MAX_MEMBER_COMPONENTS = 64 + +def require(condition, message): + if not condition: # remains active under python -O + raise ValueError(message) + +WINDOWS_DEVICE_NAMES = { + "con", "prn", "aux", "nul", "conin$", "conout$", + *(f"com{suffix}" for suffix in "123456789¹²³"), + *(f"lpt{suffix}" for suffix in "123456789¹²³"), +} + +def extraction_key(name): + """Return a component key only for portable, canonical member names.""" + is_directory = name.endswith("/") + path = name[:-1] if is_directory else name + require(path and not name.startswith("/") and "\\" not in name, + f"non-canonical archive member path: {name}") + parts = path.split("/") + require(len(parts) <= MAX_MEMBER_COMPONENTS, + "archive member depth exceeds portable extraction limit") + require( + all( + part not in {"", ".", ".."} + and not any(character in ':<>|"?*' for character in part) + and not any(ord(character) < 32 for character in part) + and not part.endswith((".", " ")) + and unicodedata.normalize("NFC", part) == part + and part.partition(".")[0].rstrip(" ").casefold() not in WINDOWS_DEVICE_NAMES + for part in parts + ), + f"non-canonical archive member path: {name}", + ) + require( + all( + len(part.encode("utf-8")) <= MAX_MEMBER_COMPONENT_BYTES + and len(part.encode("utf-16-le")) // 2 <= MAX_MEMBER_COMPONENT_UTF16_UNITS + for part in parts + ), + "archive member component exceeds portable extraction limit", + ) + require(len(path.encode("utf-8")) <= MAX_MEMBER_PATH_BYTES + and len(path.encode("utf-16-le")) // 2 <= MAX_MEMBER_PATH_UTF16_UNITS, + "archive member path exceeds portable extraction limit") + canonical = PurePosixPath(*parts).as_posix() + require(canonical == path, f"non-canonical archive member path: {name}") + key = tuple(unicodedata.normalize("NFC", part.casefold()) for part in parts) + return key, tuple(parts), is_directory + +def validate_extraction_paths(infos): + """Reject aliases and file/directory collisions with a linear component trie.""" + root = {"children": {}, "member": False, "file": False, "spelling": None} + for info in infos: + key, spellings, is_directory = extraction_key(info.filename) + node = root + for normalized, spelling in zip(key, spellings): + require(not node["file"], + "archive file and directory paths collide after extraction") + child = node["children"].get(normalized) + if child is None: + child = { + "children": {}, "member": False, "file": False, + "spelling": spelling, + } + node["children"][normalized] = child + else: + require(child["spelling"] == spelling, + "archive member path spelling collides after extraction") + node = child + require(not node["member"], "archive member paths collide after extraction") + require(is_directory or not node["children"], + "archive file and directory paths collide after extraction") + node["member"] = True + node["file"] = not is_directory + +safe_xml_parser = etree.XMLParser( + load_dtd=False, resolve_entities=False, no_network=True, + huge_tree=False, recover=False, +) + +def validate_docx_archive(archive): + """Enforce archive bounds and CRC before this same handle is extracted.""" + require(os.fstat(archive.fp.fileno()).st_size <= MAX_ARCHIVE_BYTES, + "compressed DOCX file size above limit") + infos = archive.infolist() + require(len(infos) <= MAX_MEMBERS, "archive member count above limit") + names = {info.filename for info in infos} + require(len(names) == len(infos), "duplicate archive member names are unsafe") + validate_extraction_paths(infos) + require("[Content_Types].xml" in names and "word/document.xml" in names, + "required DOCX package parts are missing") + require(sum(info.file_size for info in infos) <= MAX_TOTAL_UNCOMPRESSED, + "declared archive size exceeds the edit limit") + actual_total = 0 + for info in infos: + require(info.file_size <= MAX_ENTRY, f"oversized part: {info.filename}") + require( + info.file_size / max(info.compress_size, 1) <= MAX_COMPRESSION_RATIO, + f"suspicious compression ratio: {info.filename}", + ) + if info.filename.endswith((".xml", ".rels")): + require(info.file_size <= MAX_XML_PART, f"oversized XML part: {info.filename}") + actual_size = 0 + with archive.open(info) as stream: # streams data and verifies its CRC + while chunk := stream.read(64 * 1024): + actual_size += len(chunk) + actual_total += len(chunk) + require(actual_size <= MAX_ENTRY, f"part exceeded read limit: {info.filename}") + require(actual_total <= MAX_TOTAL_UNCOMPRESSED, + "archive exceeded total read limit") + require(actual_size == info.file_size, f"size mismatch: {info.filename}") + +input_path = Path("input.docx") +output_path = Path("output.docx").resolve() # keep output outside the temporary tree +with TemporaryDirectory(prefix="docx-edit-") as scratch: + src = Path(scratch) + with ZipFile(input_path) as archive: + validate_docx_archive(archive) # same open handle; prevents validate/extract swap + archive.extractall(src) + + # Apply the required XML edits under `src` here with `safe_xml_parser`. + # Archive bounds are a safety gate, not a semantic XML-validity gate: Tier 2 may repair XML. + content_types = src / "[Content_Types].xml" + if not content_types.is_file(): + raise FileNotFoundError(content_types) + files = sorted( + (path for path in src.rglob("*") if path.is_file() and path != content_types), + key=lambda path: path.relative_to(src).as_posix(), + ) + + with ZipFile( + output_path, "w", compression=ZIP_DEFLATED, strict_timestamps=False + ) as archive: + archive.write(content_types, "[Content_Types].xml") + for path in files: + archive.write(path, path.relative_to(src).as_posix()) +``` + + The temporary directory is deleted after repacking. This writes relative POSIX archive names, + does not add directory entries, and excludes `[Content_Types].xml` from the remaining files so + it cannot be added twice. +5. If you touched part names or added parts, update `[Content_Types].xml` and + `word/_rels/document.xml.rels` consistently - a mismatch here is the classic silent corrupt. + +## Never do + +- Blind find/replace on the raw XML string of `word/document.xml`. +- Deleting parts that look unused (styles, theme, settings) - viewers may require them. +- Editing a document that is open in Word (the save will collide with the lock file). + +## Delivery + +Save to `-edited.docx` unless the user explicitly asked to overwrite. Run the +postcheck from SKILL.md step 4 on the output. diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/read.md b/plugins/Hylouis233/document-skills/skills/docx/references/read.md new file mode 100644 index 0000000..06611a2 --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/docx/references/read.md @@ -0,0 +1,388 @@ +# Read / extract a DOCX + +Pick by fidelity needed. + +## Fast text extraction (pandoc, if installed) + +```bash +pandoc -t markdown input.docx -o extracted.md +``` + +Best structural fidelity for prose (headings become markdown headings, tables become pipe +tables). Prefer this when the goal is content, not coordinates. + +## Structured access (python-docx) + +```python +from contextlib import contextmanager +import zipfile +from pathlib import Path +from tempfile import TemporaryFile + +from docx import Document +from docx.oxml.ns import qn +from docx.table import Table +from docx.text.paragraph import Paragraph +from docx.text.run import Run +from lxml import etree + +MAX_ARCHIVE_BYTES = 200 * 1024 * 1024 +MAX_MEMBERS = 10_000 +MAX_XML_PART = 20 * 1024 * 1024 +MAX_ENTRY = 100 * 1024 * 1024 +MAX_TOTAL_UNCOMPRESSED = 500 * 1024 * 1024 +MAX_COMPRESSION_RATIO = 200 +CONTENT_TYPES_NAMESPACE = "http://schemas.openxmlformats.org/package/2006/content-types" + +def require(condition, message): + if not condition: + raise ValueError(message) + +SAFE_XML_PARSER = etree.XMLParser( + load_dtd=False, resolve_entities=False, no_network=True, + huge_tree=False, recover=False, +) + +def xml_content_type(value): + media_type = (value or "").split(";", 1)[0].strip().casefold() + return media_type in {"application/xml", "text/xml"} or media_type.endswith("+xml") + +def declared_xml_parts(archive, infos): + """Classify XML by OPC declarations, not filename spelling alone.""" + by_name = {info.filename: info for info in infos} + content_types_info = by_name.get("[Content_Types].xml") + require(content_types_info is not None, "missing [Content_Types].xml") + require(content_types_info.file_size <= MAX_XML_PART, + "oversized XML part: [Content_Types].xml") + with archive.open(content_types_info) as stream: + content_types_blob = stream.read(MAX_XML_PART + 1) + require(len(content_types_blob) <= MAX_XML_PART, + "oversized XML part: [Content_Types].xml") + root = etree.fromstring(content_types_blob, parser=SAFE_XML_PARSER) + require(root.tag == f"{{{CONTENT_TYPES_NAMESPACE}}}Types", + "invalid [Content_Types].xml root") + defaults = {} + overrides = {} + for child in root: + if child.tag == f"{{{CONTENT_TYPES_NAMESPACE}}}Default": + extension = (child.get("Extension") or "").casefold() + require(extension and extension not in defaults, + "invalid duplicate content-type default") + defaults[extension] = child.get("ContentType") or "" + elif child.tag == f"{{{CONTENT_TYPES_NAMESPACE}}}Override": + part_name = child.get("PartName") or "" + require(part_name.startswith("/") and part_name[1:] not in overrides, + "invalid duplicate content-type override") + overrides[part_name[1:]] = child.get("ContentType") or "" + xml_names = {"[Content_Types].xml"} + for info in infos: + suffix = info.filename.rsplit(".", 1)[1].casefold() if "." in info.filename else "" + content_type = overrides.get(info.filename, defaults.get(suffix, "")) + if (info.filename.casefold().endswith((".xml", ".rels")) + or xml_content_type(content_type)): + xml_names.add(info.filename) + return xml_names + +def validate_docx_archive(archive): + infos = archive.infolist() + require(len(infos) <= MAX_MEMBERS, "archive member count above limit") + names = {info.filename for info in infos} + require(len(names) == len(infos), "duplicate archive member names are unsafe") + require("[Content_Types].xml" in names and "word/document.xml" in names, + "missing required OPC members") + require(sum(info.file_size for info in infos) <= MAX_TOTAL_UNCOMPRESSED, + "declared total uncompressed size above limit") + xml_names = declared_xml_parts(archive, infos) + actual_total = 0 + for info in infos: + require(info.file_size <= MAX_ENTRY, f"oversized part: {info.filename}") + require(info.file_size / max(info.compress_size, 1) <= MAX_COMPRESSION_RATIO, + f"suspicious compression ratio: {info.filename}") + is_xml = info.filename in xml_names + if is_xml: + require(info.file_size <= MAX_XML_PART, f"oversized XML part: {info.filename}") + chunks = [] + actual_size = 0 + with archive.open(info) as stream: + while chunk := stream.read(64 * 1024): + actual_size += len(chunk) + actual_total += len(chunk) + require(actual_size <= MAX_ENTRY, f"part exceeded read limit: {info.filename}") + require(actual_total <= MAX_TOTAL_UNCOMPRESSED, + "archive exceeded total read limit") + if is_xml: + chunks.append(chunk) + require(actual_size == info.file_size, f"size mismatch: {info.filename}") + if is_xml: + etree.fromstring(b"".join(chunks), parser=SAFE_XML_PARSER) + +@contextmanager +def validated_docx_source(path): + """Yield one private, bounded snapshot for both validation and python-docx.""" + with Path(path).open("rb") as external_source, TemporaryFile() as source: + copied = 0 + while chunk := external_source.read(64 * 1024): + copied += len(chunk) + require(copied <= MAX_ARCHIVE_BYTES, "compressed DOCX file size above limit") + source.write(chunk) + source.seek(0) + with zipfile.ZipFile(source) as archive: + validate_docx_archive(archive) + source.seek(0) + yield source + +MC_NAMESPACE = "http://schemas.openxmlformats.org/markup-compatibility/2006" +MC_ALTERNATE_CONTENT = f"{{{MC_NAMESPACE}}}AlternateContent" +MC_CHOICE = f"{{{MC_NAMESPACE}}}Choice" +MC_FALLBACK = f"{{{MC_NAMESPACE}}}Fallback" +SUPPORTED_MC_NAMESPACE_URIS = { + MC_NAMESPACE, + qn("w:p").split("}", 1)[0][1:], + qn("r:id").split("}", 1)[0][1:], +} + +def alternate_content_branch(element): + """Select the first Choice whose required namespace URIs are understood.""" + fallback = None + for child in element.iterchildren(): + if child.tag == MC_CHOICE: + required_prefixes = (child.get("Requires") or "").split() + if required_prefixes and all( + child.nsmap.get(prefix) in SUPPORTED_MC_NAMESPACE_URIS + for prefix in required_prefixes + ): + return child + elif child.tag == MC_FALLBACK and fallback is None: + fallback = child + return fallback + +def unresolved_alternate_content(element): + return { + "kind": "AlternateContent", + "requires": [ + (child.get("Requires") or "").split() + for child in element.iterchildren() + if child.tag == MC_CHOICE + ], + "reason": "no supported Choice and no Fallback", + } + +def iter_effective_children(root): + """Yield one selected markup-compatibility branch, never Choice and Fallback.""" + for child in root.iterchildren(): + if child.tag != MC_ALTERNATE_CONTENT: + yield "element", child + continue + branch = alternate_content_branch(child) + if branch is None: + yield "unreadable", unresolved_alternate_content(child) + else: + yield from iter_effective_children(branch) + +def iter_part_blocks(root, parent): + """Yield each paragraph/table once, descending through block content controls.""" + for child_kind, child in iter_effective_children(root): + if child_kind == "unreadable": + yield child_kind, child + continue + if child.tag == qn("w:p"): + yield "paragraph", Paragraph(child, parent) + elif child.tag == qn("w:tbl"): + yield "table", Table(child, parent) + elif child.tag == qn("w:altChunk"): + relationship_id = child.get(qn("r:id")) + part = getattr(parent, "part", None) + relationship = None if part is None else part.rels.get(relationship_id) + yield "unreadable", { + "kind": "altChunk", + "relationship_id": relationship_id, + "target": None if relationship is None else relationship.target_ref, + "content_type": None if relationship is None or relationship.is_external + else relationship.target_part.content_type, + } + else: + yield from iter_part_blocks(child, parent) + +def iter_paragraph_items(paragraph): + """Include runs/unreadable records in their selected, rendered order.""" + def walk(element): + for child_kind, child in iter_effective_children(element): + if child_kind == "unreadable": + yield child_kind, child + continue + if child.tag == qn("w:r"): + yield "run", Run(child, paragraph) + elif child.tag != qn("w:p"): # nested text-box paragraphs are yielded separately + yield from walk(child) + yield from walk(paragraph._p) + +def iter_paragraph_runs(paragraph): + """Yield only runs; paragraph_text separately reports unresolved alternatives.""" + for kind, item in iter_paragraph_items(paragraph): + if kind == "run": + yield item + +def legacy_symbol_record(symbol): + """Report font-specific w:sym content without guessing at a Unicode mapping.""" + font = symbol.get(qn("w:font")) + character = symbol.get(qn("w:char")) + return f"[unreadable legacy symbol font={font!r} char={character!r}]" + +def run_text(run): + pieces = [] + text_tags = { + qn("w:br"), qn("w:cr"), qn("w:noBreakHyphen"), qn("w:ptab"), + qn("w:t"), qn("w:tab"), qn("w:sym"), + } + for child_kind, child in iter_effective_children(run._r): + if child_kind == "unreadable": + pieces.append(f"[unreadable {child['kind']}: {child['reason']}]") + elif child.tag in text_tags: + pieces.append(legacy_symbol_record(child) if child.tag == qn("w:sym") else str(child)) + return "".join(pieces) + +def paragraph_text(paragraph): + pieces = [] + for kind, item in iter_paragraph_items(paragraph): + pieces.append( + run_text(item) if kind == "run" + else f"[unreadable {item['kind']}: {item['reason']}]" + ) + return "".join(pieces) + +def cell_paragraph_text(paragraph): + """Render cell controls as visible markers while honoring one MC branch.""" + pieces = [] + for kind, item in iter_paragraph_items(paragraph): + if kind == "unreadable": + pieces.append(f"[unreadable {item['kind']}: {item['reason']}]") + continue + for child_kind, child in iter_effective_children(item._r): + if child_kind == "unreadable": + pieces.append(f"[unreadable {child['kind']}: {child['reason']}]") + elif child.tag == qn("w:t"): + pieces.append(child.text or "") + elif child.tag in (qn("w:tab"), qn("w:ptab")): + pieces.append("") + elif child.tag in (qn("w:br"), qn("w:cr")): + pieces.append("
") + elif child.tag == qn("w:noBreakHyphen"): + pieces.append("-") + elif child.tag == qn("w:sym"): + pieces.append(legacy_symbol_record(child)) + return "".join(pieces) + +def tc_text(tc, parent): + """Cell text rebuilt per paragraph, keeping tabs and breaks visible. + + Joining only the w:t descendants concatenates separate paragraphs + ("First" + "Second" -> "FirstSecond") and loses separators entirely. + """ + paragraphs = [] + for kind, block in iter_part_blocks(tc, parent): + if kind != "paragraph": # nested tables are represented recursively, not duplicated here + continue + paragraphs.append(cell_paragraph_text(block)) + return " / ".join(paragraphs) + +def iter_content_control_children(root, target_tag): + """Find physical rows/cells through sdtContent without entering nested tables.""" + for child_kind, child in iter_effective_children(root): + if child_kind == "unreadable": + raise ValueError( + f"unresolved AlternateContent while locating {target_tag}: {child}" + ) + if child.tag == target_tag: + yield child + elif child.tag == qn("w:sdt"): + for content in child.findall(qn("w:sdtContent")): + yield from iter_content_control_children(content, target_tag) + +def table_content(table): + rows = [] + for row_element in iter_content_control_children(table._tbl, qn("w:tr")): + rendered_cells = [] + row_properties = row_element.find(qn("w:trPr")) + grid_before_node = None if row_properties is None else row_properties.find(qn("w:gridBefore")) + grid_after_node = None if row_properties is None else row_properties.find(qn("w:gridAfter")) + grid_before = 0 if grid_before_node is None else int(grid_before_node.get(qn("w:val"), "0")) + grid_after = 0 if grid_after_node is None else int(grid_after_node.get(qn("w:val"), "0")) + column = grid_before + # row.cells repeats a merge-origin proxy for every grid position it spans. + # Walk physical w:tc elements and expose the merge structure instead. + for cell_element in iter_content_control_children(row_element, qn("w:tc")): + cell_properties = cell_element.find(qn("w:tcPr")) + grid_span = None if cell_properties is None else cell_properties.find(qn("w:gridSpan")) + colspan = 1 if grid_span is None else int(grid_span.get(qn("w:val"), "1")) + vertical = None if cell_properties is None else cell_properties.find(qn("w:vMerge")) + vertical_merge = None if vertical is None else vertical.get(qn("w:val"), "continue") + nested_tables = [] + unreadable = [] + for kind, block in iter_part_blocks(cell_element, table): + if kind == "table": + nested_tables.append(table_content(block)) + elif kind == "unreadable": + unreadable.append(block) + rendered_cells.append({ + "column": column, "colspan": colspan, + "vMerge": vertical_merge, + "text": tc_text(cell_element, table), + "tables": nested_tables, + "unreadable": unreadable, + }) + column += colspan + rows.append({ + "grid_before": grid_before, + "cells": rendered_cells, + "grid_after": grid_after, + }) + return rows + +with validated_docx_source("input.docx") as source: + doc = Document(source) + content_controls = list(doc.element.body.iter(qn("w:sdt"))) + blocks = list(iter_part_blocks(doc.element.body, doc)) + print("content controls:", len(content_controls), "top-level blocks:", len(blocks)) + for kind, block in blocks: + if kind == "paragraph": + print(block.style.name, "|", paragraph_text(block)) + elif kind == "table": + print("table |", table_content(block)) + else: + print("unreadable |", block) +``` + +Notes: + +- `doc.paragraphs` includes only direct body paragraphs, `doc.tables` includes only direct body + tables, and `Paragraph.runs` omits runs wrapped by inline content controls and other containers. + Use the XML-backed traversal above and report the content-control count. It stops descending + when a table is yielded, so table text is not also emitted as prose; `table_content()` handles + nested tables recursively and emits `grid_before`, `grid_after`, `column`, `colspan`, and + `vMerge` metadata for physical cells instead of duplicating merge-origin text through + `row.cells`. It walks physical rows and cells through `w:sdtContent`; python-docx's public row + and cell collections omit those wrapped elements. The row-level grid omissions are required + for nonuniform tables whose cells do not start in logical column zero or do not extend to the + final grid column. + `mc:AlternateContent` is evaluated once: the first Choice whose required namespace URIs this + extractor understands wins, otherwise the Fallback is used. An alternative without either a + supported Choice or a Fallback is reported as unreadable; mutually exclusive branches must + never be concatenated. + Imported `w:altChunk` HTML/RTF/document parts are not modeled by python-docx; the traversal + reports their relationship target and content type as `unreadable` instead of silently + presenting an incomplete extraction. Convert them with a trusted office renderer before + claiming their content was read. Each table-cell record carries its own `unreadable` list so + an import nested in a cell is not lost from the report. + A legacy `w:sym` code is specific to its symbol font, so the helpers preserve its position + as an explicit unreadable record containing the font and character code rather than silently + dropping visible content or guessing a Unicode character. + Text boxes, headers, footers, and footnotes still require their own collections + (`section.header/.footer`) or raw XML. +- For revision/comment metadata, inspect the XML parts directly: `word/comments.xml`, + `w:ins`/`w:del` elements in `word/document.xml`. + +## Reporting contract + +When summarizing a document for the user, lead with: heading outline, paragraph count, table +count with dimensions, and any parts that could not be read. Do not silently skip unreadable +parts. diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/review.md b/plugins/Hylouis233/document-skills/skills/docx/references/review.md new file mode 100644 index 0000000..87b5201 --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/docx/references/review.md @@ -0,0 +1,129 @@ +# Review / repair a DOCX + +## Symptom-driven triage + +| Symptom | Likely cause | Fix route +|---|---|---| +File will not open at all | broken ZIP (truncated, wrong repack) | run the bounded health check below; if entries are damaged, recover from the user's original or prior version | +Opens with "unreadable content" repair prompt | content-types / rels mismatch, invalid XML | Tier 2 surgery: validate XML parses, check `[Content_Types].xml` covers every part extension | +Text present but styles lost | document rebuilt from scratch instead of edited | redo as edit on the original package | +Images missing | media parts not repacked or rels broken | verify `word/media/*` exist and `document.xml.rels` references them | +Fonts render differently on another machine | non-embedded fonts | expected; report which fonts are referenced (`w:rFonts` values) | + +## Programmatic health check + +```python +import zipfile +from pathlib import Path +from lxml import etree + +path = "input.docx" +MAX_ARCHIVE_BYTES = 200 * 1024 * 1024 +MAX_MEMBERS = 10_000 +MAX_XML_PART = 20 * 1024 * 1024 +MAX_ENTRY = 100 * 1024 * 1024 +MAX_TOTAL_UNCOMPRESSED = 500 * 1024 * 1024 +MAX_COMPRESSION_RATIO = 200 +CONTENT_TYPES_NAMESPACE = "http://schemas.openxmlformats.org/package/2006/content-types" + +# Security limits must survive `python -O` (which strips assert statements), +# so every check raises explicitly instead of asserting. +def require(condition, message): + if not condition: + raise ValueError(message) + +safe_xml_parser = etree.XMLParser( + load_dtd=False, + resolve_entities=False, + no_network=True, + huge_tree=False, + recover=False, +) + +def xml_content_type(value): + media_type = (value or "").split(";", 1)[0].strip().casefold() + return media_type in {"application/xml", "text/xml"} or media_type.endswith("+xml") + +def declared_xml_parts(archive, infos): + """Classify XML from OPC declarations plus conventional suffixes.""" + by_name = {info.filename: info for info in infos} + content_types_info = by_name.get("[Content_Types].xml") + require(content_types_info is not None, "missing [Content_Types].xml") + require(content_types_info.file_size <= MAX_XML_PART, + "oversized XML part: [Content_Types].xml") + with archive.open(content_types_info) as stream: + content_types_blob = stream.read(MAX_XML_PART + 1) + require(len(content_types_blob) <= MAX_XML_PART, + "oversized XML part: [Content_Types].xml") + root = etree.fromstring(content_types_blob, parser=safe_xml_parser) + require(root.tag == f"{{{CONTENT_TYPES_NAMESPACE}}}Types", + "invalid [Content_Types].xml root") + defaults = {} + overrides = {} + for child in root: + if child.tag == f"{{{CONTENT_TYPES_NAMESPACE}}}Default": + extension = (child.get("Extension") or "").casefold() + require(extension and extension not in defaults, + "invalid duplicate content-type default") + defaults[extension] = child.get("ContentType") or "" + elif child.tag == f"{{{CONTENT_TYPES_NAMESPACE}}}Override": + part_name = child.get("PartName") or "" + require(part_name.startswith("/") and part_name[1:] not in overrides, + "invalid duplicate content-type override") + overrides[part_name[1:]] = child.get("ContentType") or "" + xml_names = {"[Content_Types].xml"} + for info in infos: + suffix = info.filename.rsplit(".", 1)[1].casefold() if "." in info.filename else "" + content_type = overrides.get(info.filename, defaults.get(suffix, "")) + if (info.filename.casefold().endswith((".xml", ".rels")) + or xml_content_type(content_type)): + xml_names.add(info.filename) + return xml_names +# Check the package itself before ZipFile materializes its central directory. +require(Path(path).stat().st_size <= MAX_ARCHIVE_BYTES, + "compressed DOCX file size above limit") +with zipfile.ZipFile(path) as z: + infos = z.infolist() + # Check the count before building sets, summing sizes, or opening any member. + require(len(infos) <= MAX_MEMBERS, "archive member count above limit") + names = {info.filename for info in infos} + require(len(names) == len(infos), "duplicate archive member names are unsafe") + require("[Content_Types].xml" in names and "word/document.xml" in names, + "missing required OPC members") + require(sum(info.file_size for info in infos) <= MAX_TOTAL_UNCOMPRESSED, + "declared total uncompressed size above limit") + xml_names = declared_xml_parts(z, infos) + actual_total = 0 + for info in infos: + require(info.file_size <= MAX_ENTRY, f"oversized part: {info.filename}") + ratio = info.file_size / max(info.compress_size, 1) + require(ratio <= MAX_COMPRESSION_RATIO, f"suspicious compression ratio: {info.filename}") + is_xml = info.filename in xml_names + if is_xml: + require(info.file_size <= MAX_XML_PART, f"oversized XML part: {info.filename}") + chunks = [] + actual_size = 0 + # Stream every bounded member to verify decompression and CRC. Do not call testzip() + # before the limits: it would expand every member regardless of declared risk. + with z.open(info) as stream: + while chunk := stream.read(64 * 1024): + actual_size += len(chunk) + actual_total += len(chunk) + require(actual_size <= MAX_ENTRY, f"part exceeded read limit: {info.filename}") + require(actual_total <= MAX_TOTAL_UNCOMPRESSED, "archive exceeded total read limit") + if is_xml: + chunks.append(chunk) + require(actual_size == info.file_size, f"size mismatch: {info.filename}") + if is_xml: + etree.fromstring(b"".join(chunks), parser=safe_xml_parser) +``` + +These limits are conservative review defaults, not a DOCX specification. Raise one only for an +explicitly trusted, expected large input, and keep the streaming/per-part checks in place. + +Then the SKILL.md postcheck (python-docx re-open, optional soffice PDF smoke test). + +## Report format + +State what is broken, the minimal repair applied, and what could not be verified without the +target viewer (exact pagination, field updates, embedded font rendering). diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/scenes.md b/plugins/Hylouis233/document-skills/skills/docx/references/scenes.md new file mode 100644 index 0000000..b149353 --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/docx/references/scenes.md @@ -0,0 +1,88 @@ +# Scene patterns: paper, resume, official document, contract + +When the request matches a known scene, start from its skeleton instead of improvising +structure. Every scene below still goes through the normal create/edit route and postcheck; +this file only fixes structure, typography conventions, and scene-specific verification. + +## Academic paper (中文学术论文 shape) + +Skeleton: 标题 (二号黑体 or per journal) -> 作者 -> 单位 -> 摘要 -> 关键词 -> numbered body +(1 / 1.1 / 1.1.1) -> 参考文献 -> optional 附录/致谢. + +- Headings carry the numbering as literal text when no journal template is supplied; heading + styles stay semantic (`Heading 1..4`) so navigation and TOC keep working. +- Figures/tables: caption paragraph below the object, numbered per chapter + (图 2-1 / 表 3-2); reference in text before the object appears. +- References: GB/T 7714 style is the common Chinese requirement - build each entry as a + plain paragraph with a hanging indent, not a numbered Word list (journals re-style anyway). +- Fonts by convention: 正文宋体小四, fixed or 1.5 line spacing, first-line indent two chars - + see [cjk.md](cjk.md) for the mechanics. +- Out of scope: equations as OMML and journal-specific LaTeX export - offer the PDF route or + the journal's own template instead of hand-building OMML. + +## Resume + +- One page unless the user explicitly says otherwise; the constraint is content curation, + not shrinking fonts below 10 pt. +- Structure: 姓名+联系方式 header -> 求职意向 (optional) -> 经历 (reverse chronological) -> + 教育 -> 技能. Section headings as `Heading 2`, entries as styled paragraphs, bullets via + `List Bullet`. +- ATS-safe means: real text (no text boxes, no multi-column layout, no icons), standard + section names, dates as plain text `2024-03` not fields. +- A single subtle table for the contact header is acceptable; skill-rating bars and other + graphic devices are not - they print wrong and break parsers. +- Verify: render with soffice, confirm one page, confirm all text extracts from the PDF. + +## Official document (公文, GB/T 9704 shape) + +Structure: 版头 (发文机关标志+文号) -> 标题 (二号宋体加粗 per current practice) -> 主送机关 -> +正文 (三号仿宋) -> 附件说明 -> 发文机关署名+成文日期 -> 抄送 -> 印发信息. + +- Body hierarchy is fixed: 一、 -> （一） -> 1. -> （1）; never skip or reorder levels. +- Page geometry and line spacing values: see [cjk.md](cjk.md) - the GB/T table and fixed + 28-30 pt spacing live there. +- The red 发文机关标志 header is a graphics-level element python-docx does not model; + generate the document without it and tell the user to add it in their official template, + or ask for their template file and edit inside it. +- Numbers: Arabic numerals for dates and quantities per the standard; 成文日期 as + 2026年8月16日 with Arabic numerals. + +## Contract + +Skeleton: 标题 -> 当事人 block (名称/住所/法定代表人 per party) -> 鉴于 (recitals, optional) -> +numbered 条款 -> 签署 block. + +- Clause numbering as literal text with a fixed hierarchy (第一条 / 1.1 / (a)); do not use + auto-numbered lists - counter restarts and cross-references become fragile. +- Defined terms: bold at first definition only; thereafter plain. Search for the term to + confirm it is defined exactly once before bolding. +- Cross-references as literal text ("见第 5.2 条"); after edits, grep every 第 X 条 reference + and verify the target still exists at that number. +- Signature block: a borderless 2-column table (甲方/乙方 rows for 签字、盖章、日期) at the + end. `keep_with_next` on preceding paragraphs alone does not stop table rows from splitting + across pages - apply the row-level guard to the table itself: + + ```python + from docx.oxml import OxmlElement + from docx.oxml.ns import qn + + def keep_table_together(table): + for row in table.rows: + trPr = row._tr.get_or_add_trPr() + if trPr.find(qn("w:cantSplit")) is None: + trPr.append(OxmlElement("w:cantSplit")) # a row never splits mid-row + for row in table.rows[:-1]: + for cell in row.cells: + for par in cell.paragraphs: + par.paragraph_format.keep_with_next = True # row sticks to the next row + ``` + + Then verify in the rendered PDF that the whole block landed on one page. +- Verification beyond the standard postcheck: every defined term defined once; every + cross-reference resolves; signature block on one page in the rendered PDF. + +## Routing note + +Scenes chain: pick the scene skeleton, then execute it through +[create.md](create.md) (new document) or [edit.md](edit.md) (user supplied a draft), applying +[cjk.md](cjk.md) whenever the document contains CJK text. diff --git a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md new file mode 100644 index 0000000..f877fd7 --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md @@ -0,0 +1,240 @@ +--- +name: pdf +description: Create, extract, transform, or analyze PDF documents. Use this Skill whenever a .pdf file is the input or output - generating reports, one-pagers, posters or structured documents as real selectable-text PDFs; extracting text, tables or images from existing PDFs; merging, splitting, rotating, or watermarking pages; filling forms; checking that a produced PDF has the right page count and extractable text. +--- + +# PDF workbench + +PDF tasks fail when the agent mixes incompatible libraries or generates pages as screenshots. +Route by job, use one tool per job, verify the artifact. + +## Step 0 - Check the toolchain + +```bash +python -c "import reportlab, pypdf, fitz; print('reportlab', reportlab.Version, '| pypdf ok | pymupdf ok')" +``` + +- `reportlab` - creation (preferred: selectable text, controlled layout). +- `pypdf` - page-level transforms: split, merge, rotate, encrypt, form fields. +- `pymupdf` (`fitz`) - extraction (text with coordinates, tables, images), rasterization, + page-level inspection. + +Missing any -> say so and degrade to the ones present; do not substitute screenshot pipelines. + +## Step 1 - Pick exactly one tool for the job + +| Task | Tool | Reference | +|---|---|---| +| Build a new structured PDF (report, one-pager, letter, checklist) | ReportLab | [references/create.md](references/create.md) | +| Extract text / tables / images | PyMuPDF | [references/extract.md](references/extract.md) | +| Merge, split, rotate, watermark, encrypt, forms | pypdf | [references/transform.md](references/transform.md) | +| Inspect: page count, sizes, fonts, links, blank-page detection | PyMuPDF | [references/inspect.md](references/inspect.md) | + +Chaining tools on one artifact is fine (create -> inspect). Using two creation libraries on one +file is not. + +## Step 2 - Shared rules + +1. **Text-first**: a generated PDF must contain extractable text. Image-of-text is acceptable + only when the user asked for a rasterized look or provides only images. +2. **Page geometry is explicit**: A4 = 595.27 x 841.89 pt, US Letter = 612 x 792 pt. Declare + the target size and margins up front; re-check fit after generation. +3. **Fonts**: the standard 14 fonts cover only limited encodings; they do not automatically + support arbitrary Unicode. Check that the selected face contains every requested character. + If any glyph is unsupported - including CJK, Cyrillic, Arabic, Devanagari, or emoji - embed + one or more licensed TTF/OTF fonts with the required coverage and use them for those runs. + If no suitable embeddable font is available, report the limitation instead of emitting tofu. +4. **Overflow is a defect**: content that spills past the last page or the margin must be + detected in postcheck and fixed (shrink, paginate, or cut), never shipped. +5. Write output to a new path; keep inputs untouched unless in-place was requested. +6. **Accessibility claims require tagged output.** ReportLab does not produce a tagged PDF/UA + document, so do not describe ordinary ReportLab output as accessible or screen-reader-ready. + When PDF/UA or assistive-technology compatibility is required, use a toolchain that can create + and validate tags, reading order, structure roles, language, and alternative text. If that + toolchain is unavailable, stop and report the limitation; offer an accessible DOCX or HTML + deliverable instead of silently returning an untagged PDF. + +## Step 3 - Postcheck (mandatory) + +```python +import os +import fitz +import pypdf + +output_path = "output.pdf" +expected_page_count = 1 # set this from the task; do not derive it from the output +password = os.environ.get("PDF_PASSWORD") +r = pypdf.PdfReader(output_path) +if r.is_encrypted: + # Permission-encrypted PDFs commonly have an empty user password and open normally. + if r.decrypt("") == 0: + if not password: + raise RuntimeError("set PDF_PASSWORD so the encrypted output can be postchecked") + if r.decrypt(password) == 0: + raise RuntimeError("PDF_PASSWORD could not decrypt the output") + +def require(condition, message): + # Mandatory verification must remain active under python -O. + if not condition: + raise ValueError(message) + +page_count = len(r.pages) +require( + page_count == expected_page_count, + f"expected {expected_page_count} pages, got {page_count}", +) +render_doc = fitz.open(output_path) +if render_doc.needs_pass and render_doc.authenticate("") <= 0: + if not password: + raise RuntimeError("set PDF_PASSWORD so encrypted widgets can be rendered") + if render_doc.authenticate(password) <= 0: + raise RuntimeError("PDF_PASSWORD could not decrypt the output for widget rendering") +page_texts = { + number: (page.extract_text() or "").strip() + for number, page in enumerate(r.pages, start=1) +} + +def normalized_box(box): + x0, y0, x1, y1 = (float(value) for value in box) + left, right = sorted((x0, x1)) + bottom, top = sorted((y0, y1)) + return left, bottom, right, top + +def normalized_size(box): + left, bottom, right, top = normalized_box(box) + return right - left, top - bottom + +def widget_count(page, rendered_page): + # Interactive-only pages can be valid, but hidden metadata widgets are not visible content. + non_viewable_flags = 1 | 2 | 32 # Invisible, Hidden, NoView annotation flags + count = 0 + for ref in page.get("/Annots") or []: + widget = ref.get_object() + if widget.get("/Subtype") != "/Widget": + continue + flags = int(widget.get("/F", 0)) + rectangle = widget.get("/Rect") + if flags & non_viewable_flags or rectangle is None: + continue + left, bottom, right, top = normalized_box(rectangle) + crop_left, crop_bottom, crop_right, crop_top = normalized_box(page.cropbox) + media_left, media_bottom, media_right, media_top = normalized_box(page.mediabox) + visible_left = max(crop_left, media_left) + visible_bottom = max(crop_bottom, media_bottom) + visible_right = min(crop_right, media_right) + visible_top = min(crop_top, media_top) + intersects_visible_page = ( + min(right, visible_right) > max(left, visible_left) + and min(top, visible_top) > max(bottom, visible_bottom) + ) + if right <= left or top <= bottom or not intersects_visible_page: + continue + # Do not require /AP: viewers may synthesize it from field defaults. Instead, + # render annotations on and off and require this widget region to change visibly. + xref = getattr(ref, "idnum", None) + if xref is None: + continue + try: + rendered_widget = rendered_page.load_widget(xref) + clip = ( + rendered_widget.rect * rendered_page.rotation_matrix + ) & rendered_page.rect + if clip.is_empty: + continue + with_widget = rendered_page.get_pixmap( + matrix=fitz.Matrix(2, 2), clip=clip, alpha=False, annots=True, + ) + without_widgets = rendered_page.get_pixmap( + matrix=fitz.Matrix(2, 2), clip=clip, alpha=False, annots=False, + ) + except (RuntimeError, ValueError): + continue + if with_widget.samples != without_widgets.samples: + count += 1 + return count + +intentionally_raster_only_pages = set() +missing_text_pages = [ + number for number, text in page_texts.items() + if number not in intentionally_raster_only_pages + and not text + and widget_count(r.pages[number - 1], render_doc[number - 1]) == 0 +] +require(not missing_text_pages, f"pages without extractable text: {missing_text_pages}") +# Add task-specific checks when exact copy matters, for example +# {1: ("Report title",), 2: ("Conclusion",)}. Per-page text presence is enforced above. +expected_strings_by_page = {} +for page_number, expected_strings in expected_strings_by_page.items(): + missing = [value for value in expected_strings if value not in page_texts[page_number]] + require(not missing, f"page {page_number} is missing {missing}") +# Compare width/height to the exact size used at creation with a small point tolerance. +# ReportLab A4 is about (595.2756, 841.8898), not the rounded prose value (595.27, 841.89). +expected_page_size = (595.2756, 841.8898) # replace for Letter or a task-specific size +page_size_tolerance = 0.5 +page_sizes = [ + normalized_size(page.mediabox) for page in r.pages +] +size_mismatches = [ + (number, actual) + for number, actual in enumerate(page_sizes, start=1) + if any(abs(value - expected) > page_size_tolerance + for value, expected in zip(actual, expected_page_size)) +] +require(not size_mismatches, f"unexpected page sizes: {size_mismatches}") + +# Overflow is a defect (shared rule 4): text, images, or vector drawings that run +# past the page box are clipped even though every check above still passes. Plain +# text extraction silently drops fully off-page text, so use an enlarged clip for +# text and inspect the placement boxes reported for every image and drawing. +overflow_doc = render_doc +overflow_pages = [] +for page in overflow_doc: + # These APIs report unrotated coordinates even when /Rotate is 90/270; + # page.rect uses rotated dimensions. Compare against an unrotated crop-box + # extent so valid high-y portrait content is not flagged on a rotated page. + crop_left, crop_bottom, crop_right, crop_top = normalized_box(page.cropbox) + page_box = fitz.Rect(0, 0, crop_right - crop_left, crop_top - crop_bottom) + text_rects = [ + fitz.Rect(block[:4]) + for block in page.get_text("blocks", clip=fitz.INFINITE_RECT()) + if block[6] == 0 + ] + image_rects = [ + fitz.Rect(0, 0, 1, 1) * fitz.Matrix(*image["transform"]) + for image in page.get_image_info() + ] + drawing_rects = [] + for drawing in page.get_drawings(): + rect = fitz.Rect(drawing["rect"]) + # Path rectangles exclude stroke thickness. Expand stroked paths so a + # line centered on the page edge cannot hide half its width off-page. + stroke_pad = ( + float(drawing.get("width") or 0) / 2 + if "s" in drawing.get("type", "") else 0 + ) + drawing_rects.append(fitz.Rect( + rect.x0 - stroke_pad, rect.y0 - stroke_pad, + rect.x1 + stroke_pad, rect.y1 + stroke_pad, + )) + beyond_box = any( + rect.x0 < page_box.x0 - 0.5 or rect.y0 < page_box.y0 - 0.5 + or rect.x1 > page_box.x1 + 0.5 or rect.y1 > page_box.y1 + 0.5 + for rect in text_rects + image_rects + drawing_rects + ) + if beyond_box: + overflow_pages.append(page.number + 1) +overflow_doc.close() +require(not overflow_pages, f"content extends past the page box on pages: {overflow_pages}") +# The page box is the hard bound. When the task declares specific margins, +# additionally check key blocks against them (or render and inspect visually) - +# content inside the box but past a declared margin is a softer, task-specific +# defect to report. +``` + +Confirm: page count matches the request; every page except those explicitly listed in +`intentionally_raster_only_pages` has extractable text or at least one form widget (a pure +interactive page); each requested key string is listed in `expected_strings_by_page` and +extracts on the correct page; every value in `page_sizes` is the declared size within +`page_size_tolerance`; and no page's text, image placements, or vector drawings extend past +the page box (the overflow check). Report all five. For pixel-sensitive work, render every applicable page +with PyMuPDF at 100 dpi and check that each render is non-blank (mean pixel value). diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/create.md b/plugins/Hylouis233/document-skills/skills/pdf/references/create.md new file mode 100644 index 0000000..0b91e67 --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/create.md @@ -0,0 +1,84 @@ +# Create a PDF with ReportLab + +Use flowables (the Platypus layer) so pagination, spacing, and style separation work for you. + +## Skeleton + +```python +from reportlab.lib.pagesizes import A4 +from reportlab.lib.units import mm +from reportlab.lib import colors +from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle +from reportlab.platypus import (SimpleDocTemplate, Paragraph, Spacer, Table, TableStyle, + ListFlowable, ListItem, KeepTogether) + +styles = getSampleStyleSheet() +h1 = ParagraphStyle("H1x", parent=styles["Heading1"], fontSize=16, spaceAfter=6) +body = ParagraphStyle("Bodyx", parent=styles["BodyText"], fontSize=10, leading=14) + +doc = SimpleDocTemplate( + "output.pdf", + pagesize=A4, + leftMargin=20*mm, rightMargin=20*mm, topMargin=18*mm, bottomMargin=18*mm, + title="Q3 launch checklist", author="document-skills", +) + +story = [ + Paragraph("Q3 Launch Checklist", h1), + Spacer(1, 4*mm), + Paragraph("Owner: platform team. Scope: EU region rollout.", body), + Spacer(1, 6*mm), +] + +data = [["#", "Item", "Owner", "Status"], + ["1", "Freeze scope", "PM", "done"], + ["2", "Load test at 2x", "SRE", "planned"]] +tbl = Table(data, colWidths=[10*mm, 70*mm, 30*mm, 25*mm]) +tbl.setStyle(TableStyle([ + ("BACKGROUND", (0,0), (-1,0), colors.HexColor("#1a3c6e")), + ("TEXTCOLOR", (0,0), (-1,0), colors.white), + ("FONTNAME", (0,0), (-1,0), "Helvetica-Bold"), + ("ROWBACKGROUNDS", (0,1), (-1,-1), [colors.white, colors.HexColor("#eef2f7")]), + ("GRID", (0,0), (-1,-1), 0.4, colors.HexColor("#9aa7b4")), + ("VALIGN", (0,0), (-1,-1), "MIDDLE"), + ("TOPPADDING", (0,0), (-1,-1), 4), + ("BOTTOMPADDING", (0,0), (-1,-1), 4), +])) +story.append(tbl) +story.append(Spacer(1, 6*mm)) +story.append(ListFlowable( + [ListItem(Paragraph("Dry-run in staging with production-shaped data", body))], + bulletType="bullet", +)) +doc.build(story) +``` + +## Rules + +- **Escape plain text before Paragraph**: `Paragraph` parses its string as XML-ish markup, so + user-provided prose containing `&` or `<` raises a parse error or renders wrong. Escape + everything that is data; use markup only for strings you authored as markup: + + ```python + from xml.sax.saxutils import escape + + def para(text, style): + return Paragraph(escape(text), style) # & < > become entity-safe + ``` + + The same applies to `ListItem(Paragraph(...))` and to table cell strings when they flow + through `Paragraph`. +- **Exactly-one-page constraint**: after `build`, run the postcheck. Over budget -> reduce + `leading`, tighten `spaceBefore/After`, cut rows - in that order of preference. Under budget + is usually fine; add a spacer or scale the title block. +- **Keep blocks atomic**: wrap heading + first paragraph (or a table and its caption) in + `KeepTogether` so pagination does not orphan them. +- **Two columns** only via frames/PageTemplates; never by positioning two columns on one + canvas manually unless you are in full-canvas mode with measured coordinates. +- **Full-canvas mode** (`canvas` API) is for fixed-layout artifacts: badges, certificates, + posters with absolute geometry. Measure text with `pdfmetrics.stringWidth` before placing; + never guess whether a string fits. +- **Links**: internal links via bookmarks/anchors, external via `linkURL`; declare the URL + visibly next to the link text when the PDF may be printed. +- **Images**: `Image(path, width=..., height=...)` with both dimensions set from the real + aspect ratio (PIL or PyMuPDF can measure). diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md b/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md new file mode 100644 index 0000000..0e8cd21 --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md @@ -0,0 +1,96 @@ +# Extract from a PDF (PyMuPDF) + +```python +import os +import fitz # pymupdf + +doc = fitz.open("input.pdf") +if doc.needs_pass and doc.authenticate("") <= 0: + password = os.environ.get("PDF_PASSWORD") + if not password or not doc.authenticate(password): + raise RuntimeError("set PDF_PASSWORD to the correct password before extracting") +print("pages:", doc.page_count, "metadata:", doc.metadata) + +# Extract each page; do not use `page` after this loop. +for page in doc: + page_number = page.number + 1 + + # Plain text + text = page.get_text("text", sort=True) # position-sorted reading order for simple layouts + print(f"--- page {page_number} ---") + print(text) + + # Tables: use real table detection (ruled and many borderless layouts), not span soup. + # Cells come back as strings, or None for merged/empty cells. Requires PyMuPDF >= 1.23. + try: + tables = page.find_tables() + except AttributeError: + tables = None + print(f"page {page_number}: PyMuPDF lacks find_tables; fall back to the span route below") + if tables is not None: + if not tables.tables: + print(f"page {page_number}: no table detected") + for t, table in enumerate(tables.tables, start=1): + print(f"page {page_number} table {t} bbox:", tuple(round(v) for v in table.bbox)) + for row in table.extract(): + print(row) + + # With coordinates (decide columns/reading order yourself); also the fallback + # when a table is visually present but find_tables detected nothing. + # Collect and sort first: spans arrive in content-stream order, which can + # differ from the visual order even when plain text extraction is sorted. + spans = [ + (page_number, round(span["bbox"][0]), round(span["bbox"][1]), span["text"]) + for block in page.get_text("dict")["blocks"] + for line in block.get("lines", []) + for span in line["spans"] + ] + for record in sorted(spans, key=lambda item: (item[2], item[1])): + print(*record) + + # Images. Apply a soft mask (xref at info[1]) before saving or transparency is lost. + # Pixmap keeps the image's own colorspace: convert CMYK/ICC bases to RGB first. + # This loop sees image XObjects only. + for i, info in enumerate(page.get_images(full=True), start=1): + base = fitz.Pixmap(doc, info[0]) + if base.colorspace and base.colorspace not in (fitz.csGRAY, fitz.csRGB): + base = fitz.Pixmap(fitz.csRGB, base) + if info[1] > 0: + mask = fitz.Pixmap(doc, info[1]) + pix = fitz.Pixmap(base, mask) + else: + pix = base + pix.save(f"img-p{page_number}-{i}.png") + + # Type-1 blocks include both XObjects and inline images. An xref of zero + # identifies true inline content; positive xrefs were already exported by + # the XObject loop above and must not be written a second time. + image_xrefs = { + image["number"]: image["xref"] + for image in page.get_image_info(xrefs=True) + } + for b in page.get_text("dict")["blocks"]: + if b["type"] != 1 or image_xrefs.get(b["number"]) != 0: + continue + ext = b.get("ext") or "png" + with open(f"img-p{page_number}-inline-{b['number']}.{ext}", "wb") as fh: + fh.write(b["image"]) + + # Rasterize (for visual checks or OCR preprocessing) + pix = page.get_pixmap(dpi=150) + pix.save(f"page-{page_number}.png") +``` + +## Rules + +- Scanned pages return empty `get_text`. Check each page; if its text is empty and it contains + images, report that page as "scanned, needs OCR" instead of claiming there is no text. +- Tables: run `find_tables()` first and extract rows/cells; only when detection returns nothing + but a table is visually present, reconstruct it from span coordinates - and say that + automatic detection failed. Merged cells arrive as `None`; preserve them, do not coerce to + empty strings silently. +- "Sort by position" before emitting tables: spans come in internal order, not visual order; + sort by `(round(bbox[1]), bbox[0])` for top-to-bottom, left-to-right reading. +- Two-column layouts: cluster spans by x-gap before joining lines, or text interleaves columns. +- Never send the whole raw text to the user when asked for a summary; extract, then summarize + with page references (`page 3` etc. derived from `page.number`). diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md b/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md new file mode 100644 index 0000000..ec68e5f --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md @@ -0,0 +1,448 @@ +# Inspect a PDF (PyMuPDF) + +```python +import math +import os +import re +import fitz + +doc = fitz.open("input.pdf") +if doc.needs_pass: + password = os.environ.get("PDF_PASSWORD", "") + if doc.authenticate(password) <= 0: + raise RuntimeError("Encrypted PDF: set a valid PDF_PASSWORD before inspection") + +DIRECT_CHARPROC_NAME = re.compile( + r"/(?:#[0-9A-Fa-f]{2}|[^#\s()<>\[\]{}/%])+" +) +DIRECT_CHARPROC_REFERENCE = re.compile(r"\s+([1-9]\d*)\s+\d+\s+R") + +def indirect_xref(value): + match = re.fullmatch(r"\s*([1-9]\d*)\s+\d+\s+R\s*", value or "") + return int(match.group(1)) if match else None + +def direct_charproc_xrefs(value): + """Parse only a conservative direct dictionary of name -> indirect-object entries.""" + value = (value or "").strip() + if not value.startswith("<<") or not value.endswith(">>"): + return "uninspectable", [] + body = value[2:-2] + position = 0 + references = [] + while position < len(body): + while position < len(body) and body[position].isspace(): + position += 1 + if position == len(body): + break + name_match = DIRECT_CHARPROC_NAME.match(body, position) + if name_match is None: + return "uninspectable", [] + reference_match = DIRECT_CHARPROC_REFERENCE.match(body, name_match.end()) + if reference_match is None: + return "malformed", [] + references.append(int(reference_match.group(1))) + position = reference_match.end() + return ("parsed", references) if references else ("malformed", []) + +def type3_charprocs_status(document, xref, font_type): + """Return verified, malformed, or uninspectable without reading glyph streams.""" + if font_type.replace(" ", "").casefold() != "type3": + return None + if xref <= 0: # A direct font dictionary has no xref for safe nested lookup. + return "uninspectable" + try: + charprocs_type, charprocs_value = document.xref_get_key(xref, "CharProcs") + except (RuntimeError, ValueError): + return "uninspectable" + if charprocs_type == "dict": + parse_status, glyph_xrefs = direct_charproc_xrefs(charprocs_value) + if parse_status != "parsed": + return parse_status + elif charprocs_type == "xref": + dictionary_xref = indirect_xref(charprocs_value) + if dictionary_xref is None: + return "malformed" + try: + dictionary_source = document.xref_object(dictionary_xref, compressed=True) + dictionary_is_stream = document.xref_is_stream(dictionary_xref) + except (RuntimeError, ValueError): + return "uninspectable" + dictionary_source = dictionary_source.strip() + if (dictionary_is_stream or not dictionary_source.startswith("<<") + or not dictionary_source.endswith(">>")): + return "malformed" + parse_status, glyph_xrefs = direct_charproc_xrefs(dictionary_source) + if parse_status != "parsed": + return parse_status + else: + return "malformed" + for glyph_xref in glyph_xrefs: + try: + if not document.xref_is_stream(glyph_xref): + return "malformed" + except (RuntimeError, ValueError): + return "uninspectable" + return "verified" + +def font_inventory(document, page): + """Distinguish font files, verified Type3 glyph streams, and unknown cases.""" + fonts = [] + for entry in page.get_fonts(full=True): + xref, extension, font_type, base_name, resource_name, encoding = entry[:6] + is_type3 = font_type.replace(" ", "").casefold() == "type3" + charprocs_status = type3_charprocs_status(document, xref, font_type) + embedded_bytes = 0 + if xref > 0: + try: + extracted = document.extract_font(xref) + embedded_bytes = len(extracted[3] or b"") + except (RuntimeError, ValueError): + embedded_bytes = 0 + if is_type3: + embedded = ( + True if charprocs_status == "verified" else + (False if charprocs_status == "malformed" else None) + ) + self_contained = embedded + program_source = "type3-charprocs" + else: + embedded = embedded_bytes > 0 + self_contained = None + program_source = "font-file" if embedded_bytes else None + fonts.append({ + "xref": xref, + "base_name": base_name, + "resource_name": resource_name, + "type": font_type, + "encoding": encoding, + "extension": extension, + "embedded": embedded, + "embedded_bytes": embedded_bytes, + "self_contained": self_contained, + "charprocs_status": charprocs_status, + "program_source": program_source, + }) + return fonts + +NON_VIEWABLE_ANNOTATION_FLAGS = ( + fitz.PDF_ANNOT_IS_INVISIBLE | fitz.PDF_ANNOT_IS_HIDDEN | fitz.PDF_ANNOT_IS_NO_VIEW +) + +def annotation_flags(page, item): + flags = getattr(item, "flags", None) + if flags is not None: + return int(flags) + xref = getattr(item, "xref", 0) + if not xref: + return 0 + value_type, value = page.parent.xref_get_key(xref, "F") + try: + return int(value) if value_type == "int" else 0 + except (TypeError, ValueError): + return 0 + +def visible_clip(page, rectangle, *, already_rotated=False): + try: + rectangle = fitz.Rect(rectangle) + except (TypeError, ValueError): + return None + if not all(math.isfinite(value) for value in rectangle): + return None + rectangle.normalize() + if rectangle.is_empty or rectangle.is_infinite: + return None + rotated = rectangle if already_rotated else rectangle * page.rotation_matrix + clip = rotated & page.rect + return None if clip.is_empty else clip + +MAX_IMAGE_PLACEMENTS = 1_000 +MAX_IMAGE_SOURCE_PIXELS = 25_000_000 +MAX_TOTAL_IMAGE_SOURCE_PIXELS = 50_000_000 +MAX_IMAGE_RENDER_PIXELS = 4_000_000 +MAX_TOTAL_IMAGE_RENDER_PIXELS = 20_000_000 +MAX_DRAWING_PATHS = 1_000 +MAX_DRAWING_RENDER_PIXELS = 4_000_000 +MAX_TOTAL_DRAWING_RENDER_PIXELS = 20_000_000 +MAX_TEXT_SPANS = 10_000 +MAX_TEXT_RENDER_PIXELS = 4_000_000 +MAX_TOTAL_TEXT_RENDER_PIXELS = 20_000_000 + +def viewable_text(page): + """Use text render mode, opacity, clipping, and bounded alpha rendering.""" + try: + spans = page.get_texttrace() + except (RuntimeError, ValueError): + return [], [], True + if len(spans) > MAX_TEXT_SPANS: + return spans, [], True + visible = [] + total_render_pixels = 0 + for span in spans: + try: + text = "".join(chr(character[0]) for character in span.get("chars", ())) + render_type = int(span.get("type")) + opacity = float(span.get("opacity")) + except (TypeError, ValueError, OverflowError): + return spans, visible, True + if not text.strip() or render_type > 1 or opacity <= 0: + continue + if render_type not in (0, 1) or not math.isfinite(opacity): + return spans, visible, True + clip = visible_clip(page, span.get("bbox")) + if clip is None: + continue + render_pixels = math.ceil(clip.width) * math.ceil(clip.height) + total_render_pixels += render_pixels + if (render_pixels > MAX_TEXT_RENDER_PIXELS + or total_render_pixels > MAX_TOTAL_TEXT_RENDER_PIXELS): + return spans, visible, True + try: + pixmap = page.get_pixmap(clip=clip, alpha=True, annots=False) + except (RuntimeError, ValueError): + return spans, visible, True + if not pixmap.alpha: + return spans, visible, True + if any(pixmap.samples[pixmap.n - 1::pixmap.n]): + visible.append(span) + return spans, visible, False + +def viewable_images(page): + """Render bounded placement clips; unknown visibility keeps the page nonblank.""" + try: + placements = page.get_image_info() + except (RuntimeError, ValueError): + return [], [], True + if len(placements) > MAX_IMAGE_PLACEMENTS: + return placements, [], True + visible = [] + total_source_pixels = 0 + total_render_pixels = 0 + for placement in placements: + clip = visible_clip(page, placement.get("bbox")) + if clip is None: + continue + width, height = placement.get("width"), placement.get("height") + if (not isinstance(width, int) or isinstance(width, bool) or width <= 0 + or not isinstance(height, int) or isinstance(height, bool) or height <= 0): + return placements, visible, True + source_pixels = width * height + total_source_pixels += source_pixels + if (source_pixels > MAX_IMAGE_SOURCE_PIXELS + or total_source_pixels > MAX_TOTAL_IMAGE_SOURCE_PIXELS): + return placements, visible, True + render_pixels = math.ceil(clip.width) * math.ceil(clip.height) + total_render_pixels += render_pixels + if (render_pixels > MAX_IMAGE_RENDER_PIXELS + or total_render_pixels > MAX_TOTAL_IMAGE_RENDER_PIXELS): + return placements, visible, True + try: + pixmap = page.get_pixmap(clip=clip, alpha=True, annots=False) + except (RuntimeError, ValueError): + return placements, visible, True + if not pixmap.alpha: + return placements, visible, True + if any(pixmap.samples[pixmap.n - 1::pixmap.n]): + visible.append(placement) + return placements, visible, False + +def drawing_bounds(drawing): + """Include stroke width around the path geometry returned by PyMuPDF.""" + try: + rectangle = fitz.Rect(drawing.get("rect")) + except (TypeError, ValueError): + raise ValueError("drawing has no finite bounding rectangle") from None + if not all(math.isfinite(value) for value in rectangle) or rectangle.is_infinite: + raise ValueError("drawing has no finite bounding rectangle") + rectangle.normalize() + path_type = drawing.get("type") + has_stroke = ( + "s" in path_type if isinstance(path_type, str) + else drawing.get("color") is not None + ) + if has_stroke: + width = drawing.get("width", 0) + if isinstance(width, bool): + raise ValueError("drawing has an invalid stroke width") + try: + width = float(width or 0) + except (TypeError, ValueError): + raise ValueError("drawing has an invalid stroke width") from None + if not math.isfinite(width) or width < 0: + raise ValueError("drawing has an invalid stroke width") + # PDF hairlines (width 0) still paint one device pixel at render time. + padding = max(width / 2, 0.5 if width == 0 else 0) + rectangle = fitz.Rect( + rectangle.x0 - padding, rectangle.y0 - padding, + rectangle.x1 + padding, rectangle.y1 + padding, + ) + return rectangle + +def viewable_drawings(page): + """Render bounded path clips; unknown visibility keeps the page nonblank.""" + try: + drawings = page.get_drawings() + except (RuntimeError, ValueError): + return [], [], True + if len(drawings) > MAX_DRAWING_PATHS: + return drawings, [], True + visible = [] + total_render_pixels = 0 + for drawing in drawings: + try: + bounds = drawing_bounds(drawing) + except ValueError: + return drawings, visible, True + clip = visible_clip(page, bounds) + if clip is None: + continue + render_pixels = math.ceil(clip.width) * math.ceil(clip.height) + total_render_pixels += render_pixels + if (render_pixels > MAX_DRAWING_RENDER_PIXELS + or total_render_pixels > MAX_TOTAL_DRAWING_RENDER_PIXELS): + return drawings, visible, True + try: + pixmap = page.get_pixmap(clip=clip, alpha=True, annots=False) + except (RuntimeError, ValueError): + return drawings, visible, True + if not pixmap.alpha: + return drawings, visible, True + if any(pixmap.samples[pixmap.n - 1::pixmap.n]): + visible.append(drawing) + return drawings, visible, False + +def rendered_interactives(page, items): + rendered = [] + visibility_unknown = False + for item in items: + if annotation_flags(page, item) & NON_VIEWABLE_ANNOTATION_FLAGS: + continue + clip = visible_clip(page, item.rect) + if clip is None: + continue + try: + with_annotations = page.get_pixmap( + matrix=fitz.Matrix(2, 2), clip=clip, alpha=False, annots=True, + ) + without_annotations = page.get_pixmap( + matrix=fitz.Matrix(2, 2), clip=clip, alpha=False, annots=False, + ) + except (RuntimeError, ValueError): + visibility_unknown = True + continue + if with_annotations.samples != without_annotations.samples: + rendered.append(item) + return rendered, visibility_unknown + +def page_links(page): + links = [] + link = page.first_link + while link is not None: + links.append(link) + link = link.next + return links + +def link_has_target(link): + destination = getattr(link, "dest", None) + return bool(getattr(link, "uri", None)) or ( + destination is not None and getattr(destination, "page", -1) >= 0 + ) + +def viewable_interactives(page): + widgets, widget_visibility_unknown = rendered_interactives( + page, list(page.widgets() or ()) + ) + annotations, annotation_visibility_unknown = rendered_interactives( + page, list(page.annots() or ()) + ) + # Link hit rectangles are useful without a painted appearance. PyMuPDF reports + # them in rotated page coordinates already, unlike widget / annotation rects. + links = [ + link for link in page_links(page) + if link_has_target(link) + and not annotation_flags(page, link) & NON_VIEWABLE_ANNOTATION_FLAGS + and visible_clip(page, link.rect, already_rotated=True) is not None + ] + return ( + widgets, annotations, links, + widget_visibility_unknown or annotation_visibility_unknown, + ) + +print("pages:", doc.page_count) +print("password_protected:", doc.needs_pass, + "| still_encrypted:", doc.is_encrypted, "| pdf:", doc.is_pdf) +page_geometry = [] +for page in doc: + media_size = (round(page.mediabox.width, 2), round(page.mediabox.height, 2)) + crop_size = (round(page.cropbox.width, 2), round(page.cropbox.height, 2)) + page_geometry.append({ + "page": page.number + 1, + "media_size": media_size, + "crop_size": crop_size, + "rotation": page.rotation, + }) + text_spans, visible_text, text_visibility_unknown = viewable_text(page) + image_placements, visible_images, image_visibility_unknown = viewable_images(page) + drawings, visible_drawings, drawing_visibility_unknown = viewable_drawings(page) + widgets, annotations, links, interaction_visibility_unknown = viewable_interactives(page) + is_blank = not ( + visible_text or visible_images or visible_drawings + or widgets or annotations or links + or text_visibility_unknown or image_visibility_unknown or drawing_visibility_unknown + or interaction_visibility_unknown + ) + print(page.number + 1, "media_size:", media_size, "crop_size:", crop_size, + "rotation:", page.rotation, "text_len:", len(page.get_text()), + "text_spans:", len(text_spans), "visible_text_spans:", len(visible_text), + "text_visibility_unknown:", text_visibility_unknown, + "resource_images:", len(page.get_images()), + "image_placements:", len(image_placements), + "visible_images:", len(visible_images), + "image_visibility_unknown:", image_visibility_unknown, + "drawing_paths:", len(drawings), + "visible_drawings:", len(visible_drawings), + "drawing_visibility_unknown:", drawing_visibility_unknown, + "widgets:", len(widgets), "annotations:", len(annotations), + "links:", len(links), "interaction_visibility_unknown:", + interaction_visibility_unknown, "blank:", is_blank) + print(" fonts:", font_inventory(doc, page)) +print("media_size_consistent:", len({row["media_size"] for row in page_geometry}) <= 1) +print("crop_size_consistent:", len({row["crop_size"] for row in page_geometry}) <= 1) +``` + +## Checks worth automating + +- **Blank page detection**: flag only when rendered text, visible painted image placements and vector paths, + and viewable widgets, annotations, and links are all absent. Ignore interactive objects carrying + invisible, hidden, or no-view flags, as well as empty, off-page, or unrendered appearances. + `page.get_images()` lists every image XObject resource, including unused resources, and also + misses images embedded inline in the content stream. `page.get_image_info()` reports invoked + inline and XObject placements without loading their bytes, but its boxes ignore graphics-state + clipping. Treat those boxes as diagnostics only. The bounded `viewable_images()` render checks + alpha for each page-intersecting placement; a count, source-pixel, render-pixel, or decoder limit + returns `image_visibility_unknown=True`, which must keep the page nonblank. Interactive form + `page.get_drawings()` similarly reports paths that are off-page, fully clipped, hidden by page + state, or painted with zero opacity. The bounded `viewable_drawings()` checks each + page-intersecting path clip against the actual alpha render; path-count, render-pixel, or renderer + limits return `drawing_visibility_unknown=True`, which also keeps the page nonblank. Interactive + form fields are widgets rather + than page text, so a three-content-stream predicate alone would misclassify a usable form + page as blank. A blank page after generation usually means an overflowing flowable created it. +- **Font inventory**: `page.get_fonts()` lists referenced fonts, including non-embedded base + fonts. Use `doc.extract_font(xref)` as above for conventional font files. Type3 fonts are a + separate case: report `charprocs_status="verified"` only for a nonempty `/CharProcs` dictionary + whose glyph entries point to actual streams. A direct font dictionary has no font xref and is + `uninspectable`, not falsely non-embedded. For Type3, `embedded` and `self_contained` are tri-state: + `True` for verified, `False` for malformed, and `None` when uninspectable. Its + `program_source="type3-charprocs"` remains explicit even when conventional bytes do not extract. + Other referenced faces with no extractable program may be substituted on another machine. +- **Page size consistency**: compare unrotated `(width, height)` pairs from `page.mediabox` and + `page.cropbox`, and report `page.rotation` separately. Do not compare `page.rect`: it applies + `/Rotate`, so otherwise identical paper appears to swap width and height at 90 or 270 degrees. + Report genuinely mixed media or crop sizes rather than silently normalizing them. +- **Damage**: `fitz.open` on a corrupt file raises or yields garbage - pair with + `pypdf.PdfReader` cross-check when provenance is unknown. + +Report findings as a table (page, media size, crop size, rotation, text chars, resource images, +image placements, visible images, image visibility unknown, drawing paths, visible drawings, +drawing visibility unknown, widgets, annotations, links, interaction visibility unknown, blank) - it +is what every downstream decision hangs off. diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md b/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md new file mode 100644 index 0000000..ef1d7bf --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md @@ -0,0 +1,218 @@ +# Transform a PDF (pypdf) + +One tool for page-level structure changes: + +```python +import os +from pypdf import PdfReader, PdfWriter + +def open_pdf(path): + reader = PdfReader(path) + if reader.is_encrypted: + password = os.environ.get("PDF_PASSWORD", "") + if reader.decrypt(password) == 0: + raise RuntimeError(f"Encrypted PDF {path}: set a valid PDF_PASSWORD") + return reader + +reader = open_pdf("input.pdf") +writer = PdfWriter() + +# Split: keep only pages 2-4 (0-based), remapping any retained outline entries. +writer.append(reader, pages=(1, 4), import_outline=True) + +# Merge another file at the end. append() imports bookmarks/named destinations; +# copying other.pages one by one would silently discard that navigation structure. +other = open_pdf("appendix.pdf") +writer.append(other, import_outline=True) + +# Rotate a page 90 degrees +writer.pages[0].rotate(90) + +# Metadata +writer.add_metadata({"/Title": "Merged report", "/Producer": "document-skills"}) + +with open("output.pdf", "wb") as f: + writer.write(f) +``` + +Watermark / stamp by merging a stamp page onto each page: + +```python +from pypdf import Transformation +from pypdf.generic import RectangleObject + +def rotation_transfer(page): + """Map default user space to the visible coordinates used after /Rotate.""" + media = RectangleObject(page.mediabox) + transform = ( + Transformation() + .translate( + -float(media.left + media.width / 2), + -float(media.bottom + media.height / 2), + ) + .rotate(-page.rotation) + ) + corners = [ + transform.apply_on(point) + for point in (media.lower_left, media.lower_right, media.upper_left, media.upper_right) + ] + return transform.translate( + -min(point[0] for point in corners), + -min(point[1] for point in corners), + ) + +def inverse_transformation(transform): + a, b, c, d, e, f = map(float, transform.ctm) + determinant = a * d - b * c + return Transformation(( + d / determinant, -b / determinant, + -c / determinant, a / determinant, + (c * f - d * e) / determinant, + (b * e - a * f) / determinant, + )) + +def transformed_rectangle(rectangle, transform): + rectangle = RectangleObject(rectangle) + corners = [ + transform.apply_on(point) + for point in ( + rectangle.lower_left, rectangle.lower_right, + rectangle.upper_left, rectangle.upper_right, + ) + ] + return RectangleObject(( + min(point[0] for point in corners), min(point[1] for point in corners), + max(point[0] for point in corners), max(point[1] for point in corners), + )) + +def stamp_placement(page, stamp_box): + """Fit in visible space, then map the stamp back without changing the destination page.""" + to_visual = rotation_transfer(page) + destination = transformed_rectangle(page.cropbox, to_visual) + sw, sh = float(stamp_box.width), float(stamp_box.height) + dw, dh = float(destination.width), float(destination.height) + scale = min(dw / sw, dh / sh) + tx = float(destination.left) + (dw - sw * scale) / 2 - float(stamp_box.left) * scale + ty = float(destination.bottom) + (dh - sh * scale) / 2 - float(stamp_box.bottom) * scale + visible_placement = Transformation().scale(scale).translate(tx, ty) + return visible_placement.transform(inverse_transformation(to_visual)) + +def require(condition, message): + if not condition: + raise ValueError(message) + +stamp = open_pdf("watermark.pdf").pages[0] +stamp.transfer_rotation_to_content() +stamp_text = (stamp.extract_text() or "").strip() +reader = open_pdf("input.pdf") +expected_fields = reader.get_fields() or {} +writer = PdfWriter() +writer.append(reader) # clone pages plus catalog entries such as /AcroForm + +# A plain merge_page() overlays the stamp in its own coordinates, so on a page +# with different dimensions, origin, or rotation the stamp can be clipped or +# land entirely off-page. Normalize only the stamp, fit each copy in the visible +# (rotation-aware) crop box, then map that placement back into default user space. +# This keeps the destination /Rotate, boxes, annotation /Rect values, and widget +# appearance streams intact while keeping the watermark horizontal to the viewer. +stamp_box = stamp.cropbox +for page in writer.pages: + page.merge_transformed_page(stamp, stamp_placement(page, stamp_box)) + +expected_sizes = [tuple(float(value) for value in page.mediabox) for page in writer.pages] + +with open("watermarked.pdf", "wb") as f: + writer.write(f) + +check = PdfReader("watermarked.pdf") +require(len(check.pages) == len(expected_sizes), "watermarking changed the page count") +require( + [tuple(float(value) for value in page.mediabox) for page in check.pages] == expected_sizes, + "watermarking changed page geometry", +) +if expected_fields: + require(set(expected_fields) <= set(check.get_fields() or {}), + "watermarking dropped form fields") +if stamp_text: + require(all(stamp_text in (page.extract_text() or "") for page in check.pages), + "watermark text is missing from one or more pages") +``` + +If the stamp is graphical, render every output page and visually confirm that it is present; +text extraction cannot validate a graphical watermark. + +Encryption and forms: + +- AES encryption requires pypdf's optional crypto backend. Install `pypdf[crypto]` with + `python -m pip install "pypdf[crypto]"`. Run + `python -c "import cryptography; print('AES backend ok')"` before calling + `writer.encrypt("pass", algorithm="AES-256")`. + Open the result with `PdfReader(..., password="pass")`. Report that you set a password - the + user must record it. +- AcroForm: fill fields on a cloned copy of the document, never on the readers' own pages: + + ```python + from pypdf import PdfReader, PdfWriter + + reader = open_pdf("form.pdf") + fields = reader.get_fields() or {} + if not fields: + raise ValueError("this PDF has no AcroForm form fields") + + writer = PdfWriter() + writer.append(reader) # clones every page AND the catalog /AcroForm into the writer + + def widget_field_name(widget): + """Resolve the fully qualified /T from a widget and its field ancestors.""" + parts = [] + seen = set() + while widget is not None: + object_id = id(widget) + if object_id in seen: + raise ValueError("cycle in AcroForm field parent chain") + seen.add(object_id) + partial_name = widget.get("/T") + if partial_name is not None: + parts.append(str(partial_name)) + parent = widget.get("/Parent") + widget = None if parent is None else parent.get_object() + return ".".join(reversed(parts)) if parts else None + + field_name = "applicant_name" # use the exact simple/qualified key exposed by get_fields() + target_pages = [ + page for page in writer.pages + if any( + (widget := ref.get_object()).get("/Subtype") == "/Widget" + and widget_field_name(widget) == field_name + for ref in (page.get("/Annots") or []) + ) + ] + if not target_pages: + raise KeyError(f"no widget page found for field {field_name!r}") + for page in target_pages: + writer.update_page_form_field_values(page, {field_name: "Ada Byron"}) + + with open("filled.pdf", "wb") as f: + writer.write(f) + + check = PdfReader("filled.pdf") + value = str((check.get_fields() or {}).get(field_name, {}).get("/V", "")) + if value.strip("/") != "Ada Byron": + raise ValueError(f"filled field did not round-trip: {value!r}") + ``` + + A freshly constructed `PdfWriter` is empty: `append` (or `clone_document_from_reader`) must + copy the pages and the `/AcroForm` dictionary before any `update_page_form_field_values` + call, or the write fails or silently produces a formless file. Flatten only on explicit + request; it stops later editing. + +## Rules + +- Always write a new file; transformation in place risks losing the original on a bad write. +- Open every source through `open_pdf` above before touching `.pages`; this authenticates + encrypted inputs and fails with a clear password error instead of failing mid-transform. +- Merge complete inputs with `writer.append(..., import_outline=True)` so their outlines and + named destinations are imported and remapped; page-by-page `add_page` is for intentional splits. +- After writing, re-open with `PdfReader("output.pdf")` and verify page count and page sizes. +- `rotate` is cumulative on already-rotated pages - read `/Rotate` first if the source was + scanned sideways. diff --git a/plugins/Hylouis233/document-skills/skills/pptx/SKILL.md b/plugins/Hylouis233/document-skills/skills/pptx/SKILL.md new file mode 100644 index 0000000..d20cb09 --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/pptx/SKILL.md @@ -0,0 +1,100 @@ +--- +name: pptx +description: Create, read, edit, or analyze PowerPoint .pptx presentations. Use this Skill whenever the task involves slides or a deck - building a presentation from an outline or content file; editing text, tables, or charts in an existing deck; extracting slide content for review or repurposing; checking that a produced deck opens, has the right slide count, and nothing overflows its slide. +--- + +# PPTX workbench + +A `.pptx` is a ZIP of XML parts (one per slide) plus masters, layouts, and media. python-pptx +models most of it safely; everything else is surgical. + +## Step 0 - Check the toolchain + +```bash +python -c "import pptx; print('python-pptx ok')" +``` + +`soffice` present -> the rendered-PDF smoke test becomes available; use it for anything visual. + +## Step 1 - Classify the task + +| Request | Route | +|---|---| +| Build a deck from an outline / notes / markdown | [references/create.md](references/create.md) | +| Change text, tables, or charts in an existing deck | [references/edit.md](references/edit.md) | +| Extract or summarize deck content | [references/analyze.md](references/analyze.md) | +| Deck renders wrong / opens broken | [references/analyze.md](references/analyze.md) triage section | + +## Step 2 - Shared rules + +1. **Templates first**: if the user provides a `.pptx` template or brand deck, build on it + (`Presentation("template.pptx")`), reuse its masters/layouts, and never restyle globally. +2. **Layouts carry design**: pick the closest built-in layout for each slide's purpose instead + of hand-placing empty text boxes. Hand-placement is for exceptions, measured. +3. **Text must fit its box**: estimate width (chars x ~0.5 x font size for sans-serif at a + first approximation) and step font size down (never below 12pt body) or cut words. Overflow + text is a defect the postcheck must catch. +4. **Placeholders keep semantics**: write into placeholder shapes (`.placeholders`) when + available so title/body roles survive round-trips. +5. **Images**: set both width and height from the real aspect ratio; never stretch. +6. Charts: prefer a real chart part (`chart_data` + `add_chart`) over a picture of a chart - + only a real chart stays editable and data-accurate. +7. Output to a new path (`-edited` suffix) unless in-place was explicitly requested. + +## Step 3 - Postcheck (mandatory) + +```python +from pptx import Presentation +from pptx.enum.shapes import MSO_SHAPE_TYPE + +def iter_text_frames(shapes, path=""): + """Yield every logical text frame, including group members and table cells.""" + for shape in shapes: + here = f"{path}/{shape.name}" if path else shape.name + if shape.shape_type == MSO_SHAPE_TYPE.GROUP: + yield from iter_text_frames(shape.shapes, here) + continue + if shape.has_text_frame: + yield here, shape.text_frame + if shape.has_table: + for row_index, row in enumerate(shape.table.rows): + for column_index, cell in enumerate(row.cells): + if cell.is_spanned: # a covered merge slot is intentionally empty + continue + yield f"{here}/table[{row_index},{column_index}]", cell.text_frame + +prs = Presentation("output.pptx") +print("slides:", len(prs.slides)) +for i, slide in enumerate(prs.slides): + inventory = [(location, frame.text) for location, frame in iter_text_frames(slide.shapes)] + print(i, len(slide.shapes), inventory) + +# Populate this only with locations intentionally left blank by the task. +intentionally_empty_locations = set() +unexpected_empty_locations = [ + (slide_index, location) + for slide_index, slide in enumerate(prs.slides) + for location, frame in iter_text_frames(slide.shapes) + if not frame.text.strip() + and (slide_index, location) not in intentionally_empty_locations +] +if unexpected_empty_locations: + raise ValueError(f"unexpected empty text frames: {unexpected_empty_locations}") +``` + +Confirm: slide count matches the outline; every slide has its intended title text; no text +frame is empty that should not be. If `soffice` exists, render to PDF and check the page count +equals the slide count: + +```bash +soffice --headless --convert-to pdf output.pptx --outdir +``` + +Page count is only a structural smoke test. Rasterize and inspect **every rendered slide** for +horizontal clipping and for a final line clipped or missing at the bottom; follow the text-fit +procedure in [references/analyze.md](references/analyze.md). Fix and render again if any text +overflows. If no production-equivalent renderer is available, report overflow as unverified - +do not claim that shape bounds or page count prove that text fits. + +Report: output path, slide inventory (index, layout name, title), verification done, and any +remaining layout risks (long CJK strings, tight two-column slides). diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md new file mode 100644 index 0000000..71171dc --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md @@ -0,0 +1,858 @@ +# Analyze / triage a deck + +## Content inventory + +```python +from pptx import Presentation +from pptx.enum.shapes import MSO_SHAPE_TYPE +from pptx.oxml.ns import qn +from lxml import etree +from pathlib import Path +import zipfile + +DIAGRAM_NS = "http://schemas.openxmlformats.org/drawingml/2006/diagram" +CONTENT_TYPES_NS = "http://schemas.openxmlformats.org/package/2006/content-types" +SAFE_XML = etree.XMLParser( + load_dtd=False, resolve_entities=False, no_network=True, + huge_tree=False, recover=False, +) +MAX_ARCHIVE_BYTES = 200 * 1024 * 1024 +MAX_MEMBERS = 10_000 +MAX_XML_PART = 20 * 1024 * 1024 +MAX_ENTRY = 100 * 1024 * 1024 +MAX_TOTAL_UNCOMPRESSED = 500 * 1024 * 1024 +MAX_COMPRESSION_RATIO = 200 + +def require(condition, message): + """Security gates must remain active under python -O.""" + if not condition: + raise ValueError(message) + +def validate_pptx_package(source): + """Validate the same seekable handle that python-pptx will parse.""" + source.seek(0, 2) + compressed_size = source.tell() + source.seek(0) + require(compressed_size <= MAX_ARCHIVE_BYTES, "compressed PPTX file size above limit") + with zipfile.ZipFile(source) as archive: + infos = archive.infolist() + require(len(infos) <= MAX_MEMBERS, "archive member count above limit") + names = {info.filename for info in infos} + require(len(names) == len(infos), "duplicate archive member names are unsafe") + require("[Content_Types].xml" in names and "ppt/presentation.xml" in names, + "missing required OPC members") + require(sum(info.file_size for info in infos) <= MAX_TOTAL_UNCOMPRESSED, + "declared total uncompressed size above limit") + + content_types_info = next( + info for info in infos if info.filename == "[Content_Types].xml" + ) + require(content_types_info.file_size <= MAX_XML_PART, + "oversized XML part: [Content_Types].xml") + require( + content_types_info.file_size / max(content_types_info.compress_size, 1) + <= MAX_COMPRESSION_RATIO, + "suspicious compression ratio: [Content_Types].xml", + ) + content_type_chunks = [] + content_type_size = 0 + with archive.open(content_types_info) as stream: + while chunk := stream.read(64 * 1024): + content_type_size += len(chunk) + require(content_type_size <= MAX_XML_PART, + "part exceeded read limit: [Content_Types].xml") + content_type_chunks.append(chunk) + require(content_type_size == content_types_info.file_size, + "size mismatch: [Content_Types].xml") + content_types_blob = b"".join(content_type_chunks) + content_types_root = etree.fromstring(content_types_blob, parser=SAFE_XML) + require(content_types_root.tag == f"{{{CONTENT_TYPES_NS}}}Types", + "invalid content-types root") + default_types = {} + override_types = {} + for declaration in content_types_root: + if declaration.tag == f"{{{CONTENT_TYPES_NS}}}Default": + key = (declaration.get("Extension") or "").casefold() + target = default_types + elif declaration.tag == f"{{{CONTENT_TYPES_NS}}}Override": + part_name = declaration.get("PartName") or "" + require(part_name.startswith("/"), "invalid content-type part name") + key = part_name[1:].casefold() + target = override_types + else: + continue + content_type = declaration.get("ContentType") or "" + require(key and content_type and key not in target, + "invalid or duplicate content-type declaration") + target[key] = content_type.partition(";")[0].strip().casefold() + + actual_total = 0 + for info in infos: + require(info.file_size <= MAX_ENTRY, f"oversized part: {info.filename}") + ratio = info.file_size / max(info.compress_size, 1) + require(ratio <= MAX_COMPRESSION_RATIO, + f"suspicious compression ratio: {info.filename}") + extension = info.filename.rpartition(".")[2].casefold() + content_type = override_types.get( + info.filename.casefold(), default_types.get(extension, "") + ) + is_xml = ( + info.filename.casefold().endswith((".xml", ".rels")) + or content_type in {"application/xml", "text/xml"} + or content_type.endswith("+xml") + ) + if is_xml: + require(info.file_size <= MAX_XML_PART, f"oversized XML part: {info.filename}") + + chunks = [] + actual_size = 0 + if info.filename == "[Content_Types].xml": + chunks = [content_types_blob] + actual_size = len(content_types_blob) + actual_total += actual_size + else: + with archive.open(info) as stream: + while chunk := stream.read(64 * 1024): + actual_size += len(chunk) + actual_total += len(chunk) + require(actual_size <= MAX_ENTRY, + f"part exceeded read limit: {info.filename}") + require(actual_total <= MAX_TOTAL_UNCOMPRESSED, + "archive exceeded total read limit") + if is_xml: + chunks.append(chunk) + require(actual_total <= MAX_TOTAL_UNCOMPRESSED, + "archive exceeded total read limit") + require(actual_size == info.file_size, f"size mismatch: {info.filename}") + if is_xml: + etree.fromstring(b"".join(chunks), parser=SAFE_XML) + source.seek(0) + +def open_validated_presentation(path): + """Preflight and parse one immutable open-file identity, then release the handle.""" + with Path(path).open("rb") as source: + validate_pptx_package(source) + source.seek(0) + return Presentation(source) + +OOXML_TRUE = {"1", "true"} +OOXML_FALSE = {"0", "false"} + +def ooxml_bool(element, attribute, default): + value = element.get(attribute) + if value is None: + return default + value = value.strip(" \t\r\n") + if value in OOXML_TRUE: + return True + if value in OOXML_FALSE: + return False + raise ValueError(f"invalid OOXML boolean {attribute}={value!r}") + +def shape_is_hidden(shape): + properties = shape._element.find(".//" + qn("p:cNvPr")) + return properties is not None and ooxml_bool(properties, "hidden", False) + +def iter_shapes(shapes): + """Yield visible leaf shapes, propagating a hidden group's visibility to its children.""" + for shape in shapes: + if shape_is_hidden(shape): + continue + if shape.shape_type == MSO_SHAPE_TYPE.GROUP: + yield from iter_shapes(shape.shapes) + else: + yield shape + +def layer_text_content(shapes, source, *, inherited): + """Collect visible text from one layer; inherited placeholders are template prompts.""" + records = [] + for shape in shapes: + if shape_is_hidden(shape): + continue + if shape.shape_type == MSO_SHAPE_TYPE.GROUP: + records.extend(layer_text_content(shape.shapes, source, inherited=inherited)) + continue + if inherited and shape.is_placeholder: + continue + if shape.has_text_frame and shape.text_frame.text: + records.append({ + "source": source, + "shape": shape.name, + "text": shape.text_frame.text, + }) + return records + +def slide_text_content(slide): + """Inventory rendered text layers in master -> layout -> slide order.""" + records = [] + if ooxml_bool(slide._element, "showMasterSp", True): + layout = slide.slide_layout + if ooxml_bool(layout._element, "showMasterSp", True): + records.extend(layer_text_content( + layout.slide_master.shapes, "master", inherited=True, + )) + records.extend(layer_text_content(layout.shapes, "layout", inherited=True)) + records.extend(layer_text_content(slide.shapes, "slide", inherited=False)) + return records + +def table_cells(table): + """Inventory the grid without repeating a merged cell's text in covered slots.""" + return [[ + { + "row": row_index, + "column": column_index, + "text": None if cell.is_spanned else cell.text, + "is_merge_origin": cell.is_merge_origin, + "is_spanned": cell.is_spanned, + "span_width": cell.span_width, + "span_height": cell.span_height, + } + for column_index, cell in enumerate(row.cells) + ] for row_index, row in enumerate(table.rows)] + +def picture_content(shape): + """Inventory ordinary pictures and populated picture placeholders alike.""" + if getattr(type(shape), "image", None) is None: + return None + try: + image = shape.image + return { + "name": shape.name, + "filename": image.filename, + "extension": image.ext, + "bytes": len(image.blob), + } + except (AttributeError, KeyError, OSError, ValueError): + blips = shape._element.xpath(".//a:blip") + relationship_id = ( + blips[0].get(qn("r:embed")) if len(blips) == 1 else None + ) + return { + "name": shape.name, + "status": "unreadable", + "relationship_id": relationship_id, + "reason": "missing or invalid image relationship or payload", + } + +MAX_CHART_POINTS = 100_000 +MAX_CATEGORY_LEVELS = 16 +CATEGORY_SOURCE_NAMES = {"strRef", "numRef", "multiLvlStrRef", "strLit", "numLit"} + +def cache_point_count(cache): + point_counts = cache.findall(qn("c:ptCount")) + if len(point_counts) != 1: + return None + try: + point_count = int(point_counts[0].get("val")) + except (TypeError, ValueError): + return None + return point_count if 0 <= point_count <= MAX_CHART_POINTS else None + +def consume_point_budget(point_budget, count): + if count > MAX_CHART_POINTS: + return False + if point_budget is None: + return True + if count > point_budget["remaining"]: + return False + point_budget["remaining"] -= count + return True + +def cached_numeric_points( + source, *, fill_missing=False, include_count=False, point_budget=None +): + """Return bounded indexed cached points, or None when metadata is unavailable.""" + if source is None: + return None + caches = source.xpath("./c:numRef/c:numCache | ./c:numLit") + if len(caches) != 1 or (point_count := cache_point_count(caches[0])) is None: + return None + if not consume_point_budget(point_budget, point_count): + return None + cached = {} + for point in caches[0].findall(qn("c:pt")): + value = point.find(qn("c:v")) + try: + index = int(point.get("idx")) + except (TypeError, ValueError): + return None + if value is None or not 0 <= index < point_count or index in cached: + return None + try: + parsed = float(value.text) if value.text not in (None, "") else value.text + except (TypeError, ValueError): + parsed = value.text # preserve #N/A and other error markers + cached[index] = parsed + if fill_missing: + points = [(index, cached.get(index)) for index in range(point_count)] + else: + points = sorted(cached.items()) + return {"point_count": point_count, "points": points} if include_count else points + +def cached_text_point_map(container, point_count): + values = {} + for point in container.findall(qn("c:pt")): + try: + index = int(point.get("idx")) + except (TypeError, ValueError): + return None + value_nodes = point.findall(qn("c:v")) + if (len(value_nodes) != 1 or not 0 <= index < point_count + or index in values): + return None + values[index] = value_nodes[0].text or "" + return values + +def category_source(series): + category = series._element.find(qn("c:cat")) + if category is None: + return None, None + sources = [ + child for child in category + if isinstance(child.tag, str) + and etree.QName(child).localname in CATEGORY_SOURCE_NAMES + ] + return (category, sources[0]) if len(sources) == 1 else (category, None) + +def cached_category_labels(series, point_budget=None): + """Return this series' flattened labels, or None when its cache is unavailable.""" + category, source = category_source(series) + if category is None: + return [] + if source is None: + return None + source_name = etree.QName(source).localname + + if source_name == "multiLvlStrRef": + caches = source.findall(qn("c:multiLvlStrCache")) + if len(caches) != 1 or (count := cache_point_count(caches[0])) is None: + return None + level_nodes = caches[0].findall(qn("c:lvl")) + if len(level_nodes) > MAX_CATEGORY_LEVELS or (count and not level_nodes): + return None + levels = [cached_text_point_map(level, count) for level in level_nodes] + if any(points is None for points in levels): + return None + if not consume_point_budget(point_budget, count * max(len(level_nodes), 1)): + return None + expanded_levels = [] + for level_index, points in enumerate(levels): + if level_index == 0: + expanded_levels.append([points.get(index, "") for index in range(count)]) + continue + ordered = sorted(points.items()) + if not ordered: + expanded_levels.append([""] * count) + continue + cursor = 0 + expanded = [] + for leaf_index in range(count): + while cursor + 1 < len(ordered) and ordered[cursor + 1][0] <= leaf_index: + cursor += 1 + expanded.append(ordered[cursor][1]) + expanded_levels.append(expanded) + return [ + [level[index] for level in reversed(expanded_levels)] + for index in range(count) + ] + + cache_name = {"strRef": "c:strCache", "numRef": "c:numCache"}.get(source_name) + if cache_name is None: + cache = source + else: + caches = source.findall(qn(cache_name)) + if len(caches) != 1: + return None + cache = caches[0] + if (count := cache_point_count(cache)) is None: + return None + values = cached_text_point_map(cache, count) + if values is None: + return None + if not consume_point_budget(point_budget, count): + return None + return [[values.get(index, "")] for index in range(count)] + +def category_content(series, point_budget=None): + labels = cached_category_labels(series, point_budget) + _, source = category_source(series) + formula = None if source is None else source.find(qn("c:f")) + return { + "categories": labels, + "category_formula": formula.text if formula is not None else None, + "category_cache_status": "unavailable" if labels is None else "available", + } + +def series_name_content(series): + """Return a cached/literal title, or mark a worksheet-backed title unavailable.""" + titles = series._element.xpath("./c:tx") + if not titles: + return {"name": ""} + literal = titles[0].find(qn("c:v")) + if literal is not None: + return {"name": literal.text or ""} + reference = titles[0].find(qn("c:strRef")) + if reference is None: + return {"name": None, "name_cache_status": "unavailable"} + cache = reference.find(qn("c:strCache")) + if cache is None: + return {"name": None, "name_cache_status": "unavailable"} + point_count = cache.find(qn("c:ptCount")) + points = cache.findall(qn("c:pt")) + if (point_count is None or point_count.get("val") != "1" + or len(points) != 1 or points[0].get("idx") != "0"): + return {"name": None, "name_cache_status": "unavailable"} + value = points[0].find(qn("c:v")) + if value is None: + return {"name": None, "name_cache_status": "unavailable"} + return {"name": value.text or ""} + +def series_content(series, *, include_categories=False, point_budget=None): + if point_budget is None: + point_budget = {"remaining": MAX_CHART_POINTS} + name_content = series_name_content(series) + content = { + **name_content, + **(category_content(series, point_budget) if include_categories else {}), + } + x_source = getattr(series._element, "xVal", None) + if x_source is None: # category/value chart + value_source = getattr(series._element, "val", None) + value_cache = cached_numeric_points( + value_source, fill_missing=True, include_count=True, + point_budget=point_budget, + ) + if value_cache is None: + return {**content, "values": None, "cache_status": "unavailable"} + return {**content, "values": [value for _, value in value_cache["points"]]} + x_cache = cached_numeric_points( + x_source, include_count=True, point_budget=point_budget + ) + y_cache = cached_numeric_points( + getattr(series._element, "yVal", None), include_count=True, + point_budget=point_budget, + ) + if x_cache is None or y_cache is None: + return {**content, "points": None, "cache_status": "unavailable"} + content.update({ + "x_points": x_cache["points"], "x_point_count": x_cache["point_count"], + "y_points": y_cache["points"], "y_point_count": y_cache["point_count"], + }) + size_source = getattr(series._element, "bubbleSize", None) + if size_source is not None: + bubble_cache = cached_numeric_points( + size_source, include_count=True, point_budget=point_budget + ) + if bubble_cache is None: + return {**content, "points": None, "cache_status": "unavailable"} + content.update({ + "bubble_points": bubble_cache["points"], + "bubble_point_count": bubble_cache["point_count"], + }) + return content + +def title_text(title): + """Read a chart/axis title without invoking python-pptx's mutating text accessor.""" + if title is None: + return "" + text_nodes = title.findall(qn("c:tx")) + if len(text_nodes) != 1: + return None + sources = [child for child in text_nodes[0] if isinstance(child.tag, str)] + if len(sources) != 1: + return None + if sources[0].tag == qn("c:rich"): + paragraphs = sources[0].findall(qn("a:p")) + return None if not paragraphs else "\n".join( + paragraph.text for paragraph in paragraphs + ) + if sources[0].tag != qn("c:strRef"): + return None + formulas = sources[0].findall(qn("c:f")) + if len(formulas) != 1 or not formulas[0].text: + return None + caches = sources[0].findall(qn("c:strCache")) + if (len(caches) != 1 or (count := cache_point_count(caches[0])) is None + or count != 1): + return None + values = cached_text_point_map(caches[0], count) + return None if values is None or set(values) != {0} else values[0] + +def chart_axis_text(axis): + return title_text(axis.find(qn("c:title"))) + +def chart_title_text(chart): + titles = chart._element.xpath("./c:chart/c:title") + return title_text(titles[0]) if len(titles) == 1 else ("" if not titles else None) + +def chart_axes(chart): + """Return every category, date, value, and series axis in document order.""" + axes = chart._element.xpath( + "./c:chart/c:plotArea/c:catAx | ./c:chart/c:plotArea/c:dateAx | " + "./c:chart/c:plotArea/c:valAx | ./c:chart/c:plotArea/c:serAx" + ) + return [ + { + "kind": etree.QName(axis).localname, + "id": axis.find(qn("c:axId")).get("val") if axis.find(qn("c:axId")) is not None else None, + "position": ( + axis.find(qn("c:axPos")).get("val") + if axis.find(qn("c:axPos")) is not None else None + ), + "cross_axis_id": ( + axis.find(qn("c:crossAx")).get("val") + if axis.find(qn("c:crossAx")) is not None else None + ), + "title": chart_axis_text(axis), + } + for axis in axes + ] + +def smartart_content(shape): + """Extract SmartArt data-part labels, or report why the diagram is unreadable.""" + graphic_data = shape._element.find(".//" + qn("a:graphicData")) + if graphic_data is None or graphic_data.get("uri") != DIAGRAM_NS: + return None + rel_ids = graphic_data.find(f".//{{{DIAGRAM_NS}}}relIds") + relationship_id = None if rel_ids is None else rel_ids.get(qn("r:dm")) + if not relationship_id: + return {"name": shape.name, "status": "unreadable", "reason": "missing data relationship"} + try: + data_part = shape.part.related_part(relationship_id) + except (KeyError, ValueError): + return {"name": shape.name, "status": "unreadable", "reason": relationship_id} + try: + root = etree.fromstring(data_part.blob, parser=SAFE_XML) + except etree.XMLSyntaxError as error: + return {"name": shape.name, "status": "unreadable", "reason": str(error)} + labels = [node.text for node in root.iter(qn("a:t")) if node.text] + return {"name": shape.name, "status": "ok", "text": labels} + +prs = open_validated_presentation("input.pptx") +print("slide size:", prs.slide_width, prs.slide_height) +point_budget = {"remaining": MAX_CHART_POINTS} +for i, slide in enumerate(prs.slides): + layout = slide.slide_layout.name + title_shape = slide.shapes.title + title = ( + title_shape.text_frame.text + if title_shape is not None and not shape_is_hidden(title_shape) else "" + ) + notes = slide.notes_slide.notes_text_frame.text if slide.has_notes_slide else "" + shapes = list(iter_shapes(slide.shapes)) # flattened and visible; hidden groups hide children + text = slide_text_content(slide) + tables = [ + table_cells(sh.table) + for sh in shapes if sh.has_table + ] + charts = [] + for sh in shapes: + if not sh.has_chart: + continue + chart = sh.chart + chart_title = chart_title_text(chart) + plots = [] + for plot in chart.plots: + items = list(plot.series) + has_xy_values = any(getattr(item._element, "xVal", None) is not None for item in items) + series = [ + series_content( + item, include_categories=not has_xy_values, point_budget=point_budget + ) + for item in items + ] + plots.append({ + "kind": type(plot).__name__, + "series": series, + }) + charts.append({ + "title": chart_title, + "axes": chart_axes(chart), + "plots": plots, + }) + pictures = [ + content for sh in shapes + if (content := picture_content(sh)) is not None + ] + smartart = [ + content for sh in shapes + if (content := smartart_content(sh)) is not None + ] + + # These full values - not only counts or lengths - are the evidence for summaries and + # repurposing. Keep notes verbatim so markdown output can preserve them as blockquotes. + print(f"slide {i + 1}: layout={layout!r} title={title!r}") + print(" text:", text) + print(" tables:", tables) + print(" charts:", charts) + print(" pictures:", pictures) + print(" smartart:", smartart) + print(" notes:", notes) +``` + +Inherited layout/master placeholders are deliberately excluded: their stored text is a template +prompt, not rendered slide copy, and slide placeholders already contribute their instantiated +text. Footer/date/slide-number fields and occlusion still require a rendered-slide check when +pixel-level visibility matters. + +## Triage: deck renders wrong + +| Symptom | Check | Fix | +|---|---|---| +| Shape crosses the slide edge | compare all four shape bounds with the slide bounds | move or resize the shape | +| Text is clipped or overflows its box | render every slide and inspect right/left and bottom/vertical fit; shape bounds do not measure laid-out text | reflow, resize the box, or reduce text/font size, then render again | +| Everything shifted | slide size changed between sources | normalize slide size or re-layout on the target size | +| Fonts look wrong elsewhere | non-embedded fonts (pptx rarely embeds) | report every effective font via the resolution chain below, not just explicit `run.font.name` values | +| File will not open | broken ZIP / part mismatch | run the bounded package check below before parsing parts | +| Pictures blank | media parts missing or rels broken | verify `ppt/media/*` present and slide rels reference them | + +## Bounded package health check + +The content-inventory block performs this check before its first `Presentation()` call and parses +the same open handle it validated. Do not replace that order with `Presentation(path)` followed by +a later check: package parsing already expands ZIP members. Likewise, `ZipFile.testzip()` must not +be the first check because it expands every member, including an archive bomb. + +These are conservative triage defaults, not PPTX format limits. Raise a limit only for an +explicitly trusted large deck, and retain the per-member and streaming checks. + +## Font triage with inheritance + +Most template decks set no explicit `run.font.name`; the effective face is inherited from the +placeholder, layout, master, or theme. This block is a continuation of the content-inventory +session above: reuse its already validated `prs` object rather than reopening the path. Resolve +what you can and name the fallback explicitly: + +```python +import xml.etree.ElementTree as ET +from pptx.opc.constants import RELATIONSHIP_TYPE as RT +from pptx.oxml.ns import qn + +# Reuse the visibility-aware iter_shapes() walker from the content inventory above. + +# 1. Resolve the theme related to each slide's own layout/master. A package can contain +# multiple masters with different themes, so the first /ppt/theme/* part is not a safe default. +theme_cache = {} +DRAWINGML = {"a": "http://schemas.openxmlformats.org/drawingml/2006/main"} +DIAGRAM_NS = "http://schemas.openxmlformats.org/drawingml/2006/diagram" + +def read_theme_role(root, role): + node = root.find(f".//a:{role}Font", DRAWINGML) + if node is None: + return {"latin": "", "eastAsia": "", "complexScript": "", "scripts": {}} + def typeface(tag): + child = node.find(f"a:{tag}", DRAWINGML) + return "" if child is None else child.get("typeface", "") + return { + "latin": typeface("latin"), + "eastAsia": typeface("ea"), + "complexScript": typeface("cs"), + "scripts": { + child.get("script"): child.get("typeface", "") + for child in node.findall("a:font", DRAWINGML) if child.get("script") + }, + } + +def theme_faces_for_slide(slide): + master_part = slide.slide_layout.slide_master.part + cache_key = str(master_part.partname) + if cache_key not in theme_cache: + theme_part = master_part.part_related_by(RT.THEME) + root = ET.fromstring(theme_part.blob) + theme_cache[cache_key] = { + "major": read_theme_role(root, "major"), + "minor": read_theme_role(root, "minor"), + } + return cache_key, theme_cache[cache_key] + +EAST_ASIAN_SCRIPTS = {"Hans", "Hant", "Jpan", "Hang", "Bopo"} +COMPLEX_SCRIPTS = {"Arab", "Hebr", "Deva", "Beng", "Taml", "Thai"} +HAN_RANGES = ( + (0x2E80, 0x2E99), (0x2E9B, 0x2EF3), (0x2F00, 0x2FD5), + (0x3005, 0x3005), (0x3007, 0x3007), (0x3021, 0x3029), (0x3038, 0x303B), + (0x3400, 0x4DBF), (0x4E00, 0x9FFF), + (0xF900, 0xFA6D), (0xFA70, 0xFAD9), + (0x16FE2, 0x16FE3), (0x16FF0, 0x16FF6), + (0x20000, 0x2A6DF), (0x2A700, 0x2B81D), (0x2B820, 0x2CEAD), + (0x2CEB0, 0x2EBE0), (0x2EBF0, 0x2EE5D), (0x2F800, 0x2FA1D), + (0x30000, 0x3134A), (0x31350, 0x33479), +) + +def character_tags(character): + codepoint = ord(character) + if 0x3040 <= codepoint <= 0x30FF or 0x31F0 <= codepoint <= 0x31FF: + return ["Jpan"] + if (0x1100 <= codepoint <= 0x11FF or 0x3130 <= codepoint <= 0x318F + or 0xA960 <= codepoint <= 0xA97F or 0xAC00 <= codepoint <= 0xD7AF + or 0xD7B0 <= codepoint <= 0xD7FF): + return ["Hang"] + if 0x3100 <= codepoint <= 0x312F or 0x31A0 <= codepoint <= 0x31BF: + return ["Bopo"] + if any(start <= codepoint <= end for start, end in HAN_RANGES): + return ["Hans", "Hant", "Jpan", "Hang"] # locale disambiguates Han + for tag, start, end in ( + ("Cyrl", 0x0400, 0x052F), ("Hebr", 0x0590, 0x05FF), + ("Arab", 0x0600, 0x06FF), ("Deva", 0x0900, 0x097F), + ("Beng", 0x0980, 0x09FF), ("Taml", 0x0B80, 0x0BFF), + ("Thai", 0x0E00, 0x0E7F), + ): + if start <= codepoint <= end: + return [tag] + return [] + +def script_tags(text): + tags = [] + for character in text: + tags.extend(character_tags(character)) + return list(dict.fromkeys(tags)) + +def required_slots(text): + slots = [] + for character in text: + tags = set(character_tags(character)) + slot = "eastAsia" if tags & EAST_ASIAN_SCRIPTS else ( + "complexScript" if tags & COMPLEX_SCRIPTS else "latin" + ) + if slot not in slots: + slots.append(slot) + return slots or ["latin"] + +def raw_font_slots(rpr): + slots = {} + if rpr is None: + return slots + for slot, tag in (("latin", "a:latin"), ("eastAsia", "a:ea"), + ("complexScript", "a:cs")): + child = rpr.find(qn(tag)) + if child is not None and child.get("typeface"): + slots[slot] = child.get("typeface") + return slots + +THEME_TOKENS = { + "+mj-lt": ("major", "latin"), "+mj-ea": ("major", "eastAsia"), + "+mj-cs": ("major", "complexScript"), "+mn-lt": ("minor", "latin"), + "+mn-ea": ("minor", "eastAsia"), "+mn-cs": ("minor", "complexScript"), +} + +def expand_theme_token(face, theme_fonts): + role_slot = THEME_TOKENS.get(face) + return theme_fonts[role_slot[0]][role_slot[1]] if role_slot else face + +def font_candidates(run, paragraph, theme_fonts, role): + run_slots = raw_font_slots(run._r.rPr) + ppr = paragraph._p.pPr + paragraph_slots = raw_font_slots(None if ppr is None else ppr.defRPr) + tags = script_tags(run.text) + candidates = [] + for slot in required_slots(run.text): + fallback_role = role + explicit_source = None + face = run_slots.get(slot) + if face: + explicit_source = "run" + else: + face = paragraph_slots.get(slot) + if face: + explicit_source = "paragraph defaults" + if face: + resolved = expand_theme_token(face, theme_fonts) + if resolved: + candidates.append((slot, resolved, explicit_source)) + continue + # A generic +mj/+mn slot can be empty while the same role has a matching + # script-specific theme face. Preserve the token's role for that fallback. + if face in THEME_TOKENS: + fallback_role = THEME_TOKENS[face][0] + else: + continue + role_fonts = theme_fonts[fallback_role] + slot_tags = [ + tag for tag in tags + if (slot == "eastAsia" and tag in EAST_ASIAN_SCRIPTS) + or (slot == "complexScript" and tag in COMPLEX_SCRIPTS) + or (slot == "latin" and tag not in EAST_ASIAN_SCRIPTS | COMPLEX_SCRIPTS) + ] + candidates.extend( + (slot, role_fonts["scripts"][tag], f"{fallback_role} theme script {tag}") + for tag in slot_tags if role_fonts["scripts"].get(tag) + ) + if role_fonts.get(slot): + candidates.append((slot, role_fonts[slot], f"{fallback_role} theme {slot}")) + return list(dict.fromkeys(candidates)) + +def iter_text_frames(shapes): + """Shape text frames plus every table cell's text frame (a graphic frame + has has_text_frame=False, so tables must be walked explicitly).""" + for shape in shapes: + if shape.has_text_frame: + yield shape.text_frame + if getattr(shape, "has_table", False): + for row in shape.table.rows: + for cell in row.cells: + yield cell.text_frame + +def unresolved_graphic_font_regions(shapes): + """Mark text systems whose full font cascade python-pptx does not expose.""" + for shape in shapes: + if getattr(shape, "has_chart", False): + yield {"shape": shape.name, "kind": "chart"} + graphic_data = shape._element.find(".//" + qn("a:graphicData")) + if graphic_data is not None and graphic_data.get("uri") == DIAGRAM_NS: + yield {"shape": shape.name, "kind": "SmartArt"} + +unresolved_fonts = [] +for i, slide in enumerate(prs.slides): + master_name, theme_fonts = theme_faces_for_slide(slide) + print(i, "master:", master_name, "theme:", ascii(theme_fonts)) + title_shape = slide.shapes.title + shapes = list(iter_shapes(slide.shapes)) + for frame in iter_text_frames(shapes): + holder = getattr(frame, "_parent", None) # the shape for ordinary frames + for paragraph in frame.paragraphs: + for run in paragraph.runs: + role = "major" if ( + title_shape is not None and + getattr(holder, "_element", None) is title_shape._element + ) else "minor" + faces = font_candidates(run, paragraph, theme_fonts, role) + print(i, repr(run.text[:20]), "font candidates:", ascii(faces)) + unresolved_fonts.extend( + {"slide": i + 1, **record} for record in unresolved_graphic_font_regions(shapes) + ) +if unresolved_fonts: + raise LookupError( + "font audit cannot verify chart/SmartArt text properties: " + f"{unresolved_fonts}; inspect those package parts or render with production fonts" + ) +``` + +`run.font.name` exposes only the Latin slot, so it must not short-circuit inspection of explicit +``/`` faces. python-pptx also does not evaluate the full placeholder -> layout -> +master inheritance chain; the output above remains a candidate list. Check matching layout/master +placeholders when the exact face matters. Han text needs the deck locale to distinguish Hans, +Hant, Japanese, and Korean mappings. The ranges above follow Unicode 17 `Script=Han`, including +BMP compatibility ideographs and supplementary-plane Extension J; add other script ranges when +the task uses them rather than claiming the Latin fallback is definitive. +Chart titles, legends, tick labels, data labels, and SmartArt nodes use additional DrawingML +font cascades across chart/diagram parts. The guard above deliberately fails closed until those +parts are audited directly or the deck is rendered and inspected with the production fonts. + +## Text-fit verification + +`python-pptx` exposes a text box's geometry, not the renderer's final glyph and line layout. +After any text or layout change, render all slides with the fonts used in production: + +```bash +python -c "from pathlib import Path; Path('deck-render').mkdir(exist_ok=True)" +soffice --headless --convert-to pdf --outdir deck-render input.pptx +``` + +Inspect every page of `deck-render/input.pdf` for horizontal clipping and for the final line being +clipped or missing at the bottom. Rasterize the PDF when image inspection is easier. If this must +be an automated gate, use measured text bounds from a native renderer in both axes; +`shape.left + shape.width` is only a slide-boundary check, not an overflow test. + +## Report contract + +Summarize: slide count, per-slide one-line inventory, then findings ranked by user impact. +Extraction for repurposing goes to markdown with speaker notes preserved as blockquotes. diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/create.md b/plugins/Hylouis233/document-skills/skills/pptx/references/create.md new file mode 100644 index 0000000..04909d7 --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/create.md @@ -0,0 +1,115 @@ +# Create a deck (python-pptx) + +## Skeleton using semantic layouts and placeholders + +```python +from pptx import Presentation +from pptx.enum.shapes import PP_PLACEHOLDER +from pptx.util import Inches, Pt + +prs = Presentation() +# Keep this template's slide size and layout geometry together. For 16:9 output, start with a +# real 16:9 template instead of changing only slide_width/slide_height after loading layouts. + +# Built-in template convention: 0=Title, 1=Title and Content, 5=Title Only. +# With a supplied template, inspect [(i, x.name) for i, x in enumerate(prs.slide_layouts)] +# and map these roles to its layouts instead of assuming the same indices. +title_layout = prs.slide_layouts[0] +content_layout = prs.slide_layouts[1] +title_only_layout = prs.slide_layouts[5] + +def add_slide(layout): + return prs.slides.add_slide(layout) + +def placeholder_of_type(slide, *types): + matches = [ + ph for ph in slide.placeholders + if ph.placeholder_format.type in types + ] + if len(matches) != 1: + available = [ + f"{ph.name} ({ph.placeholder_format.type})" + for ph in slide.placeholders + ] + raise ValueError( + f"expected exactly one placeholder of {types}, found {len(matches)}; " + f"available placeholders: {available or 'none'}" + ) + return matches[0] + +# P1 title slide +s = add_slide(title_layout) +s.shapes.title.text = "Service Reliability Review" +placeholder_of_type(s, PP_PLACEHOLDER.SUBTITLE).text = "Quarterly operations review" + +# P2 bullet slide +s = add_slide(content_layout) +s.shapes.title.text = "Executive summary" +tf = placeholder_of_type(s, PP_PLACEHOLDER.BODY, PP_PLACEHOLDER.OBJECT).text_frame +tf.clear(); tf.word_wrap = True +lines = ["Uptime 99.97% (+0.04 vs last quarter)", "MTTR down to 42 minutes", "Two Sev-2 incidents, both capacity-driven"] +for i, line in enumerate(lines): + par = tf.paragraphs[0] if i == 0 else tf.add_paragraph() + par.text = line; par.font.size = Pt(24) + par.space_after = Pt(12) + +# P3 table slide +s = add_slide(title_only_layout) +s.shapes.title.text = "Regional service health" +hdr = ["Region", "Error rate", "P99 latency"] +body = [ + ["Americas", "0.08%", "182 ms"], + ["Europe", "0.05%", "164 ms"], + ["Asia Pacific", "0.11%", "213 ms"], +] +rows, cols = 1 + len(body), len(hdr) +tbl_shape = s.shapes.add_table(rows, cols, Inches(0.5), Inches(1.5), Inches(9), Inches(3.5)) +table = tbl_shape.table +for j, text in enumerate(hdr): + cell = table.cell(0, j); cell.text = text + for par in cell.text_frame.paragraphs: + for run in par.runs: run.font.bold = True +for i, row in enumerate(body, start=1): + for j, text in enumerate(row): + table.cell(i, j).text = text + +# P4 chart slide (real chart part, not a picture) +from pptx.chart.data import CategoryChartData +from pptx.enum.chart import XL_CHART_TYPE +s = add_slide(title_only_layout) +s.shapes.title.text = "Deployment volume" +cd = CategoryChartData() +cd.categories = ["Jul", "Aug", "Sep"] +cd.add_series("Deploy count", (18, 22, 31)) +graphic = s.shapes.add_chart(XL_CHART_TYPE.COLUMN_CLUSTERED, + Inches(0.75), Inches(1.5), Inches(8.5), Inches(5), cd) +chart = graphic.chart +chart.has_legend = False + +prs.save("deck.pptx") +``` + +## The seven patterns + +| Pattern | Build with | +|---|---| +| Title / section divider | Title or Section Header layout; title/subtitle placeholders | +| Agenda / list | Title and Content layout; body placeholder | +| Bullets + callout | Two Content layout; use both content placeholders | +| Image + text | Picture/Content with Caption layout and its placeholders | +| Data table | Title Only layout + `add_table` when no table placeholder exists | +| Chart | Title Only layout + `add_chart` with `CategoryChartData` | +| Quote / closing | Section Header or Title Only layout; add only the missing quote box | + +## Rules + +- Select a template with the requested aspect ratio before adding slides, then keep its slide + size and layout geometry unchanged. Stay inside 0.5in margins. +- Choose a layout for the slide's purpose and populate its title/body placeholders. Use Blank + only when no template layout can express the design, then measure every added shape. +- Title top-left at a consistent y-position across content slides; consistency reads as design. +- Max ~6 bullets per slide, one line each at the chosen size - if a bullet wraps twice, split + the slide or cut. +- Speaker notes: `slide.notes_slide.notes_text_frame.text = "..."` - put the script there, + not on the slide. +- Do not touch masters/layouts unless asked; a restyled master changes every existing slide. diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md b/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md new file mode 100644 index 0000000..b6e1ae2 --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md @@ -0,0 +1,231 @@ +# Edit an existing deck + +## Locate by content, then edit narrowly + +The self-contained route below uses the same bounded `validate_pptx_package()` and +`open_validated_presentation()` preflight as `analyze.md`. Keep both copies in sync. The loader +validates and parses one open file identity; do not replace it with `Presentation(path)`, which +decompresses the package before the bounds run. + +```python +import zipfile +from pathlib import Path + +from lxml import etree +from pptx import Presentation +from pptx.enum.shapes import MSO_SHAPE_TYPE + +CONTENT_TYPES_NS = "http://schemas.openxmlformats.org/package/2006/content-types" +SAFE_XML = etree.XMLParser( + load_dtd=False, resolve_entities=False, no_network=True, + huge_tree=False, recover=False, +) +MAX_ARCHIVE_BYTES = 200 * 1024 * 1024 +MAX_MEMBERS = 10_000 +MAX_XML_PART = 20 * 1024 * 1024 +MAX_ENTRY = 100 * 1024 * 1024 +MAX_TOTAL_UNCOMPRESSED = 500 * 1024 * 1024 +MAX_COMPRESSION_RATIO = 200 + +def require(condition, message): + """Security and edit gates must remain active under python -O.""" + if not condition: + raise ValueError(message) + +def validate_pptx_package(source): + """Apply the same bounded OPC preflight used by the analysis route.""" + source.seek(0, 2) + compressed_size = source.tell() + source.seek(0) + require(compressed_size <= MAX_ARCHIVE_BYTES, "compressed PPTX file size above limit") + with zipfile.ZipFile(source) as archive: + infos = archive.infolist() + require(len(infos) <= MAX_MEMBERS, "archive member count above limit") + names = {info.filename for info in infos} + require(len(names) == len(infos), "duplicate archive member names are unsafe") + require("[Content_Types].xml" in names and "ppt/presentation.xml" in names, + "missing required OPC members") + require(sum(info.file_size for info in infos) <= MAX_TOTAL_UNCOMPRESSED, + "declared total uncompressed size above limit") + + content_types_info = next( + info for info in infos if info.filename == "[Content_Types].xml" + ) + require(content_types_info.file_size <= MAX_XML_PART, + "oversized XML part: [Content_Types].xml") + require( + content_types_info.file_size / max(content_types_info.compress_size, 1) + <= MAX_COMPRESSION_RATIO, + "suspicious compression ratio: [Content_Types].xml", + ) + content_type_chunks = [] + content_type_size = 0 + with archive.open(content_types_info) as stream: + while chunk := stream.read(64 * 1024): + content_type_size += len(chunk) + require(content_type_size <= MAX_XML_PART, + "part exceeded read limit: [Content_Types].xml") + content_type_chunks.append(chunk) + require(content_type_size == content_types_info.file_size, + "size mismatch: [Content_Types].xml") + content_types_blob = b"".join(content_type_chunks) + content_types_root = etree.fromstring(content_types_blob, parser=SAFE_XML) + require(content_types_root.tag == f"{{{CONTENT_TYPES_NS}}}Types", + "invalid content-types root") + default_types = {} + override_types = {} + for declaration in content_types_root: + if declaration.tag == f"{{{CONTENT_TYPES_NS}}}Default": + key = (declaration.get("Extension") or "").casefold() + target = default_types + elif declaration.tag == f"{{{CONTENT_TYPES_NS}}}Override": + part_name = declaration.get("PartName") or "" + require(part_name.startswith("/"), "invalid content-type part name") + key = part_name[1:].casefold() + target = override_types + else: + continue + content_type = declaration.get("ContentType") or "" + require(key and content_type and key not in target, + "invalid or duplicate content-type declaration") + target[key] = content_type.partition(";")[0].strip().casefold() + + actual_total = 0 + for info in infos: + require(info.file_size <= MAX_ENTRY, f"oversized part: {info.filename}") + ratio = info.file_size / max(info.compress_size, 1) + require(ratio <= MAX_COMPRESSION_RATIO, + f"suspicious compression ratio: {info.filename}") + extension = info.filename.rpartition(".")[2].casefold() + content_type = override_types.get( + info.filename.casefold(), default_types.get(extension, "") + ) + is_xml = ( + info.filename.casefold().endswith((".xml", ".rels")) + or content_type in {"application/xml", "text/xml"} + or content_type.endswith("+xml") + ) + if is_xml: + require(info.file_size <= MAX_XML_PART, f"oversized XML part: {info.filename}") + chunks = [] + actual_size = 0 + if info.filename == "[Content_Types].xml": + chunks = [content_types_blob] + actual_size = len(content_types_blob) + actual_total += actual_size + else: + with archive.open(info) as stream: + while chunk := stream.read(64 * 1024): + actual_size += len(chunk) + actual_total += len(chunk) + require(actual_size <= MAX_ENTRY, + f"part exceeded read limit: {info.filename}") + require(actual_total <= MAX_TOTAL_UNCOMPRESSED, + "archive exceeded total read limit") + if is_xml: + chunks.append(chunk) + require(actual_total <= MAX_TOTAL_UNCOMPRESSED, + "archive exceeded total read limit") + require(actual_size == info.file_size, f"size mismatch: {info.filename}") + if is_xml: + etree.fromstring(b"".join(chunks), parser=SAFE_XML) + source.seek(0) + +def open_validated_presentation(path): + """Preflight and parse one immutable open-file identity, then release the handle.""" + with Path(path).open("rb") as source: + validate_pptx_package(source) + source.seek(0) + return Presentation(source) + +prs = open_validated_presentation("input.pptx") + +old, new = "old wording", "new wording" +slide_index = None # Set this and shape_name when repeated text is expected. +shape_name = None +target_location = None # e.g. "Table 1/table[0,1]" for duplicate table text + +def iter_shapes(shapes, path=""): + """Yield (path, shape) for every shape, recursing into groups so text inside + grouped artwork is reachable; the path keeps the uniqueness check readable.""" + for shape in shapes: + here = f"{path}/{shape.name}" if path else shape.name + if shape.shape_type == MSO_SHAPE_TYPE.GROUP: + yield from iter_shapes(shape.shapes, here) + else: + yield here, shape + +def iter_text_targets(path, shape): + if shape.has_text_frame: + yield path, shape.text_frame + if shape.has_table: + for row_index, row in enumerate(shape.table.rows): + for column_index, cell in enumerate(row.cells): + if cell.is_spanned: # covered merge slots can retain stale, non-rendered text + continue + yield f"{path}/table[{row_index},{column_index}]", cell.text_frame + +candidates = [] +for i, slide in enumerate(prs.slides): + if slide_index is not None and i != slide_index: + continue + for path, shape in iter_shapes(slide.shapes): + if shape_name is not None and shape.name != shape_name: + continue + for location, text_frame in iter_text_targets(path, shape): + if target_location is not None and location != target_location: + continue + if old in text_frame.text: + candidates.append((i, location, text_frame)) + +locations = [(i, location) for i, location, _ in candidates] +require(len(candidates) == 1, f"expected one matching text target, found {locations}") +_, _, tf = candidates[0] + +# Replace inside one existing run so its formatting and hyperlink are retained. +require(tf.text.count(old) == 1, "target occurs more than once in the selected shape") +run_hits = [ + run + for paragraph in tf.paragraphs + for run in paragraph.runs + if old in run.text +] +require(len(run_hits) == 1 and run_hits[0].text.count(old) == 1, ( + "target is duplicated or split across runs; report it instead of flattening the paragraph" +)) +run_hits[0].text = run_hits[0].text.replace(old, new, 1) + +prs.save("input-edited.pptx") +``` + +## Rules + +1. **Never rebuild the file to make a small change.** Rewriting slides from scratch loses the + template, masters, notes, and animations. Edit in place, save to a new path. +2. Address shapes by slide index + shape name or matched text, and **require exactly one match** + with an explicit exception (never a Python `assert`, which `python -O` removes). + The locator must search both shape text frames and every editable table cell, skipping grid + slots covered by a merge and retaining a stable `/table[row,column]` suffix. If copy repeats + inside one table, set `target_location` as well as the slide/shape selectors rather than + choosing one. +3. For formatted text, change `run.text` only when the target is wholly inside one run. Assigning + `paragraph.text` or `text_frame.text` rebuilds runs and can discard run formatting and links. + If the target spans runs, stop and make an explicitly reviewed run/XML edit. +4. Table cells are edited at run level exactly like shape text: iterate + `table.cell(r, c).text_frame.paragraphs` and change `run.text`. Assigning `cell.text` + (or `.text` on the text frame) rebuilds the frame and discards per-run formatting and + hyperlinks: + + ```python + cell = table.cell(2, 1) + hits = [run for p in cell.text_frame.paragraphs for run in p.runs if old in run.text] + if len(hits) != 1 or hits[0].text.count(old) != 1: + raise ValueError("target is duplicated or split across runs in this cell") + hits[0].text = hits[0].text.replace(old, new, 1) + ``` +5. Chart data: `chart.replace_data(CategoryChartData(...))` updates the embedded workbook and + the plot together - do not hand-edit the XML series. +6. Reordering slides means moving the underlying `sldIdLst` entries; do it only on request and + verify order in the postcheck. +7. Group shapes: iterate `shape.shapes` recursively to reach members; python-pptx will not + ungroup for you - do not try to flatten groups. diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md b/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md new file mode 100644 index 0000000..a15a40f --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md @@ -0,0 +1,152 @@ +--- +name: xlsx +description: Read, edit, create, or fix spreadsheet files - .xlsx, .xlsm, .xltx, .csv, .tsv. Use this Skill whenever a spreadsheet is the primary input or output: opening and inspecting workbooks, editing cells or adding sheets, writing real formulas instead of pasted values, cleaning messy data, adding charts and formatting, converting between tabular formats, or diagnosing corrupted or slow-opening files. +--- + +# XLSX workbench + +Spreadsheets are where agents do the most damage by being clever: pasted values where formulas +belong, dates written as text, whole-number floats displayed as `3.0000000001`. Follow the +contracts below. + +## Step 0 - Check the toolchain + +```bash +python -c "import defusedxml, openpyxl; print(openpyxl.__version__, defusedxml.__version__)" +``` + +- `.csv`/`.tsv` -> standard library `csv` module is fine and often better (streaming). +- `.xlsx`/`.xlsm`/`.xltx` -> openpyxl. +- Safe package preflight requires `defusedxml==0.7.1`; do not fall back to the standard XML + parser for untrusted OOXML parts. +- Macro preservation: openpyxl keeps VBA in `.xlsm` only with `keep_vba=True` on load and save. +- Before **any** `openpyxl.load_workbook()` of an existing package, copy and use the bounded + same-handle loader in [references/package.md](references/package.md). `read_only=True` does + not bound shared strings, styles, or other package parts parsed during initialization. + +## Step 1 - Classify the task + +| Request | Route | +|---|---| +| Open, inspect, profile a workbook | [references/read.md](references/read.md) | +| Edit cells, add sheets, fix formatting | [references/edit.md](references/edit.md) | +| Build a new workbook (data + formulas + chart) | [references/create.md](references/create.md) | +| CSV/TSV in or out, messy data cleanup | [references/csv.md](references/csv.md) | +| Conditional formatting, structured tables, pivot-style aggregation | [references/formatting.md](references/formatting.md) | +| Safely open an existing OOXML package | [references/package.md](references/package.md) | + +## Step 2 - Contracts that always apply + +1. **Formulas are formulas.** If the user asks for a total/average/lookup, write `=SUM(B2:B10)` + in the cell - never the computed number - unless the user explicitly asked to freeze values. + openpyxl writes the formula; Excel/WPS/LibreOffice calculate on open. +2. **`data_only=True` reads cached values** (last calculated by a real app) and **loses + formulas on save**. Use it only for reading values; never load, edit, and save with it. +3. **Types**: write `int`/`float`/`datetime`/`bool`, never formatted strings. Dates go in as + `datetime` with `number_format='yyyy-mm-dd'`; currency as float plus + `number_format='#,##0.00'` (or the locale-appropriate currency format string). +4. **Formulas are not recalculated by openpyxl.** After writing formulas, set + `wb.calculation.fullCalcOnLoad = True` before saving so Excel/WPS/LibreOffice recalculate + on open even when the workbook (typically one you loaded, which can carry + `fullCalcOnLoad=False`) uses calculation mode `manual` - check + `wb.calculation.calcMode`. You still cannot read results back without opening the file in + a real spreadsheet app; verify formula strings and ranges structurally instead (see + postcheck). +5. **Dimensions**: treat ``, `ws.max_row`, and `ws.max_column` as untrusted hints. + Discover logical cells with the sparse worksheet-XML scan in the read route; never expand an + unknown rectangular range merely to find its bounds. +6. Save to a new path first; overwrite only on explicit request. + +## Step 3 - Postcheck (mandatory) + +Save this as `postcheck.py`, copy `load_validated_workbook()` and its dependencies from +[references/package.md](references/package.md), then pass the output path followed by every +sheet the task should produce, for example `python postcheck.py output.xlsx Sales Summary`: + +```python +import openpyxl +import sys + +if len(sys.argv) < 3: + raise SystemExit("usage: python postcheck.py OUTPUT.xlsx EXPECTED_SHEET [...]") +output_path, *expected_sheets = sys.argv[1:] +# Populate this whenever the task requested specific display formats. +expected_number_formats = { + # "Sales": {"D2": "#,##0.00", "E2": "yyyy-mm-dd"}, +} +# Populate every formula the task intends to create or preserve. +expected_formulas = { + # "Sales": {"D2": "=C2*1.08"}, +} +# Populate every expected sheet with the exact used range required by the task. +expected_dimensions = { + # "Sales": "A1:E20", + # "Summary": "A1:C8", +} +wb = load_validated_workbook(output_path) + +def require(condition, message): + if not condition: + raise ValueError(message) + +def formula_text(value): + if isinstance(value, str): + return value + if text := getattr(value, "text", None): + return text + fields = ("ref", "r1", "r2", "dt2D", "dtr", "ca", "del1", "del2") + details = ", ".join( + f"{name}={getattr(value, name)!r}" for name in fields if hasattr(value, name) + ) + return f"{type(value).__name__}({details})" + +print("sheets:", wb.sheetnames) +missing = set(expected_sheets) - set(wb.sheetnames) +require(not missing, f"missing expected sheets: {sorted(missing)}") +require( + set(expected_dimensions) == set(expected_sheets), + "expected_dimensions must declare the exact used range for every expected sheet", +) +calc = wb.calculation +print("calcMode:", getattr(calc, "calcMode", None), + "fullCalcOnLoad:", getattr(calc, "fullCalcOnLoad", None)) +if any(expected_formulas.values()): + require( + getattr(calc, "fullCalcOnLoad", False) is True + or getattr(calc, "calcMode", None) == "auto", + "formula output is not configured to recalculate in spreadsheet viewers", + ) +for ws in wb.worksheets: + print(f"{ws.title} dims:", ws.dimensions) + if ws.title in expected_dimensions: + require( + ws.dimensions == expected_dimensions[ws.title], + f"{ws.title}: expected used range {expected_dimensions[ws.title]!r}, " + f"got {ws.dimensions!r}", + ) + # `expected_formulas` is the task contract, so verify those coordinates directly. + # Never call unbounded iter_rows(): one styled extreme cell can make the rectangle huge. + actual_formulas = {} + for coordinate, expected_formula in expected_formulas.get(ws.title, {}).items(): + cell = ws[coordinate] + actual_formula = formula_text(cell.value) if cell.data_type == "f" else None + actual_formulas[coordinate] = actual_formula + require( + actual_formula == expected_formula, + f"{ws.title}!{coordinate}: expected formula {expected_formula!r}, " + f"got {actual_formula!r}", + ) + print(f"{ws.title} expected formula cells:", list(actual_formulas.items())[:10]) + for coordinate, expected_format in expected_number_formats.get(ws.title, {}).items(): + actual_format = ws[coordinate].number_format + require(actual_format == expected_format, ( + f"{ws.title}!{coordinate}: expected format {expected_format!r}, got {actual_format!r}" + )) +wb.close() +``` + +Confirm: expected sheet names exist; used range matches expectations; intended formula cells +contain formula strings; when the task wrote formulas, the printout shows `fullCalcOnLoad: True` +(or `calcMode: auto`) so viewers will recalculate — otherwise set it and re-save; every +task-specific formatted cell is listed in `expected_number_formats` and matches. Report what +was verified and note that final rendered values require opening in a spreadsheet application. diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/create.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/create.md new file mode 100644 index 0000000..6f3220d --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/create.md @@ -0,0 +1,75 @@ +# Create a workbook from scratch + +```python +import openpyxl +from openpyxl.styles import Font, PatternFill, Alignment, Border, Side +from openpyxl.utils import get_column_letter +from openpyxl.chart import BarChart, LineChart, Reference + +wb = openpyxl.Workbook() +ws = wb.active +ws.title = "Sales" + +headers = ["Region", "Product", "Units", "Unit price", "Revenue"] +ws.append(headers) +rows = [ + ("EU", "Widget", 120, 9.5), + ("EU", "Gadget", 80, 21.0), + ("US", "Widget", 200, 9.5), +] +for r_i, (region, product, units, price) in enumerate(rows, start=2): + ws.cell(row=r_i, column=1, value=region) + ws.cell(row=r_i, column=2, value=product) + ws.cell(row=r_i, column=3, value=units) + ws.cell(row=r_i, column=4, value=price) + ws.cell(row=r_i, column=5, value=f"=C{r_i}*D{r_i}") # formula, not value + +last = len(rows) + 1 +ws.cell(row=last + 1, column=4, value="Total") +ws.cell(row=last + 1, column=5, value=f"=SUM(E2:E{last})").font = Font(bold=True) + +# Header styling +for cell in ws[1]: + cell.font = Font(bold=True, color="FFFFFF") + cell.fill = PatternFill("solid", fgColor="1F4E79") + cell.alignment = Alignment(horizontal="center") + +# Number formats on data columns +for r_i in range(2, last + 2): + ws.cell(row=r_i, column=4).number_format = "0.00" + ws.cell(row=r_i, column=5).number_format = "#,##0.00" + +widths = {"A": 10, "B": 16, "C": 8, "D": 12, "E": 12} +for col, w in widths.items(): + ws.column_dimensions[col].width = w +ws.freeze_panes = "A2" + +# Native chart bound to the sheet data +chart = BarChart() +chart.type = "col" +# The source has one bar per sales row (including two separate Widget rows), not +# a product aggregate, so keep the title explicit about that granularity. +chart.title = "Revenue by transaction row" +chart.y_axis.title = "Revenue" +data = Reference(ws, min_col=5, min_row=1, max_row=last) # includes header for series name +cats = Reference(ws, min_col=2, min_row=2, max_row=last) +chart.add_data(data, titles_from_data=True) +chart.set_categories(cats) +chart.width, chart.height = 16, 9 +ws.add_chart(chart, "G2") + +wb.save("report.xlsx") +``` + +## Rules + +- `ws.append(list)` for row-oriented data; `ws.cell(row=, column=, value=)` when you need the + cell object anyway (styling, formats). +- A chart references cells - it does not copy data. Keep the referenced range contiguous and + include the header row when using `titles_from_data=True`. +- Multiple sheets: one topic per sheet, `wb.create_sheet(name)`; cross-sheet formulas use + `SheetName!A1` syntax (quote the name if it contains spaces: `'Sales Data'!A1`). +- `LineChart` for trends, `BarChart` for comparisons, `PieChart` only for few categories that + sum to a whole. +- Excel row limit 1,048,576 and column limit 16,384 are hard; streaming writes should chunk + via `write_only=True` workbooks when generating hundreds of thousands of rows. diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md new file mode 100644 index 0000000..d26f0d7 --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md @@ -0,0 +1,229 @@ +# CSV / TSV and messy data + +## Reading + +```python +import csv + +with open("input.csv", newline="", encoding="utf-8-sig") as f: # utf-8-sig strips a BOM + reader = csv.DictReader(f) + for i, row in enumerate(reader): + if i >= 5: break + print(row) +``` + +- Always pass `newline=""` to `open` on every platform - it is the documented requirement, + not a style choice. +- Sniff the dialect when provenance is unknown, and pass the detected dialect to the reader - + seeking back alone does not reconfigure it, so semicolon exports would still parse as comma: + + ```python + with open("input.csv", newline="", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + dialect = csv.Sniffer().sniff(sample) # raises csv.Error on ambiguous input + reader = csv.DictReader(f, dialect=dialect) + for i, row in enumerate(reader): + if i >= 5: break + print(row) + ``` +- Never trust inferred dtypes in CSV: everything is a string. Convert explicitly with + `try/except ValueError` per column and report counts of parse failures rather than dropping + rows silently. + +## Writing + +```python +import csv +from pathlib import Path + +FORMULA_OPERATORS = ("=", "+", "-", "@", "＝", "＋", "－", "＠") + +def begins_spreadsheet_formula(value): + index = 0 + while index < len(value) and (ord(value[index]) <= 0x20 or value[index] == "\ufeff"): + index += 1 + return value.startswith(FORMULA_OPERATORS, index) + +def spreadsheet_csv_field(value, *, mode="safe"): + if mode not in {"safe", "raw"}: + raise ValueError("mode must be 'safe' or 'raw'") + if mode == "safe" and isinstance(value, str) and begins_spreadsheet_formula(value): + return "'" + value + return value + +def delimiter_for(path): + delimiter = {".csv": ",", ".tsv": "\t"}.get(Path(path).suffix.lower()) + if delimiter is None: + raise ValueError("output must use a .csv or .tsv extension") + return delimiter + +output_path = Path("output.csv") # use .tsv when tab-separated output was requested +with output_path.open("w", newline="", encoding="utf-8") as f: + writer = csv.writer(f, delimiter=delimiter_for(output_path)) + rows = [["Region", "Units", "Note"], ["EU", 120, "=2+2"]] + writer.writerows([spreadsheet_csv_field(value) for value in row] for row in rows) +``` + +Use `mode="safe"` (the default above) when the CSV will be opened in Excel, LibreOffice, +Google Sheets, or another spreadsheet application. It scans past leading C0 controls/spaces and +BOMs, then prefixes the **complete original field** with an apostrophe when the next character is +`=`, `+`, `-`, `@`, or the corresponding fullwidth operator. Importers can strip or ignore leading +tab/CR/LF/BOM characters before formula detection, and CSV quoting does not neutralize them. +Benign control-prefixed text is preserved. Numeric values, including negative numbers represented +as numbers, are not changed. This protection deliberately changes formula-like serialized strings. + +Use `mode="raw"` only when the user explicitly requires byte-for-value interchange with a +trusted machine consumer. Raw mode preserves the exact strings and provides **no spreadsheet +formula-injection protection**; do not present a raw export as safe to open in a spreadsheet. + +## Converting + +- CSV -> XLSX: read with `csv`, write with openpyxl; convert values to real types on the way + through (dates via `datetime.strptime` with the format actually observed). Treat every + remaining CSV field as data, not a formula. In particular, force strings beginning with `=` + back to the string data type unless the user explicitly requested formula interpretation: + + ```python + def write_csv_field(cell, value): + cell.value = value + if isinstance(value, str) and value.startswith("="): + cell.data_type = "s" # openpyxl otherwise promotes it to an XLSX formula + ``` +- XLSX -> CSV: use a `data_only=True` read so formulas export the cached values users + see, not formula strings. Pair it with the sparse XML profile and report missing caches. + `openpyxl` exposes both a missing cache and a present empty-string cache as `None`, so consult + the worksheet XML. A nonempty `` is cached; an empty `` is a valid displayed blank only + when the formula cell explicitly has string-result type `t="str"`. A missing `` or the bare + empty `` that openpyxl writes for an uncalculated formula must fail closed. Cached values + can still be stale until a spreadsheet application recalculates the workbook. Copy the + bounded helpers from [package.md](package.md) and `worksheet_xml_profile()` from + [read.md](read.md) first: raw ZIP inspection and the value workbook must share one already + validated source identity. Refuse a logical CSV rectangle above the explicit cell budget; + never call an unbounded `iter_rows()` after `reset_dimensions()`. + + ```python + import csv + import os + import openpyxl + import posixpath + import zipfile + from pathlib import Path + from tempfile import mkstemp + from openpyxl.utils import get_column_letter + from openpyxl.utils.cell import coordinate_to_tuple + from xml.etree import ElementTree as ET + + FORMULA_OPERATORS = ("=", "+", "-", "@", "＝", "＋", "－", "＠") + MAIN_NS = "http://schemas.openxmlformats.org/spreadsheetml/2006/main" + DOC_REL_NS = "http://schemas.openxmlformats.org/officeDocument/2006/relationships" + PKG_REL_NS = "http://schemas.openxmlformats.org/package/2006/relationships" + CELL_TAG = f"{{{MAIN_NS}}}c" + FORMULA_TAG = f"{{{MAIN_NS}}}f" + VALUE_TAG = f"{{{MAIN_NS}}}v" + INLINE_STRING_TAG = f"{{{MAIN_NS}}}is" + MAX_EXPLICIT_CELLS = 1_000_000 + MAX_CSV_EXPORT_CELLS = 5_000_000 + + def begins_spreadsheet_formula(value): + index = 0 + while index < len(value) and (ord(value[index]) <= 0x20 or value[index] == "\ufeff"): + index += 1 + return value.startswith(FORMULA_OPERATORS, index) + + def spreadsheet_csv_field(value, *, mode="safe"): + if mode not in {"safe", "raw"}: + raise ValueError("mode must be 'safe' or 'raw'") + if mode == "safe" and isinstance(value, str) and begins_spreadsheet_formula(value): + return "'" + value + return value + + def delimiter_for(path): + delimiter = {".csv": ",", ".tsv": "\t"}.get(Path(path).suffix.lower()) + if delimiter is None: + raise ValueError("output must use a .csv or .tsv extension") + return delimiter + + def worksheet_part(archive, sheet_name): + workbook = ET.fromstring(archive.read("xl/workbook.xml")) + sheet = next( + item for item in workbook.iter(f"{{{MAIN_NS}}}sheet") + if item.attrib["name"] == sheet_name + ) + relationship_id = sheet.attrib[f"{{{DOC_REL_NS}}}id"] + relationships = ET.fromstring(archive.read("xl/_rels/workbook.xml.rels")) + target = next( + item.attrib["Target"] for item in relationships.iter(f"{{{PKG_REL_NS}}}Relationship") + if item.attrib["Id"] == relationship_id + ) + part = target.lstrip("/") if target.startswith("/") else posixpath.normpath( + posixpath.join("xl", target) + ) + require(part in archive.namelist(), f"worksheet part is missing: {part}") + return part + + input_path = "input.xlsx" + sheet_name = "Data" + export_mode = "safe" # use "raw" only for explicitly requested trusted machine interchange + output_path = Path("output.csv") + temporary_path = None + try: + descriptor, temporary_name = mkstemp( + dir=output_path.parent, prefix=f".{output_path.name}.", suffix=".tmp" + ) + temporary_path = Path(temporary_name) + os.close(descriptor) + with validated_xlsx_source(input_path) as package_source: + value_wb = openpyxl.load_workbook(package_source, read_only=True, data_only=True) + try: + package_source.seek(0) + with zipfile.ZipFile(package_source) as archive: + part = worksheet_part(archive, sheet_name) + profile = worksheet_xml_profile(archive, part) + if profile["missing_formula_count"]: + raise RuntimeError( + "formula cells have no cached value: " + f"{profile['missing_formula_samples']}" + ) + value_ws = value_wb[sheet_name] + value_ws.reset_dimensions() + if profile["bounds"] is None: + rows = iter(()) + else: + min_row, min_column, max_row, max_column = profile["bounds"] + export_cells = (max_row - min_row + 1) * (max_column - min_column + 1) + require(export_cells <= MAX_CSV_EXPORT_CELLS, + f"CSV export rectangle is too large: {export_cells} cells") + rows = value_ws.iter_rows( + min_row=min_row, min_col=min_column, + max_row=max_row, max_col=max_column, + values_only=True, + ) + with temporary_path.open("w", newline="", encoding="utf-8") as output: + writer = csv.writer(output, delimiter=delimiter_for(output_path)) + for value_row in rows: + writer.writerow([ + spreadsheet_csv_field(value, mode=export_mode) for value in value_row + ]) + finally: + value_wb.close() + except Exception: + if temporary_path is not None: + temporary_path.unlink(missing_ok=True) + raise + require(temporary_path is not None, "CSV temporary output was not created") + temporary_path.replace(output_path) + ``` + + Format numbers yourself only if the user needs a fixed display format; otherwise write raw + cached values and say so. Export formula text from the `data_only=False` workbook only when + the user explicitly requests formulas rather than displayed values. +- Large CSV -> keep it CSV or move to SQLite/Parquet; loading it all into one sheet to + "preserve" it usually exceeds limits and helps nobody. + +## Messy data cleanup contract + +1. Profile before touching: row count, per-column types, null counts, duplicate-key check. +2. Report the cleanup plan and get on with it only for mechanical transforms (trim, case, + date parsing, dedup on declared keys). +3. Every destructive step (dropping rows, overwriting values) must be counted and reported. diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md new file mode 100644 index 0000000..cf36e83 --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md @@ -0,0 +1,355 @@ +# Edit an existing workbook + +Copy the bounded loaders from [package.md](package.md) into the script. The same preflight must +run before the main editable load and before any raw ZIP/XML round-trip inventory. Define +`load_with_round_trip_audit()` from the prerequisite block later on this page before executing +the edit block; the workbook it returns is the one that must be edited and saved. + +```python +import openpyxl +from datetime import date +from openpyxl.formula import Tokenizer +from openpyxl.styles import Font +from openpyxl.utils.cell import coordinate_to_tuple, range_boundaries + +approved_feature_loss = False # set True only after showing the inventory to the user +wb, dropped_parts, stripped_extensions = load_with_round_trip_audit("input.xlsx") +if (dropped_parts or stripped_extensions) and not approved_feature_loss: + wb.close() + raise RuntimeError( + f"openpyxl would drop parts={dropped_parts!r}, extensions={stripped_extensions!r}; " + "report this exact inventory and obtain confirmation before editing" + ) +ws = wb["Data"] + +def formula_text(value): + if isinstance(value, str): + return value + if text := getattr(value, "text", None): + return text + fields = ("ref", "r1", "r2", "dt2D", "dtr", "ca", "del1", "del2") + details = ", ".join( + f"{name}={getattr(value, name)!r}" for name in fields if hasattr(value, name) + ) + return f"{type(value).__name__}({details})" + +def defined_name_values(workbook): + names = workbook.defined_names + # openpyxl 3.1 uses DefinedNameDict; 3.0 uses DefinedNameList. Support both. + return names.values() if hasattr(names, "values") else names.definedName + +def drawing_anchor_rows(drawing): + """Return 1-based rows occupied by a string/one-cell/two-cell anchor.""" + anchor = drawing.anchor + if isinstance(anchor, str): + return (coordinate_to_tuple(anchor)[0],) + rows = [] + if marker := getattr(anchor, "_from", None): + rows.append(marker.row + 1) + if marker := getattr(anchor, "to", None): + rows.append(marker.row + 1) + return tuple(rows) + +def sparse_cells(sheet): + """Walk instantiated cells only; worksheet bounds can span the full grid.""" + if not hasattr(sheet, "_cells"): + raise RuntimeError("structural edits require a normal writable Worksheet") + return sorted(sheet._cells.values(), key=lambda cell: (cell.row, cell.column)) + +def sparse_formula_cells(sheet): + return (cell for cell in sparse_cells(sheet) if cell.data_type == "f") + +def structural_references(workbook): + """Inventory formulas/ranges that insert_rows/delete_rows will not rewrite.""" + refs = [] + for item in defined_name_values(workbook): + refs.append(("defined name", item.name, item.attr_text)) + for sheet in workbook.worksheets: + owner = sheet.title + for cell in sparse_formula_cells(sheet): + refs.append(("cell formula", f"{owner}!{cell.coordinate}", + formula_text(cell.value))) + for cell in sparse_cells(sheet): + hyperlink = cell.hyperlink + if hyperlink is None: + continue + location = getattr(hyperlink, "location", None) + target = getattr(hyperlink, "target", None) + refs.append(( + "cell hyperlink", + f"{owner}!{cell.coordinate}", + (getattr(hyperlink, "ref", None), location, target), + )) + for table in sheet.tables.values(): + refs.append(("table", owner + "!" + table.name, table.ref)) + for merged_range in sheet.merged_cells.ranges: + refs.append(("merged range", owner, str(merged_range))) + if sheet.auto_filter.ref: + refs.append(("auto filter", owner, sheet.auto_filter.ref)) + for label, value in ( + ("print area", sheet.print_area), + ("print title rows", sheet.print_title_rows), + ("print title columns", sheet.print_title_cols), + ): + if value: + refs.append((label, owner, str(value))) + for validation in sheet.data_validations.dataValidation: + refs.append(("data validation range", owner, str(validation.sqref))) + for formula in (validation.formula1, validation.formula2): + if formula: + refs.append(("data validation formula", owner, str(formula))) + for conditional_range in sheet.conditional_formatting: + refs.append(("conditional formatting range", owner, str(conditional_range.sqref))) + for rule in sheet.conditional_formatting[conditional_range]: + for formula in getattr(rule, "formula", ()): + refs.append(("conditional formatting formula", owner, str(formula))) + for index, chart in enumerate(sheet._charts, start=1): + refs.append(("drawing anchor", f"{owner} chart {index}", drawing_anchor_rows(chart))) + for element in chart._write().iter(): + if element.tag.rsplit("}", 1)[-1] == "f" and element.text: + refs.append(("chart series", f"{owner} chart {index}", element.text)) + for index, image in enumerate(sheet._images, start=1): + refs.append(("drawing anchor", f"{owner} image {index}", drawing_anchor_rows(image))) + return refs + +def non_cell_references(workbook): + return [reference for reference in structural_references(workbook) + if reference[0] != "cell formula"] + +def cell_formula_references(workbook): + """Inventory ordinary, array, and data-table formulas before row/column moves.""" + refs = [] + for sheet in workbook.worksheets: + for cell in sparse_formula_cells(sheet): + value = cell.value + refs.append(( + "cell formula", + sheet.title, + cell.coordinate, + formula_text(value), + )) + return refs + +def formula_may_intersect_rows(owner_sheet, formula, shifted_sheet, start_row): + """Fail closed unless every range token is provably above/outside the shifted rows.""" + if not isinstance(formula, str) or not formula.startswith("="): + return True + tokens = Tokenizer(formula).items + # These functions can manufacture references from strings or numeric + # offsets that the RANGE-token audit below cannot see or rewrite safely. + unmodeled_reference_functions = {"indirect", "offset", "address", "hyperlink"} + if any( + token.type == "FUNC" and token.subtype == "OPEN" + and token.value.rstrip("(").rsplit(":", 1)[-1] + .lstrip("@").rsplit(".", 1)[-1].casefold() + in unmodeled_reference_functions + for token in tokens + ): + return True + for token in tokens: + if token.type != "OPERAND" or token.subtype != "RANGE": + continue + reference = token.value + target_sheet = owner_sheet + if "!" in reference: + qualifier, reference = reference.rsplit("!", 1) + if "[" in qualifier or ":" in qualifier: # external or 3-D reference + return True + target_sheet = qualifier.strip("'").replace("''", "'") + # Excel worksheet names are case-insensitive even though their spelling + # is preserved in formulas and workbook metadata. + if target_sheet.casefold() != shifted_sheet.casefold(): + continue + try: + _, min_row, _, max_row = range_boundaries(reference.replace("$", "")) + except ValueError: # named/dynamic reference: require a manual rewrite plan + return True + if min_row is None or max_row is None or max_row >= start_row: + return True + return False + +# Address cells directly; check the header to confirm column meaning first +ws["D2"] = "=C2*1.08" # real formula +ws["E2"] = date(2025, 9, 30) +ws["E2"].number_format = "yyyy-mm-dd" +ws["F2"] = 1234.5 +ws["F2"].number_format = "#,##0.00" + +# Insert/delete does not adjust formulas or non-cell dependencies. Snapshot both inventories, +# but block only formula ranges that may intersect the shifted rows (plus the conservative +# non-cell inventory) so an audited formula such as D2 = C2*1.08 can proceed. +cell_formulas_before = cell_formula_references(wb) +intersecting_formulas = [ + reference for reference in cell_formulas_before + if formula_may_intersect_rows(reference[1], reference[3], ws.title, 5) +] +unaudited_references = intersecting_formulas + non_cell_references(wb) +if unaudited_references: + for reference in unaudited_references: + print("structural-edit dependency:", reference) + raise RuntimeError( + "insert_rows is unsafe until cell formulas and non-cell references are audited" + ) +ws.insert_rows(5) + +# Append a new sheet for derived output +summary = wb.create_sheet("Summary") +summary["A1"] = "Region" +summary["B1"] = "Total" +summary["A2"] = "EU" +summary["B2"] = "=SUMIF(Data!A:A,A2,Data!C:C)" + +header_font = Font(bold=True) +for row in summary["A1:B1"]: + for cell in row: + cell.font = header_font + +# After applying the planned rewrites, rerun the same inventory and compare it with the +# pre-edit snapshot before saving. A post-edit listing alone cannot reveal a stale formula. +print("cell formulas before:", cell_formulas_before) +print("cell formulas after planned rewrites:", cell_formula_references(wb)) +print("non-cell references after planned rewrites:", non_cell_references(wb)) + +# Formula caches are not calculated by openpyxl. Force spreadsheet viewers to +# recalculate every edited formula instead of preserving a manual/stale input mode. +wb.calculation.fullCalcOnLoad = True +wb.calculation.forceFullCalc = True +wb.calculation.calcMode = "auto" +wb.save("input-edited.xlsx") +``` + +- Before editing an unknown workbook, detect what an openpyxl load/save round trip silently + changes. Dropped parts (slicers, pivot caches, power-query connections) are only half the + risk: features stored *inside* a retained part, such as `x14` extension lists in + `xl/worksheets/sheet1.xml`, can be stripped while the archive member name stays. Inventory + every worksheet extension record separately; a per-part set of coarse markers cannot detect + one lost `` when another record keeps the same URI/namespace markers alive: + + ```python + import zipfile + from collections import Counter + from tempfile import TemporaryFile + from xml.etree import ElementTree as ET + + SHEET_NS = "http://schemas.openxmlformats.org/spreadsheetml/2006/main" + EXT_TAG = f"{{{SHEET_NS}}}ext" + + def normalized_text(value): + return (value or "").strip() + + def normalized_element(element): + """Prefix/attribute-order independent, but sensitive to child values and order.""" + return ( + element.tag, # ElementTree expands prefixes to URI names + tuple(sorted(element.attrib.items())), + normalized_text(element.text), + tuple((normalized_element(child), normalized_text(child.tail)) + for child in element), + ) + + def worksheet_extension_records(archive, info): + """Return a multiset of (ext URI, normalized child content) for one worksheet.""" + records = Counter() + extension_depth = 0 + with archive.open(info) as stream: + for event, element in ET.iterparse(stream, events=("start", "end")): + if event == "start": + if extension_depth: + extension_depth += 1 + elif element.tag == EXT_TAG: + extension_depth = 1 + elif extension_depth: + if extension_depth == 1: + children = tuple( + (normalized_element(child), normalized_text(child.tail)) + for child in element + ) + records[(element.attrib.get("uri", ""), children)] += 1 + element.clear() + extension_depth = 0 + else: + extension_depth -= 1 + else: + element.clear() # keep large non-extension sheets streaming + return records + + def archive_inventory(source): + with zipfile.ZipFile(source) as archive: + names = set(archive.namelist()) + extensions = {} + for info in archive.infolist(): + if (info.filename.startswith("xl/worksheets/") + and info.filename.endswith(".xml")): + extensions[info.filename] = worksheet_extension_records(archive, info) + return names, extensions + + def stripped_extension_records(before, after, common_names): + """Return each lost (worksheet, URI, normalized children), including duplicates.""" + stripped = [] + for name in sorted(common_names): + for (uri, children), count in (before.get(name, Counter()) + - after.get(name, Counter())).items(): + stripped.extend((name, uri, children) for _ in range(count)) + return sorted(stripped, key=repr) + + def load_with_round_trip_audit_from_source(source, **load_options): + """Dry-run one workbook, then return a fresh edit copy from the same source.""" + require(not load_options.get("read_only"), + "round-trip audit requires a normal writable Workbook") + require(load_options.get("rich_text", True) is True, + "round-trip audit must preserve rich-text cell runs") + load_options["rich_text"] = True + source.seek(0) + before_names, before_extensions = archive_inventory(source) + source.seek(0) + audit_workbook = openpyxl.load_workbook(source, **load_options) + try: + with TemporaryFile() as output: + audit_workbook.save(output) + output.seek(0) + after_names, after_extensions = archive_inventory(output) + finally: + audit_workbook.close() + dropped = sorted(before_names - after_names) + stripped_extensions = stripped_extension_records( + before_extensions, after_extensions, before_names & after_names + ) + # Saving can consume image streams and other resources. Reload a fresh editable + # workbook from the same already validated identity instead of returning the dry-run copy. + source.seek(0) + editable_workbook = openpyxl.load_workbook(source, **load_options) + return editable_workbook, dropped, stripped_extensions + + def load_with_round_trip_audit(path, **load_options): + with validated_xlsx_source(path) as source: + return load_with_round_trip_audit_from_source(source, **load_options) + ``` + + (openpyxl re-serializes every sheet it touches, so byte-identity of sheets is not a + meaningful check. Expanded XML names normalize arbitrary prefixes, sorted attributes normalize + serialization order, and the per-worksheet multiset still detects one missing record when an + identical URI or even an identical duplicate record survives.) + +## Rules + +- `insert_rows`/`delete_rows` move cells but do **not** rewrite range references for you. + Before structural edits, inventory cell formulas plus workbook defined names, tables, + merged ranges, print areas/titles, autofilters, data validations, conditional formatting, and + chart-series formulas and cell hyperlinks as above. Every hyperlink rewrite plan must update + the anchor `ref` when its cell moves; an internal link must also update a destination stored in + `location` or a `target` beginning with `#` when that destination moves. Treat formula-based + `HYPERLINK()` references as dynamic and require a manual rewrite plan because their destination + is a string token, not an ordinary range token. + Refuse the insertion until every dependency that can intersect the shifted region has an + explicit rewrite; after the edit, rerun both inventories and verify the expected references. +- Styling: import `Font` and assign the style to each cell. A range such as `ws["A1:F1"]` + returns tuples of cells and cannot be styled as one object. +- Column widths: `ws.column_dimensions["A"].width = 28` - set after writing data, from the + longest value you wrote, not a fixed guess. +- Freeze panes and autofilter improve usability cheaply: + `ws.freeze_panes = "A2"; ws.auto_filter.ref = ws.dimensions`. +- Merged cells: avoid creating new merges; writing into a non-anchor merged cell raises. +- `.xlsm`: `load_validated_workbook(path, keep_vba=True)` and save with the same suffix, or macros are + stripped. +- Do not delete sheets unless asked; hide instead (`ws.sheet_state = "hidden"`) when the goal + is a cleaner tab bar. diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/formatting.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/formatting.md new file mode 100644 index 0000000..5deeaab --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/formatting.md @@ -0,0 +1,259 @@ +# Conditional formatting, tables, and pivot-style aggregation + +## Conditional formatting (openpyxl) + +Rules attach to a range and survive openpyxl round trips. Scopes smaller than the whole +column keep files fast and avoid formatting ghost rows: + +```python +from openpyxl.formatting.rule import CellIsRule, ColorScaleRule, DataBarRule, FormulaRule +from openpyxl.styles import Font, PatternFill + +red_fill = PatternFill(start_color="FFC7CE", end_color="FFC7CE", fill_type="solid") +red_font = Font(color="9C0006") + +def last_populated_row(sheet, *, first_data_row=2, min_col=1, max_col=6): + # Conditional formatting uses a normal in-memory Worksheet. Its sparse cell store avoids + # iterating/materializing every row up to an inflated max_row. + populated_rows = ( + cell.row for cell in sheet._cells.values() + if first_data_row <= cell.row + and min_col <= cell.column <= max_col + and cell.value is not None + ) + return max(populated_rows, default=first_data_row - 1) + +# Do not use ws.max_row: a styled but empty cell can inflate it far below the data. +last = last_populated_row(ws) + +# With only a header row every range below would be inverted +# ("D2:D1"); openpyxl rejects those ranges, so guard before building rules. +if last < 2: + print("skipping conditional formatting: no populated data rows below the header") +else: + # value-based rule + ws.conditional_formatting.add( + f"D2:D{last}", + CellIsRule(operator="lessThan", formula=["0"], fill=red_fill, font=red_font), + ) + + # whole-row highlight: FormulaRule anchored with $ on the key column + ws.conditional_formatting.add(f"A2:F{last}", FormulaRule(formula=["$D2<0"], fill=red_fill)) + + # gradient and data bars for magnitude scanning + ws.conditional_formatting.add( + f"C2:C{last}", + ColorScaleRule(start_type="min", start_color="FFFFFF", end_type="max", end_color="63BE7B"), + ) + ws.conditional_formatting.add( + f"E2:E{last}", + DataBarRule(start_type="min", end_type="max", color="638EC6"), + ) +``` + +Restrict the scan to the columns that define the data region. If the data is already a declared +Table, its ref is authoritative instead: use +`openpyxl.utils.cell.range_boundaries(ws.tables["TData"].ref)[3]` for `last`. + +- FormulaRule formulas are US-locale and relative to the range's top-left cell - `$D2` (lock + column, free row) is what makes the whole-row pattern work. +- Multiple rules on one range evaluate by priority; if exactly one should apply, set + `stopIfTrue=True` on the earlier rules. + +## Structured tables + +A real Table gives filter UI, banded styling, and structured references: + +```python +from openpyxl.worksheet.table import Table, TableStyleInfo + +last = last_populated_row(ws) +if last < 2: + raise ValueError("cannot create a data table without populated rows") +tbl = Table(displayName="TData", ref=f"A1:F{last}") # name has no spaces +tbl.tableStyleInfo = TableStyleInfo(name="TableStyleMedium9", showRowStripes=True) +ws.add_table(tbl) +``` + +One Table per sheet region; the ref must cover the headers. Do not also draw manual borders +over a Table range. + +## Pivot-style aggregation - the honest contract + +**openpyxl cannot create pivot tables.** It preserves existing ones on a load/save round trip, +but building the pivot cache from scratch is not supported. Offer these routes and say which +one you took: + +1. **Formula sheet (live, recalculates)** - the default. A `SUMIFS`/`COUNTIFS`/`AVERAGEIFS` + grid keyed on a unique-values column reproduces most pivot outputs and stays a formula + per contract rule 1. Build every sheet reference from the real source sheet's name - + hard-coding `Data!` breaks on any workbook whose sheet is named differently. Copy + `validated_xlsx_source()` from [package.md](package.md) and + `load_with_round_trip_audit_from_source()` plus its inventory dependencies from + [edit.md](edit.md) before running this route: + + ```python + import openpyxl + import posixpath + import zipfile + from xml.etree import ElementTree as ET + + MAIN_NS = "http://schemas.openxmlformats.org/spreadsheetml/2006/main" + DOC_REL_NS = "http://schemas.openxmlformats.org/officeDocument/2006/relationships" + PKG_REL_NS = "http://schemas.openxmlformats.org/package/2006/relationships" + MAX_AGGREGATION_ROW_SPAN = 100_000 + + def cached_formula_coordinates_for_sheet(archive, sheet_name, wanted_coordinates): + """Distinguish a typed cached blank from a formula with no cached result.""" + workbook = ET.fromstring(archive.read("xl/workbook.xml")) + sheet = next( + item for item in workbook.iter(f"{{{MAIN_NS}}}sheet") + if item.attrib["name"] == sheet_name + ) + relationship_id = sheet.attrib[f"{{{DOC_REL_NS}}}id"] + relationships = ET.fromstring(archive.read("xl/_rels/workbook.xml.rels")) + target = next( + item.attrib["Target"] + for item in relationships.iter(f"{{{PKG_REL_NS}}}Relationship") + if item.attrib["Id"] == relationship_id + ) + part = target.lstrip("/") if target.startswith("/") else posixpath.normpath( + posixpath.join("xl", target) + ) + cached = set() + coordinate = cell_type = value_text = None + has_formula = value_seen = False + with archive.open(part) as source: + for event, element in ET.iterparse(source, events=("start", "end")): + if event == "start" and element.tag == f"{{{MAIN_NS}}}c": + coordinate = element.attrib["r"] + cell_type = element.attrib.get("t") + has_formula = value_seen = False + value_text = None + elif event == "end" and coordinate is not None: + if element.tag == f"{{{MAIN_NS}}}f": + has_formula = True + elif element.tag == f"{{{MAIN_NS}}}v": + value_seen = True + value_text = element.text + elif element.tag == f"{{{MAIN_NS}}}c": + valid_cache = value_seen and ( + value_text not in (None, "") or cell_type == "str" + ) + if has_formula and valid_cache and coordinate in wanted_coordinates: + cached.add(coordinate) + coordinate = None + element.clear() + elif event == "end": + element.clear() + return cached + + def sheet_ref(sheet): + # Always quote: valid titles such as Q1-Data are ambiguous when left bare. + # Excel escapes an apostrophe inside a quoted title by doubling it. + escaped = sheet.title.replace("'", "''") + return f"'{escaped}'!" + + source_path = "input.xlsx" + source_sheet_name = "Data" # select this from the user's request / initial inventory + approved_feature_loss = False # set True only after showing the exact audit to the user + with validated_xlsx_source(source_path) as package_source: + wb, dropped_parts, stripped_extensions = load_with_round_trip_audit_from_source( + package_source, data_only=False + ) + if (dropped_parts or stripped_extensions) and not approved_feature_loss: + wb.close() + raise RuntimeError( + f"openpyxl would drop parts={dropped_parts!r}, " + f"extensions={stripped_extensions!r}; obtain confirmation before editing" + ) + ws = wb[source_sheet_name] + src = sheet_ref(ws) # e.g. "'Sales'!" or "'Raw Data'!" + require(hasattr(ws, "_cells"), + "aggregation requires a normal writable Worksheet") + source_cells = sorted( + (cell for cell in ws._cells.values() + if cell.row >= 2 and cell.column == 1 and cell.value is not None), + key=lambda cell: cell.row, + ) + if source_cells: + min_source_row, max_source_row = source_cells[0].row, source_cells[-1].row + row_span = max_source_row - min_source_row + 1 + require(row_span <= MAX_AGGREGATION_ROW_SPAN, + f"aggregation row span is too large: {row_span}") + wanted_formula_coordinates = { + cell.coordinate for cell in source_cells if cell.data_type == "f" + } + package_source.seek(0) + with zipfile.ZipFile(package_source) as archive: + cached_region_formulas = cached_formula_coordinates_for_sheet( + archive, ws.title, wanted_formula_coordinates + ) + package_source.seek(0) + value_wb = openpyxl.load_workbook( + package_source, read_only=True, data_only=True + ) + try: + value_ws = value_wb[ws.title] + value_ws.reset_dimensions() + regions = [] + seen_region_keys = set() + missing_region_caches = [] + if source_cells: + source_by_row = {cell.row: cell for cell in source_cells} + value_rows = value_ws.iter_rows( + min_row=min_source_row, max_row=max_source_row, + min_col=1, max_col=1, + ) + for row_index, value_row in enumerate(value_rows, start=min_source_row): + source_cell = source_by_row.get(row_index) + if source_cell is None: + continue + value_cell = value_row[0] + region = (value_cell.value if source_cell.data_type == "f" + else source_cell.value) + if (source_cell.data_type == "f" and region is None + and source_cell.coordinate not in cached_region_formulas): + missing_region_caches.append(source_cell.coordinate) + continue + if region is None or region == "": # keep valid falsey values: 0 and False + continue + key = (type(region), region) # keep False distinct from numeric 0 + if key not in seen_region_keys: + seen_region_keys.add(key) + regions.append(region) # stable order; no mixed-type sort + finally: + value_wb.close() + if missing_region_caches: + raise RuntimeError( + f"aggregation keys have no cached value: {missing_region_caches}" + ) + + ws2 = wb.create_sheet("ByRegion") + ws2.append(["Region", "Units", "Revenue"]) + for i, region in enumerate(regions, start=2): + ws2.cell(row=i, column=1, value=region) + ws2.cell(row=i, column=2, value=f"=SUMIF({src}A:A,A{i},{src}C:C)") + ws2.cell(row=i, column=3, value=f"=SUMIF({src}A:A,A{i},{src}D:D)") + ``` + + Unique values themselves are formulas only with array/dynamic functions - extracting them + in Python (as above) and writing them as values is the accepted split; the aggregates stay + live. + +2. **Frozen pivot values (Python-side grouping)** - when the user wants a one-shot analysis + report, not a living workbook. Group in pure Python (or pandas if already installed), + write values, and **label the sheet** ("values as of generation, not recalculated"). + +3. **User's Excel/template pivot** - when the workbook already has slicers or a pivot the + user maintains, edit around it and run the feature-loss audit from [edit.md](edit.md) before + saving. Use `load_with_round_trip_audit_from_source()` and edit only the workbook it returns. + +## Postcheck additions + +1. Re-open and count `ws.conditional_formatting` ranges; confirm the intended ranges exist + and anchor rows match the data (a rule left on `D2:D1048576` from an earlier resize is a + defect). +2. Confirm Table names are unique workbook-wide and refs cover the header row. +3. For the formula-sheet route, assert the aggregate cells contain formula strings, per the + main postcheck. diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/package.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/package.md new file mode 100644 index 0000000..372bb9a --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/package.md @@ -0,0 +1,318 @@ +# Bounded XLSX package loading + +Every route that opens an existing `.xlsx`, `.xlsm`, or `.xltx` must preflight the package +before `openpyxl` or route-specific ZIP/XML inspection parses it. Copy these helpers into the +working script. `validated_xlsx_source()` supplies a bounded raw package handle; +`load_validated_workbook()` loads a normal workbook; and `open_validated_workbook()` keeps a +read-only source handle alive until its workbook is closed. Every helper validates and parses +the same open-file identity. + +```python +from contextlib import contextmanager +from pathlib import Path, PurePosixPath +from defusedxml import ElementTree as DefusedET +from openpyxl.utils.cell import range_boundaries +from tempfile import TemporaryFile +import openpyxl +import zipfile + +CONTENT_TYPES_NS = "http://schemas.openxmlformats.org/package/2006/content-types" +SPREADSHEETML_NAMESPACES = { + "http://schemas.openxmlformats.org/spreadsheetml/2006/main", + "http://purl.oclc.org/ooxml/spreadsheetml/main", +} +WORKSHEET_CELL_TAGS = { + f"{{{namespace}}}c" for namespace in SPREADSHEETML_NAMESPACES +} +WORKSHEET_RANGE_TAGS = { + f"{{{namespace}}}{local_name}" + for namespace in SPREADSHEETML_NAMESPACES + for local_name in ("mergeCell", "hyperlink") +} +WORKSHEET_SINGLE_CELL_REF_TAGS = { + f"{{{namespace}}}comment" for namespace in SPREADSHEETML_NAMESPACES +} +WORKSHEET_MULTI_RANGE_TAGS = { + f"{{{namespace}}}{local_name}" + for namespace in SPREADSHEETML_NAMESPACES + for local_name in ("conditionalFormatting", "dataValidation", "scenarios") +} +MAX_ARCHIVE_BYTES = 200 * 1024 * 1024 +MAX_MEMBERS = 10_000 +MAX_XML_PART = 20 * 1024 * 1024 +MAX_WORKSHEET_XML = 100 * 1024 * 1024 +MAX_ENTRY = 100 * 1024 * 1024 +MAX_TOTAL_UNCOMPRESSED = 500 * 1024 * 1024 +MAX_COMPRESSION_RATIO = 200 +MAX_WORKSHEET_MATERIALIZED_CELLS = 1_000_000 +MAX_WORKSHEET_RANGE_LIST_CHARS = 65_536 +MAX_WORKSHEET_RANGE_TOKENS = 100_000 + +def require(condition, message): + """Security gates must remain active under python -O.""" + if not condition: + raise ValueError(message) + +def canonical_member_name(name): + """Reject ambiguous, absolute, or parent-traversing OPC member names.""" + require(name and "\\" not in name and "\x00" not in name, + f"invalid archive member name: {name!r}") + is_directory = name.endswith("/") + path = name[:-1] if is_directory else name + require(path and not path.startswith("/"), f"absolute archive member path: {name}") + parts = path.split("/") + require(all(part not in ("", ".", "..") for part in parts), + f"non-canonical archive member path: {name}") + canonical = PurePosixPath(*parts).as_posix() + ("/" if is_directory else "") + require(canonical == name, f"non-canonical archive member path: {name}") + return canonical.casefold() + +def safe_xml_root(blob, member_name): + try: + return DefusedET.fromstring( + blob, forbid_dtd=True, forbid_entities=True, forbid_external=True + ) + except Exception as error: + raise ValueError(f"unsafe or malformed XML part: {member_name}") from error + +def worksheet_element_cell_cost(element, member_name): + """Bound cells that openpyxl creates while binding worksheet XML.""" + tag = element.tag + if tag in WORKSHEET_CELL_TAGS: + return 1 + if tag not in WORKSHEET_RANGE_TAGS | WORKSHEET_SINGLE_CELL_REF_TAGS: + return 0 + reference = element.get("ref") or "" + require(reference and len(reference) <= 64, + f"invalid materialized worksheet range in {member_name}: {reference!r}") + try: + min_column, min_row, max_column, max_row = range_boundaries(reference) + except (TypeError, ValueError) as error: + raise ValueError( + f"invalid materialized worksheet range in {member_name}: {reference!r}" + ) from error + require( + all(isinstance(value, int) for value in + (min_column, min_row, max_column, max_row)) + and 1 <= min_column <= max_column <= 16_384 + and 1 <= min_row <= max_row <= 1_048_576, + f"invalid materialized worksheet range in {member_name}: {reference!r}", + ) + if tag in WORKSHEET_SINGLE_CELL_REF_TAGS: + require( + min_column == max_column and min_row == max_row, + f"comment reference must identify one cell in {member_name}: {reference!r}", + ) + return (max_column - min_column + 1) * (max_row - min_row + 1) + +def worksheet_element_range_token_cost(element, member_name): + """Bound MultiCellRange objects constructed from worksheet sqref lists.""" + if element.tag not in WORKSHEET_MULTI_RANGE_TAGS: + return 0 + reference = element.get("sqref") or "" + require( + len(reference) <= MAX_WORKSHEET_RANGE_LIST_CHARS, + f"worksheet range list is too long in {member_name}", + ) + return len(reference.split()) + +def read_bounded_member(archive, info, limit, *, capture): + chunks = [] if capture else None + actual_size = 0 + with archive.open(info) as stream: + while chunk := stream.read(64 * 1024): + actual_size += len(chunk) + require(actual_size <= limit, f"part exceeded read limit: {info.filename}") + if chunks is not None: + chunks.append(chunk) + require(actual_size == info.file_size, f"size mismatch: {info.filename}") + return b"".join(chunks) if chunks is not None else actual_size + +def validate_xlsx_package(source): + """Bound every member before openpyxl sees this same seekable source.""" + source.seek(0, 2) + compressed_size = source.tell() + source.seek(0) + require(compressed_size <= MAX_ARCHIVE_BYTES, "compressed XLSX file size above limit") + try: + archive_context = zipfile.ZipFile(source) + except zipfile.BadZipFile as error: + raise ValueError("input is not a valid XLSX ZIP package") from error + with archive_context as archive: + infos = archive.infolist() + require(len(infos) <= MAX_MEMBERS, "archive member count above limit") + names = {info.filename for info in infos} + require(len(names) == len(infos), "duplicate archive member names are unsafe") + canonical_names = [canonical_member_name(info.filename) for info in infos] + require(len(set(canonical_names)) == len(canonical_names), + "archive member names collide case-insensitively") + require(all(not (info.flag_bits & 1) for info in infos), + "encrypted archive members are not supported") + require( + {"[Content_Types].xml", "_rels/.rels", "xl/workbook.xml"} <= names, + "required XLSX package parts are missing", + ) + require(sum(info.file_size for info in infos) <= MAX_TOTAL_UNCOMPRESSED, + "declared total uncompressed size above limit") + + content_types_info = next( + info for info in infos if info.filename == "[Content_Types].xml" + ) + require(content_types_info.file_size <= MAX_XML_PART, + "oversized XML part: [Content_Types].xml") + require( + content_types_info.file_size / max(content_types_info.compress_size, 1) + <= MAX_COMPRESSION_RATIO, + "suspicious compression ratio: [Content_Types].xml", + ) + content_types_blob = read_bounded_member( + archive, content_types_info, MAX_XML_PART, capture=True + ) + content_types_root = safe_xml_root(content_types_blob, "[Content_Types].xml") + require(content_types_root.tag == f"{{{CONTENT_TYPES_NS}}}Types", + "invalid content-types root") + default_types = {} + override_types = {} + for declaration in content_types_root: + if declaration.tag == f"{{{CONTENT_TYPES_NS}}}Default": + key = (declaration.get("Extension") or "").casefold() + target = default_types + elif declaration.tag == f"{{{CONTENT_TYPES_NS}}}Override": + part_name = declaration.get("PartName") or "" + require(part_name.startswith("/"), "invalid content-type part name") + key = canonical_member_name(part_name[1:]) + target = override_types + else: + continue + content_type = declaration.get("ContentType") or "" + require(key and content_type and key not in target, + "invalid or duplicate content-type declaration") + target[key] = content_type.partition(";")[0].strip().casefold() + + actual_total = len(content_types_blob) + worksheet_materialized_cells = 0 + worksheet_range_tokens = 0 + for info in infos: + require(info.file_size <= MAX_ENTRY, f"oversized part: {info.filename}") + require( + info.file_size / max(info.compress_size, 1) <= MAX_COMPRESSION_RATIO, + f"suspicious compression ratio: {info.filename}", + ) + extension = info.filename.rpartition(".")[2].casefold() + content_type = override_types.get( + canonical_member_name(info.filename), default_types.get(extension, "") + ) + is_xml = ( + info.filename.casefold().endswith((".xml", ".rels")) + or content_type in {"application/xml", "text/xml"} + or content_type.endswith("+xml") + ) + is_worksheet_xml = ( + is_xml + and info.filename.casefold().startswith("xl/worksheets/") + and content_type.endswith(".worksheet+xml") + ) + if is_xml: + xml_limit = MAX_WORKSHEET_XML if is_worksheet_xml else MAX_XML_PART + require(info.file_size <= xml_limit, + f"oversized XML part: {info.filename}") + if info.filename == "[Content_Types].xml": + blob = content_types_blob + else: + member_data = read_bounded_member( + archive, info, xml_limit if is_xml else MAX_ENTRY, + capture=is_xml and not is_worksheet_xml, + ) + actual_size = ( + len(member_data) if is_xml and not is_worksheet_xml else member_data + ) + actual_total += actual_size + require(actual_total <= MAX_TOTAL_UNCOMPRESSED, + "archive exceeded total read limit") + blob = member_data if is_xml and not is_worksheet_xml else None + if is_worksheet_xml: + try: + with archive.open(info) as stream: + for _, element in DefusedET.iterparse( + stream, events=("end",), forbid_dtd=True, + forbid_entities=True, forbid_external=True, + ): + worksheet_materialized_cells += worksheet_element_cell_cost( + element, info.filename + ) + worksheet_range_tokens += worksheet_element_range_token_cost( + element, info.filename + ) + require( + worksheet_materialized_cells + <= MAX_WORKSHEET_MATERIALIZED_CELLS, + "worksheet cell materialization budget exceeded", + ) + require( + worksheet_range_tokens <= MAX_WORKSHEET_RANGE_TOKENS, + "worksheet range-token budget exceeded", + ) + element.clear() + except Exception as error: + if isinstance(error, ValueError): + raise + raise ValueError( + f"unsafe or malformed XML part: {info.filename}" + ) from error + elif is_xml and info.filename != "[Content_Types].xml": + xml_root = safe_xml_root(blob, info.filename) + for element in xml_root.iter(): + worksheet_materialized_cells += worksheet_element_cell_cost( + element, info.filename + ) + worksheet_range_tokens += worksheet_element_range_token_cost( + element, info.filename + ) + require( + worksheet_materialized_cells + <= MAX_WORKSHEET_MATERIALIZED_CELLS, + "worksheet cell materialization budget exceeded", + ) + require( + worksheet_range_tokens <= MAX_WORKSHEET_RANGE_TOKENS, + "worksheet range-token budget exceeded", + ) + source.seek(0) + +@contextmanager +def validated_xlsx_source(path): + """Yield a private validated snapshot, isolated from later path mutation.""" + with Path(path).open("rb") as external_source, TemporaryFile() as source: + copied_size = 0 + while chunk := external_source.read(64 * 1024): + copied_size += len(chunk) + require(copied_size <= MAX_ARCHIVE_BYTES, + "compressed XLSX file size above limit") + source.write(chunk) + source.flush() + source.seek(0) + validate_xlsx_package(source) + source.seek(0) + yield source + +def load_validated_workbook(path, **options): + """Load a non-streaming workbook from the exact handle that passed preflight.""" + require(not options.get("read_only"), + "use open_validated_workbook for read_only=True") + with validated_xlsx_source(path) as source: + return openpyxl.load_workbook(source, **options) + +@contextmanager +def open_validated_workbook(path, **options): + """Keep the validated source alive until its openpyxl workbook is closed.""" + with validated_xlsx_source(path) as source: + workbook = openpyxl.load_workbook(source, **options) + try: + yield workbook + finally: + workbook.close() +``` + +Do not weaken these bounds in a route-specific copy. A task may choose smaller limits. If a +valid workbook exceeds a limit, report the limit and ask the user before raising it; never retry +an untrusted package without bounds. diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md new file mode 100644 index 0000000..5e3a7aa --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md @@ -0,0 +1,206 @@ +# Read / profile a workbook + +Copy the bounded loaders from [package.md](package.md) into the script before running this +route. The package preflight must finish before either openpyxl or the raw XML scan starts. + +```python +import posixpath +import zipfile +from openpyxl.utils import get_column_letter +from openpyxl.utils.cell import coordinate_to_tuple +from xml.etree import ElementTree as ET + +MAIN_NS = "http://schemas.openxmlformats.org/spreadsheetml/2006/main" +DOC_REL_NS = "http://schemas.openxmlformats.org/officeDocument/2006/relationships" +PKG_REL_NS = "http://schemas.openxmlformats.org/package/2006/relationships" +CELL_TAG = f"{{{MAIN_NS}}}c" +FORMULA_TAG = f"{{{MAIN_NS}}}f" +VALUE_TAG = f"{{{MAIN_NS}}}v" +INLINE_STRING_TAG = f"{{{MAIN_NS}}}is" +MAX_EXPLICIT_CELLS = 1_000_000 +MAX_PROFILE_RECTANGLE_CELLS = 100_000 + +def worksheet_part(archive, sheet_name): + workbook = ET.fromstring(archive.read("xl/workbook.xml")) + sheet = next( + item for item in workbook.iter(f"{{{MAIN_NS}}}sheet") + if item.attrib["name"] == sheet_name + ) + relationship_id = sheet.attrib[f"{{{DOC_REL_NS}}}id"] + relationships = ET.fromstring(archive.read("xl/_rels/workbook.xml.rels")) + target = next( + item.attrib["Target"] for item in relationships.iter(f"{{{PKG_REL_NS}}}Relationship") + if item.attrib["Id"] == relationship_id + ) + part = target.lstrip("/") if target.startswith("/") else posixpath.normpath( + posixpath.join("xl", target) + ) + require(part in archive.namelist(), f"worksheet part is missing: {part}") + return part + +def worksheet_xml_profile(archive, part): + """Discover physical logical cells without expanding their rectangular gaps.""" + min_row = min_column = max_row = max_column = None + explicit_cell_count = 0 + formula_count = missing_formula_count = 0 + missing_formula_samples = [] + coordinate = cell_type = value_text = formula_display = None + has_formula = value_seen = inline_string_seen = False + with archive.open(part) as source: + for event, element in ET.iterparse(source, events=("start", "end")): + if event == "start" and element.tag == CELL_TAG: + coordinate = element.attrib.get("r") + require(coordinate is not None, f"cell without a coordinate in {part}") + cell_type = element.attrib.get("t") + has_formula = value_seen = inline_string_seen = False + value_text = formula_display = None + elif event == "end" and coordinate is not None: + if element.tag == FORMULA_TAG: + has_formula = True + if element.text is not None: + formula_display = "=" + element.text + else: + details = ", ".join( + f"{key}={value!r}" for key, value in sorted(element.attrib.items()) + ) + formula_display = f"" + elif element.tag == VALUE_TAG: + value_seen = True + value_text = element.text + elif element.tag == INLINE_STRING_TAG and cell_type == "inlineStr": + inline_string_seen = True + elif element.tag == CELL_TAG: + explicit_cell_count += 1 + require(explicit_cell_count <= MAX_EXPLICIT_CELLS, + f"too many explicit worksheet cells in {part}") + row_index, column_index = coordinate_to_tuple(coordinate) + require(1 <= row_index <= 1_048_576 and 1 <= column_index <= 16_384, + f"cell coordinate outside XLSX limits: {coordinate}") + scalar_value = value_seen and ( + value_text not in (None, "") or cell_type == "str" + ) + populated = has_formula or scalar_value or inline_string_seen + if populated: + min_row = row_index if min_row is None else min(min_row, row_index) + min_column = (column_index if min_column is None + else min(min_column, column_index)) + max_row = row_index if max_row is None else max(max_row, row_index) + max_column = (column_index if max_column is None + else max(max_column, column_index)) + valid_cache = value_seen and ( + value_text not in (None, "") or cell_type == "str" + ) + if has_formula: + formula_count += 1 + if not valid_cache: + missing_formula_count += 1 + if len(missing_formula_samples) < 10: + missing_formula_samples.append((coordinate, formula_display)) + coordinate = None + element.clear() + elif event == "end": + element.clear() + if max_row is None: + bounds, extent, first_populated_row = None, "A1:A1", None + else: + bounds = (min_row, min_column, max_row, max_column) + extent = ( + f"{get_column_letter(min_column)}{min_row}:" + f"{get_column_letter(max_column)}{max_row}" + ) + first_populated_row = min_row + return { + "part": part, + "bounds": bounds, + "extent": extent, + "first_populated_row": first_populated_row, + "formula_count": formula_count, + "missing_formula_count": missing_formula_count, + "missing_formula_samples": missing_formula_samples, + "explicit_cell_count": explicit_cell_count, + } + +def bounded_sample_rows(worksheet, profile, *, max_cells=MAX_PROFILE_RECTANGLE_CELLS): + """Return a header plus at most five rows, rejecting expansion before iter_rows().""" + if profile["bounds"] is None: + return iter(()) + min_row, min_column, max_row, max_column = profile["bounds"] + sample_max_row = min(max_row, min_row + 5) + sample_cells = (sample_max_row - min_row + 1) * (max_column - min_column + 1) + require(sample_cells <= max_cells, + f"sample rectangle is too large: {worksheet.title} ({sample_cells} cells)") + return worksheet.iter_rows( + min_row=min_row, min_col=min_column, + max_row=sample_max_row, max_col=max_column, + values_only=True, + ) + +def worksheet_declared_dimension(worksheet): + """Return producer metadata without forcing an unsized read-only worksheet scan.""" + if worksheet.max_row is None or worksheet.max_column is None: + return None + return worksheet.calculate_dimension() + +input_path = "input.xlsx" +with validated_xlsx_source(input_path) as package_source: + value_wb = openpyxl.load_workbook(package_source, read_only=True, data_only=True) + try: + package_source.seek(0) + with zipfile.ZipFile(package_source) as archive: + profiles = { + sheet_name: worksheet_xml_profile( + archive, worksheet_part(archive, sheet_name) + ) + for sheet_name in value_wb.sheetnames + } + print("sheets:", value_wb.sheetnames) + + # Profile EVERY sheet by default; only narrow when the task names a specific sheet. + for sheet_name in value_wb.sheetnames: + value_ws = value_wb[sheet_name] + declared = worksheet_declared_dimension(value_ws) + profile = profiles[sheet_name] + discovered = profile["extent"] + value_ws.reset_dimensions() + if discovered != declared: + print(f"--- {sheet_name} --- declared {declared!r}; discovered real extent:") + print(f"--- {sheet_name} --- dims:", discovered, + "explicit cells:", profile["explicit_cell_count"]) + + if profile["bounds"] is None: + rows, header = iter(()), None + else: + rows = bounded_sample_rows(value_ws, profile) + header = next(rows, None) + print("header:", header) + for row in rows: + print(row) + + for coordinate, formula in profile["missing_formula_samples"]: + print("formula without cached value:", coordinate, formula) + print("formulas:", profile["formula_count"], + "without cached values:", profile["missing_formula_count"]) + finally: + value_wb.close() +``` + +## Rules + +- First pass always: sheet names, per-sheet dimensions, header row, 5 sample rows. Report + those before any analysis. Multi-sheet workbooks report every sheet - a profile that + silently covers only `sheetnames[0]` is incomplete. Begin the header/sample iterator at the + discovered first populated row; leading blank rows are not a header. +- `read_only=True` streams large files; you lose random access (`ws["B2"]` works but is slow + in read_only mode - iterate instead). +- A worksheet with no `` is unsized in read-only mode. Report its declared extent + as `None` and continue with the sparse XML profile; never call + `calculate_dimension(force=True)`, which scans the worksheet before the profile budgets apply. +- `data_only=True` gives cached values. A file saved by a library (never opened in Excel) + may have no cache for a formula. Use the sparse XML profile's `` and `` inventory to + detect those cells without a second rectangular workbook traversal; array, shared, and + data-table formula records may not contain ordinary formula text. +- Mixed-type columns: profile them (`set(type(v).__name__ for v in col)`) before converting; + a column that is mostly numbers with a few text cells is a data-quality finding, not noise. +- Never use an unbounded `iter_rows()` even with a later `break`: it can manufacture empty + cells before the caller regains control. Pass all four bounds from the sparse XML profile and + enforce a rectangle budget first. diff --git a/plugins/Hylouis233/document-skills/tests/docx_fixture.py b/plugins/Hylouis233/document-skills/tests/docx_fixture.py new file mode 100644 index 0000000..8124ba5 --- /dev/null +++ b/plugins/Hylouis233/document-skills/tests/docx_fixture.py @@ -0,0 +1,1674 @@ +# Minimal runnable fixtures for the snippets called out in review: one file per format. +# Each script is self-contained, writes only scratch files into the current directory, +# and exits non-zero on failed assertions. Run from any scratch directory: +# python pdf_fixture.py (deps: reportlab, pypdf, pymupdf) +# python pptx_fixture.py (deps: python-pptx) +# python xlsx_fixture.py (deps: openpyxl) +# python docx_fixture.py (deps: python-docx, pymupdf, soffice on PATH) +import copy +import os +import subprocess +import sys +import unicodedata +import zipfile +from contextlib import contextmanager +from pathlib import Path, PurePosixPath +from tempfile import TemporaryDirectory, TemporaryFile + +import fitz +from docx import Document +from docx.enum.style import WD_STYLE_TYPE +from docx.opc.constants import RELATIONSHIP_TYPE as RT +from docx.opc.packuri import PackURI +from docx.opc.part import Part +from docx.oxml import OxmlElement +from docx.oxml.ns import qn +from docx.table import Table +from docx.text.paragraph import Paragraph +from docx.text.run import Run +from lxml import etree + +failures = [] + + +def check(name, cond, extra=""): + print(("PASS " if cond else "FAIL ") + name + ((" :: " + str(extra)) if not cond and extra else "")) + if not cond: + failures.append(name) + + +# ---- review.md health check: external entities stay unresolved --------------- +safe_xml_parser = etree.XMLParser( + load_dtd=False, + resolve_entities=False, + no_network=True, + huge_tree=False, + recover=False, +) +hostile_xml = ( + b']>' + b'&xxe;' +) +parsed = etree.fromstring(hostile_xml, parser=safe_xml_parser) +check("DOCX XML parser leaves external entities unresolved", parsed.text is None and len(parsed) == 1) + +# ---- review.md health check rejects archive bombs before expanding parts ------- +MAX_ARCHIVE_BYTES = 200 * 1024 * 1024 +MAX_MEMBERS = 10_000 +MAX_XML_PART = 20 * 1024 * 1024 +MAX_ENTRY = 100 * 1024 * 1024 +MAX_TOTAL_UNCOMPRESSED = 500 * 1024 * 1024 +MAX_COMPRESSION_RATIO = 200 +CONTENT_TYPES_NAMESPACE = "http://schemas.openxmlformats.org/package/2006/content-types" +MAX_MEMBER_COMPONENT_BYTES = 255 +MAX_MEMBER_COMPONENT_UTF16_UNITS = 255 +MAX_MEMBER_PATH_BYTES = 1024 +MAX_MEMBER_PATH_UTF16_UNITS = 240 +MAX_MEMBER_COMPONENTS = 64 + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def xml_content_type(value): + media_type = (value or "").split(";", 1)[0].strip().casefold() + return media_type in {"application/xml", "text/xml"} or media_type.endswith("+xml") + + +def declared_xml_parts(archive, infos): + by_name = {info.filename: info for info in infos} + content_types_info = by_name.get("[Content_Types].xml") + require(content_types_info is not None, "missing [Content_Types].xml") + require(content_types_info.file_size <= MAX_XML_PART, + "oversized XML part: [Content_Types].xml") + with archive.open(content_types_info) as stream: + content_types_blob = stream.read(MAX_XML_PART + 1) + require(len(content_types_blob) <= MAX_XML_PART, + "oversized XML part: [Content_Types].xml") + root = etree.fromstring(content_types_blob, parser=safe_xml_parser) + require(root.tag == f"{{{CONTENT_TYPES_NAMESPACE}}}Types", + "invalid [Content_Types].xml root") + defaults = {} + overrides = {} + for child in root: + if child.tag == f"{{{CONTENT_TYPES_NAMESPACE}}}Default": + extension = (child.get("Extension") or "").casefold() + require(extension and extension not in defaults, + "invalid duplicate content-type default") + defaults[extension] = child.get("ContentType") or "" + elif child.tag == f"{{{CONTENT_TYPES_NAMESPACE}}}Override": + part_name = child.get("PartName") or "" + require(part_name.startswith("/") and part_name[1:] not in overrides, + "invalid duplicate content-type override") + overrides[part_name[1:]] = child.get("ContentType") or "" + xml_names = {"[Content_Types].xml"} + for info in infos: + suffix = info.filename.rsplit(".", 1)[1].casefold() if "." in info.filename else "" + content_type = overrides.get(info.filename, defaults.get(suffix, "")) + if (info.filename.casefold().endswith((".xml", ".rels")) + or xml_content_type(content_type)): + xml_names.add(info.filename) + return xml_names + + +def validate_docx_package(path): + require( + Path(path).stat().st_size <= MAX_ARCHIVE_BYTES, + "compressed DOCX file size above limit", + ) + with zipfile.ZipFile(path) as archive: + infos = archive.infolist() + require(len(infos) <= MAX_MEMBERS, "archive member count above limit") + names = {info.filename for info in infos} + require(len(names) == len(infos), "duplicate archive member names are unsafe") + require( + "[Content_Types].xml" in names and "word/document.xml" in names, + "required DOCX package parts are missing", + ) + require( + sum(info.file_size for info in infos) <= MAX_TOTAL_UNCOMPRESSED, + "declared archive size exceeds the review limit", + ) + xml_names = declared_xml_parts(archive, infos) + actual_total = 0 + for info in infos: + require(info.file_size <= MAX_ENTRY, f"oversized part: {info.filename}") + require( + info.file_size / max(info.compress_size, 1) <= MAX_COMPRESSION_RATIO, + f"suspicious compression ratio: {info.filename}", + ) + is_xml = info.filename in xml_names + if is_xml: + require(info.file_size <= MAX_XML_PART, f"oversized XML part: {info.filename}") + chunks = [] + actual_size = 0 + with archive.open(info) as stream: + while chunk := stream.read(64 * 1024): + actual_size += len(chunk) + actual_total += len(chunk) + require(actual_size <= MAX_ENTRY, f"part exceeded read limit: {info.filename}") + require( + actual_total <= MAX_TOTAL_UNCOMPRESSED, + "archive exceeded total read limit", + ) + if is_xml: + chunks.append(chunk) + require(actual_size == info.file_size, f"size mismatch: {info.filename}") + if is_xml: + etree.fromstring(b"".join(chunks), parser=safe_xml_parser) + + +@contextmanager +def validated_docx_source(path): + with Path(path).open("rb") as external_source, TemporaryFile() as source: + copied = 0 + while chunk := external_source.read(64 * 1024): + copied += len(chunk) + require(copied <= MAX_ARCHIVE_BYTES, "compressed DOCX file size above limit") + source.write(chunk) + source.seek(0) + with zipfile.ZipFile(source) as archive: + infos = archive.infolist() + require(len(infos) <= MAX_MEMBERS, "archive member count above limit") + names = {info.filename for info in infos} + require(len(names) == len(infos), "duplicate archive member names are unsafe") + require("[Content_Types].xml" in names and "word/document.xml" in names, + "missing required OPC members") + require(sum(info.file_size for info in infos) <= MAX_TOTAL_UNCOMPRESSED, + "declared total uncompressed size above limit") + xml_names = declared_xml_parts(archive, infos) + actual_total = 0 + for info in infos: + require(info.file_size <= MAX_ENTRY, f"oversized part: {info.filename}") + require(info.file_size / max(info.compress_size, 1) <= MAX_COMPRESSION_RATIO, + f"suspicious compression ratio: {info.filename}") + is_xml = info.filename in xml_names + if is_xml: + require(info.file_size <= MAX_XML_PART, + f"oversized XML part: {info.filename}") + chunks = [] + actual_size = 0 + with archive.open(info) as stream: + while chunk := stream.read(64 * 1024): + actual_size += len(chunk) + actual_total += len(chunk) + require(actual_size <= MAX_ENTRY, + f"part exceeded read limit: {info.filename}") + require(actual_total <= MAX_TOTAL_UNCOMPRESSED, + "archive exceeded total read limit") + if is_xml: + chunks.append(chunk) + require(actual_size == info.file_size, f"size mismatch: {info.filename}") + if is_xml: + etree.fromstring(b"".join(chunks), parser=safe_xml_parser) + source.seek(0) + yield source + + +def load_validated_docx(path, loader=Document): + with validated_docx_source(path) as source: + return loader(source) + + +health_doc = Document() +health_doc.add_paragraph("bounded health check") +health_doc.save("healthy.docx") +try: + validate_docx_package("healthy.docx") + healthy_package_passed = True +except Exception: + healthy_package_passed = False +check("bounded package health check accepts an ordinary DOCX", healthy_package_passed) + +with zipfile.ZipFile("compressed-bomb.docx", "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("[Content_Types].xml", "") + archive.writestr("word/document.xml", "" + (" " * 2_000_000) + "") +try: + validate_docx_package("compressed-bomb.docx") + archive_bomb_rejected = False +except (AssertionError, ValueError): + archive_bomb_rejected = True +check("suspicious compression ratio is rejected before XML expansion", archive_bomb_rejected) +check( + "archive safety checks remain active under optimized Python", + __debug__ or archive_bomb_rejected, +) +loader_calls = [] +try: + load_validated_docx( + "compressed-bomb.docx", + loader=lambda source: loader_calls.append(source), + ) + read_bomb_rejected = False +except ValueError: + read_bomb_rejected = True +check("structured DOCX read rejects a bomb before python-docx is called", + read_bomb_rejected and loader_calls == [], loader_calls) +loaded_healthy = load_validated_docx("healthy.docx") +check("structured DOCX read validates and loads one private snapshot", + loaded_healthy.paragraphs[0].text == "bounded health check") + + +def docx_with_extra_part(source_path, output_path, name, payload, content_type=None): + with zipfile.ZipFile(source_path) as source: + members = {info.filename: source.read(info) for info in source.infolist()} + if content_type is not None: + root = etree.fromstring(members["[Content_Types].xml"], parser=safe_xml_parser) + override = etree.SubElement(root, f"{{{CONTENT_TYPES_NAMESPACE}}}Override") + override.set("PartName", "/" + name) + override.set("ContentType", content_type) + members["[Content_Types].xml"] = etree.tostring(root, xml_declaration=True, + encoding="UTF-8") + members[name] = payload + with zipfile.ZipFile(output_path, "w", zipfile.ZIP_DEFLATED) as output: + for member_name, member_data in members.items(): + output.writestr(member_name, member_data) + + +docx_with_extra_part("healthy.docx", "uppercase-xml.docx", "custom/BROKEN.XML", b"") + archive.writestr("word/document.xml", "") + for member_index in range(MAX_MEMBERS - 1): + archive.writestr(f"word/zero-{member_index:05d}.bin", b"") +try: + validate_docx_package("many-members.docx") + many_members_rejected = False +except ValueError as exc: + many_members_rejected = str(exc) == "archive member count above limit" +check( + "member-count gate rejects 10,001 distinct zero-byte archive members before traversal", + many_members_rejected, +) + +# ---- read.md includes block/inline content controls and controlled tables ------- +MC_NAMESPACE = "http://schemas.openxmlformats.org/markup-compatibility/2006" +MC_ALTERNATE_CONTENT = f"{{{MC_NAMESPACE}}}AlternateContent" +MC_CHOICE = f"{{{MC_NAMESPACE}}}Choice" +MC_FALLBACK = f"{{{MC_NAMESPACE}}}Fallback" +SUPPORTED_MC_NAMESPACE_URIS = { + MC_NAMESPACE, + qn("w:p").split("}", 1)[0][1:], + qn("r:id").split("}", 1)[0][1:], +} + + +def alternate_content_branch(element): + fallback = None + for child in element.iterchildren(): + if child.tag == MC_CHOICE: + required_prefixes = (child.get("Requires") or "").split() + if required_prefixes and all( + child.nsmap.get(prefix) in SUPPORTED_MC_NAMESPACE_URIS + for prefix in required_prefixes + ): + return child + elif child.tag == MC_FALLBACK and fallback is None: + fallback = child + return fallback + + +def unresolved_alternate_content(element): + return { + "kind": "AlternateContent", + "requires": [ + (child.get("Requires") or "").split() + for child in element.iterchildren() + if child.tag == MC_CHOICE + ], + "reason": "no supported Choice and no Fallback", + } + + +def iter_effective_children(root): + for child in root.iterchildren(): + if child.tag != MC_ALTERNATE_CONTENT: + yield "element", child + continue + branch = alternate_content_branch(child) + if branch is None: + yield "unreadable", unresolved_alternate_content(child) + else: + yield from iter_effective_children(branch) + + +def iter_part_blocks(root, parent): + for child_kind, child in iter_effective_children(root): + if child_kind == "unreadable": + yield child_kind, child + continue + if child.tag == qn("w:p"): + yield "paragraph", Paragraph(child, parent) + elif child.tag == qn("w:tbl"): + yield "table", Table(child, parent) + elif child.tag == qn("w:altChunk"): + relationship_id = child.get(qn("r:id")) + part = getattr(parent, "part", None) + relationship = None if part is None else part.rels.get(relationship_id) + yield "unreadable", { + "kind": "altChunk", + "relationship_id": relationship_id, + "target": None if relationship is None else relationship.target_ref, + "content_type": None if relationship is None or relationship.is_external + else relationship.target_part.content_type, + } + else: + yield from iter_part_blocks(child, parent) + + +def iter_paragraph_items(paragraph): + def walk(element): + for child_kind, child in iter_effective_children(element): + if child_kind == "unreadable": + yield child_kind, child + continue + if child.tag == qn("w:r"): + yield "run", Run(child, paragraph) + elif child.tag != qn("w:p"): + yield from walk(child) + yield from walk(paragraph._p) + + +def iter_paragraph_runs(paragraph): + for kind, item in iter_paragraph_items(paragraph): + if kind == "run": + yield item + + +def legacy_symbol_record(symbol): + font = symbol.get(qn("w:font")) + character = symbol.get(qn("w:char")) + return f"[unreadable legacy symbol font={font!r} char={character!r}]" + + +def run_text(run): + pieces = [] + text_tags = { + qn("w:br"), qn("w:cr"), qn("w:noBreakHyphen"), qn("w:ptab"), + qn("w:t"), qn("w:tab"), qn("w:sym"), + } + for child_kind, child in iter_effective_children(run._r): + if child_kind == "unreadable": + pieces.append(f"[unreadable {child['kind']}: {child['reason']}]") + elif child.tag in text_tags: + pieces.append(legacy_symbol_record(child) if child.tag == qn("w:sym") else str(child)) + return "".join(pieces) + + +def paragraph_text(paragraph): + pieces = [] + for kind, item in iter_paragraph_items(paragraph): + pieces.append( + run_text(item) if kind == "run" + else f"[unreadable {item['kind']}: {item['reason']}]" + ) + return "".join(pieces) + + +def cell_paragraph_text(paragraph): + pieces = [] + for kind, item in iter_paragraph_items(paragraph): + if kind == "unreadable": + pieces.append(f"[unreadable {item['kind']}: {item['reason']}]") + continue + for child_kind, child in iter_effective_children(item._r): + if child_kind == "unreadable": + pieces.append(f"[unreadable {child['kind']}: {child['reason']}]") + elif child.tag == qn("w:t"): + pieces.append(child.text or "") + elif child.tag in (qn("w:tab"), qn("w:ptab")): + pieces.append("") + elif child.tag in (qn("w:br"), qn("w:cr")): + pieces.append("
") + elif child.tag == qn("w:noBreakHyphen"): + pieces.append("-") + elif child.tag == qn("w:sym"): + pieces.append(legacy_symbol_record(child)) + return "".join(pieces) + + +def tc_text(tc, parent): + paragraphs = [] + for kind, block in iter_part_blocks(tc, parent): + if kind == "paragraph": + paragraphs.append(cell_paragraph_text(block)) + return " / ".join(paragraphs) + + +def iter_content_control_children(root, target_tag): + for child_kind, child in iter_effective_children(root): + if child_kind == "unreadable": + raise ValueError( + f"unresolved AlternateContent while locating {target_tag}: {child}" + ) + if child.tag == target_tag: + yield child + elif child.tag == qn("w:sdt"): + for content in child.findall(qn("w:sdtContent")): + yield from iter_content_control_children(content, target_tag) + + +def table_content(table): + rows = [] + for row_element in iter_content_control_children(table._tbl, qn("w:tr")): + rendered_cells = [] + row_properties = row_element.find(qn("w:trPr")) + grid_before_node = None if row_properties is None else row_properties.find(qn("w:gridBefore")) + grid_after_node = None if row_properties is None else row_properties.find(qn("w:gridAfter")) + grid_before = 0 if grid_before_node is None else int(grid_before_node.get(qn("w:val"), "0")) + grid_after = 0 if grid_after_node is None else int(grid_after_node.get(qn("w:val"), "0")) + column = grid_before + for cell_element in iter_content_control_children(row_element, qn("w:tc")): + cell_properties = cell_element.find(qn("w:tcPr")) + grid_span = None if cell_properties is None else cell_properties.find(qn("w:gridSpan")) + colspan = 1 if grid_span is None else int(grid_span.get(qn("w:val"), "1")) + vertical = None if cell_properties is None else cell_properties.find(qn("w:vMerge")) + vertical_merge = None if vertical is None else vertical.get(qn("w:val"), "continue") + nested_tables = [] + unreadable = [] + for kind, block in iter_part_blocks(cell_element, table): + if kind == "table": + nested_tables.append(table_content(block)) + elif kind == "unreadable": + unreadable.append(block) + rendered_cells.append({ + "column": column, "colspan": colspan, + "vMerge": vertical_merge, + "text": tc_text(cell_element, table), + "tables": nested_tables, + "unreadable": unreadable, + }) + column += colspan + rows.append({ + "grid_before": grid_before, "cells": rendered_cells, "grid_after": grid_after, + }) + return rows + + +def wrap_in_sdt(element): + sdt = OxmlElement("w:sdt") + content = OxmlElement("w:sdtContent") + element.getparent().replace(element, sdt) + content.append(element) + sdt.append(content) + + +sdt_doc = Document() +sdt_doc.add_paragraph("direct paragraph") +sdt_paragraph = sdt_doc.add_paragraph("inside content control") +wrap_in_sdt(sdt_paragraph._p) +controlled_table = sdt_doc.add_table(rows=1, cols=1) +controlled_table.cell(0, 0).text = "table한" +row_properties = controlled_table.rows[0]._tr.get_or_add_trPr() +grid_before = OxmlElement("w:gridBefore") +grid_before.set(qn("w:val"), "1") +row_properties.append(grid_before) +grid_after = OxmlElement("w:gridAfter") +grid_after.set(qn("w:val"), "2") +row_properties.append(grid_after) +nested_table = controlled_table.cell(0, 0).add_table(rows=1, cols=1) +nested_table.cell(0, 0).text = "nested한" +inline_paragraph = sdt_doc.add_paragraph("before-") +inline_run = inline_paragraph.add_run("inline한") +inline_paragraph.add_run("-after") +wrap_in_sdt(inline_run._r) +inline_paragraph.add_run("-legacy-") +legacy_symbol_run = inline_paragraph.add_run() +legacy_symbol = OxmlElement("w:sym") +legacy_symbol.set(qn("w:font"), "Wingdings") +legacy_symbol.set(qn("w:char"), "F052") +legacy_symbol_run._r.append(legacy_symbol) +inline_paragraph.add_run("-visible") +chunk_part = Part( + PackURI("/word/altChunk1.html"), "text/html", + b"IMPORTED ALTCHUNK TEXT", sdt_doc.part.package, +) +chunk_relationship = sdt_doc.part.relate_to(chunk_part, RT.A_F_CHUNK) +alt_chunk = OxmlElement("w:altChunk") +alt_chunk.set(qn("r:id"), chunk_relationship) +sdt_doc.element.body.insert(len(sdt_doc.element.body) - 1, alt_chunk) +cell_alt_chunk = OxmlElement("w:altChunk") +cell_alt_chunk.set(qn("r:id"), chunk_relationship) +controlled_table.cell(0, 0)._tc.insert( + len(controlled_table.cell(0, 0)._tc) - 1, cell_alt_chunk +) +controlled_row_element = controlled_table.rows[0]._tr +controlled_cell_element = controlled_row_element.tc_lst[0] +wrap_in_sdt(controlled_cell_element) +wrap_in_sdt(controlled_row_element) +wrap_in_sdt(controlled_table._tbl) +sdt_doc.save("content-control.docx") +sdt_reopened = Document("content-control.docx") +check("doc.paragraphs omits block content-control text (negative control)", + "inside content control" not in [paragraph.text for paragraph in sdt_reopened.paragraphs]) +walked_blocks = list(iter_part_blocks(sdt_reopened.element.body, sdt_reopened)) +check("doc.tables omits a table wrapped by a block content control (negative control)", + len(sdt_reopened.tables) == 0) +check("block walker preserves document order across content controls", + [kind for kind, _ in walked_blocks] + == ["paragraph", "paragraph", "table", "paragraph", "unreadable"], walked_blocks) +walked_paragraphs = [block for kind, block in walked_blocks if kind == "paragraph"] +walked_text = [paragraph_text(paragraph) for paragraph in walked_paragraphs] +check("content-control traversal emits the nested paragraph", "inside content control" in walked_text, walked_text) +check("content-control traversal emits inline run text", + any("before-inline한-after" in text for text in walked_text), walked_text) +legacy_symbol_marker = "[unreadable legacy symbol font='Wingdings' char='F052']" +legacy_paragraph = next(paragraph for paragraph in walked_paragraphs + if "-legacy-" in paragraph_text(paragraph)) +check("Run.text silently omits visible legacy w:sym content (negative control)", + legacy_symbol_marker not in "".join(run.text for run in iter_paragraph_runs(legacy_paragraph))) +check("paragraph extraction preserves the legacy symbol's position, font, and character code", + f"-legacy-{legacy_symbol_marker}-visible" in paragraph_text(legacy_paragraph), + paragraph_text(legacy_paragraph)) +walked_tables = [block for kind, block in walked_blocks if kind == "table"] +check("Table.rows omits a row wrapped by row-level sdtContent (negative control)", + len(walked_tables) == 1 and len(walked_tables[0].rows) == 0) +physical_rows = list(iter_content_control_children(walked_tables[0]._tbl, qn("w:tr"))) +check("tr.tc_lst omits a cell wrapped by cell-level sdtContent (negative control)", + len(physical_rows) == 1 and len(physical_rows[0].tc_lst) == 0) +rendered_tables = [table_content(table) for table in walked_tables] +check("content-control traversal emits a wrapped table", + len(walked_tables) == 1 and "table한" in str(rendered_tables), rendered_tables) +check("table traversal emits the sdt-wrapped physical row and cell", + len(rendered_tables[0]) == 1 and len(rendered_tables[0][0]["cells"]) == 1, + rendered_tables) +all_emitted = walked_text + [str(table) for table in rendered_tables] +check("table text is emitted exactly once, not again as prose", + sum(item.count("table한") for item in all_emitted) == 1, all_emitted) +check("nested-table text is emitted exactly once", + sum(item.count("nested한") for item in all_emitted) == 1, all_emitted) +check("nonuniform table rows preserve leading/trailing grid omissions", + rendered_tables[0][0]["grid_before"] == 1 + and rendered_tables[0][0]["cells"][0]["column"] == 1 + and rendered_tables[0][0]["grid_after"] == 2, + rendered_tables[0]) +unreadable_parts = [block for kind, block in walked_blocks if kind == "unreadable"] +expected_alt_chunk = { + "kind": "altChunk", + "relationship_id": chunk_relationship, + "target": "altChunk1.html", + "content_type": "text/html", +} +check("altChunk content is reported instead of silently omitted", + unreadable_parts == [expected_alt_chunk], unreadable_parts) +cell_unreadable = rendered_tables[0][0]["cells"][0]["unreadable"] +check("altChunk content inside a table cell is also reported", + cell_unreadable == [expected_alt_chunk], rendered_tables) +check("altChunk payload is not misrepresented as extracted paragraph text", + all("IMPORTED ALTCHUNK TEXT" not in text for text in walked_text), walked_text) + + +# ---- read.md chooses exactly one mc:AlternateContent branch --------------------- +W14_NAMESPACE = "http://schemas.microsoft.com/office/word/2010/wordml" +W_NAMESPACE = qn("w:p").split("}", 1)[0][1:] + + +def wml_run(text): + run = OxmlElement("w:r") + text_element = OxmlElement("w:t") + text_element.text = text + run.append(text_element) + return run + + +def wml_paragraph(text): + paragraph = OxmlElement("w:p") + paragraph.append(wml_run(text)) + return paragraph + + +def wml_table(text): + source = Document() + table = source.add_table(rows=1, cols=1) + table.cell(0, 0).text = text + return copy.deepcopy(table._tbl) + + +def alternate_content(requires, choice_elements, fallback_elements=None): + element = etree.Element( + MC_ALTERNATE_CONTENT, + nsmap={"mc": MC_NAMESPACE, "w14": W14_NAMESPACE, "w": W_NAMESPACE}, + ) + choice = etree.SubElement(element, MC_CHOICE) + choice.set("Requires", requires) + for child in choice_elements: + choice.append(child) + if fallback_elements is not None: + fallback = etree.SubElement(element, MC_FALLBACK) + for child in fallback_elements: + fallback.append(child) + return element + + +mc_doc = Document() +mc_doc.element.body.insert( + len(mc_doc.element.body) - 1, + alternate_content( + "w14", + [wml_paragraph("UNSUPPORTED-CHOICE-BLOCK"), wml_table("UNSUPPORTED-CHOICE-TABLE")], + [wml_paragraph("FALLBACK-BLOCK"), wml_table("FALLBACK-TABLE")], + ), +) +mc_doc.element.body.insert( + len(mc_doc.element.body) - 1, + alternate_content( + "w", + [wml_paragraph("SUPPORTED-CHOICE-BLOCK")], + [wml_paragraph("UNSELECTED-FALLBACK-BLOCK")], + ), +) +mc_doc.element.body.insert( + len(mc_doc.element.body) - 1, + alternate_content("w14", [wml_paragraph("UNRESOLVED-HIDDEN-BLOCK")]), +) +mc_inline = mc_doc.add_paragraph("INLINE-BEFORE-") +mc_inline._p.append( + alternate_content( + "w14", [wml_run("UNSUPPORTED-CHOICE-INLINE")], [wml_run("FALLBACK-INLINE")] + ) +) +mc_inline.add_run("-INLINE-AFTER") +mc_doc.save("alternate-content.docx") +mc_reopened = Document("alternate-content.docx") +mc_blocks = list(iter_part_blocks(mc_reopened.element.body, mc_reopened)) +mc_paragraph_texts = [paragraph_text(block) for kind, block in mc_blocks if kind == "paragraph"] +mc_tables = [table_content(block) for kind, block in mc_blocks if kind == "table"] +mc_unreadable = [block for kind, block in mc_blocks if kind == "unreadable"] +mc_emitted = mc_paragraph_texts + [str(table) for table in mc_tables] +check("unsupported AlternateContent Choice selects only its Fallback paragraph and table", + any("FALLBACK-BLOCK" in text for text in mc_emitted) + and any("FALLBACK-TABLE" in text for text in mc_emitted) + and all("UNSUPPORTED-CHOICE" not in text for text in mc_emitted), + mc_emitted) +check("supported AlternateContent Choice wins over its Fallback", + any("SUPPORTED-CHOICE-BLOCK" in text for text in mc_emitted) + and all("UNSELECTED-FALLBACK-BLOCK" not in text for text in mc_emitted), + mc_emitted) +check("inline AlternateContent contributes exactly one selected run", + "INLINE-BEFORE-FALLBACK-INLINE-INLINE-AFTER" in mc_paragraph_texts + and all("UNSUPPORTED-CHOICE-INLINE" not in text for text in mc_paragraph_texts), + mc_paragraph_texts) +check("AlternateContent without a usable branch is explicitly unreadable", + mc_unreadable == [{ + "kind": "AlternateContent", + "requires": [["w14"]], + "reason": "no supported Choice and no Fallback", + }], mc_unreadable) +check("unresolved AlternateContent payload is not misrepresented as extracted text", + all("UNRESOLVED-HIDDEN-BLOCK" not in text for text in mc_emitted), mc_emitted) + +# Per-run glyph validation must not let a different referenced font hide a missing glyph. +fixture_cmaps = {"CJK Face": {ord("漢")}, "Latin Face": {ord("A")}} +assigned_runs = [("CJK Face", "漢"), ("Latin Face", "漢")] +pooled_passes = all(any(ord(ch) in cmap for cmap in fixture_cmaps.values()) + for _, text in assigned_runs for ch in text) +per_run_missing = [(face, ch) for face, text in assigned_runs for ch in text + if ord(ch) not in fixture_cmaps[face]] +check("pooled cmap is proven unsafe (negative control)", pooled_passes) +check("per-run cmap check identifies the actual missing glyph", + per_run_missing == [("Latin Face", "漢")], per_run_missing) + + +def require_clean_cjk_glyph_audit(unresolved, missing): + if unresolved: + raise ValueError(f"font files not resolved per run: {unresolved}") + if missing: + raise ValueError(f"glyph missing from the run's effective font: {missing}") + + +try: + require_clean_cjk_glyph_audit([(0, "漢", "Unresolved Face")], []) + unresolved_face_rejected = False +except ValueError: + unresolved_face_rejected = True +check( + "mandatory CJK audit rejects unresolved font files under optimized Python", + unresolved_face_rejected, +) + +try: + require_clean_cjk_glyph_audit([], [(0, "漢", "Missing Glyph Face")]) + missing_glyph_rejected = False +except ValueError: + missing_glyph_rejected = True +check( + "mandatory CJK audit rejects missing glyphs under optimized Python", + missing_glyph_rejected, +) + +def font_slot(character): + codepoint = ord(character) + uses_east_asian_slot = ( + 0x1100 <= codepoint <= 0x11FF + or 0x2F00 <= codepoint <= 0x9FFF + or 0xA000 <= codepoint <= 0xA4CF + or 0xA960 <= codepoint <= 0xA97F + or 0xAC00 <= codepoint <= 0xD7FF + or 0xF900 <= codepoint <= 0xFAFF + or 0xFE30 <= codepoint <= 0xFE6F + or 0xFF00 <= codepoint <= 0xFFEF + or 0x20000 <= codepoint <= 0x3134F + or 0x31350 <= codepoint <= 0x33479 + ) + return "eastAsia" if uses_east_asian_slot else ("ascii" if codepoint < 128 else "hAnsi") + + +check("Han ideographs use the east-Asian font slot", font_slot("漢") == "eastAsia") +check("Hangul syllables use the east-Asian font slot", font_slot("한") == "eastAsia") +check("Hangul Jamo use the east-Asian font slot", font_slot("ᄒ") == "eastAsia") +check("Yi syllables use the east-Asian font slot", font_slot("ꀀ") == "eastAsia") +check("CJK compatibility forms use the east-Asian font slot", font_slot("︰") == "eastAsia") +check("Unicode 17 Han Extension H start uses the east-Asian font slot", + font_slot(chr(0x31350)) == "eastAsia") +check("Unicode 17 Han Extension J end uses the east-Asian font slot", + font_slot(chr(0x33479)) == "eastAsia") +check("codepoint after the Unicode 17 Han ranges stays in hAnsi", + font_slot(chr(0x3347A)) == "hAnsi") +check("ASCII text keeps the ascii font slot", font_slot("A") == "ascii") + +# Header/footer parts have no `.document`; effective styles close over the document. +font_doc = Document() +normal_style = font_doc.styles["Normal"] +normal_fonts = normal_style.element.get_or_add_rPr().get_or_add_rFonts() +normal_fonts.set(qn("w:ascii"), "Latin Face") +normal_fonts.set(qn("w:hAnsi"), "Latin Face") +normal_fonts.set(qn("w:eastAsia"), "CJK Face") +header_run = font_doc.sections[0].header.paragraphs[0].add_run("页眉") +footer_run = font_doc.sections[0].footer.paragraphs[0].add_run("页脚") + + +def face_from_rpr(rpr, slot): + if rpr is None: + return None + rfonts = rpr.find(qn("w:rFonts")) + if rfonts is None: + return None + theme_attribute = { + "ascii": "asciiTheme", "hAnsi": "hAnsiTheme", + "eastAsia": "eastAsiaTheme", "cs": "cstheme", + }[slot] + theme_token = rfonts.get(qn("w:" + theme_attribute)) + if theme_token is not None: + literal = rfonts.get(qn("w:" + slot)) + raise LookupError( + f"unresolved direct {theme_attribute}={theme_token!r}" + + (f" alongside {slot}={literal!r}" if literal is not None else "") + ) + return rfonts.get(qn("w:" + slot)) + + +def style_faces(style, slot): + while style is not None: + face = face_from_rpr(style.element.find(qn("w:rPr")), slot) + if face: + yield face + style = style.base_style + + +def table_style_faces(run, slot): + element = run._r + while element is not None and element.tag != qn("w:tbl"): + element = element.getparent() + if element is None: + return [] + table_properties = element.find(qn("w:tblPr")) + table_style = None if table_properties is None else table_properties.find(qn("w:tblStyle")) + style_id = None if table_style is None else table_style.get(qn("w:val")) + style = None if not style_id else font_doc.styles.get_by_id(style_id, WD_STYLE_TYPE.TABLE) + faces = [] + while style is not None: + rprs = [style.element.find(qn("w:rPr"))] + rprs.extend(region.find(qn("w:rPr")) + for region in style.element.findall(qn("w:tblStylePr"))) + faces.extend(face for rpr in rprs if (face := face_from_rpr(rpr, slot))) + style = style.base_style + return list(dict.fromkeys(faces)) + + +def effective_face(run, slot): + direct = face_from_rpr(run._r.find(qn("w:rPr")), slot) + if direct: + return direct + if faces := table_style_faces(run, slot): + raise LookupError(f"conditional table style face requires rendered resolution: {faces}") + for style in (run.style, run._parent.style, normal_style): + if face := next(style_faces(style, slot), None): + return face + raise LookupError(slot) + + +theme_attribute_by_slot = { + "ascii": "asciiTheme", "hAnsi": "hAnsiTheme", + "eastAsia": "eastAsiaTheme", "cs": "cstheme", +} +for slot, theme_attribute in theme_attribute_by_slot.items(): + themed_run = font_doc.add_paragraph().add_run("漢" if slot == "eastAsia" else "A") + themed_fonts = themed_run._r.get_or_add_rPr().get_or_add_rFonts() + themed_fonts.set(qn("w:" + theme_attribute), "majorEastAsia" if slot == "eastAsia" else "minorAscii") + try: + effective_face(themed_run, slot) + direct_theme_rejected = False + except LookupError: + direct_theme_rejected = True + check(f"direct {theme_attribute} does not fall through to an inherited literal face", + direct_theme_rejected) + +ambiguous_theme_run = font_doc.add_paragraph().add_run("漢") +ambiguous_fonts = ambiguous_theme_run._r.get_or_add_rPr().get_or_add_rFonts() +ambiguous_fonts.set(qn("w:eastAsia"), "Literal Face") +ambiguous_fonts.set(qn("w:eastAsiaTheme"), "majorEastAsia") +try: + effective_face(ambiguous_theme_run, "eastAsia") + literal_and_theme_rejected = False +except LookupError: + literal_and_theme_rejected = True +check("same-slot literal plus theme font declaration fails closed", + literal_and_theme_rejected) + + +check("header part has no document back-reference (negative control)", + not hasattr(header_run.part, "document")) +check("header runs resolve Normal from the owning document", + effective_face(header_run, "eastAsia") == "CJK Face") +check("footer runs resolve Normal from the owning document", + effective_face(footer_run, "eastAsia") == "CJK Face") +inline_paragraph_reopened = next( + paragraph for paragraph in walked_paragraphs if "before-" in paragraph_text(paragraph) +) +check("inline content-control runs are included in glyph traversal", + any(run.text == "inline한" for run in iter_paragraph_runs(inline_paragraph_reopened))) + +hyperlink_paragraph = font_doc.add_paragraph() +hyperlink = OxmlElement("w:hyperlink") +hyperlink.set(qn("w:anchor"), "fixture-target") +hyperlink_run = OxmlElement("w:r") +hyperlink_text = OxmlElement("w:t") +hyperlink_text.text = "链接漢" +hyperlink_run.append(hyperlink_text) +hyperlink.append(hyperlink_run) +hyperlink_paragraph._p.append(hyperlink) +walked_hyperlink_runs = list(iter_paragraph_runs(hyperlink_paragraph)) +check("Paragraph.runs omits hyperlink runs (negative control)", + all(run.text != "链接漢" for run in hyperlink_paragraph.runs)) +check("glyph traversal includes CJK text nested in a hyperlink", + [run.text for run in walked_hyperlink_runs] == ["链接漢"], + [run.text for run in walked_hyperlink_runs]) +check("hyperlink CJK text resolves through the east-Asian font slot", + effective_face(walked_hyperlink_runs[0], "eastAsia") == "CJK Face") + +conditional_style = font_doc.styles.add_style("Conditional CJK Table", WD_STYLE_TYPE.TABLE) +first_row = OxmlElement("w:tblStylePr") +first_row.set(qn("w:type"), "firstRow") +conditional_rpr = OxmlElement("w:rPr") +conditional_fonts = OxmlElement("w:rFonts") +conditional_fonts.set(qn("w:eastAsia"), "Conditional CJK Face") +conditional_rpr.append(conditional_fonts) +first_row.append(conditional_rpr) +conditional_style.element.append(first_row) +font_table = font_doc.add_table(rows=1, cols=1) +font_table.style = conditional_style +conditional_run = font_table.cell(0, 0).paragraphs[0].add_run("漢") +try: + effective_face(conditional_run, "eastAsia") + conditional_style_rejected = False +except LookupError: + conditional_style_rejected = True +check("glyph audit fails closed for conditional table-style fonts", + conditional_style_rejected) + +# ---- edit.md guarded cross-run replacement ------------------------------------ +SAFE_RUN_CHILDREN = { + qn("w:rPr"), qn("w:t"), qn("w:tab"), qn("w:cr"), +} +MODELED_PARAGRAPH_CHILDREN = {qn("w:pPr"), qn("w:r")} + + +def unsafe_run_content(run): + unsafe = [] + for child in run._r: + ordinary_break = child.tag == qn("w:br") and dict(child.attrib) in ( + {}, {qn("w:type"): "textWrapping"}, + ) + if child.tag not in SAFE_RUN_CHILDREN and not ordinary_break: + unsafe.append(child.tag.rsplit("}", 1)[-1]) + return unsafe + + +def replace_across_runs(paragraph, old, new): + if not old: + raise ValueError("old must not be empty") + unmodeled = [ + child.tag.rsplit("}", 1)[-1] + for child in paragraph._p + if child.tag not in MODELED_PARAGRAPH_CHILDREN + ] + if unmodeled: + raise ValueError(f"paragraph contains unmodeled inline containers: {unmodeled}") + runs = list(paragraph.runs) + text = "".join(run.text for run in runs) + starts = [] + position = 0 + while (start := text.find(old, position)) != -1: + starts.append(start) + position = start + len(old) + spans = [] + position = 0 + for index, run in enumerate(runs): + end = position + len(run.text) + if end > position: + spans.append((index, position, end)) + position = end + matches = [] + for start in starts: + end = start + len(old) + first, first_start, _ = next(s for s in spans if s[1] <= start < s[2]) + last, last_start, _ = next(s for s in spans if s[1] < end <= s[2]) + matches.append((start, end, first, first_start, last, last_start)) + affected_indexes = { + index + for _, _, first, _, last, _ in matches + for index in range(first, last + 1) + } + unsafe = {} + for index in affected_indexes: + if children := unsafe_run_content(runs[index]): + unsafe[index] = children + if unsafe: + raise ValueError(f"matched runs contain non-text content: {unsafe}") + for start, end, first, first_start, last, last_start in reversed(matches): + prefix = runs[first].text[:start - first_start] + suffix = runs[last].text[end - last_start:] + if first == last: + runs[first].text = prefix + new + suffix + else: + runs[first].text = prefix + new + for index in range(first + 1, last): + runs[index].text = "" + runs[last].text = suffix + return len(starts) + + +safe_doc = Document() +safe_paragraph = safe_doc.add_paragraph() +safe_first = safe_paragraph.add_run("T") +safe_first.bold = True +safe_paragraph.add_run("B") +safe_paragraph.add_run("D") +check("text-only cross-run match is replaced", replace_across_runs(safe_paragraph, "TBD", "Done") == 1) +check("safe replacement keeps first-run formatting", safe_paragraph.text == "Done" and safe_first.bold) + +icon = fitz.Pixmap(fitz.csRGB, fitz.IRect(0, 0, 4, 4), False) +icon.clear_with(200) +icon.save("inline-icon.png") +guard_doc = Document() +guard_paragraph = guard_doc.add_paragraph() +guard_run = guard_paragraph.add_run("TBD") +guard_run.add_picture("inline-icon.png") +try: + replace_across_runs(guard_paragraph, "TBD", "Done") + rejected_drawing_run = False +except ValueError: + rejected_drawing_run = True +check("replacement rejects a matched run containing a drawing", rejected_drawing_run) +check( + "rejected replacement leaves text and drawing untouched", + guard_run.text == "TBD" and len(guard_run._r.findall(qn("w:drawing"))) == 1, +) + +clear_doc = Document() +clear_paragraph = clear_doc.add_paragraph() +clear_run = clear_paragraph.add_run("TBD") +clear_break = OxmlElement("w:br") +clear_break.set(qn("w:type"), "textWrapping") +clear_break.set(qn("w:clear"), "left") +clear_run._r.append(clear_break) +clear_before = etree.tostring(clear_run._r) +try: + replace_across_runs(clear_paragraph, "TBD", "done") + clear_break_rejected = False +except ValueError: + clear_break_rejected = True +check("replacement rejects a wrapping break with clear semantics", clear_break_rejected) +check("rejected clear-break replacement is atomic", + etree.tostring(clear_run._r) == clear_before) + +container_doc = Document() +container_paragraph = container_doc.add_paragraph() +container_paragraph.add_run("T") +container_hyperlink = OxmlElement("w:hyperlink") +container_hyperlink.set(qn("w:anchor"), "fixture-target") +container_link_run = OxmlElement("w:r") +container_link_text = OxmlElement("w:t") +container_link_text.text = "link" +container_link_run.append(container_link_text) +container_hyperlink.append(container_link_run) +container_paragraph._p.append(container_hyperlink) +container_paragraph.add_run("BD") +container_before = etree.tostring(container_paragraph._p) +check("Paragraph.runs can manufacture a false match across a hyperlink (negative control)", + "".join(run.text for run in container_paragraph.runs) == "TBD" + and paragraph_text(container_paragraph) == "TlinkBD") +try: + replace_across_runs(container_paragraph, "TBD", "Done") + rejected_inline_container = False +except ValueError: + rejected_inline_container = True +check("replacement rejects unmodeled inline containers before matching", + rejected_inline_container) +check("rejected inline-container replacement is atomic", + etree.tostring(container_paragraph._p) == container_before + and paragraph_text(container_paragraph) == "TlinkBD") + + +def list_number_num_id(doc): + """The numId that the ListNumber style binds to in this document part.""" + styles = doc.part.element.body.getparent() # document.xml root; styles live in another part + styles_part = doc.part.part_related_by(RT.STYLES) + for style in styles_part.element.findall(qn("w:style")): + if style.get(qn("w:styleId")) == "ListNumber": + numPr = style.find(qn("w:pPr") + "/" + qn("w:numPr")) + if numPr is not None: + return int(numPr.find(qn("w:numId")).get(qn("w:val"))) + raise LookupError("ListNumber style has no numPr") + + +def new_restart_num_id(doc, base_num_id): + """Clone base_num_id with a startOverride so the next list restarts at 1.""" + numbering = doc.part.part_related_by(RT.NUMBERING).element + source = next( + n for n in numbering.findall(qn("w:num")) + if n.get(qn("w:numId")) == str(base_num_id) + ) + clone = copy.deepcopy(source) + new_id = max(int(n.get(qn("w:numId"))) for n in numbering.findall(qn("w:num"))) + 1 + clone.set(qn("w:numId"), str(new_id)) + level_zero_overrides = [ + item for item in clone.findall(qn("w:lvlOverride")) + if item.get(qn("w:ilvl")) == "0" + ] + if len(level_zero_overrides) > 1: + raise ValueError("base numbering has duplicate level-zero overrides") + if level_zero_overrides: + override = level_zero_overrides[0] + for old_start in override.findall(qn("w:startOverride")): + override.remove(old_start) + else: + override = OxmlElement("w:lvlOverride") + override.set(qn("w:ilvl"), "0") + clone.append(override) + start = OxmlElement("w:startOverride") + start.set(qn("w:val"), "1") + override.insert(0, start) + numbering.append(clone) + return new_id + + +def numbered_paragraph(doc, text, num_id): + p = doc.add_paragraph(text, style="List Number") + pPr = p._p.get_or_add_pPr() + numPr = OxmlElement("w:numPr") + ilvl = OxmlElement("w:ilvl") + ilvl.set(qn("w:val"), "0") + numId = OxmlElement("w:numId") + numId.set(qn("w:val"), str(num_id)) + numPr.append(ilvl) + numPr.append(numId) + pPr.append(numPr) + return p + + +# ---- document A: naive reuse of List Number (second list continues) ----------- +doc_a = Document() +doc_a.add_paragraph("First list") +for item in ("one", "two", "three"): + doc_a.add_paragraph(item, style="List Number") +doc_a.add_paragraph("Second list, same style") +for item in ("four", "five", "six"): + doc_a.add_paragraph(item, style="List Number") +doc_a.save("continuing.docx") + +# ---- document B: second list restarts via cloned numbering definition --------- +doc_b = Document() +doc_b.add_paragraph("First list") +for item in ("one", "two", "three"): + doc_b.add_paragraph(item, style="List Number") +doc_b.add_paragraph("Second list, restarted") +base_id = list_number_num_id(doc_b) +numbering_b = doc_b.part.part_related_by(RT.NUMBERING).element +base_entry = next(n for n in numbering_b.findall(qn("w:num")) + if n.get(qn("w:numId")) == str(base_id)) +preexisting_override = OxmlElement("w:lvlOverride") +preexisting_override.set(qn("w:ilvl"), "0") +preexisting_start = OxmlElement("w:startOverride") +preexisting_start.set(qn("w:val"), "7") +preexisting_override.append(preexisting_start) +base_entry.append(preexisting_override) +restart_id = new_restart_num_id(doc_b, base_id) +for item in ("four", "five", "six"): + numbered_paragraph(doc_b, item, restart_id) +doc_b.save("restarted.docx") + +# ---- structural assertions ------------------------------------------------------ +numbering_b = doc_b.part.part_related_by(RT.NUMBERING).element +nums_b = numbering_b.findall(qn("w:num")) +check("cloned num entry exists", any(n.get(qn("w:numId")) == str(restart_id) for n in nums_b)) +clone_entry = next(n for n in nums_b if n.get(qn("w:numId")) == str(restart_id)) +level_zero_overrides = [item for item in clone_entry.findall(qn("w:lvlOverride")) + if item.get(qn("w:ilvl")) == "0"] +start_overrides = ([] if not level_zero_overrides else + level_zero_overrides[0].findall(qn("w:startOverride"))) +check("clone replaces a preexisting ilvl=0 override without creating a duplicate", + len(level_zero_overrides) == 1 and len(start_overrides) == 1 + and start_overrides[0].get(qn("w:val")) == "1") +check("cloning leaves the base numbering override unchanged", + preexisting_start.get(qn("w:val")) == "7") +second_list_num_ids = [ + p._p.find(qn("w:pPr") + "/" + qn("w:numPr") + "/" + qn("w:numId")).get(qn("w:val")) + for p in doc_b.paragraphs[-3:] +] +check("restarted paragraphs reference the cloned numId", set(second_list_num_ids) == {str(restart_id)}, second_list_num_ids) + +# ---- rendered proof via LibreOffice -------------------------------------------- +subprocess.run( + ["soffice", "--headless", "--convert-to", "pdf", "--outdir", ".", "continuing.docx", "restarted.docx"], + check=True, capture_output=True, timeout=180, +) + + +def second_list_numbers(pdf_path): + text = " ".join(page.get_text() for page in fitz.open(pdf_path)) + # take the rendered numbers in front of the second list's item words + return [text.split(word)[0].split()[-1] for word in ("four", "five", "six")] + + +cont = second_list_numbers("continuing.pdf") +restart = second_list_numbers("restarted.pdf") +print("continuing.docx renders second list as:", cont) +print("restarted.docx renders second list as:", restart) +check("plain style reuse continues the sequence (negative control)", cont == ["4.", "5.", "6."], cont) +check("cloned definition restarts the second list at 1", restart == ["1.", "2.", "3."], restart) + +# ---- scenes.md snippet: keep_table_together for the signature block ------------- +doc_c = Document() +doc_c.add_heading("Contract", level=1) +# fill most of page 1 so a tall signature table would otherwise straddle the page break +for _ in range(24): + doc_c.add_paragraph("Filler paragraph to push the signature block toward the page break. " * 3) +sig = doc_c.add_table(rows=4, cols=2) +labels = [("甲方（盖章）", "乙方（盖章）"), ("签字", "签字"), ("日期", "日期"), ("备注", "备注")] +for r, pair in enumerate(labels): + sig.cell(r, 0).text, sig.cell(r, 1).text = pair + + +def keep_table_together(table): + for row in table.rows: + trPr = row._tr.get_or_add_trPr() + if trPr.find(qn("w:cantSplit")) is None: + trPr.append(OxmlElement("w:cantSplit")) # a row never splits mid-row + for row in table.rows[:-1]: + for cell in row.cells: + for par in cell.paragraphs: + par.paragraph_format.keep_with_next = True # row sticks to the next row + + +keep_table_together(sig) +doc_c.save("signature.docx") +reopened = Document("signature.docx") +check("every signature row carries cantSplit", + all(row._tr.find(qn("w:trPr")) is not None and row._tr.find(qn("w:trPr")).find(qn("w:cantSplit")) is not None + for row in reopened.tables[0].rows)) +subprocess.run( + ["soffice", "--headless", "--convert-to", "pdf", "--outdir", ".", "signature.docx"], + check=True, capture_output=True, timeout=180, +) +pages_with_labels = [ + page.number for page in fitz.open("signature.pdf") + if "甲方（盖章）" in page.get_text() and "备注" in page.get_text() +] +check("rendered signature table stays on one page", len(pages_with_labels) == 1, pages_with_labels) + + +# ---- read.md snippet: merged-cell spans survive extraction ----------------------- +def table_matrix(table): + rows = [] + for row in table.rows: + cells = [] + for tc in row._tr.tc_lst: + tc_pr = tc.find(qn("w:tcPr")) + grid_span = tc_pr.find(qn("w:gridSpan")) if tc_pr is not None else None + v_merge = tc_pr.find(qn("w:vMerge")) if tc_pr is not None else None + note = "" + if grid_span is not None: + note += "(span {})".format(grid_span.get(qn("w:val"))) + if v_merge is not None: + note += "(vmerge start)" if v_merge.get(qn("w:val")) == "restart" else "(vmerge cont.)" + text = "".join(node.text or "" for node in tc.iter(qn("w:t"))) + cells.append(text + note if note else text) + rows.append(cells) + return rows + +merge_doc = Document() +merge_table = merge_doc.add_table(rows=3, cols=3) +merge_table.cell(0, 0).text = "HEAD" # set only the origin; merging concatenates texts +merge_table.cell(0, 0).merge(merge_table.cell(0, 1)) +merge_table.cell(1, 2).text = "TOP" +merge_table.cell(2, 2).text = "BOTTOM" +merge_table.cell(1, 2).merge(merge_table.cell(2, 2)) +merge_doc.save("merged-cells.docx") +merged_reopened = Document("merged-cells.docx") +merged_table = merged_reopened.tables[0] +naive_lengths = [len(row.cells) for row in merged_table.rows] +naive_row0 = [cell.text for cell in merged_table.rows[0].cells] +matrix = table_matrix(merged_table) +check("naive row.cells expands merges to full grid width (negative control)", + naive_lengths == [3, 3, 3] and naive_row0[0] == naive_row0[1] == "HEAD", + (naive_lengths, naive_row0)) +check("matrix keeps one entry per real tc", + [len(row) for row in matrix] == [2, 3, 3], matrix) +check("horizontal merge is annotated", "(span 2)" in matrix[0][0], matrix[0]) +check("vertical merge start and continuation are annotated", + any("(vmerge start)" in cell for cell in matrix[1]) and + any("(vmerge cont.)" in cell for cell in matrix[2]), matrix) + +# ---- cjk.md snippet: Hangul routes through the East Asian slot ------------------- +def font_slot(character): + codepoint = ord(character) + return "eastAsia" if ( + 0x1100 <= codepoint <= 0x11FF or 0x2E80 <= codepoint <= 0x9FFF + or 0x3130 <= codepoint <= 0x318F or 0xA960 <= codepoint <= 0xA97F + or 0xAC00 <= codepoint <= 0xD7FF or 0xF900 <= codepoint <= 0xFAFF + or 0x20000 <= codepoint <= 0x3134F + or 0x31350 <= codepoint <= 0x33479 + ) else ("ascii" if codepoint < 128 else "hAnsi") + +check("Hangul syllables use the East Asian slot", font_slot("한") == "eastAsia") +check("Hangul jamo use the East Asian slot", font_slot("ᄀ") == "eastAsia") +check("compatibility jamo use the East Asian slot", font_slot("ㄱ") == "eastAsia") +check("Hangul extended-A uses the East Asian slot", font_slot(chr(0xA960)) == "eastAsia") +check("Hangul extended-B uses the East Asian slot", font_slot(chr(0xD7B0)) == "eastAsia") +check("U+2E80 CJK radical uses the East Asian slot", font_slot(chr(0x2E80)) == "eastAsia") +check("Unicode 17 supplementary Han endpoints use the East Asian slot", + font_slot(chr(0x31350)) == font_slot(chr(0x33479)) == "eastAsia") +check("the codepoint after Unicode 17 Han remains hAnsi", + font_slot(chr(0x3347A)) == "hAnsi") +check("Latin stays in the ascii slot", font_slot("A") == "ascii") +check("non-CJK fullwidth-range-adjacent Latin-1 stays hAnsi", font_slot("é") == "hAnsi") + + + +# ---- read.md tc_text: paragraph boundaries survive cell extraction --------------- +def tc_text(tc): + paragraphs = [] + for p in tc.iter(qn("w:p")): + pieces = [] + for node in p.iter(): + if node.tag == qn("w:t"): + pieces.append(node.text or "") + elif node.tag == qn("w:tab"): + pieces.append("") + elif node.tag in (qn("w:br"), qn("w:cr")): + pieces.append("
") + elif node.tag == qn("w:noBreakHyphen"): + pieces.append("-") + paragraphs.append("".join(pieces)) + return " / ".join(paragraphs) + +cells_doc = Document() +cells_table = cells_doc.add_table(rows=1, cols=1) +cell = cells_table.cell(0, 0) +cell.paragraphs[0].text = "First" +cell.add_paragraph("Second") +run_with_tab = cell.paragraphs[0].add_run("") +tab_element = OxmlElement("w:tab") +run_with_tab._r.append(tab_element) +run_with_tab.add_text("after tab") +hyphen_run = cell.paragraphs[0].add_run("non") +hyphen_run._r.append(OxmlElement("w:noBreakHyphen")) +hyphen_run.add_text("breaking") +cells_doc.save("cell-paragraphs.docx") +cells_reopened = Document("cell-paragraphs.docx") +cells_tc = cells_reopened.tables[0].rows[0]._tr.tc_lst[0] +joined_raw = "".join(node.text or "" for node in cells_tc.iter(qn("w:t"))) +extracted = tc_text(cells_tc) +check("raw w:t joining concatenates paragraphs (negative control)", + "Firstafter tab" in joined_raw, joined_raw) +check("tc_text preserves the paragraph boundary", " / Second" in extracted, extracted) +check("tc_text keeps tabs visible", "after tab" in extracted, extracted) +check("tc_text preserves a table-cell nonbreaking hyphen", + "non-breaking" in extracted, extracted) + + +# ---- edit.md raw OOXML repack: every input gets a fresh extraction tree --------- +media_doc = Document() +media_doc.add_paragraph("first document") +media_doc.add_picture("inline-icon.png") +media_doc.save("repack-with-media.docx") +plain_doc = Document() +plain_doc.add_paragraph("second document") +plain_doc.save("repack-without-media.docx") + + +WINDOWS_DEVICE_NAMES = { + "con", "prn", "aux", "nul", "conin$", "conout$", + *(f"com{suffix}" for suffix in "123456789¹²³"), + *(f"lpt{suffix}" for suffix in "123456789¹²³"), +} + + +def extraction_key(name): + is_directory = name.endswith("/") + path = name[:-1] if is_directory else name + require(path and not name.startswith("/") and "\\" not in name, + f"non-canonical archive member path: {name}") + parts = path.split("/") + require(len(parts) <= MAX_MEMBER_COMPONENTS, + "archive member depth exceeds portable extraction limit") + require( + all( + part not in {"", ".", ".."} + and not any(character in ':<>|"?*' for character in part) + and not any(ord(character) < 32 for character in part) + and not part.endswith((".", " ")) + and unicodedata.normalize("NFC", part) == part + and part.partition(".")[0].rstrip(" ").casefold() not in WINDOWS_DEVICE_NAMES + for part in parts + ), + f"non-canonical archive member path: {name}", + ) + require( + all( + len(part.encode("utf-8")) <= MAX_MEMBER_COMPONENT_BYTES + and len(part.encode("utf-16-le")) // 2 <= MAX_MEMBER_COMPONENT_UTF16_UNITS + for part in parts + ), + "archive member component exceeds portable extraction limit", + ) + require(len(path.encode("utf-8")) <= MAX_MEMBER_PATH_BYTES + and len(path.encode("utf-16-le")) // 2 <= MAX_MEMBER_PATH_UTF16_UNITS, + "archive member path exceeds portable extraction limit") + canonical = PurePosixPath(*parts).as_posix() + require(canonical == path, f"non-canonical archive member path: {name}") + key = tuple(unicodedata.normalize("NFC", part.casefold()) for part in parts) + return key, tuple(parts), is_directory + + +def validate_extraction_paths(infos): + root = {"children": {}, "member": False, "file": False, "spelling": None} + for info in infos: + key, spellings, is_directory = extraction_key(info.filename) + node = root + for normalized, spelling in zip(key, spellings): + require(not node["file"], + "archive file and directory paths collide after extraction") + child = node["children"].get(normalized) + if child is None: + child = { + "children": {}, "member": False, "file": False, + "spelling": spelling, + } + node["children"][normalized] = child + else: + require(child["spelling"] == spelling, + "archive member path spelling collides after extraction") + node = child + require(not node["member"], "archive member paths collide after extraction") + require(is_directory or not node["children"], + "archive file and directory paths collide after extraction") + node["member"] = True + node["file"] = not is_directory + + +def validate_docx_archive_bounds(archive): + require(os.fstat(archive.fp.fileno()).st_size <= MAX_ARCHIVE_BYTES, + "compressed DOCX file size above limit") + infos = archive.infolist() + require(len(infos) <= MAX_MEMBERS, "archive member count above limit") + names = {info.filename for info in infos} + require(len(names) == len(infos), "duplicate archive member names are unsafe") + validate_extraction_paths(infos) + require("[Content_Types].xml" in names and "word/document.xml" in names, + "required DOCX package parts are missing") + require(sum(info.file_size for info in infos) <= MAX_TOTAL_UNCOMPRESSED, + "declared archive size exceeds the edit limit") + actual_total = 0 + for info in infos: + require(info.file_size <= MAX_ENTRY, f"oversized part: {info.filename}") + require(info.file_size / max(info.compress_size, 1) <= MAX_COMPRESSION_RATIO, + f"suspicious compression ratio: {info.filename}") + if info.filename.endswith((".xml", ".rels")): + require(info.file_size <= MAX_XML_PART, f"oversized XML part: {info.filename}") + actual_size = 0 + with archive.open(info) as stream: + while chunk := stream.read(64 * 1024): + actual_size += len(chunk) + actual_total += len(chunk) + require(actual_size <= MAX_ENTRY, f"part exceeded read limit: {info.filename}") + require(actual_total <= MAX_TOTAL_UNCOMPRESSED, + "archive exceeded total read limit") + require(actual_size == info.file_size, f"size mismatch: {info.filename}") + + +def repack_tree(source, output, extraction_root): + with zipfile.ZipFile(source) as archive: + validate_docx_archive_bounds(archive) + archive.extractall(extraction_root) + content_types = Path(extraction_root) / "[Content_Types].xml" + files = sorted( + (path for path in Path(extraction_root).rglob("*") + if path.is_file() and path != content_types), + key=lambda path: path.relative_to(extraction_root).as_posix(), + ) + with zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED, strict_timestamps=False) as archive: + archive.write(content_types, "[Content_Types].xml") + for path in files: + archive.write(path, path.relative_to(extraction_root).as_posix()) + + +bomb_extract_root = Path("bomb-extract") +bomb_extract_root.mkdir() +try: + repack_tree("compressed-bomb.docx", "bomb-output.docx", bomb_extract_root) + edit_bomb_rejected_before_extract = False +except ValueError as error: + edit_bomb_rejected_before_extract = ( + str(error) == "suspicious compression ratio: word/document.xml" + and + not any(bomb_extract_root.iterdir()) and not Path("bomb-output.docx").exists() + ) +check("Tier 2 edit rejects an archive bomb before extracting any member", + edit_bomb_rejected_before_extract) +check("Tier 2 pre-extract bounds remain active under optimized Python", + __debug__ or edit_bomb_rejected_before_extract) + +with zipfile.ZipFile("noncanonical-member.docx", "w", zipfile.ZIP_STORED) as archive: + archive.writestr("[Content_Types].xml", "") + archive.writestr("word/document.xml", "") + archive.writestr("../word/document.xml", "") +with TemporaryDirectory(prefix="docx-noncanonical-") as scratch: + noncanonical_root = Path(scratch) + try: + repack_tree("noncanonical-member.docx", "noncanonical-output.docx", + noncanonical_root) + noncanonical_rejected = False + except ValueError as error: + noncanonical_rejected = ( + str(error) == "non-canonical archive member path: ../word/document.xml" + and not any(noncanonical_root.iterdir()) + and not Path("noncanonical-output.docx").exists() + ) +check("Tier 2 rejects traversal aliases before they can overwrite a validated part", + noncanonical_rejected) +check("non-canonical member rejection remains active under optimized Python", + __debug__ or noncanonical_rejected) + +with zipfile.ZipFile("normalized-collision.docx", "w", zipfile.ZIP_STORED) as archive: + archive.writestr("[Content_Types].xml", "") + archive.writestr("word/document.xml", "") + archive.writestr("custom", b"file") + archive.writestr("custom/", b"") +with TemporaryDirectory(prefix="docx-collision-") as scratch: + collision_root = Path(scratch) + try: + repack_tree("normalized-collision.docx", "collision-output.docx", collision_root) + normalized_collision_rejected = False + except ValueError as error: + normalized_collision_rejected = ( + str(error) == "archive member paths collide after extraction" + and not any(collision_root.iterdir()) + and not Path("collision-output.docx").exists() + ) +check("Tier 2 rejects distinct names that normalize to one extraction path", + normalized_collision_rejected) + +with zipfile.ZipFile("prefix-collision.docx", "w", zipfile.ZIP_STORED) as archive: + archive.writestr("[Content_Types].xml", "") + archive.writestr("word/document.xml", "") + archive.writestr("custom", b"file") + archive.writestr("custom/child.bin", b"child") +with TemporaryDirectory(prefix="docx-prefix-collision-") as scratch: + prefix_collision_root = Path(scratch) + try: + repack_tree("prefix-collision.docx", "prefix-collision-output.docx", + prefix_collision_root) + prefix_collision_rejected = False + except ValueError as error: + prefix_collision_rejected = ( + str(error) == "archive file and directory paths collide after extraction" + and not any(prefix_collision_root.iterdir()) + and not Path("prefix-collision-output.docx").exists() + ) +check("Tier 2 rejects a file path that is also an extracted directory prefix", + prefix_collision_rejected) + +with zipfile.ZipFile("case-collision.docx", "w", zipfile.ZIP_STORED) as archive: + archive.writestr("[Content_Types].xml", "") + archive.writestr("word/document.xml", "") + archive.writestr("WORD/document.xml", "") +with TemporaryDirectory(prefix="docx-case-collision-") as scratch: + case_collision_root = Path(scratch) + try: + repack_tree("case-collision.docx", "case-collision-output.docx", + case_collision_root) + case_collision_rejected = False + except ValueError as error: + case_collision_rejected = ( + str(error) == "archive member path spelling collides after extraction" + and not any(case_collision_root.iterdir()) + and not Path("case-collision-output.docx").exists() + ) +check("Tier 2 rejects case aliases before Windows extraction can overwrite a part", + case_collision_rejected) + +with zipfile.ZipFile("prefix-spelling-collision.docx", "w", zipfile.ZIP_STORED) as archive: + archive.writestr("[Content_Types].xml", "") + archive.writestr("word/document.xml", "") + archive.writestr("WORD/styles.xml", "") +with TemporaryDirectory(prefix="docx-prefix-spelling-") as scratch: + prefix_spelling_root = Path(scratch) + try: + repack_tree("prefix-spelling-collision.docx", "prefix-spelling-output.docx", + prefix_spelling_root) + prefix_spelling_rejected = False + except ValueError as error: + prefix_spelling_rejected = ( + str(error) == "archive member path spelling collides after extraction" + and not any(prefix_spelling_root.iterdir()) + and not Path("prefix-spelling-output.docx").exists() + ) +check("Tier 2 rejects case aliases in a shared directory prefix", + prefix_spelling_rejected) + +with zipfile.ZipFile("overlong-member.docx", "w", zipfile.ZIP_STORED) as archive: + archive.writestr("[Content_Types].xml", "") + archive.writestr("word/document.xml", "") + archive.writestr("word/" + "a" * 256, b"oversized component") +with TemporaryDirectory(prefix="docx-overlong-member-") as scratch: + overlong_root = Path(scratch) + try: + repack_tree("overlong-member.docx", "overlong-output.docx", overlong_root) + overlong_member_rejected = False + except ValueError as error: + overlong_member_rejected = ( + str(error) == "archive member component exceeds portable extraction limit" + and not any(overlong_root.iterdir()) + and not Path("overlong-output.docx").exists() + ) +check("Tier 2 rejects an unportable component before partial Windows extraction", + overlong_member_rejected) + +portable_path_limits_rejected = [] +for unsafe_name in ( + "/".join(["a"] * (MAX_MEMBER_COMPONENTS + 1)), + "/".join(["a" * 80] * 4), +): + try: + extraction_key(unsafe_name) + except ValueError: + portable_path_limits_rejected.append(unsafe_name) +check("portable extraction bounds total path length and component depth", + len(portable_path_limits_rejected) == 2) + +portable_name_rejections = [] +for unsafe_name in ( + "/word/document.xml", "word\\document.xml", "word/con.xml", "word/COM¹.xml", + "word/LPT².txt", "word/CONIN$.xml", "word/CONOUT$.xml", "word/NUL .xml", + "word/trailing. ", "word/cafe\u0301.xml", "word/control\x01.xml", +): + try: + extraction_key(unsafe_name) + except ValueError: + portable_name_rejections.append(unsafe_name) +check("portable extraction rejects absolute, alternate, device, non-NFC, and control names", + len(portable_name_rejections) == 11, portable_name_rejections) + +with zipfile.ZipFile("malformed-for-repair.docx", "w", zipfile.ZIP_STORED) as archive: + archive.writestr("[Content_Types].xml", "") + archive.writestr("word/document.xml", " { + const skill = await readFile(path.join(testsRoot, "..", "skills", "pdf", "SKILL.md"), "utf8"); + assert.match(skill, /ReportLab does not produce a tagged PDF\/UA\s+document/); + assert.match(skill, /do not describe ordinary ReportLab output as accessible or screen-reader-ready/); + assert.match(skill, /stop and report the limitation/); + assert.match(skill, /accessible DOCX or HTML/); +}); + +test("document-skills Python fixtures", { timeout: 12 * 60_000 }, async (context) => { + for (const name of ["xlsx", "pptx", "pdf", "docx"]) { + await context.test(name + " fixture", { timeout: 3 * 60_000 }, async () => { + const scratch = await mkdtemp(path.join(os.tmpdir(), "document-skills-" + name + "-")); + try { + await execFileAsync(python, [path.join(testsRoot, name + "_fixture.py")], { + cwd: scratch, + encoding: "utf8", + maxBuffer: 8 * 1024 * 1024, + timeout: 170_000, + windowsHide: true, + }); + } catch (error) { + const output = String(error.stdout ?? "") + "\n" + String(error.stderr ?? ""); + assert.fail(name + " fixture failed:\n" + output.slice(-16_000)); + } finally { + await rm(scratch, { recursive: true, force: true }); + } + }); + } +}); diff --git a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py new file mode 100644 index 0000000..0581a20 --- /dev/null +++ b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py @@ -0,0 +1,2097 @@ +# Minimal runnable fixtures for the snippets called out in review: one file per format. +# Each script is self-contained, writes only scratch files into the current directory, +# and exits non-zero on failed assertions. Run from any scratch directory: +# python pdf_fixture.py (deps: reportlab, pypdf, pymupdf) +# python pptx_fixture.py (deps: python-pptx) +# python xlsx_fixture.py (deps: openpyxl) +# python docx_fixture.py (deps: python-docx, pymupdf, soffice on PATH) +import math +import os +import re +import sys + +import fitz +import pypdf +import reportlab +from pypdf.generic import ( + ArrayObject, DecodedStreamObject, DictionaryObject, FloatObject, + NameObject, NullObject, NumberObject, +) +from reportlab.lib.pagesizes import A4 +from reportlab.pdfbase import pdfmetrics +from reportlab.pdfbase.ttfonts import TTFont +from reportlab.pdfgen import canvas + +failures = [] + + +def check(name, cond, extra=""): + print(("PASS " if cond else "FAIL ") + name + ((" :: " + str(extra)) if not cond and extra else "")) + if not cond: + failures.append(name) + + +def open_pdf(path, password=None): + reader = pypdf.PdfReader(path) + if reader.is_encrypted: + if not password or reader.decrypt(password) == 0: + raise RuntimeError(f"valid password required for {path}") + return reader + + +def authenticate_for_extraction(document, password=None): + """Mirror extract.md: try the valid blank user password before requiring env input.""" + if document.needs_pass and document.authenticate("") <= 0: + if not password or document.authenticate(password) <= 0: + raise RuntimeError("set PDF_PASSWORD to the correct password before extracting") + + +class BlankPasswordDocument: + needs_pass = True + + def __init__(self): + self.attempts = [] + + def authenticate(self, password): + self.attempts.append(password) + return 2 if password == "" else 0 + + +blank_password_stub = BlankPasswordDocument() +authenticate_for_extraction(blank_password_stub) +check("PDF extraction authenticates the blank password before requiring environment input", + blank_password_stub.attempts == [""], blank_password_stub.attempts) + + +# ---- build a 2-page A4 PDF with one AcroForm text field on page 1 ------------- +c = canvas.Canvas("form.pdf", pagesize=A4) +c.setFont("Helvetica", 16) +c.drawString(72, 780, "Application form") +c.acroForm.textfield(name="applicant_name", x=72, y=740, width=260, height=20, borderWidth=0) +c.showPage() +c.setFont("Helvetica", 16) +c.drawString(72, 780, "Second page content") +c.save() + +# Current PyMuPDF opens a permission-encrypted PDF with an empty user password directly +# (`needs_pass == 0`). Keep this real-file proof in addition to the branch-order stub above. +blank_user_writer = pypdf.PdfWriter() +blank_user_writer.append(pypdf.PdfReader("form.pdf")) +blank_user_writer.encrypt(user_password="", owner_password="fixture-owner") +with open("blank-user-password.pdf", "wb") as output: + blank_user_writer.write(output) +blank_user_doc = fitz.open("blank-user-password.pdf") +authenticate_for_extraction(blank_user_doc) +check("blank-user permission-encrypted PDF extracts without PDF_PASSWORD", + "Application form" in blank_user_doc[0].get_text(), blank_user_doc.needs_pass) +blank_user_doc.close() + +check("ordinary ReportLab output is not a tagged PDF/UA document", + "/StructTreeRoot" not in pypdf.PdfReader("form.pdf").trailer["/Root"]) + +# ---- SKILL.md postcheck snippet: width/height pairs from the 4-coordinate box -- +r = pypdf.PdfReader("form.pdf") +page_sizes = [ + (float(page.mediabox.width), float(page.mediabox.height)) + for page in r.pages +] +A4_TOLERANCE = 0.5 +check( + "mediabox width/height is A4 on every page", + all(abs(w - 595.2755) < A4_TOLERANCE and abs(h - 841.8897) < A4_TOLERANCE for w, h in page_sizes), + page_sizes, +) + + +def verify_page_count(reader, expected_page_count): + page_count = len(reader.pages) + if page_count != expected_page_count: + raise ValueError(f"expected {expected_page_count} pages, got {page_count}") + return page_count + + +check("postcheck accepts the requested page count", verify_page_count(r, 2) == 2) +try: + verify_page_count(r, 1) + page_count_mismatch = "" +except ValueError as exc: + page_count_mismatch = str(exc) +check("postcheck rejects a page-count mismatch even under python -O", + page_count_mismatch == "expected 1 pages, got 2", page_count_mismatch) + +# ---- inspect.md distinguishes referenced-only and embedded fonts --------------- +pdfmetrics.registerFont(TTFont("FixtureVera", os.path.join( + os.path.dirname(reportlab.__file__), "fonts", "Vera.ttf", +))) +font_canvas = canvas.Canvas("font-inventory.pdf", pagesize=A4) +font_canvas.setFont("Helvetica", 12) # standard PDF face, normally referenced only +font_canvas.drawString(72, 780, "Referenced Helvetica") +font_canvas.setFont("FixtureVera", 12) +font_canvas.drawString(72, 750, "Embedded Vera") +font_canvas.save() + +DIRECT_CHARPROC_NAME = re.compile( + r"/(?:#[0-9A-Fa-f]{2}|[^#\s()<>\[\]{}/%])+" +) +DIRECT_CHARPROC_REFERENCE = re.compile(r"\s+([1-9]\d*)\s+\d+\s+R") + + +def indirect_xref(value): + match = re.fullmatch(r"\s*([1-9]\d*)\s+\d+\s+R\s*", value or "") + return int(match.group(1)) if match else None + + +def direct_charproc_xrefs(value): + value = (value or "").strip() + if not value.startswith("<<") or not value.endswith(">>"): + return "uninspectable", [] + body = value[2:-2] + position = 0 + references = [] + while position < len(body): + while position < len(body) and body[position].isspace(): + position += 1 + if position == len(body): + break + name_match = DIRECT_CHARPROC_NAME.match(body, position) + if name_match is None: + return "uninspectable", [] + reference_match = DIRECT_CHARPROC_REFERENCE.match(body, name_match.end()) + if reference_match is None: + return "malformed", [] + references.append(int(reference_match.group(1))) + position = reference_match.end() + return ("parsed", references) if references else ("malformed", []) + + +def type3_charprocs_status(document, xref, font_type): + if font_type.replace(" ", "").casefold() != "type3": + return None + if xref <= 0: + return "uninspectable" + try: + charprocs_type, charprocs_value = document.xref_get_key(xref, "CharProcs") + except (RuntimeError, ValueError): + return "uninspectable" + if charprocs_type == "dict": + parse_status, glyph_xrefs = direct_charproc_xrefs(charprocs_value) + if parse_status != "parsed": + return parse_status + elif charprocs_type == "xref": + dictionary_xref = indirect_xref(charprocs_value) + if dictionary_xref is None: + return "malformed" + try: + dictionary_source = document.xref_object(dictionary_xref, compressed=True) + dictionary_is_stream = document.xref_is_stream(dictionary_xref) + except (RuntimeError, ValueError): + return "uninspectable" + dictionary_source = dictionary_source.strip() + if (dictionary_is_stream or not dictionary_source.startswith("<<") + or not dictionary_source.endswith(">>")): + return "malformed" + parse_status, glyph_xrefs = direct_charproc_xrefs(dictionary_source) + if parse_status != "parsed": + return parse_status + else: + return "malformed" + for glyph_xref in glyph_xrefs: + try: + if not document.xref_is_stream(glyph_xref): + return "malformed" + except (RuntimeError, ValueError): + return "uninspectable" + return "verified" + + +def font_inventory(document, page): + fonts = [] + for entry in page.get_fonts(full=True): + xref, extension, font_type, base_name, resource_name, encoding = entry[:6] + is_type3 = font_type.replace(" ", "").casefold() == "type3" + charprocs_status = type3_charprocs_status(document, xref, font_type) + embedded_bytes = 0 + if xref > 0: + try: + embedded_bytes = len((document.extract_font(xref)[3] or b"")) + except (RuntimeError, ValueError): + embedded_bytes = 0 + if is_type3: + embedded = ( + True if charprocs_status == "verified" else + (False if charprocs_status == "malformed" else None) + ) + self_contained = embedded + program_source = "type3-charprocs" + else: + embedded = embedded_bytes > 0 + self_contained = None + program_source = "font-file" if embedded_bytes else None + fonts.append({ + "base_name": base_name, "type": font_type, + "embedded": embedded, + "embedded_bytes": embedded_bytes, + "self_contained": self_contained, + "charprocs_status": charprocs_status, + "program_source": program_source, + }) + return fonts + +font_doc = fitz.open("font-inventory.pdf") +fonts = font_inventory(font_doc, font_doc[0]) +check("font inventory labels the embedded TrueType face", + any("Vera" in item["base_name"] and item["embedded"] is True for item in fonts), fonts) +check("font inventory labels referenced-only Helvetica as non-embedded", + any("Helvetica" in item["base_name"] and item["embedded"] is False for item in fonts), fonts) + +# A Type3 font stores each glyph as PDF content in /CharProcs rather than as an +# extractable conventional font file. + + +def write_type3_pdf(path, *, charprocs_kind="indirect-dict", direct_font=False): + writer = pypdf.PdfWriter() + page = writer.add_blank_page(width=612, height=792) + glyph = DecodedStreamObject() + glyph.set_data(b"500 0 0 0 500 700 d1 0 0 500 700 re f") + glyph_ref = writer._add_object(glyph) + glyph_name = NameObject( + "/A/B" if charprocs_kind in {"direct-escaped-name", "indirect-escaped-name"} + else "/A" + ) + valid_charprocs = DictionaryObject({glyph_name: glyph_ref}) + if charprocs_kind == "indirect-dict": + charprocs = writer._add_object(valid_charprocs) + elif charprocs_kind == "indirect-escaped-name": + charprocs = writer._add_object(valid_charprocs) + elif charprocs_kind == "direct-dict": + charprocs = valid_charprocs + elif charprocs_kind == "direct-escaped-name": + charprocs = valid_charprocs + elif charprocs_kind == "empty-direct": + charprocs = DictionaryObject() + elif charprocs_kind == "direct-number-entry": + charprocs = DictionaryObject({NameObject("/A"): NumberObject(42)}) + elif charprocs_kind == "indirect-array": + charprocs = writer._add_object(ArrayObject([glyph_ref])) + elif charprocs_kind == "indirect-number": + charprocs = writer._add_object(NumberObject(42)) + elif charprocs_kind == "indirect-null": + charprocs = writer._add_object(NullObject()) + elif charprocs_kind == "indirect-bad-glyph": + bad_glyph_ref = writer._add_object(NumberObject(42)) + charprocs = writer._add_object(DictionaryObject({ + NameObject("/A"): bad_glyph_ref, + })) + else: + raise ValueError(f"unsupported CharProcs fixture kind {charprocs_kind}") + encoding = DictionaryObject({ + NameObject("/Type"): NameObject("/Encoding"), + NameObject("/Differences"): ArrayObject([ + NumberObject(65), glyph_name, + ]), + }) + font = DictionaryObject({ + NameObject("/Type"): NameObject("/Font"), + NameObject("/Subtype"): NameObject("/Type3"), + NameObject("/Name"): NameObject("/FType3"), + NameObject("/FontBBox"): ArrayObject([ + NumberObject(0), NumberObject(0), NumberObject(500), NumberObject(700), + ]), + NameObject("/FontMatrix"): ArrayObject([ + FloatObject(0.001), NumberObject(0), NumberObject(0), + FloatObject(0.001), NumberObject(0), NumberObject(0), + ]), + NameObject("/CharProcs"): charprocs, + NameObject("/Encoding"): encoding, + NameObject("/FirstChar"): NumberObject(65), + NameObject("/LastChar"): NumberObject(65), + NameObject("/Widths"): ArrayObject([NumberObject(500)]), + NameObject("/Resources"): DictionaryObject(), + }) + font_object = font if direct_font else writer._add_object(font) + page[NameObject("/Resources")] = DictionaryObject({ + NameObject("/Font"): DictionaryObject({NameObject("/FType3"): font_object}), + }) + content = DecodedStreamObject() + content.set_data(b"BT /FType3 72 Tf 72 700 Td (A) Tj ET") + page[NameObject("/Contents")] = writer._add_object(content) + with open(path, "wb") as output: + writer.write(output) + + +def type3_fixture_record(path): + document = fitz.open(path) + record = next(item for item in font_inventory(document, document[0]) + if item["type"].replace(" ", "").casefold() == "type3") + return document, record + + +write_type3_pdf("type3-font.pdf") +type3_doc = fitz.open("type3-font.pdf") +type3_entry = next(item for item in type3_doc[0].get_fonts(full=True) + if item[2].replace(" ", "").casefold() == "type3") +type3_extracted_bytes = type3_doc.extract_font(type3_entry[0])[3] or b"" +type3_fonts = font_inventory(type3_doc, type3_doc[0]) +type3_record = next(item for item in type3_fonts + if item["type"].replace(" ", "").casefold() == "type3") +check("Type3 negative control has no conventional extractable font-file bytes", + type3_extracted_bytes == b"", len(type3_extracted_bytes)) +check("Type3 glyph program renders and extracts its encoded character", + type3_doc[0].get_text().strip() == "A" + and min(type3_doc[0].get_pixmap(alpha=False).samples) < 250) +check("font inventory classifies Type3 CharProcs as self-contained content", + type3_record["embedded"] and type3_record["self_contained"] + and type3_record["embedded_bytes"] == 0 + and type3_record["charprocs_status"] == "verified" + and type3_record["program_source"] == "type3-charprocs", + type3_record) + +write_type3_pdf("type3-direct-charprocs.pdf", charprocs_kind="direct-dict") +direct_charprocs_doc, direct_charprocs_record = type3_fixture_record( + "type3-direct-charprocs.pdf" +) +check("direct nonempty Type3 CharProcs with stream glyphs verifies", + direct_charprocs_record["charprocs_status"] == "verified" + and direct_charprocs_record["embedded"] is True + and direct_charprocs_doc[0].get_text().strip() == "A", + direct_charprocs_record) + +write_type3_pdf( + "type3-escaped-charproc-name.pdf", charprocs_kind="direct-escaped-name", +) +escaped_charproc_doc, escaped_charproc_record = type3_fixture_record( + "type3-escaped-charproc-name.pdf" +) +check("escaped PDF names in direct Type3 CharProcs remain verifiable", + escaped_charproc_record["charprocs_status"] == "verified" + and escaped_charproc_record["embedded"] is True + and escaped_charproc_doc[0].get_text().strip() == "A", + escaped_charproc_record) + +write_type3_pdf( + "type3-indirect-escaped-charproc-name.pdf", + charprocs_kind="indirect-escaped-name", +) +indirect_escaped_doc, indirect_escaped_record = type3_fixture_record( + "type3-indirect-escaped-charproc-name.pdf" +) +indirect_escaped_xref = next( + entry[0] for entry in indirect_escaped_doc[0].get_fonts(full=True) + if entry[2].replace(" ", "").casefold() == "type3" +) +indirect_charprocs_type, indirect_charprocs_value = indirect_escaped_doc.xref_get_key( + indirect_escaped_xref, "CharProcs" +) +indirect_charprocs_xref = indirect_xref(indirect_charprocs_value) +indirect_charprocs_source = indirect_escaped_doc.xref_object( + indirect_charprocs_xref, compressed=True +) +check("escaped slash name survives in the raw indirect CharProcs dictionary", + indirect_charprocs_type == "xref" and "#2F" in indirect_charprocs_source.upper(), + indirect_charprocs_source) +check("escaped slash name in indirect Type3 CharProcs remains verified", + indirect_escaped_record["charprocs_status"] == "verified" + and indirect_escaped_record["embedded"] is True + and indirect_escaped_doc[0].get_text().strip() == "A", + indirect_escaped_record) + +write_type3_pdf( + "type3-direct-font.pdf", charprocs_kind="direct-dict", direct_font=True, +) +direct_font_doc, direct_font_record = type3_fixture_record("type3-direct-font.pdf") +direct_font_xref = next( + entry[0] for entry in direct_font_doc[0].get_fonts(full=True) + if entry[2].replace(" ", "").casefold() == "type3" +) +check("direct Type3 font dictionary is uninspectable rather than falsely non-embedded", + direct_font_xref == 0 + and direct_font_doc[0].get_text().strip() == "A" + and direct_font_record["charprocs_status"] == "uninspectable" + and direct_font_record["embedded"] is None + and direct_font_record["self_contained"] is None, + (direct_font_xref, direct_font_record)) + +malformed_type3_records = {} +for malformed_kind in ( + "empty-direct", "direct-number-entry", "indirect-array", + "indirect-number", "indirect-null", "indirect-bad-glyph", +): + malformed_path = f"type3-{malformed_kind}.pdf" + write_type3_pdf(malformed_path, charprocs_kind=malformed_kind) + _, malformed_type3_records[malformed_kind] = type3_fixture_record(malformed_path) +check("empty, wrong-type, and non-stream Type3 CharProcs fail closed as malformed", + all(record["charprocs_status"] == "malformed" + and record["embedded"] is False + and record["self_contained"] is False + for record in malformed_type3_records.values()), + malformed_type3_records) + +# ---- SKILL.md postcheck: encrypted output is reopened with its password ------- +encrypted_writer = pypdf.PdfWriter() +encrypted_writer.append(r) +encrypted_writer.encrypt("fixture-password") +with open("encrypted.pdf", "wb") as f: + encrypted_writer.write(f) +probe = pypdf.PdfReader("encrypted.pdf") +check("encrypted fixture is detected before page access", probe.is_encrypted) + + +def postcheck_reader(path, password=None): + reader = pypdf.PdfReader(path) + if reader.is_encrypted and reader.decrypt("") == 0: + if password is None or reader.decrypt(password) == 0: + raise RuntimeError(f"valid password required to postcheck {path}") + return reader + + +encrypted_r = postcheck_reader("encrypted.pdf", "fixture-password") +check("password-authenticated postcheck can access every page", len(encrypted_r.pages) == 2) + +blank_password_writer = pypdf.PdfWriter() +blank_password_writer.append(r) +blank_password_writer.encrypt("", owner_password="fixture-owner-password") +with open("blank-user-password.pdf", "wb") as f: + blank_password_writer.write(f) +blank_password_probe = pypdf.PdfReader("blank-user-password.pdf") +check("blank-user-password fixture still reports encryption", blank_password_probe.is_encrypted) +blank_password_r = postcheck_reader("blank-user-password.pdf") +check("postcheck tries the empty user password before requiring PDF_PASSWORD", + len(blank_password_r.pages) == 2) +try: + open_pdf("encrypted.pdf") + transform_rejected_missing_password = False +except RuntimeError: + transform_rejected_missing_password = True +check("PDF transforms reject encrypted input without a password", + transform_rejected_missing_password) +check("PDF transforms authenticate before page access", + len(open_pdf("encrypted.pdf", "fixture-password").pages) == 2) +encrypted_extract = fitz.open("encrypted.pdf") +check("PyMuPDF extraction detects that authentication is required", encrypted_extract.needs_pass) +check("PyMuPDF rejects the wrong extraction password", encrypted_extract.authenticate("wrong") == 0) +check("PyMuPDF authenticates before page extraction", encrypted_extract.authenticate("fixture-password") > 0) +check("authenticated PyMuPDF extraction reaches page text", + "Application form" in encrypted_extract[0].get_text("text", sort=True)) + + +def open_pdf(path): + reader = pypdf.PdfReader(path) + if reader.is_encrypted: + password = os.environ.get("PDF_PASSWORD", "") + if reader.decrypt(password) == 0: + raise RuntimeError(f"Encrypted PDF {path}: set a valid PDF_PASSWORD") + return reader + + +os.environ["PDF_PASSWORD"] = "fixture-password" +transform_encrypted = open_pdf("encrypted.pdf") +check("transform helper authenticates encrypted input before page access", + len(transform_encrypted.pages) == 2) + +# A page containing only an AcroForm widget is interactive content, not blank. +widget_canvas = canvas.Canvas("widget-only.pdf", pagesize=A4) +widget_canvas.acroForm.textfield( + name="widget_only", x=72, y=740, width=260, height=20, borderWidth=1, +) +widget_canvas.showPage() +widget_canvas.save() +widget_doc = fitz.open("widget-only.pdf") +widget_page = widget_doc[0] + +NON_VIEWABLE_ANNOTATION_FLAGS = ( + fitz.PDF_ANNOT_IS_INVISIBLE | fitz.PDF_ANNOT_IS_HIDDEN | fitz.PDF_ANNOT_IS_NO_VIEW +) + + +def annotation_flags(page, item): + flags = getattr(item, "flags", None) + if flags is not None: + return int(flags) + xref = getattr(item, "xref", 0) + if not xref: + return 0 + value_type, value = page.parent.xref_get_key(xref, "F") + try: + return int(value) if value_type == "int" else 0 + except (TypeError, ValueError): + return 0 + + +def visible_clip(page, rectangle, *, already_rotated=False): + try: + rectangle = fitz.Rect(rectangle) + except (TypeError, ValueError): + return None + if not all(math.isfinite(value) for value in rectangle): + return None + rectangle.normalize() + if rectangle.is_empty or rectangle.is_infinite: + return None + rotated = rectangle if already_rotated else rectangle * page.rotation_matrix + clip = rotated & page.rect + return None if clip.is_empty else clip + + +MAX_IMAGE_PLACEMENTS = 1_000 +MAX_IMAGE_SOURCE_PIXELS = 25_000_000 +MAX_TOTAL_IMAGE_SOURCE_PIXELS = 50_000_000 +MAX_IMAGE_RENDER_PIXELS = 4_000_000 +MAX_TOTAL_IMAGE_RENDER_PIXELS = 20_000_000 +MAX_DRAWING_PATHS = 1_000 +MAX_DRAWING_RENDER_PIXELS = 4_000_000 +MAX_TOTAL_DRAWING_RENDER_PIXELS = 20_000_000 +MAX_TEXT_SPANS = 10_000 +MAX_TEXT_RENDER_PIXELS = 4_000_000 +MAX_TOTAL_TEXT_RENDER_PIXELS = 20_000_000 + + +def viewable_text(page): + try: + spans = page.get_texttrace() + except (RuntimeError, ValueError): + return [], [], True + if len(spans) > MAX_TEXT_SPANS: + return spans, [], True + visible = [] + total_render_pixels = 0 + for span in spans: + try: + text = "".join(chr(character[0]) for character in span.get("chars", ())) + render_type = int(span.get("type")) + opacity = float(span.get("opacity")) + except (TypeError, ValueError, OverflowError): + return spans, visible, True + if not text.strip() or render_type > 1 or opacity <= 0: + continue + if render_type not in (0, 1) or not math.isfinite(opacity): + return spans, visible, True + clip = visible_clip(page, span.get("bbox")) + if clip is None: + continue + render_pixels = math.ceil(clip.width) * math.ceil(clip.height) + total_render_pixels += render_pixels + if (render_pixels > MAX_TEXT_RENDER_PIXELS + or total_render_pixels > MAX_TOTAL_TEXT_RENDER_PIXELS): + return spans, visible, True + try: + pixmap = page.get_pixmap(clip=clip, alpha=True, annots=False) + except (RuntimeError, ValueError): + return spans, visible, True + if not pixmap.alpha: + return spans, visible, True + if any(pixmap.samples[pixmap.n - 1::pixmap.n]): + visible.append(span) + return spans, visible, False + + +def viewable_images(page): + try: + placements = page.get_image_info() + except (RuntimeError, ValueError): + return [], [], True + if len(placements) > MAX_IMAGE_PLACEMENTS: + return placements, [], True + visible = [] + total_source_pixels = 0 + total_render_pixels = 0 + for placement in placements: + clip = visible_clip(page, placement.get("bbox")) + if clip is None: + continue + width, height = placement.get("width"), placement.get("height") + if (not isinstance(width, int) or isinstance(width, bool) or width <= 0 + or not isinstance(height, int) or isinstance(height, bool) or height <= 0): + return placements, visible, True + source_pixels = width * height + total_source_pixels += source_pixels + if (source_pixels > MAX_IMAGE_SOURCE_PIXELS + or total_source_pixels > MAX_TOTAL_IMAGE_SOURCE_PIXELS): + return placements, visible, True + render_pixels = math.ceil(clip.width) * math.ceil(clip.height) + total_render_pixels += render_pixels + if (render_pixels > MAX_IMAGE_RENDER_PIXELS + or total_render_pixels > MAX_TOTAL_IMAGE_RENDER_PIXELS): + return placements, visible, True + try: + pixmap = page.get_pixmap(clip=clip, alpha=True, annots=False) + except (RuntimeError, ValueError): + return placements, visible, True + if not pixmap.alpha: + return placements, visible, True + if any(pixmap.samples[pixmap.n - 1::pixmap.n]): + visible.append(placement) + return placements, visible, False + + +def drawing_bounds(drawing): + """Include stroke width around the path geometry returned by PyMuPDF.""" + try: + rectangle = fitz.Rect(drawing.get("rect")) + except (TypeError, ValueError): + raise ValueError("drawing has no finite bounding rectangle") from None + if not all(math.isfinite(value) for value in rectangle) or rectangle.is_infinite: + raise ValueError("drawing has no finite bounding rectangle") + rectangle.normalize() + path_type = drawing.get("type") + has_stroke = ( + "s" in path_type if isinstance(path_type, str) + else drawing.get("color") is not None + ) + if has_stroke: + width = drawing.get("width", 0) + if isinstance(width, bool): + raise ValueError("drawing has an invalid stroke width") + try: + width = float(width or 0) + except (TypeError, ValueError): + raise ValueError("drawing has an invalid stroke width") from None + if not math.isfinite(width) or width < 0: + raise ValueError("drawing has an invalid stroke width") + # PDF hairlines (width 0) still paint one device pixel at render time. + padding = max(width / 2, 0.5 if width == 0 else 0) + rectangle = fitz.Rect( + rectangle.x0 - padding, rectangle.y0 - padding, + rectangle.x1 + padding, rectangle.y1 + padding, + ) + return rectangle + + +def viewable_drawings(page): + """Render bounded path clips; unknown visibility keeps the page nonblank.""" + try: + drawings = page.get_drawings() + except (RuntimeError, ValueError): + return [], [], True + if len(drawings) > MAX_DRAWING_PATHS: + return drawings, [], True + visible = [] + total_render_pixels = 0 + for drawing in drawings: + try: + bounds = drawing_bounds(drawing) + except ValueError: + return drawings, visible, True + clip = visible_clip(page, bounds) + if clip is None: + continue + render_pixels = math.ceil(clip.width) * math.ceil(clip.height) + total_render_pixels += render_pixels + if (render_pixels > MAX_DRAWING_RENDER_PIXELS + or total_render_pixels > MAX_TOTAL_DRAWING_RENDER_PIXELS): + return drawings, visible, True + try: + pixmap = page.get_pixmap(clip=clip, alpha=True, annots=False) + except (RuntimeError, ValueError): + return drawings, visible, True + if not pixmap.alpha: + return drawings, visible, True + if any(pixmap.samples[pixmap.n - 1::pixmap.n]): + visible.append(drawing) + return drawings, visible, False + + +def rendered_interactives(page, items): + rendered = [] + visibility_unknown = False + for item in items: + if annotation_flags(page, item) & NON_VIEWABLE_ANNOTATION_FLAGS: + continue + clip = visible_clip(page, item.rect) + if clip is None: + continue + try: + with_annotations = page.get_pixmap( + matrix=fitz.Matrix(2, 2), clip=clip, alpha=False, annots=True, + ) + without_annotations = page.get_pixmap( + matrix=fitz.Matrix(2, 2), clip=clip, alpha=False, annots=False, + ) + except (RuntimeError, ValueError): + visibility_unknown = True + continue + if with_annotations.samples != without_annotations.samples: + rendered.append(item) + return rendered, visibility_unknown + + +def page_links(page): + links = [] + link = page.first_link + while link is not None: + links.append(link) + link = link.next + return links + + +def link_has_target(link): + destination = getattr(link, "dest", None) + return bool(getattr(link, "uri", None)) or ( + destination is not None and getattr(destination, "page", -1) >= 0 + ) + + +def viewable_interactives(page): + widgets, widget_visibility_unknown = rendered_interactives( + page, list(page.widgets() or ()) + ) + annotations, annotation_visibility_unknown = rendered_interactives( + page, list(page.annots() or ()) + ) + links = [ + link for link in page_links(page) + if link_has_target(link) + and not annotation_flags(page, link) & NON_VIEWABLE_ANNOTATION_FLAGS + and visible_clip(page, link.rect, already_rotated=True) is not None + ] + return ( + widgets, annotations, links, + widget_visibility_unknown or annotation_visibility_unknown, + ) + + +class RenderFailurePage: + rect = fitz.Rect(0, 0, 200, 300) + rotation_matrix = fitz.Identity + + def get_pixmap(self, **kwargs): + raise RuntimeError("fixture render failure") + + +class VisibleInteractiveProbe: + flags = 0 + rect = fitz.Rect(20, 20, 80, 40) + + +failed_render_items, failed_render_unknown = rendered_interactives( + RenderFailurePage(), [VisibleInteractiveProbe()] +) +failed_render_blank = not (failed_render_items or failed_render_unknown) +check("interactive render failures keep blank-page classification fail closed", + not failed_render_items and failed_render_unknown and not failed_render_blank, + (failed_render_items, failed_render_unknown, failed_render_blank)) + + +widgets, annotations, links, interaction_visibility_unknown = viewable_interactives( + widget_page +) +_, visible_widget_drawings, widget_drawing_visibility_unknown = viewable_drawings( + widget_page +) +blank = ( + not viewable_text(widget_page)[1] and not viewable_text(widget_page)[2] + and not widget_page.get_images() + and not visible_widget_drawings and not widget_drawing_visibility_unknown + and not widgets and not annotations and not links and not interaction_visibility_unknown +) +check("widget-only form page exposes a widget", len(widgets) == 1, len(widgets)) +check("widget-aware blank-page predicate keeps form page", not blank) + +# ---- SKILL.md postcheck: interactive-only pages are exempt from the text gate --- +def normalized_box(box): + x0, y0, x1, y1 = (float(value) for value in box) + left, right = sorted((x0, x1)) + bottom, top = sorted((y0, y1)) + return left, bottom, right, top + + +def normalized_size(box): + left, bottom, right, top = normalized_box(box) + return right - left, top - bottom + + +def widget_count(page, rendered_page): + non_viewable_flags = 1 | 2 | 32 + count = 0 + for ref in page.get("/Annots") or []: + widget = ref.get_object() + if widget.get("/Subtype") != "/Widget": + continue + flags = int(widget.get("/F", 0)) + rectangle = widget.get("/Rect") + if flags & non_viewable_flags or rectangle is None: + continue + left, bottom, right, top = normalized_box(rectangle) + crop_left, crop_bottom, crop_right, crop_top = normalized_box(page.cropbox) + media_left, media_bottom, media_right, media_top = normalized_box(page.mediabox) + visible_left = max(crop_left, media_left) + visible_bottom = max(crop_bottom, media_bottom) + visible_right = min(crop_right, media_right) + visible_top = min(crop_top, media_top) + intersects_visible_page = ( + min(right, visible_right) > max(left, visible_left) + and min(top, visible_top) > max(bottom, visible_bottom) + ) + if right <= left or top <= bottom or not intersects_visible_page: + continue + xref = getattr(ref, "idnum", None) + if xref is None: + continue + try: + rendered_widget = rendered_page.load_widget(xref) + clip = ( + rendered_widget.rect * rendered_page.rotation_matrix + ) & rendered_page.rect + if clip.is_empty: + continue + with_widget = rendered_page.get_pixmap( + matrix=fitz.Matrix(2, 2), clip=clip, alpha=False, annots=True, + ) + without_widgets = rendered_page.get_pixmap( + matrix=fitz.Matrix(2, 2), clip=clip, alpha=False, annots=False, + ) + except (RuntimeError, ValueError): + continue + if with_widget.samples != without_widgets.samples: + count += 1 + return count + +widget_postcheck = pypdf.PdfReader("widget-only.pdf") +widget_render = fitz.open("widget-only.pdf") +widget_text = (widget_postcheck.pages[0].extract_text() or "").strip() +check("widget-only page extracts no text", widget_text == "", repr(widget_text)) +check("postcheck counts the visibly rendered widget annotation", + widget_count(widget_postcheck.pages[0], widget_render[0]) == 1) +check("widget-only page passes the text postcheck via the widget exemption", + bool(widget_text) or widget_count(widget_postcheck.pages[0], widget_render[0]) > 0) + +from pypdf.generic import ArrayObject, FloatObject, NameObject, NumberObject + +for label, flag in (("invisible", 1), ("hidden", 2), ("no-view", 32)): + hidden_writer = pypdf.PdfWriter() + hidden_writer.append(widget_postcheck) + hidden_widget = hidden_writer.pages[0]["/Annots"][0].get_object() + hidden_widget[NameObject("/F")] = NumberObject(flag) + hidden_path = f"widget-{label}.pdf" + with open(hidden_path, "wb") as output: + hidden_writer.write(output) + hidden_page = pypdf.PdfReader(hidden_path).pages[0] + hidden_render = fitz.open(hidden_path) + check(f"{label} widget does not exempt an otherwise blank page", + widget_count(hidden_page, hidden_render[0]) == 0 + and not bool((hidden_page.extract_text() or "").strip()) + and not any(viewable_interactives(hidden_render[0])[:3])) + +appearance_writer = pypdf.PdfWriter() +appearance_writer.append(widget_postcheck) +appearance_widget = appearance_writer.pages[0]["/Annots"][0].get_object() +del appearance_widget[NameObject("/AP")] +with open("widget-no-appearance.pdf", "wb") as output: + appearance_writer.write(output) +appearance_page = pypdf.PdfReader("widget-no-appearance.pdf").pages[0] +appearance_render = fitz.open("widget-no-appearance.pdf") +check("visible widget without /AP can still be viewer-generated and render visibly", + widget_count(appearance_page, appearance_render[0]) == 1) + +blank_appearance_writer = pypdf.PdfWriter() +blank_appearance_writer.append(widget_postcheck) +blank_appearance_widget = blank_appearance_writer.pages[0]["/Annots"][0].get_object() +for key in ("/AP", "/MK", "/BS", "/DA", "/V", "/DV"): + blank_appearance_widget.pop(NameObject(key), None) +blank_acroform = blank_appearance_writer._root_object["/AcroForm"].get_object() +for key in ("/DA", "/DR", "/NeedAppearances"): + blank_acroform.pop(NameObject(key), None) +with open("widget-blank-appearance.pdf", "wb") as output: + blank_appearance_writer.write(output) +blank_appearance_page = pypdf.PdfReader("widget-blank-appearance.pdf").pages[0] +blank_appearance_render = fitz.open("widget-blank-appearance.pdf") +check("widget with no renderable appearance does not exempt a white page", + widget_count(blank_appearance_page, blank_appearance_render[0]) == 0 + and not any(value != 255 for value in + blank_appearance_render[0].get_pixmap(alpha=False).samples)) + +for label, rectangle in ( + ("zero-area", [72, 740, 72, 760]), + ("off-page", [1000, 1000, 1100, 1100]), +): + geometry_writer = pypdf.PdfWriter() + geometry_writer.append(widget_postcheck) + geometry_widget = geometry_writer.pages[0]["/Annots"][0].get_object() + geometry_widget[NameObject("/Rect")] = ArrayObject([ + FloatObject(value) for value in rectangle + ]) + geometry_path = f"widget-{label}.pdf" + with open(geometry_path, "wb") as output: + geometry_writer.write(output) + geometry_page = pypdf.PdfReader(geometry_path).pages[0] + geometry_render = fitz.open(geometry_path) + check(f"{label} widget does not exempt an otherwise blank page", + widget_count(geometry_page, geometry_render[0]) == 0 + and not any(viewable_interactives(geometry_render[0])[:3])) + +interaction_doc = fitz.open() +interaction_page = interaction_doc.new_page(width=200, height=300) +hidden_annotation = interaction_page.add_text_annot((40, 40), "hidden note") +hidden_annotation.set_flags(fitz.PDF_ANNOT_IS_HIDDEN) +hidden_annotation.update() +interaction_page.insert_link({ + "kind": fitz.LINK_URI, + "from": fitz.Rect(500, 500, 600, 520), + "uri": "https://example.invalid", +}) +interaction_doc.save("non-viewable-interactives.pdf") +interaction_doc.close() +interaction_reopened = fitz.open("non-viewable-interactives.pdf") +filtered_widgets, filtered_annotations, filtered_links, filtered_unknown = ( + viewable_interactives(interaction_reopened[0]) +) +check("hidden annotations and off-page links do not exempt a blank page", + not filtered_widgets and not filtered_annotations and not filtered_links + and not filtered_unknown, + (filtered_widgets, filtered_annotations, filtered_links, filtered_unknown)) + +visible_interaction_doc = fitz.open() +visible_interaction_page = visible_interaction_doc.new_page(width=200, height=300) +visible_annotation = visible_interaction_page.add_text_annot((40, 40), "visible note") +visible_annotation.update() +visible_interaction_page.insert_link({ + "kind": fitz.LINK_URI, + "from": fitz.Rect(40, 80, 140, 100), + "uri": "https://example.invalid", +}) +visible_interaction_doc.save("visible-interactives.pdf") +visible_interaction_doc.close() +visible_interaction_reopened = fitz.open("visible-interactives.pdf") +visible_widgets, visible_annotations, visible_links, visible_unknown = ( + viewable_interactives(visible_interaction_reopened[0]) +) +check("visible annotations and usable links keep an interactive page nonblank", + not visible_widgets and len(visible_annotations) == 1 and len(visible_links) == 1 + and not visible_unknown, + (visible_widgets, visible_annotations, visible_links, visible_unknown)) + +reversed_writer = pypdf.PdfWriter() +reversed_writer.append(widget_postcheck) +reversed_page = reversed_writer.pages[0] +reversed_widget = reversed_page["/Annots"][0].get_object() +reversed_widget[NameObject("/Rect")] = ArrayObject([ + FloatObject(332), FloatObject(760), FloatObject(72), FloatObject(740), +]) +with open("widget-reversed-rect.pdf", "wb") as output: + reversed_writer.write(output) +reversed_reader_page = pypdf.PdfReader("widget-reversed-rect.pdf").pages[0] +reversed_render = fitz.open("widget-reversed-rect.pdf") +check("legal reversed widget rectangle is normalized and remains visible", + widget_count(reversed_reader_page, reversed_render[0]) == 1) + +reversed_boxes_writer = pypdf.PdfWriter() +reversed_boxes_writer.append(widget_postcheck) +reversed_boxes_page = reversed_boxes_writer.pages[0] +reversed_page_box = ArrayObject([ + FloatObject(A4[0]), FloatObject(A4[1]), FloatObject(0), FloatObject(0), +]) +reversed_boxes_page[NameObject("/MediaBox")] = reversed_page_box +reversed_boxes_page[NameObject("/CropBox")] = ArrayObject(reversed_page_box) +with open("widget-reversed-page-boxes.pdf", "wb") as output: + reversed_boxes_writer.write(output) +reversed_boxes_reader_page = pypdf.PdfReader("widget-reversed-page-boxes.pdf").pages[0] +reversed_boxes_render = fitz.open("widget-reversed-page-boxes.pdf") +check("legal reversed page boxes are normalized before widget intersection", + widget_count(reversed_boxes_reader_page, reversed_boxes_render[0]) == 1 + and all( + abs(actual - expected) < A4_TOLERANCE + for actual, expected in zip( + normalized_size(reversed_boxes_reader_page.mediabox), A4, + ) + )) + +rotated_widget_writer = pypdf.PdfWriter() +rotated_widget_writer.append(widget_postcheck) +rotated_widget_page = rotated_widget_writer.pages[0] +rotated_widget_page.rotate(90) +rotated_widget = rotated_widget_page["/Annots"][0].get_object() +rotated_widget[NameObject("/Rect")] = ArrayObject([ + FloatObject(72), FloatObject(72), FloatObject(332), FloatObject(92), +]) +with open("widget-rotated.pdf", "wb") as output: + rotated_widget_writer.write(output) +rotated_widget_reader_page = pypdf.PdfReader("widget-rotated.pdf").pages[0] +rotated_widget_render = fitz.open("widget-rotated.pdf") +check("visible widget on a rotated page is clipped in rotated coordinates", + widget_count(rotated_widget_reader_page, rotated_widget_render[0]) == 1) + +blank_writer = pypdf.PdfWriter() +blank_writer.add_blank_page(width=200, height=300) +with open("blank.pdf", "wb") as f: + blank_writer.write(f) +blank_r = pypdf.PdfReader("blank.pdf") +blank_render = fitz.open("blank.pdf") +check("a truly blank page still fails the text postcheck", + not (bool((blank_r.pages[0].extract_text() or "").strip()) + or widget_count(blank_r.pages[0], blank_render[0]) > 0)) + +# ---- transform.md AcroForm snippet: clone into writer, fill on writer pages ---- +from pypdf import PdfReader, PdfWriter + +reader = PdfReader("form.pdf") +fields = reader.get_fields() or {} +check("source form has the expected field", "applicant_name" in fields, list(fields)) + +writer = PdfWriter() +writer.append(reader) # clones pages AND catalog /AcroForm +writer.update_page_form_field_values( + writer.pages[0], + {"applicant_name": "Ada Byron"}, +) +with open("filled.pdf", "wb") as f: + writer.write(f) + +check_r = PdfReader("filled.pdf") +check("filled file keeps both pages", len(check_r.pages) == 2, len(check_r.pages)) +value = str((check_r.get_fields() or {}).get("applicant_name", {}).get("/V", "")) +check("field value round-trips", value.strip("/") == "Ada Byron", repr(value)) + +# A widget can live on any page; locate its annotation instead of assuming page 1. +page2_form = canvas.Canvas("form-page2.pdf", pagesize=A4) +page2_form.drawString(72, 780, "Cover page") +page2_form.showPage() +page2_form.drawString(72, 780, "Form page") +page2_form.acroForm.textfield( + name="applicant_name", x=72, y=740, width=260, height=20, borderWidth=0, +) +page2_form.showPage() +page2_form.save() + +wrong_page_writer = PdfWriter() +wrong_page_writer.append(PdfReader("form-page2.pdf")) +wrong_page_writer.update_page_form_field_values( + wrong_page_writer.pages[0], {"applicant_name": "Wrong page"}, +) +with open("form-page2-wrong.pdf", "wb") as f: + wrong_page_writer.write(f) +wrong_value = str( + (PdfReader("form-page2-wrong.pdf").get_fields() or {}) + .get("applicant_name", {}).get("/V", "") +) +check("hard-coded first-page form fill misses a page-2 widget (negative control)", + wrong_value.strip("/") != "Wrong page", repr(wrong_value)) + + +def widget_field_name(widget): + parts = [] + seen = set() + while widget is not None: + object_id = id(widget) + if object_id in seen: + raise ValueError("cycle in AcroForm field parent chain") + seen.add(object_id) + partial_name = widget.get("/T") + if partial_name is not None: + parts.append(str(partial_name)) + parent = widget.get("/Parent") + widget = None if parent is None else parent.get_object() + return ".".join(reversed(parts)) if parts else None + + +page2_writer = PdfWriter() +page2_writer.append(PdfReader("form-page2.pdf")) +field_name = "applicant_name" +target_pages = [ + page for page in page2_writer.pages + if any( + (widget := ref.get_object()).get("/Subtype") == "/Widget" + and widget_field_name(widget) == field_name + for ref in (page.get("/Annots") or []) + ) +] +for target_page in target_pages: + page2_writer.update_page_form_field_values( + target_page, {field_name: "Ada on page 2"}, + ) +with open("form-page2-filled.pdf", "wb") as f: + page2_writer.write(f) +page2_value = str( + (PdfReader("form-page2-filled.pdf").get_fields() or {}) + .get(field_name, {}).get("/V", "") +) +check("form fill locates the widget page before updating", + len(target_pages) == 1 and target_pages[0] is page2_writer.pages[1], len(target_pages)) +check("page-2 field value round-trips", page2_value.strip("/") == "Ada on page 2", + repr(page2_value)) + +# A hierarchical field stores one partial /T at each level. Build a non-terminal +# `application` parent around the page-2 widget and address the terminal field by +# the fully qualified name returned by get_fields(). +from pypdf.generic import ArrayObject, DictionaryObject, NameObject, TextStringObject + +hierarchy_writer = PdfWriter() +hierarchy_writer.append(PdfReader("form-page2.pdf")) +hierarchy_widget_ref = hierarchy_writer.pages[1]["/Annots"][0] +hierarchy_widget = hierarchy_widget_ref.get_object() +hierarchy_parent = DictionaryObject({ + NameObject("/T"): TextStringObject("application"), + NameObject("/Kids"): ArrayObject([hierarchy_widget_ref]), +}) +hierarchy_parent_ref = hierarchy_writer._add_object(hierarchy_parent) +hierarchy_widget[NameObject("/Parent")] = hierarchy_parent_ref +hierarchy_acroform = hierarchy_writer._root_object["/AcroForm"] +hierarchy_acroform[NameObject("/Fields")] = ArrayObject([hierarchy_parent_ref]) +with open("hierarchical-form.pdf", "wb") as f: + hierarchy_writer.write(f) + +hierarchy_reader = PdfReader("hierarchical-form.pdf") +hierarchical_field_name = "application.applicant_name" +check("get_fields exposes the fully qualified hierarchical field name", + hierarchical_field_name in (hierarchy_reader.get_fields() or {}), + list((hierarchy_reader.get_fields() or {}).keys())) + +hierarchy_fill_writer = PdfWriter() +hierarchy_fill_writer.append(hierarchy_reader) +hierarchy_target_pages = [ + page for page in hierarchy_fill_writer.pages + if any( + (widget := ref.get_object()).get("/Subtype") == "/Widget" + and widget_field_name(widget) == hierarchical_field_name + for ref in (page.get("/Annots") or []) + ) +] +for target_page in hierarchy_target_pages: + hierarchy_fill_writer.update_page_form_field_values( + target_page, {hierarchical_field_name: "Ada Hierarchical"}, + ) +with open("hierarchical-form-filled.pdf", "wb") as f: + hierarchy_fill_writer.write(f) +hierarchy_value = str( + (PdfReader("hierarchical-form-filled.pdf").get_fields() or {}) + .get(hierarchical_field_name, {}).get("/V", "") +) +hierarchy_literal_value = str( + (PdfReader("hierarchical-form-filled.pdf").get_fields() or {}) + .get("applicant_name", {}).get("/V", "") +) +check("qualified field lookup locates the hierarchical widget page", + len(hierarchy_target_pages) == 1 + and hierarchy_target_pages[0] is hierarchy_fill_writer.pages[1], + len(hierarchy_target_pages)) +check("hierarchical field value round-trips", + hierarchy_value.strip("/") == "Ada Hierarchical", repr(hierarchy_value)) +check("literal applicant_name lookup is a proven false negative for a qualified field", + hierarchy_literal_value == "", repr(hierarchy_literal_value)) + +# ---- transform.md merge imports outline navigation ---------------------------- +appendix_writer = PdfWriter() +appendix_writer.add_blank_page(width=200, height=300) +appendix_writer.add_outline_item("Appendix bookmark", 0) +with open("appendix-outline.pdf", "wb") as f: + appendix_writer.write(f) + +merge_writer = PdfWriter() +merge_writer.append(open_pdf("form.pdf"), pages=(1, 2), import_outline=True) +merge_writer.append(open_pdf("appendix-outline.pdf"), import_outline=True) +with open("merged-outline.pdf", "wb") as f: + merge_writer.write(f) +merged_outline = PdfReader("merged-outline.pdf").outline +check( + "append imports the appended PDF outline", + any(getattr(item, "title", "") == "Appendix bookmark" for item in merged_outline), + merged_outline, +) + +# ---- transform.md watermark snippet ------------------------------------------- +from pypdf import PdfReader as R2, Transformation +from pypdf.generic import RectangleObject + + +def rotation_transfer(page): + media = RectangleObject(page.mediabox) + transform = ( + Transformation() + .translate( + -float(media.left + media.width / 2), + -float(media.bottom + media.height / 2), + ) + .rotate(-page.rotation) + ) + corners = [ + transform.apply_on(point) + for point in (media.lower_left, media.lower_right, media.upper_left, media.upper_right) + ] + return transform.translate( + -min(point[0] for point in corners), + -min(point[1] for point in corners), + ) + + +def inverse_transformation(transform): + a, b, c, d, e, f = map(float, transform.ctm) + determinant = a * d - b * c + return Transformation(( + d / determinant, -b / determinant, + -c / determinant, a / determinant, + (c * f - d * e) / determinant, + (b * e - a * f) / determinant, + )) + + +def transformed_rectangle(rectangle, transform): + rectangle = RectangleObject(rectangle) + corners = [ + transform.apply_on(point) + for point in ( + rectangle.lower_left, rectangle.lower_right, + rectangle.upper_left, rectangle.upper_right, + ) + ] + return RectangleObject(( + min(point[0] for point in corners), min(point[1] for point in corners), + max(point[0] for point in corners), max(point[1] for point in corners), + )) + + +def stamp_placement(page, stamp_box): + to_visual = rotation_transfer(page) + destination = transformed_rectangle(page.cropbox, to_visual) + sw, sh = float(stamp_box.width), float(stamp_box.height) + dw, dh = float(destination.width), float(destination.height) + scale = min(dw / sw, dh / sh) + tx = float(destination.left) + (dw - sw * scale) / 2 - float(stamp_box.left) * scale + ty = float(destination.bottom) + (dh - sh * scale) / 2 - float(stamp_box.bottom) * scale + visible_placement = Transformation().scale(scale).translate(tx, ty) + return visible_placement.transform(inverse_transformation(to_visual)) + +stamp_src = canvas.Canvas("stamp.pdf", pagesize=A4) +stamp_src.setFont("Helvetica", 40) +stamp_src.setFillAlpha(0.35) +stamp_src.drawString(160, 400, "DRAFT") +stamp_src.save() + +stamp = R2("stamp.pdf").pages[0] +stamp.transfer_rotation_to_content() +stamp_text = (stamp.extract_text() or "").strip() +reader = R2("form.pdf") +expected_fields = reader.get_fields() or {} +writer = PdfWriter() +writer.append(reader) +stamp_box = stamp.cropbox +for page in writer.pages: + page.merge_transformed_page(stamp, stamp_placement(page, stamp_box)) +expected_sizes = [(round(float(p.mediabox.width), 2), round(float(p.mediabox.height), 2)) + for p in writer.pages] +with open("watermarked.pdf", "wb") as f: + writer.write(f) + +verify = R2("watermarked.pdf") +check("watermark written and page count kept", len(verify.pages) == 2, len(verify.pages)) +check( + "watermark page sizes unchanged", + [(round(float(p.mediabox.width), 2), round(float(p.mediabox.height), 2)) for p in verify.pages] == expected_sizes, +) +check("watermarking preserves the AcroForm catalog and fields", + set(expected_fields) <= set(verify.get_fields() or {}), verify.get_fields()) +check("stamp text present on every page", all(stamp_text in (p.extract_text() or "") for p in verify.pages)) + +# ---- inspect.md blank-page predicate includes widgets and annotations ----------- +form_only_canvas = canvas.Canvas("form-only.pdf", pagesize=A4) +form_only_canvas.acroForm.textfield( + name="widget_only", x=72, y=740, width=260, height=20, borderWidth=0 +) +form_only_canvas.showPage() +form_only_canvas.save() +form_only_doc = fitz.open("form-only.pdf") +form_only_page = form_only_doc[0] +form_only_widgets = list(form_only_page.widgets() or ()) +form_only_annotations = list(form_only_page.annots() or ()) + + +def inspected_page_is_blank(page): + _, visible_text, text_visibility_unknown = viewable_text(page) + _, visible_images, image_visibility_unknown = viewable_images(page) + _, visible_drawings, drawing_visibility_unknown = viewable_drawings(page) + widgets, annotations, links, interaction_visibility_unknown = viewable_interactives(page) + return not ( + visible_text or visible_images or visible_drawings + or widgets or annotations or links + or text_visibility_unknown or image_visibility_unknown + or drawing_visibility_unknown or interaction_visibility_unknown + ) + + +form_only_is_blank = inspected_page_is_blank(form_only_page) +check("form-only page exposes a widget", len(form_only_widgets) == 1) +check("form-only page is not classified as blank", not form_only_is_blank) + +# Extractable text can still be visually absent because of Tr=3, alpha=0, +# clipping, or off-page geometry. +text_visibility_canvas = canvas.Canvas( + "text-visibility.pdf", pagesize=(200, 200), pageCompression=0, +) +text_visibility_canvas.drawString(40, 100, "visible") +text_visibility_canvas.showPage() +invisible_text = text_visibility_canvas.beginText(40, 100) +invisible_text.setTextRenderMode(3) +invisible_text.textOut("render-mode-hidden") +text_visibility_canvas.drawText(invisible_text) +text_visibility_canvas.showPage() +text_visibility_canvas.saveState() +text_visibility_canvas.setFillAlpha(0) +text_visibility_canvas.drawString(40, 100, "zero-opacity") +text_visibility_canvas.restoreState() +text_visibility_canvas.showPage() +text_clip = text_visibility_canvas.beginPath() +text_clip.rect(0, 0, 10, 10) +text_visibility_canvas.saveState() +text_visibility_canvas.clipPath(text_clip, stroke=0, fill=0) +text_visibility_canvas.drawString(40, 100, "clipped") +text_visibility_canvas.restoreState() +text_visibility_canvas.showPage() +text_visibility_canvas.drawString(250, 100, "off-page") +text_visibility_canvas.showPage() +text_visibility_canvas.save() + +text_visibility_doc = fitz.open("text-visibility.pdf") +text_visibility_results = [] +for text_page in text_visibility_doc: + text_spans, visible_text_spans, text_visibility_unknown = viewable_text(text_page) + text_visibility_results.append(( + len(text_spans), len(visible_text_spans), text_visibility_unknown, + inspected_page_is_blank(text_page), + )) +check("blank-page text detection uses rendered visibility", + text_visibility_results == [ + (1, 1, False, False), + (1, 0, False, True), + (1, 0, False, True), + (1, 0, False, True), + (1, 0, False, True), + ], text_visibility_results) + +text_budget_results = {} +for budget_name in ( + "MAX_TEXT_SPANS", "MAX_TEXT_RENDER_PIXELS", "MAX_TOTAL_TEXT_RENDER_PIXELS", +): + original_budget = globals()[budget_name] + globals()[budget_name] = 0 + try: + spans, visible_spans, visibility_unknown = viewable_text(text_visibility_doc[0]) + page_is_blank = inspected_page_is_blank(text_visibility_doc[0]) + finally: + globals()[budget_name] = original_budget + text_budget_results[budget_name] = ( + len(spans), len(visible_spans), visibility_unknown, page_is_blank, + ) +check("every text count/render budget fails closed as visibility unknown", + all(result == (1, 0, True, False) for result in text_budget_results.values()), + text_budget_results) + +# PyMuPDF inventories invoked paths even when page geometry, clipping, or opacity +# prevents them from contributing a rendered pixel. +drawing_visibility_canvas = canvas.Canvas( + "drawing-visibility.pdf", pagesize=(200, 200), pageCompression=0, +) +drawing_visibility_canvas.setLineWidth(2) +drawing_visibility_canvas.rect(40, 40, 80, 60, stroke=1, fill=0) +drawing_visibility_canvas.showPage() +drawing_visibility_canvas.setLineWidth(2) +drawing_visibility_canvas.rect(250, 40, 80, 60, stroke=1, fill=0) +drawing_visibility_canvas.showPage() +clipping_path = drawing_visibility_canvas.beginPath() +clipping_path.rect(0, 0, 10, 10) +drawing_visibility_canvas.saveState() +drawing_visibility_canvas.clipPath(clipping_path, stroke=0, fill=0) +drawing_visibility_canvas.setLineWidth(2) +drawing_visibility_canvas.rect(40, 40, 80, 60, stroke=1, fill=0) +drawing_visibility_canvas.restoreState() +drawing_visibility_canvas.showPage() +drawing_visibility_canvas.saveState() +drawing_visibility_canvas.setStrokeAlpha(0) +drawing_visibility_canvas.setLineWidth(2) +drawing_visibility_canvas.rect(40, 40, 80, 60, stroke=1, fill=0) +drawing_visibility_canvas.restoreState() +drawing_visibility_canvas.showPage() +no_paint_path = drawing_visibility_canvas.beginPath() +no_paint_path.rect(40, 40, 80, 60) +drawing_visibility_canvas.drawPath(no_paint_path, stroke=0, fill=0) +drawing_visibility_canvas.showPage() +drawing_visibility_canvas.save() + +drawing_visibility_doc = fitz.open("drawing-visibility.pdf") +drawing_visibility_results = [] +for drawing_page in drawing_visibility_doc: + drawing_paths, visible_drawing_paths, drawing_visibility_unknown = viewable_drawings( + drawing_page + ) + drawing_visibility_results.append(( + len(drawing_paths), len(visible_drawing_paths), drawing_visibility_unknown, + inspected_page_is_blank(drawing_page), + )) +check("drawing visibility uses rendered paint rather than raw path presence", + drawing_visibility_results == [ + (1, 1, False, False), # ordinary visible stroke + (1, 0, False, True), # fully off-page stroke + (1, 0, False, True), # stroke fully excluded by the active clip + (1, 0, False, True), # zero-opacity stroke still has a path record + (0, 0, False, True), # a path ended with PDF's no-paint operator + ], drawing_visibility_results) + + +class FailingDrawingRenderPage: + rect = fitz.Rect(0, 0, 200, 200) + rotation_matrix = fitz.Identity + + def get_drawings(self): + return [{"type": "f", "rect": fitz.Rect(20, 20, 80, 80)}] + + def get_pixmap(self, **kwargs): + raise RuntimeError("fixture drawing renderer failure") + + +failed_drawing_paths, failed_visible_drawings, failed_drawing_unknown = viewable_drawings( + FailingDrawingRenderPage() +) +failed_drawing_blank = not (failed_visible_drawings or failed_drawing_unknown) +check("drawing render failures keep blank-page classification fail closed", + len(failed_drawing_paths) == 1 and not failed_visible_drawings + and failed_drawing_unknown and not failed_drawing_blank, + (failed_drawing_paths, failed_visible_drawings, + failed_drawing_unknown, failed_drawing_blank)) + +drawing_budget_results = {} +for budget_name in ( + "MAX_DRAWING_PATHS", "MAX_DRAWING_RENDER_PIXELS", + "MAX_TOTAL_DRAWING_RENDER_PIXELS", +): + original_budget = globals()[budget_name] + globals()[budget_name] = 0 + try: + budget_paths, budget_visible, budget_unknown = viewable_drawings( + drawing_visibility_doc[0] + ) + budget_page_is_blank = inspected_page_is_blank(drawing_visibility_doc[0]) + finally: + globals()[budget_name] = original_budget + drawing_budget_results[budget_name] = ( + len(budget_paths), len(budget_visible), budget_unknown, budget_page_is_blank, + ) +check("every drawing count/render budget fails closed as visibility unknown", + all(result == (1, 0, True, False) + for result in drawing_budget_results.values()), + drawing_budget_results) + +# Identical media/crop geometry remains consistent when one page has /Rotate 90. +mixed_rotation_writer = PdfWriter() +mixed_rotation_writer.append(PdfReader("form.pdf")) +mixed_rotation_writer.pages[1].rotate(90) +with open("inspect-mixed-rotation.pdf", "wb") as f: + mixed_rotation_writer.write(f) +mixed_rotation_doc = fitz.open("inspect-mixed-rotation.pdf") +mixed_media_sizes = { + (round(page.mediabox.width, 2), round(page.mediabox.height, 2)) + for page in mixed_rotation_doc +} +mixed_crop_sizes = { + (round(page.cropbox.width, 2), round(page.cropbox.height, 2)) + for page in mixed_rotation_doc +} +mixed_display_sizes = { + (round(page.rect.width, 2), round(page.rect.height, 2)) + for page in mixed_rotation_doc +} +check("page.rect falsely reports mixed sizes for identical rotated media (negative control)", + len(mixed_display_sizes) == 2, mixed_display_sizes) +check("unrotated media and crop sizes stay consistent across mixed rotation", + len(mixed_media_sizes) == 1 and len(mixed_crop_sizes) == 1, + (mixed_media_sizes, mixed_crop_sizes)) +check("inspection reports mixed rotation separately from paper size", + [page.rotation for page in mixed_rotation_doc] == [0, 90]) + +# ReportLab's drawInlineImage emits BI/ID/EI content rather than an image XObject. +inline_source = fitz.Pixmap(fitz.csRGB, fitz.IRect(0, 0, 12, 12)) +inline_source.clear_with(96) +inline_source.save("inline-only-source.png") +inline_canvas = canvas.Canvas("inline-only.pdf", pagesize=A4) +inline_canvas.drawInlineImage("inline-only-source.png", 72, 700, width=80, height=80) +inline_canvas.showPage() +inline_canvas.save() +inline_doc = fitz.open("inline-only.pdf") +inline_page = inline_doc[0] +inline_blocks = [ + block for block in inline_page.get_text("dict")["blocks"] + if block["type"] == 1 +] +check("inline-only image is absent from the XObject inventory", + inline_page.get_images() == [], inline_page.get_images()) +check("inline-only image appears as a type-1 text-dictionary block", + len(inline_blocks) == 1, inline_blocks) +check("inline-only image page is not classified as blank", + not inspected_page_is_blank(inline_page)) + +# The extraction XObject pass and inline pass are disjoint: type-1 blocks with +# positive xrefs belong to the first pass, while true inline blocks have xref 0. +xobject_canvas = canvas.Canvas("xobject-image.pdf", pagesize=A4) +xobject_canvas.drawImage("inline-only-source.png", 72, 700, width=80, height=80) +xobject_canvas.showPage() +xobject_canvas.save() + +def write_fully_clipped_image(path, *, inline): + clipped_canvas = canvas.Canvas(path, pagesize=(200, 200)) + clip_path = clipped_canvas.beginPath() + clip_path.rect(0, 0, 10, 10) + clipped_canvas.saveState() + clipped_canvas.clipPath(clip_path, stroke=0, fill=0) + draw = clipped_canvas.drawInlineImage if inline else clipped_canvas.drawImage + draw("inline-only-source.png", 60, 60, width=80, height=80) + clipped_canvas.restoreState() + clipped_canvas.showPage() + clipped_canvas.save() + + +clipped_image_results = {} +for clipped_kind, clipped_inline in (("xobject", False), ("inline", True)): + clipped_path = f"fully-clipped-{clipped_kind}.pdf" + write_fully_clipped_image(clipped_path, inline=clipped_inline) + clipped_doc = fitz.open(clipped_path) + clipped_page = clipped_doc[0] + placements, visible, visibility_unknown = viewable_images(clipped_page) + clipped_alpha = clipped_page.get_pixmap(alpha=True, annots=False) + clipped_image_results[clipped_kind] = { + "placements": len(placements), "visible": len(visible), + "unknown": visibility_unknown, + "painted_alpha": any(clipped_alpha.samples[clipped_alpha.n - 1::clipped_alpha.n]), + "blank": inspected_page_is_blank(clipped_page), + } +check("fully clipped XObject and inline placements remain diagnostic candidates", + all(result["placements"] == 1 for result in clipped_image_results.values()), + clipped_image_results) +check("alpha render excludes fully clipped XObject and inline images from blank evidence", + all(result == { + "placements": 1, "visible": 0, "unknown": False, + "painted_alpha": False, "blank": True, + } for result in clipped_image_results.values()), clipped_image_results) + +rotated_image_writer = pypdf.PdfWriter() +rotated_image_writer.append(pypdf.PdfReader("xobject-image.pdf")) +rotated_image_writer.pages[0].rotate(90) +with open("rotated-visible-image.pdf", "wb") as output: + rotated_image_writer.write(output) +rotated_image_doc = fitz.open("rotated-visible-image.pdf") +rotated_placements, rotated_visible, rotated_unknown = viewable_images(rotated_image_doc[0]) +check("visible image remains visible after page rotation", + len(rotated_placements) == len(rotated_visible) == 1 + and not rotated_unknown and not inspected_page_is_blank(rotated_image_doc[0]), + (rotated_placements, rotated_visible, rotated_unknown)) + +xobject_visibility_doc = fitz.open("xobject-image.pdf") +budget_results = {} +for budget_name in ( + "MAX_IMAGE_PLACEMENTS", "MAX_IMAGE_SOURCE_PIXELS", + "MAX_TOTAL_IMAGE_SOURCE_PIXELS", "MAX_IMAGE_RENDER_PIXELS", + "MAX_TOTAL_IMAGE_RENDER_PIXELS", +): + original_budget = globals()[budget_name] + globals()[budget_name] = 0 + try: + budget_placements, budget_visible, budget_unknown = viewable_images( + xobject_visibility_doc[0] + ) + budget_page_is_blank = inspected_page_is_blank(xobject_visibility_doc[0]) + finally: + globals()[budget_name] = original_budget + budget_results[budget_name] = ( + len(budget_placements), len(budget_visible), + budget_unknown, budget_page_is_blank, + ) +check("every image count/source/render budget fails closed as visibility unknown", + all(result == (1, 0, True, False) for result in budget_results.values()), + budget_results) + +class FailingImageRenderPage: + rotation_matrix = fitz.Matrix(1, 1) + rect = fitz.Rect(0, 0, 200, 200) + + def get_image_info(self): + return [{"bbox": (20, 20, 80, 80), "width": 12, "height": 12}] + + def get_pixmap(self, **kwargs): + raise RuntimeError("fixture image decoder failure") + + +failed_placements, failed_visible, failed_unknown = viewable_images( + FailingImageRenderPage() +) +check("image decoder failure is reported as visibility unknown", + len(failed_placements) == 1 and not failed_visible and failed_unknown) + +# Editing can remove the only Do operation while leaving the image in /Resources. +# The resource inventory is then nonempty, but no image is painted. +unused_image_writer = pypdf.PdfWriter() +unused_image_writer.append(pypdf.PdfReader("xobject-image.pdf")) +empty_page_content = DecodedStreamObject() +empty_page_content.set_data(b"") +unused_image_writer.pages[0][NameObject("/Contents")] = ( + unused_image_writer._add_object(empty_page_content) +) +with open("unused-image-resource.pdf", "wb") as output: + unused_image_writer.write(output) +unused_image_doc = fitz.open("unused-image-resource.pdf") +unused_image_page = unused_image_doc[0] +check("unused image fixture retains an XObject resource (negative control)", + bool(unused_image_page.get_images()), unused_image_page.get_images()) +check("unused image resource has no painted placement", + unused_image_page.get_image_info(xrefs=True) == [], + unused_image_page.get_image_info(xrefs=True)) +check("unused image resource does not exempt an otherwise blank rendered page", + inspected_page_is_blank(unused_image_page) + and min(unused_image_page.get_pixmap(alpha=False).samples) == 255) + +# Placement geometry, rather than mere presence in the content stream, decides visibility. +placement_canvas = canvas.Canvas("image-placement-visibility.pdf", pagesize=(200, 200)) +placement_canvas.drawImage("inline-only-source.png", -40, 80, width=80, height=80) +placement_canvas.showPage() +placement_canvas.drawImage("inline-only-source.png", 250, 80, width=80, height=80) +placement_canvas.showPage() +placement_canvas.save() +placement_doc = fitz.open("image-placement-visibility.pdf") +check("partly intersecting image placement keeps a page nonblank", + not inspected_page_is_blank(placement_doc[0]), + placement_doc[0].get_image_info(xrefs=True)) +check("fully off-page image placement does not exempt a blank page", + inspected_page_is_blank(placement_doc[1]), + placement_doc[1].get_image_info(xrefs=True)) + + +def extract_images_without_duplicates(document, page, prefix): + outputs = [] + for i, info in enumerate(page.get_images(full=True), start=1): + pix = fitz.Pixmap(document, info[0]) + output = f"{prefix}-{i}.png" + pix.save(output) + outputs.append(output) + image_xrefs = { + image["number"]: image["xref"] + for image in page.get_image_info(xrefs=True) + } + for block in page.get_text("dict")["blocks"]: + if block["type"] != 1 or image_xrefs.get(block["number"]) != 0: + continue + ext = block.get("ext") or "png" + output = f"{prefix}-inline-{block['number']}.{ext}" + with open(output, "wb") as fh: + fh.write(block["image"]) + outputs.append(output) + return outputs + + +xobject_doc = fitz.open("xobject-image.pdf") +xobject_blocks = [ + block for block in xobject_doc[0].get_text("dict")["blocks"] + if block["type"] == 1 +] +xobject_info = xobject_doc[0].get_image_info(xrefs=True) +xobject_outputs = extract_images_without_duplicates(xobject_doc, xobject_doc[0], "xobject-export") +check("ordinary XObject image info exposes a positive xref", + len(xobject_blocks) == 1 and len(xobject_info) == 1 + and xobject_info[0]["xref"] > 0, + xobject_info) +check("ordinary XObject is exported once, not duplicated by the inline pass", + len(xobject_outputs) == 1 and "-inline-" not in xobject_outputs[0], xobject_outputs) +inline_outputs = extract_images_without_duplicates(inline_doc, inline_page, "inline-export") +check("true inline image is exported by the xref-zero second pass", + len(inline_outputs) == 1 and "-inline-" in inline_outputs[0] + and os.path.getsize(inline_outputs[0]) > 0, + inline_outputs) + +# ---- extract.md CMYK conversion snippet --------------------------------------- +pix = fitz.Pixmap(fitz.csCMYK, fitz.IRect(0, 0, 24, 24)) # CMYK pixmap like a CMYK PDF image +converted = fitz.Pixmap(fitz.csRGB, pix) if pix.colorspace not in (fitz.csGRAY, fitz.csRGB) else pix +converted.save("cmyk-converted.png") +check("CMYK pixmap converts to a saved PNG", os.path.getsize("cmyk-converted.png") > 0) +rgb = fitz.Pixmap("cmyk-converted.png") +check("converted pixmap is RGB", "RGB" in str(rgb.colorspace), rgb.colorspace) + +# ---- extract.md soft-mask composition: transparent image keeps alpha ----------- +rgba = fitz.Pixmap(fitz.csRGB, fitz.IRect(0, 0, 8, 8), True) +for y in range(rgba.height): + for x in range(rgba.width): + rgba.set_pixel(x, y, (255, 0, 0, 255 if x < 4 else 64)) +rgba.save("transparent-source.png") + +transparent_pdf = canvas.Canvas("transparent-image.pdf", pagesize=A4) +transparent_pdf.drawImage( + "transparent-source.png", 72, 700, width=80, height=80, mask="auto" +) +transparent_pdf.save() + +transparent_doc = fitz.open("transparent-image.pdf") +image_info = transparent_doc[0].get_images(full=True)[0] +check("transparent PDF image exposes a soft-mask xref", image_info[1] > 0, image_info) +base = fitz.Pixmap(transparent_doc, image_info[0]) +if base.colorspace and base.colorspace not in (fitz.csGRAY, fitz.csRGB): + base = fitz.Pixmap(fitz.csRGB, base) +mask = fitz.Pixmap(transparent_doc, image_info[1]) +composited = fitz.Pixmap(base, mask) +composited.save("transparent-extracted.png") +reopened_composite = fitz.Pixmap("transparent-extracted.png") +check("soft-mask composition keeps an alpha channel", reopened_composite.alpha == 1) +check( + "soft-mask composition keeps varying transparency", + len(set(reopened_composite.samples[3::4])) > 1, + set(reopened_composite.samples[3::4]), +) + +# ---- create.md rule: escape plain text before Paragraph ------------------------ +from reportlab.lib.pagesizes import A4 as A4_SIZE +from reportlab.lib.styles import getSampleStyleSheet +from reportlab.platypus import Paragraph, SimpleDocTemplate +from xml.sax.saxutils import escape + +MESSY = "R&D spend & \"pipeline\" > forecast" +try: + SimpleDocTemplate("escaped.pdf", pagesize=A4_SIZE).build( + [Paragraph(escape(MESSY), getSampleStyleSheet()["BodyText"])] + ) + build_error = "" +except Exception as exc: # unescaped markup typically raises a paraparser error + build_error = str(exc) +check("escaped messy text builds without paraparser error", build_error == "", build_error) +esc_text = " ".join(page.get_text() for page in fitz.open("escaped.pdf")) +check("escaped text extracts with original characters", + "R&D spend " in esc_text and "\"pipeline\"" in esc_text, esc_text[:120]) + +unescaped_failed = False +try: + SimpleDocTemplate("raw.pdf", pagesize=A4_SIZE).build( + [Paragraph(MESSY, getSampleStyleSheet()["BodyText"])] + ) +except Exception: + unescaped_failed = True +if unescaped_failed: + check("unescaped markup is proven dangerous (negative control)", True) +else: + # lenient inputs build but render mangled: markup is swallowed, entities reinterpreted + raw_text = " ".join(page.get_text() for page in fitz.open("raw.pdf")) + check( + "unescaped markup is proven dangerous (negative control)", + "" not in raw_text or "R&D;" in raw_text, + raw_text[:120], + ) + +# ---- extract.md table route: find_tables instead of raw span soup --------------- +from reportlab.lib import colors +from reportlab.platypus import Table as RlTable, TableStyle + +rl_table = RlTable( + [["Region", "Sales"], ["North", "120"], ["South", "340"]], + style=TableStyle([("GRID", (0, 0), (-1, -1), 0.5, colors.black)]), +) +SimpleDocTemplate("table.pdf", pagesize=A4_SIZE).build([rl_table]) + +table_doc = fitz.open("table.pdf") +detected = table_doc[0].find_tables() +check("find_tables detects the drawn table", len(detected.tables) == 1, len(detected.tables)) +if detected.tables: + extracted_rows = detected.tables[0].extract() + check("find_tables extracts the header row", extracted_rows[0] == ["Region", "Sales"], extracted_rows) + check("find_tables extracts data rows", extracted_rows[2] == ["South", "340"], extracted_rows) + + +# ---- transform.md: stamps fit non-zero-origin and rotated destination pages ------ + +mixed_writer = PdfWriter() +mixed_writer.append(open_pdf("form.pdf")) +small_source = PdfWriter() +small_source.add_blank_page(width=200, height=300) +offset_page = small_source.add_blank_page(width=200, height=300) +offset_page.mediabox.lower_left = (100, 200) +offset_page.mediabox.upper_right = (300, 500) +offset_page.cropbox.lower_left = (100, 200) +offset_page.cropbox.upper_right = (300, 500) +rotated_page = small_source.add_blank_page(width=240, height=160) +rotated_page.rotate(90) +from pypdf.generic import ArrayObject, DictionaryObject, FloatObject, NameObject, TextStringObject +rotated_link = DictionaryObject({ + NameObject("/Type"): NameObject("/Annot"), + NameObject("/Subtype"): NameObject("/Link"), + NameObject("/Rect"): ArrayObject([FloatObject(value) for value in (20, 30, 100, 60)]), + NameObject("/Border"): ArrayObject([FloatObject(0), FloatObject(0), FloatObject(0)]), + NameObject("/A"): DictionaryObject({ + NameObject("/S"): NameObject("/URI"), + NameObject("/URI"): TextStringObject("https://example.invalid/rotated-link"), + }), +}) +rotated_page[NameObject("/Annots")] = ArrayObject([small_source._add_object(rotated_link)]) +cropped_page = small_source.add_blank_page(width=400, height=500) +cropped_page.cropbox.lower_left = (250, 300) +cropped_page.cropbox.upper_right = (390, 480) +mixed_writer.append(small_source) +with open("mixed.pdf", "wb") as f: + mixed_writer.write(f) + +def first_annotation_rect(page): + return tuple(float(value) for value in page["/Annots"][0].get_object()["/Rect"]) + +mixed_rotated_page = R2("mixed.pdf").pages[4] +rotated_geometry_before = (mixed_rotated_page.rotation, first_annotation_rect(mixed_rotated_page)) + +# Negative control: a plain merge keeps the A4 stamp's coordinates, so the text +# lands outside the small page and cannot be extracted. +plain_writer = PdfWriter() +plain_writer.append(open_pdf("mixed.pdf")) +for page in plain_writer.pages: + page.merge_page(R2("stamp.pdf").pages[0]) +with open("plain-stamped.pdf", "wb") as f: + plain_writer.write(f) + +def stamp_bboxes(path, page_number): + doc = fitz.open(path) + spans = [] + for block in doc[page_number].get_text("dict")["blocks"]: + for line in block.get("lines", []): + for span in line["spans"]: + if "DRAFT" in span["text"]: + spans.append(span["bbox"]) + return spans + +def stamp_line_directions(path, page_number): + doc = fitz.open(path) + return [ + line["dir"] + for block in doc[page_number].get_text("dict")["blocks"] + for line in block.get("lines", []) + if any("DRAFT" in span["text"] for span in line["spans"]) + ] + +# The plain merge keeps the A4 stamp's coordinates, so on the 200x300 page the +# stamp is far outside the box: PyMuPDF's positioned extraction sees no span at +# all, while pypdf's plain extractor still returns the text - proof that text +# extraction alone cannot validate visual placement. +plain_spans = stamp_bboxes("plain-stamped.pdf", 2) +check("plain merge pushes the stamp outside the small page (negative control)", + plain_spans == [] and "DRAFT" in (R2("plain-stamped.pdf").pages[2].extract_text() or ""), + plain_spans) + +# Negative control for crop-box fitting: media-box centering puts the stamp +# outside this page's small, offset visible region. +media_fit_writer = PdfWriter() +media_fit_writer.append(open_pdf("mixed.pdf"), pages=(5, 6)) +media_fit_page = media_fit_writer.pages[0] +media_destination = media_fit_page.mediabox +media_scale = min(float(media_destination.width) / float(stamp_box.width), + float(media_destination.height) / float(stamp_box.height)) +media_tx = (float(media_destination.left) + + (float(media_destination.width) - float(stamp_box.width) * media_scale) / 2 + - float(stamp_box.left) * media_scale) +media_ty = (float(media_destination.bottom) + + (float(media_destination.height) - float(stamp_box.height) * media_scale) / 2 + - float(stamp_box.bottom) * media_scale) +media_fit_page.merge_transformed_page( + stamp, Transformation().scale(media_scale).translate(media_tx, media_ty), +) +with open("media-fit-cropped.pdf", "wb") as f: + media_fit_writer.write(f) +check("media-box fitting misses an offset crop region (negative control)", + stamp_bboxes("media-fit-cropped.pdf", 0) == []) + +scaled_writer = PdfWriter() +scaled_writer.append(open_pdf("mixed.pdf")) +stamp_page = R2("stamp.pdf").pages[0] +stamp_page.transfer_rotation_to_content() +stamp_box2 = stamp_page.cropbox +sw2, sh2 = float(stamp_box2.width), float(stamp_box2.height) +for page in scaled_writer.pages: + page.merge_transformed_page(stamp_page, stamp_placement(page, stamp_box2)) +with open("scaled-stamped.pdf", "wb") as f: + scaled_writer.write(f) +scaled_check = R2("scaled-stamped.pdf") +check("scaled stamp is present on the A4 pages", + all("DRAFT" in (p.extract_text() or "") for p in scaled_check.pages[:2])) +scaled_spans = stamp_bboxes("scaled-stamped.pdf", 2) +check("scaled stamp lands inside the mixed-size small page", + bool(scaled_spans) and all(bbox[1] < 300 and bbox[3] <= 300.5 for bbox in scaled_spans), + scaled_spans) +offset_spans = stamp_bboxes("scaled-stamped.pdf", 3) +rotated_spans = stamp_bboxes("scaled-stamped.pdf", 4) +cropped_spans = stamp_bboxes("scaled-stamped.pdf", 5) +check("scaled stamp lands inside the non-zero-origin page", bool(offset_spans), offset_spans) +rotated_fitz_page = fitz.open("scaled-stamped.pdf")[4] +rotated_visible_spans = [ + fitz.Rect(bbox) * rotated_fitz_page.rotation_matrix for bbox in rotated_spans +] +rotated_visible_directions = [ + ( + direction[0] * rotated_fitz_page.rotation_matrix.a + + direction[1] * rotated_fitz_page.rotation_matrix.c, + direction[0] * rotated_fitz_page.rotation_matrix.b + + direction[1] * rotated_fitz_page.rotation_matrix.d, + ) + for direction in stamp_line_directions("scaled-stamped.pdf", 4) +] +check("scaled stamp stays horizontal in the rotated page's visible space", + bool(rotated_spans) + and all(rect.width > rect.height for rect in rotated_visible_spans) + and all(dx > 0.9 and abs(dy) < 0.1 for dx, dy in rotated_visible_directions), + (rotated_visible_spans, rotated_visible_directions)) +cropped_rect = fitz.open("scaled-stamped.pdf")[5].rect +check("scaled stamp lands inside the offset visible crop box", + bool(cropped_spans) + and all( + bbox[0] >= -0.5 and bbox[1] >= -0.5 + and bbox[2] <= cropped_rect.width + 0.5 + and bbox[3] <= cropped_rect.height + 0.5 + for bbox in cropped_spans + ), cropped_spans) +check("mixed-size pages keep their original media boxes", + [(round(float(p.mediabox.width)), round(float(p.mediabox.height))) + for p in scaled_check.pages] + == [(595, 842), (595, 842), (200, 300), (200, 300), (240, 160), (400, 500)]) +scaled_rotated_page = scaled_check.pages[4] +check("watermarking preserves rotated-page annotation geometry", + (scaled_rotated_page.rotation, first_annotation_rect(scaled_rotated_page)) + == rotated_geometry_before, + ((scaled_rotated_page.rotation, first_annotation_rect(scaled_rotated_page)), + rotated_geometry_before)) +mixed_link_before = tuple( + round(float(value), 4) + for value in fitz.open("mixed.pdf")[4].get_links()[0]["from"] +) +mixed_link_after = tuple( + round(float(value), 4) + for value in fitz.open("scaled-stamped.pdf")[4].get_links()[0]["from"] +) +check("watermarking preserves the rotated link's visible hit rectangle", + mixed_link_after == mixed_link_before, (mixed_link_before, mixed_link_after)) + + +# ---- SKILL.md overflow check: off-page text and graphics are defects ------------ +overflow_ok = canvas.Canvas("overflow.pdf", pagesize=A4) +overflow_ok.setFont("Helvetica", 16) +overflow_ok.drawString(72, 780, "fits on page") +overflow_ok.showPage() +overflow_ok.save() +overflow_bad = canvas.Canvas("overflow-bad.pdf", pagesize=A4) +overflow_bad.setFont("Helvetica", 16) +overflow_bad.drawString(72, -200, "drawn far below the page box") +overflow_bad.showPage() +overflow_bad.save() +overflow_very_far = canvas.Canvas("overflow-very-far.pdf", pagesize=A4) +overflow_very_far.setFont("Helvetica", 16) +overflow_very_far.drawString(72, -5000, "drawn beyond the old finite search window") +overflow_very_far.showPage() +overflow_very_far.save() + +rotated_source = canvas.Canvas("overflow-rotated-source.pdf", pagesize=A4) +rotated_source.setFont("Helvetica", 16) +rotated_source.drawString(72, 30, "valid near the unrotated page bottom") +rotated_source.showPage() +rotated_source.save() +rotated_writer = PdfWriter() +rotated_writer.append(R2("overflow-rotated-source.pdf")) +rotated_writer.pages[0].rotate(90) +with open("overflow-rotated.pdf", "wb") as f: + rotated_writer.write(f) + +graphics_ok = canvas.Canvas("overflow-graphics-ok.pdf", pagesize=A4) +graphics_ok.drawInlineImage("inline-only-source.png", 72, 700, width=80, height=80) +graphics_ok.setLineWidth(2) +graphics_ok.rect(72, 600, 100, 50, stroke=1, fill=0) +graphics_ok.showPage() +graphics_ok.save() + +image_bad = canvas.Canvas("overflow-image-bad.pdf", pagesize=A4) +image_bad.drawInlineImage("inline-only-source.png", 560, 700, width=80, height=80) +image_bad.showPage() +image_bad.save() + +drawing_bad = canvas.Canvas("overflow-drawing-bad.pdf", pagesize=A4) +drawing_bad.setLineWidth(2) +drawing_bad.rect(560, 600, 80, 50, stroke=1, fill=0) +drawing_bad.showPage() +drawing_bad.save() + +rotated_graphics_source = canvas.Canvas("overflow-graphics-rotated-source.pdf", pagesize=A4) +rotated_graphics_source.drawInlineImage( + "inline-only-source.png", 500, 700, width=80, height=80, +) +rotated_graphics_source.setLineWidth(2) +rotated_graphics_source.rect(20, 20, 80, 40, stroke=1, fill=0) +rotated_graphics_source.showPage() +rotated_graphics_source.save() +rotated_graphics_writer = PdfWriter() +rotated_graphics_writer.append(R2("overflow-graphics-rotated-source.pdf")) +rotated_graphics_writer.pages[0].rotate(90) +with open("overflow-graphics-rotated.pdf", "wb") as f: + rotated_graphics_writer.write(f) + +def overflow_pages(path, password=None): + doc = fitz.open(path) + if doc.needs_pass and doc.authenticate("") <= 0: + if not password: + raise RuntimeError(f"valid password required to overflow-check {path}") + if doc.authenticate(password) <= 0: + raise RuntimeError(f"password could not decrypt {path} for overflow checking") + pages = [] + for page in doc: + # Plain block extraction drops fully off-page text; disable clipping. + crop_left, crop_bottom, crop_right, crop_top = normalized_box(page.cropbox) + page_box = fitz.Rect(0, 0, crop_right - crop_left, crop_top - crop_bottom) + text_rects = [ + fitz.Rect(block[:4]) + for block in page.get_text("blocks", clip=fitz.INFINITE_RECT()) + if block[6] == 0 + ] + image_rects = [ + fitz.Rect(0, 0, 1, 1) * fitz.Matrix(*image["transform"]) + for image in page.get_image_info() + ] + drawing_rects = [drawing_bounds(drawing) for drawing in page.get_drawings()] + if any(rect.x0 < page_box.x0 - 0.5 or rect.y0 < page_box.y0 - 0.5 + or rect.x1 > page_box.x1 + 0.5 or rect.y1 > page_box.y1 + 0.5 + for rect in text_rects + image_rects + drawing_rects): + pages.append(page.number + 1) + doc.close() + return pages + +check("in-bounds PDF reports no overflow pages", overflow_pages("overflow.pdf") == []) +check("blank-user-password encrypted PDF passes the independent overflow check", + overflow_pages("blank-user-password.pdf") == []) +check("reversed page boxes remain valid through the overflow postcheck", + overflow_pages("widget-reversed-page-boxes.pdf") == []) +check("off-page text is detected by the overflow check (negative control)", + overflow_pages("overflow-bad.pdf") == [1]) +very_far_probe = fitz.open("overflow-very-far.pdf")[0] +old_finite_clip = fitz.Rect( + very_far_probe.rect.x0 - 2000, very_far_probe.rect.y0 - 2000, + very_far_probe.rect.x1 + 2000, very_far_probe.rect.y1 + 2000, +) +check("the old finite clip misses text positioned more than 2,000 points away (negative control)", + not [block for block in very_far_probe.get_text("blocks", clip=old_finite_clip) + if block[6] == 0]) +check("the unbounded overflow check detects very distant positioned text", + overflow_pages("overflow-very-far.pdf") == [1]) +check("in-bounds image and vector drawing pass the overflow check", + overflow_pages("overflow-graphics-ok.pdf") == []) +check("out-of-bounds image placement is detected", + overflow_pages("overflow-image-bad.pdf") == [1]) +check("out-of-bounds vector drawing is detected", + overflow_pages("overflow-drawing-bad.pdf") == [1]) +rotated_probe = fitz.open("overflow-rotated.pdf")[0] +rotated_blocks = rotated_probe.get_text( + "blocks", clip=fitz.Rect(-2000, -2000, 2000, 3000), +) +rotation_blind_flag = any( + block[2] > rotated_probe.rect.width + 0.5 + or block[3] > rotated_probe.rect.height + 0.5 + for block in rotated_blocks if block[6] == 0 +) +check("rotated rect comparison falsely flags valid text (negative control)", rotation_blind_flag) +check("overflow check compares rotated pages in unrotated coordinates", + overflow_pages("overflow-rotated.pdf") == []) +rotated_graphics_probe = fitz.open("overflow-graphics-rotated.pdf")[0] +rotated_graphic_rects = ( + [fitz.Rect(0, 0, 1, 1) * fitz.Matrix(*image["transform"]) + for image in rotated_graphics_probe.get_image_info()] + + [drawing_bounds(drawing) for drawing in rotated_graphics_probe.get_drawings()] +) +rotation_blind_graphics_flag = any( + rect.x1 > rotated_graphics_probe.rect.width + 0.5 + or rect.y1 > rotated_graphics_probe.rect.height + 0.5 + for rect in rotated_graphic_rects +) +check("rotated rect comparison falsely flags valid graphics (negative control)", + rotation_blind_graphics_flag, rotated_graphic_rects) +check("graphic overflow check preserves rotated coordinate correctness", + overflow_pages("overflow-graphics-rotated.pdf") == []) +check("off-page text still extracts, so extraction alone cannot catch it", + "drawn far below" in (pypdf.PdfReader("overflow-bad.pdf").pages[0].extract_text() or "")) + + +print("\n" + ("ALL PDF FIXTURES PASSED" if not failures else f"{len(failures)} FAILURES: {failures}")) +sys.exit(0 if not failures else 1) diff --git a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py new file mode 100644 index 0000000..c352f0e --- /dev/null +++ b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py @@ -0,0 +1,3031 @@ +# Minimal runnable fixtures for the snippets called out in review: one file per format. +# Each script is self-contained, writes only scratch files into the current directory, +# and exits non-zero on failed assertions. Run from any scratch directory: +# python pdf_fixture.py (deps: reportlab, pypdf, pymupdf) +# python pptx_fixture.py (deps: python-pptx) +# python xlsx_fixture.py (deps: openpyxl) +# python docx_fixture.py (deps: python-docx, pymupdf, soffice on PATH) +import base64 +import copy +import sys +import xml.etree.ElementTree as ET +import zipfile +from pathlib import Path + +from lxml import etree +from pptx import Presentation +from pptx.chart.data import BubbleChartData, ChartData, XyChartData +from pptx.dml.color import RGBColor +from pptx.enum.chart import XL_CHART_TYPE +from pptx.enum.shapes import MSO_SHAPE_TYPE, PP_PLACEHOLDER +from pptx.opc.constants import RELATIONSHIP_TYPE as RT +from pptx.util import Inches, Pt + +failures = [] + + +def check(name, cond, extra=""): + print(("PASS " if cond else "FAIL ") + name + ((" :: " + str(extra)) if not cond and extra else "")) + if not cond: + failures.append(name) + + +# ---- build the deck ------------------------------------------------------------ +prs = Presentation() +slide = prs.slides.add_slide(prs.slide_layouts[5]) # blank + +box = slide.shapes.add_textbox(Inches(1), Inches(1), Inches(8), Inches(1)) +tf = box.text_frame +p = tf.paragraphs[0] +r1 = p.add_run() +r1.text = "old wording" +r2 = p.add_run() +r2.text = " linked part" +r2.font.italic = True +r2.font.color.rgb = RGBColor(0xC0, 0x00, 0x00) +r2.hyperlink.address = "https://example.com/docs" + +table_shape = slide.shapes.add_table(2, 2, Inches(1), Inches(2.5), Inches(6), Inches(1)) +table = table_shape.table +cell = table.cell(0, 1) +ctf = cell.text_frame +cp = ctf.paragraphs[0] +cr = cp.add_run() +cr.text = "old cell text" +cr.font.bold = True +cr.font.color.rgb = RGBColor(0x00, 0x70, 0xC0) + +chart_data = ChartData() +chart_data.categories = ["EU", "US"] +chart_data.add_series("Units", (120, 80)) +chart = slide.shapes.add_chart( + XL_CHART_TYPE.COLUMN_CLUSTERED, + Inches(7.2), Inches(2.5), Inches(2), Inches(2), + chart_data, +).chart +chart.has_title = True +chart.chart_title.text_frame.text = "Units by region" +chart.category_axis.has_title = True +chart.category_axis.axis_title.text_frame.text = "Region" +chart.value_axis.has_title = True +chart.value_axis.axis_title.text_frame.text = "Units sold" + +xy_data = XyChartData() +xy_series = xy_data.add_series("Trend") +xy_series.add_data_point(1, 2) +xy_series.add_data_point(3, 4) +xy_chart = slide.shapes.add_chart( + XL_CHART_TYPE.XY_SCATTER, Inches(7.2), Inches(4.7), Inches(2), Inches(2), xy_data, +).chart +xy_chart.has_title = True +xy_chart.chart_title.text_frame.text = "XY trend" + +bubble_data = BubbleChartData() +bubble_series = bubble_data.add_series("Risk") +bubble_series.add_data_point(3, 4, 5) +bubble_chart = slide.shapes.add_chart( + XL_CHART_TYPE.BUBBLE, Inches(5), Inches(4.7), Inches(2), Inches(2), bubble_data, +).chart +bubble_chart.has_title = True +bubble_chart.chart_title.text_frame.text = "Bubble risk" +slide.notes_slide.notes_text_frame.text = "Speaker note: explain the regional split." + +prs.save("input.pptx") + +# ---- create.md skeleton: explicit placeholder selection and populated table ---- +def placeholder_of_type(slide, *types): + matches = [ + ph for ph in slide.placeholders + if ph.placeholder_format.type in types + ] + if len(matches) != 1: + available = [ + f"{ph.name} ({ph.placeholder_format.type})" + for ph in slide.placeholders + ] + raise ValueError( + f"expected exactly one placeholder of {types}, found {len(matches)}; " + f"available placeholders: {available or 'none'}" + ) + return matches[0] + + +selector_prs = Presentation() +no_object_slide = selector_prs.slides.add_slide(selector_prs.slide_layouts[6]) +try: + placeholder_of_type(no_object_slide, PP_PLACEHOLDER.OBJECT) + missing_placeholder_message = None +except ValueError as exc: + missing_placeholder_message = str(exc) +check( + "placeholder selector explicitly rejects zero matches", + missing_placeholder_message is not None + and "found 0" in missing_placeholder_message + and "available placeholders: none" in missing_placeholder_message, + missing_placeholder_message, +) + +two_object_slide = selector_prs.slides.add_slide(selector_prs.slide_layouts[3]) +try: + placeholder_of_type(two_object_slide, PP_PLACEHOLDER.OBJECT) + ambiguous_placeholder_message = None +except ValueError as exc: + ambiguous_placeholder_message = str(exc) +check( + "placeholder selector explicitly rejects multiple matches", + ambiguous_placeholder_message is not None + and "found 2" in ambiguous_placeholder_message + and "Content Placeholder 2" in ambiguous_placeholder_message + and "Content Placeholder 3" in ambiguous_placeholder_message, + ambiguous_placeholder_message, +) + +skeleton_prs = Presentation() +skeleton_slide = skeleton_prs.slides.add_slide(skeleton_prs.slide_layouts[5]) +skeleton_slide.shapes.title.text = "Regional service health" +skeleton_headers = ["Region", "Error rate", "P99 latency"] +skeleton_body = [ + ["Americas", "0.08%", "182 ms"], + ["Europe", "0.05%", "164 ms"], + ["Asia Pacific", "0.11%", "213 ms"], +] +skeleton_table = skeleton_slide.shapes.add_table( + 1 + len(skeleton_body), len(skeleton_headers), + Inches(0.5), Inches(1.5), Inches(9), Inches(3.5), +).table +for column_index, text in enumerate(skeleton_headers): + skeleton_table.cell(0, column_index).text = text +for row_index, row in enumerate(skeleton_body, start=1): + for column_index, text in enumerate(row): + skeleton_table.cell(row_index, column_index).text = text +skeleton_prs.save("create-skeleton.pptx") +skeleton_reopened = Presentation("create-skeleton.pptx") +skeleton_reopened_table = next( + shape.table for shape in skeleton_reopened.slides[0].shapes if shape.has_table +) +skeleton_values = [ + [cell.text.strip() for cell in row.cells] + for row in skeleton_reopened_table.rows +] +check( + "creation skeleton table has no unexpected empty cells", + skeleton_values == [skeleton_headers, *skeleton_body] + and all(text for row in skeleton_values for text in row), + skeleton_values, +) + +# ---- analyze.md bounded package check rejects archive bombs before expansion --- +MAX_ARCHIVE_BYTES = 200 * 1024 * 1024 +MAX_MEMBERS = 10_000 +MAX_XML_PART = 20 * 1024 * 1024 +MAX_ENTRY = 100 * 1024 * 1024 +MAX_TOTAL_UNCOMPRESSED = 500 * 1024 * 1024 +MAX_COMPRESSION_RATIO = 200 +CONTENT_TYPES_NS = "http://schemas.openxmlformats.org/package/2006/content-types" +SAFE_XML = etree.XMLParser( + load_dtd=False, + resolve_entities=False, + no_network=True, + huge_tree=False, + recover=False, +) + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def validate_pptx_package(source): + source.seek(0, 2) + compressed_size = source.tell() + source.seek(0) + require(compressed_size <= MAX_ARCHIVE_BYTES, + "compressed PPTX file size above limit") + with zipfile.ZipFile(source) as archive: + infos = archive.infolist() + require(len(infos) <= MAX_MEMBERS, "archive member count above limit") + names = {info.filename for info in infos} + require(len(names) == len(infos), "duplicate archive member names are unsafe") + require("[Content_Types].xml" in names and "ppt/presentation.xml" in names, + "missing required OPC members") + require(sum(info.file_size for info in infos) <= MAX_TOTAL_UNCOMPRESSED, + "declared total uncompressed size above limit") + + content_types_info = next( + info for info in infos if info.filename == "[Content_Types].xml" + ) + require(content_types_info.file_size <= MAX_XML_PART, + "oversized XML part: [Content_Types].xml") + require( + content_types_info.file_size / max(content_types_info.compress_size, 1) + <= MAX_COMPRESSION_RATIO, + "suspicious compression ratio: [Content_Types].xml", + ) + content_type_chunks = [] + content_type_size = 0 + with archive.open(content_types_info) as stream: + while chunk := stream.read(64 * 1024): + content_type_size += len(chunk) + require(content_type_size <= MAX_XML_PART, + "part exceeded read limit: [Content_Types].xml") + content_type_chunks.append(chunk) + require(content_type_size == content_types_info.file_size, + "size mismatch: [Content_Types].xml") + content_types_blob = b"".join(content_type_chunks) + content_types_root = etree.fromstring(content_types_blob, parser=SAFE_XML) + require(content_types_root.tag == f"{{{CONTENT_TYPES_NS}}}Types", + "invalid content-types root") + default_types = {} + override_types = {} + for declaration in content_types_root: + if declaration.tag == f"{{{CONTENT_TYPES_NS}}}Default": + key = (declaration.get("Extension") or "").casefold() + target = default_types + elif declaration.tag == f"{{{CONTENT_TYPES_NS}}}Override": + part_name = declaration.get("PartName") or "" + require(part_name.startswith("/"), "invalid content-type part name") + key = part_name[1:].casefold() + target = override_types + else: + continue + content_type = declaration.get("ContentType") or "" + require(key and content_type and key not in target, + "invalid or duplicate content-type declaration") + target[key] = content_type.partition(";")[0].strip().casefold() + + actual_total = 0 + for info in infos: + require(info.file_size <= MAX_ENTRY, f"oversized part: {info.filename}") + ratio = info.file_size / max(info.compress_size, 1) + require(ratio <= MAX_COMPRESSION_RATIO, + f"suspicious compression ratio: {info.filename}") + extension = info.filename.rpartition(".")[2].casefold() + content_type = override_types.get( + info.filename.casefold(), default_types.get(extension, "") + ) + is_xml = ( + info.filename.casefold().endswith((".xml", ".rels")) + or content_type in {"application/xml", "text/xml"} + or content_type.endswith("+xml") + ) + if is_xml: + require(info.file_size <= MAX_XML_PART, f"oversized XML part: {info.filename}") + chunks = [] + actual_size = 0 + if info.filename == "[Content_Types].xml": + chunks = [content_types_blob] + actual_size = len(content_types_blob) + actual_total += actual_size + else: + with archive.open(info) as stream: + while chunk := stream.read(64 * 1024): + actual_size += len(chunk) + actual_total += len(chunk) + require(actual_size <= MAX_ENTRY, + f"part exceeded read limit: {info.filename}") + require(actual_total <= MAX_TOTAL_UNCOMPRESSED, + "archive exceeded total read limit") + if is_xml: + chunks.append(chunk) + require(actual_total <= MAX_TOTAL_UNCOMPRESSED, + "archive exceeded total read limit") + require(actual_size == info.file_size, f"size mismatch: {info.filename}") + if is_xml: + etree.fromstring(b"".join(chunks), parser=SAFE_XML) + source.seek(0) + + +def open_validated_presentation(path): + with Path(path).open("rb") as source: + validate_pptx_package(source) + source.seek(0) + return Presentation(source) + + +try: + with Path("input.pptx").open("rb") as source: + validate_pptx_package(source) + healthy_pptx_passed = True +except Exception: + healthy_pptx_passed = False +check("bounded PPTX check accepts an ordinary deck", healthy_pptx_passed) + +Path("oversized-before-open.pptx").write_bytes(b"not a ZIP package") +original_archive_limit = MAX_ARCHIVE_BYTES +MAX_ARCHIVE_BYTES = 0 +try: + with Path("oversized-before-open.pptx").open("rb") as source: + validate_pptx_package(source) + compressed_size_rejected_before_open = False +except ValueError as exc: + compressed_size_rejected_before_open = ( + str(exc) == "compressed PPTX file size above limit" + ) +finally: + MAX_ARCHIVE_BYTES = original_archive_limit +check( + "compressed PPTX size is bounded before ZipFile opens the package", + compressed_size_rejected_before_open, +) + +with zipfile.ZipFile("too-many-members.pptx", "w", zipfile.ZIP_STORED) as archive: + for member_index in range(MAX_MEMBERS + 1): + archive.writestr(f"zero-{member_index:05d}.bin", b"") +try: + with Path("too-many-members.pptx").open("rb") as source: + validate_pptx_package(source) + many_members_rejected = False +except ValueError as exc: + many_members_rejected = str(exc) == "archive member count above limit" +check( + "member-count gate rejects 10,001 zero-byte members before traversal", + many_members_rejected, +) + +with zipfile.ZipFile("input.pptx") as source_archive: + compressed_bomb_payload = { + info.filename: source_archive.read(info) for info in source_archive.infolist() + } +compressed_bomb_member = next( + name for name in compressed_bomb_payload + if not name.casefold().endswith((".xml", ".rels")) +) +compressed_bomb_payload[compressed_bomb_member] = b"x" * 2_000_000 +with zipfile.ZipFile("compressed-bomb.pptx", "w", zipfile.ZIP_DEFLATED) as archive: + for member_name, member_data in compressed_bomb_payload.items(): + archive.writestr(member_name, member_data) +try: + with Path("compressed-bomb.pptx").open("rb") as source: + validate_pptx_package(source) + pptx_bomb_rejected = False +except ValueError as error: + pptx_bomb_rejected = ( + str(error) == f"suspicious compression ratio: {compressed_bomb_member}" + ) +check("valid-manifest PPTX compression bomb reaches the ratio gate", + pptx_bomb_rejected, compressed_bomb_member) + +with zipfile.ZipFile("input.pptx") as source_archive: + uppercase_xml_payload = { + info.filename: source_archive.read(info) for info in source_archive.infolist() + } +uppercase_slide = uppercase_xml_payload.pop("ppt/slides/slide1.xml") +uppercase_xml_limit = 64 * 1024 +uppercase_xml_payload["ppt/slides/slide1.XML"] = uppercase_slide.replace( + b"", b" " * uppercase_xml_limit + b"", 1, +) +uppercase_xml_payload["[Content_Types].xml"] = uppercase_xml_payload[ + "[Content_Types].xml" +].replace(b"/ppt/slides/slide1.xml", b"/ppt/slides/slide1.XML") +uppercase_xml_payload["ppt/_rels/presentation.xml.rels"] = uppercase_xml_payload[ + "ppt/_rels/presentation.xml.rels" +].replace(b"slides/slide1.xml", b"slides/slide1.XML") +with zipfile.ZipFile("uppercase-xml-part.pptx", "w", zipfile.ZIP_STORED) as archive: + for member_name, member_data in uppercase_xml_payload.items(): + archive.writestr(member_name, member_data) +uppercase_route_parses = len(Presentation("uppercase-xml-part.pptx").slides) == 1 +original_xml_limit = MAX_XML_PART +MAX_XML_PART = uppercase_xml_limit +try: + with Path("uppercase-xml-part.pptx").open("rb") as source: + validate_pptx_package(source) + uppercase_oversized_xml_rejected = False +except ValueError as error: + uppercase_oversized_xml_rejected = ( + str(error) == "oversized XML part: ppt/slides/slide1.XML" + ) +finally: + MAX_XML_PART = original_xml_limit +check("uppercase XML part names cannot bypass the XML size and parser gate", + uppercase_route_parses and uppercase_oversized_xml_rejected) + +with zipfile.ZipFile("input.pptx") as source_archive: + typed_xml_payload = { + info.filename: source_archive.read(info) for info in source_archive.infolist() + } +typed_slide = typed_xml_payload.pop("ppt/slides/slide1.xml") +typed_xml_payload["ppt/slides/Slide1.DaT"] = typed_slide.replace( + b"", b" " * uppercase_xml_limit + b"", 1, +) +typed_xml_payload["[Content_Types].xml"] = typed_xml_payload[ + "[Content_Types].xml" +].replace(b"/ppt/slides/slide1.xml", b"/PPT/SLIDES/SLIDE1.DAT") +typed_xml_payload["ppt/_rels/presentation.xml.rels"] = typed_xml_payload[ + "ppt/_rels/presentation.xml.rels" +].replace(b"slides/slide1.xml", b"slides/Slide1.DaT") +with zipfile.ZipFile("content-typed-xml-part.pptx", "w", zipfile.ZIP_STORED) as archive: + for member_name, member_data in typed_xml_payload.items(): + archive.writestr(member_name, member_data) +typed_xml_route_parses = len(Presentation("content-typed-xml-part.pptx").slides) == 1 +original_xml_limit = MAX_XML_PART +MAX_XML_PART = uppercase_xml_limit +try: + with Path("content-typed-xml-part.pptx").open("rb") as source: + validate_pptx_package(source) + typed_oversized_xml_rejected = False +except ValueError as error: + typed_oversized_xml_rejected = ( + str(error) == "oversized XML part: ppt/slides/Slide1.DaT" + ) +finally: + MAX_XML_PART = original_xml_limit +check("case-insensitive XML content types cannot bypass bounds with an arbitrary extension", + typed_xml_route_parses and typed_oversized_xml_rejected) + +mixed_case_xml_rejections = [] +for mixed_case_name in ("custom.XmL", "custom.ReLs"): + with zipfile.ZipFile("input.pptx") as source_archive: + mixed_case_payload = { + info.filename: source_archive.read(info) for info in source_archive.infolist() + } + mixed_case_payload[mixed_case_name] = ( + b"" + b" " * uppercase_xml_limit + b"" + ) + mixed_case_declaration = ( + f'' + ).encode() + mixed_case_payload["[Content_Types].xml"] = mixed_case_payload[ + "[Content_Types].xml" + ].replace(b"", mixed_case_declaration + b"", 1) + mixed_case_path = "mixed-case-" + mixed_case_name.replace(".", "-") + ".pptx" + with zipfile.ZipFile(mixed_case_path, "w", zipfile.ZIP_STORED) as archive: + for member_name, member_data in mixed_case_payload.items(): + archive.writestr(member_name, member_data) + original_xml_limit = MAX_XML_PART + MAX_XML_PART = uppercase_xml_limit + try: + with Path(mixed_case_path).open("rb") as source: + validate_pptx_package(source) + except ValueError as error: + if str(error) == f"oversized XML part: {mixed_case_name}": + mixed_case_xml_rejections.append(mixed_case_name) + finally: + MAX_XML_PART = original_xml_limit +check("mixed-case XML and relationship suffixes keep XML bounds", + mixed_case_xml_rejections == ["custom.XmL", "custom.ReLs"], + mixed_case_xml_rejections) + +with zipfile.ZipFile("input.pptx") as source_archive: + default_xml_payload = { + info.filename: source_archive.read(info) for info in source_archive.infolist() + } +default_xml_payload["custom.payload"] = ( + b"" + b" " * uppercase_xml_limit + b"" +) +default_xml_declaration = ( + b'' +) +default_xml_payload["[Content_Types].xml"] = default_xml_payload[ + "[Content_Types].xml" +].replace(b"", default_xml_declaration + b"", 1) +with zipfile.ZipFile("default-content-type-xml.pptx", "w", zipfile.ZIP_STORED) as archive: + for member_name, member_data in default_xml_payload.items(): + archive.writestr(member_name, member_data) +original_xml_limit = MAX_XML_PART +MAX_XML_PART = uppercase_xml_limit +try: + with Path("default-content-type-xml.pptx").open("rb") as source: + validate_pptx_package(source) + default_typed_xml_rejected = False +except ValueError as error: + default_typed_xml_rejected = str(error) == "oversized XML part: custom.payload" +finally: + MAX_XML_PART = original_xml_limit +check("default +xml content types apply XML bounds to arbitrary extensions", + default_typed_xml_rejected) + +duplicate_override_payload = dict(typed_xml_payload) +duplicate_override = ( + b'' +) +duplicate_override_payload["[Content_Types].xml"] = duplicate_override_payload[ + "[Content_Types].xml" +].replace(b"", duplicate_override + b"", 1) +with zipfile.ZipFile("duplicate-case-override.pptx", "w", zipfile.ZIP_STORED) as archive: + for member_name, member_data in duplicate_override_payload.items(): + archive.writestr(member_name, member_data) +try: + with Path("duplicate-case-override.pptx").open("rb") as source: + validate_pptx_package(source) + duplicate_case_override_rejected = False +except ValueError as error: + duplicate_case_override_rejected = ( + str(error) == "invalid or duplicate content-type declaration" + ) +check("case-insensitive duplicate content-type overrides fail closed", + duplicate_case_override_rejected) + +presentation_loader_calls = [] +real_presentation_loader = Presentation + + +def unexpected_presentation_loader(source): + presentation_loader_calls.append(source) + return real_presentation_loader(source) + + +Presentation = unexpected_presentation_loader +try: + open_validated_presentation("compressed-bomb.pptx") + bomb_rejected_before_presentation = False +except ValueError: + bomb_rejected_before_presentation = not presentation_loader_calls +finally: + Presentation = real_presentation_loader +check("content inventory rejects a package bomb before Presentation parses it", + bomb_rejected_before_presentation, len(presentation_loader_calls)) + +validated_sources = [] +parsed_sources = [] +parsed_source_open_states = [] +real_package_validator = validate_pptx_package +real_presentation_loader = Presentation + + +def tracking_package_validator(source): + validated_sources.append(source) + return real_package_validator(source) + + +def tracking_presentation_loader(source): + parsed_sources.append(source) + parsed_source_open_states.append(not source.closed) + return real_presentation_loader(source) + + +validate_pptx_package = tracking_package_validator +Presentation = tracking_presentation_loader +try: + same_handle_prs = open_validated_presentation("input.pptx") +finally: + validate_pptx_package = real_package_validator + Presentation = real_presentation_loader +check("validated presentation parses the exact same open handle before closing it", + len(same_handle_prs.slides) == 1 + and len(validated_sources) == len(parsed_sources) == 1 + and validated_sources[0] is parsed_sources[0] + and parsed_source_open_states == [True] + and validated_sources[0].closed) + +# ---- edit.md snippet: validated single-shape run replace keeps styling/link ---- +edit_loader_calls = [] +real_presentation_loader = Presentation + + +def unexpected_edit_loader(source): + edit_loader_calls.append(source) + return real_presentation_loader(source) + + +Presentation = unexpected_edit_loader +try: + open_validated_presentation("compressed-bomb.pptx") + edit_bomb_rejected_before_parse = False +except ValueError: + edit_bomb_rejected_before_parse = not edit_loader_calls +finally: + Presentation = real_presentation_loader +check("edit route rejects a package bomb before Presentation parses it", + edit_bomb_rejected_before_parse, edit_loader_calls) + +prs = open_validated_presentation("input.pptx") +old, new = "old wording", "new wording" +candidates = [] +for i, s in enumerate(prs.slides): + for shape in s.shapes: + if shape.has_text_frame and old in shape.text_frame.text: + candidates.append((i, shape.name, shape)) +require(len(candidates) == 1, "expected exactly one text target") +_, _, target_shape = candidates[0] +tf = target_shape.text_frame +run_hits = [run for par in tf.paragraphs for run in par.runs if old in run.text] +require(len(run_hits) == 1, "target is duplicated or split across runs") +run_hits[0].text = run_hits[0].text.replace(old, new, 1) + +edited_link = [run for par in tf.paragraphs for run in par.runs if run.hyperlink.address] +check("run replace keeps the other run's hyperlink", len(edited_link) == 1 and edited_link[0].hyperlink.address == "https://example.com/docs") +styled = [run for par in tf.paragraphs for run in par.runs if run.font.italic] +check("run replace keeps sibling run styling", len(styled) == 1 and styled[0].font.color.rgb == RGBColor(0xC0, 0x00, 0x00)) +prs.save("edited.pptx") + +# ---- edit.md snippet: table cell edited at run level --------------------------- +prs2 = Presentation("input.pptx") +old_cell, new_cell = "old cell text", "new cell text" +tbl = next(sh for sh in prs2.slides[0].shapes if sh.has_table).table +cell = tbl.cell(0, 1) +hits = [run for par in cell.text_frame.paragraphs for run in par.runs if old_cell in run.text] +require(len(hits) == 1, "target is duplicated or split across runs in this cell") +hits[0].text = hits[0].text.replace(old_cell, new_cell, 1) + +prs2.save("cell-edited.pptx") +prs3 = Presentation("cell-edited.pptx") +cell3 = next(sh for sh in prs3.slides[0].shapes if sh.has_table).table.cell(0, 1) +after_runs = [run for par in cell3.text_frame.paragraphs for run in par.runs] +check("cell run edit keeps bold", any(r.font.bold for r in after_runs)) +check("cell run edit keeps color", any(r.font.color and r.font.color.rgb == RGBColor(0x00, 0x70, 0xC0) for r in after_runs)) +check("cell run edit changed the text", cell3.text_frame.text == "new cell text") + +# the dangerous variant for contrast: assigning cell.text drops run properties +prs4 = Presentation("input.pptx") +tbl4 = next(sh for sh in prs4.slides[0].shapes if sh.has_table).table +tbl4.cell(0, 1).text = "new cell text" +prs4.save("cell-flattened.pptx") +prs5 = Presentation("cell-flattened.pptx") +flat_runs = [run for par in next(sh for sh in prs5.slides[0].shapes if sh.has_table).table.cell(0, 1).text_frame.paragraphs for run in par.runs] +check("cell.text assignment is proven lossy (negative control)", not any(r.font.bold for r in flat_runs)) + +# ---- analyze.md snippet: grouped-shape walker ---------------------------------- +from pptx.oxml import parse_xml +from pptx.oxml.ns import qn +from pptx.oxml.xmlchemy import OxmlElement + +prs6 = Presentation() +slide6 = prs6.slides.add_slide(prs6.slide_layouts[5]) +pic_holder = slide6.shapes.add_textbox(Inches(1), Inches(1), Inches(4), Inches(1)) +pic_holder.text_frame.text = "nested member" +pic_holder.text_frame.paragraphs[0].runs[0].font.name = "Grouped Face" + +GRP = ( + '' + "" + "" + "" + '' + '' + "" +) + + +def iter_shapes(shapes): + for shape in shapes: + if shape_is_hidden(shape): + continue + if shape.shape_type == MSO_SHAPE_TYPE.GROUP: + yield from iter_shapes(shape.shapes) + else: + yield shape + + +OOXML_TRUE = {"1", "true"} +OOXML_FALSE = {"0", "false"} + + +def ooxml_bool(element, attribute, default): + value = element.get(attribute) + if value is None: + return default + value = value.strip(" \t\r\n") + if value in OOXML_TRUE: + return True + if value in OOXML_FALSE: + return False + raise ValueError(f"invalid OOXML boolean {attribute}={value!r}") + + +def shape_is_hidden(shape): + properties = shape._element.find(".//" + qn("p:cNvPr")) + return properties is not None and ooxml_bool(properties, "hidden", False) + + +def layer_text_content(shapes, source, *, inherited): + records = [] + for shape in shapes: + if shape_is_hidden(shape): + continue + if shape.shape_type == MSO_SHAPE_TYPE.GROUP: + records.extend(layer_text_content(shape.shapes, source, inherited=inherited)) + continue + if inherited and shape.is_placeholder: + continue + if shape.has_text_frame and shape.text_frame.text: + records.append({"source": source, "shape": shape.name, + "text": shape.text_frame.text}) + return records + + +def slide_text_content(slide): + records = [] + if ooxml_bool(slide._element, "showMasterSp", True): + layout = slide.slide_layout + if ooxml_bool(layout._element, "showMasterSp", True): + records.extend(layer_text_content( + layout.slide_master.shapes, "master", inherited=True, + )) + records.extend(layer_text_content(layout.shapes, "layout", inherited=True)) + records.extend(layer_text_content(slide.shapes, "slide", inherited=False)) + return records + + +def table_cells(table): + return [[ + { + "row": row_index, + "column": column_index, + "text": None if cell.is_spanned else cell.text, + "is_merge_origin": cell.is_merge_origin, + "is_spanned": cell.is_spanned, + "span_width": cell.span_width, + "span_height": cell.span_height, + } + for column_index, cell in enumerate(row.cells) + ] for row_index, row in enumerate(table.rows)] + + +def picture_content(shape): + if getattr(type(shape), "image", None) is None: + return None + try: + image = shape.image + return { + "name": shape.name, + "filename": image.filename, + "extension": image.ext, + "bytes": len(image.blob), + } + except (AttributeError, KeyError, OSError, ValueError): + blips = shape._element.xpath(".//a:blip") + relationship_id = ( + blips[0].get(qn("r:embed")) if len(blips) == 1 else None + ) + return { + "name": shape.name, + "status": "unreadable", + "relationship_id": relationship_id, + "reason": "missing or invalid image relationship or payload", + } + + +MAX_CHART_POINTS = 100_000 +MAX_CATEGORY_LEVELS = 16 +CATEGORY_SOURCE_NAMES = {"strRef", "numRef", "multiLvlStrRef", "strLit", "numLit"} + + +def cache_point_count(cache): + point_counts = cache.findall(qn("c:ptCount")) + if len(point_counts) != 1: + return None + try: + point_count = int(point_counts[0].get("val")) + except (TypeError, ValueError): + return None + return point_count if 0 <= point_count <= MAX_CHART_POINTS else None + + +def consume_point_budget(point_budget, count): + if count > MAX_CHART_POINTS: + return False + if point_budget is None: + return True + if count > point_budget["remaining"]: + return False + point_budget["remaining"] -= count + return True + + +def cached_numeric_points( + source, *, fill_missing=False, include_count=False, point_budget=None +): + if source is None: + return None + caches = source.xpath("./c:numRef/c:numCache | ./c:numLit") + if len(caches) != 1 or (point_count := cache_point_count(caches[0])) is None: + return None + if not consume_point_budget(point_budget, point_count): + return None + cached = {} + for point in caches[0].findall(qn("c:pt")): + value = point.find(qn("c:v")) + try: + index = int(point.get("idx")) + except (TypeError, ValueError): + return None + if value is None or not 0 <= index < point_count or index in cached: + return None + try: + parsed = float(value.text) if value.text not in (None, "") else value.text + except (TypeError, ValueError): + parsed = value.text + cached[index] = parsed + if fill_missing: + points = [(index, cached.get(index)) for index in range(point_count)] + else: + points = sorted(cached.items()) + return {"point_count": point_count, "points": points} if include_count else points + + +def cached_text_point_map(container, point_count): + values = {} + for point in container.findall(qn("c:pt")): + try: + index = int(point.get("idx")) + except (TypeError, ValueError): + return None + value_nodes = point.findall(qn("c:v")) + if (len(value_nodes) != 1 or not 0 <= index < point_count + or index in values): + return None + values[index] = value_nodes[0].text or "" + return values + + +def category_source(series): + category = series._element.find(qn("c:cat")) + if category is None: + return None, None + sources = [ + child for child in category + if isinstance(child.tag, str) + and etree.QName(child).localname in CATEGORY_SOURCE_NAMES + ] + return (category, sources[0]) if len(sources) == 1 else (category, None) + + +def cached_category_labels(series, point_budget=None): + category, source = category_source(series) + if category is None: + return [] + if source is None: + return None + source_name = etree.QName(source).localname + + if source_name == "multiLvlStrRef": + caches = source.findall(qn("c:multiLvlStrCache")) + if len(caches) != 1 or (count := cache_point_count(caches[0])) is None: + return None + level_nodes = caches[0].findall(qn("c:lvl")) + if len(level_nodes) > MAX_CATEGORY_LEVELS or (count and not level_nodes): + return None + levels = [cached_text_point_map(level, count) for level in level_nodes] + if any(points is None for points in levels): + return None + if not consume_point_budget(point_budget, count * max(len(level_nodes), 1)): + return None + expanded_levels = [] + for level_index, points in enumerate(levels): + if level_index == 0: + expanded_levels.append([points.get(index, "") for index in range(count)]) + continue + ordered = sorted(points.items()) + if not ordered: + expanded_levels.append([""] * count) + continue + cursor = 0 + expanded = [] + for leaf_index in range(count): + while cursor + 1 < len(ordered) and ordered[cursor + 1][0] <= leaf_index: + cursor += 1 + expanded.append(ordered[cursor][1]) + expanded_levels.append(expanded) + return [ + [level[index] for level in reversed(expanded_levels)] + for index in range(count) + ] + + cache_name = {"strRef": "c:strCache", "numRef": "c:numCache"}.get(source_name) + if cache_name is None: + cache = source + else: + caches = source.findall(qn(cache_name)) + if len(caches) != 1: + return None + cache = caches[0] + if (count := cache_point_count(cache)) is None: + return None + values = cached_text_point_map(cache, count) + if values is None: + return None + if not consume_point_budget(point_budget, count): + return None + return [[values.get(index, "")] for index in range(count)] + + +def category_content(series, point_budget=None): + labels = cached_category_labels(series, point_budget) + _, source = category_source(series) + formula = None if source is None else source.find(qn("c:f")) + return { + "categories": labels, + "category_formula": formula.text if formula is not None else None, + "category_cache_status": "unavailable" if labels is None else "available", + } + + +def series_name_content(series): + titles = series._element.xpath("./c:tx") + if not titles: + return {"name": ""} + literal = titles[0].find(qn("c:v")) + if literal is not None: + return {"name": literal.text or ""} + reference = titles[0].find(qn("c:strRef")) + if reference is None: + return {"name": None, "name_cache_status": "unavailable"} + cache = reference.find(qn("c:strCache")) + if cache is None: + return {"name": None, "name_cache_status": "unavailable"} + point_count = cache.find(qn("c:ptCount")) + points = cache.findall(qn("c:pt")) + if (point_count is None or point_count.get("val") != "1" + or len(points) != 1 or points[0].get("idx") != "0"): + return {"name": None, "name_cache_status": "unavailable"} + value = points[0].find(qn("c:v")) + if value is None: + return {"name": None, "name_cache_status": "unavailable"} + return {"name": value.text or ""} + + +def series_content(series, *, include_categories=False, point_budget=None): + if point_budget is None: + point_budget = {"remaining": MAX_CHART_POINTS} + name_content = series_name_content(series) + content = { + **name_content, + **(category_content(series, point_budget) if include_categories else {}), + } + x_source = getattr(series._element, "xVal", None) + if x_source is None: + value_source = getattr(series._element, "val", None) + value_cache = cached_numeric_points( + value_source, fill_missing=True, include_count=True, + point_budget=point_budget, + ) + if value_cache is None: + return {**content, "values": None, "cache_status": "unavailable"} + return {**content, "values": [value for _, value in value_cache["points"]]} + x_cache = cached_numeric_points( + x_source, include_count=True, point_budget=point_budget + ) + y_cache = cached_numeric_points( + getattr(series._element, "yVal", None), include_count=True, + point_budget=point_budget, + ) + if x_cache is None or y_cache is None: + return {**content, "points": None, "cache_status": "unavailable"} + content.update({ + "x_points": x_cache["points"], "x_point_count": x_cache["point_count"], + "y_points": y_cache["points"], "y_point_count": y_cache["point_count"], + }) + size_source = getattr(series._element, "bubbleSize", None) + if size_source is not None: + bubble_cache = cached_numeric_points( + size_source, include_count=True, point_budget=point_budget + ) + if bubble_cache is None: + return {**content, "points": None, "cache_status": "unavailable"} + content.update({ + "bubble_points": bubble_cache["points"], + "bubble_point_count": bubble_cache["point_count"], + }) + return content + + +def title_text(title): + if title is None: + return "" + text_nodes = title.findall(qn("c:tx")) + if len(text_nodes) != 1: + return None + sources = [child for child in text_nodes[0] if isinstance(child.tag, str)] + if len(sources) != 1: + return None + if sources[0].tag == qn("c:rich"): + paragraphs = sources[0].findall(qn("a:p")) + return None if not paragraphs else "\n".join( + paragraph.text for paragraph in paragraphs + ) + if sources[0].tag != qn("c:strRef"): + return None + formulas = sources[0].findall(qn("c:f")) + if len(formulas) != 1 or not formulas[0].text: + return None + caches = sources[0].findall(qn("c:strCache")) + if (len(caches) != 1 or (count := cache_point_count(caches[0])) is None + or count != 1): + return None + values = cached_text_point_map(caches[0], count) + return None if values is None or set(values) != {0} else values[0] + + +def chart_axis_text(axis): + return title_text(axis.find(qn("c:title"))) + + +def chart_title_text(chart): + titles = chart._element.xpath("./c:chart/c:title") + return title_text(titles[0]) if len(titles) == 1 else ("" if not titles else None) + + +def chart_axes(chart): + axes = chart._element.xpath( + "./c:chart/c:plotArea/c:catAx | ./c:chart/c:plotArea/c:dateAx | " + "./c:chart/c:plotArea/c:valAx | ./c:chart/c:plotArea/c:serAx" + ) + return [ + { + "kind": etree.QName(axis).localname, + "id": axis.find(qn("c:axId")).get("val") if axis.find(qn("c:axId")) is not None else None, + "position": ( + axis.find(qn("c:axPos")).get("val") + if axis.find(qn("c:axPos")) is not None else None + ), + "cross_axis_id": ( + axis.find(qn("c:crossAx")).get("val") + if axis.find(qn("c:crossAx")) is not None else None + ), + "title": chart_axis_text(axis), + } + for axis in axes + ] + + +DIAGRAM_NS = "http://schemas.openxmlformats.org/drawingml/2006/diagram" +SAFE_DIAGRAM_XML = etree.XMLParser(load_dtd=False, resolve_entities=False, no_network=True) + + +def smartart_content(shape): + graphic_data = shape._element.find(".//" + qn("a:graphicData")) + if graphic_data is None or graphic_data.get("uri") != DIAGRAM_NS: + return None + rel_ids = graphic_data.find(f".//{{{DIAGRAM_NS}}}relIds") + relationship_id = None if rel_ids is None else rel_ids.get(qn("r:dm")) + if not relationship_id: + return {"name": shape.name, "status": "unreadable", "reason": "missing data relationship"} + try: + data_part = shape.part.related_part(relationship_id) + except (KeyError, ValueError): + return {"name": shape.name, "status": "unreadable", "reason": relationship_id} + try: + root = etree.fromstring(data_part.blob, parser=SAFE_DIAGRAM_XML) + except etree.XMLSyntaxError as error: + return {"name": shape.name, "status": "unreadable", "reason": str(error)} + labels = [node.text for node in root.iter(qn("a:t")) if node.text] + return {"name": shape.name, "status": "ok", "text": labels} + + +def extract_slide_content(slide, point_budget=None): + shapes = list(iter_shapes(slide.shapes)) + if point_budget is None: + point_budget = {"remaining": MAX_CHART_POINTS} + text = slide_text_content(slide) + tables = [ + table_cells(sh.table) + for sh in shapes if sh.has_table + ] + charts = [] + for sh in shapes: + if not sh.has_chart: + continue + chart = sh.chart + chart_title = chart_title_text(chart) + plots = [] + for plot in chart.plots: + items = list(plot.series) + has_xy_values = any(getattr(item._element, "xVal", None) is not None for item in items) + series = [ + series_content( + item, include_categories=not has_xy_values, point_budget=point_budget + ) + for item in items + ] + plots.append({ + "kind": type(plot).__name__, + "series": series, + }) + charts.append({ + "title": chart_title, + "axes": chart_axes(chart), + "plots": plots, + }) + notes = slide.notes_slide.notes_text_frame.text if slide.has_notes_slide else "" + pictures = [ + info for shape in shapes + if (info := picture_content(shape)) is not None + ] + smartart = [ + info for shape in shapes + if (info := smartart_content(shape)) is not None + ] + return {"text": text, "tables": tables, "charts": charts, + "pictures": pictures, "smartart": smartart, "notes": notes} + + +content = extract_slide_content(open_validated_presentation("input.pptx").slides[0]) +check("content inventory emits body text", + any("old wording" in value["text"] for value in content["text"]), content) +check("content inventory emits table cell text", + content["tables"][0][0][1]["text"] == "old cell text", content["tables"]) +check( + "content inventory emits chart title, categories, series, and values", + content["charts"][0]["title"] == "Units by region" + and content["charts"][0]["plots"][0]["series"][0]["categories"] + == [["EU"], ["US"]] + and content["charts"][0]["plots"][0]["series"][0]["category_cache_status"] + == "available" + and content["charts"][0]["plots"][0]["series"][0]["values"] == [120.0, 80.0], + content["charts"], +) +check("content inventory emits raw category and value axis metadata", + {axis["title"] for axis in content["charts"][0]["axes"]} + == {"Region", "Units sold"} + and all(axis["id"] and axis["position"] and axis["cross_axis_id"] + for axis in content["charts"][0]["axes"]), content["charts"][0]) + +chart_title_prs = Presentation("input.pptx") +chart_title_chart = next( + shape.chart for shape in chart_title_prs.slides[0].shapes if shape.has_chart +) +chart_title_element = chart_title_chart._element.xpath("./c:chart/c:title")[0] +chart_title_tx = chart_title_element.find(qn("c:tx")) +for child in list(chart_title_tx): + chart_title_tx.remove(child) +chart_title_ref = OxmlElement("c:strRef") +chart_title_formula = OxmlElement("c:f") +chart_title_formula.text = "Sheet1!$G$1" +chart_title_cache = OxmlElement("c:strCache") +chart_title_count = OxmlElement("c:ptCount") +chart_title_count.set("val", "1") +chart_title_point = OxmlElement("c:pt") +chart_title_point.set("idx", "0") +chart_title_value = OxmlElement("c:v") +chart_title_value.text = "Cached chart title" +chart_title_point.append(chart_title_value) +chart_title_cache.extend([chart_title_count, chart_title_point]) +chart_title_ref.extend([chart_title_formula, chart_title_cache]) +chart_title_tx.append(chart_title_ref) +chart_title_prs.save("chart-title-strref.pptx") + +chart_title_reopened = Presentation("chart-title-strref.pptx") +raw_title_chart = next( + shape.chart for shape in chart_title_reopened.slides[0].shapes if shape.has_chart +) +raw_title_element = raw_title_chart._element.xpath("./c:chart/c:title")[0] +title_xml_before = etree.tostring(raw_title_element) +cached_chart_title = chart_title_text(raw_title_chart) +title_xml_after = etree.tostring(raw_title_element) +check("raw chart title reads a worksheet cache without mutating strRef", + cached_chart_title == "Cached chart title" + and title_xml_before == title_xml_after + and raw_title_element.xpath("./c:tx/c:strRef/c:f")[0].text == "Sheet1!$G$1" + and not raw_title_element.xpath("./c:tx/c:rich"), + cached_chart_title) +inventory_title_xml_before = etree.tostring(raw_title_element) +cached_title_inventory = extract_slide_content(chart_title_reopened.slides[0]) +inventory_title_xml_after = etree.tostring(raw_title_element) +check("full content inventory keeps a worksheet-backed chart title and its XML intact", + cached_title_inventory["charts"][0]["title"] == "Cached chart title" + and inventory_title_xml_before == inventory_title_xml_after + and raw_title_element.xpath("./c:tx/c:strRef/c:f")[0].text == "Sheet1!$G$1" + and not raw_title_element.xpath("./c:tx/c:rich"), + cached_title_inventory["charts"][0]) + +cacheless_chart_title = copy.deepcopy(raw_title_element) +cacheless_chart_title.xpath("./c:tx/c:strRef")[0].remove( + cacheless_chart_title.xpath("./c:tx/c:strRef/c:strCache")[0] +) +empty_chart_title = copy.deepcopy(raw_title_element) +empty_chart_title.xpath("./c:tx/c:strRef/c:strCache/c:pt/c:v")[0].text = None +ambiguous_chart_title = copy.deepcopy(raw_title_element) +ambiguous_chart_title.find(qn("c:tx")).append(OxmlElement("c:v")) +literal_only_chart_title = copy.deepcopy(raw_title_element) +literal_only_tx = literal_only_chart_title.find(qn("c:tx")) +literal_only_tx.remove(literal_only_tx.find(qn("c:strRef"))) +literal_only_value = OxmlElement("c:v") +literal_only_value.text = "invalid literal title" +literal_only_tx.append(literal_only_value) +foreign_namespace_title = copy.deepcopy(raw_title_element) +foreign_namespace_tx = foreign_namespace_title.find(qn("c:tx")) +foreign_namespace_ref = foreign_namespace_tx.find(qn("c:strRef")) +foreign_namespace_ref.tag = "{urn:foreign-chart-title}strRef" +empty_rich_title = copy.deepcopy(raw_title_element) +empty_rich_tx = empty_rich_title.find(qn("c:tx")) +empty_rich_tx.remove(empty_rich_tx.find(qn("c:strRef"))) +empty_rich_tx.append(OxmlElement("c:rich")) +duplicate_choice_title = copy.deepcopy(raw_title_element) +duplicate_choice_tx = duplicate_choice_title.find(qn("c:tx")) +duplicate_choice_tx.append(copy.deepcopy(duplicate_choice_tx.find(qn("c:strRef")))) +duplicate_cache_title = copy.deepcopy(raw_title_element) +duplicate_cache_ref = duplicate_cache_title.xpath("./c:tx/c:strRef")[0] +duplicate_cache_ref.append(copy.deepcopy(duplicate_cache_ref.find(qn("c:strCache")))) +missing_formula_title = copy.deepcopy(raw_title_element) +missing_formula_ref = missing_formula_title.xpath("./c:tx/c:strRef")[0] +missing_formula_ref.remove(missing_formula_ref.find(qn("c:f"))) +duplicate_formula_title = copy.deepcopy(raw_title_element) +duplicate_formula_ref = duplicate_formula_title.xpath("./c:tx/c:strRef")[0] +duplicate_formula_ref.append(copy.deepcopy(duplicate_formula_ref.find(qn("c:f")))) +missing_count_title = copy.deepcopy(raw_title_element) +missing_count_cache = missing_count_title.xpath("./c:tx/c:strRef/c:strCache")[0] +missing_count_cache.remove(missing_count_cache.find(qn("c:ptCount"))) +missing_point_title = copy.deepcopy(raw_title_element) +missing_point_cache = missing_point_title.xpath("./c:tx/c:strRef/c:strCache")[0] +missing_point_cache.remove(missing_point_cache.find(qn("c:pt"))) +missing_value_title = copy.deepcopy(raw_title_element) +missing_value_point = missing_value_title.xpath("./c:tx/c:strRef/c:strCache/c:pt")[0] +missing_value_point.remove(missing_value_point.find(qn("c:v"))) +duplicate_value_title = copy.deepcopy(raw_title_element) +duplicate_value_point = duplicate_value_title.xpath("./c:tx/c:strRef/c:strCache/c:pt")[0] +duplicate_value = OxmlElement("c:v") +duplicate_value.text = "second value" +duplicate_value_point.append(duplicate_value) + +rich_paragraph_title = copy.deepcopy(raw_title_element) +rich_paragraph_tx = rich_paragraph_title.find(qn("c:tx")) +rich_paragraph_tx.remove(rich_paragraph_tx.find(qn("c:strRef"))) +rich_title = OxmlElement("c:rich") +for text_value in ("First paragraph", "Second paragraph"): + paragraph = OxmlElement("a:p") + run = OxmlElement("a:r") + text_node = OxmlElement("a:t") + text_node.text = text_value + run.append(text_node) + paragraph.append(run) + rich_title.append(paragraph) +rich_paragraph_tx.append(rich_title) +check("chart title cache parser distinguishes unavailable, empty, and ambiguous choices", + title_text(cacheless_chart_title) is None + and title_text(empty_chart_title) == "" + and title_text(ambiguous_chart_title) is None + and title_text(literal_only_chart_title) is None + and title_text(foreign_namespace_title) is None + and title_text(empty_rich_title) is None + and title_text(duplicate_choice_title) is None + and title_text(duplicate_cache_title) is None + and title_text(missing_formula_title) is None + and title_text(duplicate_formula_title) is None + and title_text(missing_count_title) is None + and title_text(missing_point_title) is None + and title_text(missing_value_title) is None + and title_text(duplicate_value_title) is None + and title_text(rich_paragraph_title) == "First paragraph\nSecond paragraph") + + +def install_inherited_text(target_shapes, scratch_slide, text, *, grouped=False, hidden=False): + if grouped: + source_shape = scratch_slide.shapes.add_group_shape() + group_texts = (text,) if isinstance(text, str) else tuple(text) + for index, text_value in enumerate(group_texts): + child = source_shape.shapes.add_textbox( + Inches(1), Inches(1 + index), Inches(5), Inches(0.5) + ) + child.text = text_value + else: + source_shape = scratch_slide.shapes.add_textbox( + Inches(1), Inches(1), Inches(5), Inches(0.5) + ) + source_shape.text = text + copied = copy.deepcopy(source_shape._element) + next_shape_id = target_shapes._next_shape_id + for non_visual_properties in copied.iter(qn("p:cNvPr")): + non_visual_properties.set("id", str(next_shape_id)) + next_shape_id += 1 + if hidden: + copied.find(".//" + qn("p:cNvPr")).set("hidden", "1") + target_shapes._spTree.insert_element_before(copied, "p:extLst") + scratch_slide.shapes._spTree.remove(source_shape._element) + + +inheritance_prs = Presentation() +inheritance_layout = inheritance_prs.slide_layouts[5] +inheritance_slide = inheritance_prs.slides.add_slide(inheritance_layout) +inheritance_slide.shapes.title.text = "Actual slide title" +inheritance_layout_title = next( + shape for shape in inheritance_layout.placeholders + if shape.placeholder_format.type in (PP_PLACEHOLDER.TITLE, PP_PLACEHOLDER.CENTER_TITLE) +) +inheritance_layout_title.text = "TEMPLATE PLACEHOLDER PROMPT" +inheritance_master_placeholder = next(iter(inheritance_layout.slide_master.placeholders)) +inheritance_master_placeholder.text = "MASTER TEMPLATE PLACEHOLDER PROMPT" +install_inherited_text( + inheritance_layout.slide_master.shapes, inheritance_slide, "Master disclaimer" +) +install_inherited_text(inheritance_layout.shapes, inheritance_slide, "Layout disclaimer") +install_inherited_text( + inheritance_layout.shapes, inheritance_slide, + ("Grouped layout first", "Grouped layout second"), grouped=True, +) +install_inherited_text( + inheritance_layout.slide_master.shapes, inheritance_slide, + "Hidden master copy", hidden=True, +) +install_inherited_text( + inheritance_layout.shapes, inheritance_slide, + "Hidden grouped layout copy", grouped=True, hidden=True, +) +install_inherited_text( + inheritance_layout.shapes, inheritance_slide, "Whitespace-hidden layout copy" +) +whitespace_hidden_shape = next( + shape for shape in inheritance_layout.shapes + if shape.has_text_frame and shape.text_frame.text == "Whitespace-hidden layout copy" +) +whitespace_hidden_shape._element.find(".//" + qn("p:cNvPr")).set( + "hidden", " \ttrue\r\n" +) +hidden_slide_shape = inheritance_slide.shapes.add_textbox( + Inches(1), Inches(6), Inches(5), Inches(0.5) +) +hidden_slide_shape.text = "Hidden slide copy" +hidden_slide_shape._element.find(".//" + qn("p:cNvPr")).set("hidden", "1") +inheritance_prs.save("inherited-text.pptx") + +inherited_slide = open_validated_presentation("inherited-text.pptx").slides[0] +inherited_text = extract_slide_content(inherited_slide)["text"] +inherited_pairs = [(item["source"], item["text"]) for item in inherited_text] +inherited_master_ids = [ + node.get("id") for node in inherited_slide.slide_layout.slide_master._element.xpath( + ".//p:cNvPr" + ) +] +inherited_layout_ids = [ + node.get("id") for node in inherited_slide.slide_layout._element.xpath(".//p:cNvPr") +] +check("inherited-text fixture keeps unique non-visual shape ids per part", + len(inherited_master_ids) == len(set(inherited_master_ids)) + and len(inherited_layout_ids) == len(set(inherited_layout_ids)), + (inherited_master_ids, inherited_layout_ids)) +check("content inventory includes visible master and layout text with provenance", + inherited_pairs == [ + ("master", "Master disclaimer"), + ("layout", "Layout disclaimer"), + ("layout", "Grouped layout first"), + ("layout", "Grouped layout second"), + ("slide", "Actual slide title"), + ], + inherited_pairs) + +slide_hidden_prs = Presentation("inherited-text.pptx") +slide_hidden_prs.slides[0]._element.set("showMasterSp", "0") +slide_hidden_prs.save("slide-hides-inherited-text.pptx") +slide_hidden_pairs = [ + (item["source"], item["text"]) + for item in extract_slide_content( + open_validated_presentation("slide-hides-inherited-text.pptx").slides[0] + )["text"] +] +check("slide showMasterSp=false hides both layout and master copy", + ("slide", "Actual slide title") in slide_hidden_pairs + and not any(source in {"layout", "master"} for source, _ in slide_hidden_pairs), + slide_hidden_pairs) + +layout_hidden_prs = Presentation("inherited-text.pptx") +layout_hidden_prs.slides[0].slide_layout._element.set("showMasterSp", "false") +layout_hidden_prs.save("layout-hides-master-text.pptx") +layout_hidden_pairs = [ + (item["source"], item["text"]) + for item in extract_slide_content( + open_validated_presentation("layout-hides-master-text.pptx").slides[0] + )["text"] +] +check("layout showMasterSp=false hides only master copy", + ("layout", "Layout disclaimer") in layout_hidden_pairs + and ("slide", "Actual slide title") in layout_hidden_pairs + and not any(source == "master" for source, _ in layout_hidden_pairs), + layout_hidden_pairs) + +whitespace_false_prs = Presentation("inherited-text.pptx") +whitespace_false_prs.slides[0]._element.set("showMasterSp", " \t0\r\n") +whitespace_false_prs.save("whitespace-false-inherited-text.pptx") +whitespace_false_text = extract_slide_content( + open_validated_presentation("whitespace-false-inherited-text.pptx").slides[0] +)["text"] +check("xsd whitespace around false hides inherited layers", + all(item["source"] == "slide" for item in whitespace_false_text), + whitespace_false_text) + +whitespace_true_prs = Presentation("inherited-text.pptx") +whitespace_true_prs.slides[0]._element.set("showMasterSp", "\ttrue\n") +whitespace_true_prs.save("whitespace-true-inherited-text.pptx") +whitespace_true_pairs = [ + (item["source"], item["text"]) + for item in extract_slide_content( + open_validated_presentation("whitespace-true-inherited-text.pptx").slides[0] + )["text"] +] +check("xsd whitespace around true preserves inherited layers", + ("master", "Master disclaimer") in whitespace_true_pairs + and ("layout", "Layout disclaimer") in whitespace_true_pairs, + whitespace_true_pairs) + +invalid_visibility_values = ( + "maybe", "on", "off", "yes", "no", "TRUE", "False", "true false", +) +invalid_visibility_rejections = [] +for invalid_value in invalid_visibility_values: + invalid_visibility_slide = Presentation("inherited-text.pptx").slides[0] + invalid_visibility_slide._element.set("showMasterSp", invalid_value) + try: + extract_slide_content(invalid_visibility_slide) + except ValueError: + invalid_visibility_rejections.append(invalid_value) +check("non-xsd showMasterSp values fail closed under optimized Python", + invalid_visibility_rejections == list(invalid_visibility_values), + invalid_visibility_rejections) + +secondary_axis_prs = Presentation("input.pptx") +secondary_axis_chart = next( + shape.chart for shape in secondary_axis_prs.slides[0].shapes + if shape.has_chart and shape.chart.has_title + and shape.chart.chart_title.text_frame.text == "Units by region" +) +plot_area = secondary_axis_chart._element.xpath("./c:chart/c:plotArea")[0] +primary_category_axis = plot_area.find(qn("c:catAx")) +primary_value_axis = plot_area.find(qn("c:valAx")) +primary_category_id = primary_category_axis.find(qn("c:axId")).get("val") +primary_value_id = primary_value_axis.find(qn("c:axId")).get("val") +secondary_category_id, secondary_value_id = "91000001", "91000002" + +secondary_plot = copy.deepcopy(plot_area.xpath("./c:barChart")[0]) +for axis_id in secondary_plot.findall(qn("c:axId")): + axis_id.set("val", ( + secondary_category_id if axis_id.get("val") == primary_category_id + else secondary_value_id + )) +first_axis_index = min(plot_area.index(primary_category_axis), plot_area.index(primary_value_axis)) +plot_area.insert(first_axis_index, secondary_plot) + +secondary_category_axis = copy.deepcopy(primary_category_axis) +secondary_category_axis.find(qn("c:axId")).set("val", secondary_category_id) +secondary_category_axis.find(qn("c:axPos")).set("val", "t") +secondary_category_axis.find(qn("c:crossAx")).set("val", secondary_value_id) +secondary_category_text = secondary_category_axis.xpath("./c:title//a:t")[0] +secondary_category_text.text = "Secondary" +secondary_category_run = secondary_category_text.getparent() +secondary_category_break = OxmlElement("a:br") +secondary_category_tail = OxmlElement("a:r") +secondary_category_tail_text = OxmlElement("a:t") +secondary_category_tail_text.text = "region" +secondary_category_tail.append(secondary_category_tail_text) +secondary_category_run.addnext(secondary_category_break) +secondary_category_break.addnext(secondary_category_tail) + +secondary_value_axis = copy.deepcopy(primary_value_axis) +secondary_value_axis.find(qn("c:axId")).set("val", secondary_value_id) +secondary_value_axis.find(qn("c:axPos")).set("val", "r") +secondary_value_axis.find(qn("c:crossAx")).set("val", secondary_category_id) +secondary_title_tx = secondary_value_axis.find(qn("c:title")).find(qn("c:tx")) +for child in list(secondary_title_tx): + secondary_title_tx.remove(child) +secondary_title_ref = OxmlElement("c:strRef") +secondary_title_formula = OxmlElement("c:f") +secondary_title_formula.text = "Sheet1!$F$1" +secondary_title_cache = OxmlElement("c:strCache") +secondary_title_count = OxmlElement("c:ptCount") +secondary_title_count.set("val", "1") +secondary_title_point = OxmlElement("c:pt") +secondary_title_point.set("idx", "0") +secondary_title_value = OxmlElement("c:v") +secondary_title_value.text = "Percent" +secondary_title_point.append(secondary_title_value) +secondary_title_cache.extend([secondary_title_count, secondary_title_point]) +secondary_title_ref.extend([secondary_title_formula, secondary_title_cache]) +secondary_title_tx.append(secondary_title_ref) +empty_secondary_axis = copy.deepcopy(secondary_value_axis) +empty_secondary_axis.xpath("./c:title/c:tx/c:strRef/c:strCache/c:pt/c:v")[0].text = None +check("raw worksheet-backed axis title preserves an explicit empty cache value", + chart_axis_text(empty_secondary_axis) == "", chart_axis_text(empty_secondary_axis)) +multi_point_axis = copy.deepcopy(secondary_value_axis) +multi_point_axis.xpath("./c:title/c:tx/c:strRef/c:strCache/c:ptCount")[0].set("val", "2") +check("worksheet-backed axis title rejects a non-scalar cache before expansion", + chart_axis_text(multi_point_axis) is None) + +plot_area.extend([secondary_category_axis, secondary_value_axis]) +secondary_axis_prs.save("secondary-axes.pptx") +secondary_axis_reopened = Presentation("secondary-axes.pptx") +secondary_axis_content = extract_slide_content( + secondary_axis_reopened.slides[0] +)["charts"][0] +secondary_axes = {axis["title"]: axis for axis in secondary_axis_content["axes"]} +secondary_axis_reopened_chart = next( + shape.chart for shape in secondary_axis_reopened.slides[0].shapes + if shape.has_chart and shape.chart.has_title + and shape.chart.chart_title.text_frame.text == "Units by region" +) +secondary_title_refs = secondary_axis_reopened_chart._element.xpath( + f'./c:chart/c:plotArea/c:valAx[c:axId[@val="{secondary_value_id}"]]' + '/c:title/c:tx/c:strRef' +) +check("combination-chart inventory includes all primary and secondary axes", + len(secondary_axis_content["plots"]) == 2 + and set(secondary_axes) == {"Region", "Units sold", "Secondary\vregion", "Percent"} + and secondary_axes["Secondary\vregion"] == { + "kind": "catAx", "id": secondary_category_id, "position": "t", + "cross_axis_id": secondary_value_id, "title": "Secondary\vregion", + } + and secondary_axes["Percent"] == { + "kind": "valAx", "id": secondary_value_id, "position": "r", + "cross_axis_id": secondary_category_id, "title": "Percent", + }, secondary_axis_content) +check("raw axis-title inventory preserves a worksheet-backed strRef", + len(secondary_title_refs) == 1 + and secondary_title_refs[0].find(qn("c:f")).text == "Sheet1!$F$1", + secondary_title_refs) +chart_by_title = {item["title"]: item for item in content["charts"]} +check("content inventory emits XY x/y points", + chart_by_title["XY trend"]["plots"][0]["series"][0]["x_points"] + == [(0, 1.0), (1, 3.0)] + and chart_by_title["XY trend"]["plots"][0]["series"][0]["x_point_count"] == 2 + and chart_by_title["XY trend"]["plots"][0]["series"][0]["y_points"] + == [(0, 2.0), (1, 4.0)] + and chart_by_title["XY trend"]["plots"][0]["series"][0]["y_point_count"] == 2) +check("content inventory emits bubble x/y/size points", + chart_by_title["Bubble risk"]["plots"][0]["series"][0]["x_points"] == [(0, 3.0)] + and chart_by_title["Bubble risk"]["plots"][0]["series"][0]["y_points"] == [(0, 4.0)] + and chart_by_title["Bubble risk"]["plots"][0]["series"][0]["bubble_points"] + == [(0, 5.0)] + and chart_by_title["Bubble risk"]["plots"][0]["series"][0]["bubble_point_count"] + == 1) +check("content inventory emits notes text", "regional split" in content["notes"], content["notes"]) + +# Category/value series can retain an external workbook formula without a numCache. +category_prs = Presentation() +category_slide = category_prs.slides.add_slide(category_prs.slide_layouts[6]) +category_data = ChartData() +category_data.categories = ["A", "B"] +category_data.add_series("Missing cache", (1, 2)) +category_data.add_series("Cached series", (3, 4)) +category_slide.shapes.add_chart( + XL_CHART_TYPE.COLUMN_CLUSTERED, + Inches(0.5), Inches(0.5), Inches(6), Inches(4), category_data, +) +category_prs.save("category-cache-source.pptx") + +shared_deck_budget = {"remaining": 10} +budget_source = Presentation("category-cache-source.pptx").slides[0] +first_budgeted_slide = extract_slide_content(budget_source, shared_deck_budget) +second_budgeted_slide = extract_slide_content(budget_source, shared_deck_budget) +first_budgeted_items = first_budgeted_slide["charts"][0]["plots"][0]["series"] +second_budgeted_items = second_budgeted_slide["charts"][0]["plots"][0]["series"] +check("one shared logical-point budget bounds aggregate inventory across slides", + all(item["values"] is not None for item in first_budgeted_items) + and second_budgeted_items[0]["categories"] == [["A"], ["B"]] + and second_budgeted_items[0]["values"] is None + and second_budgeted_items[1]["categories"] is None + and shared_deck_budget["remaining"] == 0, + (first_budgeted_items, second_budgeted_items, shared_deck_budget)) + +series_name_mutated = Presentation("category-cache-source.pptx") +series_name_chart = next( + shape.chart for shape in series_name_mutated.slides[0].shapes if shape.has_chart +) +series_name_ref = list(series_name_chart.plots[0].series)[0]._element.xpath( + "./c:tx/c:strRef" +)[0] +series_name_formula = series_name_ref.find(qn("c:f")).text +series_name_ref.remove(series_name_ref.find(qn("c:strCache"))) +series_name_mutated.save("series-name-cacheless.pptx") +series_name_reopened = Presentation("series-name-cacheless.pptx") +series_name_chart = next( + shape.chart for shape in series_name_reopened.slides[0].shapes if shape.has_chart +) +series_name_series = list(series_name_chart.plots[0].series)[0] +series_name_result = series_content(series_name_series) +check("cacheless series title retains its worksheet formula", + series_name_series._element.xpath("./c:tx/c:strRef/c:f")[0].text + == series_name_formula) +check("cacheless series title is unavailable without aborting value inventory", + series_name_result == { + "name": None, "name_cache_status": "unavailable", "values": [1.0, 2.0], + }, series_name_result) +series_name_inventory = extract_slide_content(series_name_reopened.slides[0]) +series_name_inventory_items = series_name_inventory["charts"][0]["plots"][0]["series"] +check("deck inventory continues through cached siblings after an unavailable series title", + series_name_inventory_items[0]["name"] is None + and series_name_inventory_items[0]["values"] == [1.0, 2.0] + and series_name_inventory_items[1]["name"] == "Cached series" + and series_name_inventory_items[1]["values"] == [3.0, 4.0], + series_name_inventory_items) + + +class RaisingNameSeries: + def __init__(self, element, values): + self._element = element + self.values = values + + @property + def name(self): + raise RuntimeError("series.name must not be accessed") + + +raising_name_result = series_content(RaisingNameSeries( + series_name_series._element, list(series_name_series.values), +)) +check("cacheless name path never touches the python-pptx name property", + raising_name_result == series_name_result, raising_name_result) + +source_name_series = list(series_name_chart.plots[0].series)[1] +malformed_name_element = copy.deepcopy(source_name_series._element) +malformed_cache = malformed_name_element.find(qn("c:tx")).find( + qn("c:strRef") +).find(qn("c:strCache")) +for point in malformed_cache.findall(qn("c:pt")): + malformed_cache.remove(point) +malformed_name_result = series_content(RaisingNameSeries( + malformed_name_element, list(source_name_series.values), +)) +check("incomplete series-name cache is unavailable while values remain readable", + malformed_name_result == { + "name": None, "name_cache_status": "unavailable", "values": [3.0, 4.0], + }, malformed_name_result) + +literal_name_element = copy.deepcopy(source_name_series._element) +literal_tx = literal_name_element.find(qn("c:tx")) +literal_tx.remove(literal_tx.find(qn("c:strRef"))) +literal_value = OxmlElement("c:v") +literal_value.text = "Literal series" +literal_tx.append(literal_value) +literal_name_result = series_content(RaisingNameSeries( + literal_name_element, list(source_name_series.values), +)) +check("literal series title is read directly without the name property", + literal_name_result["name"] == "Literal series", literal_name_result) + +untitled_element = copy.deepcopy(source_name_series._element) +untitled_element.remove(untitled_element.find(qn("c:tx"))) +untitled_result = series_content(RaisingNameSeries( + untitled_element, list(source_name_series.values), +)) +check("series without a title remains a readable unnamed series", + untitled_result["name"] == "", untitled_result) + +category_mutated = Presentation("category-cache-source.pptx") +category_chart = next( + shape.chart for shape in category_mutated.slides[0].shapes if shape.has_chart +) +category_series = list(category_chart.plots[0].series) +missing_value_ref = category_series[0]._element.val.find(qn("c:numRef")) +missing_formula_before = missing_value_ref.find(qn("c:f")).text +missing_value_ref.remove(missing_value_ref.find(qn("c:numCache"))) +category_mutated.save("category-cacheless.pptx") + +category_reopened = Presentation("category-cacheless.pptx") +category_reopened_chart = next( + shape.chart for shape in category_reopened.slides[0].shapes if shape.has_chart +) +category_reopened_series = list(category_reopened_chart.plots[0].series) +reopened_value_ref = category_reopened_series[0]._element.val.find(qn("c:numRef")) +reopened_formula = reopened_value_ref.find(qn("c:f")) +with zipfile.ZipFile("category-cacheless.pptx") as category_archive: + embedded_workbook_remains = any( + name.startswith("ppt/embeddings/") for name in category_archive.namelist() + ) +check( + "cacheless category series retains its formula and embedded workbook", + reopened_formula is not None + and reopened_formula.text == missing_formula_before + and reopened_value_ref.find(qn("c:numCache")) is None + and embedded_workbook_remains, +) +category_inventory = extract_slide_content(category_reopened.slides[0]) +category_inventory_series = category_inventory["charts"][0]["plots"][0]["series"] +check( + "cacheless category series is unavailable without aborting inventory", + category_inventory_series[0]["name"] == "Missing cache" + and category_inventory_series[0]["values"] is None + and category_inventory_series[0]["cache_status"] == "unavailable", + category_inventory_series, +) +check( + "cached category series still reports its values after a cacheless sibling", + category_inventory_series[1]["name"] == "Cached series" + and category_inventory_series[1]["values"] == [3.0, 4.0], + category_inventory_series, +) + +marker_element = copy.deepcopy(category_series[1]._element) +marker_values = marker_element.xpath("./c:val/c:numRef/c:numCache/c:pt") +marker_values[0].find(qn("c:v")).text = "#N/A" +marker_values[1].find(qn("c:v")).text = None +marker_series = type("CachedMarkerSeries", (), {"_element": marker_element})() +marker_result = series_content(marker_series, include_categories=True) +check("category series preserves #N/A and blank cache markers without series.values", + marker_result["values"] == ["#N/A", None], marker_result) +omitted_marker_element = copy.deepcopy(marker_element) +omitted_marker_cache = omitted_marker_element.xpath("./c:val/c:numRef/c:numCache")[0] +omitted_marker_cache.remove(omitted_marker_cache.findall(qn("c:pt"))[1]) +omitted_marker_series = type( + "SparseCategorySeries", (), {"_element": omitted_marker_element} +)() +omitted_marker_result = series_content(omitted_marker_series, include_categories=True) +check("category series densifies an omitted sparse point as blank", + omitted_marker_result["values"] == ["#N/A", None], omitted_marker_result) + +divergent_prs = Presentation("category-cache-source.pptx") +divergent_chart = next( + shape.chart for shape in divergent_prs.slides[0].shapes if shape.has_chart +) +divergent_series = list(divergent_chart.plots[0].series) +second_category = divergent_series[1]._element.find(qn("c:cat")) +second_category.find(qn("c:strRef")).find(qn("c:f")).text = "Sheet1!$D$2:$D$3" +second_points = second_category.xpath("./c:strRef/c:strCache/c:pt") +second_points[0].find(qn("c:v")).text = "North" +second_points[1].find(qn("c:v")).text = "South" +divergent_inventory = extract_slide_content(divergent_prs.slides[0]) +divergent_items = divergent_inventory["charts"][0]["plots"][0]["series"] +check("each category series retains its own formula and cached labels", + divergent_items[0]["categories"] == [["A"], ["B"]] + and divergent_items[1]["categories"] == [["North"], ["South"]] + and divergent_items[0]["category_formula"] != divergent_items[1]["category_formula"], + divergent_items) +commented_category_element = copy.deepcopy(divergent_series[0]._element) +commented_category_element.find(qn("c:cat")).insert(0, etree.Comment("source follows")) +commented_category_series = type( + "CommentedCategorySeries", (), {"_element": commented_category_element} +)() +check("category source selection ignores preserved XML comments", + category_content(commented_category_series)["categories"] == [["A"], ["B"]]) +malformed_category_element = copy.deepcopy(divergent_series[1]._element) +malformed_category_cache = malformed_category_element.xpath( + "./c:cat/c:strRef/c:strCache" +)[0] +malformed_category_cache.append(copy.deepcopy( + malformed_category_cache.findall(qn("c:pt"))[0] +)) +malformed_category_series = type( + "MalformedCategorySeries", (), {"_element": malformed_category_element} +)() +malformed_category_result = category_content(malformed_category_series) +check("duplicate category cache indexes fail closed for only that series", + malformed_category_result["categories"] is None + and malformed_category_result["category_cache_status"] == "unavailable", + malformed_category_result) + +duplicate_count_element = copy.deepcopy(divergent_series[1]._element) +duplicate_count_cache = duplicate_count_element.xpath("./c:cat/c:strRef/c:strCache")[0] +duplicate_count_cache.insert(1, copy.deepcopy(duplicate_count_cache.find(qn("c:ptCount")))) +duplicate_count_series = type( + "DuplicateCategoryCountSeries", (), {"_element": duplicate_count_element} +)() +duplicate_count_budget = {"remaining": 2} +duplicate_count_result = category_content(duplicate_count_series, duplicate_count_budget) +valid_after_malformed = category_content(divergent_series[0], duplicate_count_budget) +check("duplicate category ptCount fails closed without starving a valid sibling", + duplicate_count_result["categories"] is None + and valid_after_malformed["categories"] == [["A"], ["B"]] + and duplicate_count_budget["remaining"] == 0, + (duplicate_count_result, valid_after_malformed, duplicate_count_budget)) + +choice_conflict_element = copy.deepcopy(divergent_series[1]._element) +choice_conflict_category = choice_conflict_element.find(qn("c:cat")) +choice_conflict_category.append(parse_xml( + '' + 'Sheet1!$E$2:$E$3' + 'X' + 'Y' + '' +)) +choice_conflict_series = type( + "ChoiceConflictSeries", (), {"_element": choice_conflict_element} +)() +choice_conflict_result = category_content(choice_conflict_series) +check("conflicting category source choices fail closed without mixing formula and labels", + choice_conflict_result["categories"] is None + and choice_conflict_result["category_formula"] is None + and choice_conflict_result["category_cache_status"] == "unavailable", + choice_conflict_result) + +huge_category_element = copy.deepcopy(divergent_series[1]._element) +huge_category_element.xpath("./c:cat/c:strRef/c:strCache/c:ptCount")[0].set( + "val", "4294967295" +) +huge_category_series = type( + "HugeCategoryCountSeries", (), {"_element": huge_category_element} +)() +check("oversized logical category counts fail before allocation", + category_content(huge_category_series)["categories"] is None) + +aggregate_budget = {"remaining": 3} +first_budgeted_categories = category_content(divergent_series[0], aggregate_budget) +second_budgeted_categories = category_content(divergent_series[1], aggregate_budget) +check("shared category budget bounds aggregate dense inventory", + first_budgeted_categories["category_cache_status"] == "available" + and second_budgeted_categories["category_cache_status"] == "unavailable" + and aggregate_budget["remaining"] == 1, + (first_budgeted_categories, second_budgeted_categories, aggregate_budget)) + +multi_level_element = copy.deepcopy(divergent_series[0]._element) +multi_level_category = multi_level_element.find(qn("c:cat")) +multi_level_category.remove(next(iter(multi_level_category))) +multi_level_category.append(parse_xml( + '' + 'Sheet1!$A$2:$A$5' + 'AB' + 'CD' + 'G1' + 'G2' + '' +)) +multi_level_series = type("MultiLevelSeries", (), {"_element": multi_level_element})() +multi_level_result = category_content(multi_level_series) +check("multi-level categories flatten parent-to-child with linear carry-forward", + multi_level_result["categories"] + == [["G1", "A"], ["G1", "B"], ["G2", "C"], ["G2", "D"]] + and multi_level_result["category_formula"] == "Sheet1!$A$2:$A$5", + multi_level_result) + +category_labels_mutated = Presentation("category-cache-source.pptx") +category_labels_chart = next( + shape.chart for shape in category_labels_mutated.slides[0].shapes if shape.has_chart +) +category_ref = category_labels_chart.plots[0]._element.xpath( + "./c:ser[1]/c:cat/c:strRef" +)[0] +category_formula_before = category_ref.find(qn("c:f")).text +category_ref.remove(category_ref.find(qn("c:strCache"))) +category_labels_mutated.save("category-label-cacheless.pptx") +category_labels_reopened = Presentation("category-label-cacheless.pptx") +category_labels_chart = next( + shape.chart for shape in category_labels_reopened.slides[0].shapes if shape.has_chart +) +category_ref = category_labels_chart.plots[0]._element.xpath( + "./c:ser[1]/c:cat/c:strRef" +)[0] +with zipfile.ZipFile("category-label-cacheless.pptx") as category_archive: + category_workbook_remains = any( + name.startswith("ppt/embeddings/") for name in category_archive.namelist() + ) +category_labels_inventory = extract_slide_content(category_labels_reopened.slides[0]) +category_plot_inventory = category_labels_inventory["charts"][0]["plots"][0] +check("cacheless categories retain their worksheet formula and embedded workbook", + category_ref.find(qn("c:f")).text == category_formula_before + and category_ref.find(qn("c:strCache")) is None + and category_workbook_remains) +check("missing category cache is reported without aborting later chart inventory", + category_plot_inventory["series"][0]["categories"] is None + and category_plot_inventory["series"][0]["category_cache_status"] == "unavailable" + and category_plot_inventory["series"][1]["categories"] == [["A"], ["B"]] + and category_plot_inventory["series"][1]["values"] == [3.0, 4.0], + category_plot_inventory) + +diagram_frame = etree.fromstring(f''' + +'''.encode()) +diagram_data = f''' + Revenue +'''.encode() +diagram_part = type("DiagramPart", (), {"blob": diagram_data})() +diagram_owner = type("SlidePart", (), { + "related_part": lambda self, relationship_id: {"rIdSmart": diagram_part}[relationship_id], +})() +diagram_shape = type("SmartArtShape", (), { + "name": "SmartArt 1", "_element": diagram_frame, "part": diagram_owner, +})() +check("SmartArt inventory extracts text from the diagram data part", + smartart_content(diagram_shape) + == {"name": "SmartArt 1", "status": "ok", "text": ["Revenue"]}) +missing_diagram_shape = type("SmartArtShape", (), { + "name": "Broken SmartArt", "_element": diagram_frame, + "part": type("SlidePart", (), { + "related_part": lambda self, relationship_id: {}[relationship_id], + })(), +})() +check("SmartArt inventory reports an unresolved diagram relationship", + smartart_content(missing_diagram_shape)["status"] == "unreadable") + +placeholder_png = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9ZlRYAAAAASUVORK5CYII=" +) +with open("placeholder-picture.png", "wb") as stream: + stream.write(placeholder_png) +placeholder_prs = Presentation() +picture_layout = next( + layout for layout in placeholder_prs.slide_layouts + if any( + shape.placeholder_format.type == PP_PLACEHOLDER.PICTURE + for shape in layout.placeholders + ) +) +placeholder_slide = placeholder_prs.slides.add_slide(picture_layout) +picture_placeholder = next( + shape for shape in placeholder_slide.placeholders + if shape.placeholder_format.type == PP_PLACEHOLDER.PICTURE +) +placeholder_picture = picture_placeholder.insert_picture("placeholder-picture.png") +placeholder_prs.save("picture-placeholder.pptx") +placeholder_reopened = Presentation("picture-placeholder.pptx").slides[0] +reopened_picture = next( + shape for shape in placeholder_reopened.placeholders if hasattr(shape, "image") +) +placeholder_inventory = extract_slide_content(placeholder_reopened)["pictures"] +check("picture placeholder remains a placeholder after image insertion", + reopened_picture.shape_type == MSO_SHAPE_TYPE.PLACEHOLDER, + reopened_picture.shape_type) +check("picture inventory includes populated picture placeholders and image metadata", + len(placeholder_inventory) == 1 + and placeholder_inventory[0]["extension"] == "png" + and placeholder_inventory[0]["bytes"] == len(placeholder_png), + placeholder_inventory) + +# A dangling r:embed must produce an unreadable record and allow later slides to inventory. +broken_picture_source = Presentation("picture-placeholder.pptx") +later_picture_slide = broken_picture_source.slides.add_slide( + broken_picture_source.slide_layouts[6] +) +later_picture_slide.shapes.add_picture( + "placeholder-picture.png", Inches(1), Inches(1), Inches(1), Inches(1) +) +broken_picture_source.save("broken-picture-source.pptx") +with zipfile.ZipFile("broken-picture-source.pptx") as source_archive: + broken_picture_payload = { + info.filename: source_archive.read(info) for info in source_archive.infolist() + } +broken_slide_root = etree.fromstring(broken_picture_payload["ppt/slides/slide1.xml"]) +broken_blips = broken_slide_root.findall(".//" + qn("a:blip")) +require(len(broken_blips) == 1, "broken-picture fixture expected one slide-1 image") +broken_relationship_id = broken_blips[0].get(qn("r:embed")) +relationships_path = "ppt/slides/_rels/slide1.xml.rels" +relationships_root = etree.fromstring(broken_picture_payload[relationships_path]) +package_relationship_ns = "http://schemas.openxmlformats.org/package/2006/relationships" +broken_relationships = [ + relation for relation in relationships_root.findall( + f"{{{package_relationship_ns}}}Relationship" + ) + if relation.get("Id") == broken_relationship_id +] +require(len(broken_relationships) == 1, + "broken-picture fixture expected one matching relationship") +relationships_root.remove(broken_relationships[0]) +broken_picture_payload[relationships_path] = etree.tostring( + relationships_root, xml_declaration=True, encoding="UTF-8", standalone=True, +) +with zipfile.ZipFile("broken-picture-relationship.pptx", "w", zipfile.ZIP_DEFLATED) as archive: + for member_name, member_data in broken_picture_payload.items(): + archive.writestr(member_name, member_data) +broken_picture_deck = open_validated_presentation("broken-picture-relationship.pptx") +broken_picture_inventory = [ + extract_slide_content(slide)["pictures"] for slide in broken_picture_deck.slides +] +check("broken picture relationship is explicit instead of aborting inventory", + broken_picture_inventory[0] == [{ + "name": reopened_picture.name, + "status": "unreadable", + "relationship_id": broken_relationship_id, + "reason": "missing or invalid image relationship or payload", + }], broken_picture_inventory[0]) +check("picture inventory continues to a healthy later slide after a broken relationship", + len(broken_picture_inventory[1]) == 1 + and broken_picture_inventory[1][0]["extension"] == "png", + broken_picture_inventory[1]) + +# Directly hidden leaf shapes and children of a hidden group must be absent from every +# non-text inventory, while visible siblings after them remain discoverable. +hidden_inventory_prs = Presentation() +hidden_inventory_slide = hidden_inventory_prs.slides.add_slide( + hidden_inventory_prs.slide_layouts[6] +) + +def set_shape_hidden(shape): + properties = shape._element.find(".//" + qn("p:cNvPr")) + require(properties is not None, f"shape {shape.name!r} has no cNvPr") + properties.set("hidden", "1") + +hidden_table_shape = hidden_inventory_slide.shapes.add_table( + 1, 1, Inches(0.2), Inches(0.2), Inches(1), Inches(0.5) +) +hidden_table_shape.table.cell(0, 0).text = "secret table" +set_shape_hidden(hidden_table_shape) + +hidden_chart_data = ChartData() +hidden_chart_data.categories = ["secret"] +hidden_chart_data.add_series("secret", (9,)) +hidden_chart_shape = hidden_inventory_slide.shapes.add_chart( + XL_CHART_TYPE.COLUMN_CLUSTERED, + Inches(1.3), Inches(0.2), Inches(1.5), Inches(1), hidden_chart_data, +) +set_shape_hidden(hidden_chart_shape) + +hidden_picture_shape = hidden_inventory_slide.shapes.add_picture( + "placeholder-picture.png", Inches(3), Inches(0.2), Inches(1), Inches(1) +) +set_shape_hidden(hidden_picture_shape) + +def smartart_frame(shape_id, name, relationship_id, *, hidden=False): + hidden_attribute = ' hidden="1"' if hidden else "" + return etree.fromstring(f''' + + + + + + + + + '''.encode()) + +hidden_inventory_slide.shapes._spTree.append( + smartart_frame(910, "Hidden SmartArt", "rIdHiddenSmart", hidden=True) +) + +nested_hidden_table = hidden_inventory_slide.shapes.add_table( + 1, 1, Inches(4.2), Inches(0.2), Inches(1), Inches(0.5) +) +nested_hidden_table.table.cell(0, 0).text = "secret grouped table" +nested_hidden_element = nested_hidden_table._element +hidden_group_element = parse_xml(GRP) +hidden_group_element.find(".//" + qn("p:cNvPr")).set("hidden", "1") +nested_hidden_element.getparent().replace(nested_hidden_element, hidden_group_element) +hidden_group_element.append(nested_hidden_element) + +visible_table_shape = hidden_inventory_slide.shapes.add_table( + 1, 1, Inches(0.2), Inches(2), Inches(1), Inches(0.5) +) +visible_table_shape.table.cell(0, 0).text = "visible table" +visible_chart_data = ChartData() +visible_chart_data.categories = ["visible"] +visible_chart_data.add_series("visible", (1,)) +visible_chart_shape = hidden_inventory_slide.shapes.add_chart( + XL_CHART_TYPE.COLUMN_CLUSTERED, + Inches(1.3), Inches(2), Inches(1.5), Inches(1), visible_chart_data, +) +visible_picture_shape = hidden_inventory_slide.shapes.add_picture( + "placeholder-picture.png", Inches(3), Inches(2), Inches(1), Inches(1) +) +hidden_inventory_slide.shapes._spTree.append( + smartart_frame(911, "Visible SmartArt", "rIdVisibleSmart") +) +hidden_inventory_prs.save("hidden-nontext-inventory.pptx") +hidden_inventory_reopened = open_validated_presentation("hidden-nontext-inventory.pptx") +hidden_inventory_slide = hidden_inventory_reopened.slides[0] +hidden_raw_names = [shape.name for shape in hidden_inventory_slide.shapes] +hidden_visible_names = [shape.name for shape in iter_shapes(hidden_inventory_slide.shapes)] +hidden_inventory = extract_slide_content(hidden_inventory_slide) +check("hidden non-text fixture retains direct and group-hidden objects (negative control)", + {"Hidden SmartArt", hidden_table_shape.name, hidden_chart_shape.name, + hidden_picture_shape.name, "demo group"} <= set(hidden_raw_names), + hidden_raw_names) +check("visibility walker prunes direct hidden leaves and every child of a hidden group", + "Hidden SmartArt" not in hidden_visible_names + and hidden_table_shape.name not in hidden_visible_names + and hidden_chart_shape.name not in hidden_visible_names + and hidden_picture_shape.name not in hidden_visible_names + and "secret grouped table" not in str(hidden_inventory), + (hidden_visible_names, hidden_inventory)) +check("all non-text inventories retain only visible siblings after hidden shapes", + len(hidden_inventory["tables"]) == 1 + and hidden_inventory["tables"][0][0][0]["text"] == "visible table" + and len(hidden_inventory["charts"]) == 1 + and hidden_inventory["charts"][0]["plots"][0]["series"][0]["values"] == [1.0] + and len(hidden_inventory["pictures"]) == 1 + and hidden_inventory["pictures"][0]["extension"] == "png" + and hidden_inventory["smartart"] == [{ + "name": "Visible SmartArt", "status": "unreadable", + "reason": "rIdVisibleSmart", + }], hidden_inventory) + +hidden_title_prs = Presentation() +hidden_title_slide = hidden_title_prs.slides.add_slide(hidden_title_prs.slide_layouts[5]) +hidden_title_slide.shapes.title.text = "confidential hidden title" +set_shape_hidden(hidden_title_slide.shapes.title) +hidden_title_prs.save("hidden-title.pptx") +hidden_title_slide = open_validated_presentation("hidden-title.pptx").slides[0] +hidden_title_shape = hidden_title_slide.shapes.title +guarded_title = ( + hidden_title_shape.text_frame.text + if hidden_title_shape is not None and not shape_is_hidden(hidden_title_shape) else "" +) +check("main slide summary does not expose a hidden title", guarded_title == "", guarded_title) + +merged_prs = Presentation() +merged_slide = merged_prs.slides.add_slide(merged_prs.slide_layouts[6]) +merged_table = merged_slide.shapes.add_table( + 3, 3, Inches(1), Inches(1), Inches(6), Inches(3) +).table +merged_table.cell(0, 0).text = "Merged heading" +merged_table.cell(0, 0).merge(merged_table.cell(1, 1)) +merged_table.cell(2, 0).text = "ordinary cell" +merged_prs.save("merged-table.pptx") +merged_inventory = extract_slide_content( + Presentation("merged-table.pptx").slides[0] +)["tables"][0] +merged_origin = merged_inventory[0][0] +covered_slots = [merged_inventory[0][1], merged_inventory[1][0], merged_inventory[1][1]] +check( + "table inventory records a merged origin and its row/column spans", + merged_origin["is_merge_origin"] + and not merged_origin["is_spanned"] + and merged_origin["span_height"] == 2 + and merged_origin["span_width"] == 2, + merged_origin, +) +check( + "table inventory marks covered slots without repeating merged text", + all(item["is_spanned"] and item["text"] is None for item in covered_slots) + and sum( + item["text"] == "Merged heading" + for row in merged_inventory for item in row + ) == 1, + merged_inventory, +) + +# XY scatter and bubble plots do not have category/value-series semantics. +xy_prs = Presentation() +xy_slide = xy_prs.slides.add_slide(xy_prs.slide_layouts[6]) +xy_data = XyChartData() +xy_series = xy_data.add_series("XY series") +xy_series.add_data_point(1, 2) +xy_series.add_data_point(3, 4) +xy_slide.shapes.add_chart( + XL_CHART_TYPE.XY_SCATTER, + Inches(0.5), Inches(0.5), Inches(4), Inches(2.5), xy_data, +) +bubble_data = BubbleChartData() +bubble_series = bubble_data.add_series("Bubble series") +bubble_series.add_data_point(5, 6, 7) +xy_slide.shapes.add_chart( + XL_CHART_TYPE.BUBBLE, + Inches(0.5), Inches(3.5), Inches(4), Inches(2.5), bubble_data, +) +xy_prs.save("xy-bubble.pptx") +xy_content = extract_slide_content(Presentation("xy-bubble.pptx").slides[0]) +xy_plots = [plot for chart in xy_content["charts"] for plot in chart["plots"]] +check( + "scatter inventory emits x and y caches without category access", + any( + plot["kind"] == "XyPlot" + and plot["series"][0]["x_points"] == [(0, 1.0), (1, 3.0)] + and plot["series"][0]["y_points"] == [(0, 2.0), (1, 4.0)] + for plot in xy_plots + ), + xy_plots, +) +check( + "bubble inventory emits x, y, and bubble-size caches", + any( + plot["kind"] == "BubblePlot" + and plot["series"][0]["x_points"] == [(0, 5.0)] + and plot["series"][0]["y_points"] == [(0, 6.0)] + and plot["series"][0]["bubble_points"] == [(0, 7.0)] + for plot in xy_plots + ), + xy_plots, +) + +# Missing caches and missing ptCount are unavailable, not a proven empty series. +xy_plot = next(plot for chart in Presentation("xy-bubble.pptx").slides[0].shapes + if chart.has_chart for plot in chart.chart.plots + if type(plot).__name__ == "XyPlot") +xy_item = list(xy_plot.series)[0] +x_source = xy_item._element.xVal +missing_cache = copy.deepcopy(x_source) +missing_cache_ref = missing_cache.find(qn("c:numRef")) +missing_cache_ref.remove(missing_cache_ref.find(qn("c:numCache"))) +check("numeric cache without numCache reports unavailable", cached_numeric_points(missing_cache) is None) +missing_count = copy.deepcopy(x_source) +missing_count_cache = missing_count.find(qn("c:numRef") + "/" + qn("c:numCache")) +missing_count_cache.remove(missing_count_cache.find(qn("c:ptCount"))) +check("numeric cache without ptCount reports unavailable", cached_numeric_points(missing_count) is None) +marker_source = copy.deepcopy(x_source) +marker_points = marker_source.xpath("./c:numRef/c:numCache/c:pt") +marker_points[0].find(qn("c:v")).text = "#N/A" +marker_points[1].find(qn("c:v")).text = None +check("numeric cache preserves #N/A and explicit blank markers", + cached_numeric_points(marker_source) == [(0, "#N/A"), (1, None)], + cached_numeric_points(marker_source)) +sparse_source = copy.deepcopy(x_source) +sparse_cache = sparse_source.xpath("./c:numRef/c:numCache")[0] +sparse_cache.find(qn("c:ptCount")).set("val", "3") +sparse_cache.remove(sparse_cache.findall(qn("c:pt"))[1]) +check("XY numeric cache retains only present sparse indexed points", + cached_numeric_points(sparse_source) == [(0, 1.0)], + cached_numeric_points(sparse_source)) +check("sparse XY cache reports ptCount without densifying trailing blanks", + cached_numeric_points(sparse_source, include_count=True) + == {"point_count": 3, "points": [(0, 1.0)]}, + cached_numeric_points(sparse_source, include_count=True)) +sparse_declared_budget = {"remaining": 3} +check("sparse numeric cache charges declared ptCount before materializing present points", + cached_numeric_points(sparse_source, point_budget=sparse_declared_budget) + == [(0, 1.0)] and sparse_declared_budget["remaining"] == 0, + sparse_declared_budget) +check("category numeric cache can densify sparse positions as blanks", + cached_numeric_points(sparse_source, fill_missing=True) + == [(0, 1.0), (1, None), (2, None)], + cached_numeric_points(sparse_source, fill_missing=True)) +duplicate_source = copy.deepcopy(x_source) +duplicate_cache = duplicate_source.xpath("./c:numRef/c:numCache")[0] +duplicate_cache.append(copy.deepcopy(duplicate_cache.findall(qn("c:pt"))[0])) +check("duplicate numeric cache indexes fail closed", + cached_numeric_points(duplicate_source) is None) +huge_count_source = copy.deepcopy(x_source) +huge_count_source.xpath("./c:numRef/c:numCache/c:ptCount")[0].set( + "val", "4294967295" +) +check("oversized logical numeric counts fail before allocation", + cached_numeric_points(huge_count_source, fill_missing=True) is None) + +# Sparse XY/bubble caches reserve their declared logical counts from one shared deck +# budget before iterating or materializing the few present nodes. +budgeted_xy_element = copy.deepcopy(xy_item._element) +for source_name in ("xVal", "yVal"): + numeric_cache = budgeted_xy_element.find(qn(f"c:{source_name}")) + numeric_cache = numeric_cache.find(qn("c:numRef") + "/" + qn("c:numCache")) + numeric_cache.find(qn("c:ptCount")).set("val", "60000") + for point in numeric_cache.findall(qn("c:pt"))[1:]: + numeric_cache.remove(point) +budgeted_xy_series = type("BudgetedXySeries", (), { + "_element": budgeted_xy_element, +})() +xy_declared_budget = {"remaining": MAX_CHART_POINTS} +xy_budgeted_content = series_content( + budgeted_xy_series, point_budget=xy_declared_budget +) +check("scatter X/Y caches charge declared sparse counts from the shared budget", + xy_budgeted_content.get("cache_status") == "unavailable" + and xy_declared_budget["remaining"] == 40000, + (xy_budgeted_content, xy_declared_budget)) + +bubble_plot = next( + plot for shape in Presentation("xy-bubble.pptx").slides[0].shapes + if shape.has_chart for plot in shape.chart.plots + if type(plot).__name__ == "BubblePlot" +) +bubble_item = list(bubble_plot.series)[0] +budgeted_bubble_element = copy.deepcopy(bubble_item._element) +for source_name in ("xVal", "yVal", "bubbleSize"): + numeric_cache = budgeted_bubble_element.find(qn(f"c:{source_name}")) + numeric_cache = numeric_cache.find(qn("c:numRef") + "/" + qn("c:numCache")) + numeric_cache.find(qn("c:ptCount")).set("val", "40000") + for point in numeric_cache.findall(qn("c:pt"))[1:]: + numeric_cache.remove(point) +budgeted_bubble_series = type("BudgetedBubbleSeries", (), { + "_element": budgeted_bubble_element, +})() +bubble_declared_budget = {"remaining": MAX_CHART_POINTS} +bubble_budgeted_content = series_content( + budgeted_bubble_series, point_budget=bubble_declared_budget +) +check("bubble X/Y/size caches share one declared-count budget", + bubble_budgeted_content.get("cache_status") == "unavailable" + and bubble_declared_budget["remaining"] == 20000, + (bubble_budgeted_content, bubble_declared_budget)) + +actual_cache = x_source.xpath("./c:numRef/c:numCache")[0] +x_source.xpath("./c:numRef")[0].remove(actual_cache) +check("XY series with an unavailable cache is explicit", + series_content(xy_item).get("cache_status") == "unavailable", series_content(xy_item)) + + +sp_element = slide6.shapes[-1]._element # the textbox; layout 5 still carries a Title placeholder +group_element = parse_xml(GRP) +sp_element.getparent().replace(sp_element, group_element) +group_element.append(sp_element) + +flat = list(iter_shapes(slide6.shapes)) +check( + "walker finds the shape nested in the group", + any(getattr(sh, "text_frame", None) is not None and sh.text_frame.text == "nested member" for sh in flat), +) +check( + "top-level shapes list hides the nested member (negative control)", + not any(getattr(sh, "text_frame", None) is not None and sh.text_frame.text == "nested member" for sh in slide6.shapes), +) +grouped_faces = [ + run.font.name + for shape in iter_shapes(slide6.shapes) + if shape.has_text_frame + for paragraph in shape.text_frame.paragraphs + for run in paragraph.runs + if run.font.name +] +check("font triage reaches runs nested in groups", "Grouped Face" in grouped_faces, grouped_faces) + +# ---- edit.md locator: candidate collection must recurse into groups ------------ +old_w, new_w = "nested member", "renamed member" + + +def iter_shapes_with_path(shapes, path=""): + for shape in shapes: + here = f"{path}/{shape.name}" if path else shape.name + if shape.shape_type == MSO_SHAPE_TYPE.GROUP: + yield from iter_shapes_with_path(shape.shapes, here) + else: + yield here, shape + + +def iter_text_targets(path, shape): + if shape.has_text_frame: + yield path, shape.text_frame + if shape.has_table: + for row_index, row in enumerate(shape.table.rows): + for column_index, cell in enumerate(row.cells): + if cell.is_spanned: + continue + yield f"{path}/table[{row_index},{column_index}]", cell.text_frame + + +merged_edit_old, merged_edit_new = "Merged edit target", "Edited merged target" +merged_edit_prs = Presentation() +merged_edit_slide = merged_edit_prs.slides.add_slide(merged_edit_prs.slide_layouts[6]) +merged_edit_table = merged_edit_slide.shapes.add_table( + 2, 2, Inches(1), Inches(1), Inches(4), Inches(2) +).table +merged_edit_table.cell(0, 0).text = merged_edit_old +merged_edit_table.cell(0, 0).merge(merged_edit_table.cell(1, 1)) +# Covered cells can retain stale text in a valid OPC package even though it is not rendered. +for merged_edit_coordinate in ((0, 1), (1, 0), (1, 1)): + merged_edit_table.cell(*merged_edit_coordinate).text = merged_edit_old +merged_edit_prs.save("merged-edit-source.pptx") + +merged_edit_reopened = Presentation("merged-edit-source.pptx") +merged_edit_shape = next( + shape for shape in merged_edit_reopened.slides[0].shapes if shape.has_table +) +merged_edit_candidates = [ + (location, text_frame) + for location, text_frame in iter_text_targets(merged_edit_shape.name, merged_edit_shape) + if merged_edit_old in text_frame.text +] +check( + "merged-table locator ignores stale text in covered merge slots", + len(merged_edit_candidates) == 1 + and merged_edit_candidates[0][0].endswith("/table[0,0]"), + [location for location, _ in merged_edit_candidates], +) +merged_edit_frame = merged_edit_candidates[0][1] +merged_edit_hits = [ + run + for paragraph in merged_edit_frame.paragraphs + for run in paragraph.runs + if merged_edit_old in run.text +] +require(len(merged_edit_hits) == 1, "expected one run in the merged-cell edit target") +merged_edit_hits[0].text = merged_edit_hits[0].text.replace( + merged_edit_old, merged_edit_new, 1 +) +merged_edit_reopened.save("merged-edit-result.pptx") + +merged_edit_result = Presentation("merged-edit-result.pptx") +merged_edit_result_table = next( + shape.table for shape in merged_edit_result.slides[0].shapes if shape.has_table +) +merged_edit_origin = merged_edit_result_table.cell(0, 0) +merged_edit_covered = [ + merged_edit_result_table.cell(*coordinate) + for coordinate in ((0, 1), (1, 0), (1, 1)) +] +check( + "merged-cell run edit persists without changing the merge topology", + merged_edit_origin.text == merged_edit_new + and merged_edit_origin.is_merge_origin + and not merged_edit_origin.is_spanned + and merged_edit_origin.span_width == 2 + and merged_edit_origin.span_height == 2 + and all(cell.is_spanned for cell in merged_edit_covered), + { + "origin": merged_edit_origin.text, + "covered": [(cell.text, cell.is_spanned) for cell in merged_edit_covered], + }, +) + + +def iter_postcheck_text_frames(shapes, path=""): + for shape in shapes: + here = f"{path}/{shape.name}" if path else shape.name + if shape.shape_type == MSO_SHAPE_TYPE.GROUP: + yield from iter_postcheck_text_frames(shape.shapes, here) + continue + if shape.has_text_frame: + yield here, shape.text_frame + if shape.has_table: + for row_index, row in enumerate(shape.table.rows): + for column_index, cell in enumerate(row.cells): + if not cell.is_spanned: + yield f"{here}/table[{row_index},{column_index}]", cell.text_frame + + +group_postcheck = { + location: frame.text for location, frame in iter_postcheck_text_frames(slide6.shapes) +} +table_postcheck = { + location: frame.text + for location, frame in iter_postcheck_text_frames(Presentation("input.pptx").slides[0].shapes) +} +check("mandatory postcheck reaches grouped text", + "nested member" in group_postcheck.values(), group_postcheck) +check("mandatory postcheck reaches table-cell text", + "old cell text" in table_postcheck.values(), table_postcheck) +check("mandatory postcheck inventories unexpected empty table cells", + any(location.endswith("/table[0,0]") and not text.strip() + for location, text in table_postcheck.items()), table_postcheck) + + +candidates = [ + (i, location, text_frame) + for i, s in enumerate(prs6.slides) + for p, sh in iter_shapes_with_path(s.shapes) + for location, text_frame in iter_text_targets(p, sh) + if old_w in text_frame.text +] +check("locator reaches text inside the group", len(candidates) == 1, [(i, p) for i, p, _ in candidates]) +check("locator reports a stable nested path", "/" in candidates[0][1], candidates[0][1]) +_, _, target = candidates[0] +target.paragraphs[0].runs[0].text = new_w +prs6.save("group-edited.pptx") +prs_g = Presentation("group-edited.pptx") +found = [sh for sh in iter_shapes(prs_g.slides[0].shapes) + if getattr(sh, "text_frame", None) is not None and sh.text_frame.text == new_w] +check("group member edit persists after save", len(found) == 1) + +table_candidates = [ + (i, location, text_frame) + for i, s in enumerate(Presentation("input.pptx").slides) + for p, sh in iter_shapes_with_path(s.shapes) + for location, text_frame in iter_text_targets(p, sh) + if old_cell in text_frame.text +] +check("locator reaches wording stored only in a table cell", len(table_candidates) == 1) +check("table-cell locator retains row and column", + table_candidates[0][1].endswith("/table[0,1]"), table_candidates[0][1]) + +duplicate_prs = Presentation("input.pptx") +duplicate_shape = next(shape for shape in duplicate_prs.slides[0].shapes if shape.has_table) +duplicate_shape.table.cell(1, 0).text = old_cell +all_table_candidates = [ + (i, location, text_frame) + for i, slide_item in enumerate(duplicate_prs.slides) + for path, shape in iter_shapes_with_path(slide_item.shapes) + for location, text_frame in iter_text_targets(path, shape) + if old_cell in text_frame.text +] +target_location = f"{duplicate_shape.name}/table[0,1]" +selected_table_candidates = [ + candidate for candidate in all_table_candidates if candidate[1] == target_location +] +check("duplicate table text requires a location selector", len(all_table_candidates) == 2) +check("table location selector chooses one row/column", + len(selected_table_candidates) == 1, [item[1] for item in all_table_candidates]) +try: + require(len(all_table_candidates) == 1, "target is not unique") + optimized_duplicate_guard_rejected = False +except ValueError: + optimized_duplicate_guard_rejected = True +check("explicit uniqueness guard rejects duplicates even under python -O", + optimized_duplicate_guard_rejected) + +split_prs = Presentation() +split_slide = split_prs.slides.add_slide(split_prs.slide_layouts[6]) +split_frame = split_slide.shapes.add_textbox( + Inches(1), Inches(1), Inches(4), Inches(1) +).text_frame +split_frame.paragraphs[0].add_run().text = "old " +split_frame.paragraphs[0].add_run().text = "wording" +split_hits = [ + run for paragraph in split_frame.paragraphs for run in paragraph.runs + if old in run.text +] +try: + require(len(split_hits) == 1 and split_hits[0].text.count(old) == 1, + "target is split across runs") + optimized_run_guard_rejected = False +except ValueError: + optimized_run_guard_rejected = True +check("explicit run-boundary guard survives python -O", optimized_run_guard_rejected) + +repeated_frame = split_slide.shapes.add_textbox( + Inches(1), Inches(2), Inches(4), Inches(1) +).text_frame +repeated_frame.text = f"{old} / {old}" +try: + require(repeated_frame.text.count(old) == 1, + "target occurs more than once in the selected shape") + optimized_repeated_guard_rejected = False +except ValueError: + optimized_repeated_guard_rejected = True +check("explicit repeated-text guard survives python -O", optimized_repeated_guard_rejected) + +# ---- analyze.md snippet: per-master, script-aware theme font resolution -------- + +prs7 = Presentation("input.pptx") +theme_cache = {} +DRAWINGML = {"a": "http://schemas.openxmlformats.org/drawingml/2006/main"} + + +def read_theme_role(root, role): + node = root.find(f".//a:{role}Font", DRAWINGML) + if node is None: + return {"latin": "", "eastAsia": "", "complexScript": "", "scripts": {}} + + def typeface(tag): + child = node.find(f"a:{tag}", DRAWINGML) + return "" if child is None else child.get("typeface", "") + + return { + "latin": typeface("latin"), + "eastAsia": typeface("ea"), + "complexScript": typeface("cs"), + "scripts": { + child.get("script"): child.get("typeface", "") + for child in node.findall("a:font", DRAWINGML) if child.get("script") + }, + } + + +def theme_faces_for_slide(slide): + master_part = slide.slide_layout.slide_master.part + cache_key = str(master_part.partname) + if cache_key not in theme_cache: + theme_part = master_part.part_related_by(RT.THEME) + root = ET.fromstring(theme_part.blob) + theme_cache[cache_key] = { + "major": read_theme_role(root, "major"), + "minor": read_theme_role(root, "minor"), + } + return cache_key, theme_cache[cache_key] + + +EAST_ASIAN_SCRIPTS = {"Hans", "Hant", "Jpan", "Hang", "Bopo"} +COMPLEX_SCRIPTS = {"Arab", "Hebr", "Deva", "Beng", "Taml", "Thai"} +HAN_RANGES = ( + (0x2E80, 0x2E99), (0x2E9B, 0x2EF3), (0x2F00, 0x2FD5), + (0x3005, 0x3005), (0x3007, 0x3007), (0x3021, 0x3029), (0x3038, 0x303B), + (0x3400, 0x4DBF), (0x4E00, 0x9FFF), + (0xF900, 0xFA6D), (0xFA70, 0xFAD9), + (0x16FE2, 0x16FE3), (0x16FF0, 0x16FF6), + (0x20000, 0x2A6DF), (0x2A700, 0x2B81D), (0x2B820, 0x2CEAD), + (0x2CEB0, 0x2EBE0), (0x2EBF0, 0x2EE5D), (0x2F800, 0x2FA1D), + (0x30000, 0x3134A), (0x31350, 0x33479), +) + + +def character_tags(character): + codepoint = ord(character) + if 0x3040 <= codepoint <= 0x30FF or 0x31F0 <= codepoint <= 0x31FF: + return ["Jpan"] + if (0x1100 <= codepoint <= 0x11FF or 0x3130 <= codepoint <= 0x318F + or 0xA960 <= codepoint <= 0xA97F or 0xAC00 <= codepoint <= 0xD7AF + or 0xD7B0 <= codepoint <= 0xD7FF): + return ["Hang"] + if 0x3100 <= codepoint <= 0x312F or 0x31A0 <= codepoint <= 0x31BF: + return ["Bopo"] + if any(start <= codepoint <= end for start, end in HAN_RANGES): + return ["Hans", "Hant", "Jpan", "Hang"] + for tag, start, end in ( + ("Cyrl", 0x0400, 0x052F), ("Hebr", 0x0590, 0x05FF), + ("Arab", 0x0600, 0x06FF), ("Deva", 0x0900, 0x097F), + ("Beng", 0x0980, 0x09FF), ("Taml", 0x0B80, 0x0BFF), + ("Thai", 0x0E00, 0x0E7F), + ): + if start <= codepoint <= end: + return [tag] + return [] + + +def script_tags(text): + tags = [] + for character in text: + tags.extend(character_tags(character)) + return list(dict.fromkeys(tags)) + + +def required_slots(text): + slots = [] + for character in text: + tags = set(character_tags(character)) + slot = "eastAsia" if tags & EAST_ASIAN_SCRIPTS else ( + "complexScript" if tags & COMPLEX_SCRIPTS else "latin" + ) + if slot not in slots: + slots.append(slot) + return slots or ["latin"] + + +han_examples = [ + chr(codepoint) + for start, end in HAN_RANGES + for codepoint in {start, end} +] +han_gap_examples = [ + chr(codepoint) for codepoint in ( + 0x2E9A, 0x2EF4, 0x2FD6, 0x3006, 0x3105, 0x3131, 0x4DC0, + 0xFA6E, 0xFA6F, 0xFADA, 0x16FE1, 0x16FE4, 0x16FEF, 0x16FF7, + 0x2A6E0, 0x2B81E, 0x2CEAE, 0x2EBE1, 0x2EE5E, 0x2FA1E, + 0x3134B, 0x3347A, 0x33480, + ) +] +han_tags = {"Hans", "Hant", "Jpan", "Hang"} +supplementary_han_classified = all( + set(character_tags(character)) == han_tags + and required_slots(character) == ["eastAsia"] + for character in han_examples +) +han_gaps_stay_latin = all( + set(character_tags(character)) != han_tags + for character in han_gap_examples +) +check("compatibility and supplementary-plane Han use the east-Asian slot", + supplementary_han_classified) +check("Unicode 17 Han gaps and neighboring scripts are not mislabeled as Han", + han_gaps_stay_latin) +check("Bopomofo and compatibility Jamo keep distinct East-Asian script tags", + character_tags("ㄅ") == ["Bopo"] and required_slots("ㄅ") == ["eastAsia"] + and character_tags("ㄱ") == ["Hang"] and required_slots("ㄱ") == ["eastAsia"]) +check("Katakana phonetic extensions remain Japanese after narrowing Han ranges", + character_tags("ㇰ") == ["Jpan"] and required_slots("ㇰ") == ["eastAsia"]) +check("supplementary Han range checks remain active under optimized Python", + __debug__ or (supplementary_han_classified and han_gaps_stay_latin)) + + +def raw_font_slots(rpr): + slots = {} + if rpr is None: + return slots + for slot, tag in (("latin", "a:latin"), ("eastAsia", "a:ea"), + ("complexScript", "a:cs")): + child = rpr.find(qn(tag)) + if child is not None and child.get("typeface"): + slots[slot] = child.get("typeface") + return slots + + +THEME_TOKENS = { + "+mj-lt": ("major", "latin"), "+mj-ea": ("major", "eastAsia"), + "+mj-cs": ("major", "complexScript"), "+mn-lt": ("minor", "latin"), + "+mn-ea": ("minor", "eastAsia"), "+mn-cs": ("minor", "complexScript"), +} + + +def expand_theme_token(face, theme_fonts): + role_slot = THEME_TOKENS.get(face) + return theme_fonts[role_slot[0]][role_slot[1]] if role_slot else face + + +def font_candidates(run, paragraph, theme_fonts, role): + run_slots = raw_font_slots(run._r.rPr) + ppr = paragraph._p.pPr + paragraph_slots = raw_font_slots(None if ppr is None else ppr.defRPr) + tags = script_tags(run.text) + candidates = [] + for slot in required_slots(run.text): + fallback_role = role + explicit_source = None + face = run_slots.get(slot) + if face: + explicit_source = "run" + else: + face = paragraph_slots.get(slot) + if face: + explicit_source = "paragraph defaults" + if face: + resolved = expand_theme_token(face, theme_fonts) + if resolved: + candidates.append((slot, resolved, explicit_source)) + continue + if face in THEME_TOKENS: + fallback_role = THEME_TOKENS[face][0] + else: + continue + role_fonts = theme_fonts[fallback_role] + slot_tags = [ + tag for tag in tags + if (slot == "eastAsia" and tag in EAST_ASIAN_SCRIPTS) + or (slot == "complexScript" and tag in COMPLEX_SCRIPTS) + or (slot == "latin" and tag not in EAST_ASIAN_SCRIPTS | COMPLEX_SCRIPTS) + ] + candidates.extend( + (slot, role_fonts["scripts"][tag], f"{fallback_role} theme script {tag}") + for tag in slot_tags if role_fonts["scripts"].get(tag) + ) + if role_fonts.get(slot): + candidates.append((slot, role_fonts[slot], f"{fallback_role} theme {slot}")) + return list(dict.fromkeys(candidates)) + + +master_name, theme_fonts = theme_faces_for_slide(prs7.slides[0]) +check( + "theme major/minor fonts resolve through the slide master relationship", + theme_fonts["major"]["latin"] and theme_fonts["minor"]["latin"], + (master_name, theme_fonts), +) +print("theme fonts:", ascii(theme_fonts)) + + +class StubThemePart: + def __init__(self, major, minor, east_asian="", complex_script="", scripts=None): + script_nodes = "".join( + f'' + for script, face in (scripts or {}).items() + ) + self.blob = ( + f'' + f'' + f'{script_nodes}' + f'' + f'{script_nodes}' + f'' + ).encode() + + +class StubMasterPart: + def __init__(self, name, major, minor, **theme_options): + self.partname = name + self.theme_part = StubThemePart(major, minor, **theme_options) + + def part_related_by(self, relationship_type): + assert relationship_type == RT.THEME + return self.theme_part + + +def stub_slide(name, major, minor, **theme_options): + master = type("Master", (), {"part": StubMasterPart(name, major, minor, **theme_options)})() + layout = type("Layout", (), {"slide_master": master})() + return type("Slide", (), {"slide_layout": layout})() + + +_, first_fonts = theme_faces_for_slide(stub_slide("/ppt/slideMasters/one.xml", "Head One", "Body One")) +_, second_fonts = theme_faces_for_slide(stub_slide("/ppt/slideMasters/two.xml", "Head Two", "Body Two")) +check( + "different slide masters resolve their own theme faces", + first_fonts["major"]["latin"] == "Head One" + and second_fonts["major"]["latin"] == "Head Two" + and first_fonts != second_fonts, + (first_fonts, second_fonts), +) + +_, script_fonts = theme_faces_for_slide(stub_slide( + "/ppt/slideMasters/scripts.xml", "Latin Theme", "Latin Body", + east_asian="East Asian Theme", complex_script="Complex Script Theme", + scripts={ + "Hans": "Simplified Chinese Theme", "Cyrl": "Cyrillic Theme", + "Thai": "Thai Theme", + }, +)) + +check( + "unstyled runs report as inherited, not as a concrete face", + all(run.font.name is None for sh in prs7.slides[0].shapes if sh.has_text_frame for par in sh.text_frame.paragraphs for run in par.runs), +) + +font_box = prs7.slides[0].shapes.add_textbox(Inches(1), Inches(4), Inches(4), Inches(1)) +font_paragraph = font_box.text_frame.paragraphs[0] +font_paragraph.font.name = "Paragraph Face" +paragraph_run = font_paragraph.add_run() +paragraph_run.text = "paragraph default" +explicit_run = font_paragraph.add_run() +explicit_run.text = "run override" +explicit_run.font.name = "Run Face" +cjk_theme_run = font_paragraph.add_run() +cjk_theme_run.text = "汉字" +thai_theme_run = font_paragraph.add_run() +thai_theme_run.text = "ไทย" +mixed_run = font_paragraph.add_run() +mixed_run.text = "A汉ก" +mixed_run.font.name = "Latin Explicit" +for tag, face in (("a:ea", "East Explicit"), ("a:cs", "Complex Explicit")): + node = OxmlElement(tag) + node.set("typeface", face) + mixed_run._r.get_or_add_rPr().append(node) +token_run = font_paragraph.add_run() +token_run.text = "汉" +token_ea = OxmlElement("a:ea") +token_ea.set("typeface", "+mj-ea") +token_run._r.get_or_add_rPr().append(token_ea) + +partial_box = prs7.slides[0].shapes.add_textbox(Inches(5), Inches(4), Inches(4), Inches(1)) +partial_paragraph = partial_box.text_frame.paragraphs[0] +latin_only_run = partial_paragraph.add_run() +latin_only_run.text = "A汉" +latin_only_run.font.name = "Latin Only" +east_only_run = partial_paragraph.add_run() +east_only_run.text = "A汉" +east_only = OxmlElement("a:ea") +east_only.set("typeface", "East Only") +east_only_run._r.get_or_add_rPr().append(east_only) +cyrillic_cjk_run = partial_paragraph.add_run() +cyrillic_cjk_run.text = "Тест 汉" + +token_fallback_fonts = { + "major": {"latin": "Major Latin", "eastAsia": "", "complexScript": "", + "scripts": {"Hans": "Major Hans"}}, + "minor": {"latin": "Minor Latin", "eastAsia": "Minor East", "complexScript": "", + "scripts": {"Hans": "Minor Hans"}}, +} + +detected_faces = { + run.text: font_candidates(run, font_paragraph, script_fonts, "minor") + for run in font_paragraph.runs +} +latin_only_faces = font_candidates(latin_only_run, partial_paragraph, script_fonts, "minor") +east_only_faces = font_candidates(east_only_run, partial_paragraph, script_fonts, "minor") +cyrillic_cjk_faces = font_candidates( + cyrillic_cjk_run, partial_paragraph, script_fonts, "minor" +) +check("font triage reports the run face and source", + ("latin", "Run Face", "run") in detected_faces["run override"], detected_faces) +check( + "font triage reports the paragraph face and source", + ("latin", "Paragraph Face", "paragraph defaults") + in detected_faces["paragraph default"], + detected_faces, +) +check( + "CJK inherited-font triage includes east-Asian and script-specific faces", + {"East Asian Theme", "Simplified Chinese Theme"} + <= {face for _, face, _ in detected_faces["汉字"]}, detected_faces["汉字"], +) +check("Thai inherited-font triage uses its script mapping", + "Thai Theme" in {face for _, face, _ in detected_faces["ไทย"]}, detected_faces["ไทย"]) +check("explicit Latin/East-Asian/complex-script slots resolve independently", + {(slot, face) for slot, face, _ in detected_faces["A汉ก"]} + == {("latin", "Latin Explicit"), ("eastAsia", "East Explicit"), + ("complexScript", "Complex Explicit")}, detected_faces["A汉ก"]) +check("mixed run combines direct Latin with inherited east-Asian faces", + {("latin", "Latin Only"), ("eastAsia", "East Asian Theme")} + <= {(slot, face) for slot, face, _ in latin_only_faces}, + latin_only_faces) +check("mixed run combines inherited Latin with a direct east-Asian face", + {("latin", "Latin Body"), ("eastAsia", "East Only")} + <= {(slot, face) for slot, face, _ in east_only_faces}, + east_only_faces) +check("Cyrillic plus CJK requires both Latin and east-Asian theme slots", + {("latin", "Cyrillic Theme"), ("eastAsia", "Simplified Chinese Theme")} + <= {(slot, face) for slot, face, _ in cyrillic_cjk_faces}, + cyrillic_cjk_faces) +check("empty +mj-ea generic face falls back to the major script mapping", + ("eastAsia", "Major Hans", "major theme script Hans") + in font_candidates(token_run, font_paragraph, token_fallback_fonts, "minor"), + font_candidates(token_run, font_paragraph, token_fallback_fonts, "minor")) + + +# ---- analyze.md snippet: sparse cache points keep their idx ---------------------- +sparse_chart_xml = ( + '' + 'series' + 'General' + '' + '1' + '5' + '9' + '' +) +from pptx.oxml.ns import qn as pptx_qn + +def cached_numeric_points(series_element, element_name): + return [ + (int(pt.get("idx")), pt.find(pptx_qn("c:v")).text) + for pt in series_element.xpath(f"./c:{element_name}//c:pt") + if pt.get("idx") is not None and pt.find(pptx_qn("c:v")) is not None + ] + +from pptx.oxml import parse_xml as pptx_parse_xml +sparse_series = pptx_parse_xml(sparse_chart_xml) +points = cached_numeric_points(sparse_series, "xVal") +check("sparse cache points keep their idx", points == [(0, "1"), (2, "5"), (5, "9")], points) +compact = [value for _, value in points] +check("compact extraction is provably lossy (negative control)", + compact == ["1", "5", "9"] and len({idx for idx, _ in points}) == len(points)) + + + +# ---- analyze.md: script-aware run faces and table-cell triage -------------------- +from pptx.oxml import parse_xml as pptx_parse_xml2 +from pptx.oxml.ns import qn + +def required_font_slots(text, script_tags_fn): + tags = script_tags_fn(text) + slots = [] + if any(ch.isascii() and ch.isalnum() for ch in text): + slots.append("latin") + if any(tag in ("Hans", "Hant", "Jpan", "Hang") for tag in tags): + slots.append("eastAsia") + if any(tag in ("Arab", "Hebr", "Deva") for tag in tags): + slots.append("complexScript") + return slots or ["latin"] + +def explicit_run_faces(run): + rPr = run._r.find(qn("a:rPr")) + if rPr is None: + return {} + declared = {} + for slot, tag in (("latin", "a:latin"), ("eastAsia", "a:ea"), ("complexScript", "a:cs")): + node = rPr.find(qn(tag)) + if node is not None and node.get("typeface"): + declared[slot] = node.get("typeface") + return declared + +def resolve_faces(run, text, role_fonts, script_tags_fn): + direct = explicit_run_faces(run) + tags = script_tags_fn(text) + result = {} + for slot in required_font_slots(text, script_tags_fn): + if slot in direct: + result[slot] = [direct[slot]] + continue + relevant = { + "eastAsia": {"Hans", "Hant", "Jpan", "Hang"}, + "complexScript": {"Arab", "Hebr", "Deva"}, + }.get(slot, set()) + candidates = [role_fonts[slot]] + candidates.extend(role_fonts["scripts"].get(tag, "") for tag in tags if tag in relevant) + result[slot] = [face for face in dict.fromkeys(candidates) if face] + return result + +run_xml = ( + '' + '' + '\u6d4b\u8bd5' +) +dual_run = pptx_parse_xml2(run_xml) +class FakeRun: + _r = dual_run +check("a CJK run with latin+ea declared resolves to the eastAsia face", + resolve_faces(FakeRun(), "\u6d4b\u8bd5", { + "latin": "Theme Latin", "eastAsia": "Theme East", "complexScript": "Theme CS", + "scripts": {}, + }, script_tags) == {"eastAsia": ["SimSun"]}) +check("a mixed Latin+CJK run reports both applicable declared faces", + resolve_faces(FakeRun(), "Q3 \u6d4b\u8bd5", { + "latin": "Theme Latin", "eastAsia": "Theme East", "complexScript": "Theme CS", + "scripts": {}, + }, script_tags) == {"latin": ["Arial"], "eastAsia": ["SimSun"]}) + +latin_only_xml = ( + '' + 'Q3 \u6d4b\u8bd5' +) +latin_only_run = pptx_parse_xml2(latin_only_xml) +class LatinOnlyRun: + _r = latin_only_run +partial_resolution = resolve_faces(LatinOnlyRun(), "Q3 \u6d4b\u8bd5", { + "latin": "Theme Latin", "eastAsia": "Theme East", "complexScript": "Theme CS", + "scripts": {"Hans": "Theme Hans"}, +}, script_tags) +check("a direct Latin face does not suppress inherited CJK candidates", + partial_resolution["latin"] == ["Arial"] + and partial_resolution["eastAsia"] == ["Theme East", "Theme Hans"], + partial_resolution) +check("run.font.name alone would report only the Latin face (negative control)", + dual_run.find(qn("a:rPr")).find(qn("a:latin")).get("typeface") == "Arial") + +triage_deck = Presentation() +triage_slide = triage_deck.slides.add_slide(triage_deck.slide_layouts[6]) +triage_shape = triage_slide.shapes.add_table(2, 2, 0, 0, 4000000, 2000000) +triage_shape.table.cell(0, 0).text_frame.text = "\u8868\u683c\u6587\u672c" +triage_deck.save("triage-table.pptx") +triage_reopened = Presentation("triage-table.pptx") +triage_table_shape = next(s for s in triage_reopened.slides[0].shapes if s.has_table) +def iter_text_frames(shapes): + for shape in shapes: + if shape.has_text_frame: + yield shape.text_frame + if getattr(shape, "has_table", False): + for row in shape.table.rows: + for cell in row.cells: + yield cell.text_frame + + +def unresolved_graphic_font_regions(shapes): + for shape in shapes: + if getattr(shape, "has_chart", False): + yield {"shape": shape.name, "kind": "chart"} + graphic_data = shape._element.find(".//" + qn("a:graphicData")) + if graphic_data is not None and graphic_data.get("uri") == DIAGRAM_NS: + yield {"shape": shape.name, "kind": "SmartArt"} + + +frames = list(iter_text_frames(triage_reopened.slides[0].shapes)) +check("table-cell text frames are reached by the triage walker", + any("\u8868\u683c\u6587\u672c" in f.text for f in frames), [f.text for f in frames]) +check("the table graphic frame itself has no text frame (negative control)", + not triage_table_shape.has_text_frame) + +chart_shape_for_font_audit = next( + shape for shape in Presentation("input.pptx").slides[0].shapes if shape.has_chart +) +unresolved_graphics = list(unresolved_graphic_font_regions([ + chart_shape_for_font_audit, diagram_shape, triage_table_shape, +])) +check("font triage marks both charts and SmartArt as unresolved regions", + {item["kind"] for item in unresolved_graphics} == {"chart", "SmartArt"}, + unresolved_graphics) +check("ordinary table graphic frames are not mislabeled as chart or SmartArt", + all(item["shape"] != triage_table_shape.name for item in unresolved_graphics), + unresolved_graphics) +hidden_deck_unresolved_graphics = list(unresolved_graphic_font_regions( + iter_shapes(hidden_inventory_slide.shapes) +)) +check("font-region inventory excludes hidden charts and SmartArt while retaining visible siblings", + {item["shape"] for item in hidden_deck_unresolved_graphics} + == {visible_chart_shape.name, "Visible SmartArt"}, + hidden_deck_unresolved_graphics) +try: + if unresolved_graphics: + raise LookupError(f"unresolved chart/SmartArt fonts: {unresolved_graphics}") + graphic_font_audit_rejected = False +except LookupError: + graphic_font_audit_rejected = True +check("unresolved chart/SmartArt font regions fail closed under optimized Python", + graphic_font_audit_rejected) + +print("\n" + ("ALL PPTX FIXTURES PASSED" if not failures else f"{len(failures)} FAILURES: {failures}")) +sys.exit(0 if not failures else 1) diff --git a/plugins/Hylouis233/document-skills/tests/requirements-fixtures.txt b/plugins/Hylouis233/document-skills/tests/requirements-fixtures.txt new file mode 100644 index 0000000..39891dc --- /dev/null +++ b/plugins/Hylouis233/document-skills/tests/requirements-fixtures.txt @@ -0,0 +1,9 @@ +lxml==6.0.2 +openpyxl==3.1.5 +defusedxml==0.7.1 +Pillow==11.3.0 +pymupdf==1.27.2.2 +pypdf==6.9.2 +python-docx==1.2.0 +python-pptx==1.0.2 +reportlab==4.4.10 diff --git a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py new file mode 100644 index 0000000..221a3e6 --- /dev/null +++ b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py @@ -0,0 +1,2227 @@ +# Minimal runnable fixtures for the snippets called out in review: one file per format. +# Each script is self-contained, writes only scratch files into the current directory, +# and exits non-zero on failed assertions. Run from any scratch directory: +# python pdf_fixture.py (deps: reportlab, pypdf, pymupdf) +# python pptx_fixture.py (deps: python-pptx) +# python xlsx_fixture.py (deps: openpyxl) +# python docx_fixture.py (deps: python-docx, pymupdf, soffice on PATH) +import base64 +import csv +import os +import sys +import zipfile +from pathlib import Path +from tempfile import TemporaryFile, mkstemp + +import openpyxl +from openpyxl.cell.rich_text import CellRichText, TextBlock +from openpyxl.cell.text import InlineFont + +failures = [] + + +def check(name, cond, extra=""): + print(("PASS " if cond else "FAIL ") + name + ((" :: " + str(extra)) if not cond and extra else "")) + if not cond: + failures.append(name) + + +# Execute the canonical package.md snippet itself so fixtures cannot drift from the Skill. +package_reference = ( + Path(__file__).resolve().parents[1] / "skills" / "xlsx" / "references" / "package.md" +) +package_markdown = package_reference.read_text(encoding="utf-8") +package_code = package_markdown.split("```python", 1)[1].split("```", 1)[0] +exec(compile(package_code, str(package_reference), "exec"), globals()) + + +# ---- csv.md snippet: sniffed dialect actually reaches the reader --------------- +semicolon_csv = "region;units;note\nEU;120;first\nUS;80;second\n" +with open("input.csv", "w", newline="", encoding="utf-8") as f: + f.write(semicolon_csv) + +with open("input.csv", newline="", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + dialect = csv.Sniffer().sniff(sample) + reader = csv.DictReader(f, dialect=dialect) + rows = list(reader) + +check("sniffer detects the semicolon dialect", dialect.delimiter == ";", dialect.delimiter) +check("rows parse into per-column values", rows[0] == {"region": "EU", "units": "120", "note": "first"}, rows[0]) + +# negative control: default comma reader splits the whole line as one key +with open("input.csv", newline="", encoding="utf-8-sig") as f: + naive = next(csv.DictReader(f)) +check("default reader is proven wrong here (negative control)", list(naive.keys())[0] == "region;units;note", naive) + +# ---- csv.md conversion: formula-looking input remains literal text ----------- +formula_looking = '=HYPERLINK("https://example.invalid", "click")' +csv_wb = openpyxl.Workbook() +csv_cell = csv_wb.active["A1"] +csv_cell.value = formula_looking +csv_cell.data_type = "s" +csv_wb.save("csv-text.xlsx") +csv_reopened = openpyxl.load_workbook("csv-text.xlsx", data_only=False) +check("formula-looking CSV field keeps its exact text", csv_reopened.active["A1"].value == formula_looking) +check("formula-looking CSV field is not an XLSX formula", csv_reopened.active["A1"].data_type == "s") + +unsafe_wb = openpyxl.Workbook() +unsafe_wb.active["A1"] = formula_looking +check("plain assignment is proven unsafe (negative control)", unsafe_wb.active["A1"].data_type == "f") + +CSV_FORMULA_OPERATORS = ("=", "+", "-", "@", "＝", "＋", "－", "＠") + + +def begins_spreadsheet_formula(value): + index = 0 + while index < len(value) and (ord(value[index]) <= 0x20 or value[index] == "\ufeff"): + index += 1 + return value.startswith(CSV_FORMULA_OPERATORS, index) + + +def spreadsheet_csv_field(value, *, mode="safe"): + if mode not in {"safe", "raw"}: + raise ValueError("mode must be 'safe' or 'raw'") + if mode == "safe" and isinstance(value, str) and begins_spreadsheet_formula(value): + return "'" + value + return value + + +def delimiter_for(path): + delimiter = {".csv": ",", ".tsv": "\t"}.get(Path(path).suffix.lower()) + if delimiter is None: + raise ValueError("output must use a .csv or .tsv extension") + return delimiter + + +formula_like_fields = [ + "\ufeff=1+1", "=1+1", "+SUM(A1:A2)", "-2+3", "@cmd", + "\t=1+1", "\r@cmd", "\n-2+3", "\x00\t +SUM(A1:A2)", + "＝1+1", "＋SUM(A1:A2)", "－2+3", "＠cmd", "-7", +] +benign_fields = ["plain", -7, "\tplain", "\rplain", "\nplain", "\ufeffplain", "\x00 plain"] +with open("spreadsheet-safe.csv", "w", newline="", encoding="utf-8") as output: + csv.writer(output).writerow([ + spreadsheet_csv_field(value) for value in formula_like_fields + benign_fields + ]) +with open("spreadsheet-safe.csv", newline="", encoding="utf-8-sig") as exported: + safe_fields = next(csv.reader(exported)) +check("spreadsheet-safe CSV neutralizes operator, control, BOM, and fullwidth prefixes", + safe_fields[:len(formula_like_fields)] + == ["'" + value for value in formula_like_fields], safe_fields) +check("safe CSV preserves benign text and numeric values", + safe_fields[len(formula_like_fields):] + == [str(value) for value in benign_fields], safe_fields) +check("raw CSV mode preserves exact formula-like literal strings", + [spreadsheet_csv_field(value, mode="raw") for value in formula_like_fields] + == formula_like_fields) +try: + spreadsheet_csv_field("=1+1", mode="unknown") + invalid_csv_mode_rejected = False +except ValueError: + invalid_csv_mode_rejected = True +check("CSV export rejects an ambiguous safety mode", invalid_csv_mode_rejected) + +tabular_rows = [["Region", "Units", "Note"], ["EU, West", 120, "\t=1+1"]] +safe_tabular_rows = [ + [spreadsheet_csv_field(value) for value in row] for row in tabular_rows +] +with open("spreadsheet-safe.TSV", "w", newline="", encoding="utf-8") as output: + csv.writer(output, delimiter=delimiter_for("spreadsheet-safe.TSV")).writerows(safe_tabular_rows) +with open("spreadsheet-safe.TSV", newline="", encoding="utf-8") as exported: + tsv_rows = list(csv.reader(exported, delimiter="\t")) +check("TSV export selects a tab delimiter case-insensitively", + tsv_rows == [[str(value) for value in row] for row in safe_tabular_rows], tsv_rows) +check("TSV safe mode neutralizes a control-prefixed formula after round-trip", + tsv_rows[1][2] == "'\t=1+1", tsv_rows) +check("CSV export retains its comma delimiter", delimiter_for("output.csv") == ",") +try: + delimiter_for("output.txt") + unknown_tabular_suffix_rejected = False +except ValueError: + unknown_tabular_suffix_rejected = True +check("tabular export rejects an unknown extension", unknown_tabular_suffix_rejected) + +# ---- edit.md snippet: round_trip_changes detects dropped parts AND stripped extensions ---- +from collections import Counter +from xml.etree import ElementTree as ET + +wb = openpyxl.Workbook() +ws = wb.active +ws.title = "Data" +ws.append(["Region", "Units"]) +ws.append(["EU", 120]) +wb.create_sheet("Keep")["A1"] = "keep" +wb.save("plain.xlsx") + +with zipfile.ZipFile("plain.xlsx") as zin: + payload = {name: zin.read(name) for name in zin.namelist()} + +# ---- package.md: preflight runs before openpyxl and bounds every package part ----- +try: + with open_validated_workbook("plain.xlsx") as validated_plain: + ordinary_package_loaded = validated_plain.sheetnames == ["Data", "Keep"] +except Exception as error: + ordinary_package_loaded = False + ordinary_package_error = error +check("bounded XLSX loader accepts an ordinary package", ordinary_package_loaded, + locals().get("ordinary_package_error")) + +compressed_payload = dict(payload) +compressed_payload["xl/styles.xml"] = ( + b'' + + b"A" * (1024 * 1024) + + b"" +) +with zipfile.ZipFile("compressed-xlsx-bomb.xlsx", "w", zipfile.ZIP_DEFLATED) as archive: + for name, data in compressed_payload.items(): + archive.writestr(name, data) +real_load_workbook = openpyxl.load_workbook +preflight_load_calls = [] +openpyxl.load_workbook = lambda *args, **kwargs: preflight_load_calls.append(args) or None +try: + with open_validated_workbook("compressed-xlsx-bomb.xlsx"): + compressed_bomb_rejected = False +except ValueError as error: + compressed_bomb_rejected = "suspicious compression ratio: xl/styles.xml" in str(error) +finally: + openpyxl.load_workbook = real_load_workbook +check("XLSX compression bomb is rejected before openpyxl runs", + compressed_bomb_rejected and preflight_load_calls == [], + (compressed_bomb_rejected, preflight_load_calls)) +check("XLSX preflight gates remain active under optimized Python", + __debug__ or compressed_bomb_rejected) + +original_member_limit = MAX_MEMBERS +try: + with zipfile.ZipFile("plain.xlsx") as archive: + MAX_MEMBERS = len(archive.infolist()) - 1 + with validated_xlsx_source("plain.xlsx"): + member_limit_rejected = False +except ValueError as error: + member_limit_rejected = str(error) == "archive member count above limit" +finally: + MAX_MEMBERS = original_member_limit +check("XLSX member-count limit rejects a real package before parsing", member_limit_rejected) + +metadata_names = [ + name for name in payload + if name.endswith((".xml", ".rels")) and not name.startswith("xl/worksheets/") +] +metadata_test_limit = max(len(payload[name]) for name in metadata_names) + 128 +oversized_metadata = dict(payload) +oversized_metadata["xl/styles.xml"] = ( + b'" +) +with zipfile.ZipFile("oversized-metadata.xlsx", "w", zipfile.ZIP_STORED) as archive: + for name, data in oversized_metadata.items(): + archive.writestr(name, data) +original_xml_limit = MAX_XML_PART +try: + MAX_XML_PART = metadata_test_limit + with validated_xlsx_source("oversized-metadata.xlsx"): + oversized_metadata_rejected = False +except ValueError as error: + oversized_metadata_rejected = str(error) == "oversized XML part: xl/styles.xml" +finally: + MAX_XML_PART = original_xml_limit +check("metadata XML part limit is enforced before openpyxl", oversized_metadata_rejected) + +large_worksheet_payload = dict(payload) +sheet_name = "xl/worksheets/sheet1.xml" +large_worksheet_payload[sheet_name] = large_worksheet_payload[sheet_name].replace( + b"", b"" +) +with zipfile.ZipFile("large-streamed-worksheet.xlsx", "w", zipfile.ZIP_STORED) as archive: + for name, data in large_worksheet_payload.items(): + archive.writestr(name, data) +original_xml_limit = MAX_XML_PART +original_worksheet_limit = MAX_WORKSHEET_XML +try: + MAX_XML_PART = metadata_test_limit + MAX_WORKSHEET_XML = len(large_worksheet_payload[sheet_name]) + 128 + with validated_xlsx_source("large-streamed-worksheet.xlsx"): + streamed_worksheet_passed = True +except Exception as error: + streamed_worksheet_passed = False + streamed_worksheet_error = error +finally: + MAX_XML_PART = original_xml_limit + MAX_WORKSHEET_XML = original_worksheet_limit +check("large worksheet XML uses the bounded streaming limit, not metadata limit", + streamed_worksheet_passed, locals().get("streamed_worksheet_error")) + +unsafe_xml_payload = dict(payload) +unsafe_xml_payload["xl/unsafe.xml"] = ( + '' + ']>&x;' +).encode("utf-16") +with zipfile.ZipFile("unsafe-utf16-xml.xlsx", "w", zipfile.ZIP_STORED) as archive: + for name, data in unsafe_xml_payload.items(): + archive.writestr(name, data) +try: + with validated_xlsx_source("unsafe-utf16-xml.xlsx"): + utf16_dtd_rejected = False +except ValueError as error: + utf16_dtd_rejected = "unsafe or malformed XML part: xl/unsafe.xml" in str(error) +check("defused XML parsing rejects UTF-16 DTD/entity parts", utf16_dtd_rejected) + +semantic_range_payloads = { + "merge-range-bomb.xlsx": ( + b'' + ), + "hyperlink-range-bomb.xlsx": ( + b'' + b'' + ), +} +for malicious_path, range_markup in semantic_range_payloads.items(): + malicious_payload = dict(payload) + malicious_payload[sheet_name] = malicious_payload[sheet_name].replace( + b"", range_markup + b"" + ) + with zipfile.ZipFile(malicious_path, "w", zipfile.ZIP_DEFLATED) as archive: + for name, data in malicious_payload.items(): + archive.writestr(name, data) + +from openpyxl.comments import Comment + +comment_wb = openpyxl.Workbook() +comment_wb.active["A1"].comment = Comment("bounded note", "fixture") +comment_wb.save("comment-base.xlsx") +comment_wb.close() +with zipfile.ZipFile("comment-base.xlsx") as archive: + comment_payload = {name: archive.read(name) for name in archive.namelist()} +comment_part = next( + name for name in comment_payload + if name.casefold().startswith("xl/comments/") and name.casefold().endswith(".xml") +) +comment_payload[comment_part] = comment_payload[comment_part].replace( + b'ref="A1"', b'ref="A1:XFD1048576"', 1 +) +with zipfile.ZipFile("comment-range-bomb.xlsx", "w", zipfile.ZIP_DEFLATED) as archive: + for name, data in comment_payload.items(): + archive.writestr(name, data) + +semantic_expected_errors = { + **{path: "worksheet cell materialization budget exceeded" + for path in semantic_range_payloads}, + "comment-range-bomb.xlsx": "comment reference must identify one cell", +} + +real_load_workbook = openpyxl.load_workbook +semantic_preflight_load_calls = [] +semantic_range_results = {} +openpyxl.load_workbook = ( + lambda *args, **kwargs: semantic_preflight_load_calls.append(args) or None +) +try: + for malicious_path, expected_error in semantic_expected_errors.items(): + try: + with open_validated_workbook(malicious_path): + semantic_range_results[malicious_path] = False + except ValueError as error: + semantic_range_results[malicious_path] = ( + expected_error in str(error) + ) + + original_materialized_cell_limit = MAX_WORKSHEET_MATERIALIZED_CELLS + try: + MAX_WORKSHEET_MATERIALIZED_CELLS = 1 + with open_validated_workbook("plain.xlsx"): + explicit_cell_limit_rejected = False + except ValueError as error: + explicit_cell_limit_rejected = ( + str(error) == "worksheet cell materialization budget exceeded" + ) + finally: + MAX_WORKSHEET_MATERIALIZED_CELLS = original_materialized_cell_limit +finally: + openpyxl.load_workbook = real_load_workbook + +check( + "merge, hyperlink, and comment ranges are bounded before openpyxl runs", + all(semantic_range_results.values()) and semantic_preflight_load_calls == [], + (semantic_range_results, semantic_preflight_load_calls), +) +check( + "explicit worksheet cells share the pre-load materialization budget", + explicit_cell_limit_rejected and semantic_preflight_load_calls == [], + (explicit_cell_limit_rejected, semantic_preflight_load_calls), +) +with open_validated_workbook("comment-base.xlsx") as validated_comment_wb: + valid_comment_loaded = ( + validated_comment_wb.active["A1"].comment is not None + and validated_comment_wb.active["A1"].comment.text == "bounded note" + ) +check("a valid single-cell comment survives bounded loading", valid_comment_loaded) + +range_list_payloads = { + "range-list-budget.xlsx": b'', + "range-list-whole-sheet.xlsx": ( + b'' + ), +} +for range_list_path, range_list_markup in range_list_payloads.items(): + range_list_payload = dict(payload) + range_list_payload[sheet_name] = range_list_payload[sheet_name].replace( + b"", range_list_markup + b"" + ) + with zipfile.ZipFile(range_list_path, "w", zipfile.ZIP_DEFLATED) as archive: + for name, data in range_list_payload.items(): + archive.writestr(name, data) + +original_range_token_limit = MAX_WORKSHEET_RANGE_TOKENS +original_range_list_char_limit = MAX_WORKSHEET_RANGE_LIST_CHARS +real_load_workbook = openpyxl.load_workbook +range_list_load_calls = [] +openpyxl.load_workbook = lambda *args, **kwargs: range_list_load_calls.append(args) or None +try: + MAX_WORKSHEET_RANGE_TOKENS = 2 + try: + with open_validated_workbook("range-list-budget.xlsx"): + range_token_budget_rejected = False + except ValueError as error: + range_token_budget_rejected = str(error) == "worksheet range-token budget exceeded" + MAX_WORKSHEET_RANGE_TOKENS = 1 + try: + with validated_xlsx_source("range-list-whole-sheet.xlsx"): + whole_sheet_range_accepted = True + except Exception as error: + whole_sheet_range_accepted = False + whole_sheet_range_error = error + MAX_WORKSHEET_RANGE_LIST_CHARS = 8 + try: + with validated_xlsx_source("range-list-whole-sheet.xlsx"): + long_range_list_rejected = False + except ValueError as error: + long_range_list_rejected = "worksheet range list is too long" in str(error) +finally: + MAX_WORKSHEET_RANGE_TOKENS = original_range_token_limit + MAX_WORKSHEET_RANGE_LIST_CHARS = original_range_list_char_limit + openpyxl.load_workbook = real_load_workbook + +check( + "worksheet range-list token growth is bounded before openpyxl runs", + range_token_budget_rejected and range_list_load_calls == [], + (range_token_budget_rejected, range_list_load_calls), +) +check( + "one whole-sheet sqref costs one range token rather than its cell area", + whole_sheet_range_accepted, + locals().get("whole_sheet_range_error"), +) +check("worksheet sqref character growth is bounded", long_range_list_rejected) + +Path("snapshot-source.xlsx").write_bytes(Path("plain.xlsx").read_bytes()) +replacement_wb = openpyxl.Workbook() +replacement_wb.active["A1"] = "UNVALIDATED" +replacement_wb.save("snapshot-replacement.xlsx") +replacement_wb.close() +with validated_xlsx_source("snapshot-source.xlsx") as snapshot_source: + Path("snapshot-source.xlsx").write_bytes( + Path("snapshot-replacement.xlsx").read_bytes() + ) + snapshot_wb = openpyxl.load_workbook(snapshot_source, read_only=True, data_only=True) + try: + snapshot_value = snapshot_wb.active["A1"].value + finally: + snapshot_wb.close() +check( + "validated source is a private snapshot isolated from later path changes", + snapshot_value == "Region", + snapshot_value, +) + +# simulate an unsupported extension part (what a slicer/queries part looks like in the zip) +payload["xl/slicers/slicer1.xml"] = b"" +payload["xl/media/large.bin"] = b"x14:" * 32_768 # binary payload must never be marker-scanned + +# Two records deliberately share a URI/namespace but carry different child content. A coarse +# presence set cannot see one disappear while the other survives. +SHEET_NS = "http://schemas.openxmlformats.org/spreadsheetml/2006/main" +X14_NS = "http://schemas.microsoft.com/office/spreadsheetml/2009/9/main" +EXT_URI = "{00000000-0000-0000-0000-000000000000}" +KEEP_EXT = ( + f'' +).encode() +KEEP_EXT_ALT_PREFIX = ( + f'' +).encode() +DROP_EXT = ( + f'' +).encode() +EXT_LIST_BOTH = b"" + KEEP_EXT + DROP_EXT + b"" +EXT_LIST_KEEP = b"" + KEEP_EXT_ALT_PREFIX + b"" +EXT_LIST_DROP = b"" + DROP_EXT + b"" +payload["xl/worksheets/sheet1.xml"] = payload["xl/worksheets/sheet1.xml"].replace( + b"", EXT_LIST_BOTH + b"") +payload["xl/worksheets/sheet2.xml"] = payload["xl/worksheets/sheet2.xml"].replace( + b"", EXT_LIST_DROP + b"") + +with zipfile.ZipFile("extended.xlsx", "w", zipfile.ZIP_STORED) as zout: + for name, data in payload.items(): + zout.writestr(name, data) + +partial_payload = dict(payload) +partial_payload["xl/worksheets/sheet1.xml"] = partial_payload[ + "xl/worksheets/sheet1.xml" +].replace(EXT_LIST_BOTH, EXT_LIST_KEEP) +with zipfile.ZipFile("partial-extension.xlsx", "w", zipfile.ZIP_STORED) as zout: + for name, data in partial_payload.items(): + zout.writestr(name, data) + +EXT_TAG = f"{{{SHEET_NS}}}ext" + + +def normalized_text(value): + return (value or "").strip() + + +def normalized_element(element): + return ( + element.tag, + tuple(sorted(element.attrib.items())), + normalized_text(element.text), + tuple((normalized_element(child), normalized_text(child.tail)) for child in element), + ) + + +def worksheet_extension_records(archive, info): + records = Counter() + extension_depth = 0 + with archive.open(info) as stream: + for event, element in ET.iterparse(stream, events=("start", "end")): + if event == "start": + if extension_depth: + extension_depth += 1 + elif element.tag == EXT_TAG: + extension_depth = 1 + elif extension_depth: + if extension_depth == 1: + children = tuple( + (normalized_element(child), normalized_text(child.tail)) + for child in element + ) + records[(element.attrib.get("uri", ""), children)] += 1 + element.clear() + extension_depth = 0 + else: + extension_depth -= 1 + else: + element.clear() + return records + + +def archive_inventory(source): + with zipfile.ZipFile(source) as archive: + names = set(archive.namelist()) + extensions = {} + for info in archive.infolist(): + if (info.filename.startswith("xl/worksheets/") + and info.filename.endswith(".xml")): + extensions[info.filename] = worksheet_extension_records(archive, info) + return names, extensions + + +def stripped_extension_records(before, after, common_names): + stripped = [] + for name in sorted(common_names): + for (uri, children), count in (before.get(name, Counter()) + - after.get(name, Counter())).items(): + stripped.extend((name, uri, children) for _ in range(count)) + return sorted(stripped, key=repr) + + +def load_with_round_trip_audit_from_source(source, **load_options): + require(not load_options.get("read_only"), + "round-trip audit requires a normal writable Workbook") + require(load_options.get("rich_text", True) is True, + "round-trip audit must preserve rich-text cell runs") + load_options["rich_text"] = True + source.seek(0) + before_names, before_extensions = archive_inventory(source) + source.seek(0) + audit_workbook = openpyxl.load_workbook(source, **load_options) + try: + with TemporaryFile() as output: + audit_workbook.save(output) + output.seek(0) + after_names, after_extensions = archive_inventory(output) + finally: + audit_workbook.close() + dropped = sorted(before_names - after_names) + stripped_extensions = stripped_extension_records( + before_extensions, after_extensions, before_names & after_names + ) + source.seek(0) + editable_workbook = openpyxl.load_workbook(source, **load_options) + return editable_workbook, dropped, stripped_extensions + + +def load_with_round_trip_audit(path, **load_options): + with validated_xlsx_source(path) as source: + return load_with_round_trip_audit_from_source(source, **load_options) + + +audited_extended, dropped, stripped = load_with_round_trip_audit("extended.xlsx") +audited_extended.close() +check("injected slicer-like part is detected as dropped", "xl/slicers/slicer1.xml" in dropped, dropped) +check("each stripped worksheet extension record is reported with worksheet and URI", + sum(item[0] == "xl/worksheets/sheet1.xml" and item[1] == EXT_URI + for item in stripped) == 2, + stripped) +audited_plain, clean_dropped, clean_stripped = load_with_round_trip_audit("plain.xlsx") +check("clean workbook reports nothing", (clean_dropped, clean_stripped) == ([], [])) +check("round-trip audit returns the actual editable workbook identity", + audited_plain["Data"]["A2"].value == "EU") +audited_plain.close() + +rich_input = openpyxl.Workbook() +rich_input.active.title = "Rich" +rich_input.active["A1"] = CellRichText( + TextBlock(InlineFont(b=True), "Bold"), " and plain" +) +rich_input.save("rich-roundtrip.xlsx") +rich_editable, rich_dropped, rich_stripped = load_with_round_trip_audit( + "rich-roundtrip.xlsx" +) +rich_editable["Rich"]["B1"] = "edited" +rich_editable.save("rich-roundtrip-edited.xlsx") +rich_editable.close() +rich_reopened = openpyxl.load_workbook("rich-roundtrip-edited.xlsx", rich_text=True) +rich_value = rich_reopened["Rich"]["A1"].value +check( + "round-trip audit preserves rich-text runs while editing another cell", + rich_dropped == [] and rich_stripped == [] + and isinstance(rich_value, CellRichText) + and isinstance(rich_value[0], TextBlock) and rich_value[0].font.b is True + and str(rich_value) == "Bold and plain" + and rich_reopened["Rich"]["B1"].value == "edited", + (rich_dropped, rich_stripped, type(rich_value), rich_value), +) +rich_reopened.close() +try: + load_with_round_trip_audit("plain.xlsx", rich_text=False) + rich_text_opt_out_rejected = False +except ValueError as error: + rich_text_opt_out_rejected = "must preserve rich-text" in str(error) +check("round-trip audit rejects rich-text flattening opt-outs", rich_text_opt_out_rejected) + +from openpyxl.drawing.image import Image as AuditImage +Path("audit-image.png").write_bytes(base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9ZlRYAAAAASUVORK5CYII=" +)) +image_input_wb = openpyxl.Workbook() +image_input_wb.active.title = "Data" +image_input_wb.active["A1"] = "before" +image_input_wb.active.add_image(AuditImage("audit-image.png"), "C3") +image_input_wb.save("image-roundtrip.xlsx") +image_editable, image_dropped, image_stripped = load_with_round_trip_audit( + "image-roundtrip.xlsx" +) +image_editable["Data"]["A1"] = "after" +image_editable.save("image-roundtrip-edited.xlsx") +image_editable.close() +image_reopened = openpyxl.load_workbook("image-roundtrip-edited.xlsx") +check("round-trip audit returns a fresh workbook whose image streams can be saved again", + image_dropped == [] and image_stripped == [] + and image_reopened["Data"]["A1"].value == "after" + and len(image_reopened["Data"]._images) == 1, + (image_dropped, image_stripped, len(image_reopened["Data"]._images))) +image_reopened.close() + +real_archive_inventory = archive_inventory +preflight_inventory_calls = [] +def tracking_archive_inventory(source): + preflight_inventory_calls.append(source) + return real_archive_inventory(source) +archive_inventory = tracking_archive_inventory +try: + load_with_round_trip_audit("compressed-xlsx-bomb.xlsx") + audit_bomb_rejected = False +except ValueError as error: + audit_bomb_rejected = "suspicious compression ratio: xl/styles.xml" in str(error) +finally: + archive_inventory = real_archive_inventory +check("round-trip audit rejects a package bomb before raw inventory", + audit_bomb_rejected and preflight_inventory_calls == [], + (audit_bomb_rejected, preflight_inventory_calls)) +inventory_names, inventory_extensions = archive_inventory("extended.xlsx") +check("binary archive parts are named but never marker-scanned", + "xl/media/large.bin" in inventory_names + and "xl/media/large.bin" not in inventory_extensions) + +partial_names, partial_extensions = archive_inventory("partial-extension.xlsx") +coarse_needles = (b"extLst", X14_NS.encode(), EXT_URI.encode()) +coarse_before = {needle for needle in coarse_needles + if needle in payload["xl/worksheets/sheet1.xml"]} +coarse_after = {needle for needle in coarse_needles + if needle in partial_payload["xl/worksheets/sheet1.xml"]} +check("coarse marker presence misses one removed extension record (negative control)", + coarse_before == coarse_after == set(coarse_needles), + (coarse_before, coarse_after)) +partial_loss = stripped_extension_records( + inventory_extensions, partial_extensions, inventory_names & partial_names +) +check("granular extension inventory detects only the removed child-content record", + len(partial_loss) == 1 + and partial_loss[0][0] == "xl/worksheets/sheet1.xml" + and partial_loss[0][1] == EXT_URI + and "drop" in repr(partial_loss[0][2]), + partial_loss) +check("extension inventory keeps records separated by worksheet identity", + inventory_extensions["xl/worksheets/sheet2.xml"] + == partial_extensions["xl/worksheets/sheet2.xml"] + and bool(partial_extensions["xl/worksheets/sheet2.xml"]), + partial_extensions) + +# ---- formatting.md snippet: sheet references built from the real sheet title ------- +wb_f = openpyxl.Workbook() +src = wb_f.active +src.title = "Raw Data" # space forces quoting +src.append(["Region", "Units", "Price"]) +src.append(["EU", 3, 10]) +src.append(["US", 4, 20]) +agg = wb_f.create_sheet("ByRegion") + + +def sheet_ref(sheet): + escaped = sheet.title.replace("'", "''") + return f"'{escaped}'!" + + +ref = sheet_ref(src) +agg.append(["Region", "Units"]) +agg["A2"] = "EU" +agg["B2"] = f"=SUMIF({ref}A:A,A2,{ref}B:B)" +wb_f.save("agg.xlsx") +wb_g = openpyxl.load_workbook("agg.xlsx") +check("formula references the real sheet name", wb_g["ByRegion"]["B2"].value == "=SUMIF('Raw Data'!A:A,A2,'Raw Data'!B:B)", wb_g["ByRegion"]["B2"].value) +apostrophe_sheet = wb_f.create_sheet("O'Brien") +apostrophe_sheet["A1"] = 1 +agg["B3"] = f"=SUM({sheet_ref(apostrophe_sheet)}A:A)" +wb_f.save("apostrophe-agg.xlsx") +apostrophe_formula = openpyxl.load_workbook("apostrophe-agg.xlsx")["ByRegion"]["B3"].value +check( + "quoted sheet reference doubles apostrophes", + apostrophe_formula == "=SUM('O''Brien'!A:A)", + apostrophe_formula, +) +hyphen_sheet = wb_f.create_sheet("Q1-Data") +hyphen_sheet["A1"] = 1 +agg["B4"] = f"=SUM({sheet_ref(hyphen_sheet)}A:A)" +wb_f.save("hyphen-agg.xlsx") +hyphen_formula = openpyxl.load_workbook("hyphen-agg.xlsx")["ByRegion"]["B4"].value +check("ambiguous punctuation is protected by quoting", hyphen_formula == "=SUM('Q1-Data'!A:A)", hyphen_formula) + +# Falsey values are valid categories; blank filtering must not discard or conflate them. +falsey_ws = openpyxl.Workbook().active +for value in ("Category", 0, False, "", None, 0, False): + falsey_ws.append([value]) +regions = [] +seen_region_keys = set() +for (region,) in falsey_ws.iter_rows(min_row=2, min_col=1, max_col=1, values_only=True): + if region is None or region == "": + continue + key = (type(region), region) + if key not in seen_region_keys: + seen_region_keys.add(key) + regions.append(region) +check( + "aggregation preserves numeric zero and boolean false as distinct categories", + len(regions) == 2 + and type(regions[0]) is int and regions[0] == 0 + and type(regions[1]) is bool and regions[1] is False, + [(type(value).__name__, value) for value in regions], +) + +# ---- edit.md structural audit includes non-cell dependencies ------------------ +from openpyxl.chart import BarChart, Reference +from openpyxl.drawing.image import Image +from openpyxl.formula import Tokenizer +from openpyxl.formatting.rule import CellIsRule, ColorScaleRule, DataBarRule, FormulaRule +from openpyxl.utils import get_column_letter +from openpyxl.utils.cell import coordinate_to_tuple, range_boundaries +from openpyxl.workbook.defined_name import DefinedName +from openpyxl.worksheet.datavalidation import DataValidation +from openpyxl.worksheet.hyperlink import Hyperlink +from openpyxl.worksheet.table import Table + + +# ---- create.md chart: duplicate products are described as row-level sales ---- +row_chart_wb = openpyxl.Workbook() +row_chart_ws = row_chart_wb.active +row_chart_ws.title = "Sales" +row_chart_ws.append(["Region", "Product", "Revenue"]) +for row in (("EU", "Widget", 1140), ("EU", "Gadget", 1680), ("US", "Widget", 1900)): + row_chart_ws.append(row) +row_chart = BarChart() +row_chart.type = "col" +row_chart.title = "Revenue by transaction row" +row_chart.add_data(Reference(row_chart_ws, min_col=3, min_row=1, max_row=4), + titles_from_data=True) +row_chart.set_categories(Reference(row_chart_ws, min_col=2, min_row=2, max_row=4)) +row_chart_ws.add_chart(row_chart, "E2") +row_chart_wb.save("row-level-sales-chart.xlsx") +row_chart_reopened = openpyxl.load_workbook("row-level-sales-chart.xlsx", data_only=False) +reopened_row_chart = row_chart_reopened["Sales"]._charts[0] +row_chart_title = reopened_row_chart.title.tx.rich.p[0].r[0].t +check("duplicate product labels are explicitly charted per transaction row", + [row_chart_reopened["Sales"][f"B{row}"].value for row in range(2, 5)].count("Widget") == 2 + and row_chart_title == "Revenue by transaction row" + and reopened_row_chart.ser[0].cat.numRef.f == "'Sales'!$B$2:$B$4" + and reopened_row_chart.ser[0].val.numRef.f == "'Sales'!$C$2:$C$4" + and reopened_row_chart.ser[0].tx.strRef.f == "'Sales'!C1", + row_chart_title) +row_chart_reopened.close() + + +def formula_text(value): + if isinstance(value, str): + return value + if text := getattr(value, "text", None): + return text + fields = ("ref", "r1", "r2", "dt2D", "dtr", "ca", "del1", "del2") + details = ", ".join( + f"{name}={getattr(value, name)!r}" for name in fields if hasattr(value, name) + ) + return f"{type(value).__name__}({details})" + + +def defined_name_values(workbook): + names = workbook.defined_names + return names.values() if hasattr(names, "values") else names.definedName + + +def drawing_anchor_rows(drawing): + anchor = drawing.anchor + if isinstance(anchor, str): + return (coordinate_to_tuple(anchor)[0],) + rows = [] + if marker := getattr(anchor, "_from", None): + rows.append(marker.row + 1) + if marker := getattr(anchor, "to", None): + rows.append(marker.row + 1) + return tuple(rows) + + +def sparse_cells(sheet): + if not hasattr(sheet, "_cells"): + raise RuntimeError("structural edits require a normal writable Worksheet") + return sorted(sheet._cells.values(), key=lambda cell: (cell.row, cell.column)) + + +def sparse_formula_cells(sheet): + return (cell for cell in sparse_cells(sheet) if cell.data_type == "f") + + +def structural_references(workbook): + refs = [] + for item in defined_name_values(workbook): + refs.append(("defined name", item.name, item.attr_text)) + for sheet in workbook.worksheets: + owner = sheet.title + for cell in sparse_formula_cells(sheet): + refs.append(("cell formula", f"{owner}!{cell.coordinate}", + formula_text(cell.value))) + for cell in sparse_cells(sheet): + hyperlink = cell.hyperlink + if hyperlink is None: + continue + location = getattr(hyperlink, "location", None) + target = getattr(hyperlink, "target", None) + refs.append(( + "cell hyperlink", + f"{owner}!{cell.coordinate}", + (getattr(hyperlink, "ref", None), location, target), + )) + for table in sheet.tables.values(): + refs.append(("table", owner + "!" + table.name, table.ref)) + for merged_range in sheet.merged_cells.ranges: + refs.append(("merged range", owner, str(merged_range))) + if sheet.auto_filter.ref: + refs.append(("auto filter", owner, sheet.auto_filter.ref)) + for label, value in ( + ("print area", sheet.print_area), + ("print title rows", sheet.print_title_rows), + ("print title columns", sheet.print_title_cols), + ): + if value: + refs.append((label, owner, str(value))) + for validation in sheet.data_validations.dataValidation: + refs.append(("data validation range", owner, str(validation.sqref))) + for formula in (validation.formula1, validation.formula2): + if formula: + refs.append(("data validation formula", owner, str(formula))) + for conditional_range in sheet.conditional_formatting: + refs.append(("conditional formatting range", owner, str(conditional_range.sqref))) + for rule in sheet.conditional_formatting[conditional_range]: + for formula in getattr(rule, "formula", ()): + refs.append(("conditional formatting formula", owner, str(formula))) + for index, chart in enumerate(sheet._charts, start=1): + refs.append(("drawing anchor", f"{owner} chart {index}", drawing_anchor_rows(chart))) + for element in chart._write().iter(): + if element.tag.rsplit("}", 1)[-1] == "f" and element.text: + refs.append(("chart series", f"{owner} chart {index}", element.text)) + for index, image in enumerate(sheet._images, start=1): + refs.append(("drawing anchor", f"{owner} image {index}", drawing_anchor_rows(image))) + return refs + + +def non_cell_references(workbook): + return [reference for reference in structural_references(workbook) + if reference[0] != "cell formula"] + + +def cell_formula_references(workbook): + refs = [] + for sheet in workbook.worksheets: + for cell in sparse_formula_cells(sheet): + value = cell.value + refs.append(( + "cell formula", + sheet.title, + cell.coordinate, + formula_text(value), + )) + return refs + + +def formula_may_intersect_rows(owner_sheet, formula, shifted_sheet, start_row): + if not isinstance(formula, str) or not formula.startswith("="): + return True + tokens = Tokenizer(formula).items + unmodeled_reference_functions = {"indirect", "offset", "address", "hyperlink"} + if any( + token.type == "FUNC" and token.subtype == "OPEN" + and token.value.rstrip("(").rsplit(":", 1)[-1] + .lstrip("@").rsplit(".", 1)[-1].casefold() + in unmodeled_reference_functions + for token in tokens + ): + return True + for token in tokens: + if token.type != "OPERAND" or token.subtype != "RANGE": + continue + reference = token.value + target_sheet = owner_sheet + if "!" in reference: + qualifier, reference = reference.rsplit("!", 1) + if "[" in qualifier or ":" in qualifier: + return True + target_sheet = qualifier.strip("'").replace("''", "'") + if target_sheet.casefold() != shifted_sheet.casefold(): + continue + try: + _, min_row, _, max_row = range_boundaries(reference.replace("$", "")) + except ValueError: + return True + if min_row is None or max_row is None or max_row >= start_row: + return True + return False + +class LegacyDefinedNames: + """Minimal openpyxl 3.0-style DefinedNameList surface.""" + definedName = [DefinedName("LegacyRange", attr_text="'Legacy'!$A$1")] + + +class LegacyWorkbook: + defined_names = LegacyDefinedNames() + worksheets = [] + + +check( + "structural audit supports openpyxl 3.0 DefinedNameList", + structural_references(LegacyWorkbook()) + == [("defined name", "LegacyRange", "'Legacy'!$A$1")], +) + + +audit_wb = openpyxl.Workbook() +audit_ws = audit_wb.active +audit_ws.title = "Audit" +audit_ws.append(["Value"]) +audit_ws.append([1]) +audit_ws.append([2]) +audit_ws["C1"] = "=SUM(A2:A3)" +audit_wb.defined_names.add(DefinedName("AuditRange", attr_text="'Audit'!$A$2:$A$3")) +audit_ws.add_table(Table(displayName="AuditTable", ref="A1:A3")) +audit_ws.merge_cells("B2:B3") +audit_ws.auto_filter.ref = "A1:A3" +audit_ws.print_area = "A1:B3" +audit_ws.print_title_rows = "1:1" +audit_ws.print_title_cols = "A:A" +validation = DataValidation(type="whole", formula1="'Audit'!$A$2") +validation.add("A2:A3") +audit_ws.add_data_validation(validation) +audit_ws.conditional_formatting.add("A2:A3", FormulaRule(formula=["A2>0"])) +chart = BarChart() +chart.add_data(Reference(audit_ws, min_col=1, min_row=1, max_row=3), titles_from_data=True) +audit_ws.add_chart(chart, "C1") +with open("anchor.png", "wb") as stream: + stream.write(base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9ZlRYAAAAASUVORK5CYII=" + )) +audit_ws.add_image(Image("anchor.png"), "E5") +audit_references = structural_references(audit_wb) +reference_kinds = {kind for kind, _, _ in audit_references} +formula_references = cell_formula_references(audit_wb) +check( + "structural audit covers names, tables, filters, validation, formatting, and charts", + {"defined name", "cell formula", "table", "merged range", "auto filter", "print area", + "print title rows", "print title columns", "data validation range", + "data validation formula", "conditional formatting range", + "conditional formatting formula", "chart series", "drawing anchor"} <= reference_kinds, + reference_kinds, +) +check("drawing audit records an image anchored at the insertion row", + ("drawing anchor", "Audit image 1", (5,)) in audit_references, audit_references) +check( + "structural audit snapshots ordinary cell formulas before row insertion", + formula_references == [("cell formula", "Audit", "C1", "=SUM(A2:A3)")], + formula_references, +) + +hyperlink_wb = openpyxl.Workbook() +hyperlink_ws = hyperlink_wb.active +hyperlink_ws.title = "Data" +hyperlink_ws["A10"] = "destination" +hyperlink_ws["B2"] = "location link" +hyperlink_ws["B2"].hyperlink = Hyperlink(ref="B2", location="'Data'!A10") +hyperlink_ws["B3"] = "hash target link" +hyperlink_ws["B3"].hyperlink = Hyperlink(ref="B3", target="#'Data'!A10") +hyperlink_ws["B4"] = "external link" +hyperlink_ws["B4"].hyperlink = "https://example.com/" +hyperlink_references = non_cell_references(hyperlink_wb) +check( + "structural inventory records every hyperlink anchor and destination form", + ("cell hyperlink", "Data!B2", ("B2", "'Data'!A10", None)) + in hyperlink_references + and ("cell hyperlink", "Data!B3", ("B3", None, "#'Data'!A10")) + in hyperlink_references + and ("cell hyperlink", "Data!B4", ("B4", None, "https://example.com/")) + in hyperlink_references, + hyperlink_references, +) +check("cell hyperlinks block structural edits before insert_rows runs", + bool(hyperlink_references) and hyperlink_ws["A10"].value == "destination") + +# Negative controls prove why both destination and anchor metadata need a rewrite plan. +destination_stale_wb = openpyxl.Workbook() +destination_stale_ws = destination_stale_wb.active +destination_stale_ws.title = "Data" +destination_stale_ws["A10"] = "destination" +destination_stale_ws["B2"] = "jump" +destination_stale_ws["B2"].hyperlink = Hyperlink(ref="B2", location="'Data'!A10") +destination_stale_ws.insert_rows(5) +check("insert_rows leaves an internal hyperlink destination stale (negative control)", + destination_stale_ws["A11"].value == "destination" + and destination_stale_ws["B2"].hyperlink.location == "'Data'!A10") + +anchor_stale_wb = openpyxl.Workbook() +anchor_stale_ws = anchor_stale_wb.active +anchor_stale_ws.title = "Data" +anchor_stale_ws["B8"] = "moving link" +anchor_stale_ws["B8"].hyperlink = Hyperlink(ref="B8", target="https://example.com/") +anchor_stale_ws.insert_rows(5) +check("insert_rows leaves an external hyperlink anchor ref stale (negative control)", + anchor_stale_ws["B9"].value == "moving link" + and anchor_stale_ws["B9"].hyperlink.ref == "B8") +anchor_stale_wb.save("stale-hyperlink-anchor.xlsx") +anchor_stale_reopened = openpyxl.load_workbook("stale-hyperlink-anchor.xlsx") +check("stale hyperlink ref detaches from moved text after save/reopen (negative control)", + anchor_stale_reopened["Data"]["B8"].hyperlink is not None + and anchor_stale_reopened["Data"]["B9"].value == "moving link" + and anchor_stale_reopened["Data"]["B9"].hyperlink is None) +anchor_stale_reopened.close() + +sparse_scan_wb = openpyxl.Workbook() +sparse_scan_ws = sparse_scan_wb.active +sparse_scan_ws.title = "Sparse" +sparse_scan_ws["D2"] = "=1+1" +sparse_scan_ws["XFD1048576"].number_format = "0.00" +sparse_scan_wb.save("sparse-structural-scan.xlsx") +sparse_scan_wb.close() +sparse_scan_wb = openpyxl.load_workbook("sparse-structural-scan.xlsx", data_only=False) +sparse_scan_ws = sparse_scan_wb["Sparse"] +original_sparse_iter_rows = sparse_scan_ws.iter_rows +sparse_scan_ws.iter_rows = lambda *args, **kwargs: (_ for _ in ()).throw( + RuntimeError("rectangular formula scan must not run") +) +try: + sparse_structural = structural_references(sparse_scan_wb) + sparse_formulas = cell_formula_references(sparse_scan_wb) +finally: + sparse_scan_ws.iter_rows = original_sparse_iter_rows +check("structural formula inventory walks sparse cells at worksheet limits", + ("cell formula", "Sparse!D2", "=1+1") in sparse_structural + and sparse_formulas == [("cell formula", "Sparse", "D2", "=1+1")], + (sparse_structural, sparse_formulas)) +sparse_scan_wb.close() +check("intersecting formula ranges are blocked before row insertion", + formula_may_intersect_rows("Audit", "=SUM(A2:A3)", "Audit", 3)) +check("audited formula ranges above the insertion can proceed", + not formula_may_intersect_rows("Data", "=C2*1.08", "Data", 5)) +check("sheet qualifiers are matched case-insensitively", + formula_may_intersect_rows("Summary", "=SUM(data!A5:A6)", "Data", 5)) +check("a genuinely different sheet remains outside the shifted rows", + not formula_may_intersect_rows("Summary", "=SUM(Archive!A5:A6)", "Data", 5)) +check("INDIRECT string references require a manual structural rewrite plan", + formula_may_intersect_rows("Data", '=SUM(INDIRECT("A5:A6"))', "Data", 5)) +check("OFFSET numeric row references require a manual structural rewrite plan", + formula_may_intersect_rows("Data", "=SUM(OFFSET(A1,4,0,2,1))", "Data", 5)) +check("HYPERLINK string destinations require a manual structural rewrite plan", + formula_may_intersect_rows( + "Data", '=HYPERLINK("#\'Data\'!A10","jump")', "Data", 5 + )) +check("implicit-intersection INDIRECT references require a manual rewrite plan", + formula_may_intersect_rows("Data", '=@INDIRECT("A5:A6")', "Data", 5)) +check("OFFSET used by the range operator requires a manual rewrite plan", + formula_may_intersect_rows("Data", "=SUM(A1:OFFSET(A1,5,0))", "Data", 5)) +check("INDIRECT text inside a string does not create a dynamic reference", + not formula_may_intersect_rows( + "Data", '=IF(A1="INDIRECT(A5:A6)",1,0)', "Data", 5 + )) + +stale_wb = openpyxl.Workbook() +stale_ws = stale_wb.active +stale_ws["A5"], stale_ws["A6"] = 10, 20 +stale_ws["B1"] = "=SUM(A5:A6)" +stale_formula_before = cell_formula_references(stale_wb) +stale_ws.insert_rows(5) +check( + "insert_rows leaves intersecting formulas stale (negative control)", + stale_formula_before == [("cell formula", "Sheet", "B1", "=SUM(A5:A6)")] + and stale_ws["B1"].value == "=SUM(A5:A6)" + and (stale_ws["A6"].value, stale_ws["A7"].value) == (10, 20), + (stale_formula_before, stale_ws["B1"].value), +) + +safe_wb = openpyxl.Workbook() +safe_ws = safe_wb.active +safe_ws.title = "Data" +safe_ws["C2"], safe_ws["D2"] = 100, "=C2*1.08" +safe_before = cell_formula_references(safe_wb) +safe_dependencies = [ + reference for reference in safe_before + if formula_may_intersect_rows(reference[1], reference[3], "Data", 5) +] +if not safe_dependencies: + safe_ws.insert_rows(5) + safe_wb.calculation.fullCalcOnLoad = True + safe_wb.calculation.forceFullCalc = True + safe_wb.calculation.calcMode = "auto" + safe_wb.save("audited-structural-edit.xlsx") +safe_reopened = openpyxl.load_workbook("audited-structural-edit.xlsx", data_only=False) +check("audited non-intersecting formula path reaches save", + safe_reopened["Data"]["D2"].value == "=C2*1.08") +check("formula edit forces recalculation after save", + safe_reopened.calculation.fullCalcOnLoad is True + and safe_reopened.calculation.calcMode == "auto") + +legacy_name = DefinedName("LegacyName", attr_text="Audit!$A$1") +legacy_names = type("LegacyDefinedNames", (), {"definedName": [legacy_name]})() +legacy_workbook = type("LegacyWorkbook", (), {"defined_names": legacy_names})() +check("defined-name adapter supports openpyxl 3.0 collections", + list(defined_name_values(legacy_workbook)) == [legacy_name]) + +formula_only = openpyxl.Workbook() +formula_only.active["A1"] = "=Data!A5" +check("structural guard sees formulas before row insertion", + structural_references(formula_only) + == [("cell formula", "Sheet!A1", "=Data!A5")], + structural_references(formula_only)) + +audited_wb = openpyxl.Workbook() +audited_ws = audited_wb.active +audited_ws.title = "Data" +audited_ws["B2"] = "=A2*2" +audited_ws["A5"] = "shift me" +audited_references_before = structural_references(audited_wb) +audited_unchanged = {("cell formula", "Data!B2", "=A2*2")} +unaudited_references = [ + reference for reference in audited_references_before + if reference not in audited_unchanged +] +if not unaudited_references: + audited_ws.insert_rows(5) + audited_ws["D2"] = "=C2*1.08" + audited_wb.save("audited-edit.xlsx") +audited_reopened = openpyxl.load_workbook("audited-edit.xlsx", data_only=False) +check("exact dependency allowlist lets an audited edit reach save", + audited_references_before == [("cell formula", "Data!B2", "=A2*2")] + and unaudited_references == [] + and audited_reopened["Data"]["A6"].value == "shift me" + and audited_reopened["Data"]["B2"].value == "=A2*2" + and audited_reopened["Data"]["D2"].value == "=C2*1.08") + +unsafe_wb = openpyxl.Workbook() +unsafe_ws = unsafe_wb.active +unsafe_ws.title = "Data" +unsafe_ws["A5"], unsafe_ws["A6"] = 10, 20 +unsafe_ws["B1"] = "=SUM(A5:A6)" +unsafe_dependencies = [ + reference for reference in structural_references(unsafe_wb) + if reference not in audited_unchanged +] +unsafe_blocked_before_insert = bool(unsafe_dependencies) +check("intersecting formula not on the exact allowlist is blocked before insertion", + unsafe_blocked_before_insert + and unsafe_ws["A5"].value == 10 and unsafe_ws["A6"].value == 20, + unsafe_dependencies) + +# and the edit itself still works after the warning path +wb2 = openpyxl.load_workbook("plain.xlsx") +wb2["Data"]["B2"] = "=B2*1" # formula stays a formula +wb2["Data"]["B2"].number_format = "#,##0.00" +wb2.save("edited.xlsx") +wb3 = openpyxl.load_workbook("edited.xlsx") +check("edited cell keeps a formula string", isinstance(wb3["Data"]["B2"].value, str) and wb3["Data"]["B2"].value.startswith("=")) +expected_number_formats = {"Data": {"B2": "#,##0.00"}} +format_matches = all( + wb3[sheet][coordinate].number_format == expected + for sheet, cells in expected_number_formats.items() + for coordinate, expected in cells.items() +) +check("task-specific number format mapping is verified", format_matches) + +# ---- read.md snippet: multi-sheet profiles cover every sheet ---------------------- +import posixpath +from openpyxl.worksheet.formula import ArrayFormula, DataTableFormula + +MAIN_NS = "http://schemas.openxmlformats.org/spreadsheetml/2006/main" +DOC_REL_NS = "http://schemas.openxmlformats.org/officeDocument/2006/relationships" +PKG_REL_NS = "http://schemas.openxmlformats.org/package/2006/relationships" +CELL_TAG = f"{{{MAIN_NS}}}c" +FORMULA_TAG = f"{{{MAIN_NS}}}f" +VALUE_TAG = f"{{{MAIN_NS}}}v" +INLINE_STRING_TAG = f"{{{MAIN_NS}}}is" +MAX_EXPLICIT_CELLS = 1_000_000 +MAX_PROFILE_RECTANGLE_CELLS = 100_000 + + +def worksheet_part(archive, sheet_name): + workbook = ET.fromstring(archive.read("xl/workbook.xml")) + sheet = next( + item for item in workbook.iter(f"{{{MAIN_NS}}}sheet") + if item.attrib["name"] == sheet_name + ) + relationship_id = sheet.attrib[f"{{{DOC_REL_NS}}}id"] + relationships = ET.fromstring(archive.read("xl/_rels/workbook.xml.rels")) + target = next( + item.attrib["Target"] for item in relationships.iter(f"{{{PKG_REL_NS}}}Relationship") + if item.attrib["Id"] == relationship_id + ) + part = target.lstrip("/") if target.startswith("/") else posixpath.normpath( + posixpath.join("xl", target) + ) + require(part in archive.namelist(), f"worksheet part is missing: {part}") + return part + + +def worksheet_xml_profile(archive, part): + min_row = min_column = max_row = max_column = None + explicit_cell_count = 0 + formula_count = missing_formula_count = 0 + missing_formula_samples = [] + coordinate = cell_type = value_text = formula_display = None + has_formula = value_seen = inline_string_seen = False + with archive.open(part) as source: + for event, element in ET.iterparse(source, events=("start", "end")): + if event == "start" and element.tag == CELL_TAG: + coordinate = element.attrib.get("r") + require(coordinate is not None, f"cell without a coordinate in {part}") + cell_type = element.attrib.get("t") + has_formula = value_seen = inline_string_seen = False + value_text = formula_display = None + elif event == "end" and coordinate is not None: + if element.tag == FORMULA_TAG: + has_formula = True + if element.text is not None: + formula_display = "=" + element.text + else: + details = ", ".join( + f"{key}={value!r}" for key, value in sorted(element.attrib.items()) + ) + formula_display = f"" + elif element.tag == VALUE_TAG: + value_seen = True + value_text = element.text + elif element.tag == INLINE_STRING_TAG and cell_type == "inlineStr": + inline_string_seen = True + elif element.tag == CELL_TAG: + explicit_cell_count += 1 + require(explicit_cell_count <= MAX_EXPLICIT_CELLS, + f"too many explicit worksheet cells in {part}") + row_index, column_index = coordinate_to_tuple(coordinate) + require(1 <= row_index <= 1_048_576 and 1 <= column_index <= 16_384, + f"cell coordinate outside XLSX limits: {coordinate}") + scalar_value = value_seen and ( + value_text not in (None, "") or cell_type == "str" + ) + populated = has_formula or scalar_value or inline_string_seen + if populated: + min_row = row_index if min_row is None else min(min_row, row_index) + min_column = (column_index if min_column is None + else min(min_column, column_index)) + max_row = row_index if max_row is None else max(max_row, row_index) + max_column = (column_index if max_column is None + else max(max_column, column_index)) + valid_cache = value_seen and ( + value_text not in (None, "") or cell_type == "str" + ) + if has_formula: + formula_count += 1 + if not valid_cache: + missing_formula_count += 1 + if len(missing_formula_samples) < 10: + missing_formula_samples.append((coordinate, formula_display)) + coordinate = None + element.clear() + elif event == "end": + element.clear() + if max_row is None: + bounds, extent, first_populated_row = None, "A1:A1", None + else: + bounds = (min_row, min_column, max_row, max_column) + extent = (f"{get_column_letter(min_column)}{min_row}:" + f"{get_column_letter(max_column)}{max_row}") + first_populated_row = min_row + return { + "part": part, + "bounds": bounds, + "extent": extent, + "first_populated_row": first_populated_row, + "formula_count": formula_count, + "missing_formula_count": missing_formula_count, + "missing_formula_samples": missing_formula_samples, + "explicit_cell_count": explicit_cell_count, + } + + +def bounded_sample_rows(worksheet, profile, *, max_cells=MAX_PROFILE_RECTANGLE_CELLS): + if profile["bounds"] is None: + return iter(()) + min_row, min_column, max_row, max_column = profile["bounds"] + sample_max_row = min(max_row, min_row + 5) + sample_cells = (sample_max_row - min_row + 1) * (max_column - min_column + 1) + require(sample_cells <= max_cells, + f"sample rectangle is too large: {worksheet.title} ({sample_cells} cells)") + return worksheet.iter_rows( + min_row=min_row, min_col=min_column, + max_row=sample_max_row, max_col=max_column, + values_only=True, + ) + + +def worksheet_declared_dimension(worksheet): + if worksheet.max_row is None or worksheet.max_column is None: + return None + return worksheet.calculate_dimension() + +wb_h = openpyxl.Workbook() +wb_h.active.title = "First" +wb_h.active["A1"] = "=1+1" +wb_h.active["A2"] = ArrayFormula("A2:A3", "=ROW(A2:A3)") +wb_h.active["C1"] = "=literal" +wb_h.active["C1"].data_type = "s" +wb_h.active["D1"] = "+literal" +wb_h.active["E1"] = "-literal" +wb_h.active["F1"] = "@literal" +second = wb_h.create_sheet("Second") +second["A1"] = "plain" +second["A2"] = "=2+2" +second["B1"] = DataTableFormula(ref="B1:B2", r1="C1") +wb_h.save("multi.xlsx") +# openpyxl writes an empty for uncached formulas. Remove those elements explicitly so +# this fixture represents a truly absent cache rather than a cached displayed blank. +with zipfile.ZipFile("multi.xlsx") as archive: + multi_members = {name: archive.read(name) for name in archive.namelist()} +for name in [item for item in multi_members if item.startswith("xl/worksheets/") + and item.endswith(".xml")]: + root = ET.fromstring(multi_members[name]) + for cell in root.iter(CELL_TAG): + if cell.find(FORMULA_TAG) is not None: + cached_value = cell.find(VALUE_TAG) + if cached_value is not None: + cell.remove(cached_value) + multi_members[name] = ET.tostring(root, encoding="utf-8", xml_declaration=True) +with zipfile.ZipFile("multi.xlsx", "w", zipfile.ZIP_DEFLATED) as archive: + for name, data in multi_members.items(): + archive.writestr(name, data) +with validated_xlsx_source("multi.xlsx") as multi_source: + formula_wb = openpyxl.load_workbook(multi_source, read_only=True, data_only=False) + multi_source.seek(0) + value_wb = openpyxl.load_workbook(multi_source, read_only=True, data_only=True) + try: + multi_source.seek(0) + with zipfile.ZipFile(multi_source) as archive: + profiled = list(value_wb.sheetnames) + multi_profiles = { + sheet_name: worksheet_xml_profile( + archive, worksheet_part(archive, sheet_name) + ) + for sheet_name in profiled + } + uncached = { + (sheet_name, coordinate, formula) + for sheet_name, profile in multi_profiles.items() + for coordinate, formula in profile["missing_formula_samples"] + } + paired_same_source = ( + next(formula_wb["First"].iter_rows(min_row=1, max_row=1))[0].value == "=1+1" + and next(value_wb["First"].iter_rows(min_row=1, max_row=1))[0].value is None + ) + finally: + formula_wb.close() + value_wb.close() +check("multi-sheet profile iterates every sheet", profiled == ["First", "Second"], profiled) +check("formula/value read-only streams share one validated source identity", + paired_same_source) +check("uncached formulas found on both sheets", {item[0] for item in uncached} == {"First", "Second"}, uncached) +check("array-formula objects are detected by data_type", ("First", "A2", "=ROW(A2:A3)") in uncached, uncached) +data_table_entry = next(item for item in uncached if item[:2] == ("Second", "B1")) +check("data-table formulas have stable diagnostic text", + data_table_entry[2].startswith("= {"A1", "A2"}, missing_caches) + + +def write_formula_cache_fixture(path, cache_kind): + workbook = openpyxl.Workbook() + sheet = workbook.active + sheet.title = "Data" + sheet["A1"] = ('=IF(TRUE,"","x")' if cache_kind == "empty-string" else "=1+1") + workbook.save(path) + with zipfile.ZipFile(path) as archive: + members = {name: archive.read(name) for name in archive.namelist()} + root = ET.fromstring(members["xl/worksheets/sheet1.xml"]) + cell = next(item for item in root.iter(CELL_TAG) if item.attrib["r"] == "A1") + cached_value = cell.find(VALUE_TAG) + if cache_kind == "nonempty": + cell.attrib.pop("t", None) + if cached_value is None: + cached_value = ET.SubElement(cell, VALUE_TAG) + cached_value.text = "2" + elif cache_kind == "empty-string": + cell.set("t", "str") + if cached_value is None: + cached_value = ET.SubElement(cell, VALUE_TAG) + cached_value.text = None + elif cache_kind == "bare-empty": + cell.attrib.pop("t", None) + if cached_value is None: + cached_value = ET.SubElement(cell, VALUE_TAG) + cached_value.text = None + elif cache_kind == "absent": + cell.attrib.pop("t", None) + if cached_value is not None: + cell.remove(cached_value) + else: + raise ValueError(cache_kind) + members["xl/worksheets/sheet1.xml"] = ET.tostring( + root, encoding="utf-8", xml_declaration=True, + ) + with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as archive: + for name, data in members.items(): + archive.writestr(name, data) + + +MAX_CSV_EXPORT_CELLS = 5_000_000 + + +def export_formula_values(source_path, output_path, *, max_cells=MAX_CSV_EXPORT_CELLS): + sheet_name = "Data" + destination = Path(output_path) + temporary = None + try: + descriptor, temporary_name = mkstemp( + dir=destination.parent, prefix=f".{destination.name}.", suffix=".tmp" + ) + temporary = Path(temporary_name) + os.close(descriptor) + with validated_xlsx_source(source_path) as package_source: + value_book = openpyxl.load_workbook( + package_source, read_only=True, data_only=True + ) + try: + package_source.seek(0) + with zipfile.ZipFile(package_source) as archive: + profile = worksheet_xml_profile( + archive, worksheet_part(archive, sheet_name) + ) + if profile["missing_formula_count"]: + raise RuntimeError( + "formula cells have no cached value: " + f"{profile['missing_formula_samples']}" + ) + value_sheet = value_book[sheet_name] + value_sheet.reset_dimensions() + if profile["bounds"] is None: + rows = iter(()) + else: + min_row, min_column, max_row, max_column = profile["bounds"] + export_cells = ( + (max_row - min_row + 1) * (max_column - min_column + 1) + ) + require(export_cells <= max_cells, + f"CSV export rectangle is too large: {export_cells} cells") + rows = value_sheet.iter_rows( + min_row=min_row, min_col=min_column, + max_row=max_row, max_col=max_column, + values_only=True, + ) + with temporary.open("w", newline="", encoding="utf-8") as output: + writer = csv.writer(output) + for value_row in rows: + writer.writerow([ + spreadsheet_csv_field(value) for value in value_row + ]) + finally: + value_book.close() + except Exception: + if temporary is not None: + temporary.unlink(missing_ok=True) + raise + require(temporary is not None, "CSV temporary output was not created") + temporary.replace(destination) + + +write_formula_cache_fixture("cached-value.xlsx", "nonempty") +write_formula_cache_fixture("cached-empty.xlsx", "empty-string") +write_formula_cache_fixture("bare-empty-cache.xlsx", "bare-empty") +write_formula_cache_fixture("missing-cache.xlsx", "absent") +with zipfile.ZipFile("cached-value.xlsx") as archive: + cached_value_cells = cached_formula_coordinates(archive, worksheet_part(archive, "Data")) +with zipfile.ZipFile("cached-empty.xlsx") as archive: + cached_empty_cells = cached_formula_coordinates(archive, worksheet_part(archive, "Data")) +with zipfile.ZipFile("bare-empty-cache.xlsx") as archive: + bare_empty_cells = cached_formula_coordinates(archive, worksheet_part(archive, "Data")) +with zipfile.ZipFile("missing-cache.xlsx") as archive: + truly_missing_cells = cached_formula_coordinates(archive, worksheet_part(archive, "Data")) +check("XML cache inventory accepts nonempty and typed empty-string caches", + cached_value_cells == cached_empty_cells == {"A1"}, + (cached_value_cells, cached_empty_cells)) +check("XML cache inventory rejects bare empty and an absent cache", + bare_empty_cells == truly_missing_cells == set(), + (bare_empty_cells, truly_missing_cells)) + + +def profile_missing_formula_caches(source_path): + with validated_xlsx_source(source_path) as package_source: + with zipfile.ZipFile(package_source) as archive: + profile = worksheet_xml_profile( + archive, worksheet_part(archive, "Data") + ) + return [coordinate for coordinate, _ in profile["missing_formula_samples"]] + + +check("workbook profiling accepts nonempty and typed blank formula caches", + profile_missing_formula_caches("cached-value.xlsx") == [] + and profile_missing_formula_caches("cached-empty.xlsx") == []) +check("workbook profiling reports absent and untyped empty formula caches", + profile_missing_formula_caches("missing-cache.xlsx") == ["A1"] + and profile_missing_formula_caches("bare-empty-cache.xlsx") == ["A1"]) + + +def write_region_key_fixture(path, cache_kind): + workbook = openpyxl.Workbook() + sheet = workbook.active + sheet.title = "Data" + sheet.append(["Region", "Units", "Revenue", "Source Region"]) + sheet.append(["=D2", 2, 4, "EU"]) + workbook.save(path) + with zipfile.ZipFile(path) as archive: + members = {name: archive.read(name) for name in archive.namelist()} + root = ET.fromstring(members["xl/worksheets/sheet1.xml"]) + cell = next(item for item in root.iter(CELL_TAG) if item.attrib["r"] == "A2") + value = cell.find(VALUE_TAG) + if cache_kind == "nonempty": + cell.set("t", "str") + if value is None: + value = ET.SubElement(cell, VALUE_TAG) + value.text = "EU" + elif cache_kind == "empty-string": + cell.set("t", "str") + if value is None: + value = ET.SubElement(cell, VALUE_TAG) + value.text = None + elif cache_kind == "absent": + cell.attrib.pop("t", None) + if value is not None: + cell.remove(value) + else: + raise ValueError(cache_kind) + members["xl/worksheets/sheet1.xml"] = ET.tostring( + root, encoding="utf-8", xml_declaration=True, + ) + with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as archive: + for name, data in members.items(): + archive.writestr(name, data) + + +def aggregation_regions(source_path, *, max_row_span=100_000, + approved_feature_loss=False): + with validated_xlsx_source(source_path) as package_source: + formula_book, dropped_parts, stripped_extensions = ( + load_with_round_trip_audit_from_source(package_source, data_only=False) + ) + if (dropped_parts or stripped_extensions) and not approved_feature_loss: + formula_book.close() + raise RuntimeError( + f"openpyxl would drop parts={dropped_parts!r}, " + f"extensions={stripped_extensions!r}" + ) + value_book = None + try: + source_sheet = formula_book["Data"] + source_cells = sorted( + (cell for cell in source_sheet._cells.values() + if cell.row >= 2 and cell.column == 1 and cell.value is not None), + key=lambda cell: cell.row, + ) + if source_cells: + min_source_row, max_source_row = source_cells[0].row, source_cells[-1].row + row_span = max_source_row - min_source_row + 1 + require(row_span <= max_row_span, + f"aggregation row span is too large: {row_span}") + wanted_formula_coordinates = { + cell.coordinate for cell in source_cells if cell.data_type == "f" + } + package_source.seek(0) + with zipfile.ZipFile(package_source) as archive: + cached_cells = cached_formula_coordinates( + archive, worksheet_part(archive, "Data"), wanted_formula_coordinates + ) + package_source.seek(0) + value_book = openpyxl.load_workbook( + package_source, read_only=True, data_only=True + ) + value_sheet = value_book["Data"] + value_sheet.reset_dimensions() + regions = [] + missing = [] + if source_cells: + source_by_row = {cell.row: cell for cell in source_cells} + value_rows = value_sheet.iter_rows( + min_row=min_source_row, max_row=max_source_row, + min_col=1, max_col=1, + ) + for row_index, value_row in enumerate(value_rows, start=min_source_row): + source_cell = source_by_row.get(row_index) + if source_cell is None: + continue + value_cell = value_row[0] + region = (value_cell.value if source_cell.data_type == "f" + else source_cell.value) + if (source_cell.data_type == "f" and region is None + and source_cell.coordinate not in cached_cells): + missing.append(source_cell.coordinate) + continue + if region is not None and region != "": + regions.append(region) + finally: + if value_book is not None: + value_book.close() + formula_book.close() + if missing: + raise RuntimeError(f"aggregation keys have no cached value: {missing}") + return regions + + +write_region_key_fixture("region-key-cached.xlsx", "nonempty") +write_region_key_fixture("region-key-empty.xlsx", "empty-string") +write_region_key_fixture("region-key-missing.xlsx", "absent") +cached_regions = aggregation_regions("region-key-cached.xlsx") +aggregate_book = openpyxl.Workbook() +aggregate_book.active["A1"] = cached_regions[0] +check("formula-backed aggregation key is copied from its cached displayed value", + aggregate_book.active["A1"].value == "EU" + and aggregate_book.active["A1"].data_type != "f") +check("typed cached blank aggregation keys are skipped", + aggregation_regions("region-key-empty.xlsx") == []) +try: + aggregation_regions("region-key-missing.xlsx") + missing_region_cache_rejected = False +except RuntimeError as error: + missing_region_cache_rejected = "A2" in str(error) +check("aggregation fails closed when a formula key has no cached value", + missing_region_cache_rejected) + +feature_loss_payload = { + name: data for name, data in payload.items() if name != "xl/media/large.bin" +} +with zipfile.ZipFile("aggregation-feature-loss.xlsx", "w", zipfile.ZIP_STORED) as archive: + for name, data in feature_loss_payload.items(): + archive.writestr(name, data) +try: + aggregation_regions("aggregation-feature-loss.xlsx") + aggregation_feature_loss_rejected = False +except RuntimeError as error: + aggregation_feature_loss_rejected = "openpyxl would drop parts=" in str(error) +check("aggregation feature-loss audit gates the exact workbook later edited", + aggregation_feature_loss_rejected) + +export_formula_values("cached-value.xlsx", "cached-value.csv") +with open("cached-value.csv", newline="", encoding="utf-8") as exported: + cached_value_rows = list(csv.reader(exported)) +check("nonempty cached formula result exports its displayed value", + cached_value_rows == [["2"]], cached_value_rows) +export_formula_values("cached-empty.xlsx", "cached-empty.csv") +with open("cached-empty.csv", newline="", encoding="utf-8") as exported: + cached_empty_rows = list(csv.reader(exported)) +check("cached empty-string formula result exports as a displayed blank", + cached_empty_rows == [[""]], cached_empty_rows) +Path("missing-cache.csv").write_text("sentinel\n", encoding="utf-8") +Path("missing-cache.csv.tmp").write_text("unrelated temporary\n", encoding="utf-8") +try: + export_formula_values("missing-cache.xlsx", "missing-cache.csv") + missing_cache_rejected = False +except RuntimeError as error: + missing_cache_rejected = "A1" in str(error) +check("formula with no XML cache element is rejected", missing_cache_rejected) +check("failed cache audit preserves the prior destination and removes its temporary file", + Path("missing-cache.csv").read_text(encoding="utf-8") == "sentinel\n" + and Path("missing-cache.csv.tmp").read_text(encoding="utf-8") + == "unrelated temporary\n" + and list(Path(".").glob(".missing-cache.csv.*.tmp")) == []) +Path("bare-empty-cache.csv").write_text("sentinel\n", encoding="utf-8") +Path("bare-empty-cache.csv.tmp").write_text("unrelated temporary\n", encoding="utf-8") +try: + export_formula_values("bare-empty-cache.xlsx", "bare-empty-cache.csv") + bare_empty_rejected = False +except RuntimeError as error: + bare_empty_rejected = "A1" in str(error) +check("untyped bare empty from an uncalculated formula is rejected", + bare_empty_rejected) +check("bare-empty rejection preserves destination and removes temporary output", + Path("bare-empty-cache.csv").read_text(encoding="utf-8") == "sentinel\n" + and Path("bare-empty-cache.csv.tmp").read_text(encoding="utf-8") + == "unrelated temporary\n" + and list(Path(".").glob(".bare-empty-cache.csv.*.tmp")) == []) + +csv_sparse_wb = openpyxl.Workbook() +csv_sparse_ws = csv_sparse_wb.active +csv_sparse_ws.title = "Data" +csv_sparse_ws["A1"], csv_sparse_ws["A2"] = "Region", "EU" +csv_sparse_ws["XFD1048576"].number_format = "0.00" +csv_sparse_wb.save("csv-style-extreme.xlsx") +export_formula_values("csv-style-extreme.xlsx", "csv-style-extreme.csv") +with open("csv-style-extreme.csv", newline="", encoding="utf-8") as exported: + csv_sparse_rows = list(csv.reader(exported)) +check("CSV export ignores a style-only extreme cell without rectangular expansion", + csv_sparse_rows == [["Region"], ["EU"]], csv_sparse_rows) + +csv_far_wb = openpyxl.Workbook() +csv_far_ws = csv_far_wb.active +csv_far_ws.title = "Data" +csv_far_ws["A1"], csv_far_ws["XFD1048576"] = "near", "far" +csv_far_wb.save("csv-far-values.xlsx") +Path("csv-far-values.csv").write_text("sentinel\n", encoding="utf-8") +Path("csv-far-values.csv.tmp").write_text("unrelated temporary\n", encoding="utf-8") +try: + export_formula_values("csv-far-values.xlsx", "csv-far-values.csv", max_cells=1_000) + csv_rectangle_rejected = False +except ValueError as error: + csv_rectangle_rejected = "CSV export rectangle is too large" in str(error) +check("CSV export rejects a far-apart logical rectangle before iteration", + csv_rectangle_rejected + and Path("csv-far-values.csv").read_text(encoding="utf-8") == "sentinel\n" + and Path("csv-far-values.csv.tmp").read_text(encoding="utf-8") + == "unrelated temporary\n" + and list(Path(".").glob(".csv-far-values.csv.*.tmp")) == []) + +aggregation_sparse_wb = openpyxl.Workbook() +aggregation_sparse_ws = aggregation_sparse_wb.active +aggregation_sparse_ws.title = "Data" +aggregation_sparse_ws["A1"], aggregation_sparse_ws["A2"] = "Region", "EU" +aggregation_sparse_ws["XFD1048576"].number_format = "0.00" +aggregation_sparse_wb.save("aggregation-style-extreme.xlsx") +check("aggregation ignores an extreme style-only cell without iterating a million rows", + aggregation_regions("aggregation-style-extreme.xlsx") == ["EU"]) + +aggregation_far_wb = openpyxl.Workbook() +aggregation_far_ws = aggregation_far_wb.active +aggregation_far_ws.title = "Data" +aggregation_far_ws["A1"], aggregation_far_ws["A2"] = "Region", "near" +aggregation_far_ws["A1048576"] = "far" +aggregation_far_wb.save("aggregation-far-values.xlsx") +try: + aggregation_regions("aggregation-far-values.xlsx", max_row_span=1_000) + aggregation_span_rejected = False +except ValueError as error: + aggregation_span_rejected = "aggregation row span is too large" in str(error) +check("aggregation rejects far-apart keys before iter_rows", aggregation_span_rejected) + +real_worksheet_xml_profile = worksheet_xml_profile +bomb_profile_calls = [] +def tracking_worksheet_xml_profile(*args, **kwargs): + bomb_profile_calls.append(args) + return real_worksheet_xml_profile(*args, **kwargs) +worksheet_xml_profile = tracking_worksheet_xml_profile +try: + export_formula_values("compressed-xlsx-bomb.xlsx", "bomb-export.csv") + csv_bomb_rejected = False +except ValueError as error: + csv_bomb_rejected = "suspicious compression ratio: xl/styles.xml" in str(error) +finally: + worksheet_xml_profile = real_worksheet_xml_profile +check("CSV route rejects a package bomb before raw worksheet inventory", + csv_bomb_rejected and bomb_profile_calls == [], + (csv_bomb_rejected, bomb_profile_calls)) + +# SKILL.md contract: fullCalcOnLoad makes viewers recalculate even in manual calc mode. +calc_wb = openpyxl.Workbook() +calc_ws = calc_wb.active +calc_ws["A1"] = 1 +calc_ws["A2"] = 2 +calc_ws["A3"] = "=SUM(A1:A2)" +calc_wb.calculation.calcMode = "manual" +calc_wb.calculation.fullCalcOnLoad = False # simulate a source that does not recalc on load +calc_wb.save("stale-calc.xlsx") +stale_reopened = openpyxl.load_workbook("stale-calc.xlsx") +check("workbook without fullCalcOnLoad round-trips the stale flag (negative control)", + not bool(getattr(stale_reopened.calculation, "fullCalcOnLoad", False)), + stale_reopened.calculation) +stale_reopened.calculation.fullCalcOnLoad = True +stale_reopened.save("manual-calc.xlsx") +calc_reopened = openpyxl.load_workbook("manual-calc.xlsx") +check("fullCalcOnLoad survives save/reload", + bool(getattr(calc_reopened.calculation, "fullCalcOnLoad", False)), + calc_reopened.calculation) +check("manual calc mode survives save/reload", + getattr(calc_reopened.calculation, "calcMode", None) == "manual", + calc_reopened.calculation) +check("reloaded formula cell still holds the formula string", + calc_reopened.active["A3"].value == "=SUM(A1:A2)", calc_reopened.active["A3"].value) +calc_reopened.close() + +# ---- SKILL.md postcheck: formula inventory stays sparse at worksheet limits ---- +formula_bound_wb = openpyxl.Workbook() +formula_bound_ws = formula_bound_wb.active +formula_bound_ws["D2"] = "=1+1" +formula_bound_ws["XFD1048576"].number_format = "0.00" # styled but empty extreme cell +formula_bound_wb.save("formula-bound.xlsx") +formula_bound_reopened = openpyxl.load_workbook("formula-bound.xlsx", data_only=False) +formula_bound_ws = formula_bound_reopened.active + + +def expected_formula_inventory(sheet, expected): + actual = {} + for coordinate, expected_formula in expected.items(): + cell = sheet[coordinate] + actual_formula = formula_text(cell.value) if cell.data_type == "f" else None + if actual_formula != expected_formula: + raise ValueError( + f"{coordinate}: expected {expected_formula!r}, got {actual_formula!r}" + ) + actual[coordinate] = actual_formula + return actual + + +check("extreme styled cell inflates the rectangular worksheet bounds (negative control)", + formula_bound_ws.max_row == 1_048_576 and formula_bound_ws.max_column == 16_384, + (formula_bound_ws.max_row, formula_bound_ws.max_column)) +original_iter_rows = formula_bound_ws.iter_rows +formula_bound_ws.iter_rows = lambda *args, **kwargs: (_ for _ in ()).throw( + RuntimeError("unbounded iter_rows must not run") +) +try: + bounded_formulas = expected_formula_inventory(formula_bound_ws, {"D2": "=1+1"}) +finally: + formula_bound_ws.iter_rows = original_iter_rows +check("formula postcheck uses bounded public coordinate lookups", + bounded_formulas == {"D2": "=1+1"}, bounded_formulas) + +dimension_contract_wb = openpyxl.Workbook() +dimension_contract_ws = dimension_contract_wb.active +dimension_contract_ws.title = "Expected" +dimension_contract_ws["A1"] = "header" +dimension_contract_ws["C3"] = "tail" +expected_dimension_contract = {"Expected": "A1:C3"} +try: + require( + dimension_contract_ws.dimensions == expected_dimension_contract["Expected"], + "unexpected used range", + ) + exact_dimension_passed = True +except ValueError: + exact_dimension_passed = False +dimension_contract_ws["D4"] = "unintended" +try: + require( + dimension_contract_ws.dimensions == expected_dimension_contract["Expected"], + "unexpected used range", + ) + stale_dimension_rejected = False +except ValueError: + stale_dimension_rejected = True +check("postcheck accepts the exact task-declared worksheet range", exact_dimension_passed) +check("postcheck rejects unintended cells outside the task-declared range", + stale_dimension_rejected, dimension_contract_ws.dimensions) +formula_bound_reopened.close() + + +# ---- edit.md snippet: extension detection is prefix-independent ------------------ +X14_URI = b"http://schemas.microsoft.com/office/spreadsheetml/2009/9/main" +MC_URI = b"http://schemas.openxmlformats.org/markup-compatibility/2006" +EXTENSION_MARKERS = { + "extLst": b"extLst", + "x14 namespace": X14_URI, + "markup compatibility": MC_URI, +} + +def markers_in(data): + return {label for label, marker in EXTENSION_MARKERS.items() if marker in data} + +custom_prefix_sheet = ( + b'' + b'' +) +found = markers_in(custom_prefix_sheet) +check("namespace markers detect custom-prefix x14 extensions", "x14 namespace" in found, found) +check("namespace markers detect custom-prefix markup compatibility", "markup compatibility" in found, found) +check("local-name marker detects any-prefix extLst", "extLst" in found, found) +legacy_prefix_markers = {b"x14:", b"mc:AlternateContent"} +check("prefix markers are provably blind to custom prefixes (negative control)", + not any(marker in custom_prefix_sheet for marker in legacy_prefix_markers)) + +# ---- formatting.md guard: header-only sheets skip conditional formatting --------- +def last_populated_row(sheet, *, first_data_row=2, min_col=1, max_col=6): + populated_rows = ( + cell.row for cell in sheet._cells.values() + if first_data_row <= cell.row + and min_col <= cell.column <= max_col + and cell.value is not None + ) + return max(populated_rows, default=first_data_row - 1) + + +def add_demo_formatting(sheet): + last = last_populated_row(sheet) + if last < 2: + return 0 + sheet.conditional_formatting.add( + f"D2:D{last}", CellIsRule(operator="lessThan", formula=["0"]), + ) + sheet.conditional_formatting.add( + f"A2:F{last}", FormulaRule(formula=["$D2<0"]), + ) + sheet.conditional_formatting.add( + f"C2:C{last}", + ColorScaleRule(start_type="min", start_color="FFFFFF", + end_type="max", end_color="63BE7B"), + ) + sheet.conditional_formatting.add( + f"E2:E{last}", DataBarRule(start_type="min", end_type="max", color="638EC6"), + ) + return 4 + + +header_wb = openpyxl.Workbook() +header_ws = header_wb.active +header_ws.append(["A", "B", "C", "D", "E", "F"]) +header_ws["F100"].number_format = "0.00" # styled empty cell inflates max_row +try: + header_ws.conditional_formatting.add( + "D2:D1", CellIsRule(operator="lessThan", formula=["0"]), + ) + inverted_range_rejected = False +except (TypeError, ValueError): + inverted_range_rejected = True +check("unguarded header-only range is rejected (negative control)", inverted_range_rejected) +check("style-only ghost row inflates max_row (negative control)", header_ws.max_row == 100) +check("populated-row scan ignores a style-only ghost row", last_populated_row(header_ws) == 1) +check("header-only guard skips all four formatting rules", add_demo_formatting(header_ws) == 0) +header_wb.save("header-only-formatting.xlsx") +header_reopened = openpyxl.load_workbook("header-only-formatting.xlsx") +check("header-only workbook saves and reopens with no conditional formatting", + len(header_reopened.active.conditional_formatting) == 0) + +data_wb = openpyxl.Workbook() +data_ws = data_wb.active +data_ws.append(["A", "B", "C", "D", "E", "F"]) +data_ws.append([1, 2, 3, -1, 5, 6]) +data_ws["F100"].number_format = "0.00" +check("data rows receive all four formatting rules", add_demo_formatting(data_ws) == 4) +data_ranges = {str(item.sqref) for item in data_ws.conditional_formatting} +check("conditional formatting stops at the last populated row despite ghost styles", + data_ranges == {"D2", "A2:F2", "C2", "E2"}, data_ranges) +data_wb.save("data-formatting.xlsx") +data_reopened = openpyxl.load_workbook("data-formatting.xlsx") +check("all four formatting rules survive save/reopen", + len(data_reopened.active.conditional_formatting) == 4) + + +# ---- read.md: implausible is reset before streaming ------------------ +dim_wb = openpyxl.Workbook() +dim_ws = dim_wb.active +dim_ws.append(["h1", "h2"]) +dim_ws.append([1, 2]) +dim_ws.append([3, 4]) +dim_ws["C4"] = "=SUM(A2:B3)" +dim_wb.save("dimension.xlsx") +# Corrupt the sheet's dimension metadata the way non-Excel producers do. +import zipfile as dim_zip +with dim_zip.ZipFile("dimension.xlsx") as archive: + members = {name: archive.read(name) for name in archive.namelist()} +members["xl/worksheets/sheet1.xml"] = members["xl/worksheets/sheet1.xml"].replace( + b"", b"" +) +with dim_zip.ZipFile("dimension.xlsx", "w") as archive: + for name, data in members.items(): + archive.writestr(name, data) + +with validated_xlsx_source("dimension.xlsx") as dimension_source: + dim_value = openpyxl.load_workbook( + dimension_source, read_only=True, data_only=True + ) + try: + dim_ws_ro = dim_value.active + check("plausible but truncated dimension limits streaming (negative control)", + dim_ws_ro.calculate_dimension() == "A1:B2" and dim_ws_ro.max_row == 2, + (dim_ws_ro.calculate_dimension(), dim_ws_ro.max_row)) + original_dimension_iter_rows = dim_ws_ro.iter_rows + dim_ws_ro.iter_rows = lambda *args, **kwargs: (_ for _ in ()).throw( + RuntimeError("XML discovery must not expand a worksheet rectangle") + ) + dimension_source.seek(0) + with zipfile.ZipFile(dimension_source) as archive: + dimension_profile = worksheet_xml_profile( + archive, worksheet_part(archive, dim_ws_ro.title) + ) + dim_ws_ro.iter_rows = original_dimension_iter_rows + dim_ws_ro.reset_dimensions() + dimension_rows = list(bounded_sample_rows(dim_ws_ro, dimension_profile)) + finally: + dim_value.close() +check("sparse XML profile ignores truncated dimension and retains uncached formulas", + dimension_profile["extent"] == "A1:C4" + and dimension_profile["first_populated_row"] == 1 + and dimension_profile["formula_count"] == 1 + and dimension_profile["missing_formula_samples"][0][0] == "C4", + dimension_profile) +check("bounded sampling uses the XML extent after resetting producer metadata", + len(dimension_rows) == 4 + and dimension_rows[2][:2] == (3, 4), + dimension_rows) + +missing_dimension_wb = openpyxl.Workbook() +missing_dimension_ws = missing_dimension_wb.active +missing_dimension_ws.append(["Region", "Units"]) +missing_dimension_ws.append(["EU", 120]) +missing_dimension_ws.append(["US", 80]) +missing_dimension_wb.save("missing-dimension.xlsx") +with zipfile.ZipFile("missing-dimension.xlsx") as archive: + missing_dimension_members = { + name: archive.read(name) for name in archive.namelist() + } +missing_dimension_xml = missing_dimension_members["xl/worksheets/sheet1.xml"] +missing_dimension_tag = b'' +check("missing-dimension fixture starts with the expected producer metadata", + missing_dimension_tag in missing_dimension_xml) +missing_dimension_members["xl/worksheets/sheet1.xml"] = ( + missing_dimension_xml.replace(missing_dimension_tag, b"") +) +with zipfile.ZipFile("missing-dimension.xlsx", "w") as archive: + for name, data in missing_dimension_members.items(): + archive.writestr(name, data) + +with validated_xlsx_source("missing-dimension.xlsx") as missing_dimension_source: + missing_dimension_value_wb = openpyxl.load_workbook( + missing_dimension_source, read_only=True, data_only=True + ) + try: + missing_dimension_value_ws = missing_dimension_value_wb.active + forced_dimension_scans = [] + original_forced_dimension_scan = missing_dimension_value_ws._calculate_dimension + + def reject_forced_dimension_scan(): + forced_dimension_scans.append(True) + raise RuntimeError("read route must not force a worksheet dimension scan") + + missing_dimension_value_ws._calculate_dimension = reject_forced_dimension_scan + missing_dimension_source.seek(0) + with zipfile.ZipFile(missing_dimension_source) as archive: + missing_dimension_profile = worksheet_xml_profile( + archive, worksheet_part(archive, missing_dimension_value_ws.title) + ) + missing_dimension_declared = worksheet_declared_dimension( + missing_dimension_value_ws + ) + missing_dimension_value_ws.reset_dimensions() + missing_dimension_rows = list(bounded_sample_rows( + missing_dimension_value_ws, missing_dimension_profile + )) + missing_dimension_value_ws._calculate_dimension = original_forced_dimension_scan + finally: + missing_dimension_value_wb.close() +check("missing dimension remains unsized before raw XML routing", + missing_dimension_declared is None + and missing_dimension_profile["extent"] == "A1:B3", + (missing_dimension_declared, missing_dimension_profile)) +check("missing dimension route samples from XML bounds without a forced scan", + forced_dimension_scans == [] + and missing_dimension_rows == [ + ("Region", "Units"), ("EU", 120), ("US", 80) + ], + (forced_dimension_scans, missing_dimension_rows)) + +csv_formula_wb = openpyxl.load_workbook("dimension.xlsx", read_only=True, data_only=False) +csv_value_wb = openpyxl.load_workbook("dimension.xlsx", read_only=True, data_only=True) +csv_formula_ws, csv_value_ws = csv_formula_wb.active, csv_value_wb.active +check("both CSV source streams initially trust the truncated dimension (negative control)", + csv_formula_ws.calculate_dimension() == "A1:B2" + and csv_value_ws.calculate_dimension() == "A1:B2") +csv_formula_ws.reset_dimensions() +csv_value_ws.reset_dimensions() +csv_stream_rows = list(zip(csv_formula_ws.iter_rows(), csv_value_ws.iter_rows())) +check("CSV export resets both paired streams before iterating", + len(csv_stream_rows) == 4 + and len(csv_stream_rows[-1][0]) == 3 + and len(csv_stream_rows[-1][1]) == 3 + and csv_stream_rows[-1][0][2].data_type == "f" + and csv_stream_rows[-1][1][2].value is None, + [(len(formula_row), len(value_row)) for formula_row, value_row in csv_stream_rows]) +csv_formula_wb.close() +csv_value_wb.close() + +offset_wb = openpyxl.Workbook() +offset_ws = offset_wb.active +offset_ws["A7"], offset_ws["B7"] = "Region", "Units" +offset_ws["A8"], offset_ws["B8"] = "EU", 120 +offset_ws["A2"].number_format = "0.00" # styled but empty: not part of the data range +offset_ws["C8"] = "=SUM(B8)" # uncached formula: remains part of the range +offset_ws["XFD1048576"].number_format = "0.00" # extreme style-only physical cell +offset_wb.save("leading-blank-rows.xlsx") +with validated_xlsx_source("leading-blank-rows.xlsx") as offset_source: + offset_value_wb = openpyxl.load_workbook( + offset_source, read_only=True, data_only=True, + ) + try: + offset_value_ws = offset_value_wb.active + original_offset_iter_rows = offset_value_ws.iter_rows + offset_value_ws.iter_rows = lambda *args, **kwargs: (_ for _ in ()).throw( + RuntimeError("XML profile must not call iter_rows") + ) + offset_source.seek(0) + with zipfile.ZipFile(offset_source) as archive: + offset_profile = worksheet_xml_profile( + archive, worksheet_part(archive, offset_value_ws.title) + ) + offset_value_ws.iter_rows = original_offset_iter_rows + offset_value_ws.reset_dimensions() + offset_rows = bounded_sample_rows(offset_value_ws, offset_profile) + offset_header = next(offset_rows, None) + offset_sample = next(offset_rows, None) + finally: + offset_value_wb.close() +check("styled empty cells do not move the formula-preserving logical range", + (offset_profile["extent"], offset_profile["first_populated_row"]) + == ("A7:C8", 7), offset_profile) +check("header sampling skips six leading blank rows", + offset_header[:2] == ("Region", "Units") + and offset_sample[:2] == ("EU", 120), + (offset_header, offset_sample)) + +far_wb = openpyxl.Workbook() +far_ws = far_wb.active +far_ws["A1"] = "near" +far_ws["XFD1048576"] = "far" +far_wb.save("far-logical-cells.xlsx") +with validated_xlsx_source("far-logical-cells.xlsx") as far_source: + far_value_wb = openpyxl.load_workbook(far_source, read_only=True, data_only=True) + try: + far_source.seek(0) + with zipfile.ZipFile(far_source) as archive: + far_profile = worksheet_xml_profile( + archive, worksheet_part(archive, far_value_wb.active.title) + ) + far_iter_calls = [] + far_value_wb.active.iter_rows = ( + lambda *args, **kwargs: far_iter_calls.append((args, kwargs)) or iter(()) + ) + try: + bounded_sample_rows(far_value_wb.active, far_profile, max_cells=1_000) + far_budget_rejected = False + except ValueError as error: + far_budget_rejected = "sample rectangle is too large" in str(error) + finally: + far_value_wb.close() +check("real far-apart values hit the rectangle budget before iter_rows", + far_budget_rejected and far_iter_calls == [], (far_profile, far_iter_calls)) + +empty_dimension_wb = openpyxl.Workbook() +empty_dimension_wb.save("empty-dimension.xlsx") +with validated_xlsx_source("empty-dimension.xlsx") as empty_source: + with zipfile.ZipFile(empty_source) as archive: + empty_profile = worksheet_xml_profile( + archive, worksheet_part(archive, "Sheet") + ) +check("dimension scan handles an actually empty worksheet", + empty_profile["extent"] == "A1:A1" + and empty_profile["first_populated_row"] is None) + + +print("\n" + ("ALL XLSX FIXTURES PASSED" if not failures else f"{len(failures)} FAILURES: {failures}")) +sys.exit(0 if not failures else 1) diff --git a/test/hosted-plugins.test.mjs b/test/hosted-plugins.test.mjs index 4e58656..400ac3f 100644 --- a/test/hosted-plugins.test.mjs +++ b/test/hosted-plugins.test.mjs @@ -36,7 +36,7 @@ test('contributor can scaffold a hosted Skill plugin with one command', async (c assert.match(readme, /# Hello World/u); assert.match(license, /Apache License/u); assert.match(skill, /^---\nname: hello-world\n/mu); - assert.match(stdout, /plugins\/alice\/hello-world/u); + assert.match(stdout, /plugins[\\/]alice[\\/]hello-world/u); }); test('hosted Plugin is valid when its package and contribution docs are complete', async (context) => {