From 26c09763c05696cf207beb64021e587ae5103345 Mon Sep 17 00:00:00 2001 From: Hylouis233 Date: Sat, 15 Aug 2026 03:43:19 +0800 Subject: [PATCH 01/34] Add plugin document-skills Four original workbench Skills (docx, pdf, pptx, xlsx) for creating, reading, editing, and verifying Office and PDF documents with standard open-source Python tooling. - lives at plugins/Hylouis233/document-skills - plugin.json + README + Apache-2.0 LICENSE + four Skills with references - original guidance written for this plugin; no proprietary content reused - no network, no credentials, no MCP server; scripts referenced are the user's own installed Python libraries (python-docx, python-pptx, openpyxl, pypdf, reportlab, pymupdf), not bundled - npm run validate passes; documented patterns smoke-tested end to end --- plugins/Hylouis233/document-skills/LICENSE | 192 ++++++++++++++++++ plugins/Hylouis233/document-skills/README.md | 90 ++++++++ .../Hylouis233/document-skills/plugin.json | 22 ++ .../document-skills/skills/docx/SKILL.md | 73 +++++++ .../skills/docx/references/create.md | 80 ++++++++ .../skills/docx/references/edit.md | 68 +++++++ .../skills/docx/references/read.md | 38 ++++ .../skills/docx/references/review.md | 35 ++++ .../document-skills/skills/pdf/SKILL.md | 60 ++++++ .../skills/pdf/references/create.md | 71 +++++++ .../skills/pdf/references/extract.md | 37 ++++ .../skills/pdf/references/inspect.md | 25 +++ .../skills/pdf/references/transform.md | 53 +++++ .../document-skills/skills/pptx/SKILL.md | 64 ++++++ .../skills/pptx/references/analyze.md | 36 ++++ .../skills/pptx/references/create.md | 82 ++++++++ .../skills/pptx/references/edit.md | 46 +++++ .../document-skills/skills/xlsx/SKILL.md | 63 ++++++ .../skills/xlsx/references/create.md | 73 +++++++ .../skills/xlsx/references/csv.md | 49 +++++ .../skills/xlsx/references/edit.md | 46 +++++ .../skills/xlsx/references/read.md | 30 +++ 22 files changed, 1333 insertions(+) create mode 100644 plugins/Hylouis233/document-skills/LICENSE create mode 100644 plugins/Hylouis233/document-skills/README.md create mode 100644 plugins/Hylouis233/document-skills/plugin.json create mode 100644 plugins/Hylouis233/document-skills/skills/docx/SKILL.md create mode 100644 plugins/Hylouis233/document-skills/skills/docx/references/create.md create mode 100644 plugins/Hylouis233/document-skills/skills/docx/references/edit.md create mode 100644 plugins/Hylouis233/document-skills/skills/docx/references/read.md create mode 100644 plugins/Hylouis233/document-skills/skills/docx/references/review.md create mode 100644 plugins/Hylouis233/document-skills/skills/pdf/SKILL.md create mode 100644 plugins/Hylouis233/document-skills/skills/pdf/references/create.md create mode 100644 plugins/Hylouis233/document-skills/skills/pdf/references/extract.md create mode 100644 plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md create mode 100644 plugins/Hylouis233/document-skills/skills/pdf/references/transform.md create mode 100644 plugins/Hylouis233/document-skills/skills/pptx/SKILL.md create mode 100644 plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md create mode 100644 plugins/Hylouis233/document-skills/skills/pptx/references/create.md create mode 100644 plugins/Hylouis233/document-skills/skills/pptx/references/edit.md create mode 100644 plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md create mode 100644 plugins/Hylouis233/document-skills/skills/xlsx/references/create.md create mode 100644 plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md create mode 100644 plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md create mode 100644 plugins/Hylouis233/document-skills/skills/xlsx/references/read.md diff --git a/plugins/Hylouis233/document-skills/LICENSE b/plugins/Hylouis233/document-skills/LICENSE new file mode 100644 index 0000000..125be1b --- /dev/null +++ b/plugins/Hylouis233/document-skills/LICENSE @@ -0,0 +1,192 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + Copyright 2026 MCode Plugins contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/plugins/Hylouis233/document-skills/README.md b/plugins/Hylouis233/document-skills/README.md new file mode 100644 index 0000000..bf4de01 --- /dev/null +++ b/plugins/Hylouis233/document-skills/README.md @@ -0,0 +1,90 @@ +# Document Skills + +## The problem + +Users keep asking coding agents for document deliverables — a Word report, a PDF handout, a slide +deck, a data workbook — and get unreliable results: ZIP files rewritten by hand and corrupted, +spreadsheets where formulas were typed as text, slide decks that no viewer opens, PDFs that are a +stack of screenshots instead of text. Office and PDF documents are container formats with strict +internal contracts, and an agent without format knowledge breaks them silently. + +This Plugin installs four workbench Skills — `docx`, `pdf`, `pptx`, `xlsx` — that route each +task to the right standard tool (python-docx, python-pptx, openpyxl, pypdf, ReportLab, PyMuPDF), +enforce the container contracts (valid ZIP, correct content types, calculated dimensions), and +verify output before it is handed back. + +## Try it + +```text +Use the document-skills plugin: open sales-2024.xlsx, add a sheet "Summary" with per-region +totals computed by formula, a bar chart of the top 5 products, and currency formatting. +``` + +Expected result: the agent loads the workbook with openpyxl, inspects sheet names, headers, and +dtypes, writes `SUMIF`/`COUNTIF` formulas (not pasted values), adds a native `BarChart` anchored +to the new sheet, applies number formats, and reports the cell ranges it changed. + +```text +Use the document-skills plugin: create a PDF one-pager "Q3 launch checklist" from this outline +with a title block and a two-column checklist that fits exactly one A4 page. +``` + +Expected result: the agent generates the page with ReportLab flowables on an A4 canvas, measures +the checklist blocks, and confirms with pypdf that the output is exactly 1 page with extractable +text — not a screenshot. + +## What the Skills do + +Shared spine (all four Skills follow it): + +1. Classify the request as **create**, **read**, **edit**, or **review**. +2. Check tool availability first and report missing dependencies instead of improvising. +3. Follow format-specific rules (packages below). +4. Run the post-generation verification checklist; fix and re-verify until it passes. +5. Report the output path, the page/sheet/slide inventory, and any remaining caveats. + +Per format: + +- **docx** — create with python-docx from a heading outline; edit existing files by direct + `word/document.xml` surgery (python-docx cannot open-and-save arbitrary files losslessly); + extract text with python-docx or `pandoc -t markdown`; postcheck with python-docx re-open and + `soffice --headless --convert-to` PDF smoke test when LibreOffice is present. +- **xlsx** — openpyxl for reading, editing, styling, and native charts; formulas as formulas, + never as pasted results; `data_only=True` only for reading cached values; date/number formats + applied explicitly; recalculation contract documented (openpyxl writes formulas, the viewer + calculates). +- **pptx** — python-pptx to build decks (7 common slide patterns: title, agenda, bullet, two + image+text, table, chart, quote/closing); edit only named, existing shapes, never blind + rewriting of the whole XML; text measured against shape width with font-size reduction rules; + presentation-level verification via `python-pptx` re-open plus a rendered PDF smoke test when + LibreOffice is available. +- **pdf** — creation prefers ReportLab (structured, accessible text) over HTML-to-print paths; + extraction and splitting/merging with pypdf; analysis and rasterization with PyMuPDF; explicit + one-tool-per-job table so the agent stops mixing libraries mid-task. + +## Verification-first output + +Every Skill ends with the same rule: do not hand back a file you have not re-opened. The checklists +are specific (re-open the archive, confirm the sheet count and formula presence, confirm the slide +count, confirm page count and text extraction) and the Skills require reporting what was verified +versus what was assumed. + +## Requirements + +- Python 3.9+ with `python-docx`, `python-pptx`, `openpyxl`, `pypdf`, `reportlab`, + `pymupdf` (`pip install python-docx python-pptx openpyxl pypdf reportlab pymupdf`). +- Optional: LibreOffice (`soffice`) for PDF smoke tests of DOCX/PPTX output; `pandoc` for + markdown extraction from DOCX. +- Works on Windows, macOS, and Linux. All commands are given in cross-platform form; the Skills + say how to resolve the skill directory path on each platform. + +## Data and network + +- No network access. All processing is local file conversion and generation. +- No credentials required. +- The Skills only read and write document files the user points at; temporary files go to the + system temp directory and are cleaned up. + +## License + +Apache-2.0. See [LICENSE](LICENSE). diff --git a/plugins/Hylouis233/document-skills/plugin.json b/plugins/Hylouis233/document-skills/plugin.json new file mode 100644 index 0000000..7fe7217 --- /dev/null +++ b/plugins/Hylouis233/document-skills/plugin.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json", + "name": "document-skills", + "version": "0.1.0", + "description": "Workbench Skills for creating, reading, editing, and checking DOCX, PDF, PPTX, and XLSX documents with standard open-source tooling.", + "author": { + "name": "Hylouis233", + "url": "https://github.com/Hylouis233" + }, + "license": "Apache-2.0", + "keywords": [ + "minimax-code", + "plugin", + "documents", + "docx", + "pdf", + "pptx", + "xlsx", + "office", + "ooxml" + ] +} diff --git a/plugins/Hylouis233/document-skills/skills/docx/SKILL.md b/plugins/Hylouis233/document-skills/skills/docx/SKILL.md new file mode 100644 index 0000000..cdfaf2c --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/docx/SKILL.md @@ -0,0 +1,73 @@ +--- +name: docx +description: Create, read, edit, or review Microsoft Word .docx documents. Use this Skill whenever the task involves a .docx file - generating reports, letters, or structured documents from scratch; extracting or reorganizing content; applying consistent formatting (headings, tables, styles, headers/footers, page numbers); editing an existing document while preserving its layout; or checking that a produced document actually opens and renders. +--- + +# DOCX workbench + +A `.docx` file is a ZIP archive of XML parts around one `word/document.xml`. Treat the format +with respect: silent corruption is the default failure mode when agents edit the XML blind. + +## Step 0 — Resolve paths and check the toolchain + +Resolve this Skill's directory once (`SKILL_DIR` = the folder containing this SKILL.md), then +check what is installed before promising anything: + +```bash +python -c "import docx; print(docx.__version__ if hasattr(docx,'__version__') else 'ok')" +``` + +- `python-docx` present -> full create/read/edit support. +- Missing -> report it (`pip install python-docx`) and stop; do not hand-write OOXML. +- `pandoc` present -> prefer it for text/markdown extraction (fast, faithful). +- `soffice` present -> enables the PDF smoke test in postcheck. + +## Step 1 — Classify the task + +| Request looks like | Route | Go to | +|---|---|---| +| "Write / create / generate a document" (no input file) | Create | [references/create.md](references/create.md) | +| "Edit / change / add to this document" (input file given) | Edit | [references/edit.md](references/edit.md) | +| "Read / extract / summarize / convert" | Read | [references/read.md](references/read.md) | +| "Check / fix / why is it broken" | Review | [references/review.md](references/review.md) | + +Two routes can chain (read -> edit, create -> review). Never skip classification; it decides +between python-docx generation and raw XML surgery, which have opposite safety profiles. + +## Step 2 — Shared rules that always apply + +1. **Never edit `word/document.xml` with find/replace on rendered text.** Run-level splits mean + the visible word "contract" may be spread across three `` nodes. Use the edit route's + paragraph-index addressing instead. +2. **Prefer modifying an existing document with python-docx when the change is structural** + (add paragraphs, tables, images, styles); drop to raw XML only for things python-docx cannot + express (field codes, exotic properties, tracked-change surgery). +3. **Always write to a new output path first.** Only overwrite the source when the user explicitly + asked for in-place modification, and then only after the postcheck passes. +4. **Keep styles semantic.** Use `Heading 1..4` styles instead of "bold 16pt text" so + navigation, TOC fields, and accessibility keep working. +5. **Units**: lengths in the XML are twentieths of a point (twips); python-docx accepts + `.inches`/`.cm`/`.Pt` helpers - use the helpers. +6. **Non-ASCII**: write files as UTF-8 without BOM; declare encoding when opening text side + files. + +## Step 3 — Execute the route + +Follow the referenced file for concrete code patterns, then continue to postcheck. Do not +improvise container-level operations (re-zipping, renaming parts) outside the patterns in +[references/edit.md](references/edit.md). + +## Step 4 — Postcheck (mandatory before handing back the file) + +Run every applicable item and report results explicitly: + +1. Re-open the output with python-docx (`Document(path)`); confirm it parses and count + paragraphs/tables. +2. If `soffice` exists: `soffice --headless --convert-to pdf --outdir ` + must exit 0; the PDF is a smoke test, keep or delete it per the user's ask. +3. Verify requested features landed: search the re-opened document for the expected heading + texts, table row counts, or inserted image count. +4. Report: output path, paragraph/table counts, what was verified, what could not be verified + locally (e.g. exact pagination in MS Word), and any font substitutions expected. + +If a check fails, fix and re-run; do not deliver with a known-broken archive. diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/create.md b/plugins/Hylouis233/document-skills/skills/docx/references/create.md new file mode 100644 index 0000000..ad3f916 --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/docx/references/create.md @@ -0,0 +1,80 @@ +# Create a new DOCX + +Use python-docx from an outline. Never assemble the ZIP by hand. + +## Pattern + +```python +from docx import Document +from docx.shared import Pt, Cm, RGBColor +from docx.enum.text import WD_ALIGN_PARAGRAPH + +doc = Document() + +# 1. Page geometry (A4 default is US Letter in python-docx - set explicitly) +for section in doc.sections: + section.page_width, section.page_height = Cm(21.0), Cm(29.7) + section.left_margin = section.right_margin = Cm(2.5) + section.top_margin = section.bottom_margin = Cm(2.5) + +# 2. Semantic outline: title + headings drive navigation and TOC +doc.add_heading("Quarterly Reliability Report", level=0) # uses Title style +doc.add_heading("Summary", level=1) +doc.add_heading("Incident review", level=2) + +# 3. Body paragraphs; set spacing once via style, not per paragraph +style = doc.styles["Normal"] +style.font.name = "Calibri" +style.font.size = Pt(11) +style.paragraph_format.space_after = Pt(6) +style.paragraph_format.line_spacing = 1.15 + +p = doc.add_paragraph("Text with ") +p.add_run("bold segment").bold = True +p.add_run(" and regular text.") + +# 4. Tables with header row emphasis +table = doc.add_table(rows=1, cols=3) +table.style = "Light Grid Accent 1" +hdr = table.rows[0].cells +for cell, text in zip(hdr, ["Region", "Incidents", "MTTR (h)"]): + cell.text = text + for par in cell.paragraphs: + for run in par.runs: + run.font.bold = True +row = table.add_row().cells +row[0].text, row[1].text, row[2].text = "eu-1", "3", "0.8" + +# 5. Optional footer with page number field +from docx.oxml.ns import qn +from docx.oxml import OxmlElement +footer = doc.sections[0].footer +fp = footer.paragraphs[0] +fp.alignment = WD_ALIGN_PARAGRAPH.CENTER +fld = OxmlElement("w:fldSimple") +fld.set(qn("w:instr"), "PAGE") +fp._p.append(fld) + +doc.save("report.docx") +``` + +## Rules + +- Build the heading outline **before** writing prose; a wrong outline is the most expensive + late fix. +- One table style per document. Cell-level font overrides are for exceptions only. +- Bullet/numbered lists: `add_paragraph(text, style="List Bullet")` / + `"List Number"`. Restart numbering per list by using separate list style instances. +- Images: `doc.add_picture(path, width=Cm(14))` - always set width so oversized images do not + overflow the text column. Keep aspect ratio by setting only one dimension. +- A real TOC is a **field**, it renders after the user opens the file and updates fields + (Word prompts, or Ctrl+A then F9). If a TOC is requested, insert the field and tell the user + it needs one field refresh; optionally pre-populate static entries from your outline. +- Do not attempt pixel-exact page design (complex multi-column covers, inline floating + wrap). If the user wants that fidelity, produce the PDF route instead. + +## Quality bar + +- Heading levels never skip (no Heading 1 straight to Heading 3). +- No empty sections: every heading is followed by content or removed. +- Numbers in tables right-aligned; units declared in the header. diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/edit.md b/plugins/Hylouis233/document-skills/skills/docx/references/edit.md new file mode 100644 index 0000000..a7c9b93 --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/docx/references/edit.md @@ -0,0 +1,68 @@ +# Edit an existing DOCX + +Editing has two tiers. Pick the lowest tier that can express the change. + +## Tier 1 - python-docx structural edits (preferred) + +python-docx opens the real package and rewrites it safely. Use it for: adding/removing +paragraphs, tables, images; changing styles; editing text of a specific run; find-replace over +paragraph text. + +```python +from docx import Document + +doc = Document("input.docx") + +# Address paragraphs by index over doc.paragraphs (body level). Tables' cells hold their own +# paragraphs: table.rows[i].cells[j].paragraphs +for i, par in enumerate(doc.paragraphs): + if "TBD" in par.text: + # Run-level replace keeps formatting of untouched runs + for run in par.runs: + if "TBD" in run.text: + run.text = run.text.replace("TBD", "To be decided") + +# Append content at a specific position: manipulate the XML tree +target = doc.paragraphs[7]._p +new_par = doc.add_paragraph("Inserted after the target.") +target.addnext(new_par._p) + +doc.save("input.edited.docx") +``` + +## Tier 2 - raw OOXML surgery (only when Tier 1 cannot express it) + +For field codes, sectPr surgery, tracked changes, or parts python-docx does not model. +Rules that keep the archive valid: + +1. Operate on a **copy** of the file. +2. Unzip preserving structure: `python -m zipfile -e input.docx work/` or use `zipfile` in + Python with `ZIP_DEFLATED` on repack. +3. Parse XML with `lxml`/`xml.etree` - never string replace. Text lives in `w:t` inside runs + (`w:r`) inside paragraphs (`w:p`); a logical sentence can span several runs. +4. Repack with `[Content_Types].xml` first and stored/deflated entries only: + +```python +import zipfile, pathlib +src = pathlib.Path("work") +with zipfile.ZipFile("output.docx", "w", zipfile.ZIP_DEFLATED) as z: + for name in ["[Content_Types].xml"] + [p.name for p in src.rglob("*") if p.is_file()]: + pass # write real paths in archive order; see note +``` + + Concretely, walk the tree and `z.write(p, p.relative_to(src).as_posix())`, writing + `[Content_Types].xml` first. Do not add directories as entries, do not preserve absolute + timestamps incorrectly (ZIP wants local time or 0). +5. If you touched part names or added parts, update `[Content_Types].xml` and + `word/_rels/document.xml.rels` consistently - a mismatch here is the classic silent corrupt. + +## Never do + +- Blind find/replace on the raw XML string of `word/document.xml`. +- Deleting parts that look unused (styles, theme, settings) - viewers may require them. +- Editing a document that is open in Word (the save will collide with the lock file). + +## Delivery + +Save to `-edited.docx` unless the user explicitly asked to overwrite. Run the +postcheck from SKILL.md step 4 on the output. diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/read.md b/plugins/Hylouis233/document-skills/skills/docx/references/read.md new file mode 100644 index 0000000..a41ad89 --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/docx/references/read.md @@ -0,0 +1,38 @@ +# Read / extract a DOCX + +Pick by fidelity needed. + +## Fast text extraction (pandoc, if installed) + +```bash +pandoc -t markdown input.docx -o extracted.md +``` + +Best structural fidelity for prose (headings become markdown headings, tables become pipe +tables). Prefer this when the goal is content, not coordinates. + +## Structured access (python-docx) + +```python +from docx import Document +doc = Document("input.docx") +for par in doc.paragraphs: + print(par.style.name, "|", par.text) +for t, table in enumerate(doc.tables): + for r, row in enumerate(table.rows): + print(t, r, [c.text for c in row.cells]) +``` + +Notes: + +- `doc.paragraphs` is body-level only. Text inside text boxes, headers, footers, footnotes is + reached via their own collections (`section.header/.footer`) or raw XML. +- `doc.tables` is top-level only; nested tables require walking cells. +- For revision/comment metadata, inspect the XML parts directly: `word/comments.xml`, + `w:ins`/`w:del` elements in `word/document.xml`. + +## Reporting contract + +When summarizing a document for the user, lead with: heading outline, paragraph count, table +count with dimensions, and any parts that could not be read. Do not silently skip unreadable +parts. diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/review.md b/plugins/Hylouis233/document-skills/skills/docx/references/review.md new file mode 100644 index 0000000..369dce5 --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/docx/references/review.md @@ -0,0 +1,35 @@ +# Review / repair a DOCX + +## Symptom-driven triage + +| Symptom | Likely cause | Fix route +|---|---|---| +File will not open at all | broken ZIP (truncated, wrong repack) | inspect with `zipfile.ZipFile(path).testzip()`; if entries are damaged, recover from the user's original or prior version | +Opens with "unreadable content" repair prompt | content-types / rels mismatch, invalid XML | Tier 2 surgery: validate XML parses, check `[Content_Types].xml` covers every part extension | +Text present but styles lost | document rebuilt from scratch instead of edited | redo as edit on the original package | +Images missing | media parts not repacked or rels broken | verify `word/media/*` exist and `document.xml.rels` references them | +Fonts render differently on another machine | non-embedded fonts | expected; report which fonts are referenced (`w:rFonts` values) | + +## Programmatic health check + +```python +import zipfile +from lxml import etree + +path = "input.docx" +with zipfile.ZipFile(path) as z: + bad = z.testzip() + assert bad is None, f"corrupt entry: {bad}" + names = z.namelist() + assert "[Content_Types].xml" in names and "word/document.xml" in names + for part in names: + if part.endswith(('.xml', '.rels')): + etree.fromstring(z.read(part)) # raises on malformed XML +``` + +Then the SKILL.md postcheck (python-docx re-open, optional soffice PDF smoke test). + +## Report format + +State what is broken, the minimal repair applied, and what could not be verified without the +target viewer (exact pagination, field updates, embedded font rendering). diff --git a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md new file mode 100644 index 0000000..73dac79 --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md @@ -0,0 +1,60 @@ +--- +name: pdf +description: Create, extract, transform, or analyze PDF documents. Use this Skill whenever a .pdf file is the input or output - generating reports, one-pagers, posters or structured documents as real selectable-text PDFs; extracting text, tables or images from existing PDFs; merging, splitting, rotating, or watermarking pages; filling forms; checking that a produced PDF has the right page count and extractable text. +--- + +# PDF workbench + +PDF tasks fail when the agent mixes incompatible libraries or generates pages as screenshots. +Route by job, use one tool per job, verify the artifact. + +## Step 0 - Check the toolchain + +```bash +python -c "import reportlab, pypdf, fitz; print('reportlab', reportlab.Version, '| pypdf ok | pymupdf ok')" +``` + +- `reportlab` - creation (preferred: real text, real structure). +- `pypdf` - page-level transforms: split, merge, rotate, encrypt, form fields. +- `pymupdf` (`fitz`) - extraction (text with coordinates, tables, images), rasterization, + page-level inspection. + +Missing any -> say so and degrade to the ones present; do not substitute screenshot pipelines. + +## Step 1 - Pick exactly one tool for the job + +| Task | Tool | Reference | +|---|---|---| +| Build a new structured PDF (report, one-pager, letter, checklist) | ReportLab | [references/create.md](references/create.md) | +| Extract text / tables / images | PyMuPDF | [references/extract.md](references/extract.md) | +| Merge, split, rotate, watermark, encrypt, forms | pypdf | [references/transform.md](references/transform.md) | +| Inspect: page count, sizes, fonts, links, blank-page detection | PyMuPDF | [references/inspect.md](references/inspect.md) | + +Chaining tools on one artifact is fine (create -> inspect). Using two creation libraries on one +file is not. + +## Step 2 - Shared rules + +1. **Text-first**: a generated PDF must contain extractable text. Image-of-text is acceptable + only when the user asked for a rasterized look or provides only images. +2. **Page geometry is explicit**: A4 = 595.27 x 841.89 pt, US Letter = 612 x 792 pt. Declare + the target size and margins up front; re-check fit after generation. +3. **Fonts**: standard 14 fonts always work; embedding a custom TTF is allowed only with its + license permitting distribution. CJK requires an embedded font - there is no built-in CJK + face; if unavailable, report the limitation instead of emitting tofu. +4. **Overflow is a defect**: content that spills past the last page or the margin must be + detected in postcheck and fixed (shrink, paginate, or cut), never shipped. +5. Write output to a new path; keep inputs untouched unless in-place was requested. + +## Step 3 - Postcheck (mandatory) + +```python +import pypdf +r = pypdf.PdfReader("output.pdf") +page_count = len(r.pages) +first_text = (r.pages[0].extract_text() or "").strip() +``` + +Confirm: page count matches the request; the key title/heading text extracts non-empty; page +size is the declared size (`r.pages[0].mediabox`). Report all three. For pixel-sensitive work, +add a PyMuPDF render of page 1 at 100 dpi and check it is non-blank (mean pixel value). diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/create.md b/plugins/Hylouis233/document-skills/skills/pdf/references/create.md new file mode 100644 index 0000000..2e96f98 --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/create.md @@ -0,0 +1,71 @@ +# Create a PDF with ReportLab + +Use flowables (the Platypus layer) so pagination, spacing, and style separation work for you. + +## Skeleton + +```python +from reportlab.lib.pagesizes import A4 +from reportlab.lib.units import mm +from reportlab.lib import colors +from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle +from reportlab.platypus import (SimpleDocTemplate, Paragraph, Spacer, Table, TableStyle, + ListFlowable, ListItem, KeepTogether) + +styles = getSampleStyleSheet() +h1 = ParagraphStyle("H1x", parent=styles["Heading1"], fontSize=16, spaceAfter=6) +body = ParagraphStyle("Bodyx", parent=styles["BodyText"], fontSize=10, leading=14) + +doc = SimpleDocTemplate( + "output.pdf", + pagesize=A4, + leftMargin=20*mm, rightMargin=20*mm, topMargin=18*mm, bottomMargin=18*mm, + title="Q3 launch checklist", author="document-skills", +) + +story = [ + Paragraph("Q3 Launch Checklist", h1), + Spacer(1, 4*mm), + Paragraph("Owner: platform team. Scope: EU region rollout.", body), + Spacer(1, 6*mm), +] + +data = [["#", "Item", "Owner", "Status"], + ["1", "Freeze scope", "PM", "done"], + ["2", "Load test at 2x", "SRE", "planned"]] +tbl = Table(data, colWidths=[10*mm, 70*mm, 30*mm, 25*mm]) +tbl.setStyle(TableStyle([ + ("BACKGROUND", (0,0), (-1,0), colors.HexColor("#1a3c6e")), + ("TEXTCOLOR", (0,0), (-1,0), colors.white), + ("FONTNAME", (0,0), (-1,0), "Helvetica-Bold"), + ("ROWBACKGROUNDS", (0,1), (-1,-1), [colors.white, colors.HexColor("#eef2f7")]), + ("GRID", (0,0), (-1,-1), 0.4, colors.HexColor("#9aa7b4")), + ("VALIGN", (0,0), (-1,-1), "MIDDLE"), + ("TOPPADDING", (0,0), (-1,-1), 4), + ("BOTTOMPADDING", (0,0), (-1,-1), 4), +])) +story.append(tbl) +story.append(Spacer(1, 6*mm)) +story.append(ListFlowable( + [ListItem(Paragraph("Dry-run in staging with production-shaped data", body))], + bulletType="bullet", +)) +doc.build(story) +``` + +## Rules + +- **Exactly-one-page constraint**: after `build`, run the postcheck. Over budget -> reduce + `leading`, tighten `spaceBefore/After`, cut rows - in that order of preference. Under budget + is usually fine; add a spacer or scale the title block. +- **Keep blocks atomic**: wrap heading + first paragraph (or a table and its caption) in + `KeepTogether` so pagination does not orphan them. +- **Two columns** only via frames/PageTemplates; never by positioning two columns on one + canvas manually unless you are in full-canvas mode with measured coordinates. +- **Full-canvas mode** (`canvas` API) is for fixed-layout artifacts: badges, certificates, + posters with absolute geometry. Measure text with `pdfmetrics.stringWidth` before placing; + never guess whether a string fits. +- **Links**: internal links via bookmarks/anchors, external via `linkURL`; declare the URL + visibly next to the link text when the PDF may be printed. +- **Images**: `Image(path, width=..., height=...)` with both dimensions set from the real + aspect ratio (PIL or PyMuPDF can measure). diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md b/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md new file mode 100644 index 0000000..9061b56 --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md @@ -0,0 +1,37 @@ +# Extract from a PDF (PyMuPDF) + +```python +import fitz # pymupdf + +doc = fitz.open("input.pdf") +print("pages:", doc.page_count, "metadata:", doc.metadata) + +# Plain text, page by page +for page in doc: + text = page.get_text("text") # reading order text + +# With coordinates (decide columns/reading order yourself) +for block in page.get_text("dict")["blocks"]: + for line in block.get("lines", []): + for span in line["spans"]: + print(round(span["bbox"][0]), round(span["bbox"][1]), span["text"]) + +# Images +for i, info in enumerate(page.get_images(full=True)): + pix = fitz.Pixmap(doc, info[0]) + pix.save(f"img-p{page.number}-{i}.png") + +# Rasterize a page (for visual checks or OCR preprocessing) +pix = page.get_pixmap(dpi=150) +pix.save(f"page-{page.number}.png") +``` + +## Rules + +- Scanned pages return empty `get_text`. Detect: first page text empty + images present -> + report "scanned, needs OCR" instead of claiming there is no text. +- "Sort by position" before emitting tables: spans come in internal order, not visual order; + sort by `(round(bbox[1]), bbox[0])` for top-to-bottom, left-to-right reading. +- Two-column layouts: cluster spans by x-gap before joining lines, or text interleaves columns. +- Never send the whole raw text to the user when asked for a summary; extract, then summarize + with page references (`page 3` etc. derived from `page.number`). diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md b/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md new file mode 100644 index 0000000..e44ae35 --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md @@ -0,0 +1,25 @@ +# Inspect a PDF (PyMuPDF) + +```python +import fitz +doc = fitz.open("input.pdf") +print("pages:", doc.page_count) +print("encrypted:", doc.needs_pass, "| tagged:", doc.is_pdf, "pdf:", doc.is_pdf) +for page in doc: + print(page.number, page.rect, "text_len:", len(page.get_text()), + "images:", len(page.get_images()), "links:", len(page.get_links())) +``` + +## Checks worth automating + +- **Blank page detection**: `len(page.get_text()) == 0 and not page.get_images()` -> flag; a + blank page after generation usually means an overflowing flowable created an empty page. +- **Font inventory**: `page.get_fonts()` lists embedded names - needed when the user reports + "looks different on machine X". +- **Page size consistency**: mixed `page.rect` sizes in one file break duplex printing; report + it rather than silently normalizing. +- **Damage**: `fitz.open` on a corrupt file raises or yields garbage - pair with + `pypdf.PdfReader` cross-check when provenance is unknown. + +Report findings as a table (page, size, text chars, images, links) - it is what every +downstream decision hangs off. diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md b/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md new file mode 100644 index 0000000..aedac52 --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md @@ -0,0 +1,53 @@ +# Transform a PDF (pypdf) + +One tool for page-level structure changes: + +```python +from pypdf import PdfReader, PdfWriter + +reader = PdfReader("input.pdf") +writer = PdfWriter() + +# Split: keep only pages 2-4 (0-based) +for i in range(1, 4): + writer.add_page(reader.pages[i]) + +# Merge another file at the end +other = PdfReader("appendix.pdf") +for page in other.pages: + writer.add_page(page) + +# Rotate a page 90 degrees +writer.pages[0].rotate(90) + +# Metadata +writer.add_metadata({"/Title": "Merged report", "/Producer": "document-skills"}) + +with open("output.pdf", "wb") as f: + writer.write(f) +``` + +Watermark / stamp by merging a stamp page onto each page: + +```python +stamp = PdfReader("watermark.pdf").pages[0] +writer = PdfWriter() +for page in PdfReader("input.pdf").pages: + page.merge_page(stamp) # stamp content on top; use merge_transformed_page to place + writer.add_page(page) +``` + +Encryption and forms: + +- `writer.encrypt("pass", algorithm="AES-256")` to protect; `PdfReader(..., password="pass")` + to open. Report that you set a password - the user must record it. +- AcroForm fields: `reader.get_fields()` to enumerate; `writer.update_page_form_field_values( + page, {"fieldname": "value"})` to fill. Flatten only on explicit request; it stops later + editing. + +## Rules + +- Always write a new file; transformation in place risks losing the original on a bad write. +- After writing, re-open with `PdfReader("output.pdf")` and verify page count and page sizes. +- `rotate` is cumulative on already-rotated pages - read `/Rotate` first if the source was + scanned sideways. diff --git a/plugins/Hylouis233/document-skills/skills/pptx/SKILL.md b/plugins/Hylouis233/document-skills/skills/pptx/SKILL.md new file mode 100644 index 0000000..b1fce22 --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/pptx/SKILL.md @@ -0,0 +1,64 @@ +--- +name: pptx +description: Create, read, edit, or analyze PowerPoint .pptx presentations. Use this Skill whenever the task involves slides or a deck - building a presentation from an outline or content file; editing text, tables, or charts in an existing deck; extracting slide content for review or repurposing; checking that a produced deck opens, has the right slide count, and nothing overflows its slide. +--- + +# PPTX workbench + +A `.pptx` is a ZIP of XML parts (one per slide) plus masters, layouts, and media. python-pptx +models most of it safely; everything else is surgical. + +## Step 0 - Check the toolchain + +```bash +python -c "import pptx; print('python-pptx ok')" +``` + +`soffice` present -> the rendered-PDF smoke test becomes available; use it for anything visual. + +## Step 1 - Classify the task + +| Request | Route | +|---|---| +| Build a deck from an outline / notes / markdown | [references/create.md](references/create.md) | +| Change text, tables, or charts in an existing deck | [references/edit.md](references/edit.md) | +| Extract or summarize deck content | [references/analyze.md](references/analyze.md) | +| Deck renders wrong / opens broken | [references/analyze.md](references/analyze.md) triage section | + +## Step 2 - Shared rules + +1. **Templates first**: if the user provides a `.pptx` template or brand deck, build on it + (`Presentation("template.pptx")`), reuse its masters/layouts, and never restyle globally. +2. **Layouts carry design**: pick the closest built-in layout for each slide's purpose instead + of hand-placing empty text boxes. Hand-placement is for exceptions, measured. +3. **Text must fit its box**: estimate width (chars x ~0.5 x font size for sans-serif at a + first approximation) and step font size down (never below 12pt body) or cut words. Overflow + text is a defect the postcheck must catch. +4. **Placeholders keep semantics**: write into placeholder shapes (`.placeholders`) when + available so title/body roles survive round-trips. +5. **Images**: set both width and height from the real aspect ratio; never stretch. +6. Charts: prefer a real chart part (`chart_data` + `add_chart`) over a picture of a chart - + only a real chart stays editable and data-accurate. +7. Output to a new path (`-edited` suffix) unless in-place was explicitly requested. + +## Step 3 - Postcheck (mandatory) + +```python +from pptx import Presentation +prs = Presentation("output.pptx") +print("slides:", len(prs.slides)) +for i, slide in enumerate(prs.slides): + texts = [sh.text_frame.text for sh in slide.shapes if sh.has_text_frame] + print(i, len(slide.shapes), texts[:3]) +``` + +Confirm: slide count matches the outline; every slide has its intended title text; no text +frame is empty that should not be. If `soffice` exists, render to PDF and check the page count +equals the slide count: + +```bash +soffice --headless --convert-to pdf output.pptx --outdir +``` + +Report: output path, slide inventory (index, layout name, title), verification done, and any +layout risks (long CJK strings, tight two-column slides) that only a human eye can confirm. diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md new file mode 100644 index 0000000..c7be922 --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md @@ -0,0 +1,36 @@ +# Analyze / triage a deck + +## Content inventory + +```python +from pptx import Presentation +prs = Presentation("input.pptx") +print("slide size:", prs.slide_width, prs.slide_height) +for i, slide in enumerate(prs.slides): + layout = slide.slide_layout.name + title = slide.shapes.title.text_frame.text if slide.shapes.title is not None else "" + notes = slide.notes_slide.notes_text_frame.text if slide.has_notes_slide else "" + n_tables = sum(1 for sh in slide.shapes if sh.has_table) + n_charts = sum(1 for sh in slide.shapes if sh.has_chart) + n_pics = sum(1 for sh in slide.shapes if sh.shape_type == 13) + print(i, layout, repr(title[:40]), "tables:", n_tables, "charts:", n_charts, + "pics:", n_pics, "notes:", len(notes)) +``` + +(Simplify the title lookup to `slide.shapes.title` when present; the defensive loop is for +layouts where the title placeholder is missing.) + +## Triage: deck renders wrong + +| Symptom | Check | Fix | +|---|---|---| +Text overflows the slide edge | shape left+width vs `prs.slide_width` | move/resize the shape, or shrink font | +Everything shifted | slide size changed between sources | normalize slide size or re-layout on the target size | +Fonts look wrong elsewhere | non-embedded fonts (pptx rarely embeds) | report referenced fonts (`run.font.name`) | +File will not open | broken ZIP / part mismatch | same programmatic health check as DOCX: `zipfile.testzip()`, parse every `.xml` part | +Pictures blank | media parts missing or rels broken | verify `ppt/media/*` present and slide rels reference them | + +## Report contract + +Summarize: slide count, per-slide one-line inventory, then findings ranked by user impact. +Extraction for repurposing goes to markdown with speaker notes preserved as blockquotes. \ No newline at end of file diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/create.md b/plugins/Hylouis233/document-skills/skills/pptx/references/create.md new file mode 100644 index 0000000..db3f1a8 --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/create.md @@ -0,0 +1,82 @@ +# Create a deck (python-pptx) + +## Skeleton with the seven workhorse slide patterns + +```python +from pptx import Presentation +from pptx.util import Inches, Pt, Emu +from pptx.dml.color import RGBColor +from pptx.enum.text import PP_ALIGN + +prs = Presentation() # 16:9 default in modern python-pptx; else set slide size +prs.slide_width, prs.slide_height = Inches(13.333), Inches(7.5) + +blank = prs.slide_layouts[6] # index 6 = Blank in the default template + +def add_slide(): + return prs.slides.add_slide(blank) + +# P1 title slide +s = add_slide() +box = s.shapes.add_textbox(Inches(0.8), Inches(2.6), Inches(11.7), Inches(1.6)) +tf = box.text_frame; tf.word_wrap = True +p = tf.paragraphs[0]; p.text = "Service Reliability Review"; p.font.size = Pt(44); p.font.bold = True + +# P2 bullet slide +s = add_slide() +box = s.shapes.add_textbox(Inches(0.8), Inches(1.2), Inches(11.7), Inches(5.6)) +tf = box.text_frame; tf.word_wrap = True +lines = ["Uptime 99.97% (+0.04 vs last quarter)", "MTTR down to 42 minutes", "Two Sev-2 incidents, both capacity-driven"] +for i, line in enumerate(lines): + par = tf.paragraphs[0] if i == 0 else tf.add_paragraph() + par.text = line; par.font.size = Pt(24) + par.space_after = Pt(12) + +# P3 table slide +from pptx.util import Inches +s = add_slide() +rows, cols = 4, 3 +tbl_shape = s.shapes.add_table(rows, cols, Inches(0.8), Inches(1.5), Inches(11.7), Inches(3.5)) +table = tbl_shape.table +hdr = ["Region", "Error rate", "P99 latency"] +for j, text in enumerate(hdr): + cell = table.cell(0, j); cell.text = text + for par in cell.text_frame.paragraphs: + for run in par.runs: run.font.bold = True + +# P4 chart slide (real chart part, not a picture) +from pptx.chart.data import CategoryChartData +from pptx.enum.chart import XL_CHART_TYPE +s = add_slide() +cd = CategoryChartData() +cd.categories = ["Jul", "Aug", "Sep"] +cd.add_series("Deploy count", (18, 22, 31)) +graphic = s.shapes.add_chart(XL_CHART_TYPE.COLUMN_CLUSTERED, + Inches(1.5), Inches(1.5), Inches(10), Inches(5), cd) +chart = graphic.chart +chart.has_legend = False + +prs.save("deck.pptx") +``` + +## The seven patterns + +| Pattern | Build with | +|---|---| +| Title / section divider | one large textbox, 40-48pt | +| Agenda / list | single textbox, 22-28pt, space_after 10-14pt | +| Bullets + callout | two textboxes: bullets left, highlight right | +| Image + text | `add_picture` (aspect-true) + textbox beside it | +| Data table | `add_table`, bold header row, zebra fills optional | +| Chart | `add_chart` with `CategoryChartData` | +| Quote / closing | centered italic 28-32pt + attribution 16pt | + +## Rules + +- Set slide size once up front (16:9 = 13.333 x 7.5 in) and stay inside 0.6in margins. +- Title top-left at a consistent y-position across content slides; consistency reads as design. +- Max ~6 bullets per slide, one line each at the chosen size - if a bullet wraps twice, split + the slide or cut. +- Speaker notes: `slide.notes_slide.notes_text_frame.text = "..."` - put the script there, + not on the slide. +- Do not touch masters/layouts unless asked; a restyled master changes every existing slide. diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md b/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md new file mode 100644 index 0000000..f883ee4 --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md @@ -0,0 +1,46 @@ +# Edit an existing deck + +## Locate by content, then edit narrowly + +```python +from pptx import Presentation +from pptx.util import Pt + +prs = Presentation("input.pptx") + +target_slide, target_shape = None, None +for slide in prs.slides: + for shape in slide.shapes: + if shape.has_text_frame and "old wording" in shape.text_frame.text: + target_slide, target_shape = slide, shape + +assert target_shape is not None, "target text not found - report instead of guessing" + +# Replace whole-paragraph text but keep the first run's formatting +tf = target_shape.text_frame +for par in tf.paragraphs: + if "old wording" in par.text: + template_run = par.runs[0] if par.runs else None + par.text = par.text.replace("old wording", "new wording") + if template_run is not None: + for run in par.runs: + run.font.size = template_run.font.size + run.font.bold = template_run.font.bold + +prs.save("input-edited.pptx") +``` + +## Rules + +1. **Never rebuild the file to make a small change.** Rewriting slides from scratch loses the + template, masters, notes, and animations. Edit in place, save to a new path. +2. Address shapes by slide index + shape name or matched text, and **assert the match** - a + silent no-op edit is worse than a loud failure. +3. Table cells: `table.cell(r, c).text = ...`; keep the change inside the cell's text frame so + its formatting survives. +4. Chart data: `chart.replace_data(CategoryChartData(...))` updates the embedded workbook and + the plot together - do not hand-edit the XML series. +5. Reordering slides means moving the underlying `sldIdLst` entries; do it only on request and + verify order in the postcheck. +6. Group shapes: iterate `shape.shapes` recursively to reach members; python-pptx will not + ungroup for you - do not try to flatten groups. diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md b/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md new file mode 100644 index 0000000..e324ba0 --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md @@ -0,0 +1,63 @@ +--- +name: xlsx +description: Read, edit, create, or fix spreadsheet files - .xlsx, .xlsm, .xltx, .csv, .tsv. Use this Skill whenever a spreadsheet is the primary input or output: opening and inspecting workbooks, editing cells or adding sheets, writing real formulas instead of pasted values, cleaning messy data, adding charts and formatting, converting between tabular formats, or diagnosing corrupted or slow-opening files. +--- + +# XLSX workbench + +Spreadsheets are where agents do the most damage by being clever: pasted values where formulas +belong, dates written as text, whole-number floats displayed as `3.0000000001`. Follow the +contracts below. + +## Step 0 - Check the toolchain + +```bash +python -c "import openpyxl; print(openpyxl.__version__)" +``` + +- `.csv`/`.tsv` -> standard library `csv` module is fine and often better (streaming). +- `.xlsx`/`.xlsm`/`.xltx` -> openpyxl. +- Macro preservation: openpyxl keeps VBA in `.xlsm` only with `keep_vba=True` on load and save. + +## Step 1 - Classify the task + +| Request | Route | +|---|---| +| Open, inspect, profile a workbook | [references/read.md](references/read.md) | +| Edit cells, add sheets, fix formatting | [references/edit.md](references/edit.md) | +| Build a new workbook (data + formulas + chart) | [references/create.md](references/create.md) | +| CSV/TSV in or out, messy data cleanup | [references/csv.md](references/csv.md) | + +## Step 2 - Contracts that always apply + +1. **Formulas are formulas.** If the user asks for a total/average/lookup, write `=SUM(B2:B10)` + in the cell - never the computed number - unless the user explicitly asked to freeze values. + openpyxl writes the formula; Excel/WPS/LibreOffice calculate on open. +2. **`data_only=True` reads cached values** (last calculated by a real app) and **loses + formulas on save**. Use it only for reading values; never load, edit, and save with it. +3. **Types**: write `int`/`float`/`datetime`/`bool`, never formatted strings. Dates go in as + `datetime` with `number_format='yyyy-mm-dd'`; currency as float plus + `number_format='#,##0.00'` (or the locale-appropriate currency format string). +4. **Formulas are not recalculated by openpyxl.** After writing formulas you cannot read their + results back without opening the file in a real spreadsheet app; verify formula strings and + ranges structurally instead (see postcheck). +5. **Dimensions**: `ws.max_row`/`ws.max_column` reflect used range - trust them over guesses; + but scan for trailing blank-but-formatted rows when a file "looks" bigger than its data. +6. Save to a new path first; overwrite only on explicit request. + +## Step 3 - Postcheck (mandatory) + +```python +import openpyxl +wb = openpyxl.load_workbook("output.xlsx") +print("sheets:", wb.sheetnames) +ws = wb["Summary"] +print("dims:", ws.dimensions) +formulas = [(c.coordinate, c.value) for row in ws.iter_rows() for c in row + if isinstance(c.value, str) and c.value.startswith("=")] +print("formula cells:", formulas[:10]) +``` + +Confirm: expected sheet names exist; used range matches expectations; intended formula cells +contain formula strings; number formats survive. Report what was verified and note that final +rendered values require opening in a spreadsheet application. diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/create.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/create.md new file mode 100644 index 0000000..5658b82 --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/create.md @@ -0,0 +1,73 @@ +# Create a workbook from scratch + +```python +import openpyxl +from openpyxl.styles import Font, PatternFill, Alignment, Border, Side +from openpyxl.utils import get_column_letter +from openpyxl.chart import BarChart, LineChart, Reference + +wb = openpyxl.Workbook() +ws = wb.active +ws.title = "Sales" + +headers = ["Region", "Product", "Units", "Unit price", "Revenue"] +ws.append(headers) +rows = [ + ("EU", "Widget", 120, 9.5), + ("EU", "Gadget", 80, 21.0), + ("US", "Widget", 200, 9.5), +] +for r_i, (region, product, units, price) in enumerate(rows, start=2): + ws.cell(row=r_i, column=1, value=region) + ws.cell(row=r_i, column=2, value=product) + ws.cell(row=r_i, column=3, value=units) + ws.cell(row=r_i, column=4, value=price) + ws.cell(row=r_i, column=5, value=f"=C{r_i}*D{r_i}") # formula, not value + +last = len(rows) + 1 +ws.cell(row=last + 1, column=4, value="Total") +ws.cell(row=last + 1, column=5, value=f"=SUM(E2:E{last})").font = Font(bold=True) + +# Header styling +for cell in ws[1]: + cell.font = Font(bold=True, color="FFFFFF") + cell.fill = PatternFill("solid", fgColor="1F4E79") + cell.alignment = Alignment(horizontal="center") + +# Number formats on data columns +for r_i in range(2, last + 2): + ws.cell(row=r_i, column=4).number_format = "0.00" + ws.cell(row=r_i, column=5).number_format = "#,##0.00" + +widths = {"A": 10, "B": 16, "C": 8, "D": 12, "E": 12} +for col, w in widths.items(): + ws.column_dimensions[col].width = w +ws.freeze_panes = "A2" + +# Native chart bound to the sheet data +chart = BarChart() +chart.type = "col" +chart.title = "Revenue by product" +chart.y_axis.title = "Revenue" +data = Reference(ws, min_col=5, min_row=1, max_row=last) # includes header for series name +cats = Reference(ws, min_col=2, min_row=2, max_row=last) +chart.add_data(data, titles_from_data=True) +chart.set_categories(cats) +chart.width, chart.height = 16, 9 +ws.add_chart(chart, "G2") + +wb.save("report.xlsx") +``` + +## Rules + +- `ws.append(list)` for row-oriented data; `ws.cell(row=, column=, value=)` when you need the + cell object anyway (styling, formats). +- A chart references cells - it does not copy data. Keep the referenced range contiguous and + include the header row when using `titles_from_data=True`. +- Multiple sheets: one topic per sheet, `wb.create_sheet(name)`; cross-sheet formulas use + `SheetName!A1` syntax (quote the name if it contains spaces: `'Sales Data'!A1`). +- `LineChart` for trends, `BarChart` for comparisons, `PieChart` only for few categories that + sum to a whole. +- Excel row limit 1,048,576 and column limit 16,384 are hard; streaming writes should chunk + via `write_only=True` workbooks when generating hundreds of thousands of rows. diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md new file mode 100644 index 0000000..4ba0374 --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md @@ -0,0 +1,49 @@ +# CSV / TSV and messy data + +## Reading + +```python +import csv + +with open("input.csv", newline="", encoding="utf-8-sig") as f: # utf-8-sig strips a BOM + reader = csv.DictReader(f) + for i, row in enumerate(reader): + if i >= 5: break + print(row) +``` + +- Always pass `newline=""` to `open` on every platform - it is the documented requirement, + not a style choice. +- Sniff the dialect when provenance is unknown: + `csv.Sniffer().sniff(f.read(2048))` then `f.seek(0)`; semicolon-delimited exports are common + in European locales. +- Never trust inferred dtypes in CSV: everything is a string. Convert explicitly with + `try/except ValueError` per column and report counts of parse failures rather than dropping + rows silently. + +## Writing + +```python +import csv + +with open("output.csv", "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f) + writer.writerow(["Region", "Units"]) + writer.writerow(["EU", 120]) +``` + +## Converting + +- CSV -> XLSX: read with `csv`, write with openpyxl; convert values to real types on the way + through (dates via `datetime.strptime` with the format actually observed). +- XLSX -> CSV: `iter_rows(values_only=True)`; format numbers yourself only if the user needs a + fixed display format - otherwise write raw values and say so. +- Large CSV -> keep it CSV or move to SQLite/Parquet; loading it all into one sheet to + "preserve" it usually exceeds limits and helps nobody. + +## Messy data cleanup contract + +1. Profile before touching: row count, per-column types, null counts, duplicate-key check. +2. Report the cleanup plan and get on with it only for mechanical transforms (trim, case, + date parsing, dedup on declared keys). +3. Every destructive step (dropping rows, overwriting values) must be counted and reported. diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md new file mode 100644 index 0000000..cc41026 --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md @@ -0,0 +1,46 @@ +# Edit an existing workbook + +```python +import openpyxl +from datetime import date + +wb = openpyxl.load_workbook("input.xlsx") # NOT data_only: that would drop all formulas +ws = wb["Data"] + +# Address cells directly; check the header to confirm column meaning first +ws["D2"] = "=C2*1.08" # real formula +ws["E2"] = date(2025, 9, 30) +ws["E2"].number_format = "yyyy-mm-dd" +ws["F2"] = 1234.5 +ws["F2"].number_format = "#,##0.00" + +# Insert a row at a position (formulas in shifted rows do NOT auto-adjust - rewrite them) +ws.insert_rows(5) + +# Append a new sheet for derived output +summary = wb.create_sheet("Summary") +summary["A1"] = "Region" +summary["B1"] = "Total" +summary["A2"] = "EU" +summary["B2"] = "=SUMIF(Data!A:A,A2,Data!C:C)" + +wb.save("input-edited.xlsx") +``` + +## Rules + +- `insert_rows`/`delete_rows` move cells but do **not** rewrite range references for you. + After structural edits, audit every formula that references the shifted region: + `for row in ws.iter_rows(): for c in row: ...` checking `c.value` is a str starting with + `=`. +- Styling: set on the range once (`ws["A1:F1"].font = Font(bold=True)`), not per cell in a + loop. +- Column widths: `ws.column_dimensions["A"].width = 28` - set after writing data, from the + longest value you wrote, not a fixed guess. +- Freeze panes and autofilter improve usability cheaply: + `ws.freeze_panes = "A2"; ws.auto_filter.ref = ws.dimensions`. +- Merged cells: avoid creating new merges; writing into a non-anchor merged cell raises. +- `.xlsm`: `load_workbook(path, keep_vba=True)` and save with the same suffix, or macros are + stripped. +- Do not delete sheets unless asked; hide instead (`ws.sheet_state = "hidden"`) when the goal + is a cleaner tab bar. diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md new file mode 100644 index 0000000..8b0a7ef --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md @@ -0,0 +1,30 @@ +# Read / profile a workbook + +```python +import openpyxl + +wb = openpyxl.load_workbook("input.xlsx", read_only=True, data_only=True) +print("sheets:", wb.sheetnames) +ws = wb[wb.sheetnames[0]] +print("dims:", ws.calculate_dimension()) + +rows = ws.iter_rows(values_only=True) +header = next(rows) +print("header:", header) +for i, row in enumerate(rows): + if i >= 5: break + print(row) +``` + +## Rules + +- First pass always: sheet names, dimensions, header row, 5 sample rows. Report those before + any analysis. +- `read_only=True` streams large files; you lose random access (`ws["B2"]` works but is slow + in read_only mode - iterate instead). +- `data_only=True` gives cached values. A file saved by a library (never opened in Excel) + has **no cached values** - formulas read as `None`. Detect and report this instead of + claiming cells are empty. +- Mixed-type columns: profile them (`set(type(v).__name__ for v in col)`) before converting; + a column that is mostly numbers with a few text cells is a data-quality finding, not noise. +- Never load the full sheet into memory to "look at it" when `iter_rows` with a break would do. From 2907c51b66573508f13cd3270363389ebbb66b8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=98=E6=89=AC?= Date: Sat, 15 Aug 2026 10:20:43 +0800 Subject: [PATCH 02/34] Fix document skill review feedback --- .../skills/docx/references/edit.md | 81 +++++++++++++++---- .../document-skills/skills/pdf/SKILL.md | 26 +++++- .../skills/pdf/references/extract.md | 42 +++++----- .../skills/pdf/references/inspect.md | 21 +++-- .../skills/pdf/references/transform.md | 25 +++++- .../skills/pptx/references/analyze.md | 28 +++++-- .../skills/pptx/references/create.md | 75 ++++++++++------- .../skills/pptx/references/edit.md | 61 ++++++++------ .../document-skills/skills/xlsx/SKILL.md | 23 ++++-- .../skills/xlsx/references/edit.md | 22 +++-- .../skills/xlsx/references/read.md | 31 +++++-- 11 files changed, 315 insertions(+), 120 deletions(-) diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/edit.md b/plugins/Hylouis233/document-skills/skills/docx/references/edit.md index a7c9b93..e49e5e2 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/edit.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/edit.md @@ -11,16 +11,52 @@ paragraph text. ```python from docx import Document +def replace_across_runs(paragraph, old, new): + """Replace non-overlapping matches, including matches split across runs.""" + if not old: + raise ValueError("old must not be empty") + + runs = list(paragraph.runs) + text = "".join(run.text for run in runs) + starts = [] + position = 0 + while (start := text.find(old, position)) != -1: + starts.append(start) + position = start + len(old) + + # Map each non-empty run to its character range in the original paragraph text. + spans = [] + position = 0 + for index, run in enumerate(runs): + end = position + len(run.text) + if end > position: + spans.append((index, position, end)) + position = end + + # Work backwards so changing a later match cannot move an earlier match. + for start in reversed(starts): + end = start + len(old) + first, first_start, _ = next(s for s in spans if s[1] <= start < s[2]) + last, last_start, _ = next(s for s in spans if s[1] < end <= s[2]) + prefix = runs[first].text[:start - first_start] + suffix = runs[last].text[end - last_start:] + + if first == last: + runs[first].text = prefix + new + suffix + else: + runs[first].text = prefix + new + for index in range(first + 1, last): + runs[index].text = "" + runs[last].text = suffix + + return len(starts) + doc = Document("input.docx") # Address paragraphs by index over doc.paragraphs (body level). Tables' cells hold their own # paragraphs: table.rows[i].cells[j].paragraphs -for i, par in enumerate(doc.paragraphs): - if "TBD" in par.text: - # Run-level replace keeps formatting of untouched runs - for run in par.runs: - if "TBD" in run.text: - run.text = run.text.replace("TBD", "To be decided") +for par in doc.paragraphs: + replace_across_runs(par, "TBD", "To be decided") # Append content at a specific position: manipulate the XML tree target = doc.paragraphs[7]._p @@ -30,6 +66,10 @@ target.addnext(new_par._p) doc.save("input.edited.docx") ``` +The replacement text inherits the first matched run's formatting. Unmatched text before and +after it stays in its original runs, so its formatting is preserved. Use raw OOXML for fields, +tracked changes, or other content that `paragraph.runs` does not expose. + ## Tier 2 - raw OOXML surgery (only when Tier 1 cannot express it) For field codes, sectPr surgery, tracked changes, or parts python-docx does not model. @@ -43,16 +83,29 @@ Rules that keep the archive valid: 4. Repack with `[Content_Types].xml` first and stored/deflated entries only: ```python -import zipfile, pathlib -src = pathlib.Path("work") -with zipfile.ZipFile("output.docx", "w", zipfile.ZIP_DEFLATED) as z: - for name in ["[Content_Types].xml"] + [p.name for p in src.rglob("*") if p.is_file()]: - pass # write real paths in archive order; see note +from pathlib import Path +from zipfile import ZIP_DEFLATED, ZipFile + +src = Path("work") +content_types = src / "[Content_Types].xml" +if not content_types.is_file(): + raise FileNotFoundError(content_types) + +files = sorted( + (path for path in src.rglob("*") if path.is_file() and path != content_types), + key=lambda path: path.relative_to(src).as_posix(), +) + +with ZipFile( + "output.docx", "w", compression=ZIP_DEFLATED, strict_timestamps=False +) as archive: + archive.write(content_types, "[Content_Types].xml") + for path in files: + archive.write(path, path.relative_to(src).as_posix()) ``` - Concretely, walk the tree and `z.write(p, p.relative_to(src).as_posix())`, writing - `[Content_Types].xml` first. Do not add directories as entries, do not preserve absolute - timestamps incorrectly (ZIP wants local time or 0). + This writes relative POSIX archive names, does not add directory entries, and excludes + `[Content_Types].xml` from the remaining files so it cannot be added twice. 5. If you touched part names or added parts, update `[Content_Types].xml` and `word/_rels/document.xml.rels` consistently - a mismatch here is the classic silent corrupt. diff --git a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md index 73dac79..7a1312b 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md @@ -52,9 +52,27 @@ file is not. import pypdf r = pypdf.PdfReader("output.pdf") page_count = len(r.pages) -first_text = (r.pages[0].extract_text() or "").strip() +page_texts = { + number: (page.extract_text() or "").strip() + for number, page in enumerate(r.pages, start=1) +} +intentionally_raster_only_pages = set() +missing_text_pages = [ + number for number, text in page_texts.items() + if number not in intentionally_raster_only_pages and not text +] +assert not missing_text_pages, f"pages without extractable text: {missing_text_pages}" +# Add task-specific checks when exact copy matters, for example +# {1: ("Report title",), 2: ("Conclusion",)}. Per-page text presence is enforced above. +expected_strings_by_page = {} +for page_number, expected_strings in expected_strings_by_page.items(): + missing = [value for value in expected_strings if value not in page_texts[page_number]] + assert not missing, f"page {page_number} is missing {missing}" +page_sizes = [tuple(page.mediabox) for page in r.pages] ``` -Confirm: page count matches the request; the key title/heading text extracts non-empty; page -size is the declared size (`r.pages[0].mediabox`). Report all three. For pixel-sensitive work, -add a PyMuPDF render of page 1 at 100 dpi and check it is non-blank (mean pixel value). +Confirm: page count matches the request; every page except those explicitly listed in +`intentionally_raster_only_pages` has extractable text; each requested key string is listed in +`expected_strings_by_page` and extracts on the correct page; every value in `page_sizes` is the +declared size. Report all four. For pixel-sensitive work, render every applicable page with +PyMuPDF at 100 dpi and check that each render is non-blank (mean pixel value). diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md b/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md index 9061b56..6224b09 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md @@ -6,30 +6,36 @@ import fitz # pymupdf doc = fitz.open("input.pdf") print("pages:", doc.page_count, "metadata:", doc.metadata) -# Plain text, page by page +# Extract each page; do not use `page` after this loop. for page in doc: - text = page.get_text("text") # reading order text - -# With coordinates (decide columns/reading order yourself) -for block in page.get_text("dict")["blocks"]: - for line in block.get("lines", []): - for span in line["spans"]: - print(round(span["bbox"][0]), round(span["bbox"][1]), span["text"]) + page_number = page.number + 1 -# Images -for i, info in enumerate(page.get_images(full=True)): - pix = fitz.Pixmap(doc, info[0]) - pix.save(f"img-p{page.number}-{i}.png") - -# Rasterize a page (for visual checks or OCR preprocessing) -pix = page.get_pixmap(dpi=150) -pix.save(f"page-{page.number}.png") + # Plain text + text = page.get_text("text") # reading order text + print(f"--- page {page_number} ---") + print(text) + + # With coordinates (decide columns/reading order yourself) + for block in page.get_text("dict")["blocks"]: + for line in block.get("lines", []): + for span in line["spans"]: + print(page_number, round(span["bbox"][0]), + round(span["bbox"][1]), span["text"]) + + # Images + for i, info in enumerate(page.get_images(full=True), start=1): + pix = fitz.Pixmap(doc, info[0]) + pix.save(f"img-p{page_number}-{i}.png") + + # Rasterize (for visual checks or OCR preprocessing) + pix = page.get_pixmap(dpi=150) + pix.save(f"page-{page_number}.png") ``` ## Rules -- Scanned pages return empty `get_text`. Detect: first page text empty + images present -> - report "scanned, needs OCR" instead of claiming there is no text. +- Scanned pages return empty `get_text`. Check each page; if its text is empty and it contains + images, report that page as "scanned, needs OCR" instead of claiming there is no text. - "Sort by position" before emitting tables: spans come in internal order, not visual order; sort by `(round(bbox[1]), bbox[0])` for top-to-bottom, left-to-right reading. - Two-column layouts: cluster spans by x-gap before joining lines, or text interleaves columns. diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md b/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md index e44ae35..50f50f7 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md @@ -1,19 +1,30 @@ # Inspect a PDF (PyMuPDF) ```python +import os import fitz + doc = fitz.open("input.pdf") +if doc.needs_pass: + password = os.environ.get("PDF_PASSWORD", "") + if doc.authenticate(password) <= 0: + raise RuntimeError("Encrypted PDF: set a valid PDF_PASSWORD before inspection") + print("pages:", doc.page_count) -print("encrypted:", doc.needs_pass, "| tagged:", doc.is_pdf, "pdf:", doc.is_pdf) +print("password_protected:", doc.needs_pass, + "| still_encrypted:", doc.is_encrypted, "| pdf:", doc.is_pdf) for page in doc: + drawings = page.get_drawings() print(page.number, page.rect, "text_len:", len(page.get_text()), - "images:", len(page.get_images()), "links:", len(page.get_links())) + "images:", len(page.get_images()), "drawings:", len(drawings), + "links:", len(page.get_links())) ``` ## Checks worth automating -- **Blank page detection**: `len(page.get_text()) == 0 and not page.get_images()` -> flag; a - blank page after generation usually means an overflowing flowable created an empty page. +- **Blank page detection**: `not page.get_text().strip() and not page.get_images() and not + page.get_drawings()` -> flag; checking drawings avoids misclassifying vector-only pages as + blank. A blank page after generation usually means an overflowing flowable created it. - **Font inventory**: `page.get_fonts()` lists embedded names - needed when the user reports "looks different on machine X". - **Page size consistency**: mixed `page.rect` sizes in one file break duplex printing; report @@ -21,5 +32,5 @@ for page in doc: - **Damage**: `fitz.open` on a corrupt file raises or yields garbage - pair with `pypdf.PdfReader` cross-check when provenance is unknown. -Report findings as a table (page, size, text chars, images, links) - it is what every +Report findings as a table (page, size, text chars, images, drawings, links) - it is what every downstream decision hangs off. diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md b/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md index aedac52..61134da 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md @@ -31,16 +31,35 @@ Watermark / stamp by merging a stamp page onto each page: ```python stamp = PdfReader("watermark.pdf").pages[0] +stamp_text = (stamp.extract_text() or "").strip() +reader = PdfReader("input.pdf") +expected_sizes = [tuple(float(value) for value in page.mediabox) for page in reader.pages] writer = PdfWriter() -for page in PdfReader("input.pdf").pages: +for page in reader.pages: page.merge_page(stamp) # stamp content on top; use merge_transformed_page to place writer.add_page(page) + +with open("watermarked.pdf", "wb") as f: + writer.write(f) + +check = PdfReader("watermarked.pdf") +assert len(check.pages) == len(expected_sizes) +assert [tuple(float(value) for value in page.mediabox) for page in check.pages] == expected_sizes +if stamp_text: + assert all(stamp_text in (page.extract_text() or "") for page in check.pages) ``` +If the stamp is graphical, render every output page and visually confirm that it is present; +text extraction cannot validate a graphical watermark. + Encryption and forms: -- `writer.encrypt("pass", algorithm="AES-256")` to protect; `PdfReader(..., password="pass")` - to open. Report that you set a password - the user must record it. +- AES encryption requires pypdf's optional crypto backend. Install `pypdf[crypto]` with + `python -m pip install "pypdf[crypto]"`. Run + `python -c "import cryptography; print('AES backend ok')"` before calling + `writer.encrypt("pass", algorithm="AES-256")`. + Open the result with `PdfReader(..., password="pass")`. Report that you set a password - the + user must record it. - AcroForm fields: `reader.get_fields()` to enumerate; `writer.update_page_form_field_values( page, {"fieldname": "value"})` to fill. Flatten only on explicit request; it stops later editing. diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md index c7be922..e52f3cf 100644 --- a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md @@ -24,13 +24,29 @@ layouts where the title placeholder is missing.) | Symptom | Check | Fix | |---|---|---| -Text overflows the slide edge | shape left+width vs `prs.slide_width` | move/resize the shape, or shrink font | -Everything shifted | slide size changed between sources | normalize slide size or re-layout on the target size | -Fonts look wrong elsewhere | non-embedded fonts (pptx rarely embeds) | report referenced fonts (`run.font.name`) | -File will not open | broken ZIP / part mismatch | same programmatic health check as DOCX: `zipfile.testzip()`, parse every `.xml` part | -Pictures blank | media parts missing or rels broken | verify `ppt/media/*` present and slide rels reference them | +| Shape crosses the slide edge | compare all four shape bounds with the slide bounds | move or resize the shape | +| Text is clipped or overflows its box | render every slide and inspect right/left and bottom/vertical fit; shape bounds do not measure laid-out text | reflow, resize the box, or reduce text/font size, then render again | +| Everything shifted | slide size changed between sources | normalize slide size or re-layout on the target size | +| Fonts look wrong elsewhere | non-embedded fonts (pptx rarely embeds) | report referenced fonts (`run.font.name`) | +| File will not open | broken ZIP / part mismatch | same programmatic health check as DOCX: `zipfile.testzip()`, parse every `.xml` part | +| Pictures blank | media parts missing or rels broken | verify `ppt/media/*` present and slide rels reference them | + +## Text-fit verification + +`python-pptx` exposes a text box's geometry, not the renderer's final glyph and line layout. +After any text or layout change, render all slides with the fonts used in production: + +```bash +python -c "from pathlib import Path; Path('deck-render').mkdir(exist_ok=True)" +soffice --headless --convert-to pdf --outdir deck-render input.pptx +``` + +Inspect every page of `deck-render/input.pdf` for horizontal clipping and for the final line being +clipped or missing at the bottom. Rasterize the PDF when image inspection is easier. If this must +be an automated gate, use measured text bounds from a native renderer in both axes; +`shape.left + shape.width` is only a slide-boundary check, not an overflow test. ## Report contract Summarize: slide count, per-slide one-line inventory, then findings ranked by user impact. -Extraction for repurposing goes to markdown with speaker notes preserved as blockquotes. \ No newline at end of file +Extraction for repurposing goes to markdown with speaker notes preserved as blockquotes. diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/create.md b/plugins/Hylouis233/document-skills/skills/pptx/references/create.md index db3f1a8..080544b 100644 --- a/plugins/Hylouis233/document-skills/skills/pptx/references/create.md +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/create.md @@ -1,31 +1,44 @@ # Create a deck (python-pptx) -## Skeleton with the seven workhorse slide patterns +## Skeleton using semantic layouts and placeholders ```python from pptx import Presentation -from pptx.util import Inches, Pt, Emu -from pptx.dml.color import RGBColor -from pptx.enum.text import PP_ALIGN +from pptx.enum.shapes import PP_PLACEHOLDER +from pptx.util import Inches, Pt -prs = Presentation() # 16:9 default in modern python-pptx; else set slide size -prs.slide_width, prs.slide_height = Inches(13.333), Inches(7.5) +prs = Presentation() +# Keep this template's slide size and layout geometry together. For 16:9 output, start with a +# real 16:9 template instead of changing only slide_width/slide_height after loading layouts. -blank = prs.slide_layouts[6] # index 6 = Blank in the default template +# Built-in template convention: 0=Title, 1=Title and Content, 5=Title Only. +# With a supplied template, inspect [(i, x.name) for i, x in enumerate(prs.slide_layouts)] +# and map these roles to its layouts instead of assuming the same indices. +title_layout = prs.slide_layouts[0] +content_layout = prs.slide_layouts[1] +title_only_layout = prs.slide_layouts[5] -def add_slide(): - return prs.slides.add_slide(blank) +def add_slide(layout): + return prs.slides.add_slide(layout) + +def placeholder_of_type(slide, *types): + matches = [ + ph for ph in slide.placeholders + if ph.placeholder_format.type in types + ] + assert len(matches) == 1, f"expected one placeholder of {types}, found {len(matches)}" + return matches[0] # P1 title slide -s = add_slide() -box = s.shapes.add_textbox(Inches(0.8), Inches(2.6), Inches(11.7), Inches(1.6)) -tf = box.text_frame; tf.word_wrap = True -p = tf.paragraphs[0]; p.text = "Service Reliability Review"; p.font.size = Pt(44); p.font.bold = True +s = add_slide(title_layout) +s.shapes.title.text = "Service Reliability Review" +placeholder_of_type(s, PP_PLACEHOLDER.SUBTITLE).text = "Quarterly operations review" # P2 bullet slide -s = add_slide() -box = s.shapes.add_textbox(Inches(0.8), Inches(1.2), Inches(11.7), Inches(5.6)) -tf = box.text_frame; tf.word_wrap = True +s = add_slide(content_layout) +s.shapes.title.text = "Executive summary" +tf = placeholder_of_type(s, PP_PLACEHOLDER.BODY, PP_PLACEHOLDER.OBJECT).text_frame +tf.clear(); tf.word_wrap = True lines = ["Uptime 99.97% (+0.04 vs last quarter)", "MTTR down to 42 minutes", "Two Sev-2 incidents, both capacity-driven"] for i, line in enumerate(lines): par = tf.paragraphs[0] if i == 0 else tf.add_paragraph() @@ -33,10 +46,10 @@ for i, line in enumerate(lines): par.space_after = Pt(12) # P3 table slide -from pptx.util import Inches -s = add_slide() +s = add_slide(title_only_layout) +s.shapes.title.text = "Regional service health" rows, cols = 4, 3 -tbl_shape = s.shapes.add_table(rows, cols, Inches(0.8), Inches(1.5), Inches(11.7), Inches(3.5)) +tbl_shape = s.shapes.add_table(rows, cols, Inches(0.5), Inches(1.5), Inches(9), Inches(3.5)) table = tbl_shape.table hdr = ["Region", "Error rate", "P99 latency"] for j, text in enumerate(hdr): @@ -47,12 +60,13 @@ for j, text in enumerate(hdr): # P4 chart slide (real chart part, not a picture) from pptx.chart.data import CategoryChartData from pptx.enum.chart import XL_CHART_TYPE -s = add_slide() +s = add_slide(title_only_layout) +s.shapes.title.text = "Deployment volume" cd = CategoryChartData() cd.categories = ["Jul", "Aug", "Sep"] cd.add_series("Deploy count", (18, 22, 31)) graphic = s.shapes.add_chart(XL_CHART_TYPE.COLUMN_CLUSTERED, - Inches(1.5), Inches(1.5), Inches(10), Inches(5), cd) + Inches(0.75), Inches(1.5), Inches(8.5), Inches(5), cd) chart = graphic.chart chart.has_legend = False @@ -63,17 +77,20 @@ prs.save("deck.pptx") | Pattern | Build with | |---|---| -| Title / section divider | one large textbox, 40-48pt | -| Agenda / list | single textbox, 22-28pt, space_after 10-14pt | -| Bullets + callout | two textboxes: bullets left, highlight right | -| Image + text | `add_picture` (aspect-true) + textbox beside it | -| Data table | `add_table`, bold header row, zebra fills optional | -| Chart | `add_chart` with `CategoryChartData` | -| Quote / closing | centered italic 28-32pt + attribution 16pt | +| Title / section divider | Title or Section Header layout; title/subtitle placeholders | +| Agenda / list | Title and Content layout; body placeholder | +| Bullets + callout | Two Content layout; use both content placeholders | +| Image + text | Picture/Content with Caption layout and its placeholders | +| Data table | Title Only layout + `add_table` when no table placeholder exists | +| Chart | Title Only layout + `add_chart` with `CategoryChartData` | +| Quote / closing | Section Header or Title Only layout; add only the missing quote box | ## Rules -- Set slide size once up front (16:9 = 13.333 x 7.5 in) and stay inside 0.6in margins. +- Select a template with the requested aspect ratio before adding slides, then keep its slide + size and layout geometry unchanged. Stay inside 0.5in margins. +- Choose a layout for the slide's purpose and populate its title/body placeholders. Use Blank + only when no template layout can express the design, then measure every added shape. - Title top-left at a consistent y-position across content slides; consistency reads as design. - Max ~6 bullets per slide, one line each at the chosen size - if a bullet wraps twice, split the slide or cut. diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md b/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md index f883ee4..fd1b816 100644 --- a/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md @@ -4,28 +4,40 @@ ```python from pptx import Presentation -from pptx.util import Pt prs = Presentation("input.pptx") -target_slide, target_shape = None, None -for slide in prs.slides: - for shape in slide.shapes: - if shape.has_text_frame and "old wording" in shape.text_frame.text: - target_slide, target_shape = slide, shape - -assert target_shape is not None, "target text not found - report instead of guessing" +old, new = "old wording", "new wording" +slide_index = None # Set this and shape_name when repeated text is expected. +shape_name = None -# Replace whole-paragraph text but keep the first run's formatting +candidates = [] +for i, slide in enumerate(prs.slides): + for shape in slide.shapes: + if slide_index is not None and i != slide_index: + continue + if shape_name is not None and shape.name != shape_name: + continue + if shape.has_text_frame and old in shape.text_frame.text: + candidates.append((i, shape.name, shape)) + +locations = [(i, name) for i, name, _ in candidates] +assert len(candidates) == 1, f"expected one matching shape, found {locations}" +_, _, target_shape = candidates[0] + +# Replace inside one existing run so its formatting and hyperlink are retained. tf = target_shape.text_frame -for par in tf.paragraphs: - if "old wording" in par.text: - template_run = par.runs[0] if par.runs else None - par.text = par.text.replace("old wording", "new wording") - if template_run is not None: - for run in par.runs: - run.font.size = template_run.font.size - run.font.bold = template_run.font.bold +assert tf.text.count(old) == 1, "target occurs more than once in the selected shape" +run_hits = [ + run + for paragraph in tf.paragraphs + for run in paragraph.runs + if old in run.text +] +assert len(run_hits) == 1 and run_hits[0].text.count(old) == 1, ( + "target is duplicated or split across runs; report it instead of flattening the paragraph" +) +run_hits[0].text = run_hits[0].text.replace(old, new, 1) prs.save("input-edited.pptx") ``` @@ -34,13 +46,16 @@ prs.save("input-edited.pptx") 1. **Never rebuild the file to make a small change.** Rewriting slides from scratch loses the template, masters, notes, and animations. Edit in place, save to a new path. -2. Address shapes by slide index + shape name or matched text, and **assert the match** - a - silent no-op edit is worse than a loud failure. -3. Table cells: `table.cell(r, c).text = ...`; keep the change inside the cell's text frame so +2. Address shapes by slide index + shape name or matched text, and **assert exactly one match**. + If copy repeats, set both selectors rather than silently choosing the last shape. +3. For formatted text, change `run.text` only when the target is wholly inside one run. Assigning + `paragraph.text` or `text_frame.text` rebuilds runs and can discard run formatting and links. + If the target spans runs, stop and make an explicitly reviewed run/XML edit. +4. Table cells: `table.cell(r, c).text = ...`; keep the change inside the cell's text frame so its formatting survives. -4. Chart data: `chart.replace_data(CategoryChartData(...))` updates the embedded workbook and +5. Chart data: `chart.replace_data(CategoryChartData(...))` updates the embedded workbook and the plot together - do not hand-edit the XML series. -5. Reordering slides means moving the underlying `sldIdLst` entries; do it only on request and +6. Reordering slides means moving the underlying `sldIdLst` entries; do it only on request and verify order in the postcheck. -6. Group shapes: iterate `shape.shapes` recursively to reach members; python-pptx will not +7. Group shapes: iterate `shape.shapes` recursively to reach members; python-pptx will not ungroup for you - do not try to flatten groups. diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md b/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md index e324ba0..3bea9bf 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md @@ -47,15 +47,26 @@ python -c "import openpyxl; print(openpyxl.__version__)" ## Step 3 - Postcheck (mandatory) +Save this as `postcheck.py`, then pass the output path followed by every sheet the task should +produce, for example `python postcheck.py output.xlsx Sales Summary`: + ```python import openpyxl -wb = openpyxl.load_workbook("output.xlsx") +import sys + +if len(sys.argv) < 3: + raise SystemExit("usage: python postcheck.py OUTPUT.xlsx EXPECTED_SHEET [...]") +output_path, *expected_sheets = sys.argv[1:] +wb = openpyxl.load_workbook(output_path) print("sheets:", wb.sheetnames) -ws = wb["Summary"] -print("dims:", ws.dimensions) -formulas = [(c.coordinate, c.value) for row in ws.iter_rows() for c in row - if isinstance(c.value, str) and c.value.startswith("=")] -print("formula cells:", formulas[:10]) +missing = set(expected_sheets) - set(wb.sheetnames) +assert not missing, f"missing expected sheets: {sorted(missing)}" +for ws in wb.worksheets: + print(f"{ws.title} dims:", ws.dimensions) + formulas = [(c.coordinate, c.value) for row in ws.iter_rows() for c in row + if isinstance(c.value, str) and c.value.startswith("=")] + print(f"{ws.title} formula cells:", formulas[:10]) +wb.close() ``` Confirm: expected sheet names exist; used range matches expectations; intended formula cells diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md index cc41026..69c3c8c 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md @@ -3,6 +3,7 @@ ```python import openpyxl from datetime import date +from openpyxl.styles import Font wb = openpyxl.load_workbook("input.xlsx") # NOT data_only: that would drop all formulas ws = wb["Data"] @@ -24,17 +25,28 @@ summary["B1"] = "Total" summary["A2"] = "EU" summary["B2"] = "=SUMIF(Data!A:A,A2,Data!C:C)" +header_font = Font(bold=True) +for row in summary["A1:B1"]: + for cell in row: + cell.font = header_font + +# References to the shifted region may live on any sheet; inspect every formula +for formula_ws in wb.worksheets: + for row in formula_ws.iter_rows(): + for cell in row: + if isinstance(cell.value, str) and cell.value.startswith("="): + print(f"{formula_ws.title}!{cell.coordinate}: {cell.value}") + wb.save("input-edited.xlsx") ``` ## Rules - `insert_rows`/`delete_rows` move cells but do **not** rewrite range references for you. - After structural edits, audit every formula that references the shifted region: - `for row in ws.iter_rows(): for c in row: ...` checking `c.value` is a str starting with - `=`. -- Styling: set on the range once (`ws["A1:F1"].font = Font(bold=True)`), not per cell in a - loop. + After structural edits, traverse `wb.worksheets` as above and update every formula that + references the shifted region, including formulas on other sheets. +- Styling: import `Font` and assign the style to each cell. A range such as `ws["A1:F1"]` + returns tuples of cells and cannot be styled as one object. - Column widths: `ws.column_dimensions["A"].width = 28` - set after writing data, from the longest value you wrote, not a fixed guess. - Freeze panes and autofilter improve usability cheaply: diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md index 8b0a7ef..94380d2 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md @@ -3,17 +3,34 @@ ```python import openpyxl -wb = openpyxl.load_workbook("input.xlsx", read_only=True, data_only=True) -print("sheets:", wb.sheetnames) -ws = wb[wb.sheetnames[0]] -print("dims:", ws.calculate_dimension()) +formula_wb = openpyxl.load_workbook("input.xlsx", read_only=True, data_only=False) +value_wb = openpyxl.load_workbook("input.xlsx", read_only=True, data_only=True) +print("sheets:", value_wb.sheetnames) +sheet_name = value_wb.sheetnames[0] +formula_ws = formula_wb[sheet_name] +value_ws = value_wb[sheet_name] +print("dims:", value_ws.calculate_dimension()) -rows = ws.iter_rows(values_only=True) +rows = value_ws.iter_rows(values_only=True) header = next(rows) print("header:", header) for i, row in enumerate(rows): if i >= 5: break print(row) + +missing_cache_count = 0 +for formula_row, value_row in zip(formula_ws.iter_rows(), value_ws.iter_rows()): + for formula_cell, value_cell in zip(formula_row, value_row): + if (isinstance(formula_cell.value, str) + and formula_cell.value.startswith("=") + and value_cell.value is None): + missing_cache_count += 1 + if missing_cache_count <= 10: + print("formula without cached value:", formula_cell.coordinate, + formula_cell.value) +print("formulas without cached values:", missing_cache_count) +formula_wb.close() +value_wb.close() ``` ## Rules @@ -23,8 +40,8 @@ for i, row in enumerate(rows): - `read_only=True` streams large files; you lose random access (`ws["B2"]` works but is slow in read_only mode - iterate instead). - `data_only=True` gives cached values. A file saved by a library (never opened in Excel) - has **no cached values** - formulas read as `None`. Detect and report this instead of - claiming cells are empty. + may return `None` for formulas with no cache. Compare each cell with the corresponding cell + from a `data_only=False` workbook, as above, before claiming it is empty. - Mixed-type columns: profile them (`set(type(v).__name__ for v in col)`) before converting; a column that is mostly numbers with a few text cells is a data-quality finding, not noise. - Never load the full sheet into memory to "look at it" when `iter_rows` with a break would do. From f803513495e85081e58d560590b0cb4514ee0f74 Mon Sep 17 00:00:00 2001 From: Hylouis233 Date: Sun, 16 Aug 2026 00:54:48 +0800 Subject: [PATCH 03/34] Fix review round 2 and add depth references MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review fixes (all fixture-verified in tests/): - PDF: AcroForm route now clones pages + /AcroForm into the writer via append() and fills fields on writer pages; mediabox postcheck reduced to width/height pairs; CMYK/ICC pixmaps converted to RGB before PNG save - PPTX: table cells edited at run level (cell.text rebuild proven lossy); inventory walker recurses into group shapes; font triage resolves inherited fonts with theme major/minor fallback - XLSX: sniffed CSV dialect passed to DictReader; round_trip_losses() detects parts an openpyxl round trip drops before saving - DOCX: numbering restart via cloned + startOverride, rendered proof in fixture (restarts at 1 vs style-reuse continuing at 4) Depth references (original content, no proprietary material): - docx cjk.md: east-asian font slot, 字号 table, char-based indent, fixed line spacing, GB/T 9704 page geometry, font availability risks - docx scenes.md: academic paper / resume / official document / contract skeletons with scene-specific verification - xlsx formatting.md: conditional formatting rules, structured tables, honest pivot aggregation routes (openpyxl cannot create pivots) tests/: one runnable fixture per format, 30 assertions, all passing locally (pdf 9, pptx 10, xlsx 6, docx 5 incl. soffice-rendered proof) --- plugins/Hylouis233/document-skills/README.md | 14 +- .../document-skills/skills/docx/SKILL.md | 5 + .../skills/docx/references/cjk.md | 96 +++++++++++ .../skills/docx/references/create.md | 45 ++++- .../skills/docx/references/scenes.md | 71 ++++++++ .../document-skills/skills/pdf/SKILL.md | 6 +- .../skills/pdf/references/extract.md | 5 +- .../skills/pdf/references/transform.md | 34 +++- .../skills/pptx/references/analyze.md | 63 ++++++- .../skills/pptx/references/edit.md | 13 +- .../document-skills/skills/xlsx/SKILL.md | 1 + .../skills/xlsx/references/csv.md | 16 +- .../skills/xlsx/references/edit.md | 23 +++ .../skills/xlsx/references/formatting.md | 95 +++++++++++ .../document-skills/tests/docx_fixture.py | 131 ++++++++++++++ .../document-skills/tests/pdf_fixture.py | 107 ++++++++++++ .../document-skills/tests/pptx_fixture.py | 161 ++++++++++++++++++ .../document-skills/tests/xlsx_fixture.py | 86 ++++++++++ 18 files changed, 954 insertions(+), 18 deletions(-) create mode 100644 plugins/Hylouis233/document-skills/skills/docx/references/cjk.md create mode 100644 plugins/Hylouis233/document-skills/skills/docx/references/scenes.md create mode 100644 plugins/Hylouis233/document-skills/skills/xlsx/references/formatting.md create mode 100644 plugins/Hylouis233/document-skills/tests/docx_fixture.py create mode 100644 plugins/Hylouis233/document-skills/tests/pdf_fixture.py create mode 100644 plugins/Hylouis233/document-skills/tests/pptx_fixture.py create mode 100644 plugins/Hylouis233/document-skills/tests/xlsx_fixture.py diff --git a/plugins/Hylouis233/document-skills/README.md b/plugins/Hylouis233/document-skills/README.md index bf4de01..030599f 100644 --- a/plugins/Hylouis233/document-skills/README.md +++ b/plugins/Hylouis233/document-skills/README.md @@ -48,11 +48,15 @@ Per format: - **docx** — create with python-docx from a heading outline; edit existing files by direct `word/document.xml` surgery (python-docx cannot open-and-save arbitrary files losslessly); extract text with python-docx or `pandoc -t markdown`; postcheck with python-docx re-open and - `soffice --headless --convert-to` PDF smoke test when LibreOffice is present. + `soffice --headless --convert-to` PDF smoke test when LibreOffice is present. Depth + references: CJK typography (east-asian font slots, 字号 table, char-based indents, GB/T 9704 + page geometry) and scene patterns (academic paper, resume, official document, contract). - **xlsx** — openpyxl for reading, editing, styling, and native charts; formulas as formulas, never as pasted results; `data_only=True` only for reading cached values; date/number formats applied explicitly; recalculation contract documented (openpyxl writes formulas, the viewer - calculates). + calculates). Depth references: conditional formatting rules, structured tables, and honest + pivot-style aggregation (openpyxl cannot create pivot tables; the reference gives the + formula-sheet, frozen-values, and user-template routes). - **pptx** — python-pptx to build decks (7 common slide patterns: title, agenda, bullet, two image+text, table, chart, quote/closing); edit only named, existing shapes, never blind rewriting of the whole XML; text measured against shape width with font-size reduction rules; @@ -69,6 +73,12 @@ are specific (re-open the archive, confirm the sheet count and formula presence, count, confirm page count and text extraction) and the Skills require reporting what was verified versus what was assumed. +The [`tests/`](tests/) directory ships one runnable fixture script per format covering the +snippets with the worst silent-failure modes (PDF AcroForm clone-and-fill, watermark write, +CMYK pixmap conversion; PPTX run-preserving edits, table-cell run edits, grouped-shape walking; +XLSX dialect sniffing, round-trip part-loss detection; DOCX numbering restart rendered through +LibreOffice). Each script is self-contained and exits non-zero on failure. + ## Requirements - Python 3.9+ with `python-docx`, `python-pptx`, `openpyxl`, `pypdf`, `reportlab`, diff --git a/plugins/Hylouis233/document-skills/skills/docx/SKILL.md b/plugins/Hylouis233/document-skills/skills/docx/SKILL.md index cdfaf2c..cbc587e 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/SKILL.md +++ b/plugins/Hylouis233/document-skills/skills/docx/SKILL.md @@ -31,6 +31,11 @@ python -c "import docx; print(docx.__version__ if hasattr(docx,'__version__') el | "Read / extract / summarize / convert" | Read | [references/read.md](references/read.md) | | "Check / fix / why is it broken" | Review | [references/review.md](references/review.md) | +Two depth references slot in before create/edit: [references/cjk.md](references/cjk.md) +whenever the document contains CJK text (east-asian fonts, 字号, char-based indents, 公文 +geometry), and [references/scenes.md](references/scenes.md) when the request matches a known +scene - academic paper, resume, official document (公文), or contract. + Two routes can chain (read -> edit, create -> review). Never skip classification; it decides between python-docx generation and raw XML surgery, which have opposite safety profiles. diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md b/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md new file mode 100644 index 0000000..249422b --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md @@ -0,0 +1,96 @@ +# CJK typography in DOCX (Chinese, Japanese, Korean) + +python-docx is Latin-first: `run.font.name` writes only the `w:ascii` and `w:hAnsi` slots. +Chinese text renders from the `w:eastAsia` slot, so a "font is wrong in Word but fine in +LibreOffice" report almost always means the east-asian face was never set. + +## Set the CJK face explicitly + +```python +from docx.oxml.ns import qn + +def set_fonts(run, latin="Times New Roman", east_asian="宋体"): + run.font.name = latin # writes w:ascii + w:hAnsi + rPr = run._element.get_or_add_rPr() + rFonts = rPr.get_or_add_rFonts() + rFonts.set(qn("w:eastAsia"), east_asian) # python-docx has no helper for this slot + +# Style level: do the same on the style so body text inherits it +style = doc.styles["Normal"] +style.font.name = latin +style.element.get_or_add_rPr().get_or_add_rFonts().set(qn("w:eastAsia"), east_asian) +``` + +Convention for mixed-script body text: CJK glyphs from the east-asian face, digits and Latin +from a Latin face (Times New Roman or Arial). Both slots set = deterministic rendering. + +## Chinese font-size table (字号) + +Word's Chinese UI names map to point sizes; scripts must use the points: + +| 字号 | pt | typical use | 字号 | pt | +|---|---|---|---|---| +| 初号 | 42 | big covers | 小三 | 15 | +| 小初 | 36 | covers | 四号 | 14 | +| 一号 | 26 | 公文标题辅助 | 小四 | 12 | +| 小一 | 24 | | 五号 | 10.5 | +| 二号 | 22 | 公文标题 | 小五 | 9 | +| 小二 | 18 | section covers | 六号 | 7.5 | +| 三号 | 16 | 公文正文 | 七号 | 5.5 | +| | | | 八号 | 5 | + +Common pairings: 正文宋体小四 (reports), 仿宋三号 (official documents), 黑体 for headings at +one size step above the body. + +## Paragraph properties CJK actually needs + +```python +from docx.shared import Pt + +pPr = p._p.get_or_add_pPr() + +# First-line indent of exactly two characters - use the char-based attribute, not twips, +# so the indent survives font-size changes +ind = pPr.get_or_add_ind() +ind.set(qn("w:firstLineChars"), "200") # units are 1/100 of a character + +# Fixed line spacing (公文 practice: 28-30 pt fixed; pick one value and stay consistent) +p.paragraph_format.line_spacing = Pt(28) # writes w:line=560, w:lineRule="exact" +``` + +- Word's CJK defaults (kinsoku line-breaking, auto space between CJK and Latin) live in + document defaults and need no action unless the source file disabled them. +- Do not fake a two-char indent with spaces or full-width spaces; `firstLineChars` is the + durable mechanism and survives re-flow. +- Justify body text (`WD_ALIGN_PARAGRAPH.JUSTIFY`); CJK justification is the expected look. + +## Official-document page geometry (GB/T 9704 family) + +Public-standard values for 党政机关公文 style documents on A4 - always confirm the user's +edition before promising compliance: + +```python +from docx.shared import Cm + +section = doc.sections[0] +section.top_margin, section.bottom_margin = Cm(3.7), Cm(3.5) +section.left_margin, section.right_margin = Cm(2.8), Cm(2.6) +# conventional target density with 三号仿宋 body: ~22 lines per page, ~28 chars per line +``` + +## Font availability is a delivery risk + +- `宋体`/`黑体`/`楷体` exist on Chinese Windows; `仿宋_GB2312` may not. Non-Chinese systems + substitute silently. Report the faces you referenced and expected substitutions in the + postcheck summary. +- python-docx cannot embed fonts. If the recipient machine is unknown and the layout must be + exact, say so and suggest embedding from Word (File > Options > Save > Embed fonts), or + deliver a PDF alongside. + +## Postcheck additions for CJK documents + +1. Re-open the output and assert the east-asian slot is set on body runs and styles + (`.get(qn("w:eastAsia"))` is not None) - not just `run.font.name`. +2. Render with soffice and confirm no tofu (missing-glyph boxes) by extracting text from the + rendered PDF and comparing against the source strings. +3. Confirm `firstLineChars` survived on body paragraphs if the indent was requested. diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/create.md b/plugins/Hylouis233/document-skills/skills/docx/references/create.md index ad3f916..628e15b 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/create.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/create.md @@ -63,8 +63,49 @@ doc.save("report.docx") - Build the heading outline **before** writing prose; a wrong outline is the most expensive late fix. - One table style per document. Cell-level font overrides are for exceptions only. -- Bullet/numbered lists: `add_paragraph(text, style="List Bullet")` / - `"List Number"`. Restart numbering per list by using separate list style instances. +- Bullet lists: `add_paragraph(text, style="List Bullet")`. Numbered lists: `style="List + Number"` - but reusing that style makes every list **continue** the same sequence, because + all paragraphs share one numbering definition. When a second list must restart at 1, clone + the numbering definition and point the new list's paragraphs at the clone: + + ```python + import copy + from docx.oxml.ns import qn + from docx.oxml import OxmlElement + from docx.opc.constants import RELATIONSHIP_TYPE as RT + + def new_restart_num_id(doc, base_num_id): + """Clone base_num_id with a startOverride so the next list restarts at 1.""" + numbering = doc.part.part_related_by(RT.NUMBERING).element + source = next( + n for n in numbering.findall(qn("w:num")) + if n.get(qn("w:numId")) == str(base_num_id) + ) + clone = copy.deepcopy(source) + new_id = max(int(n.get(qn("w:numId"))) for n in numbering.findall(qn("w:num"))) + 1 + clone.set(qn("w:numId"), str(new_id)) + override = OxmlElement("w:lvlOverride") + override.set(qn("w:ilvl"), "0") + start = OxmlElement("w:startOverride") + start.set(qn("w:val"), "1") + override.append(start) + clone.append(override) + numbering.append(clone) + return new_id + + def numbered_paragraph(doc, text, num_id): + p = doc.add_paragraph(text, style="List Number") + pPr = p._p.get_or_add_pPr() + numPr = OxmlElement("w:numPr") + ilvl = OxmlElement("w:ilvl"); ilvl.set(qn("w:val"), "0") + numId = OxmlElement("w:numId"); numId.set(qn("w:val"), str(num_id)) + numPr.append(ilvl); numPr.append(numId) + pPr.append(numPr) + return p + + # Find the numId behind "List Number" in numbering.xml (inspect it once, then hard-code), + # then give each independent list its own cloned definition. + ``` - Images: `doc.add_picture(path, width=Cm(14))` - always set width so oversized images do not overflow the text column. Keep aspect ratio by setting only one dimension. - A real TOC is a **field**, it renders after the user opens the file and updates fields diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/scenes.md b/plugins/Hylouis233/document-skills/skills/docx/references/scenes.md new file mode 100644 index 0000000..12fce1f --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/docx/references/scenes.md @@ -0,0 +1,71 @@ +# Scene patterns: paper, resume, official document, contract + +When the request matches a known scene, start from its skeleton instead of improvising +structure. Every scene below still goes through the normal create/edit route and postcheck; +this file only fixes structure, typography conventions, and scene-specific verification. + +## Academic paper (中文学术论文 shape) + +Skeleton: 标题 (二号黑体 or per journal) -> 作者 -> 单位 -> 摘要 -> 关键词 -> numbered body +(1 / 1.1 / 1.1.1) -> 参考文献 -> optional 附录/致谢. + +- Headings carry the numbering as literal text when no journal template is supplied; heading + styles stay semantic (`Heading 1..4`) so navigation and TOC keep working. +- Figures/tables: caption paragraph below the object, numbered per chapter + (图 2-1 / 表 3-2); reference in text before the object appears. +- References: GB/T 7714 style is the common Chinese requirement - build each entry as a + plain paragraph with a hanging indent, not a numbered Word list (journals re-style anyway). +- Fonts by convention: 正文宋体小四, fixed or 1.5 line spacing, first-line indent two chars - + see [cjk.md](cjk.md) for the mechanics. +- Out of scope: equations as OMML and journal-specific LaTeX export - offer the PDF route or + the journal's own template instead of hand-building OMML. + +## Resume + +- One page unless the user explicitly says otherwise; the constraint is content curation, + not shrinking fonts below 10 pt. +- Structure: 姓名+联系方式 header -> 求职意向 (optional) -> 经历 (reverse chronological) -> + 教育 -> 技能. Section headings as `Heading 2`, entries as styled paragraphs, bullets via + `List Bullet`. +- ATS-safe means: real text (no text boxes, no multi-column layout, no icons), standard + section names, dates as plain text `2024-03` not fields. +- A single subtle table for the contact header is acceptable; skill-rating bars and other + graphic devices are not - they print wrong and break parsers. +- Verify: render with soffice, confirm one page, confirm all text extracts from the PDF. + +## Official document (公文, GB/T 9704 shape) + +Structure: 版头 (发文机关标志+文号) -> 标题 (二号宋体加粗 per current practice) -> 主送机关 -> +正文 (三号仿宋) -> 附件说明 -> 发文机关署名+成文日期 -> 抄送 -> 印发信息. + +- Body hierarchy is fixed: 一、 -> （一） -> 1. -> （1）; never skip or reorder levels. +- Page geometry and line spacing values: see [cjk.md](cjk.md) - the GB/T table and fixed + 28-30 pt spacing live there. +- The red 发文机关标志 header is a graphics-level element python-docx does not model; + generate the document without it and tell the user to add it in their official template, + or ask for their template file and edit inside it. +- Numbers: Arabic numerals for dates and quantities per the standard; 成文日期 as + 2026年8月16日 with Arabic numerals. + +## Contract + +Skeleton: 标题 -> 当事人 block (名称/住所/法定代表人 per party) -> 鉴于 (recitals, optional) -> +numbered 条款 -> 签署 block. + +- Clause numbering as literal text with a fixed hierarchy (第一条 / 1.1 / (a)); do not use + auto-numbered lists - counter restarts and cross-references become fragile. +- Defined terms: bold at first definition only; thereafter plain. Search for the term to + confirm it is defined exactly once before bolding. +- Cross-references as literal text ("见第 5.2 条"); after edits, grep every 第 X 条 reference + and verify the target still exists at that number. +- Signature block: a borderless 2-column table (甲方/乙方 rows for 签字、盖章、日期) at the + end; set `keep_with_next = True` on the paragraphs right before it so the block never + splits across pages, and re-check in the rendered PDF. +- Verification beyond the standard postcheck: every defined term defined once; every + cross-reference resolves; signature block on one page in the rendered PDF. + +## Routing note + +Scenes chain: pick the scene skeleton, then execute it through +[create.md](create.md) (new document) or [edit.md](edit.md) (user supplied a draft), applying +[cjk.md](cjk.md) whenever the document contains CJK text. diff --git a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md index 7a1312b..e272a4c 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md @@ -68,7 +68,11 @@ expected_strings_by_page = {} for page_number, expected_strings in expected_strings_by_page.items(): missing = [value for value in expected_strings if value not in page_texts[page_number]] assert not missing, f"page {page_number} is missing {missing}" -page_sizes = [tuple(page.mediabox) for page in r.pages] +# mediabox is (left, bottom, right, top); reduce it to the width/height pair you declared +page_sizes = [ + (round(float(page.mediabox.width), 2), round(float(page.mediabox.height), 2)) + for page in r.pages +] ``` Confirm: page count matches the request; every page except those explicitly listed in diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md b/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md index 6224b09..734019e 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md @@ -22,9 +22,12 @@ for page in doc: print(page_number, round(span["bbox"][0]), round(span["bbox"][1]), span["text"]) - # Images + # Images. Pixmap keeps the image's own colorspace: CMYK/ICC pixmaps cannot be + # saved as PNG directly, so convert anything not Gray/RGB to RGB first. for i, info in enumerate(page.get_images(full=True), start=1): pix = fitz.Pixmap(doc, info[0]) + if pix.colorspace and pix.colorspace not in (fitz.csGRAY, fitz.csRGB): + pix = fitz.Pixmap(fitz.csRGB, pix) pix.save(f"img-p{page_number}-{i}.png") # Rasterize (for visual checks or OCR preprocessing) diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md b/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md index 61134da..299ead5 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md @@ -60,9 +60,37 @@ Encryption and forms: `writer.encrypt("pass", algorithm="AES-256")`. Open the result with `PdfReader(..., password="pass")`. Report that you set a password - the user must record it. -- AcroForm fields: `reader.get_fields()` to enumerate; `writer.update_page_form_field_values( - page, {"fieldname": "value"})` to fill. Flatten only on explicit request; it stops later - editing. +- AcroForm: fill fields on a cloned copy of the document, never on the readers' own pages: + + ```python + from pypdf import PdfReader, PdfWriter + + reader = PdfReader("form.pdf") + fields = reader.get_fields() or {} + assert fields, "this PDF has no AcroForm form fields" + + writer = PdfWriter() + writer.append(reader) # clones every page AND the catalog /AcroForm into the writer + + # update fields on the writer's page copies; a field widget can sit on any page, + # so pass the writer page that actually carries the field you are filling + writer.update_page_form_field_values( + writer.pages[0], + {"applicant_name": "Ada Byron"}, + ) + + with open("filled.pdf", "wb") as f: + writer.write(f) + + check = PdfReader("filled.pdf") + value = str((check.get_fields() or {}).get("applicant_name", {}).get("/V", "")) + assert value.strip("/") == "Ada Byron" + ``` + + A freshly constructed `PdfWriter` is empty: `append` (or `clone_document_from_reader`) must + copy the pages and the `/AcroForm` dictionary before any `update_page_form_field_values` + call, or the write fails or silently produces a formless file. Flatten only on explicit + request; it stops later editing. ## Rules diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md index e52f3cf..0ca131b 100644 --- a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md @@ -4,15 +4,26 @@ ```python from pptx import Presentation +from pptx.enum.shapes import MSO_SHAPE_TYPE + +def iter_shapes(shapes): + """Walk shapes recursively so content nested inside group shapes is counted too.""" + for shape in shapes: + if shape.shape_type == MSO_SHAPE_TYPE.GROUP: + yield from iter_shapes(shape.shapes) + else: + yield shape + prs = Presentation("input.pptx") print("slide size:", prs.slide_width, prs.slide_height) for i, slide in enumerate(prs.slides): layout = slide.slide_layout.name title = slide.shapes.title.text_frame.text if slide.shapes.title is not None else "" notes = slide.notes_slide.notes_text_frame.text if slide.has_notes_slide else "" - n_tables = sum(1 for sh in slide.shapes if sh.has_table) - n_charts = sum(1 for sh in slide.shapes if sh.has_chart) - n_pics = sum(1 for sh in slide.shapes if sh.shape_type == 13) + shapes = list(iter_shapes(slide.shapes)) # flattened; groups are common in template decks + n_tables = sum(1 for sh in shapes if sh.has_table) + n_charts = sum(1 for sh in shapes if sh.has_chart) + n_pics = sum(1 for sh in shapes if sh.shape_type == MSO_SHAPE_TYPE.PICTURE) print(i, layout, repr(title[:40]), "tables:", n_tables, "charts:", n_charts, "pics:", n_pics, "notes:", len(notes)) ``` @@ -27,10 +38,54 @@ layouts where the title placeholder is missing.) | Shape crosses the slide edge | compare all four shape bounds with the slide bounds | move or resize the shape | | Text is clipped or overflows its box | render every slide and inspect right/left and bottom/vertical fit; shape bounds do not measure laid-out text | reflow, resize the box, or reduce text/font size, then render again | | Everything shifted | slide size changed between sources | normalize slide size or re-layout on the target size | -| Fonts look wrong elsewhere | non-embedded fonts (pptx rarely embeds) | report referenced fonts (`run.font.name`) | +| Fonts look wrong elsewhere | non-embedded fonts (pptx rarely embeds) | report every effective font via the resolution chain below, not just explicit `run.font.name` values | | File will not open | broken ZIP / part mismatch | same programmatic health check as DOCX: `zipfile.testzip()`, parse every `.xml` part | | Pictures blank | media parts missing or rels broken | verify `ppt/media/*` present and slide rels reference them | +## Font triage with inheritance + +Most template decks set no explicit `run.font.name`; the effective face is inherited from the +placeholder, layout, master, or theme. Resolve what you can and name the fallback explicitly: + +```python +import re +from pptx import Presentation + +prs = Presentation("deck.pptx") + +# 1. Theme fonts are the final fallback for inherited text. +theme_xml = next( + part.blob.decode("utf-8", "ignore") + for part in prs.part.package.iter_parts() + if str(part.partname).startswith("/ppt/theme/") +) +theme = { + "major (headings)": re.search(r'\s*\s*", match.group(1) if match else "(not set)") + +# 2. Per run: explicit value, else paragraph defaults, else report as inherited. +for i, slide in enumerate(prs.slides): + for shape in slide.shapes: + if not shape.has_text_frame: + continue + for paragraph in shape.text_frame.paragraphs: + for run in paragraph.runs: + if run.font.name: + source = "run" + elif paragraph.font.name: + source = "paragraph defaults" + else: + source = "inherited (placeholder/layout/master chain, theme fallback)" + print(i, shape.name, repr(run.text[:20]), source) +``` + +python-pptx does not evaluate the full placeholder -> layout -> master inheritance chain; when +a run reports inherited, list the theme fallback above and, if the exact face matters, check +the layout and master placeholder of the same index for an explicit ``. + ## Text-fit verification `python-pptx` exposes a text box's geometry, not the renderer's final glyph and line layout. diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md b/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md index fd1b816..e74e257 100644 --- a/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md @@ -51,8 +51,17 @@ prs.save("input-edited.pptx") 3. For formatted text, change `run.text` only when the target is wholly inside one run. Assigning `paragraph.text` or `text_frame.text` rebuilds runs and can discard run formatting and links. If the target spans runs, stop and make an explicitly reviewed run/XML edit. -4. Table cells: `table.cell(r, c).text = ...`; keep the change inside the cell's text frame so - its formatting survives. +4. Table cells are edited at run level exactly like shape text: iterate + `table.cell(r, c).text_frame.paragraphs` and change `run.text`. Assigning `cell.text` + (or `.text` on the text frame) rebuilds the frame and discards per-run formatting and + hyperlinks: + + ```python + cell = table.cell(2, 1) + hits = [run for p in cell.text_frame.paragraphs for run in p.runs if old in run.text] + assert len(hits) == 1, "target is duplicated or split across runs in this cell" + hits[0].text = hits[0].text.replace(old, new, 1) + ``` 5. Chart data: `chart.replace_data(CategoryChartData(...))` updates the embedded workbook and the plot together - do not hand-edit the XML series. 6. Reordering slides means moving the underlying `sldIdLst` entries; do it only on request and diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md b/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md index 3bea9bf..ad65dd6 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md @@ -27,6 +27,7 @@ python -c "import openpyxl; print(openpyxl.__version__)" | Edit cells, add sheets, fix formatting | [references/edit.md](references/edit.md) | | Build a new workbook (data + formulas + chart) | [references/create.md](references/create.md) | | CSV/TSV in or out, messy data cleanup | [references/csv.md](references/csv.md) | +| Conditional formatting, structured tables, pivot-style aggregation | [references/formatting.md](references/formatting.md) | ## Step 2 - Contracts that always apply diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md index 4ba0374..749ab6c 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md @@ -14,9 +14,19 @@ with open("input.csv", newline="", encoding="utf-8-sig") as f: # utf-8-sig str - Always pass `newline=""` to `open` on every platform - it is the documented requirement, not a style choice. -- Sniff the dialect when provenance is unknown: - `csv.Sniffer().sniff(f.read(2048))` then `f.seek(0)`; semicolon-delimited exports are common - in European locales. +- Sniff the dialect when provenance is unknown, and pass the detected dialect to the reader - + seeking back alone does not reconfigure it, so semicolon exports would still parse as comma: + + ```python + with open("input.csv", newline="", encoding="utf-8-sig") as f: + sample = f.read(2048) + f.seek(0) + dialect = csv.Sniffer().sniff(sample) # raises csv.Error on ambiguous input + reader = csv.DictReader(f, dialect=dialect) + for i, row in enumerate(reader): + if i >= 5: break + print(row) + ``` - Never trust inferred dtypes in CSV: everything is a string. Convert explicitly with `try/except ValueError` per column and report counts of parse failures rather than dropping rows silently. diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md index 69c3c8c..e1f7bf6 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md @@ -40,6 +40,29 @@ for formula_ws in wb.worksheets: wb.save("input-edited.xlsx") ``` +- Before editing an unknown workbook, detect parts an openpyxl load/save round trip silently + drops (slicers, pivot caches, power-query connections are the common casualties). Save a + copy to memory, compare archive contents, and report the loss before overwriting the file: + + ```python + import zipfile + from io import BytesIO + + def round_trip_losses(path, **load_options): + with zipfile.ZipFile(path) as z: + before = set(z.namelist()) + wb = openpyxl.load_workbook(path, **load_options) # same options as the real edit + buf = BytesIO() + wb.save(buf) + with zipfile.ZipFile(buf) as z: + return sorted(before - set(z.namelist())) + + losses = round_trip_losses("input.xlsx") + if losses: + print("WARNING: saving with openpyxl will drop:", losses) + # report to the user and get confirmation before the first save + ``` + ## Rules - `insert_rows`/`delete_rows` move cells but do **not** rewrite range references for you. diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/formatting.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/formatting.md new file mode 100644 index 0000000..f05aa47 --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/formatting.md @@ -0,0 +1,95 @@ +# Conditional formatting, tables, and pivot-style aggregation + +## Conditional formatting (openpyxl) + +Rules attach to a range and survive openpyxl round trips. Scopes smaller than the whole +column keep files fast and avoid formatting ghost rows: + +```python +from openpyxl.formatting.rule import CellIsRule, ColorScaleRule, DataBarRule, FormulaRule +from openpyxl.styles import Font, PatternFill + +red_fill = PatternFill(start_color="FFC7CE", end_color="FFC7CE", fill_type="solid") +red_font = Font(color="9C0006") +last = ws.max_row # real data boundary, not the column + +# value-based rule +ws.conditional_formatting.add( + f"D2:D{last}", + CellIsRule(operator="lessThan", formula=["0"], fill=red_fill, font=red_font), +) + +# whole-row highlight: FormulaRule anchored with $ on the key column +ws.conditional_formatting.add(f"A2:F{last}", FormulaRule(formula=["$D2<0"], fill=red_fill)) + +# gradient and data bars for magnitude scanning +ws.conditional_formatting.add( + f"C2:C{last}", + ColorScaleRule(start_type="min", start_color="FFFFFF", end_type="max", end_color="63BE7B"), +) +ws.conditional_formatting.add( + f"E2:E{last}", + DataBarRule(start_type="min", end_type="max", color="638EC6"), +) +``` + +- FormulaRule formulas are US-locale and relative to the range's top-left cell - `$D2` (lock + column, free row) is what makes the whole-row pattern work. +- Multiple rules on one range evaluate by priority; if exactly one should apply, set + `stopIfTrue=True` on the earlier rules. + +## Structured tables + +A real Table gives filter UI, banded styling, and structured references: + +```python +from openpyxl.worksheet.table import Table, TableStyleInfo + +tbl = Table(displayName="TData", ref=f"A1:F{ws.max_row}") # name has no spaces +tbl.tableStyleInfo = TableStyleInfo(name="TableStyleMedium9", showRowStripes=True) +ws.add_table(tbl) +``` + +One Table per sheet region; the ref must cover the headers. Do not also draw manual borders +over a Table range. + +## Pivot-style aggregation - the honest contract + +**openpyxl cannot create pivot tables.** It preserves existing ones on a load/save round trip, +but building the pivot cache from scratch is not supported. Offer these routes and say which +one you took: + +1. **Formula sheet (live, recalculates)** - the default. A `SUMIFS`/`COUNTIFS`/`AVERAGEIFS` + grid keyed on a unique-values column reproduces most pivot outputs and stays a formula + per contract rule 1: + + ```python + ws2 = wb.create_sheet("ByRegion") + ws2.append(["Region", "Units", "Revenue"]) + regions = sorted({r[0] for r in ws.iter_rows(min_row=2, min_col=1, values_only=True) if r[0]}) + for i, region in enumerate(regions, start=2): + ws2.cell(row=i, column=1, value=region) + ws2.cell(row=i, column=2, value=f'=SUMIF(Data!A:A,A{i},Data!C:C)') + ws2.cell(row=i, column=3, value=f'=SUMIF(Data!A:A,A{i},Data!D:D)') + ``` + + Unique values themselves are formulas only with array/dynamic functions - extracting them + in Python (as above) and writing them as values is the accepted split; the aggregates stay + live. + +2. **Frozen pivot values (Python-side grouping)** - when the user wants a one-shot analysis + report, not a living workbook. Group in pure Python (or pandas if already installed), + write values, and **label the sheet** ("values as of generation, not recalculated"). + +3. **User's Excel/template pivot** - when the workbook already has slicers or a pivot the + user maintains, edit around it and re-run the `round_trip_losses` check from + [edit.md](edit.md) before saving. + +## Postcheck additions + +1. Re-open and count `ws.conditional_formatting` ranges; confirm the intended ranges exist + and anchor rows match the data (a rule left on `D2:D1048576` from an earlier resize is a + defect). +2. Confirm Table names are unique workbook-wide and refs cover the header row. +3. For the formula-sheet route, assert the aggregate cells contain formula strings, per the + main postcheck. diff --git a/plugins/Hylouis233/document-skills/tests/docx_fixture.py b/plugins/Hylouis233/document-skills/tests/docx_fixture.py new file mode 100644 index 0000000..36d3e08 --- /dev/null +++ b/plugins/Hylouis233/document-skills/tests/docx_fixture.py @@ -0,0 +1,131 @@ +# Minimal runnable fixtures for the snippets called out in review: one file per format. +# Each script is self-contained, writes only scratch files into the current directory, +# and exits non-zero on failed assertions. Run from any scratch directory: +# python pdf_fixture.py (deps: reportlab, pypdf, pymupdf) +# python pptx_fixture.py (deps: python-pptx) +# python xlsx_fixture.py (deps: openpyxl) +# python docx_fixture.py (deps: python-docx, pymupdf, soffice on PATH) +import copy +import subprocess +import sys + +import fitz +from docx import Document +from docx.opc.constants import RELATIONSHIP_TYPE as RT +from docx.oxml import OxmlElement +from docx.oxml.ns import qn + +failures = [] + + +def check(name, cond, extra=""): + print(("PASS " if cond else "FAIL ") + name + ((" :: " + str(extra)) if not cond and extra else "")) + if not cond: + failures.append(name) + + +def list_number_num_id(doc): + """The numId that the ListNumber style binds to in this document part.""" + styles = doc.part.element.body.getparent() # document.xml root; styles live in another part + styles_part = doc.part.part_related_by(RT.STYLES) + for style in styles_part.element.findall(qn("w:style")): + if style.get(qn("w:styleId")) == "ListNumber": + numPr = style.find(qn("w:pPr") + "/" + qn("w:numPr")) + if numPr is not None: + return int(numPr.find(qn("w:numId")).get(qn("w:val"))) + raise LookupError("ListNumber style has no numPr") + + +def new_restart_num_id(doc, base_num_id): + """Clone base_num_id with a startOverride so the next list restarts at 1.""" + numbering = doc.part.part_related_by(RT.NUMBERING).element + source = next( + n for n in numbering.findall(qn("w:num")) + if n.get(qn("w:numId")) == str(base_num_id) + ) + clone = copy.deepcopy(source) + new_id = max(int(n.get(qn("w:numId"))) for n in numbering.findall(qn("w:num"))) + 1 + clone.set(qn("w:numId"), str(new_id)) + override = OxmlElement("w:lvlOverride") + override.set(qn("w:ilvl"), "0") + start = OxmlElement("w:startOverride") + start.set(qn("w:val"), "1") + override.append(start) + clone.append(override) + numbering.append(clone) + return new_id + + +def numbered_paragraph(doc, text, num_id): + p = doc.add_paragraph(text, style="List Number") + pPr = p._p.get_or_add_pPr() + numPr = OxmlElement("w:numPr") + ilvl = OxmlElement("w:ilvl") + ilvl.set(qn("w:val"), "0") + numId = OxmlElement("w:numId") + numId.set(qn("w:val"), str(num_id)) + numPr.append(ilvl) + numPr.append(numId) + pPr.append(numPr) + return p + + +# ---- document A: naive reuse of List Number (second list continues) ----------- +doc_a = Document() +doc_a.add_paragraph("First list") +for item in ("one", "two", "three"): + doc_a.add_paragraph(item, style="List Number") +doc_a.add_paragraph("Second list, same style") +for item in ("four", "five", "six"): + doc_a.add_paragraph(item, style="List Number") +doc_a.save("continuing.docx") + +# ---- document B: second list restarts via cloned numbering definition --------- +doc_b = Document() +doc_b.add_paragraph("First list") +for item in ("one", "two", "three"): + doc_b.add_paragraph(item, style="List Number") +doc_b.add_paragraph("Second list, restarted") +base_id = list_number_num_id(doc_b) +restart_id = new_restart_num_id(doc_b, base_id) +for item in ("four", "five", "six"): + numbered_paragraph(doc_b, item, restart_id) +doc_b.save("restarted.docx") + +# ---- structural assertions ------------------------------------------------------ +numbering_b = doc_b.part.part_related_by(RT.NUMBERING).element +nums_b = numbering_b.findall(qn("w:num")) +check("cloned num entry exists", any(n.get(qn("w:numId")) == str(restart_id) for n in nums_b)) +clone_entry = next(n for n in nums_b if n.get(qn("w:numId")) == str(restart_id)) +override = clone_entry.find(qn("w:lvlOverride")) +check("clone carries startOverride=1 at ilvl 0", + override is not None and override.get(qn("w:ilvl")) == "0" + and override.find(qn("w:startOverride")).get(qn("w:val")) == "1") +second_list_num_ids = [ + p._p.find(qn("w:pPr") + "/" + qn("w:numPr") + "/" + qn("w:numId")).get(qn("w:val")) + for p in doc_b.paragraphs[-3:] +] +check("restarted paragraphs reference the cloned numId", set(second_list_num_ids) == {str(restart_id)}, second_list_num_ids) + +# ---- rendered proof via LibreOffice -------------------------------------------- +subprocess.run( + ["soffice", "--headless", "--convert-to", "pdf", "--outdir", ".", "continuing.docx", "restarted.docx"], + check=True, capture_output=True, timeout=180, +) + + +def second_list_numbers(pdf_path): + text = " ".join(page.get_text() for page in fitz.open(pdf_path)) + # take the rendered numbers in front of the second list's item words + return [text.split(word)[0].split()[-1] for word in ("four", "five", "six")] + + +cont = second_list_numbers("continuing.pdf") +restart = second_list_numbers("restarted.pdf") +print("continuing.docx renders second list as:", cont) +print("restarted.docx renders second list as:", restart) +check("plain style reuse continues the sequence (negative control)", cont == ["4.", "5.", "6."], cont) +check("cloned definition restarts the second list at 1", restart == ["1.", "2.", "3."], restart) + +print("\n" + ("ALL DOCX FIXTURES PASSED" if not failures else f"{len(failures)} FAILURES: {failures}")) +sys.exit(0 if not failures else 1) diff --git a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py new file mode 100644 index 0000000..60f53fc --- /dev/null +++ b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py @@ -0,0 +1,107 @@ +# Minimal runnable fixtures for the snippets called out in review: one file per format. +# Each script is self-contained, writes only scratch files into the current directory, +# and exits non-zero on failed assertions. Run from any scratch directory: +# python pdf_fixture.py (deps: reportlab, pypdf, pymupdf) +# python pptx_fixture.py (deps: python-pptx) +# python xlsx_fixture.py (deps: openpyxl) +# python docx_fixture.py (deps: python-docx, pymupdf, soffice on PATH) +import sys + +import fitz +import pypdf +from reportlab.lib.pagesizes import A4 +from reportlab.pdfgen import canvas + +failures = [] + + +def check(name, cond, extra=""): + print(("PASS " if cond else "FAIL ") + name + ((" :: " + str(extra)) if not cond and extra else "")) + if not cond: + failures.append(name) + + +# ---- build a 2-page A4 PDF with one AcroForm text field on page 1 ------------- +c = canvas.Canvas("form.pdf", pagesize=A4) +c.setFont("Helvetica", 16) +c.drawString(72, 780, "Application form") +c.acroForm.textfield(name="applicant_name", x=72, y=740, width=260, height=20, borderWidth=0) +c.showPage() +c.setFont("Helvetica", 16) +c.drawString(72, 780, "Second page content") +c.save() + +# ---- SKILL.md postcheck snippet: width/height pairs from the 4-coordinate box -- +r = pypdf.PdfReader("form.pdf") +page_sizes = [ + (round(float(page.mediabox.width), 2), round(float(page.mediabox.height), 2)) + for page in r.pages +] +A4_TOLERANCE = 0.5 # reportlab A4 is 595.28 x 841.89 after rounding; compare with tolerance +check( + "mediabox width/height is A4 on every page", + all(abs(w - 595.2755) < A4_TOLERANCE and abs(h - 841.8897) < A4_TOLERANCE for w, h in page_sizes), + page_sizes, +) + +# ---- transform.md AcroForm snippet: clone into writer, fill on writer pages ---- +from pypdf import PdfReader, PdfWriter + +reader = PdfReader("form.pdf") +fields = reader.get_fields() or {} +check("source form has the expected field", "applicant_name" in fields, list(fields)) + +writer = PdfWriter() +writer.append(reader) # clones pages AND catalog /AcroForm +writer.update_page_form_field_values( + writer.pages[0], + {"applicant_name": "Ada Byron"}, +) +with open("filled.pdf", "wb") as f: + writer.write(f) + +check_r = PdfReader("filled.pdf") +check("filled file keeps both pages", len(check_r.pages) == 2, len(check_r.pages)) +value = str((check_r.get_fields() or {}).get("applicant_name", {}).get("/V", "")) +check("field value round-trips", value.strip("/") == "Ada Byron", repr(value)) + +# ---- transform.md watermark snippet ------------------------------------------- +from pypdf import PdfReader as R2 + +stamp_src = canvas.Canvas("stamp.pdf", pagesize=A4) +stamp_src.setFont("Helvetica", 40) +stamp_src.setFillAlpha(0.35) +stamp_src.drawString(160, 400, "DRAFT") +stamp_src.save() + +stamp = R2("stamp.pdf").pages[0] +stamp_text = (stamp.extract_text() or "").strip() +reader = R2("form.pdf") +expected_sizes = [(round(float(p.mediabox.width), 2), round(float(p.mediabox.height), 2)) for p in reader.pages] +writer = PdfWriter() +for page in reader.pages: + page.merge_page(stamp) + writer.add_page(page) +with open("watermarked.pdf", "wb") as f: + writer.write(f) + +verify = R2("watermarked.pdf") +check("watermark written and page count kept", len(verify.pages) == 2, len(verify.pages)) +check( + "watermark page sizes unchanged", + [(round(float(p.mediabox.width), 2), round(float(p.mediabox.height), 2)) for p in verify.pages] == expected_sizes, +) +check("stamp text present on every page", all(stamp_text in (p.extract_text() or "") for p in verify.pages)) + +# ---- extract.md CMYK conversion snippet --------------------------------------- +pix = fitz.Pixmap(fitz.csCMYK, fitz.IRect(0, 0, 24, 24)) # CMYK pixmap like a CMYK PDF image +converted = fitz.Pixmap(fitz.csRGB, pix) if pix.colorspace not in (fitz.csGRAY, fitz.csRGB) else pix +converted.save("cmyk-converted.png") +import os + +check("CMYK pixmap converts to a saved PNG", os.path.getsize("cmyk-converted.png") > 0) +rgb = fitz.Pixmap("cmyk-converted.png") +check("converted pixmap is RGB", "RGB" in str(rgb.colorspace), rgb.colorspace) + +print("\n" + ("ALL PDF FIXTURES PASSED" if not failures else f"{len(failures)} FAILURES: {failures}")) +sys.exit(0 if not failures else 1) diff --git a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py new file mode 100644 index 0000000..a0e3faa --- /dev/null +++ b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py @@ -0,0 +1,161 @@ +# Minimal runnable fixtures for the snippets called out in review: one file per format. +# Each script is self-contained, writes only scratch files into the current directory, +# and exits non-zero on failed assertions. Run from any scratch directory: +# python pdf_fixture.py (deps: reportlab, pypdf, pymupdf) +# python pptx_fixture.py (deps: python-pptx) +# python xlsx_fixture.py (deps: openpyxl) +# python docx_fixture.py (deps: python-docx, pymupdf, soffice on PATH) +import sys + +from pptx import Presentation +from pptx.dml.color import RGBColor +from pptx.enum.shapes import MSO_SHAPE_TYPE +from pptx.util import Inches, Pt + +failures = [] + + +def check(name, cond, extra=""): + print(("PASS " if cond else "FAIL ") + name + ((" :: " + str(extra)) if not cond and extra else "")) + if not cond: + failures.append(name) + + +# ---- build the deck ------------------------------------------------------------ +prs = Presentation() +slide = prs.slides.add_slide(prs.slide_layouts[5]) # blank + +box = slide.shapes.add_textbox(Inches(1), Inches(1), Inches(8), Inches(1)) +tf = box.text_frame +p = tf.paragraphs[0] +r1 = p.add_run() +r1.text = "old wording" +r2 = p.add_run() +r2.text = " linked part" +r2.font.italic = True +r2.font.color.rgb = RGBColor(0xC0, 0x00, 0x00) +r2.hyperlink.address = "https://example.com/docs" + +table_shape = slide.shapes.add_table(2, 2, Inches(1), Inches(2.5), Inches(6), Inches(1)) +table = table_shape.table +cell = table.cell(0, 1) +ctf = cell.text_frame +cp = ctf.paragraphs[0] +cr = cp.add_run() +cr.text = "old cell text" +cr.font.bold = True +cr.font.color.rgb = RGBColor(0x00, 0x70, 0xC0) + +prs.save("input.pptx") + +# ---- edit.md snippet: single-shape run replace keeps styling and hyperlink ----- +prs = Presentation("input.pptx") +old, new = "old wording", "new wording" +candidates = [] +for i, s in enumerate(prs.slides): + for shape in s.shapes: + if shape.has_text_frame and old in shape.text_frame.text: + candidates.append((i, shape.name, shape)) +assert len(candidates) == 1 +_, _, target_shape = candidates[0] +tf = target_shape.text_frame +run_hits = [run for par in tf.paragraphs for run in par.runs if old in run.text] +assert len(run_hits) == 1 +run_hits[0].text = run_hits[0].text.replace(old, new, 1) + +edited_link = [run for par in tf.paragraphs for run in par.runs if run.hyperlink.address] +check("run replace keeps the other run's hyperlink", len(edited_link) == 1 and edited_link[0].hyperlink.address == "https://example.com/docs") +styled = [run for par in tf.paragraphs for run in par.runs if run.font.italic] +check("run replace keeps sibling run styling", len(styled) == 1 and styled[0].font.color.rgb == RGBColor(0xC0, 0x00, 0x00)) +prs.save("edited.pptx") + +# ---- edit.md snippet: table cell edited at run level --------------------------- +prs2 = Presentation("input.pptx") +old_cell, new_cell = "old cell text", "new cell text" +tbl = next(sh for sh in prs2.slides[0].shapes if sh.has_table).table +cell = tbl.cell(0, 1) +hits = [run for par in cell.text_frame.paragraphs for run in par.runs if old_cell in run.text] +assert len(hits) == 1, "target is duplicated or split across runs in this cell" +hits[0].text = hits[0].text.replace(old_cell, new_cell, 1) + +prs2.save("cell-edited.pptx") +prs3 = Presentation("cell-edited.pptx") +cell3 = next(sh for sh in prs3.slides[0].shapes if sh.has_table).table.cell(0, 1) +after_runs = [run for par in cell3.text_frame.paragraphs for run in par.runs] +check("cell run edit keeps bold", any(r.font.bold for r in after_runs)) +check("cell run edit keeps color", any(r.font.color and r.font.color.rgb == RGBColor(0x00, 0x70, 0xC0) for r in after_runs)) +check("cell run edit changed the text", cell3.text_frame.text == "new cell text") + +# the dangerous variant for contrast: assigning cell.text drops run properties +prs4 = Presentation("input.pptx") +tbl4 = next(sh for sh in prs4.slides[0].shapes if sh.has_table).table +tbl4.cell(0, 1).text = "new cell text" +prs4.save("cell-flattened.pptx") +prs5 = Presentation("cell-flattened.pptx") +flat_runs = [run for par in next(sh for sh in prs5.slides[0].shapes if sh.has_table).table.cell(0, 1).text_frame.paragraphs for run in par.runs] +check("cell.text assignment is proven lossy (negative control)", not any(r.font.bold for r in flat_runs)) + +# ---- analyze.md snippet: grouped-shape walker ---------------------------------- +from pptx.oxml import parse_xml + +prs6 = Presentation() +slide6 = prs6.slides.add_slide(prs6.slide_layouts[5]) +pic_holder = slide6.shapes.add_textbox(Inches(1), Inches(1), Inches(4), Inches(1)) +pic_holder.text_frame.text = "nested member" + +GRP = ( + '' + "" + "" + "" + '' + '' + "" +) + + +def iter_shapes(shapes): + for shape in shapes: + if shape.shape_type == MSO_SHAPE_TYPE.GROUP: + yield from iter_shapes(shape.shapes) + else: + yield shape + + +sp_element = slide6.shapes[-1]._element # the textbox; layout 5 still carries a Title placeholder +group_element = parse_xml(GRP) +sp_element.getparent().replace(sp_element, group_element) +group_element.append(sp_element) + +flat = list(iter_shapes(slide6.shapes)) +check( + "walker finds the shape nested in the group", + any(getattr(sh, "text_frame", None) is not None and sh.text_frame.text == "nested member" for sh in flat), +) +check( + "top-level shapes list hides the nested member (negative control)", + not any(getattr(sh, "text_frame", None) is not None and sh.text_frame.text == "nested member" for sh in slide6.shapes), +) + +# ---- analyze.md snippet: theme font resolution --------------------------------- +import re + +prs7 = Presentation("input.pptx") +theme_xml = next( + part.blob.decode("utf-8", "ignore") + for part in prs7.part.package.iter_parts() + if str(part.partname).startswith("/ppt/theme/") +) +major = re.search(r'\s*\s*" +with zipfile.ZipFile("extended.xlsx", "w", zipfile.ZIP_DEFLATED) as zout: + for name, data in payload.items(): + zout.writestr(name, data) + + +def round_trip_losses(path, **load_options): + with zipfile.ZipFile(path) as z: + before = set(z.namelist()) + wb = openpyxl.load_workbook(path, **load_options) # same options as the real edit + buf = BytesIO() + wb.save(buf) + with zipfile.ZipFile(buf) as z: + return sorted(before - set(z.namelist())) + + +losses = round_trip_losses("extended.xlsx") +check("injected slicer-like part is detected as a loss", "xl/slicers/slicer1.xml" in losses, losses) +check("clean workbook reports no losses", round_trip_losses("plain.xlsx") == []) + +# and the edit itself still works after the warning path +wb2 = openpyxl.load_workbook("plain.xlsx") +wb2["Data"]["B2"] = "=B2*1" # formula stays a formula +wb2.save("edited.xlsx") +wb3 = openpyxl.load_workbook("edited.xlsx") +check("edited cell keeps a formula string", isinstance(wb3["Data"]["B2"].value, str) and wb3["Data"]["B2"].value.startswith("=")) + +print("\n" + ("ALL XLSX FIXTURES PASSED" if not failures else f"{len(failures)} FAILURES: {failures}")) +sys.exit(0 if not failures else 1) From 2f0900859cea2cd7e1ffa1e24f2e3fb14c964184 Mon Sep 17 00:00:00 2001 From: Hylouis233 Date: Sun, 16 Aug 2026 01:08:17 +0800 Subject: [PATCH 04/34] Fix round-3 review findings on f803513 - xlsx edit: round_trip_changes() now also detects extensions stripped from retained parts (x14/extLst markers), not just dropped archive members - docx scenes: signature blocks get row-level cantSplit + keep_with_next on all rows (rendered fixture proves the table stays on one page) - docx cjk: tofu postcheck switched to glyph-coverage (fontTools cmap) since text extraction cannot detect missing-glyph boxes; style snippet now defines font names before use (NameError fix) - pptx edit: locator candidate collection recurses into groups with a stable nested path for the uniqueness assertion - xlsx formatting: aggregation formulas build sheet refs from ws.title (quoting when needed) instead of hard-coded Data! - xlsx read: profiles every sheet by default, not just sheetnames[0] - pdf create: escape() rule for plain text into Paragraph; fixture shows unescaped markup silently swallows <...> runs Fixtures extended to 44 assertions, all passing locally --- .../skills/docx/references/cjk.md | 30 +++++-- .../skills/docx/references/scenes.md | 21 ++++- .../skills/pdf/references/create.md | 13 +++ .../skills/pptx/references/edit.md | 21 +++-- .../skills/xlsx/references/edit.md | 36 +++++--- .../skills/xlsx/references/formatting.md | 13 ++- .../skills/xlsx/references/read.md | 49 ++++++----- .../document-skills/tests/docx_fixture.py | 39 +++++++++ .../document-skills/tests/pdf_fixture.py | 37 ++++++++ .../document-skills/tests/pptx_fixture.py | 29 +++++++ .../document-skills/tests/xlsx_fixture.py | 85 ++++++++++++++++--- 11 files changed, 310 insertions(+), 63 deletions(-) diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md b/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md index 249422b..3d21035 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md @@ -9,7 +9,9 @@ LibreOffice" report almost always means the east-asian face was never set. ```python from docx.oxml.ns import qn -def set_fonts(run, latin="Times New Roman", east_asian="宋体"): +LATIN, EAST_ASIAN = "Times New Roman", "宋体" # define both names once, up front + +def set_fonts(run, latin=LATIN, east_asian=EAST_ASIAN): run.font.name = latin # writes w:ascii + w:hAnsi rPr = run._element.get_or_add_rPr() rFonts = rPr.get_or_add_rFonts() @@ -17,14 +19,12 @@ def set_fonts(run, latin="Times New Roman", east_asian="宋体"): # Style level: do the same on the style so body text inherits it style = doc.styles["Normal"] -style.font.name = latin -style.element.get_or_add_rPr().get_or_add_rFonts().set(qn("w:eastAsia"), east_asian) +style.font.name = LATIN +style.element.get_or_add_rPr().get_or_add_rFonts().set(qn("w:eastAsia"), EAST_ASIAN) ``` Convention for mixed-script body text: CJK glyphs from the east-asian face, digits and Latin -from a Latin face (Times New Roman or Arial). Both slots set = deterministic rendering. - -## Chinese font-size table (字号) +from a Latin face (Times New Roman or Arial). Both slots set = deterministic rendering.## Chinese font-size table (字号) Word's Chinese UI names map to point sizes; scripts must use the points: @@ -91,6 +91,20 @@ section.left_margin, section.right_margin = Cm(2.8), Cm(2.6) 1. Re-open the output and assert the east-asian slot is set on body runs and styles (`.get(qn("w:eastAsia"))` is not None) - not just `run.font.name`. -2. Render with soffice and confirm no tofu (missing-glyph boxes) by extracting text from the - rendered PDF and comparing against the source strings. +2. Tofu (missing-glyph boxes) cannot be detected by text extraction: the rendered PDF's text + layer keeps the original codepoint while the glyph is a box, so "extracted text matches" + proves nothing about rendering. Check glyph coverage directly instead - with fontTools + (`pip install fonttools`) when the referenced faces can be located: + + ```python + from fontTools.ttLib import TTFont + + cmaps = [TTFont(font_path).getBestCmap() for font_path in referenced_font_files] + emitted = {ch for ch in source_text if ord(ch) >= 0x2E80} # CJK and beyond + missing = {ch for ch in emitted if not any(ord(ch) in cmap for cmap in cmaps)} + assert not missing, f"codepoints with no glyph in any referenced font: {sorted(missing)}" + ``` + + When font files cannot be located, rasterize the rendered pages with PyMuPDF and inspect + the images visually, and say in the report that glyph rendering was not machine-verified. 3. Confirm `firstLineChars` survived on body paragraphs if the indent was requested. diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/scenes.md b/plugins/Hylouis233/document-skills/skills/docx/references/scenes.md index 12fce1f..b149353 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/scenes.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/scenes.md @@ -59,8 +59,25 @@ numbered 条款 -> 签署 block. - Cross-references as literal text ("见第 5.2 条"); after edits, grep every 第 X 条 reference and verify the target still exists at that number. - Signature block: a borderless 2-column table (甲方/乙方 rows for 签字、盖章、日期) at the - end; set `keep_with_next = True` on the paragraphs right before it so the block never - splits across pages, and re-check in the rendered PDF. + end. `keep_with_next` on preceding paragraphs alone does not stop table rows from splitting + across pages - apply the row-level guard to the table itself: + + ```python + from docx.oxml import OxmlElement + from docx.oxml.ns import qn + + def keep_table_together(table): + for row in table.rows: + trPr = row._tr.get_or_add_trPr() + if trPr.find(qn("w:cantSplit")) is None: + trPr.append(OxmlElement("w:cantSplit")) # a row never splits mid-row + for row in table.rows[:-1]: + for cell in row.cells: + for par in cell.paragraphs: + par.paragraph_format.keep_with_next = True # row sticks to the next row + ``` + + Then verify in the rendered PDF that the whole block landed on one page. - Verification beyond the standard postcheck: every defined term defined once; every cross-reference resolves; signature block on one page in the rendered PDF. diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/create.md b/plugins/Hylouis233/document-skills/skills/pdf/references/create.md index 2e96f98..0b91e67 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/references/create.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/create.md @@ -55,6 +55,19 @@ doc.build(story) ## Rules +- **Escape plain text before Paragraph**: `Paragraph` parses its string as XML-ish markup, so + user-provided prose containing `&` or `<` raises a parse error or renders wrong. Escape + everything that is data; use markup only for strings you authored as markup: + + ```python + from xml.sax.saxutils import escape + + def para(text, style): + return Paragraph(escape(text), style) # & < > become entity-safe + ``` + + The same applies to `ListItem(Paragraph(...))` and to table cell strings when they flow + through `Paragraph`. - **Exactly-one-page constraint**: after `build`, run the postcheck. Over budget -> reduce `leading`, tighten `spaceBefore/After`, cut rows - in that order of preference. Under budget is usually fine; add a spacer or scale the title block. diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md b/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md index e74e257..c3668d7 100644 --- a/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md @@ -4,6 +4,7 @@ ```python from pptx import Presentation +from pptx.enum.shapes import MSO_SHAPE_TYPE prs = Presentation("input.pptx") @@ -11,17 +12,27 @@ old, new = "old wording", "new wording" slide_index = None # Set this and shape_name when repeated text is expected. shape_name = None +def iter_shapes(shapes, path=""): + """Yield (path, shape) for every shape, recursing into groups so text inside + grouped artwork is reachable; the path keeps the uniqueness check readable.""" + for shape in shapes: + here = f"{path}/{shape.name}" if path else shape.name + if shape.shape_type == MSO_SHAPE_TYPE.GROUP: + yield from iter_shapes(shape.shapes, here) + else: + yield here, shape + candidates = [] for i, slide in enumerate(prs.slides): - for shape in slide.shapes: - if slide_index is not None and i != slide_index: - continue + if slide_index is not None and i != slide_index: + continue + for path, shape in iter_shapes(slide.shapes): if shape_name is not None and shape.name != shape_name: continue if shape.has_text_frame and old in shape.text_frame.text: - candidates.append((i, shape.name, shape)) + candidates.append((i, path, shape)) -locations = [(i, name) for i, name, _ in candidates] +locations = [(i, path) for i, path, _ in candidates] assert len(candidates) == 1, f"expected one matching shape, found {locations}" _, _, target_shape = candidates[0] diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md index e1f7bf6..5364c04 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md @@ -40,29 +40,45 @@ for formula_ws in wb.worksheets: wb.save("input-edited.xlsx") ``` -- Before editing an unknown workbook, detect parts an openpyxl load/save round trip silently - drops (slicers, pivot caches, power-query connections are the common casualties). Save a - copy to memory, compare archive contents, and report the loss before overwriting the file: +- Before editing an unknown workbook, detect what an openpyxl load/save round trip silently + changes. Dropped parts (slicers, pivot caches, power-query connections) are only half the + risk: features stored *inside* a retained part, such as `x14` extension lists in + `xl/worksheets/sheet1.xml`, can be stripped while the archive member name stays. Compare + part contents too, not just names, and report both lists before overwriting the file: ```python import zipfile from io import BytesIO - def round_trip_losses(path, **load_options): + EXTENSION_MARKERS = (b"= 5: break - print(row) +# Profile EVERY sheet by default; only narrow when the task names a specific sheet. +for sheet_name in value_wb.sheetnames: + formula_ws = formula_wb[sheet_name] + value_ws = value_wb[sheet_name] + print(f"--- {sheet_name} --- dims:", value_ws.calculate_dimension()) -missing_cache_count = 0 -for formula_row, value_row in zip(formula_ws.iter_rows(), value_ws.iter_rows()): - for formula_cell, value_cell in zip(formula_row, value_row): - if (isinstance(formula_cell.value, str) - and formula_cell.value.startswith("=") - and value_cell.value is None): - missing_cache_count += 1 - if missing_cache_count <= 10: - print("formula without cached value:", formula_cell.coordinate, - formula_cell.value) -print("formulas without cached values:", missing_cache_count) + rows = value_ws.iter_rows(values_only=True) + header = next(rows, None) + print("header:", header) + for i, row in enumerate(rows): + if i >= 5: break + print(row) + + missing_cache_count = 0 + for formula_row, value_row in zip(formula_ws.iter_rows(), value_ws.iter_rows()): + for formula_cell, value_cell in zip(formula_row, value_row): + if (isinstance(formula_cell.value, str) + and formula_cell.value.startswith("=") + and value_cell.value is None): + missing_cache_count += 1 + if missing_cache_count <= 10: + print("formula without cached value:", formula_cell.coordinate, + formula_cell.value) + print("formulas without cached values:", missing_cache_count) formula_wb.close() value_wb.close() ``` ## Rules -- First pass always: sheet names, dimensions, header row, 5 sample rows. Report those before - any analysis. +- First pass always: sheet names, per-sheet dimensions, header row, 5 sample rows. Report + those before any analysis. Multi-sheet workbooks report every sheet - a profile that + silently covers only `sheetnames[0]` is incomplete. - `read_only=True` streams large files; you lose random access (`ws["B2"]` works but is slow in read_only mode - iterate instead). - `data_only=True` gives cached values. A file saved by a library (never opened in Excel) diff --git a/plugins/Hylouis233/document-skills/tests/docx_fixture.py b/plugins/Hylouis233/document-skills/tests/docx_fixture.py index 36d3e08..b4ea585 100644 --- a/plugins/Hylouis233/document-skills/tests/docx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/docx_fixture.py @@ -127,5 +127,44 @@ def second_list_numbers(pdf_path): check("plain style reuse continues the sequence (negative control)", cont == ["4.", "5.", "6."], cont) check("cloned definition restarts the second list at 1", restart == ["1.", "2.", "3."], restart) +# ---- scenes.md snippet: keep_table_together for the signature block ------------- +doc_c = Document() +doc_c.add_heading("Contract", level=1) +# fill most of page 1 so a tall signature table would otherwise straddle the page break +for _ in range(24): + doc_c.add_paragraph("Filler paragraph to push the signature block toward the page break. " * 3) +sig = doc_c.add_table(rows=4, cols=2) +labels = [("甲方（盖章）", "乙方（盖章）"), ("签字", "签字"), ("日期", "日期"), ("备注", "备注")] +for r, pair in enumerate(labels): + sig.cell(r, 0).text, sig.cell(r, 1).text = pair + + +def keep_table_together(table): + for row in table.rows: + trPr = row._tr.get_or_add_trPr() + if trPr.find(qn("w:cantSplit")) is None: + trPr.append(OxmlElement("w:cantSplit")) # a row never splits mid-row + for row in table.rows[:-1]: + for cell in row.cells: + for par in cell.paragraphs: + par.paragraph_format.keep_with_next = True # row sticks to the next row + + +keep_table_together(sig) +doc_c.save("signature.docx") +reopened = Document("signature.docx") +check("every signature row carries cantSplit", + all(row._tr.find(qn("w:trPr")) is not None and row._tr.find(qn("w:trPr")).find(qn("w:cantSplit")) is not None + for row in reopened.tables[0].rows)) +subprocess.run( + ["soffice", "--headless", "--convert-to", "pdf", "--outdir", ".", "signature.docx"], + check=True, capture_output=True, timeout=180, +) +pages_with_labels = [ + page.number for page in fitz.open("signature.pdf") + if "甲方（盖章）" in page.get_text() and "备注" in page.get_text() +] +check("rendered signature table stays on one page", len(pages_with_labels) == 1, pages_with_labels) + print("\n" + ("ALL DOCX FIXTURES PASSED" if not failures else f"{len(failures)} FAILURES: {failures}")) sys.exit(0 if not failures else 1) diff --git a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py index 60f53fc..2634489 100644 --- a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py @@ -103,5 +103,42 @@ def check(name, cond, extra=""): rgb = fitz.Pixmap("cmyk-converted.png") check("converted pixmap is RGB", "RGB" in str(rgb.colorspace), rgb.colorspace) +# ---- create.md rule: escape plain text before Paragraph ------------------------ +from reportlab.lib.pagesizes import A4 as A4_SIZE +from reportlab.lib.styles import getSampleStyleSheet +from reportlab.platypus import Paragraph, SimpleDocTemplate +from xml.sax.saxutils import escape + +MESSY = "R&D spend & \"pipeline\" > forecast" +try: + SimpleDocTemplate("escaped.pdf", pagesize=A4_SIZE).build( + [Paragraph(escape(MESSY), getSampleStyleSheet()["BodyText"])] + ) + build_error = "" +except Exception as exc: # unescaped markup typically raises a paraparser error + build_error = str(exc) +check("escaped messy text builds without paraparser error", build_error == "", build_error) +esc_text = " ".join(page.get_text() for page in fitz.open("escaped.pdf")) +check("escaped text extracts with original characters", + "R&D spend " in esc_text and "\"pipeline\"" in esc_text, esc_text[:120]) + +unescaped_failed = False +try: + SimpleDocTemplate("raw.pdf", pagesize=A4_SIZE).build( + [Paragraph(MESSY, getSampleStyleSheet()["BodyText"])] + ) +except Exception: + unescaped_failed = True +if unescaped_failed: + check("unescaped markup is proven dangerous (negative control)", True) +else: + # lenient inputs build but render mangled: markup is swallowed, entities reinterpreted + raw_text = " ".join(page.get_text() for page in fitz.open("raw.pdf")) + check( + "unescaped markup is proven dangerous (negative control)", + "" not in raw_text or "R&D;" in raw_text, + raw_text[:120], + ) + print("\n" + ("ALL PDF FIXTURES PASSED" if not failures else f"{len(failures)} FAILURES: {failures}")) sys.exit(0 if not failures else 1) diff --git a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py index a0e3faa..5cdb8d0 100644 --- a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py @@ -138,6 +138,35 @@ def iter_shapes(shapes): not any(getattr(sh, "text_frame", None) is not None and sh.text_frame.text == "nested member" for sh in slide6.shapes), ) +# ---- edit.md locator: candidate collection must recurse into groups ------------ +old_w, new_w = "nested member", "renamed member" + + +def iter_shapes_with_path(shapes, path=""): + for shape in shapes: + here = f"{path}/{shape.name}" if path else shape.name + if shape.shape_type == MSO_SHAPE_TYPE.GROUP: + yield from iter_shapes_with_path(shape.shapes, here) + else: + yield here, shape + + +candidates = [ + (i, p, sh) + for i, s in enumerate(prs6.slides) + for p, sh in iter_shapes_with_path(s.shapes) + if sh.has_text_frame and old_w in sh.text_frame.text +] +check("locator reaches text inside the group", len(candidates) == 1, [(i, p) for i, p, _ in candidates]) +check("locator reports a stable nested path", "/" in candidates[0][1], candidates[0][1]) +_, _, target = candidates[0] +target.text_frame.paragraphs[0].runs[0].text = new_w +prs6.save("group-edited.pptx") +prs_g = Presentation("group-edited.pptx") +found = [sh for sh in iter_shapes(prs_g.slides[0].shapes) + if getattr(sh, "text_frame", None) is not None and sh.text_frame.text == new_w] +check("group member edit persists after save", len(found) == 1) + # ---- analyze.md snippet: theme font resolution --------------------------------- import re diff --git a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py index 31b2513..b30693c 100644 --- a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py @@ -41,7 +41,7 @@ def check(name, cond, extra=""): naive = next(csv.DictReader(f)) check("default reader is proven wrong here (negative control)", list(naive.keys())[0] == "region;units;note", naive) -# ---- edit.md snippet: round_trip_losses detects dropped parts ------------------ +# ---- edit.md snippet: round_trip_changes detects dropped parts AND stripped extensions ---- from io import BytesIO wb = openpyxl.Workbook() @@ -51,29 +51,70 @@ def check(name, cond, extra=""): ws.append(["EU", 120]) wb.save("plain.xlsx") -# simulate an unsupported extension part (what a slicer/queries part looks like in the zip) with zipfile.ZipFile("plain.xlsx") as zin: - names = zin.namelist() - payload = {name: zin.read(name) for name in names} + payload = {name: zin.read(name) for name in zin.namelist()} + +# simulate an unsupported extension part (what a slicer/queries part looks like in the zip) payload["xl/slicers/slicer1.xml"] = b"" + +# simulate an in-part x14 extension that openpyxl will strip while keeping the part name +EXT = (b"" + b"") +payload["xl/worksheets/sheet1.xml"] = payload["xl/worksheets/sheet1.xml"].replace( + b"", EXT + b"") + with zipfile.ZipFile("extended.xlsx", "w", zipfile.ZIP_DEFLATED) as zout: for name, data in payload.items(): zout.writestr(name, data) +EXTENSION_MARKERS = (b" Date: Sun, 16 Aug 2026 11:05:18 +0800 Subject: [PATCH 05/34] Fix document skill round-3 review feedback --- .../skills/docx/references/review.md | 9 ++++- .../document-skills/skills/pdf/SKILL.md | 10 +++++- .../skills/pptx/references/analyze.md | 17 +++++---- .../document-skills/skills/xlsx/SKILL.md | 12 ++++++- .../skills/xlsx/references/csv.md | 11 +++++- .../skills/xlsx/references/formatting.md | 7 ++-- .../document-skills/tests/docx_fixture.py | 17 +++++++++ .../document-skills/tests/pdf_fixture.py | 11 ++++++ .../document-skills/tests/pptx_fixture.py | 21 +++++++++++ .../document-skills/tests/xlsx_fixture.py | 36 ++++++++++++++++++- 10 files changed, 137 insertions(+), 14 deletions(-) diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/review.md b/plugins/Hylouis233/document-skills/skills/docx/references/review.md index 369dce5..5dd1078 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/review.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/review.md @@ -17,6 +17,13 @@ import zipfile from lxml import etree path = "input.docx" +safe_xml_parser = etree.XMLParser( + load_dtd=False, + resolve_entities=False, + no_network=True, + huge_tree=False, + recover=False, +) with zipfile.ZipFile(path) as z: bad = z.testzip() assert bad is None, f"corrupt entry: {bad}" @@ -24,7 +31,7 @@ with zipfile.ZipFile(path) as z: assert "[Content_Types].xml" in names and "word/document.xml" in names for part in names: if part.endswith(('.xml', '.rels')): - etree.fromstring(z.read(part)) # raises on malformed XML + etree.fromstring(z.read(part), parser=safe_xml_parser) # malformed XML still raises ``` Then the SKILL.md postcheck (python-docx re-open, optional soffice PDF smoke test). diff --git a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md index e272a4c..1bc610a 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md @@ -49,8 +49,16 @@ file is not. ## Step 3 - Postcheck (mandatory) ```python +import os import pypdf -r = pypdf.PdfReader("output.pdf") + +output_path = "output.pdf" +password = os.environ.get("PDF_PASSWORD") +r = pypdf.PdfReader(output_path) +if r.is_encrypted: + if not password: + raise RuntimeError("set PDF_PASSWORD so the encrypted output can be postchecked") + r = pypdf.PdfReader(output_path, password=password) # wrong passwords fail here page_count = len(r.pages) page_texts = { number: (page.extract_text() or "").strip() diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md index 0ca131b..2c3fc65 100644 --- a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md @@ -63,8 +63,12 @@ theme = { "major (headings)": re.search(r'\s*\s*", match.group(1) if match else "(not set)") +theme_faces = { + kind: match.group(1) if match else "(not set)" + for kind, match in theme.items() +} +for kind, face in theme_faces.items(): + print("theme", kind, "->", face) # 2. Per run: explicit value, else paragraph defaults, else report as inherited. for i, slide in enumerate(prs.slides): @@ -74,12 +78,13 @@ for i, slide in enumerate(prs.slides): for paragraph in shape.text_frame.paragraphs: for run in paragraph.runs: if run.font.name: - source = "run" + face, source = run.font.name, "run" elif paragraph.font.name: - source = "paragraph defaults" + face, source = paragraph.font.name, "paragraph defaults" else: - source = "inherited (placeholder/layout/master chain, theme fallback)" - print(i, shape.name, repr(run.text[:20]), source) + face = f"major={theme_faces['major (headings)']}; minor={theme_faces['minor (body)']}" + source = "inherited candidate (verify placeholder/layout/master chain)" + print(i, shape.name, repr(run.text[:20]), "font:", face, "source:", source) ``` python-pptx does not evaluate the full placeholder -> layout -> master inheritance chain; when diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md b/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md index ad65dd6..12bf9ea 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md @@ -58,6 +58,10 @@ import sys if len(sys.argv) < 3: raise SystemExit("usage: python postcheck.py OUTPUT.xlsx EXPECTED_SHEET [...]") output_path, *expected_sheets = sys.argv[1:] +# Populate this whenever the task requested specific display formats. +expected_number_formats = { + # "Sales": {"D2": "#,##0.00", "E2": "yyyy-mm-dd"}, +} wb = openpyxl.load_workbook(output_path) print("sheets:", wb.sheetnames) missing = set(expected_sheets) - set(wb.sheetnames) @@ -67,9 +71,15 @@ for ws in wb.worksheets: formulas = [(c.coordinate, c.value) for row in ws.iter_rows() for c in row if isinstance(c.value, str) and c.value.startswith("=")] print(f"{ws.title} formula cells:", formulas[:10]) + for coordinate, expected_format in expected_number_formats.get(ws.title, {}).items(): + actual_format = ws[coordinate].number_format + assert actual_format == expected_format, ( + f"{ws.title}!{coordinate}: expected format {expected_format!r}, got {actual_format!r}" + ) wb.close() ``` Confirm: expected sheet names exist; used range matches expectations; intended formula cells -contain formula strings; number formats survive. Report what was verified and note that final +contain formula strings; every task-specific formatted cell is listed in +`expected_number_formats` and matches. Report what was verified and note that final rendered values require opening in a spreadsheet application. diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md index 749ab6c..88657b1 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md @@ -45,7 +45,16 @@ with open("output.csv", "w", newline="", encoding="utf-8") as f: ## Converting - CSV -> XLSX: read with `csv`, write with openpyxl; convert values to real types on the way - through (dates via `datetime.strptime` with the format actually observed). + through (dates via `datetime.strptime` with the format actually observed). Treat every + remaining CSV field as data, not a formula. In particular, force strings beginning with `=` + back to the string data type unless the user explicitly requested formula interpretation: + + ```python + def write_csv_field(cell, value): + cell.value = value + if isinstance(value, str) and value.startswith("="): + cell.data_type = "s" # openpyxl otherwise promotes it to an XLSX formula + ``` - XLSX -> CSV: `iter_rows(values_only=True)`; format numbers yourself only if the user needs a fixed display format - otherwise write raw values and say so. - Large CSV -> keep it CSV or move to SQLite/Parquet; loading it all into one sheet to diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/formatting.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/formatting.md index 627c87d..148e1ab 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/formatting.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/formatting.md @@ -69,8 +69,9 @@ one you took: ws2.append(["Region", "Units", "Revenue"]) def sheet_ref(sheet): - # quote the title only when it contains characters a formula would misread - return f"'{sheet.title}'!" if any(c in sheet.title for c in " !'") else f"{sheet.title}!" + # Excel escapes an apostrophe inside a quoted sheet title by doubling it. + escaped = sheet.title.replace("'", "''") + return f"'{escaped}'!" if any(c in sheet.title for c in " !'") else f"{escaped}!" src = sheet_ref(ws) # e.g. "Sales!" or "'Raw Data'!" regions = sorted({r[0] for r in ws.iter_rows(min_row=2, min_col=1, values_only=True) if r[0]}) @@ -89,7 +90,7 @@ one you took: write values, and **label the sheet** ("values as of generation, not recalculated"). 3. **User's Excel/template pivot** - when the workbook already has slicers or a pivot the - user maintains, edit around it and re-run the `round_trip_losses` check from + user maintains, edit around it and re-run the `round_trip_changes` check from [edit.md](edit.md) before saving. ## Postcheck additions diff --git a/plugins/Hylouis233/document-skills/tests/docx_fixture.py b/plugins/Hylouis233/document-skills/tests/docx_fixture.py index b4ea585..4bda6c4 100644 --- a/plugins/Hylouis233/document-skills/tests/docx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/docx_fixture.py @@ -14,6 +14,7 @@ from docx.opc.constants import RELATIONSHIP_TYPE as RT from docx.oxml import OxmlElement from docx.oxml.ns import qn +from lxml import etree failures = [] @@ -24,6 +25,22 @@ def check(name, cond, extra=""): failures.append(name) +# ---- review.md health check: external entities stay unresolved --------------- +safe_xml_parser = etree.XMLParser( + load_dtd=False, + resolve_entities=False, + no_network=True, + huge_tree=False, + recover=False, +) +hostile_xml = ( + b']>' + b'&xxe;' +) +parsed = etree.fromstring(hostile_xml, parser=safe_xml_parser) +check("DOCX XML parser leaves external entities unresolved", parsed.text is None and len(parsed) == 1) + + def list_number_num_id(doc): """The numId that the ListNumber style binds to in this document part.""" styles = doc.part.element.body.getparent() # document.xml root; styles live in another part diff --git a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py index 2634489..b542d7d 100644 --- a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py @@ -44,6 +44,17 @@ def check(name, cond, extra=""): page_sizes, ) +# ---- SKILL.md postcheck: encrypted output is reopened with its password ------- +encrypted_writer = pypdf.PdfWriter() +encrypted_writer.append(r) +encrypted_writer.encrypt("fixture-password") +with open("encrypted.pdf", "wb") as f: + encrypted_writer.write(f) +probe = pypdf.PdfReader("encrypted.pdf") +check("encrypted fixture is detected before page access", probe.is_encrypted) +encrypted_r = pypdf.PdfReader("encrypted.pdf", password="fixture-password") +check("password-authenticated postcheck can access every page", len(encrypted_r.pages) == 2) + # ---- transform.md AcroForm snippet: clone into writer, fill on writer pages ---- from pypdf import PdfReader, PdfWriter diff --git a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py index 5cdb8d0..d24d596 100644 --- a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py @@ -186,5 +186,26 @@ def iter_shapes_with_path(shapes, path=""): all(run.font.name is None for sh in prs7.slides[0].shapes if sh.has_text_frame for par in sh.text_frame.paragraphs for run in par.runs), ) +font_box = prs7.slides[0].shapes.add_textbox(Inches(1), Inches(4), Inches(4), Inches(1)) +font_paragraph = font_box.text_frame.paragraphs[0] +font_paragraph.font.name = "Paragraph Face" +paragraph_run = font_paragraph.add_run() +paragraph_run.text = "paragraph default" +explicit_run = font_paragraph.add_run() +explicit_run.text = "run override" +explicit_run.font.name = "Run Face" +detected_faces = [] +for run in font_paragraph.runs: + if run.font.name: + detected_faces.append((run.font.name, "run")) + elif font_paragraph.font.name: + detected_faces.append((font_paragraph.font.name, "paragraph defaults")) +check("font triage reports the run face and source", ("Run Face", "run") in detected_faces, detected_faces) +check( + "font triage reports the paragraph face and source", + ("Paragraph Face", "paragraph defaults") in detected_faces, + detected_faces, +) + print("\n" + ("ALL PPTX FIXTURES PASSED" if not failures else f"{len(failures)} FAILURES: {failures}")) sys.exit(0 if not failures else 1) diff --git a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py index b30693c..7badd09 100644 --- a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py @@ -41,6 +41,21 @@ def check(name, cond, extra=""): naive = next(csv.DictReader(f)) check("default reader is proven wrong here (negative control)", list(naive.keys())[0] == "region;units;note", naive) +# ---- csv.md conversion: formula-looking input remains literal text ----------- +formula_looking = '=HYPERLINK("https://example.invalid", "click")' +csv_wb = openpyxl.Workbook() +csv_cell = csv_wb.active["A1"] +csv_cell.value = formula_looking +csv_cell.data_type = "s" +csv_wb.save("csv-text.xlsx") +csv_reopened = openpyxl.load_workbook("csv-text.xlsx", data_only=False) +check("formula-looking CSV field keeps its exact text", csv_reopened.active["A1"].value == formula_looking) +check("formula-looking CSV field is not an XLSX formula", csv_reopened.active["A1"].data_type == "s") + +unsafe_wb = openpyxl.Workbook() +unsafe_wb.active["A1"] = formula_looking +check("plain assignment is proven unsafe (negative control)", unsafe_wb.active["A1"].data_type == "f") + # ---- edit.md snippet: round_trip_changes detects dropped parts AND stripped extensions ---- from io import BytesIO @@ -105,7 +120,8 @@ def round_trip_changes(path, **load_options): def sheet_ref(sheet): - return f"'{sheet.title}'!" if any(c in sheet.title for c in " !'") else f"{sheet.title}!" + escaped = sheet.title.replace("'", "''") + return f"'{escaped}'!" if any(c in sheet.title for c in " !'") else f"{escaped}!" ref = sheet_ref(src) @@ -115,13 +131,31 @@ def sheet_ref(sheet): wb_f.save("agg.xlsx") wb_g = openpyxl.load_workbook("agg.xlsx") check("formula references the real sheet name", wb_g["ByRegion"]["B2"].value == "=SUMIF('Raw Data'!A:A,A2,'Raw Data'!B:B)", wb_g["ByRegion"]["B2"].value) +apostrophe_sheet = wb_f.create_sheet("O'Brien") +apostrophe_sheet["A1"] = 1 +agg["B3"] = f"=SUM({sheet_ref(apostrophe_sheet)}A:A)" +wb_f.save("apostrophe-agg.xlsx") +apostrophe_formula = openpyxl.load_workbook("apostrophe-agg.xlsx")["ByRegion"]["B3"].value +check( + "quoted sheet reference doubles apostrophes", + apostrophe_formula == "=SUM('O''Brien'!A:A)", + apostrophe_formula, +) # and the edit itself still works after the warning path wb2 = openpyxl.load_workbook("plain.xlsx") wb2["Data"]["B2"] = "=B2*1" # formula stays a formula +wb2["Data"]["B2"].number_format = "#,##0.00" wb2.save("edited.xlsx") wb3 = openpyxl.load_workbook("edited.xlsx") check("edited cell keeps a formula string", isinstance(wb3["Data"]["B2"].value, str) and wb3["Data"]["B2"].value.startswith("=")) +expected_number_formats = {"Data": {"B2": "#,##0.00"}} +format_matches = all( + wb3[sheet][coordinate].number_format == expected + for sheet, cells in expected_number_formats.items() + for coordinate, expected in cells.items() +) +check("task-specific number format mapping is verified", format_matches) # ---- read.md snippet: multi-sheet profiles cover every sheet ---------------------- wb_h = openpyxl.Workbook() From 64049ebb7c7cbfdeca5f0bbd8b492dfe278dc939 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=98=E6=89=AC?= Date: Sun, 16 Aug 2026 11:15:20 +0800 Subject: [PATCH 06/34] Harden document reference edge cases --- .../document-skills/skills/pptx/references/analyze.md | 2 +- .../skills/xlsx/references/formatting.md | 7 ++++--- .../Hylouis233/document-skills/tests/pptx_fixture.py | 10 ++++++++++ .../Hylouis233/document-skills/tests/xlsx_fixture.py | 8 +++++++- 4 files changed, 22 insertions(+), 5 deletions(-) diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md index 2c3fc65..8e018ee 100644 --- a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md @@ -72,7 +72,7 @@ for kind, face in theme_faces.items(): # 2. Per run: explicit value, else paragraph defaults, else report as inherited. for i, slide in enumerate(prs.slides): - for shape in slide.shapes: + for shape in iter_shapes(slide.shapes): if not shape.has_text_frame: continue for paragraph in shape.text_frame.paragraphs: diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/formatting.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/formatting.md index 148e1ab..d1c3d0f 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/formatting.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/formatting.md @@ -69,11 +69,12 @@ one you took: ws2.append(["Region", "Units", "Revenue"]) def sheet_ref(sheet): - # Excel escapes an apostrophe inside a quoted sheet title by doubling it. + # Always quote: valid titles such as Q1-Data are ambiguous when left bare. + # Excel escapes an apostrophe inside a quoted title by doubling it. escaped = sheet.title.replace("'", "''") - return f"'{escaped}'!" if any(c in sheet.title for c in " !'") else f"{escaped}!" + return f"'{escaped}'!" - src = sheet_ref(ws) # e.g. "Sales!" or "'Raw Data'!" + src = sheet_ref(ws) # e.g. "'Sales'!" or "'Raw Data'!" regions = sorted({r[0] for r in ws.iter_rows(min_row=2, min_col=1, values_only=True) if r[0]}) for i, region in enumerate(regions, start=2): ws2.cell(row=i, column=1, value=region) diff --git a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py index d24d596..ee9a3d2 100644 --- a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py @@ -102,6 +102,7 @@ def check(name, cond, extra=""): slide6 = prs6.slides.add_slide(prs6.slide_layouts[5]) pic_holder = slide6.shapes.add_textbox(Inches(1), Inches(1), Inches(4), Inches(1)) pic_holder.text_frame.text = "nested member" +pic_holder.text_frame.paragraphs[0].runs[0].font.name = "Grouped Face" GRP = ( ' Date: Sun, 16 Aug 2026 12:04:19 +0800 Subject: [PATCH 07/34] fix(document-skills): address current review gaps --- plugins/Hylouis233/document-skills/README.md | 7 +- .../skills/docx/references/edit.md | 47 +++++- .../document-skills/skills/pdf/SKILL.md | 8 +- .../skills/pdf/references/extract.md | 17 ++- .../document-skills/skills/pptx/SKILL.md | 8 +- .../skills/pptx/references/analyze.md | 83 ++++++++--- .../skills/xlsx/references/edit.md | 27 ++-- .../skills/xlsx/references/formatting.md | 10 +- .../document-skills/tests/docx_fixture.py | 91 ++++++++++++ .../document-skills/tests/pdf_fixture.py | 30 ++++ .../document-skills/tests/pptx_fixture.py | 138 ++++++++++++++++-- .../document-skills/tests/xlsx_fixture.py | 52 ++++++- 12 files changed, 453 insertions(+), 65 deletions(-) diff --git a/plugins/Hylouis233/document-skills/README.md b/plugins/Hylouis233/document-skills/README.md index 030599f..4118cc1 100644 --- a/plugins/Hylouis233/document-skills/README.md +++ b/plugins/Hylouis233/document-skills/README.md @@ -75,9 +75,10 @@ versus what was assumed. The [`tests/`](tests/) directory ships one runnable fixture script per format covering the snippets with the worst silent-failure modes (PDF AcroForm clone-and-fill, watermark write, -CMYK pixmap conversion; PPTX run-preserving edits, table-cell run edits, grouped-shape walking; -XLSX dialect sniffing, round-trip part-loss detection; DOCX numbering restart rendered through -LibreOffice). Each script is self-contained and exits non-zero on failure. +CMYK conversion, soft-mask extraction; PPTX run-preserving edits, actual content extraction, +grouped-shape walking, per-master themes; XLSX dialect sniffing, independent extension-marker +loss, falsey aggregation categories; DOCX guarded cross-run replacement and numbering restart +rendered through LibreOffice). Each script is self-contained and exits non-zero on failure. ## Requirements diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/edit.md b/plugins/Hylouis233/document-skills/skills/docx/references/edit.md index e49e5e2..592b742 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/edit.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/edit.md @@ -10,9 +10,25 @@ paragraph text. ```python from docx import Document +from docx.oxml.ns import qn + +SAFE_RUN_CHILDREN = { + qn("w:rPr"), qn("w:t"), qn("w:tab"), qn("w:br"), qn("w:cr"), +} + +def unsafe_run_content(run): + unsafe = [] + for child in run._r: + # Assigning run.text can reconstruct text, tabs, and ordinary line breaks only. + typed_break = child.tag == qn("w:br") and child.get(qn("w:type")) not in ( + None, "textWrapping", + ) + if child.tag not in SAFE_RUN_CHILDREN or typed_break: + unsafe.append(child.tag.rsplit("}", 1)[-1]) + return unsafe def replace_across_runs(paragraph, old, new): - """Replace non-overlapping matches, including matches split across runs.""" + """Replace text-only matches; reject drawings, fields, and other lossy run content.""" if not old: raise ValueError("old must not be empty") @@ -33,11 +49,30 @@ def replace_across_runs(paragraph, old, new): spans.append((index, position, end)) position = end - # Work backwards so changing a later match cannot move an earlier match. - for start in reversed(starts): + matches = [] + for start in starts: end = start + len(old) first, first_start, _ = next(s for s in spans if s[1] <= start < s[2]) last, last_start, _ = next(s for s in spans if s[1] < end <= s[2]) + matches.append((start, end, first, first_start, last, last_start)) + + # Validate every affected run before mutating any of them. Assigning run.text replaces the + # run XML and would otherwise silently delete an inline drawing, field, footnote reference, + # or a page/column break. + affected_indexes = { + index + for _, _, first, _, last, _ in matches + for index in range(first, last + 1) + } + unsafe = {} + for index in affected_indexes: + if children := unsafe_run_content(runs[index]): + unsafe[index] = children + if unsafe: + raise ValueError(f"matched runs contain non-text content: {unsafe}") + + # Work backwards so changing a later match cannot move an earlier match. + for start, end, first, first_start, last, last_start in reversed(matches): prefix = runs[first].text[:start - first_start] suffix = runs[last].text[end - last_start:] @@ -67,8 +102,10 @@ doc.save("input.edited.docx") ``` The replacement text inherits the first matched run's formatting. Unmatched text before and -after it stays in its original runs, so its formatting is preserved. Use raw OOXML for fields, -tracked changes, or other content that `paragraph.runs` does not expose. +after it stays in its original runs, so its formatting is preserved. The routine fails before +making changes if any matched run contains a drawing, field, reference, or typed page/column +break that `run.text` would destroy. Use raw OOXML for those cases and for tracked changes or +other content that `paragraph.runs` does not expose. ## Tier 2 - raw OOXML surgery (only when Tier 1 cannot express it) diff --git a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md index 1bc610a..e60fa1c 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md @@ -39,9 +39,11 @@ file is not. only when the user asked for a rasterized look or provides only images. 2. **Page geometry is explicit**: A4 = 595.27 x 841.89 pt, US Letter = 612 x 792 pt. Declare the target size and margins up front; re-check fit after generation. -3. **Fonts**: standard 14 fonts always work; embedding a custom TTF is allowed only with its - license permitting distribution. CJK requires an embedded font - there is no built-in CJK - face; if unavailable, report the limitation instead of emitting tofu. +3. **Fonts**: the standard 14 fonts cover only limited encodings; they do not automatically + support arbitrary Unicode. Check that the selected face contains every requested character. + If any glyph is unsupported - including CJK, Cyrillic, Arabic, Devanagari, or emoji - embed + one or more licensed TTF/OTF fonts with the required coverage and use them for those runs. + If no suitable embeddable font is available, report the limitation instead of emitting tofu. 4. **Overflow is a defect**: content that spills past the last page or the margin must be detected in postcheck and fixed (shrink, paginate, or cut), never shipped. 5. Write output to a new path; keep inputs untouched unless in-place was requested. diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md b/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md index 734019e..b3c3eb9 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md @@ -11,7 +11,7 @@ for page in doc: page_number = page.number + 1 # Plain text - text = page.get_text("text") # reading order text + text = page.get_text("text", sort=True) # position-sorted reading order for simple layouts print(f"--- page {page_number} ---") print(text) @@ -22,12 +22,17 @@ for page in doc: print(page_number, round(span["bbox"][0]), round(span["bbox"][1]), span["text"]) - # Images. Pixmap keeps the image's own colorspace: CMYK/ICC pixmaps cannot be - # saved as PNG directly, so convert anything not Gray/RGB to RGB first. + # Images. Apply a soft mask (xref at info[1]) before saving or transparency is lost. + # Pixmap keeps the image's own colorspace: convert CMYK/ICC bases to RGB first. for i, info in enumerate(page.get_images(full=True), start=1): - pix = fitz.Pixmap(doc, info[0]) - if pix.colorspace and pix.colorspace not in (fitz.csGRAY, fitz.csRGB): - pix = fitz.Pixmap(fitz.csRGB, pix) + base = fitz.Pixmap(doc, info[0]) + if base.colorspace and base.colorspace not in (fitz.csGRAY, fitz.csRGB): + base = fitz.Pixmap(fitz.csRGB, base) + if info[1] > 0: + mask = fitz.Pixmap(doc, info[1]) + pix = fitz.Pixmap(base, mask) + else: + pix = base pix.save(f"img-p{page_number}-{i}.png") # Rasterize (for visual checks or OCR preprocessing) diff --git a/plugins/Hylouis233/document-skills/skills/pptx/SKILL.md b/plugins/Hylouis233/document-skills/skills/pptx/SKILL.md index b1fce22..6432d81 100644 --- a/plugins/Hylouis233/document-skills/skills/pptx/SKILL.md +++ b/plugins/Hylouis233/document-skills/skills/pptx/SKILL.md @@ -60,5 +60,11 @@ equals the slide count: soffice --headless --convert-to pdf output.pptx --outdir ``` +Page count is only a structural smoke test. Rasterize and inspect **every rendered slide** for +horizontal clipping and for a final line clipped or missing at the bottom; follow the text-fit +procedure in [references/analyze.md](references/analyze.md). Fix and render again if any text +overflows. If no production-equivalent renderer is available, report overflow as unverified - +do not claim that shape bounds or page count prove that text fits. + Report: output path, slide inventory (index, layout name, title), verification done, and any -layout risks (long CJK strings, tight two-column slides) that only a human eye can confirm. +remaining layout risks (long CJK strings, tight two-column slides). diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md index 8e018ee..59d55b6 100644 --- a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md @@ -21,11 +21,42 @@ for i, slide in enumerate(prs.slides): title = slide.shapes.title.text_frame.text if slide.shapes.title is not None else "" notes = slide.notes_slide.notes_text_frame.text if slide.has_notes_slide else "" shapes = list(iter_shapes(slide.shapes)) # flattened; groups are common in template decks - n_tables = sum(1 for sh in shapes if sh.has_table) - n_charts = sum(1 for sh in shapes if sh.has_chart) - n_pics = sum(1 for sh in shapes if sh.shape_type == MSO_SHAPE_TYPE.PICTURE) - print(i, layout, repr(title[:40]), "tables:", n_tables, "charts:", n_charts, - "pics:", n_pics, "notes:", len(notes)) + text = [sh.text_frame.text for sh in shapes if sh.has_text_frame and sh.text_frame.text] + tables = [ + [[cell.text for cell in row.cells] for row in sh.table.rows] + for sh in shapes if sh.has_table + ] + charts = [] + for sh in shapes: + if not sh.has_chart: + continue + chart = sh.chart + chart_title = ( + chart.chart_title.text_frame.text + if chart.has_title and chart.chart_title.has_text_frame else "" + ) + plots = [] + for plot in chart.plots: + categories = [ + [str(level) for level in label] + for label in plot.categories.flattened_labels + ] + series = [ + {"name": item.name, "values": list(item.values)} + for item in plot.series + ] + plots.append({"categories": categories, "series": series}) + charts.append({"title": chart_title, "plots": plots}) + pictures = [sh.name for sh in shapes if sh.shape_type == MSO_SHAPE_TYPE.PICTURE] + + # These full values - not only counts or lengths - are the evidence for summaries and + # repurposing. Keep notes verbatim so markdown output can preserve them as blockquotes. + print(f"slide {i + 1}: layout={layout!r} title={title!r}") + print(" text:", text) + print(" tables:", tables) + print(" charts:", charts) + print(" pictures:", pictures) + print(" notes:", notes) ``` (Simplify the title lookup to `slide.shapes.title` when present; the defensive loop is for @@ -50,28 +81,38 @@ placeholder, layout, master, or theme. Resolve what you can and name the fallbac ```python import re from pptx import Presentation +from pptx.opc.constants import RELATIONSHIP_TYPE as RT prs = Presentation("deck.pptx") -# 1. Theme fonts are the final fallback for inherited text. -theme_xml = next( - part.blob.decode("utf-8", "ignore") - for part in prs.part.package.iter_parts() - if str(part.partname).startswith("/ppt/theme/") -) -theme = { - "major (headings)": re.search(r'\s*\s*", face) +# 1. Resolve the theme related to each slide's own layout/master. A package can contain +# multiple masters with different themes, so the first /ppt/theme/* part is not a safe default. +theme_cache = {} + +def theme_faces_for_slide(slide): + master_part = slide.slide_layout.slide_master.part + cache_key = str(master_part.partname) + if cache_key not in theme_cache: + theme_part = master_part.part_related_by(RT.THEME) + theme_xml = theme_part.blob.decode("utf-8", "ignore") + matches = { + "major (headings)": re.search( + r'\s*\s*` survives but its `x14:` content does not.) ## Rules diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/formatting.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/formatting.md index d1c3d0f..533d7a2 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/formatting.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/formatting.md @@ -75,7 +75,15 @@ one you took: return f"'{escaped}'!" src = sheet_ref(ws) # e.g. "'Sales'!" or "'Raw Data'!" - regions = sorted({r[0] for r in ws.iter_rows(min_row=2, min_col=1, values_only=True) if r[0]}) + regions = [] + seen_region_keys = set() + for (region,) in ws.iter_rows(min_row=2, min_col=1, max_col=1, values_only=True): + if region is None or region == "": # keep valid falsey values: 0 and False + continue + key = (type(region), region) # do not collapse False and numeric 0 + if key not in seen_region_keys: + seen_region_keys.add(key) + regions.append(region) # stable source order; no mixed-type sort for i, region in enumerate(regions, start=2): ws2.cell(row=i, column=1, value=region) ws2.cell(row=i, column=2, value=f"=SUMIF({src}A:A,A{i},{src}C:C)") diff --git a/plugins/Hylouis233/document-skills/tests/docx_fixture.py b/plugins/Hylouis233/document-skills/tests/docx_fixture.py index 4bda6c4..a945f48 100644 --- a/plugins/Hylouis233/document-skills/tests/docx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/docx_fixture.py @@ -40,6 +40,97 @@ def check(name, cond, extra=""): parsed = etree.fromstring(hostile_xml, parser=safe_xml_parser) check("DOCX XML parser leaves external entities unresolved", parsed.text is None and len(parsed) == 1) +# ---- edit.md guarded cross-run replacement ------------------------------------ +SAFE_RUN_CHILDREN = { + qn("w:rPr"), qn("w:t"), qn("w:tab"), qn("w:br"), qn("w:cr"), +} + + +def unsafe_run_content(run): + unsafe = [] + for child in run._r: + typed_break = child.tag == qn("w:br") and child.get(qn("w:type")) not in ( + None, "textWrapping", + ) + if child.tag not in SAFE_RUN_CHILDREN or typed_break: + unsafe.append(child.tag.rsplit("}", 1)[-1]) + return unsafe + + +def replace_across_runs(paragraph, old, new): + if not old: + raise ValueError("old must not be empty") + runs = list(paragraph.runs) + text = "".join(run.text for run in runs) + starts = [] + position = 0 + while (start := text.find(old, position)) != -1: + starts.append(start) + position = start + len(old) + spans = [] + position = 0 + for index, run in enumerate(runs): + end = position + len(run.text) + if end > position: + spans.append((index, position, end)) + position = end + matches = [] + for start in starts: + end = start + len(old) + first, first_start, _ = next(s for s in spans if s[1] <= start < s[2]) + last, last_start, _ = next(s for s in spans if s[1] < end <= s[2]) + matches.append((start, end, first, first_start, last, last_start)) + affected_indexes = { + index + for _, _, first, _, last, _ in matches + for index in range(first, last + 1) + } + unsafe = {} + for index in affected_indexes: + if children := unsafe_run_content(runs[index]): + unsafe[index] = children + if unsafe: + raise ValueError(f"matched runs contain non-text content: {unsafe}") + for start, end, first, first_start, last, last_start in reversed(matches): + prefix = runs[first].text[:start - first_start] + suffix = runs[last].text[end - last_start:] + if first == last: + runs[first].text = prefix + new + suffix + else: + runs[first].text = prefix + new + for index in range(first + 1, last): + runs[index].text = "" + runs[last].text = suffix + return len(starts) + + +safe_doc = Document() +safe_paragraph = safe_doc.add_paragraph() +safe_first = safe_paragraph.add_run("T") +safe_first.bold = True +safe_paragraph.add_run("B") +safe_paragraph.add_run("D") +check("text-only cross-run match is replaced", replace_across_runs(safe_paragraph, "TBD", "Done") == 1) +check("safe replacement keeps first-run formatting", safe_paragraph.text == "Done" and safe_first.bold) + +icon = fitz.Pixmap(fitz.csRGB, fitz.IRect(0, 0, 4, 4), False) +icon.clear_with(200) +icon.save("inline-icon.png") +guard_doc = Document() +guard_paragraph = guard_doc.add_paragraph() +guard_run = guard_paragraph.add_run("TBD") +guard_run.add_picture("inline-icon.png") +try: + replace_across_runs(guard_paragraph, "TBD", "Done") + rejected_drawing_run = False +except ValueError: + rejected_drawing_run = True +check("replacement rejects a matched run containing a drawing", rejected_drawing_run) +check( + "rejected replacement leaves text and drawing untouched", + guard_run.text == "TBD" and len(guard_run._r.findall(qn("w:drawing"))) == 1, +) + def list_number_num_id(doc): """The numId that the ListNumber style binds to in this document part.""" diff --git a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py index b542d7d..de84a8b 100644 --- a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py @@ -114,6 +114,36 @@ def check(name, cond, extra=""): rgb = fitz.Pixmap("cmyk-converted.png") check("converted pixmap is RGB", "RGB" in str(rgb.colorspace), rgb.colorspace) +# ---- extract.md soft-mask composition: transparent image keeps alpha ----------- +rgba = fitz.Pixmap(fitz.csRGB, fitz.IRect(0, 0, 8, 8), True) +for y in range(rgba.height): + for x in range(rgba.width): + rgba.set_pixel(x, y, (255, 0, 0, 255 if x < 4 else 64)) +rgba.save("transparent-source.png") + +transparent_pdf = canvas.Canvas("transparent-image.pdf", pagesize=A4) +transparent_pdf.drawImage( + "transparent-source.png", 72, 700, width=80, height=80, mask="auto" +) +transparent_pdf.save() + +transparent_doc = fitz.open("transparent-image.pdf") +image_info = transparent_doc[0].get_images(full=True)[0] +check("transparent PDF image exposes a soft-mask xref", image_info[1] > 0, image_info) +base = fitz.Pixmap(transparent_doc, image_info[0]) +if base.colorspace and base.colorspace not in (fitz.csGRAY, fitz.csRGB): + base = fitz.Pixmap(fitz.csRGB, base) +mask = fitz.Pixmap(transparent_doc, image_info[1]) +composited = fitz.Pixmap(base, mask) +composited.save("transparent-extracted.png") +reopened_composite = fitz.Pixmap("transparent-extracted.png") +check("soft-mask composition keeps an alpha channel", reopened_composite.alpha == 1) +check( + "soft-mask composition keeps varying transparency", + len(set(reopened_composite.samples[3::4])) > 1, + set(reopened_composite.samples[3::4]), +) + # ---- create.md rule: escape plain text before Paragraph ------------------------ from reportlab.lib.pagesizes import A4 as A4_SIZE from reportlab.lib.styles import getSampleStyleSheet diff --git a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py index ee9a3d2..cb9c414 100644 --- a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py @@ -8,8 +8,11 @@ import sys from pptx import Presentation +from pptx.chart.data import ChartData from pptx.dml.color import RGBColor +from pptx.enum.chart import XL_CHART_TYPE from pptx.enum.shapes import MSO_SHAPE_TYPE +from pptx.opc.constants import RELATIONSHIP_TYPE as RT from pptx.util import Inches, Pt failures = [] @@ -46,6 +49,18 @@ def check(name, cond, extra=""): cr.font.bold = True cr.font.color.rgb = RGBColor(0x00, 0x70, 0xC0) +chart_data = ChartData() +chart_data.categories = ["EU", "US"] +chart_data.add_series("Units", (120, 80)) +chart = slide.shapes.add_chart( + XL_CHART_TYPE.COLUMN_CLUSTERED, + Inches(7.2), Inches(2.5), Inches(2), Inches(2), + chart_data, +).chart +chart.has_title = True +chart.chart_title.text_frame.text = "Units by region" +slide.notes_slide.notes_text_frame.text = "Speaker note: explain the regional split." + prs.save("input.pptx") # ---- edit.md snippet: single-shape run replace keeps styling and hyperlink ----- @@ -124,6 +139,52 @@ def iter_shapes(shapes): yield shape +def extract_slide_content(slide): + shapes = list(iter_shapes(slide.shapes)) + text = [sh.text_frame.text for sh in shapes if sh.has_text_frame and sh.text_frame.text] + tables = [ + [[cell.text for cell in row.cells] for row in sh.table.rows] + for sh in shapes if sh.has_table + ] + charts = [] + for sh in shapes: + if not sh.has_chart: + continue + chart = sh.chart + chart_title = ( + chart.chart_title.text_frame.text + if chart.has_title and chart.chart_title.has_text_frame else "" + ) + plots = [] + for plot in chart.plots: + categories = [ + [str(level) for level in label] + for label in plot.categories.flattened_labels + ] + series = [ + {"name": item.name, "values": list(item.values)} + for item in plot.series + ] + plots.append({"categories": categories, "series": series}) + charts.append({"title": chart_title, "plots": plots}) + notes = slide.notes_slide.notes_text_frame.text if slide.has_notes_slide else "" + return {"text": text, "tables": tables, "charts": charts, "notes": notes} + + +content = extract_slide_content(Presentation("input.pptx").slides[0]) +check("content inventory emits body text", any("old wording" in value for value in content["text"]), content) +check("content inventory emits table cell text", content["tables"][0][0][1] == "old cell text", content["tables"]) +check( + "content inventory emits chart title, categories, series, and values", + content["charts"][0]["title"] == "Units by region" + and content["charts"][0]["plots"][0]["categories"] == [["EU"], ["US"]] + and content["charts"][0]["plots"][0]["series"] + == [{"name": "Units", "values": [120.0, 80.0]}], + content["charts"], +) +check("content inventory emits notes text", "regional split" in content["notes"], content["notes"]) + + sp_element = slide6.shapes[-1]._element # the textbox; layout 5 still carries a Title placeholder group_element = parse_xml(GRP) sp_element.getparent().replace(sp_element, group_element) @@ -177,19 +238,78 @@ def iter_shapes_with_path(shapes, path=""): if getattr(sh, "text_frame", None) is not None and sh.text_frame.text == new_w] check("group member edit persists after save", len(found) == 1) -# ---- analyze.md snippet: theme font resolution --------------------------------- +# ---- analyze.md snippet: per-master theme font resolution ---------------------- import re prs7 = Presentation("input.pptx") -theme_xml = next( - part.blob.decode("utf-8", "ignore") - for part in prs7.part.package.iter_parts() - if str(part.partname).startswith("/ppt/theme/") +theme_cache = {} + + +def theme_faces_for_slide(slide): + master_part = slide.slide_layout.slide_master.part + cache_key = str(master_part.partname) + if cache_key not in theme_cache: + theme_part = master_part.part_related_by(RT.THEME) + theme_xml = theme_part.blob.decode("utf-8", "ignore") + matches = { + "major (headings)": re.search( + r'\s*\s*' + f'' + f'' + f'' + ).encode() + + +class StubMasterPart: + def __init__(self, name, major, minor): + self.partname = name + self.theme_part = StubThemePart(major, minor) + + def part_related_by(self, relationship_type): + assert relationship_type == RT.THEME + return self.theme_part + + +def stub_slide(name, major, minor): + master = type("Master", (), {"part": StubMasterPart(name, major, minor)})() + layout = type("Layout", (), {"slide_master": master})() + return type("Slide", (), {"slide_layout": layout})() + + +_, first_faces = theme_faces_for_slide(stub_slide("/ppt/slideMasters/one.xml", "Head One", "Body One")) +_, second_faces = theme_faces_for_slide(stub_slide("/ppt/slideMasters/two.xml", "Head Two", "Body Two")) +check( + "different slide masters resolve their own theme faces", + first_faces["major (headings)"] == "Head One" + and second_faces["major (headings)"] == "Head Two" + and first_faces != second_faces, + (first_faces, second_faces), ) -major = re.search(r'\s*\s*"} +partial_after = {"xl/worksheets/sheet1.xml": b""} +check( + "marker comparison catches x14 loss while extLst survives", + stripped_extension_markers(partial_before, partial_after) + == [("xl/worksheets/sheet1.xml", "x14")], + stripped_extension_markers(partial_before, partial_after), +) # ---- formatting.md snippet: sheet references built from the real sheet title ------- wb_f = openpyxl.Workbook() @@ -148,6 +165,27 @@ def sheet_ref(sheet): hyphen_formula = openpyxl.load_workbook("hyphen-agg.xlsx")["ByRegion"]["B4"].value check("ambiguous punctuation is protected by quoting", hyphen_formula == "=SUM('Q1-Data'!A:A)", hyphen_formula) +# Falsey values are valid categories; blank filtering must not discard or conflate them. +falsey_ws = openpyxl.Workbook().active +for value in ("Category", 0, False, "", None, 0, False): + falsey_ws.append([value]) +regions = [] +seen_region_keys = set() +for (region,) in falsey_ws.iter_rows(min_row=2, min_col=1, max_col=1, values_only=True): + if region is None or region == "": + continue + key = (type(region), region) + if key not in seen_region_keys: + seen_region_keys.add(key) + regions.append(region) +check( + "aggregation preserves numeric zero and boolean false as distinct categories", + len(regions) == 2 + and type(regions[0]) is int and regions[0] == 0 + and type(regions[1]) is bool and regions[1] is False, + [(type(value).__name__, value) for value in regions], +) + # and the edit itself still works after the warning path wb2 = openpyxl.load_workbook("plain.xlsx") wb2["Data"]["B2"] = "=B2*1" # formula stays a formula From 12d9542c43421f68c4981c78a9a0e9313b8cb2c9 Mon Sep 17 00:00:00 2001 From: Hylouis233 Date: Sun, 16 Aug 2026 12:54:08 +0800 Subject: [PATCH 08/34] fix(document-skills): address latest review round --- plugins/Hylouis233/document-skills/README.md | 21 ++- .../skills/docx/references/cjk.md | 76 +++++++- .../skills/docx/references/read.md | 23 ++- .../skills/docx/references/review.md | 40 ++++- .../document-skills/skills/pdf/SKILL.md | 19 +- .../skills/pdf/references/extract.md | 5 + .../skills/pdf/references/transform.md | 7 +- .../skills/pptx/references/analyze.md | 84 +++++++-- .../skills/pptx/references/edit.md | 23 ++- .../document-skills/skills/xlsx/SKILL.md | 6 +- .../skills/xlsx/references/csv.md | 35 +++- .../skills/xlsx/references/edit.md | 115 ++++++++++-- .../skills/xlsx/references/read.md | 14 +- .../document-skills/tests/docx_fixture.py | 95 ++++++++++ .../document-skills/tests/pdf_fixture.py | 17 +- .../document-skills/tests/pptx_fixture.py | 157 +++++++++++++---- .../document-skills/tests/xlsx_fixture.py | 166 +++++++++++++++--- 17 files changed, 761 insertions(+), 142 deletions(-) diff --git a/plugins/Hylouis233/document-skills/README.md b/plugins/Hylouis233/document-skills/README.md index 4118cc1..fba9fc7 100644 --- a/plugins/Hylouis233/document-skills/README.md +++ b/plugins/Hylouis233/document-skills/README.md @@ -45,9 +45,10 @@ Shared spine (all four Skills follow it): Per format: -- **docx** — create with python-docx from a heading outline; edit existing files by direct - `word/document.xml` surgery (python-docx cannot open-and-save arbitrary files losslessly); - extract text with python-docx or `pandoc -t markdown`; postcheck with python-docx re-open and +- **docx** — create with python-docx from a heading outline; use python-docx first for routine + structural edits (paragraphs, tables, images, styles, and text runs), and reserve direct OOXML + surgery for fields, tracked changes, or package features python-docx cannot express; extract + text with python-docx or `pandoc -t markdown`; postcheck with python-docx re-open and `soffice --headless --convert-to` PDF smoke test when LibreOffice is present. Depth references: CJK typography (east-asian font slots, 字号 table, char-based indents, GB/T 9704 page geometry) and scene patterns (academic paper, resume, official document, contract). @@ -63,8 +64,9 @@ Per format: presentation-level verification via `python-pptx` re-open plus a rendered PDF smoke test when LibreOffice is available. - **pdf** — creation prefers ReportLab (structured, accessible text) over HTML-to-print paths; - extraction and splitting/merging with pypdf; analysis and rasterization with PyMuPDF; explicit - one-tool-per-job table so the agent stops mixing libraries mid-task. + extraction (text, coordinates, tables, images) and rasterization use PyMuPDF, while pypdf is + reserved for page-level transforms such as split, merge, rotate, watermark, encryption, and + forms; an explicit one-tool-per-job table prevents accidental API mixing. ## Verification-first output @@ -75,10 +77,11 @@ versus what was assumed. The [`tests/`](tests/) directory ships one runnable fixture script per format covering the snippets with the worst silent-failure modes (PDF AcroForm clone-and-fill, watermark write, -CMYK conversion, soft-mask extraction; PPTX run-preserving edits, actual content extraction, -grouped-shape walking, per-master themes; XLSX dialect sniffing, independent extension-marker -loss, falsey aggregation categories; DOCX guarded cross-run replacement and numbering restart -rendered through LibreOffice). Each script is self-contained and exits non-zero on failure. +encrypted extraction, CMYK conversion, soft masks; PPTX run-preserving edits, table-cell locating, +actual content extraction, grouped shapes, script-aware per-master themes; XLSX dialect sniffing, +array formulas, streamed extension checks, structural-reference audits; DOCX bounded package review, +content controls, per-run glyph checks, guarded replacement, and a LibreOffice-rendered numbering +restart). Each script is self-contained and exits non-zero on failure. ## Requirements diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md b/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md index 3d21035..cd43ca0 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md @@ -97,12 +97,78 @@ section.left_margin, section.right_margin = Cm(2.8), Cm(2.6) (`pip install fonttools`) when the referenced faces can be located: ```python + from docx import Document from fontTools.ttLib import TTFont - - cmaps = [TTFont(font_path).getBestCmap() for font_path in referenced_font_files] - emitted = {ch for ch in source_text if ord(ch) >= 0x2E80} # CJK and beyond - missing = {ch for ch in emitted if not any(ord(ch) in cmap for cmap in cmaps)} - assert not missing, f"codepoints with no glyph in any referenced font: {sorted(missing)}" + from docx.oxml.ns import qn + from docx.text.paragraph import Paragraph + + doc = Document("output.docx") + + def xml_runs(element, parent): + # Covers direct paragraphs, tables, nested tables, and block content controls. + for paragraph_element in element.iter(qn("w:p")): + yield from Paragraph(paragraph_element, parent).runs + + emitted_runs = list(xml_runs(doc.element.body, doc)) + for section in doc.sections: + emitted_runs.extend(xml_runs(section.header._element, section.header)) + emitted_runs.extend(xml_runs(section.footer._element, section.footer)) + + def face_from_rpr(rpr, slot): + if rpr is None: + return None + rfonts = rpr.find(qn("w:rFonts")) + return None if rfonts is None else rfonts.get(qn("w:" + slot)) + + def style_faces(style, slot): + while style is not None: + face = face_from_rpr(style.element.find(qn("w:rPr")), slot) + if face: + yield face + style = style.base_style + + def effective_face(run, slot): + direct = face_from_rpr(run._r.find(qn("w:rPr")), slot) + if direct: + return direct + for style in (run.style, run._parent.style, run.part.document.styles["Normal"]): + if face := next(style_faces(style, slot), None): + return face + raise LookupError(f"no resolved {slot} face for run {run.text!r}; resolve theme defaults") + + def font_slot(character): + codepoint = ord(character) + return "eastAsia" if ( + 0x2E80 <= codepoint <= 0x9FFF or 0xF900 <= codepoint <= 0xFAFF + or 0x20000 <= codepoint <= 0x3134F + ) else ("ascii" if codepoint < 128 else "hAnsi") + + # Resolve installed files by exact face name first; do not pool their cmaps. + # TTC collections need the face's fontNumber; ordinary TTF files use -1. + font_files_by_face = { + "宋体": ("path/to/simsun.ttc", 0), + "Times New Roman": ("path/to/times.ttf", -1), + } + cmaps = {} + missing = [] + unresolved = [] + for run_index, run in enumerate(emitted_runs): + for character in run.text: + if character.isspace(): + continue + face = effective_face(run, font_slot(character)) + font_spec = font_files_by_face.get(face) + if not font_spec: + unresolved.append((run_index, character, face)) + continue + if font_spec not in cmaps: + font_path, font_number = font_spec + cmaps[font_spec] = TTFont(font_path, fontNumber=font_number).getBestCmap() + cmap = cmaps[font_spec] + if ord(character) not in cmap: + missing.append((run_index, character, face)) + assert not unresolved, f"font files not resolved per run: {unresolved}" + assert not missing, f"glyph missing from the run's effective font: {missing}" ``` When font files cannot be located, rasterize the rendered pages with PyMuPDF and inspect diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/read.md b/plugins/Hylouis233/document-skills/skills/docx/references/read.md index a41ad89..d7f5b0b 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/read.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/read.md @@ -15,8 +15,23 @@ tables). Prefer this when the goal is content, not coordinates. ```python from docx import Document +from docx.oxml.ns import qn +from docx.text.paragraph import Paragraph + +def iter_body_paragraphs(parent, document): + """Yield direct body paragraphs plus paragraphs nested in block content controls.""" + for child in parent.iterchildren(): + if child.tag == qn("w:p"): + yield Paragraph(child, document) + elif child.tag == qn("w:sdt"): + content = child.find(qn("w:sdtContent")) + if content is not None: + yield from iter_body_paragraphs(content, document) + doc = Document("input.docx") -for par in doc.paragraphs: +content_controls = list(doc.element.body.iter(qn("w:sdt"))) +print("block content controls:", len(content_controls)) +for par in iter_body_paragraphs(doc.element.body, doc): print(par.style.name, "|", par.text) for t, table in enumerate(doc.tables): for r, row in enumerate(table.rows): @@ -25,8 +40,10 @@ for t, table in enumerate(doc.tables): Notes: -- `doc.paragraphs` is body-level only. Text inside text boxes, headers, footers, footnotes is - reached via their own collections (`section.header/.footer`) or raw XML. +- `doc.paragraphs` includes only direct body paragraphs; it omits paragraphs nested in block + content controls (`w:sdt`). Use the traversal above and report the content-control count. + Text boxes, headers, footers, and footnotes still require their own collections + (`section.header/.footer`) or raw XML. - `doc.tables` is top-level only; nested tables require walking cells. - For revision/comment metadata, inspect the XML parts directly: `word/comments.xml`, `w:ins`/`w:del` elements in `word/document.xml`. diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/review.md b/plugins/Hylouis233/document-skills/skills/docx/references/review.md index 5dd1078..e06a894 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/review.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/review.md @@ -17,6 +17,10 @@ import zipfile from lxml import etree path = "input.docx" +MAX_XML_PART = 20 * 1024 * 1024 +MAX_ENTRY = 100 * 1024 * 1024 +MAX_TOTAL_UNCOMPRESSED = 500 * 1024 * 1024 +MAX_COMPRESSION_RATIO = 200 safe_xml_parser = etree.XMLParser( load_dtd=False, resolve_entities=False, @@ -25,15 +29,39 @@ safe_xml_parser = etree.XMLParser( recover=False, ) with zipfile.ZipFile(path) as z: - bad = z.testzip() - assert bad is None, f"corrupt entry: {bad}" - names = z.namelist() + infos = z.infolist() + names = {info.filename for info in infos} + assert len(names) == len(infos), "duplicate archive member names are unsafe" assert "[Content_Types].xml" in names and "word/document.xml" in names - for part in names: - if part.endswith(('.xml', '.rels')): - etree.fromstring(z.read(part), parser=safe_xml_parser) # malformed XML still raises + assert sum(info.file_size for info in infos) <= MAX_TOTAL_UNCOMPRESSED + actual_total = 0 + for info in infos: + assert info.file_size <= MAX_ENTRY, f"oversized part: {info.filename}" + ratio = info.file_size / max(info.compress_size, 1) + assert ratio <= MAX_COMPRESSION_RATIO, f"suspicious compression ratio: {info.filename}" + is_xml = info.filename.endswith((".xml", ".rels")) + if is_xml: + assert info.file_size <= MAX_XML_PART, f"oversized XML part: {info.filename}" + chunks = [] + actual_size = 0 + # Stream every bounded member to verify decompression and CRC. Do not call testzip() + # before the limits: it would expand every member regardless of declared risk. + with z.open(info) as stream: + while chunk := stream.read(64 * 1024): + actual_size += len(chunk) + actual_total += len(chunk) + assert actual_size <= MAX_ENTRY, f"part exceeded read limit: {info.filename}" + assert actual_total <= MAX_TOTAL_UNCOMPRESSED, "archive exceeded total read limit" + if is_xml: + chunks.append(chunk) + assert actual_size == info.file_size, f"size mismatch: {info.filename}" + if is_xml: + etree.fromstring(b"".join(chunks), parser=safe_xml_parser) ``` +These limits are conservative review defaults, not a DOCX specification. Raise one only for an +explicitly trusted, expected large input, and keep the streaming/per-part checks in place. + Then the SKILL.md postcheck (python-docx re-open, optional soffice PDF smoke test). ## Report format diff --git a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md index e60fa1c..fc779fa 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md @@ -78,15 +78,26 @@ expected_strings_by_page = {} for page_number, expected_strings in expected_strings_by_page.items(): missing = [value for value in expected_strings if value not in page_texts[page_number]] assert not missing, f"page {page_number} is missing {missing}" -# mediabox is (left, bottom, right, top); reduce it to the width/height pair you declared +# Compare width/height to the exact size used at creation with a small point tolerance. +# ReportLab A4 is about (595.2756, 841.8898), not the rounded prose value (595.27, 841.89). +expected_page_size = (595.2756, 841.8898) # replace for Letter or a task-specific size +page_size_tolerance = 0.5 page_sizes = [ - (round(float(page.mediabox.width), 2), round(float(page.mediabox.height), 2)) + (float(page.mediabox.width), float(page.mediabox.height)) for page in r.pages ] +size_mismatches = [ + (number, actual) + for number, actual in enumerate(page_sizes, start=1) + if any(abs(value - expected) > page_size_tolerance + for value, expected in zip(actual, expected_page_size)) +] +assert not size_mismatches, f"unexpected page sizes: {size_mismatches}" ``` Confirm: page count matches the request; every page except those explicitly listed in `intentionally_raster_only_pages` has extractable text; each requested key string is listed in `expected_strings_by_page` and extracts on the correct page; every value in `page_sizes` is the -declared size. Report all four. For pixel-sensitive work, render every applicable page with -PyMuPDF at 100 dpi and check that each render is non-blank (mean pixel value). +declared size within `page_size_tolerance`. Report all four. For pixel-sensitive work, render +every applicable page with PyMuPDF at 100 dpi and check that each render is non-blank (mean pixel +value). diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md b/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md index b3c3eb9..b341f06 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md @@ -1,9 +1,14 @@ # Extract from a PDF (PyMuPDF) ```python +import os import fitz # pymupdf doc = fitz.open("input.pdf") +if doc.needs_pass: + password = os.environ.get("PDF_PASSWORD") + if not password or not doc.authenticate(password): + raise RuntimeError("set PDF_PASSWORD to the correct password before extracting") print("pages:", doc.page_count, "metadata:", doc.metadata) # Extract each page; do not use `page` after this loop. diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md b/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md index 299ead5..68e25d6 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md @@ -34,10 +34,11 @@ stamp = PdfReader("watermark.pdf").pages[0] stamp_text = (stamp.extract_text() or "").strip() reader = PdfReader("input.pdf") expected_sizes = [tuple(float(value) for value in page.mediabox) for page in reader.pages] +expected_fields = reader.get_fields() or {} writer = PdfWriter() -for page in reader.pages: +writer.append(reader) # clone pages plus catalog entries such as /AcroForm +for page in writer.pages: page.merge_page(stamp) # stamp content on top; use merge_transformed_page to place - writer.add_page(page) with open("watermarked.pdf", "wb") as f: writer.write(f) @@ -45,6 +46,8 @@ with open("watermarked.pdf", "wb") as f: check = PdfReader("watermarked.pdf") assert len(check.pages) == len(expected_sizes) assert [tuple(float(value) for value in page.mediabox) for page in check.pages] == expected_sizes +if expected_fields: + assert set(expected_fields) <= set(check.get_fields() or {}), "watermarking dropped form fields" if stamp_text: assert all(stamp_text in (page.extract_text() or "") for page in check.pages) ``` diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md index 59d55b6..75e50d4 100644 --- a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md @@ -79,7 +79,7 @@ Most template decks set no explicit `run.font.name`; the effective face is inher placeholder, layout, master, or theme. Resolve what you can and name the fallback explicitly: ```python -import re +import xml.etree.ElementTree as ET from pptx import Presentation from pptx.opc.constants import RELATIONSHIP_TYPE as RT @@ -88,31 +88,72 @@ prs = Presentation("deck.pptx") # 1. Resolve the theme related to each slide's own layout/master. A package can contain # multiple masters with different themes, so the first /ppt/theme/* part is not a safe default. theme_cache = {} +DRAWINGML = {"a": "http://schemas.openxmlformats.org/drawingml/2006/main"} + +def read_theme_role(root, role): + node = root.find(f".//a:{role}Font", DRAWINGML) + if node is None: + return {"latin": "", "eastAsia": "", "complexScript": "", "scripts": {}} + def typeface(tag): + child = node.find(f"a:{tag}", DRAWINGML) + return "" if child is None else child.get("typeface", "") + return { + "latin": typeface("latin"), + "eastAsia": typeface("ea"), + "complexScript": typeface("cs"), + "scripts": { + child.get("script"): child.get("typeface", "") + for child in node.findall("a:font", DRAWINGML) if child.get("script") + }, + } def theme_faces_for_slide(slide): master_part = slide.slide_layout.slide_master.part cache_key = str(master_part.partname) if cache_key not in theme_cache: theme_part = master_part.part_related_by(RT.THEME) - theme_xml = theme_part.blob.decode("utf-8", "ignore") - matches = { - "major (headings)": re.search( - r'\s*\s* layout -> master inheritance chain; when -a run reports inherited, list the theme fallback above and, if the exact face matters, check -the layout and master placeholder of the same index for an explicit ``. +a run reports inherited, list the script-aware theme candidates above and, if the exact face +matters, check the layout and master placeholder of the same index for explicit ``, +``, ``, and script-specific `` mappings. Han text also needs the deck locale +to distinguish Hans, Hant, Japanese, and Korean theme mappings. ## Text-fit verification diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md b/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md index c3668d7..ff4d335 100644 --- a/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md @@ -22,6 +22,14 @@ def iter_shapes(shapes, path=""): else: yield here, shape +def iter_text_targets(path, shape): + if shape.has_text_frame: + yield path, shape.text_frame + if shape.has_table: + for row_index, row in enumerate(shape.table.rows): + for column_index, cell in enumerate(row.cells): + yield f"{path}/table[{row_index},{column_index}]", cell.text_frame + candidates = [] for i, slide in enumerate(prs.slides): if slide_index is not None and i != slide_index: @@ -29,15 +37,15 @@ for i, slide in enumerate(prs.slides): for path, shape in iter_shapes(slide.shapes): if shape_name is not None and shape.name != shape_name: continue - if shape.has_text_frame and old in shape.text_frame.text: - candidates.append((i, path, shape)) + for location, text_frame in iter_text_targets(path, shape): + if old in text_frame.text: + candidates.append((i, location, text_frame)) -locations = [(i, path) for i, path, _ in candidates] -assert len(candidates) == 1, f"expected one matching shape, found {locations}" -_, _, target_shape = candidates[0] +locations = [(i, location) for i, location, _ in candidates] +assert len(candidates) == 1, f"expected one matching text target, found {locations}" +_, _, tf = candidates[0] # Replace inside one existing run so its formatting and hyperlink are retained. -tf = target_shape.text_frame assert tf.text.count(old) == 1, "target occurs more than once in the selected shape" run_hits = [ run @@ -58,7 +66,8 @@ prs.save("input-edited.pptx") 1. **Never rebuild the file to make a small change.** Rewriting slides from scratch loses the template, masters, notes, and animations. Edit in place, save to a new path. 2. Address shapes by slide index + shape name or matched text, and **assert exactly one match**. - If copy repeats, set both selectors rather than silently choosing the last shape. + The locator must search both shape text frames and every table cell, retaining a stable + `/table[row,column]` suffix. If copy repeats, set both selectors rather than choosing one. 3. For formatted text, change `run.text` only when the target is wholly inside one run. Assigning `paragraph.text` or `text_frame.text` rebuilds runs and can discard run formatting and links. If the target spans runs, stop and make an explicitly reviewed run/XML edit. diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md b/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md index 12bf9ea..ba12b4f 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md @@ -68,8 +68,10 @@ missing = set(expected_sheets) - set(wb.sheetnames) assert not missing, f"missing expected sheets: {sorted(missing)}" for ws in wb.worksheets: print(f"{ws.title} dims:", ws.dimensions) - formulas = [(c.coordinate, c.value) for row in ws.iter_rows() for c in row - if isinstance(c.value, str) and c.value.startswith("=")] + formulas = [ + (c.coordinate, getattr(c.value, "text", None) or str(c.value)) + for row in ws.iter_rows() for c in row if c.data_type == "f" + ] print(f"{ws.title} formula cells:", formulas[:10]) for coordinate, expected_format in expected_number_formats.get(ws.title, {}).items(): actual_format = ws[coordinate].number_format diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md index 88657b1..e37000d 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md @@ -55,8 +55,39 @@ with open("output.csv", "w", newline="", encoding="utf-8") as f: if isinstance(value, str) and value.startswith("="): cell.data_type = "s" # openpyxl otherwise promotes it to an XLSX formula ``` -- XLSX -> CSV: `iter_rows(values_only=True)`; format numbers yourself only if the user needs a - fixed display format - otherwise write raw values and say so. +- XLSX -> CSV: use a separate `data_only=True` read so formulas export the cached values users + see, not formula strings. Pair it with a formula-preserving read and report missing caches; + cached values can also be stale until a spreadsheet application recalculates the workbook: + + ```python + import csv + import openpyxl + from pathlib import Path + + formula_wb = openpyxl.load_workbook("input.xlsx", read_only=True, data_only=False) + value_wb = openpyxl.load_workbook("input.xlsx", read_only=True, data_only=True) + formula_ws, value_ws = formula_wb["Data"], value_wb["Data"] + missing_caches = [] + output_path = Path("output.csv") + temporary_path = output_path.with_suffix(output_path.suffix + ".tmp") + with temporary_path.open("w", newline="", encoding="utf-8") as output: + writer = csv.writer(output) + for formula_row, value_row in zip(formula_ws.iter_rows(), value_ws.iter_rows()): + for formula_cell, value_cell in zip(formula_row, value_row): + if formula_cell.data_type == "f" and value_cell.value is None: + missing_caches.append(formula_cell.coordinate) + writer.writerow([cell.value for cell in value_row]) + formula_wb.close() + value_wb.close() + if missing_caches: + temporary_path.unlink(missing_ok=True) + raise RuntimeError(f"formula cells have no cached value: {missing_caches}") + temporary_path.replace(output_path) + ``` + + Format numbers yourself only if the user needs a fixed display format; otherwise write raw + cached values and say so. Export formula text from the `data_only=False` workbook only when + the user explicitly requests formulas rather than displayed values. - Large CSV -> keep it CSV or move to SQLite/Parquet; loading it all into one sheet to "preserve" it usually exceeds limits and helps nobody. diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md index a369dc3..5d9bbeb 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md @@ -8,6 +8,42 @@ from openpyxl.styles import Font wb = openpyxl.load_workbook("input.xlsx") # NOT data_only: that would drop all formulas ws = wb["Data"] +def non_cell_references(workbook): + """Inventory ranges/formulas that insert_rows/delete_rows will not rewrite.""" + refs = [] + for item in workbook.defined_names.values(): + refs.append(("defined name", item.name, item.attr_text)) + for sheet in workbook.worksheets: + owner = sheet.title + for table in sheet.tables.values(): + refs.append(("table", owner + "!" + table.name, table.ref)) + for merged_range in sheet.merged_cells.ranges: + refs.append(("merged range", owner, str(merged_range))) + if sheet.auto_filter.ref: + refs.append(("auto filter", owner, sheet.auto_filter.ref)) + for label, value in ( + ("print area", sheet.print_area), + ("print title rows", sheet.print_title_rows), + ("print title columns", sheet.print_title_cols), + ): + if value: + refs.append((label, owner, str(value))) + for validation in sheet.data_validations.dataValidation: + refs.append(("data validation range", owner, str(validation.sqref))) + for formula in (validation.formula1, validation.formula2): + if formula: + refs.append(("data validation formula", owner, str(formula))) + for conditional_range in sheet.conditional_formatting: + refs.append(("conditional formatting range", owner, str(conditional_range.sqref))) + for rule in sheet.conditional_formatting[conditional_range]: + for formula in getattr(rule, "formula", ()): + refs.append(("conditional formatting formula", owner, str(formula))) + for index, chart in enumerate(sheet._charts, start=1): + for element in chart._write().iter(): + if element.tag.rsplit("}", 1)[-1] == "f" and element.text: + refs.append(("chart series", f"{owner} chart {index}", element.text)) + return refs + # Address cells directly; check the header to confirm column meaning first ws["D2"] = "=C2*1.08" # real formula ws["E2"] = date(2025, 9, 30) @@ -15,7 +51,15 @@ ws["E2"].number_format = "yyyy-mm-dd" ws["F2"] = 1234.5 ws["F2"].number_format = "#,##0.00" -# Insert a row at a position (formulas in shifted rows do NOT auto-adjust - rewrite them) +# Insert/delete does not adjust formulas or non-cell dependencies. Fail closed until every +# reported reference that can intersect the shifted region has an explicit rewrite plan. +references_before = non_cell_references(wb) +if references_before: + for reference in references_before: + print("structural-edit dependency:", reference) + raise RuntimeError( + "insert_rows is unsafe until chart/name/table/filter/validation/format references are audited" + ) ws.insert_rows(5) # Append a new sheet for derived output @@ -30,12 +74,14 @@ for row in summary["A1:B1"]: for cell in row: cell.font = header_font -# References to the shifted region may live on any sheet; inspect every formula +# References to the shifted region may live on any sheet; inspect every formula, then rerun +# non_cell_references and verify every planned rewrite before saving. for formula_ws in wb.worksheets: for row in formula_ws.iter_rows(): for cell in row: - if isinstance(cell.value, str) and cell.value.startswith("="): - print(f"{formula_ws.title}!{cell.coordinate}: {cell.value}") + if cell.data_type == "f": + formula = getattr(cell.value, "text", None) or str(cell.value) + print(f"{formula_ws.title}!{cell.coordinate}: {formula}") wb.save("input-edited.xlsx") ``` @@ -48,7 +94,7 @@ wb.save("input-edited.xlsx") ```python import zipfile - from io import BytesIO + from tempfile import TemporaryFile EXTENSION_MARKERS = { "extLst": b"") + archive.writestr("word/document.xml", "" + (" " * 2_000_000) + "") +try: + validate_docx_package("compressed-bomb.docx") + archive_bomb_rejected = False +except AssertionError: + archive_bomb_rejected = True +check("suspicious compression ratio is rejected before XML expansion", archive_bomb_rejected) + +# ---- read.md includes block paragraphs nested in content controls -------------- +def iter_body_paragraphs(parent, document): + for child in parent.iterchildren(): + if child.tag == qn("w:p"): + yield Paragraph(child, document) + elif child.tag == qn("w:sdt"): + content = child.find(qn("w:sdtContent")) + if content is not None: + yield from iter_body_paragraphs(content, document) + + +sdt_doc = Document() +sdt_doc.add_paragraph("direct paragraph") +sdt_paragraph = sdt_doc.add_paragraph("inside content control") +sdt = OxmlElement("w:sdt") +sdt_content = OxmlElement("w:sdtContent") +sdt_paragraph._p.getparent().replace(sdt_paragraph._p, sdt) +sdt_content.append(sdt_paragraph._p) +sdt.append(sdt_content) +sdt_doc.save("content-control.docx") +sdt_reopened = Document("content-control.docx") +check("doc.paragraphs omits block content-control text (negative control)", + "inside content control" not in [paragraph.text for paragraph in sdt_reopened.paragraphs]) +walked_text = [paragraph.text for paragraph in iter_body_paragraphs(sdt_reopened.element.body, sdt_reopened)] +check("content-control traversal emits the nested paragraph", "inside content control" in walked_text, walked_text) + +# Per-run glyph validation must not let a different referenced font hide a missing glyph. +fixture_cmaps = {"CJK Face": {ord("漢")}, "Latin Face": {ord("A")}} +assigned_runs = [("CJK Face", "漢"), ("Latin Face", "漢")] +pooled_passes = all(any(ord(ch) in cmap for cmap in fixture_cmaps.values()) + for _, text in assigned_runs for ch in text) +per_run_missing = [(face, ch) for face, text in assigned_runs for ch in text + if ord(ch) not in fixture_cmaps[face]] +check("pooled cmap is proven unsafe (negative control)", pooled_passes) +check("per-run cmap check identifies the actual missing glyph", + per_run_missing == [("Latin Face", "漢")], per_run_missing) + # ---- edit.md guarded cross-run replacement ------------------------------------ SAFE_RUN_CHILDREN = { qn("w:rPr"), qn("w:t"), qn("w:tab"), qn("w:br"), qn("w:cr"), diff --git a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py index de84a8b..156e24a 100644 --- a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py @@ -34,10 +34,10 @@ def check(name, cond, extra=""): # ---- SKILL.md postcheck snippet: width/height pairs from the 4-coordinate box -- r = pypdf.PdfReader("form.pdf") page_sizes = [ - (round(float(page.mediabox.width), 2), round(float(page.mediabox.height), 2)) + (float(page.mediabox.width), float(page.mediabox.height)) for page in r.pages ] -A4_TOLERANCE = 0.5 # reportlab A4 is 595.28 x 841.89 after rounding; compare with tolerance +A4_TOLERANCE = 0.5 check( "mediabox width/height is A4 on every page", all(abs(w - 595.2755) < A4_TOLERANCE and abs(h - 841.8897) < A4_TOLERANCE for w, h in page_sizes), @@ -54,6 +54,12 @@ def check(name, cond, extra=""): check("encrypted fixture is detected before page access", probe.is_encrypted) encrypted_r = pypdf.PdfReader("encrypted.pdf", password="fixture-password") check("password-authenticated postcheck can access every page", len(encrypted_r.pages) == 2) +encrypted_extract = fitz.open("encrypted.pdf") +check("PyMuPDF extraction detects that authentication is required", encrypted_extract.needs_pass) +check("PyMuPDF rejects the wrong extraction password", encrypted_extract.authenticate("wrong") == 0) +check("PyMuPDF authenticates before page extraction", encrypted_extract.authenticate("fixture-password") > 0) +check("authenticated PyMuPDF extraction reaches page text", + "Application form" in encrypted_extract[0].get_text("text", sort=True)) # ---- transform.md AcroForm snippet: clone into writer, fill on writer pages ---- from pypdf import PdfReader, PdfWriter @@ -89,10 +95,11 @@ def check(name, cond, extra=""): stamp_text = (stamp.extract_text() or "").strip() reader = R2("form.pdf") expected_sizes = [(round(float(p.mediabox.width), 2), round(float(p.mediabox.height), 2)) for p in reader.pages] +expected_fields = reader.get_fields() or {} writer = PdfWriter() -for page in reader.pages: +writer.append(reader) +for page in writer.pages: page.merge_page(stamp) - writer.add_page(page) with open("watermarked.pdf", "wb") as f: writer.write(f) @@ -102,6 +109,8 @@ def check(name, cond, extra=""): "watermark page sizes unchanged", [(round(float(p.mediabox.width), 2), round(float(p.mediabox.height), 2)) for p in verify.pages] == expected_sizes, ) +check("watermarking preserves the AcroForm catalog and fields", + set(expected_fields) <= set(verify.get_fields() or {}), verify.get_fields()) check("stamp text present on every page", all(stamp_text in (p.extract_text() or "") for p in verify.pages)) # ---- extract.md CMYK conversion snippet --------------------------------------- diff --git a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py index cb9c414..7815f26 100644 --- a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py @@ -222,27 +222,69 @@ def iter_shapes_with_path(shapes, path=""): yield here, shape +def iter_text_targets(path, shape): + if shape.has_text_frame: + yield path, shape.text_frame + if shape.has_table: + for row_index, row in enumerate(shape.table.rows): + for column_index, cell in enumerate(row.cells): + yield f"{path}/table[{row_index},{column_index}]", cell.text_frame + + candidates = [ - (i, p, sh) + (i, location, text_frame) for i, s in enumerate(prs6.slides) for p, sh in iter_shapes_with_path(s.shapes) - if sh.has_text_frame and old_w in sh.text_frame.text + for location, text_frame in iter_text_targets(p, sh) + if old_w in text_frame.text ] check("locator reaches text inside the group", len(candidates) == 1, [(i, p) for i, p, _ in candidates]) check("locator reports a stable nested path", "/" in candidates[0][1], candidates[0][1]) _, _, target = candidates[0] -target.text_frame.paragraphs[0].runs[0].text = new_w +target.paragraphs[0].runs[0].text = new_w prs6.save("group-edited.pptx") prs_g = Presentation("group-edited.pptx") found = [sh for sh in iter_shapes(prs_g.slides[0].shapes) if getattr(sh, "text_frame", None) is not None and sh.text_frame.text == new_w] check("group member edit persists after save", len(found) == 1) -# ---- analyze.md snippet: per-master theme font resolution ---------------------- -import re +table_candidates = [ + (i, location, text_frame) + for i, s in enumerate(Presentation("input.pptx").slides) + for p, sh in iter_shapes_with_path(s.shapes) + for location, text_frame in iter_text_targets(p, sh) + if old_cell in text_frame.text +] +check("locator reaches wording stored only in a table cell", len(table_candidates) == 1) +check("table-cell locator retains row and column", + table_candidates[0][1].endswith("/table[0,1]"), table_candidates[0][1]) + +# ---- analyze.md snippet: per-master, script-aware theme font resolution -------- +import xml.etree.ElementTree as ET prs7 = Presentation("input.pptx") theme_cache = {} +DRAWINGML = {"a": "http://schemas.openxmlformats.org/drawingml/2006/main"} + + +def read_theme_role(root, role): + node = root.find(f".//a:{role}Font", DRAWINGML) + if node is None: + return {"latin": "", "eastAsia": "", "complexScript": "", "scripts": {}} + + def typeface(tag): + child = node.find(f"a:{tag}", DRAWINGML) + return "" if child is None else child.get("typeface", "") + + return { + "latin": typeface("latin"), + "eastAsia": typeface("ea"), + "complexScript": typeface("cs"), + "scripts": { + child.get("script"): child.get("typeface", "") + for child in node.findall("a:font", DRAWINGML) if child.get("script") + }, + } def theme_faces_for_slide(slide): @@ -250,65 +292,112 @@ def theme_faces_for_slide(slide): cache_key = str(master_part.partname) if cache_key not in theme_cache: theme_part = master_part.part_related_by(RT.THEME) - theme_xml = theme_part.blob.decode("utf-8", "ignore") - matches = { - "major (headings)": re.search( - r'\s*\s*' + for script, face in (scripts or {}).items() + ) self.blob = ( - f'' - f'' - f'' + f'' + f'' + f'{script_nodes}' + f'' + f'{script_nodes}' f'' ).encode() class StubMasterPart: - def __init__(self, name, major, minor): + def __init__(self, name, major, minor, **theme_options): self.partname = name - self.theme_part = StubThemePart(major, minor) + self.theme_part = StubThemePart(major, minor, **theme_options) def part_related_by(self, relationship_type): assert relationship_type == RT.THEME return self.theme_part -def stub_slide(name, major, minor): - master = type("Master", (), {"part": StubMasterPart(name, major, minor)})() +def stub_slide(name, major, minor, **theme_options): + master = type("Master", (), {"part": StubMasterPart(name, major, minor, **theme_options)})() layout = type("Layout", (), {"slide_master": master})() return type("Slide", (), {"slide_layout": layout})() -_, first_faces = theme_faces_for_slide(stub_slide("/ppt/slideMasters/one.xml", "Head One", "Body One")) -_, second_faces = theme_faces_for_slide(stub_slide("/ppt/slideMasters/two.xml", "Head Two", "Body Two")) +_, first_fonts = theme_faces_for_slide(stub_slide("/ppt/slideMasters/one.xml", "Head One", "Body One")) +_, second_fonts = theme_faces_for_slide(stub_slide("/ppt/slideMasters/two.xml", "Head Two", "Body Two")) check( "different slide masters resolve their own theme faces", - first_faces["major (headings)"] == "Head One" - and second_faces["major (headings)"] == "Head Two" - and first_faces != second_faces, - (first_faces, second_faces), + first_fonts["major"]["latin"] == "Head One" + and second_fonts["major"]["latin"] == "Head Two" + and first_fonts != second_fonts, + (first_fonts, second_fonts), +) + +_, script_fonts = theme_faces_for_slide(stub_slide( + "/ppt/slideMasters/scripts.xml", "Latin Theme", "Latin Body", + east_asian="East Asian Theme", complex_script="Complex Script Theme", + scripts={"Hans": "Simplified Chinese Theme", "Cyrl": "Cyrillic Theme"}, +)) +check( + "CJK inherited-font triage includes east-Asian and script-specific faces", + {"East Asian Theme", "Simplified Chinese Theme"} + <= set(theme_candidates(script_fonts["major"], "汉字")), + theme_candidates(script_fonts["major"], "汉字"), +) +check( + "non-Latin inherited-font triage uses script mappings", + "Cyrillic Theme" in theme_candidates(script_fonts["minor"], "текст"), + theme_candidates(script_fonts["minor"], "текст"), ) check( diff --git a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py index e122b58..6b4c5d4 100644 --- a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py @@ -6,9 +6,9 @@ # python xlsx_fixture.py (deps: openpyxl) # python docx_fixture.py (deps: python-docx, pymupdf, soffice on PATH) import csv -import io import sys import zipfile +from tempfile import TemporaryFile import openpyxl @@ -57,7 +57,6 @@ def check(name, cond, extra=""): check("plain assignment is proven unsafe (negative control)", unsafe_wb.active["A1"].data_type == "f") # ---- edit.md snippet: round_trip_changes detects dropped parts AND stripped extensions ---- -from io import BytesIO wb = openpyxl.Workbook() ws = wb.active @@ -71,6 +70,7 @@ def check(name, cond, extra=""): # simulate an unsupported extension part (what a slicer/queries part looks like in the zip) payload["xl/slicers/slicer1.xml"] = b"" +payload["xl/media/large.bin"] = b"x14:" * 32_768 # binary payload must never be marker-scanned # simulate an in-part x14 extension that openpyxl will strip while keeping the part name EXT = (b""} -partial_after = {"xl/worksheets/sheet1.xml": b""} +inventory_names, inventory_markers = archive_inventory("extended.xlsx") +check("binary archive parts are named but never marker-scanned", + "xl/media/large.bin" in inventory_names and "xl/media/large.bin" not in inventory_markers) +partial_before = {"xl/worksheets/sheet1.xml": {"extLst", "x14"}} +partial_after = {"xl/worksheets/sheet1.xml": {"extLst"}} check( "marker comparison catches x14 loss while extLst survives", - stripped_extension_markers(partial_before, partial_after) + stripped_extension_markers(partial_before, partial_after, set(partial_before)) == [("xl/worksheets/sheet1.xml", "x14")], - stripped_extension_markers(partial_before, partial_after), + stripped_extension_markers(partial_before, partial_after, set(partial_before)), ) # ---- formatting.md snippet: sheet references built from the real sheet title ------- @@ -186,6 +216,80 @@ def sheet_ref(sheet): [(type(value).__name__, value) for value in regions], ) +# ---- edit.md structural audit includes non-cell dependencies ------------------ +from openpyxl.chart import BarChart, Reference +from openpyxl.formatting.rule import FormulaRule +from openpyxl.workbook.defined_name import DefinedName +from openpyxl.worksheet.datavalidation import DataValidation +from openpyxl.worksheet.table import Table + + +def non_cell_references(workbook): + refs = [] + for item in workbook.defined_names.values(): + refs.append(("defined name", item.name, item.attr_text)) + for sheet in workbook.worksheets: + owner = sheet.title + for table in sheet.tables.values(): + refs.append(("table", owner + "!" + table.name, table.ref)) + for merged_range in sheet.merged_cells.ranges: + refs.append(("merged range", owner, str(merged_range))) + if sheet.auto_filter.ref: + refs.append(("auto filter", owner, sheet.auto_filter.ref)) + for label, value in ( + ("print area", sheet.print_area), + ("print title rows", sheet.print_title_rows), + ("print title columns", sheet.print_title_cols), + ): + if value: + refs.append((label, owner, str(value))) + for validation in sheet.data_validations.dataValidation: + refs.append(("data validation range", owner, str(validation.sqref))) + for formula in (validation.formula1, validation.formula2): + if formula: + refs.append(("data validation formula", owner, str(formula))) + for conditional_range in sheet.conditional_formatting: + refs.append(("conditional formatting range", owner, str(conditional_range.sqref))) + for rule in sheet.conditional_formatting[conditional_range]: + for formula in getattr(rule, "formula", ()): + refs.append(("conditional formatting formula", owner, str(formula))) + for index, chart in enumerate(sheet._charts, start=1): + for element in chart._write().iter(): + if element.tag.rsplit("}", 1)[-1] == "f" and element.text: + refs.append(("chart series", f"{owner} chart {index}", element.text)) + return refs + + +audit_wb = openpyxl.Workbook() +audit_ws = audit_wb.active +audit_ws.title = "Audit" +audit_ws.append(["Value"]) +audit_ws.append([1]) +audit_ws.append([2]) +audit_wb.defined_names.add(DefinedName("AuditRange", attr_text="'Audit'!$A$2:$A$3")) +audit_ws.add_table(Table(displayName="AuditTable", ref="A1:A3")) +audit_ws.merge_cells("B2:B3") +audit_ws.auto_filter.ref = "A1:A3" +audit_ws.print_area = "A1:B3" +audit_ws.print_title_rows = "1:1" +audit_ws.print_title_cols = "A:A" +validation = DataValidation(type="whole", formula1="'Audit'!$A$2") +validation.add("A2:A3") +audit_ws.add_data_validation(validation) +audit_ws.conditional_formatting.add("A2:A3", FormulaRule(formula=["A2>0"])) +chart = BarChart() +chart.add_data(Reference(audit_ws, min_col=1, min_row=1, max_row=3), titles_from_data=True) +audit_ws.add_chart(chart, "C1") +reference_kinds = {kind for kind, _, _ in non_cell_references(audit_wb)} +check( + "structural audit covers names, tables, filters, validation, formatting, and charts", + {"defined name", "table", "merged range", "auto filter", "print area", + "print title rows", "print title columns", "data validation range", + "data validation formula", "conditional formatting range", + "conditional formatting formula", "chart series"} <= reference_kinds, + reference_kinds, +) + # and the edit itself still works after the warning path wb2 = openpyxl.load_workbook("plain.xlsx") wb2["Data"]["B2"] = "=B2*1" # formula stays a formula @@ -202,9 +306,12 @@ def sheet_ref(sheet): check("task-specific number format mapping is verified", format_matches) # ---- read.md snippet: multi-sheet profiles cover every sheet ---------------------- +from openpyxl.worksheet.formula import ArrayFormula + wb_h = openpyxl.Workbook() wb_h.active.title = "First" wb_h.active["A1"] = "=1+1" +wb_h.active["A2"] = ArrayFormula("A2:A3", "=ROW(A2:A3)") second = wb_h.create_sheet("Second") second["A1"] = "plain" second["A2"] = "=2+2" @@ -212,12 +319,29 @@ def sheet_ref(sheet): formula_wb = openpyxl.load_workbook("multi.xlsx", read_only=True, data_only=False) value_wb = openpyxl.load_workbook("multi.xlsx", read_only=True, data_only=True) profiled = list(value_wb.sheetnames) -uncached = {sn for sn in profiled +uncached = {(sn, fc.coordinate, getattr(fc.value, "text", None) or str(fc.value)) + for sn in profiled for frow, vrow in zip(formula_wb[sn].iter_rows(), value_wb[sn].iter_rows()) for fc, vc in zip(frow, vrow) - if isinstance(fc.value, str) and fc.value.startswith("=") and vc.value is None} + if fc.data_type == "f" and vc.value is None} check("multi-sheet profile iterates every sheet", profiled == ["First", "Second"], profiled) -check("uncached formulas found on both sheets", uncached == {"First", "Second"}, uncached) +check("uncached formulas found on both sheets", {item[0] for item in uncached} == {"First", "Second"}, uncached) +check("array-formula objects are detected by data_type", ("First", "A2", "=ROW(A2:A3)") in uncached, uncached) + +# csv.md: value export uses the cached-value workbook and reports every missing cache. +missing_caches = [] +with open("formula-values.csv", "w", newline="", encoding="utf-8") as output: + writer = csv.writer(output) + for formula_row, value_row in zip(formula_wb["First"].iter_rows(), value_wb["First"].iter_rows()): + for formula_cell, value_cell in zip(formula_row, value_row): + if formula_cell.data_type == "f" and value_cell.value is None: + missing_caches.append(formula_cell.coordinate) + writer.writerow([cell.value for cell in value_row]) +with open("formula-values.csv", newline="", encoding="utf-8") as exported: + exported_values = [value for row in csv.reader(exported) for value in row] +check("XLSX-to-CSV reports formulas with no cached value", set(missing_caches) >= {"A1", "A2"}, missing_caches) +check("XLSX-to-CSV does not leak formula strings into value output", + not any(value.startswith("=") for value in exported_values), exported_values) formula_wb.close() value_wb.close() From f2afc97a6eb8a6d2389c4ba2a42c7a5d3c88a62c Mon Sep 17 00:00:00 2001 From: Hylouis233 Date: Sun, 16 Aug 2026 13:57:35 +0800 Subject: [PATCH 09/34] fix(document-skills): address latest validation gaps --- .../skills/docx/references/cjk.md | 5 +- .../skills/docx/references/review.md | 2 +- .../skills/pdf/references/inspect.md | 17 +- .../skills/pdf/references/transform.md | 30 ++-- .../skills/pptx/references/analyze.md | 98 ++++++++++-- .../skills/xlsx/references/edit.md | 9 +- .../document-skills/tests/docx_fixture.py | 38 +++++ .../document-skills/tests/pdf_fixture.py | 58 ++++++- .../document-skills/tests/pptx_fixture.py | 146 ++++++++++++++++-- .../document-skills/tests/xlsx_fixture.py | 25 ++- 10 files changed, 385 insertions(+), 43 deletions(-) diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md b/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md index cd43ca0..66cc30a 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md @@ -103,6 +103,7 @@ section.left_margin, section.right_margin = Cm(2.8), Cm(2.6) from docx.text.paragraph import Paragraph doc = Document("output.docx") + normal_style = doc.styles["Normal"] def xml_runs(element, parent): # Covers direct paragraphs, tables, nested tables, and block content controls. @@ -131,7 +132,9 @@ section.left_margin, section.right_margin = Cm(2.8), Cm(2.6) direct = face_from_rpr(run._r.find(qn("w:rPr")), slot) if direct: return direct - for style in (run.style, run._parent.style, run.part.document.styles["Normal"]): + # Header/footer runs belong to a HeaderPart/FooterPart, which has no .document. + # Resolve the owning document's Normal style once rather than via run.part. + for style in (run.style, run._parent.style, normal_style): if face := next(style_faces(style, slot), None): return face raise LookupError(f"no resolved {slot} face for run {run.text!r}; resolve theme defaults") diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/review.md b/plugins/Hylouis233/document-skills/skills/docx/references/review.md index e06a894..99ad11a 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/review.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/review.md @@ -4,7 +4,7 @@ | Symptom | Likely cause | Fix route |---|---|---| -File will not open at all | broken ZIP (truncated, wrong repack) | inspect with `zipfile.ZipFile(path).testzip()`; if entries are damaged, recover from the user's original or prior version | +File will not open at all | broken ZIP (truncated, wrong repack) | run the bounded health check below; if entries are damaged, recover from the user's original or prior version | Opens with "unreadable content" repair prompt | content-types / rels mismatch, invalid XML | Tier 2 surgery: validate XML parses, check `[Content_Types].xml` covers every part extension | Text present but styles lost | document rebuilt from scratch instead of edited | redo as edit on the original package | Images missing | media parts not repacked or rels broken | verify `word/media/*` exist and `document.xml.rels` references them | diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md b/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md index 50f50f7..3602695 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md @@ -15,16 +15,21 @@ print("password_protected:", doc.needs_pass, "| still_encrypted:", doc.is_encrypted, "| pdf:", doc.is_pdf) for page in doc: drawings = page.get_drawings() + widgets = list(page.widgets() or ()) + annotations = list(page.annots() or ()) + links = page.get_links() print(page.number, page.rect, "text_len:", len(page.get_text()), "images:", len(page.get_images()), "drawings:", len(drawings), - "links:", len(page.get_links())) + "widgets:", len(widgets), "annotations:", len(annotations), + "links:", len(links)) ``` ## Checks worth automating -- **Blank page detection**: `not page.get_text().strip() and not page.get_images() and not - page.get_drawings()` -> flag; checking drawings avoids misclassifying vector-only pages as - blank. A blank page after generation usually means an overflowing flowable created it. +- **Blank page detection**: flag only when text, images, drawings, widgets, annotations, and + links are all absent. Interactive form fields are widgets rather than page text, so the + three-content-stream predicate alone would misclassify a usable form page as blank. A blank + page after generation usually means an overflowing flowable created it. - **Font inventory**: `page.get_fonts()` lists embedded names - needed when the user reports "looks different on machine X". - **Page size consistency**: mixed `page.rect` sizes in one file break duplex printing; report @@ -32,5 +37,5 @@ for page in doc: - **Damage**: `fitz.open` on a corrupt file raises or yields garbage - pair with `pypdf.PdfReader` cross-check when provenance is unknown. -Report findings as a table (page, size, text chars, images, drawings, links) - it is what every -downstream decision hangs off. +Report findings as a table (page, size, text chars, images, drawings, widgets, annotations, +links) - it is what every downstream decision hangs off. diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md b/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md index 68e25d6..9b62770 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md @@ -3,19 +3,27 @@ One tool for page-level structure changes: ```python +import os from pypdf import PdfReader, PdfWriter -reader = PdfReader("input.pdf") +def open_pdf(path): + reader = PdfReader(path) + if reader.is_encrypted: + password = os.environ.get("PDF_PASSWORD", "") + if reader.decrypt(password) == 0: + raise RuntimeError(f"Encrypted PDF {path}: set a valid PDF_PASSWORD") + return reader + +reader = open_pdf("input.pdf") writer = PdfWriter() -# Split: keep only pages 2-4 (0-based) -for i in range(1, 4): - writer.add_page(reader.pages[i]) +# Split: keep only pages 2-4 (0-based), remapping any retained outline entries. +writer.append(reader, pages=(1, 4), import_outline=True) -# Merge another file at the end -other = PdfReader("appendix.pdf") -for page in other.pages: - writer.add_page(page) +# Merge another file at the end. append() imports bookmarks/named destinations; +# copying other.pages one by one would silently discard that navigation structure. +other = open_pdf("appendix.pdf") +writer.append(other, import_outline=True) # Rotate a page 90 degrees writer.pages[0].rotate(90) @@ -30,9 +38,9 @@ with open("output.pdf", "wb") as f: Watermark / stamp by merging a stamp page onto each page: ```python -stamp = PdfReader("watermark.pdf").pages[0] +stamp = open_pdf("watermark.pdf").pages[0] stamp_text = (stamp.extract_text() or "").strip() -reader = PdfReader("input.pdf") +reader = open_pdf("input.pdf") expected_sizes = [tuple(float(value) for value in page.mediabox) for page in reader.pages] expected_fields = reader.get_fields() or {} writer = PdfWriter() @@ -68,7 +76,7 @@ Encryption and forms: ```python from pypdf import PdfReader, PdfWriter - reader = PdfReader("form.pdf") + reader = open_pdf("form.pdf") fields = reader.get_fields() or {} assert fields, "this PDF has no AcroForm form fields" diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md index 75e50d4..e19218d 100644 --- a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md @@ -14,6 +14,13 @@ def iter_shapes(shapes): else: yield shape +def cached_numeric_values(series, element_name): + """Read the cached numeric points python-pptx does not expose for XY/bubble axes.""" + return [ + node.text for node in + series._element.xpath(f"./c:{element_name}//c:pt/c:v") + ] + prs = Presentation("input.pptx") print("slide size:", prs.slide_width, prs.slide_height) for i, slide in enumerate(prs.slides): @@ -37,15 +44,29 @@ for i, slide in enumerate(prs.slides): ) plots = [] for plot in chart.plots: - categories = [ - [str(level) for level in label] - for label in plot.categories.flattened_labels - ] - series = [ - {"name": item.name, "values": list(item.values)} - for item in plot.series - ] - plots.append({"categories": categories, "series": series}) + plot_kind = type(plot).__name__ + if plot_kind in {"XyPlot", "BubblePlot"}: + series = [] + for item in plot.series: + values = { + "name": item.name, + "x_values": cached_numeric_values(item, "xVal"), + "y_values": cached_numeric_values(item, "yVal"), + } + if plot_kind == "BubblePlot": + values["bubble_sizes"] = cached_numeric_values(item, "bubbleSize") + series.append(values) + plots.append({"kind": plot_kind, "series": series}) + else: + categories = [ + [str(level) for level in label] + for label in plot.categories.flattened_labels + ] + series = [ + {"name": item.name, "values": list(item.values)} + for item in plot.series + ] + plots.append({"kind": plot_kind, "categories": categories, "series": series}) charts.append({"title": chart_title, "plots": plots}) pictures = [sh.name for sh in shapes if sh.shape_type == MSO_SHAPE_TYPE.PICTURE] @@ -70,9 +91,66 @@ layouts where the title placeholder is missing.) | Text is clipped or overflows its box | render every slide and inspect right/left and bottom/vertical fit; shape bounds do not measure laid-out text | reflow, resize the box, or reduce text/font size, then render again | | Everything shifted | slide size changed between sources | normalize slide size or re-layout on the target size | | Fonts look wrong elsewhere | non-embedded fonts (pptx rarely embeds) | report every effective font via the resolution chain below, not just explicit `run.font.name` values | -| File will not open | broken ZIP / part mismatch | same programmatic health check as DOCX: `zipfile.testzip()`, parse every `.xml` part | +| File will not open | broken ZIP / part mismatch | run the bounded package health check below before parsing every XML part | | Pictures blank | media parts missing or rels broken | verify `ppt/media/*` present and slide rels reference them | +## Bounded package health check + +Inspect declared sizes and compression ratios before decompressing anything. `ZipFile.testzip()` +must not be the first check because it expands every member, including an archive bomb. + +```python +import zipfile +from lxml import etree + +path = "input.pptx" +MAX_XML_PART = 20 * 1024 * 1024 +MAX_ENTRY = 100 * 1024 * 1024 +MAX_TOTAL_UNCOMPRESSED = 500 * 1024 * 1024 +MAX_COMPRESSION_RATIO = 200 +safe_xml_parser = etree.XMLParser( + load_dtd=False, + resolve_entities=False, + no_network=True, + huge_tree=False, + recover=False, +) + +with zipfile.ZipFile(path) as archive: + infos = archive.infolist() + names = {info.filename for info in infos} + assert len(names) == len(infos), "duplicate archive member names are unsafe" + assert "[Content_Types].xml" in names and "ppt/presentation.xml" in names + assert sum(info.file_size for info in infos) <= MAX_TOTAL_UNCOMPRESSED + + actual_total = 0 + for info in infos: + assert info.file_size <= MAX_ENTRY, f"oversized part: {info.filename}" + ratio = info.file_size / max(info.compress_size, 1) + assert ratio <= MAX_COMPRESSION_RATIO, f"suspicious compression ratio: {info.filename}" + is_xml = info.filename.endswith((".xml", ".rels")) + if is_xml: + assert info.file_size <= MAX_XML_PART, f"oversized XML part: {info.filename}" + + chunks = [] + actual_size = 0 + # Streaming to EOF verifies decompression and CRC only after metadata limits pass. + with archive.open(info) as stream: + while chunk := stream.read(64 * 1024): + actual_size += len(chunk) + actual_total += len(chunk) + assert actual_size <= MAX_ENTRY, f"part exceeded read limit: {info.filename}" + assert actual_total <= MAX_TOTAL_UNCOMPRESSED, "archive exceeded total read limit" + if is_xml: + chunks.append(chunk) + assert actual_size == info.file_size, f"size mismatch: {info.filename}" + if is_xml: + etree.fromstring(b"".join(chunks), parser=safe_xml_parser) +``` + +These are conservative triage defaults, not PPTX format limits. Raise a limit only for an +explicitly trusted large deck, and retain the per-member and streaming checks. + ## Font triage with inheritance Most template decks set no explicit `run.font.name`; the effective face is inherited from the diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md index 5d9bbeb..aa85b5f 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md @@ -11,7 +11,14 @@ ws = wb["Data"] def non_cell_references(workbook): """Inventory ranges/formulas that insert_rows/delete_rows will not rewrite.""" refs = [] - for item in workbook.defined_names.values(): + defined_names = workbook.defined_names + # openpyxl 3.1 exposes a dict-like mapping; 3.0 uses DefinedNameList. + defined_name_items = ( + defined_names.values() + if hasattr(defined_names, "values") + else defined_names.definedName + ) + for item in defined_name_items: refs.append(("defined name", item.name, item.attr_text)) for sheet in workbook.worksheets: owner = sheet.title diff --git a/plugins/Hylouis233/document-skills/tests/docx_fixture.py b/plugins/Hylouis233/document-skills/tests/docx_fixture.py index a77b90f..205c38f 100644 --- a/plugins/Hylouis233/document-skills/tests/docx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/docx_fixture.py @@ -135,6 +135,44 @@ def iter_body_paragraphs(parent, document): check("per-run cmap check identifies the actual missing glyph", per_run_missing == [("Latin Face", "漢")], per_run_missing) +# Header/footer runs do not expose run.part.document; resolve Normal from doc once. +header_doc = Document() +normal_style = header_doc.styles["Normal"] +normal_rfonts = normal_style._element.get_or_add_rPr().get_or_add_rFonts() +normal_rfonts.set(qn("w:eastAsia"), "Header CJK Face") +header_run = header_doc.sections[0].header.paragraphs[0].add_run("漢") + + +def fixture_face_from_rpr(rpr, slot): + if rpr is None: + return None + rfonts = rpr.find(qn("w:rFonts")) + return None if rfonts is None else rfonts.get(qn("w:" + slot)) + + +def fixture_style_faces(style, slot): + while style is not None: + face = fixture_face_from_rpr(style.element.find(qn("w:rPr")), slot) + if face: + yield face + style = style.base_style + + +def fixture_effective_face(run, slot): + direct = fixture_face_from_rpr(run._r.find(qn("w:rPr")), slot) + if direct: + return direct + for style in (run.style, run._parent.style, normal_style): + if face := next(fixture_style_faces(style, slot), None): + return face + raise LookupError(slot) + + +check("header part has no document back-reference (negative control)", + not hasattr(header_run.part, "document")) +check("header glyph validation resolves document Normal style", + fixture_effective_face(header_run, "eastAsia") == "Header CJK Face") + # ---- edit.md guarded cross-run replacement ------------------------------------ SAFE_RUN_CHILDREN = { qn("w:rPr"), qn("w:t"), qn("w:tab"), qn("w:br"), qn("w:cr"), diff --git a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py index 156e24a..f10b157 100644 --- a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py @@ -5,6 +5,7 @@ # python pptx_fixture.py (deps: python-pptx) # python xlsx_fixture.py (deps: openpyxl) # python docx_fixture.py (deps: python-docx, pymupdf, soffice on PATH) +import os import sys import fitz @@ -61,6 +62,42 @@ def check(name, cond, extra=""): check("authenticated PyMuPDF extraction reaches page text", "Application form" in encrypted_extract[0].get_text("text", sort=True)) + +def open_pdf(path): + reader = pypdf.PdfReader(path) + if reader.is_encrypted: + password = os.environ.get("PDF_PASSWORD", "") + if reader.decrypt(password) == 0: + raise RuntimeError(f"Encrypted PDF {path}: set a valid PDF_PASSWORD") + return reader + + +os.environ["PDF_PASSWORD"] = "fixture-password" +transform_encrypted = open_pdf("encrypted.pdf") +check("transform helper authenticates encrypted input before page access", + len(transform_encrypted.pages) == 2) + +# A page containing only an AcroForm widget is interactive content, not blank. +widget_canvas = canvas.Canvas("widget-only.pdf", pagesize=A4) +widget_canvas.acroForm.textfield( + name="widget_only", x=72, y=740, width=260, height=20, borderWidth=1, +) +widget_canvas.showPage() +widget_canvas.save() +widget_doc = fitz.open("widget-only.pdf") +widget_page = widget_doc[0] +widgets = list(widget_page.widgets() or ()) +annotations = list(widget_page.annots() or ()) +links = widget_page.get_links() +blank = ( + not widget_page.get_text().strip() + and not widget_page.get_images() + and not widget_page.get_drawings() + and not widgets and not annotations and not links +) +check("widget-only form page exposes a widget", len(widgets) == 1, len(widgets)) +check("widget-aware blank-page predicate keeps form page", not blank) + # ---- transform.md AcroForm snippet: clone into writer, fill on writer pages ---- from pypdf import PdfReader, PdfWriter @@ -82,6 +119,25 @@ def check(name, cond, extra=""): value = str((check_r.get_fields() or {}).get("applicant_name", {}).get("/V", "")) check("field value round-trips", value.strip("/") == "Ada Byron", repr(value)) +# ---- transform.md merge imports outline navigation ---------------------------- +appendix_writer = PdfWriter() +appendix_writer.add_blank_page(width=200, height=300) +appendix_writer.add_outline_item("Appendix bookmark", 0) +with open("appendix-outline.pdf", "wb") as f: + appendix_writer.write(f) + +merge_writer = PdfWriter() +merge_writer.append(open_pdf("form.pdf"), pages=(1, 2), import_outline=True) +merge_writer.append(open_pdf("appendix-outline.pdf"), import_outline=True) +with open("merged-outline.pdf", "wb") as f: + merge_writer.write(f) +merged_outline = PdfReader("merged-outline.pdf").outline +check( + "append imports the appended PDF outline", + any(getattr(item, "title", "") == "Appendix bookmark" for item in merged_outline), + merged_outline, +) + # ---- transform.md watermark snippet ------------------------------------------- from pypdf import PdfReader as R2 @@ -117,8 +173,6 @@ def check(name, cond, extra=""): pix = fitz.Pixmap(fitz.csCMYK, fitz.IRect(0, 0, 24, 24)) # CMYK pixmap like a CMYK PDF image converted = fitz.Pixmap(fitz.csRGB, pix) if pix.colorspace not in (fitz.csGRAY, fitz.csRGB) else pix converted.save("cmyk-converted.png") -import os - check("CMYK pixmap converts to a saved PNG", os.path.getsize("cmyk-converted.png") > 0) rgb = fitz.Pixmap("cmyk-converted.png") check("converted pixmap is RGB", "RGB" in str(rgb.colorspace), rgb.colorspace) diff --git a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py index 7815f26..fa147f8 100644 --- a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py @@ -6,9 +6,11 @@ # python xlsx_fixture.py (deps: openpyxl) # python docx_fixture.py (deps: python-docx, pymupdf, soffice on PATH) import sys +import zipfile +from lxml import etree from pptx import Presentation -from pptx.chart.data import ChartData +from pptx.chart.data import BubbleChartData, ChartData, XyChartData from pptx.dml.color import RGBColor from pptx.enum.chart import XL_CHART_TYPE from pptx.enum.shapes import MSO_SHAPE_TYPE @@ -139,6 +141,13 @@ def iter_shapes(shapes): yield shape +def cached_numeric_values(series, element_name): + return [ + node.text for node in + series._element.xpath(f"./c:{element_name}//c:pt/c:v") + ] + + def extract_slide_content(slide): shapes = list(iter_shapes(slide.shapes)) text = [sh.text_frame.text for sh in shapes if sh.has_text_frame and sh.text_frame.text] @@ -157,15 +166,29 @@ def extract_slide_content(slide): ) plots = [] for plot in chart.plots: - categories = [ - [str(level) for level in label] - for label in plot.categories.flattened_labels - ] - series = [ - {"name": item.name, "values": list(item.values)} - for item in plot.series - ] - plots.append({"categories": categories, "series": series}) + plot_kind = type(plot).__name__ + if plot_kind in {"XyPlot", "BubblePlot"}: + series = [] + for item in plot.series: + values = { + "name": item.name, + "x_values": cached_numeric_values(item, "xVal"), + "y_values": cached_numeric_values(item, "yVal"), + } + if plot_kind == "BubblePlot": + values["bubble_sizes"] = cached_numeric_values(item, "bubbleSize") + series.append(values) + plots.append({"kind": plot_kind, "series": series}) + else: + categories = [ + [str(level) for level in label] + for label in plot.categories.flattened_labels + ] + series = [ + {"name": item.name, "values": list(item.values)} + for item in plot.series + ] + plots.append({"kind": plot_kind, "categories": categories, "series": series}) charts.append({"title": chart_title, "plots": plots}) notes = slide.notes_slide.notes_text_frame.text if slide.has_notes_slide else "" return {"text": text, "tables": tables, "charts": charts, "notes": notes} @@ -184,6 +207,49 @@ def extract_slide_content(slide): ) check("content inventory emits notes text", "regional split" in content["notes"], content["notes"]) +# XY scatter and bubble plots do not have category/value-series semantics. +xy_prs = Presentation() +xy_slide = xy_prs.slides.add_slide(xy_prs.slide_layouts[6]) +xy_data = XyChartData() +xy_series = xy_data.add_series("XY series") +xy_series.add_data_point(1, 2) +xy_series.add_data_point(3, 4) +xy_slide.shapes.add_chart( + XL_CHART_TYPE.XY_SCATTER, + Inches(0.5), Inches(0.5), Inches(4), Inches(2.5), xy_data, +) +bubble_data = BubbleChartData() +bubble_series = bubble_data.add_series("Bubble series") +bubble_series.add_data_point(5, 6, 7) +xy_slide.shapes.add_chart( + XL_CHART_TYPE.BUBBLE, + Inches(0.5), Inches(3.5), Inches(4), Inches(2.5), bubble_data, +) +xy_prs.save("xy-bubble.pptx") +xy_content = extract_slide_content(Presentation("xy-bubble.pptx").slides[0]) +xy_plots = [plot for chart in xy_content["charts"] for plot in chart["plots"]] +check( + "scatter inventory emits x and y caches without category access", + any( + plot["kind"] == "XyPlot" + and plot["series"][0]["x_values"] == ["1", "3"] + and plot["series"][0]["y_values"] == ["2", "4"] + for plot in xy_plots + ), + xy_plots, +) +check( + "bubble inventory emits x, y, and bubble-size caches", + any( + plot["kind"] == "BubblePlot" + and plot["series"][0]["x_values"] == ["5"] + and plot["series"][0]["y_values"] == ["6"] + and plot["series"][0]["bubble_sizes"] == ["7"] + for plot in xy_plots + ), + xy_plots, +) + sp_element = slide6.shapes[-1]._element # the textbox; layout 5 still carries a Title placeholder group_element = parse_xml(GRP) @@ -426,5 +492,65 @@ def stub_slide(name, major, minor, **theme_options): detected_faces, ) +# ---- analyze.md bounded ZIP/XML health check ---------------------------------- +MAX_XML_PART = 20 * 1024 * 1024 +MAX_ENTRY = 100 * 1024 * 1024 +MAX_TOTAL_UNCOMPRESSED = 500 * 1024 * 1024 +MAX_COMPRESSION_RATIO = 200 +safe_xml_parser = etree.XMLParser( + load_dtd=False, + resolve_entities=False, + no_network=True, + huge_tree=False, + recover=False, +) + + +def validate_pptx_package(path): + with zipfile.ZipFile(path) as archive: + infos = archive.infolist() + names = {info.filename for info in infos} + assert len(names) == len(infos) + assert "[Content_Types].xml" in names and "ppt/presentation.xml" in names + assert sum(info.file_size for info in infos) <= MAX_TOTAL_UNCOMPRESSED + actual_total = 0 + for info in infos: + assert info.file_size <= MAX_ENTRY + assert info.file_size / max(info.compress_size, 1) <= MAX_COMPRESSION_RATIO + is_xml = info.filename.endswith((".xml", ".rels")) + if is_xml: + assert info.file_size <= MAX_XML_PART + chunks = [] + actual_size = 0 + with archive.open(info) as stream: + while chunk := stream.read(64 * 1024): + actual_size += len(chunk) + actual_total += len(chunk) + assert actual_size <= MAX_ENTRY + assert actual_total <= MAX_TOTAL_UNCOMPRESSED + if is_xml: + chunks.append(chunk) + assert actual_size == info.file_size + if is_xml: + etree.fromstring(b"".join(chunks), parser=safe_xml_parser) + + +try: + validate_pptx_package("input.pptx") + healthy_package_passed = True +except Exception: + healthy_package_passed = False +check("bounded package health check accepts an ordinary PPTX", healthy_package_passed) + +with zipfile.ZipFile("compressed-bomb.pptx", "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("[Content_Types].xml", "") + archive.writestr("ppt/presentation.xml", "" + (" " * 2_000_000) + "") +try: + validate_pptx_package("compressed-bomb.pptx") + archive_bomb_rejected = False +except AssertionError: + archive_bomb_rejected = True +check("PPTX compression bomb is rejected before XML expansion", archive_bomb_rejected) + print("\n" + ("ALL PPTX FIXTURES PASSED" if not failures else f"{len(failures)} FAILURES: {failures}")) sys.exit(0 if not failures else 1) diff --git a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py index 6b4c5d4..8d91f55 100644 --- a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py @@ -226,7 +226,13 @@ def sheet_ref(sheet): def non_cell_references(workbook): refs = [] - for item in workbook.defined_names.values(): + defined_names = workbook.defined_names + defined_name_items = ( + defined_names.values() + if hasattr(defined_names, "values") + else defined_names.definedName + ) + for item in defined_name_items: refs.append(("defined name", item.name, item.attr_text)) for sheet in workbook.worksheets: owner = sheet.title @@ -260,6 +266,23 @@ def non_cell_references(workbook): return refs +class LegacyDefinedNames: + """Minimal openpyxl 3.0-style DefinedNameList surface.""" + definedName = [DefinedName("LegacyRange", attr_text="'Legacy'!$A$1")] + + +class LegacyWorkbook: + defined_names = LegacyDefinedNames() + worksheets = [] + + +check( + "structural audit supports openpyxl 3.0 DefinedNameList", + non_cell_references(LegacyWorkbook()) + == [("defined name", "LegacyRange", "'Legacy'!$A$1")], +) + + audit_wb = openpyxl.Workbook() audit_ws = audit_wb.active audit_ws.title = "Audit" From 746e663d1bf9c5626a5b40e3acf396d10999d70a Mon Sep 17 00:00:00 2001 From: Hylouis233 Date: Sun, 16 Aug 2026 14:51:20 +0800 Subject: [PATCH 10/34] fix(document-skills): close latest review gaps - pdf extract: real table detection via page.find_tables() with span fallback - pdf postcheck: interactive-only pages (AcroForm widgets) exempt from text gate - docx read: unified block walker yields tables inside w:sdtContent - README: drop the untagged-ReportLab accessibility claim, state the limitation - xlsx: fullCalcOnLoad contract so manual-calc workbooks recalculate on open - fixtures: table detection, widget exemption, sdt table walker, calc flags (all green) --- plugins/Hylouis233/document-skills/README.md | 15 ++++--- .../skills/docx/references/read.md | 31 ++++++++------ .../document-skills/skills/pdf/SKILL.md | 23 ++++++++--- .../skills/pdf/references/extract.md | 22 +++++++++- .../document-skills/skills/xlsx/SKILL.md | 20 ++++++--- .../document-skills/tests/docx_fixture.py | 40 ++++++++++++++++++ .../document-skills/tests/pdf_fixture.py | 41 +++++++++++++++++++ .../document-skills/tests/xlsx_fixture.py | 26 ++++++++++++ 8 files changed, 186 insertions(+), 32 deletions(-) diff --git a/plugins/Hylouis233/document-skills/README.md b/plugins/Hylouis233/document-skills/README.md index fba9fc7..c0ffb91 100644 --- a/plugins/Hylouis233/document-skills/README.md +++ b/plugins/Hylouis233/document-skills/README.md @@ -54,8 +54,8 @@ Per format: page geometry) and scene patterns (academic paper, resume, official document, contract). - **xlsx** — openpyxl for reading, editing, styling, and native charts; formulas as formulas, never as pasted results; `data_only=True` only for reading cached values; date/number formats - applied explicitly; recalculation contract documented (openpyxl writes formulas, the viewer - calculates). Depth references: conditional formatting rules, structured tables, and honest + applied explicitly; recalculation contract documented (openpyxl writes formulas, the Skills + set `fullCalcOnLoad` so even manual-calc workbooks recalculate when a viewer opens them). Depth references: conditional formatting rules, structured tables, and honest pivot-style aggregation (openpyxl cannot create pivot tables; the reference gives the formula-sheet, frozen-values, and user-template routes). - **pptx** — python-pptx to build decks (7 common slide patterns: title, agenda, bullet, two @@ -63,10 +63,13 @@ Per format: rewriting of the whole XML; text measured against shape width with font-size reduction rules; presentation-level verification via `python-pptx` re-open plus a rendered PDF smoke test when LibreOffice is available. -- **pdf** — creation prefers ReportLab (structured, accessible text) over HTML-to-print paths; - extraction (text, coordinates, tables, images) and rasterization use PyMuPDF, while pypdf is - reserved for page-level transforms such as split, merge, rotate, watermark, encryption, and - forms; an explicit one-tool-per-job table prevents accidental API mixing. +- **pdf** — creation prefers ReportLab (real, selectable text with flowable structure) over + HTML-to-print paths; extraction (text, coordinates, tables, images) and rasterization use + PyMuPDF, while pypdf is reserved for page-level transforms such as split, merge, rotate, + watermark, encryption, and forms; an explicit one-tool-per-job table prevents accidental API + mixing. Note: ReportLab output is not tagged PDF/UA — when the user needs an accessible + (screen-reader-ready) PDF, the pdf Skill says to report that limitation honestly instead of + claiming accessibility. ## Verification-first output diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/read.md b/plugins/Hylouis233/document-skills/skills/docx/references/read.md index d7f5b0b..5991a2e 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/read.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/read.md @@ -16,35 +16,40 @@ tables). Prefer this when the goal is content, not coordinates. ```python from docx import Document from docx.oxml.ns import qn +from docx.table import Table from docx.text.paragraph import Paragraph -def iter_body_paragraphs(parent, document): - """Yield direct body paragraphs plus paragraphs nested in block content controls.""" +def iter_body_blocks(parent, document): + """Yield ("p", paragraph) / ("table", table) blocks, recursing into block content controls.""" for child in parent.iterchildren(): if child.tag == qn("w:p"): - yield Paragraph(child, document) + yield ("p", Paragraph(child, document)) + elif child.tag == qn("w:tbl"): + yield ("table", Table(child, document)) elif child.tag == qn("w:sdt"): content = child.find(qn("w:sdtContent")) if content is not None: - yield from iter_body_paragraphs(content, document) + yield from iter_body_blocks(content, document) doc = Document("input.docx") content_controls = list(doc.element.body.iter(qn("w:sdt"))) print("block content controls:", len(content_controls)) -for par in iter_body_paragraphs(doc.element.body, doc): - print(par.style.name, "|", par.text) -for t, table in enumerate(doc.tables): - for r, row in enumerate(table.rows): - print(t, r, [c.text for c in row.cells]) +for kind, block in iter_body_blocks(doc.element.body, doc): + if kind == "p": + print("p", block.style.name, "|", block.text) + else: + print("table", len(block.rows), "x", len(block.columns), + [[cell.text for cell in row.cells] for row in block.rows]) ``` Notes: -- `doc.paragraphs` includes only direct body paragraphs; it omits paragraphs nested in block - content controls (`w:sdt`). Use the traversal above and report the content-control count. - Text boxes, headers, footers, and footnotes still require their own collections +- `doc.paragraphs` includes only direct body paragraphs and `doc.tables` only top-level + tables; both omit content nested in block content controls (`w:sdt`). The walker above + yields paragraphs and tables inside `w:sdtContent` too, and reports the content-control + count. Tables nested inside table cells still require walking `cell.tables`; text boxes, + headers, footers, and footnotes still require their own collections (`section.header/.footer`) or raw XML. -- `doc.tables` is top-level only; nested tables require walking cells. - For revision/comment metadata, inspect the XML parts directly: `word/comments.xml`, `w:ins`/`w:del` elements in `word/document.xml`. diff --git a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md index fc779fa..c89e04f 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md @@ -66,10 +66,21 @@ page_texts = { number: (page.extract_text() or "").strip() for number, page in enumerate(r.pages, start=1) } + +def widget_count(page): + # Interactive-only pages (pure AcroForm screens) extract no text yet are valid output. + count = 0 + for ref in page.get("/Annots") or []: + if ref.get_object().get("/Subtype") == "/Widget": + count += 1 + return count + intentionally_raster_only_pages = set() missing_text_pages = [ number for number, text in page_texts.items() - if number not in intentionally_raster_only_pages and not text + if number not in intentionally_raster_only_pages + and not text + and widget_count(r.pages[number - 1]) == 0 ] assert not missing_text_pages, f"pages without extractable text: {missing_text_pages}" # Add task-specific checks when exact copy matters, for example @@ -96,8 +107,8 @@ assert not size_mismatches, f"unexpected page sizes: {size_mismatches}" ``` Confirm: page count matches the request; every page except those explicitly listed in -`intentionally_raster_only_pages` has extractable text; each requested key string is listed in -`expected_strings_by_page` and extracts on the correct page; every value in `page_sizes` is the -declared size within `page_size_tolerance`. Report all four. For pixel-sensitive work, render -every applicable page with PyMuPDF at 100 dpi and check that each render is non-blank (mean pixel -value). +`intentionally_raster_only_pages` has extractable text or at least one form widget (a pure +interactive page); each requested key string is listed in `expected_strings_by_page` and +extracts on the correct page; every value in `page_sizes` is the declared size within +`page_size_tolerance`. Report all four. For pixel-sensitive work, render every applicable page +with PyMuPDF at 100 dpi and check that each render is non-blank (mean pixel value). diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md b/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md index b341f06..3993ca1 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md @@ -20,7 +20,23 @@ for page in doc: print(f"--- page {page_number} ---") print(text) - # With coordinates (decide columns/reading order yourself) + # Tables: use real table detection (ruled and many borderless layouts), not span soup. + # Cells come back as strings, or None for merged/empty cells. Requires PyMuPDF >= 1.23. + try: + tables = page.find_tables() + except AttributeError: + tables = None + print(f"page {page_number}: PyMuPDF lacks find_tables; fall back to the span route below") + if tables is not None: + if not tables.tables: + print(f"page {page_number}: no table detected") + for t, table in enumerate(tables.tables, start=1): + print(f"page {page_number} table {t} bbox:", tuple(round(v) for v in table.bbox)) + for row in table.extract(): + print(row) + + # With coordinates (decide columns/reading order yourself); also the fallback + # when a table is visually present but find_tables detected nothing. for block in page.get_text("dict")["blocks"]: for line in block.get("lines", []): for span in line["spans"]: @@ -49,6 +65,10 @@ for page in doc: - Scanned pages return empty `get_text`. Check each page; if its text is empty and it contains images, report that page as "scanned, needs OCR" instead of claiming there is no text. +- Tables: run `find_tables()` first and extract rows/cells; only when detection returns nothing + but a table is visually present, reconstruct it from span coordinates - and say that + automatic detection failed. Merged cells arrive as `None`; preserve them, do not coerce to + empty strings silently. - "Sort by position" before emitting tables: spans come in internal order, not visual order; sort by `(round(bbox[1]), bbox[0])` for top-to-bottom, left-to-right reading. - Two-column layouts: cluster spans by x-gap before joining lines, or text interleaves columns. diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md b/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md index ba12b4f..6dbfc49 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md @@ -39,9 +39,13 @@ python -c "import openpyxl; print(openpyxl.__version__)" 3. **Types**: write `int`/`float`/`datetime`/`bool`, never formatted strings. Dates go in as `datetime` with `number_format='yyyy-mm-dd'`; currency as float plus `number_format='#,##0.00'` (or the locale-appropriate currency format string). -4. **Formulas are not recalculated by openpyxl.** After writing formulas you cannot read their - results back without opening the file in a real spreadsheet app; verify formula strings and - ranges structurally instead (see postcheck). +4. **Formulas are not recalculated by openpyxl.** After writing formulas, set + `wb.calculation.fullCalcOnLoad = True` before saving so Excel/WPS/LibreOffice recalculate + on open even when the workbook (typically one you loaded, which can carry + `fullCalcOnLoad=False`) uses calculation mode `manual` - check + `wb.calculation.calcMode`. You still cannot read results back without opening the file in + a real spreadsheet app; verify formula strings and ranges structurally instead (see + postcheck). 5. **Dimensions**: `ws.max_row`/`ws.max_column` reflect used range - trust them over guesses; but scan for trailing blank-but-formatted rows when a file "looks" bigger than its data. 6. Save to a new path first; overwrite only on explicit request. @@ -66,6 +70,9 @@ wb = openpyxl.load_workbook(output_path) print("sheets:", wb.sheetnames) missing = set(expected_sheets) - set(wb.sheetnames) assert not missing, f"missing expected sheets: {sorted(missing)}" +calc = wb.calculation +print("calcMode:", getattr(calc, "calcMode", None), + "fullCalcOnLoad:", getattr(calc, "fullCalcOnLoad", None)) for ws in wb.worksheets: print(f"{ws.title} dims:", ws.dimensions) formulas = [ @@ -82,6 +89,7 @@ wb.close() ``` Confirm: expected sheet names exist; used range matches expectations; intended formula cells -contain formula strings; every task-specific formatted cell is listed in -`expected_number_formats` and matches. Report what was verified and note that final -rendered values require opening in a spreadsheet application. +contain formula strings; when the task wrote formulas, the printout shows `fullCalcOnLoad: True` +(or `calcMode: auto`) so viewers will recalculate — otherwise set it and re-save; every +task-specific formatted cell is listed in `expected_number_formats` and matches. Report what +was verified and note that final rendered values require opening in a spreadsheet application. diff --git a/plugins/Hylouis233/document-skills/tests/docx_fixture.py b/plugins/Hylouis233/document-skills/tests/docx_fixture.py index 205c38f..2fc46cc 100644 --- a/plugins/Hylouis233/document-skills/tests/docx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/docx_fixture.py @@ -15,6 +15,7 @@ from docx.opc.constants import RELATIONSHIP_TYPE as RT from docx.oxml import OxmlElement from docx.oxml.ns import qn +from docx.table import Table from docx.text.paragraph import Paragraph from lxml import etree @@ -109,6 +110,18 @@ def iter_body_paragraphs(parent, document): yield from iter_body_paragraphs(content, document) +def iter_body_blocks(parent, document): + for child in parent.iterchildren(): + if child.tag == qn("w:p"): + yield ("p", Paragraph(child, document)) + elif child.tag == qn("w:tbl"): + yield ("table", Table(child, document)) + elif child.tag == qn("w:sdt"): + content = child.find(qn("w:sdtContent")) + if content is not None: + yield from iter_body_blocks(content, document) + + sdt_doc = Document() sdt_doc.add_paragraph("direct paragraph") sdt_paragraph = sdt_doc.add_paragraph("inside content control") @@ -117,6 +130,7 @@ def iter_body_paragraphs(parent, document): sdt_paragraph._p.getparent().replace(sdt_paragraph._p, sdt) sdt_content.append(sdt_paragraph._p) sdt.append(sdt_content) +sdt_doc.add_paragraph("trailing paragraph") sdt_doc.save("content-control.docx") sdt_reopened = Document("content-control.docx") check("doc.paragraphs omits block content-control text (negative control)", @@ -124,6 +138,32 @@ def iter_body_paragraphs(parent, document): walked_text = [paragraph.text for paragraph in iter_body_paragraphs(sdt_reopened.element.body, sdt_reopened)] check("content-control traversal emits the nested paragraph", "inside content control" in walked_text, walked_text) +# A table placed inside w:sdtContent is invisible to doc.tables; the block walker reaches it. +table_doc = Document() +table_doc.add_paragraph("before the control") +sdt_table = table_doc.add_table(rows=2, cols=2) +sdt_table.cell(0, 0).text = "ctrl-a1" +sdt_table.cell(1, 1).text = "ctrl-b2" +table_sdt = OxmlElement("w:sdt") +table_sdt_content = OxmlElement("w:sdtContent") +sdt_table._tbl.getparent().replace(sdt_table._tbl, table_sdt) +table_sdt_content.append(sdt_table._tbl) +table_sdt.append(table_sdt_content) +table_doc.add_paragraph("after the control") +table_doc.save("content-control-table.docx") +table_reopened = Document("content-control-table.docx") +check("doc.tables omits content-control tables (negative control)", len(table_reopened.tables) == 0) +blocks = list(iter_body_blocks(table_reopened.element.body, table_reopened)) +kinds = [kind for kind, _ in blocks] +nested_tables = [block for kind, block in blocks if kind == "table"] +check("block walker keeps document order across the control", + kinds == ["p", "table", "p"], kinds) +check("block walker surfaces the content-control table", len(nested_tables) == 1) +if nested_tables: + check("content-control table cells are readable", + nested_tables[0].cell(0, 0).text == "ctrl-a1" and nested_tables[0].cell(1, 1).text == "ctrl-b2", + [[cell.text for cell in row.cells] for row in nested_tables[0].rows]) + # Per-run glyph validation must not let a different referenced font hide a missing glyph. fixture_cmaps = {"CJK Face": {ord("漢")}, "Latin Face": {ord("A")}} assigned_runs = [("CJK Face", "漢"), ("Latin Face", "漢")] diff --git a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py index f10b157..fa027a7 100644 --- a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py @@ -98,6 +98,29 @@ def open_pdf(path): check("widget-only form page exposes a widget", len(widgets) == 1, len(widgets)) check("widget-aware blank-page predicate keeps form page", not blank) +# ---- SKILL.md postcheck: interactive-only pages are exempt from the text gate --- +def widget_count(page): + count = 0 + for ref in page.get("/Annots") or []: + if ref.get_object().get("/Subtype") == "/Widget": + count += 1 + return count + +widget_postcheck = pypdf.PdfReader("widget-only.pdf") +widget_text = (widget_postcheck.pages[0].extract_text() or "").strip() +check("widget-only page extracts no text", widget_text == "", repr(widget_text)) +check("postcheck counts the widget annotation", widget_count(widget_postcheck.pages[0]) == 1) +check("widget-only page passes the text postcheck via the widget exemption", + bool(widget_text) or widget_count(widget_postcheck.pages[0]) > 0) + +blank_writer = pypdf.PdfWriter() +blank_writer.add_blank_page(width=200, height=300) +with open("blank.pdf", "wb") as f: + blank_writer.write(f) +blank_r = pypdf.PdfReader("blank.pdf") +check("a truly blank page still fails the text postcheck", + not (bool((blank_r.pages[0].extract_text() or "").strip()) or widget_count(blank_r.pages[0]) > 0)) + # ---- transform.md AcroForm snippet: clone into writer, fill on writer pages ---- from pypdf import PdfReader, PdfWriter @@ -244,5 +267,23 @@ def open_pdf(path): raw_text[:120], ) +# ---- extract.md table route: find_tables instead of raw span soup --------------- +from reportlab.lib import colors +from reportlab.platypus import Table as RlTable, TableStyle + +rl_table = RlTable( + [["Region", "Sales"], ["North", "120"], ["South", "340"]], + style=TableStyle([("GRID", (0, 0), (-1, -1), 0.5, colors.black)]), +) +SimpleDocTemplate("table.pdf", pagesize=A4_SIZE).build([rl_table]) + +table_doc = fitz.open("table.pdf") +detected = table_doc[0].find_tables() +check("find_tables detects the drawn table", len(detected.tables) == 1, len(detected.tables)) +if detected.tables: + extracted_rows = detected.tables[0].extract() + check("find_tables extracts the header row", extracted_rows[0] == ["Region", "Sales"], extracted_rows) + check("find_tables extracts data rows", extracted_rows[2] == ["South", "340"], extracted_rows) + print("\n" + ("ALL PDF FIXTURES PASSED" if not failures else f"{len(failures)} FAILURES: {failures}")) sys.exit(0 if not failures else 1) diff --git a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py index 8d91f55..3a579de 100644 --- a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py @@ -368,5 +368,31 @@ class LegacyWorkbook: formula_wb.close() value_wb.close() +# SKILL.md contract: fullCalcOnLoad makes viewers recalculate even in manual calc mode. +calc_wb = openpyxl.Workbook() +calc_ws = calc_wb.active +calc_ws["A1"] = 1 +calc_ws["A2"] = 2 +calc_ws["A3"] = "=SUM(A1:A2)" +calc_wb.calculation.calcMode = "manual" +calc_wb.calculation.fullCalcOnLoad = False # simulate a source that does not recalc on load +calc_wb.save("stale-calc.xlsx") +stale_reopened = openpyxl.load_workbook("stale-calc.xlsx") +check("workbook without fullCalcOnLoad round-trips the stale flag (negative control)", + not bool(getattr(stale_reopened.calculation, "fullCalcOnLoad", False)), + stale_reopened.calculation) +stale_reopened.calculation.fullCalcOnLoad = True +stale_reopened.save("manual-calc.xlsx") +calc_reopened = openpyxl.load_workbook("manual-calc.xlsx") +check("fullCalcOnLoad survives save/reload", + bool(getattr(calc_reopened.calculation, "fullCalcOnLoad", False)), + calc_reopened.calculation) +check("manual calc mode survives save/reload", + getattr(calc_reopened.calculation, "calcMode", None) == "manual", + calc_reopened.calculation) +check("reloaded formula cell still holds the formula string", + calc_reopened.active["A3"].value == "=SUM(A1:A2)", calc_reopened.active["A3"].value) +calc_reopened.close() + print("\n" + ("ALL XLSX FIXTURES PASSED" if not failures else f"{len(failures)} FAILURES: {failures}")) sys.exit(0 if not failures else 1) From b6b0fdb7ea31476401430551b54c61685d23e7ca Mon Sep 17 00:00:00 2001 From: Hylouis233 Date: Sun, 16 Aug 2026 15:52:32 +0800 Subject: [PATCH 11/34] fix(document-skills): close latest review round - pptx analyze: sparse XY/bubble cache points keep their idx so x/y/bubble values pair correctly across blank points - docx cjk: Hangul (jamo, compatibility jamo, extended-A, syllables) routes through the eastAsia font slot - xlsx edit: extension detection matches namespace URIs and the local name extLst instead of arbitrary XML prefixes - xlsx formatting: header-only sheets skip conditional formatting instead of building inverted ranges openpyxl rejects - docx read: table extraction walks real w:tc elements and annotates gridSpan/vMerge instead of the merge-expanded row.cells view - pdf transform: stamps are scaled and centered per destination page via merge_transformed_page; rotated pages flagged for visual verification - fixtures extended for all six (with negative controls) --- .../skills/docx/references/cjk.md | 6 +- .../skills/docx/references/read.md | 27 +++++++- .../skills/pdf/references/transform.md | 15 ++++- .../skills/pptx/references/analyze.md | 23 ++++--- .../skills/xlsx/references/edit.md | 9 ++- .../skills/xlsx/references/formatting.md | 41 ++++++------ .../document-skills/tests/docx_fixture.py | 60 ++++++++++++++++++ .../document-skills/tests/pdf_fixture.py | 62 +++++++++++++++++++ .../document-skills/tests/pptx_fixture.py | 30 +++++++++ .../document-skills/tests/xlsx_fixture.py | 37 ++++++++++- 10 files changed, 277 insertions(+), 33 deletions(-) diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md b/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md index 66cc30a..26fee57 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md @@ -141,8 +141,12 @@ section.left_margin, section.right_margin = Cm(2.8), Cm(2.6) def font_slot(character): codepoint = ord(character) + # Word renders Korean through the East Asian slot as well: Hangul Jamo, + # compatibility Jamo, extended-A, and the syllable blocks. return "eastAsia" if ( - 0x2E80 <= codepoint <= 0x9FFF or 0xF900 <= codepoint <= 0xFAFF + 0x1100 <= codepoint <= 0x11FF or 0x2E80 <= codepoint <= 0x9FFF + or 0x3130 <= codepoint <= 0x318F or 0xA960 <= codepoint <= 0xA97F + or 0xAC00 <= codepoint <= 0xD7FF or 0xF900 <= codepoint <= 0xFAFF or 0x20000 <= codepoint <= 0x3134F ) else ("ascii" if codepoint < 128 else "hAnsi") diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/read.md b/plugins/Hylouis233/document-skills/skills/docx/references/read.md index 5991a2e..6b13eea 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/read.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/read.md @@ -31,6 +31,30 @@ def iter_body_blocks(parent, document): if content is not None: yield from iter_body_blocks(content, document) +def table_matrix(table): + """Rows of cell text with merge structure preserved. + + python-docx's row.cells repeats a merge-origin cell across every grid + position it spans, which hides the real structure. Walk the actual w:tc + elements and annotate gridSpan/vMerge instead. + """ + rows = [] + for row in table.rows: + cells = [] + for tc in row._tr.tc_lst: + tc_pr = tc.find(qn("w:tcPr")) + grid_span = tc_pr.find(qn("w:gridSpan")) if tc_pr is not None else None + v_merge = tc_pr.find(qn("w:vMerge")) if tc_pr is not None else None + note = "" + if grid_span is not None: + note += "(span {})".format(grid_span.get(qn("w:val"))) + if v_merge is not None: + note += "(vmerge start)" if v_merge.get(qn("w:val")) == "restart" else "(vmerge cont.)" + text = "".join(node.text or "" for node in tc.iter(qn("w:t"))) + cells.append(text + note if note else text) + rows.append(cells) + return rows + doc = Document("input.docx") content_controls = list(doc.element.body.iter(qn("w:sdt"))) print("block content controls:", len(content_controls)) @@ -38,8 +62,7 @@ for kind, block in iter_body_blocks(doc.element.body, doc): if kind == "p": print("p", block.style.name, "|", block.text) else: - print("table", len(block.rows), "x", len(block.columns), - [[cell.text for cell in row.cells] for row in block.rows]) + print("table", len(block.rows), "x", len(block.columns), table_matrix(block)) ``` Notes: diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md b/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md index 9b62770..6b45905 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md @@ -38,6 +38,8 @@ with open("output.pdf", "wb") as f: Watermark / stamp by merging a stamp page onto each page: ```python +from pypdf import Transformation + stamp = open_pdf("watermark.pdf").pages[0] stamp_text = (stamp.extract_text() or "").strip() reader = open_pdf("input.pdf") @@ -45,8 +47,19 @@ expected_sizes = [tuple(float(value) for value in page.mediabox) for page in rea expected_fields = reader.get_fields() or {} writer = PdfWriter() writer.append(reader) # clone pages plus catalog entries such as /AcroForm + +# A plain merge_page() overlays the stamp in its own coordinates, so on a page +# with different dimensions, origin, or rotation the stamp can be clipped or +# land entirely off-page. Scale each copy to fit its destination and center it. +sw, sh = float(stamp.mediabox.width), float(stamp.mediabox.height) for page in writer.pages: - page.merge_page(stamp) # stamp content on top; use merge_transformed_page to place + dw, dh = float(page.mediabox.width), float(page.mediabox.height) + scale = min(dw / sw, dh / sh) + tx = (dw - sw * scale) / 2 - float(stamp.mediabox.left) * scale + ty = (dh - sh * scale) / 2 - float(stamp.mediabox.bottom) * scale + page.merge_transformed_page(stamp, Transformation().scale(scale).translate(tx, ty)) +# Rotated destination pages (/Rotate != 0) apply the transformation in +# unrotated page space: verify rotated pages visually after stamping. with open("watermarked.pdf", "wb") as f: writer.write(f) diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md index e19218d..e0b3973 100644 --- a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md @@ -5,6 +5,7 @@ ```python from pptx import Presentation from pptx.enum.shapes import MSO_SHAPE_TYPE +from pptx.oxml.ns import qn def iter_shapes(shapes): """Walk shapes recursively so content nested inside group shapes is counted too.""" @@ -14,11 +15,17 @@ def iter_shapes(shapes): else: yield shape -def cached_numeric_values(series, element_name): - """Read the cached numeric points python-pptx does not expose for XY/bubble axes.""" +def cached_numeric_points(series, element_name): + """Read cached numeric points python-pptx does not expose for XY/bubble axes. + + Returns (idx, value) pairs: a series with blank points omits those + entries while later points keep their original idx, so values extracted + without their indices cannot be paired across the x/y/bubble axes. + """ return [ - node.text for node in - series._element.xpath(f"./c:{element_name}//c:pt/c:v") + (int(pt.get("idx")), pt.find(qn("c:v")).text) + for pt in series._element.xpath(f"./c:{element_name}//c:pt") + if pt.get("idx") is not None and pt.find(qn("c:v")) is not None ] prs = Presentation("input.pptx") @@ -50,11 +57,13 @@ for i, slide in enumerate(prs.slides): for item in plot.series: values = { "name": item.name, - "x_values": cached_numeric_values(item, "xVal"), - "y_values": cached_numeric_values(item, "yVal"), + # Pair x/y/bubble entries by idx; a missing idx marks a + # blank point and must not shift the pairing. + "x_points": cached_numeric_points(item, "xVal"), + "y_points": cached_numeric_points(item, "yVal"), } if plot_kind == "BubblePlot": - values["bubble_sizes"] = cached_numeric_values(item, "bubbleSize") + values["bubble_points"] = cached_numeric_points(item, "bubbleSize") series.append(values) plots.append({"kind": plot_kind, "series": series}) else: diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md index aa85b5f..519234b 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md @@ -103,10 +103,13 @@ wb.save("input-edited.xlsx") import zipfile from tempfile import TemporaryFile + # XML prefixes are arbitrary - a valid workbook may bind the x14 namespace to + # "sx" or the markup-compatibility namespace to anything. Match the namespace + # URIs (and the prefix-independent local name extLst), never prefixes. EXTENSION_MARKERS = { - "extLst": b"' + 'series' + 'General' + '' + '1' + '5' + '9' + '' +) +from pptx.oxml.ns import qn as pptx_qn + +def cached_numeric_points(series_element, element_name): + return [ + (int(pt.get("idx")), pt.find(pptx_qn("c:v")).text) + for pt in series_element.xpath(f"./c:{element_name}//c:pt") + if pt.get("idx") is not None and pt.find(pptx_qn("c:v")) is not None + ] + +from pptx.oxml import parse_xml as pptx_parse_xml +sparse_series = pptx_parse_xml(sparse_chart_xml) +points = cached_numeric_points(sparse_series, "xVal") +check("sparse cache points keep their idx", points == [(0, "1"), (2, "5"), (5, "9")], points) +compact = [value for _, value in points] +check("compact extraction is provably lossy (negative control)", + compact == ["1", "5", "9"] and len({idx for idx, _ in points}) == len(points)) + + print("\n" + ("ALL PPTX FIXTURES PASSED" if not failures else f"{len(failures)} FAILURES: {failures}")) sys.exit(0 if not failures else 1) diff --git a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py index 3a579de..76e18d8 100644 --- a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py @@ -218,7 +218,7 @@ def sheet_ref(sheet): # ---- edit.md structural audit includes non-cell dependencies ------------------ from openpyxl.chart import BarChart, Reference -from openpyxl.formatting.rule import FormulaRule +from openpyxl.formatting.rule import CellIsRule, FormulaRule from openpyxl.workbook.defined_name import DefinedName from openpyxl.worksheet.datavalidation import DataValidation from openpyxl.worksheet.table import Table @@ -394,5 +394,40 @@ class LegacyWorkbook: calc_reopened.active["A3"].value == "=SUM(A1:A2)", calc_reopened.active["A3"].value) calc_reopened.close() + +# ---- edit.md snippet: extension detection is prefix-independent ------------------ +X14_URI = b"http://schemas.microsoft.com/office/spreadsheetml/2009/9/main" +MC_URI = b"http://schemas.openxmlformats.org/markup-compatibility/2006" +EXTENSION_MARKERS = { + "extLst": b"extLst", + "x14 namespace": X14_URI, + "markup compatibility": MC_URI, +} + +def markers_in(data): + return {label for label, marker in EXTENSION_MARKERS.items() if marker in data} + +custom_prefix_sheet = ( + b'' + b'' +) +found = markers_in(custom_prefix_sheet) +check("namespace markers detect custom-prefix x14 extensions", "x14 namespace" in found, found) +check("namespace markers detect custom-prefix markup compatibility", "markup compatibility" in found, found) +check("local-name marker detects any-prefix extLst", "extLst" in found, found) +legacy_prefix_markers = {b"x14:", b"mc:AlternateContent"} +check("prefix markers are provably blind to custom prefixes (negative control)", + not any(marker in custom_prefix_sheet for marker in legacy_prefix_markers)) + +# ---- formatting.md guard: header-only sheets skip conditional formatting --------- +header_wb = openpyxl.Workbook() +header_ws = header_wb.active +header_ws.append(["Qty", "Note"]) +guard_last = header_ws.max_row +if guard_last < 2: + pass # guarded route: skip building rules + print("\n" + ("ALL XLSX FIXTURES PASSED" if not failures else f"{len(failures)} FAILURES: {failures}")) sys.exit(0 if not failures else 1) From 98af2fc0938a034ed82e63c69b238a657df30c01 Mon Sep 17 00:00:00 2001 From: Hylouis233 Date: Sun, 16 Aug 2026 16:20:59 +0800 Subject: [PATCH 12/34] fix(document-skills): close latest review round - pdf postcheck: real overflow check - text blocks extracted through an enlarged clip rectangle and compared to the page box (plain extraction silently drops fully off-page text) - docx read: table cells rebuilt per paragraph with visible tab/break markers instead of concatenated w:t text - docx/pptx health checks: explicit require() raises so every archive limit survives python -O - pptx analyze: run fonts resolve a:latin/a:ea/a:cs by the run's scripts before falling back to the Latin-only font.name; triage walks table cells (graphic frames have no text frame) - docx SKILL: pandoc-only installs can serve read/extract requests - pdf extract: inline content-stream images enumerated via dict image blocks alongside the XObject loop; coordinate spans position-sorted before emission - xlsx read: implausible sheet reset before streaming - fixtures extended for the overflow check (with negative control), cell paragraph boundaries, dimension corruption, script faces, table triage --- .../document-skills/skills/docx/SKILL.md | 7 +- .../skills/docx/references/read.md | 21 ++++- .../skills/docx/references/review.md | 27 ++++-- .../document-skills/skills/pdf/SKILL.md | 31 ++++++- .../skills/pdf/references/extract.md | 27 ++++-- .../skills/pptx/references/analyze.md | 83 +++++++++++++++---- .../skills/xlsx/references/read.md | 9 ++ .../document-skills/tests/docx_fixture.py | 38 ++++++++- .../document-skills/tests/pdf_fixture.py | 33 ++++++++ .../document-skills/tests/pptx_fixture.py | 59 ++++++++++++- .../document-skills/tests/xlsx_fixture.py | 32 +++++++ 11 files changed, 330 insertions(+), 37 deletions(-) diff --git a/plugins/Hylouis233/document-skills/skills/docx/SKILL.md b/plugins/Hylouis233/document-skills/skills/docx/SKILL.md index cbc587e..08b26dc 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/SKILL.md +++ b/plugins/Hylouis233/document-skills/skills/docx/SKILL.md @@ -18,7 +18,12 @@ python -c "import docx; print(docx.__version__ if hasattr(docx,'__version__') el ``` - `python-docx` present -> full create/read/edit support. -- Missing -> report it (`pip install python-docx`) and stop; do not hand-write OOXML. +- `python-docx` missing but `pandoc` present -> read/extract requests can still + be served by the pandoc route in [references/read.md](references/read.md); + report that structural reads, edits, and creation are unavailable + (`pip install python-docx`) and stop only for those routes. Never hand-write + OOXML as a substitute. +- Both missing -> report the missing tools and stop. - `pandoc` present -> prefer it for text/markdown extraction (fast, faithful). - `soffice` present -> enables the PDF smoke test in postcheck. diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/read.md b/plugins/Hylouis233/document-skills/skills/docx/references/read.md index 6b13eea..31cb092 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/read.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/read.md @@ -31,6 +31,25 @@ def iter_body_blocks(parent, document): if content is not None: yield from iter_body_blocks(content, document) +def tc_text(tc): + """Cell text rebuilt per paragraph, keeping tabs and breaks visible. + + Joining only the w:t descendants concatenates separate paragraphs + ("First" + "Second" -> "FirstSecond") and loses separators entirely. + """ + paragraphs = [] + for p in tc.iter(qn("w:p")): + pieces = [] + for node in p.iter(): + if node.tag == qn("w:t"): + pieces.append(node.text or "") + elif node.tag == qn("w:tab"): + pieces.append("") + elif node.tag in (qn("w:br"), qn("w:cr")): + pieces.append("
") + paragraphs.append("".join(pieces)) + return " / ".join(paragraphs) + def table_matrix(table): """Rows of cell text with merge structure preserved. @@ -50,7 +69,7 @@ def table_matrix(table): note += "(span {})".format(grid_span.get(qn("w:val"))) if v_merge is not None: note += "(vmerge start)" if v_merge.get(qn("w:val")) == "restart" else "(vmerge cont.)" - text = "".join(node.text or "" for node in tc.iter(qn("w:t"))) + text = tc_text(tc) cells.append(text + note if note else text) rows.append(cells) return rows diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/review.md b/plugins/Hylouis233/document-skills/skills/docx/references/review.md index 99ad11a..8e5c1c8 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/review.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/review.md @@ -21,6 +21,13 @@ MAX_XML_PART = 20 * 1024 * 1024 MAX_ENTRY = 100 * 1024 * 1024 MAX_TOTAL_UNCOMPRESSED = 500 * 1024 * 1024 MAX_COMPRESSION_RATIO = 200 + +# Security limits must survive `python -O` (which strips assert statements), +# so every check raises explicitly instead of asserting. +def require(condition, message): + if not condition: + raise ValueError(message) + safe_xml_parser = etree.XMLParser( load_dtd=False, resolve_entities=False, @@ -31,17 +38,19 @@ safe_xml_parser = etree.XMLParser( with zipfile.ZipFile(path) as z: infos = z.infolist() names = {info.filename for info in infos} - assert len(names) == len(infos), "duplicate archive member names are unsafe" - assert "[Content_Types].xml" in names and "word/document.xml" in names - assert sum(info.file_size for info in infos) <= MAX_TOTAL_UNCOMPRESSED + require(len(names) == len(infos), "duplicate archive member names are unsafe") + require("[Content_Types].xml" in names and "word/document.xml" in names, + "missing required OPC members") + require(sum(info.file_size for info in infos) <= MAX_TOTAL_UNCOMPRESSED, + "declared total uncompressed size above limit") actual_total = 0 for info in infos: - assert info.file_size <= MAX_ENTRY, f"oversized part: {info.filename}" + require(info.file_size <= MAX_ENTRY, f"oversized part: {info.filename}") ratio = info.file_size / max(info.compress_size, 1) - assert ratio <= MAX_COMPRESSION_RATIO, f"suspicious compression ratio: {info.filename}" + require(ratio <= MAX_COMPRESSION_RATIO, f"suspicious compression ratio: {info.filename}") is_xml = info.filename.endswith((".xml", ".rels")) if is_xml: - assert info.file_size <= MAX_XML_PART, f"oversized XML part: {info.filename}" + require(info.file_size <= MAX_XML_PART, f"oversized XML part: {info.filename}") chunks = [] actual_size = 0 # Stream every bounded member to verify decompression and CRC. Do not call testzip() @@ -50,11 +59,11 @@ with zipfile.ZipFile(path) as z: while chunk := stream.read(64 * 1024): actual_size += len(chunk) actual_total += len(chunk) - assert actual_size <= MAX_ENTRY, f"part exceeded read limit: {info.filename}" - assert actual_total <= MAX_TOTAL_UNCOMPRESSED, "archive exceeded total read limit" + require(actual_size <= MAX_ENTRY, f"part exceeded read limit: {info.filename}") + require(actual_total <= MAX_TOTAL_UNCOMPRESSED, "archive exceeded total read limit") if is_xml: chunks.append(chunk) - assert actual_size == info.file_size, f"size mismatch: {info.filename}" + require(actual_size == info.file_size, f"size mismatch: {info.filename}") if is_xml: etree.fromstring(b"".join(chunks), parser=safe_xml_parser) ``` diff --git a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md index c89e04f..57a2b52 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md @@ -104,11 +104,40 @@ size_mismatches = [ for value, expected in zip(actual, expected_page_size)) ] assert not size_mismatches, f"unexpected page sizes: {size_mismatches}" + +# Overflow is a defect (shared rule 4): text that runs past the page box is +# clipped or off-page even though every check above still passes. Plain block +# extraction silently drops fully off-page text, so extract through an +# explicitly enlarged clip rectangle and compare the block boxes to the page. +import fitz + +overflow_doc = fitz.open(output_path) +if overflow_doc.needs_pass: + if not password: + raise RuntimeError("set PDF_PASSWORD so the encrypted output can be overflow-checked") + overflow_doc.authenticate(password) +overflow_pages = [] +for page in overflow_doc: + clip = fitz.Rect(-2000, -2000, page.rect.width + 2000, page.rect.height + 2000) + text_blocks = [b for b in page.get_text("blocks", clip=clip) if b[6] == 0] + beyond_box = any( + b[0] < -0.5 or b[1] < -0.5 or b[2] > page.rect.width + 0.5 or b[3] > page.rect.height + 0.5 + for b in text_blocks + ) + if beyond_box: + overflow_pages.append(page.number + 1) +overflow_doc.close() +assert not overflow_pages, f"text blocks extend past the page box on pages: {overflow_pages}" +# The page box is the hard bound. When the task declares specific margins, +# additionally check key blocks against them (or render and inspect visually) - +# content inside the box but past a declared margin is a softer, task-specific +# defect to report. ``` Confirm: page count matches the request; every page except those explicitly listed in `intentionally_raster_only_pages` has extractable text or at least one form widget (a pure interactive page); each requested key string is listed in `expected_strings_by_page` and extracts on the correct page; every value in `page_sizes` is the declared size within -`page_size_tolerance`. Report all four. For pixel-sensitive work, render every applicable page +`page_size_tolerance`; and no page's text blocks extend past the page box (the overflow +check). Report all five. For pixel-sensitive work, render every applicable page with PyMuPDF at 100 dpi and check that each render is non-blank (mean pixel value). diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md b/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md index 3993ca1..6eb5629 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md @@ -37,14 +37,20 @@ for page in doc: # With coordinates (decide columns/reading order yourself); also the fallback # when a table is visually present but find_tables detected nothing. - for block in page.get_text("dict")["blocks"]: - for line in block.get("lines", []): - for span in line["spans"]: - print(page_number, round(span["bbox"][0]), - round(span["bbox"][1]), span["text"]) + # Collect and sort first: spans arrive in content-stream order, which can + # differ from the visual order even when plain text extraction is sorted. + spans = [ + (page_number, round(span["bbox"][0]), round(span["bbox"][1]), span["text"]) + for block in page.get_text("dict")["blocks"] + for line in block.get("lines", []) + for span in line["spans"] + ] + for record in sorted(spans, key=lambda item: (item[2], item[1])): + print(*record) # Images. Apply a soft mask (xref at info[1]) before saving or transparency is lost. # Pixmap keeps the image's own colorspace: convert CMYK/ICC bases to RGB first. + # This loop sees image XObjects only. for i, info in enumerate(page.get_images(full=True), start=1): base = fitz.Pixmap(doc, info[0]) if base.colorspace and base.colorspace not in (fitz.csGRAY, fitz.csRGB): @@ -56,6 +62,17 @@ for page in doc: pix = base pix.save(f"img-p{page_number}-{i}.png") + # Inline images live in the page content stream, not the XObject table, so + # get_images() never lists them. Enumerate image blocks from the dict pass + # and report both lists: the xref loop is authoritative for XObjects, the + # block pass catches inline placements. + for b in page.get_text("dict")["blocks"]: + if b["type"] != 1: + continue + ext = b.get("ext") or "png" + with open(f"img-p{page_number}-inline-{b['number']}.{ext}", "wb") as fh: + fh.write(b["image"]) + # Rasterize (for visual checks or OCR preprocessing) pix = page.get_pixmap(dpi=150) pix.save(f"page-{page_number}.png") diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md index e0b3973..5b3de7d 100644 --- a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md @@ -117,6 +117,13 @@ MAX_XML_PART = 20 * 1024 * 1024 MAX_ENTRY = 100 * 1024 * 1024 MAX_TOTAL_UNCOMPRESSED = 500 * 1024 * 1024 MAX_COMPRESSION_RATIO = 200 + +# Security limits must survive `python -O` (which strips assert statements), +# so every check raises explicitly instead of asserting. +def require(condition, message): + if not condition: + raise ValueError(message) + safe_xml_parser = etree.XMLParser( load_dtd=False, resolve_entities=False, @@ -128,18 +135,20 @@ safe_xml_parser = etree.XMLParser( with zipfile.ZipFile(path) as archive: infos = archive.infolist() names = {info.filename for info in infos} - assert len(names) == len(infos), "duplicate archive member names are unsafe" - assert "[Content_Types].xml" in names and "ppt/presentation.xml" in names - assert sum(info.file_size for info in infos) <= MAX_TOTAL_UNCOMPRESSED + require(len(names) == len(infos), "duplicate archive member names are unsafe") + require("[Content_Types].xml" in names and "ppt/presentation.xml" in names, + "missing required OPC members") + require(sum(info.file_size for info in infos) <= MAX_TOTAL_UNCOMPRESSED, + "declared total uncompressed size above limit") actual_total = 0 for info in infos: - assert info.file_size <= MAX_ENTRY, f"oversized part: {info.filename}" + require(info.file_size <= MAX_ENTRY, f"oversized part: {info.filename}") ratio = info.file_size / max(info.compress_size, 1) - assert ratio <= MAX_COMPRESSION_RATIO, f"suspicious compression ratio: {info.filename}" + require(ratio <= MAX_COMPRESSION_RATIO, f"suspicious compression ratio: {info.filename}") is_xml = info.filename.endswith((".xml", ".rels")) if is_xml: - assert info.file_size <= MAX_XML_PART, f"oversized XML part: {info.filename}" + require(info.file_size <= MAX_XML_PART, f"oversized XML part: {info.filename}") chunks = [] actual_size = 0 @@ -148,11 +157,11 @@ with zipfile.ZipFile(path) as archive: while chunk := stream.read(64 * 1024): actual_size += len(chunk) actual_total += len(chunk) - assert actual_size <= MAX_ENTRY, f"part exceeded read limit: {info.filename}" - assert actual_total <= MAX_TOTAL_UNCOMPRESSED, "archive exceeded total read limit" + require(actual_size <= MAX_ENTRY, f"part exceeded read limit: {info.filename}") + require(actual_total <= MAX_TOTAL_UNCOMPRESSED, "archive exceeded total read limit") if is_xml: chunks.append(chunk) - assert actual_size == info.file_size, f"size mismatch: {info.filename}" + require(actual_size == info.file_size, f"size mismatch: {info.filename}") if is_xml: etree.fromstring(b"".join(chunks), parser=safe_xml_parser) ``` @@ -237,26 +246,64 @@ def theme_candidates(role_fonts, text): return [face for face in dict.fromkeys(faces) if face] # 2. Per run: explicit value, else paragraph defaults, else report as inherited. +# run.font.name exposes only the LATIN typeface; a run that also declares a:ea +# or a:cs must be resolved per the scripts present in its own text first. +def explicit_run_face(run, text): + """The run's direct a:latin/a:ea/a:cs typeface chosen by its scripts (or None).""" + rPr = run._r.find(qn("a:rPr")) + if rPr is None: + return None + declared = {} + for slot, tag in (("latin", "a:latin"), ("eastAsia", "a:ea"), ("complexScript", "a:cs")): + node = rPr.find(qn(tag)) + if node is not None and node.get("typeface"): + declared[slot] = node.get("typeface") + if not declared: + return None + tags = script_tags(text) + if any(tag in ("Hans", "Hant", "Jpan", "Hang") for tag in tags) and "eastAsia" in declared: + return declared["eastAsia"] + if any(tag in ("Arab", "Hebr", "Deva") for tag in tags) and "complexScript" in declared: + return declared["complexScript"] + return declared.get("latin") + +def iter_text_frames(shapes): + """Shape text frames plus every table cell's text frame (a graphic frame + has has_text_frame=False, so tables must be walked explicitly).""" + for shape in shapes: + if shape.has_text_frame: + yield shape.text_frame + if getattr(shape, "has_table", False): + for row in shape.table.rows: + for cell in row.cells: + yield cell.text_frame + for i, slide in enumerate(prs.slides): master_name, theme_fonts = theme_faces_for_slide(slide) print(i, "master:", master_name, "theme:", ascii(theme_fonts)) title_shape = slide.shapes.title - for shape in iter_shapes(slide.shapes): - if not shape.has_text_frame: - continue - for paragraph in shape.text_frame.paragraphs: + for frame in iter_text_frames(iter_shapes(slide.shapes)): + holder = getattr(frame, "_parent", None) # the shape for ordinary frames + for paragraph in frame.paragraphs: for run in paragraph.runs: - if run.font.name: - face, source = run.font.name, "run" + explicit = explicit_run_face(run, run.text) + if explicit: + face, source = explicit, "run script face (a:latin/a:ea/a:cs)" + elif run.font.name: + face, source = run.font.name, "run latin" elif paragraph.font.name: - face, source = paragraph.font.name, "paragraph defaults" + # paragraph.font is the LATIN slot of a:pPr/a:defRPr only; + # script-specific paragraph defaults are not modeled by + # python-pptx and need raw XML for full coverage. + face, source = paragraph.font.name, "paragraph defaults (latin)" else: role = "major" if ( - title_shape is not None and shape._element is title_shape._element + title_shape is not None and + getattr(holder, "_element", None) is title_shape._element ) else "minor" face = theme_candidates(theme_fonts[role], run.text) source = f"inherited {role} theme candidates (verify placeholder chain/locale)" - print(i, shape.name, repr(run.text[:20]), "font:", ascii(face), "source:", source) + print(i, repr(run.text[:20]), "font:", ascii(face), "source:", source) ``` python-pptx does not evaluate the full placeholder -> layout -> master inheritance chain; when diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md index d0a64e3..3978007 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md @@ -16,6 +16,15 @@ def formula_text(cell): for sheet_name in value_wb.sheetnames: formula_ws = formula_wb[sheet_name] value_ws = value_wb[sheet_name] + # Read-only iteration is bounded by the sheet's metadata. + # Non-Excel producers write wrong dimensions, which silently truncates the + # stream; when the declared extent looks implausible, reset it and let + # openpyxl discover the real used range. + declared = value_ws.calculate_dimension() + if value_ws.max_row in (None, 0) or declared in ("A1:A1", "A1"): + value_ws.reset_dimensions() + formula_ws.reset_dimensions() + print(f"--- {sheet_name} --- implausible dimension {declared!r}; reset, real extent:") print(f"--- {sheet_name} --- dims:", value_ws.calculate_dimension()) rows = value_ws.iter_rows(values_only=True) diff --git a/plugins/Hylouis233/document-skills/tests/docx_fixture.py b/plugins/Hylouis233/document-skills/tests/docx_fixture.py index 5bc7e70..43de2bc 100644 --- a/plugins/Hylouis233/document-skills/tests/docx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/docx_fixture.py @@ -95,7 +95,7 @@ def validate_docx_package(path): try: validate_docx_package("compressed-bomb.docx") archive_bomb_rejected = False -except AssertionError: +except (AssertionError, ValueError): archive_bomb_rejected = True check("suspicious compression ratio is rejected before XML expansion", archive_bomb_rejected) @@ -507,5 +507,41 @@ def font_slot(character): check("non-CJK fullwidth-range-adjacent Latin-1 stays hAnsi", font_slot("é") == "hAnsi") + +# ---- read.md tc_text: paragraph boundaries survive cell extraction --------------- +def tc_text(tc): + paragraphs = [] + for p in tc.iter(qn("w:p")): + pieces = [] + for node in p.iter(): + if node.tag == qn("w:t"): + pieces.append(node.text or "") + elif node.tag == qn("w:tab"): + pieces.append("") + elif node.tag in (qn("w:br"), qn("w:cr")): + pieces.append("
") + paragraphs.append("".join(pieces)) + return " / ".join(paragraphs) + +cells_doc = Document() +cells_table = cells_doc.add_table(rows=1, cols=1) +cell = cells_table.cell(0, 0) +cell.paragraphs[0].text = "First" +cell.add_paragraph("Second") +run_with_tab = cell.paragraphs[0].add_run("") +tab_element = OxmlElement("w:tab") +run_with_tab._r.append(tab_element) +run_with_tab.add_text("after tab") +cells_doc.save("cell-paragraphs.docx") +cells_reopened = Document("cell-paragraphs.docx") +cells_tc = cells_reopened.tables[0].rows[0]._tr.tc_lst[0] +joined_raw = "".join(node.text or "" for node in cells_tc.iter(qn("w:t"))) +extracted = tc_text(cells_tc) +check("raw w:t joining concatenates paragraphs (negative control)", + "Firstafter tab" in joined_raw, joined_raw) +check("tc_text preserves the paragraph boundary", " / Second" in extracted, extracted) +check("tc_text keeps tabs visible", "after tab" in extracted, extracted) + + print("\n" + ("ALL DOCX FIXTURES PASSED" if not failures else f"{len(failures)} FAILURES: {failures}")) sys.exit(0 if not failures else 1) diff --git a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py index 53b0fb2..41e885b 100644 --- a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py @@ -347,5 +347,38 @@ def stamp_bboxes(path, page_number): check("mixed-size pages keep their original dimensions", [round(float(p.mediabox.width)) for p in scaled_check.pages] == [595, 595, 200]) + +# ---- SKILL.md overflow check: off-page text is a defect ------------------------- +overflow_ok = canvas.Canvas("overflow.pdf", pagesize=A4) +overflow_ok.setFont("Helvetica", 16) +overflow_ok.drawString(72, 780, "fits on page") +overflow_ok.showPage() +overflow_ok.save() +overflow_bad = canvas.Canvas("overflow-bad.pdf", pagesize=A4) +overflow_bad.setFont("Helvetica", 16) +overflow_bad.drawString(72, -200, "drawn far below the page box") +overflow_bad.showPage() +overflow_bad.save() + +def overflow_pages(path): + doc = fitz.open(path) + pages = [] + for page in doc: + # Plain block extraction drops fully off-page text; enlarge the clip. + clip = fitz.Rect(-2000, -2000, page.rect.width + 2000, page.rect.height + 2000) + blocks = [b for b in page.get_text("blocks", clip=clip) if b[6] == 0] + if any(b[0] < -0.5 or b[1] < -0.5 or b[2] > page.rect.width + 0.5 or b[3] > page.rect.height + 0.5 + for b in blocks): + pages.append(page.number + 1) + doc.close() + return pages + +check("in-bounds PDF reports no overflow pages", overflow_pages("overflow.pdf") == []) +check("off-page text is detected by the overflow check (negative control)", + overflow_pages("overflow-bad.pdf") == [1]) +check("off-page text still extracts, so extraction alone cannot catch it", + "drawn far below" in (pypdf.PdfReader("overflow-bad.pdf").pages[0].extract_text() or "")) + + print("\n" + ("ALL PDF FIXTURES PASSED" if not failures else f"{len(failures)} FAILURES: {failures}")) sys.exit(0 if not failures else 1) diff --git a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py index 1334718..fc6d64e 100644 --- a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py @@ -548,7 +548,7 @@ def validate_pptx_package(path): try: validate_pptx_package("compressed-bomb.pptx") archive_bomb_rejected = False -except AssertionError: +except (AssertionError, ValueError): archive_bomb_rejected = True check("PPTX compression bomb is rejected before XML expansion", archive_bomb_rejected) @@ -582,5 +582,62 @@ def cached_numeric_points(series_element, element_name): compact == ["1", "5", "9"] and len({idx for idx, _ in points}) == len(points)) + +# ---- analyze.md: script-aware run faces and table-cell triage -------------------- +from pptx.oxml import parse_xml as pptx_parse_xml2 +from pptx.oxml.ns import qn + +def explicit_run_face(run, text, script_tags_fn): + rPr = run._r.find(qn("a:rPr")) + if rPr is None: + return None + declared = {} + for slot, tag in (("latin", "a:latin"), ("eastAsia", "a:ea"), ("complexScript", "a:cs")): + node = rPr.find(qn(tag)) + if node is not None and node.get("typeface"): + declared[slot] = node.get("typeface") + if not declared: + return None + tags = script_tags_fn(text) + if any(tag in ("Hans", "Hant", "Jpan", "Hang") for tag in tags) and "eastAsia" in declared: + return declared["eastAsia"] + if any(tag in ("Arab", "Hebr", "Deva") for tag in tags) and "complexScript" in declared: + return declared["complexScript"] + return declared.get("latin") + +run_xml = ( + '' + '' + '\u6d4b\u8bd5' +) +dual_run = pptx_parse_xml2(run_xml) +class FakeRun: + _r = dual_run +check("a CJK run with latin+ea declared resolves to the eastAsia face", + explicit_run_face(FakeRun(), "\u6d4b\u8bd5", script_tags) == "SimSun") +check("run.font.name alone would report only the Latin face (negative control)", + dual_run.find(qn("a:rPr")).find(qn("a:latin")).get("typeface") == "Arial") + +triage_deck = Presentation() +triage_slide = triage_deck.slides.add_slide(triage_deck.slide_layouts[6]) +triage_shape = triage_slide.shapes.add_table(2, 2, 0, 0, 4000000, 2000000) +triage_shape.table.cell(0, 0).text_frame.text = "\u8868\u683c\u6587\u672c" +triage_deck.save("triage-table.pptx") +triage_reopened = Presentation("triage-table.pptx") +triage_table_shape = next(s for s in triage_reopened.slides[0].shapes if s.has_table) +def iter_text_frames(shapes): + for shape in shapes: + if shape.has_text_frame: + yield shape.text_frame + if getattr(shape, "has_table", False): + for row in shape.table.rows: + for cell in row.cells: + yield cell.text_frame +frames = list(iter_text_frames(triage_reopened.slides[0].shapes)) +check("table-cell text frames are reached by the triage walker", + any("\u8868\u683c\u6587\u672c" in f.text for f in frames), [f.text for f in frames]) +check("the table graphic frame itself has no text frame (negative control)", + not triage_table_shape.has_text_frame) + print("\n" + ("ALL PPTX FIXTURES PASSED" if not failures else f"{len(failures)} FAILURES: {failures}")) sys.exit(0 if not failures else 1) diff --git a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py index 76e18d8..99ac13b 100644 --- a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py @@ -429,5 +429,37 @@ def markers_in(data): if guard_last < 2: pass # guarded route: skip building rules + +# ---- read.md: implausible is reset before streaming ------------------ +dim_wb = openpyxl.Workbook() +dim_ws = dim_wb.active +dim_ws.append(["h1", "h2"]) +dim_ws.append([1, 2]) +dim_ws.append([3, 4]) +dim_wb.save("dimension.xlsx") +# Corrupt the sheet's dimension metadata the way non-Excel producers do. +import zipfile as dim_zip +with dim_zip.ZipFile("dimension.xlsx") as archive: + members = {name: archive.read(name) for name in archive.namelist()} +members["xl/worksheets/sheet1.xml"] = members["xl/worksheets/sheet1.xml"].replace( + b"", b"" +) +with dim_zip.ZipFile("dimension.xlsx", "w") as archive: + for name, data in members.items(): + archive.writestr(name, data) + +dim_value = openpyxl.load_workbook("dimension.xlsx", read_only=True, data_only=True) +dim_ws_ro = dim_value.active +check("corrupted dimension truncates streaming (negative control)", + dim_ws_ro.max_row == 1, dim_ws_ro.max_row) +streamed_before_reset = [row for row in dim_ws_ro.iter_rows(values_only=True)] +dim_ws_ro.reset_dimensions() +streamed_after_reset = [row for row in dim_ws_ro.iter_rows(values_only=True)] +dim_value.close() +check("reset_dimensions restores the real extent", + len(streamed_after_reset) == 3 and streamed_after_reset[2] == (3, 4), + (len(streamed_before_reset), streamed_after_reset[-1])) + + print("\n" + ("ALL XLSX FIXTURES PASSED" if not failures else f"{len(failures)} FAILURES: {failures}")) sys.exit(0 if not failures else 1) From 328f552d0ad8bf86c3f0998d670954bd7db20fd4 Mon Sep 17 00:00:00 2001 From: Hylouis233 Date: Sun, 16 Aug 2026 17:33:58 +0800 Subject: [PATCH 13/34] fix(document-skills): address current review gaps --- .../skills/pptx/references/analyze.md | 45 +++++++++++----- .../skills/xlsx/references/edit.md | 36 ++++++++----- .../skills/xlsx/references/read.md | 4 +- .../document-skills/tests/pptx_fixture.py | 52 +++++++++++++++---- .../document-skills/tests/xlsx_fixture.py | 39 ++++++++++++++ 5 files changed, 140 insertions(+), 36 deletions(-) diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md index 5b3de7d..c9e06e4 100644 --- a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md @@ -37,7 +37,13 @@ for i, slide in enumerate(prs.slides): shapes = list(iter_shapes(slide.shapes)) # flattened; groups are common in template decks text = [sh.text_frame.text for sh in shapes if sh.has_text_frame and sh.text_frame.text] tables = [ - [[cell.text for cell in row.cells] for row in sh.table.rows] + [[{ + "text": cell.text, + "is_merge_origin": cell.is_merge_origin, + "is_spanned": cell.is_spanned, + "span_width": cell.span_width, + "span_height": cell.span_height, + } for cell in row.cells] for row in sh.table.rows] for sh in shapes if sh.has_table ] charts = [] @@ -177,7 +183,17 @@ placeholder, layout, master, or theme. Resolve what you can and name the fallbac ```python import xml.etree.ElementTree as ET from pptx import Presentation +from pptx.enum.shapes import MSO_SHAPE_TYPE from pptx.opc.constants import RELATIONSHIP_TYPE as RT +from pptx.oxml.ns import qn + +def iter_shapes(shapes): + """Self-contained recursive walker for this independently runnable block.""" + for shape in shapes: + if shape.shape_type == MSO_SHAPE_TYPE.GROUP: + yield from iter_shapes(shape.shapes) + else: + yield shape prs = Presentation("deck.pptx") @@ -248,24 +264,29 @@ def theme_candidates(role_fonts, text): # 2. Per run: explicit value, else paragraph defaults, else report as inherited. # run.font.name exposes only the LATIN typeface; a run that also declares a:ea # or a:cs must be resolved per the scripts present in its own text first. -def explicit_run_face(run, text): - """The run's direct a:latin/a:ea/a:cs typeface chosen by its scripts (or None).""" +def explicit_run_faces(run, text): + """Distinct direct a:latin/a:ea/a:cs faces applicable to scripts in this run.""" rPr = run._r.find(qn("a:rPr")) if rPr is None: - return None + return [] declared = {} for slot, tag in (("latin", "a:latin"), ("eastAsia", "a:ea"), ("complexScript", "a:cs")): node = rPr.find(qn(tag)) if node is not None and node.get("typeface"): declared[slot] = node.get("typeface") if not declared: - return None + return [] tags = script_tags(text) - if any(tag in ("Hans", "Hant", "Jpan", "Hang") for tag in tags) and "eastAsia" in declared: - return declared["eastAsia"] - if any(tag in ("Arab", "Hebr", "Deva") for tag in tags) and "complexScript" in declared: - return declared["complexScript"] - return declared.get("latin") + slots = [] + if any(ch.isascii() and ch.isalnum() for ch in text): + slots.append("latin") + if any(tag in ("Hans", "Hant", "Jpan", "Hang") for tag in tags): + slots.append("eastAsia") + if any(tag in ("Arab", "Hebr", "Deva") for tag in tags): + slots.append("complexScript") + if not slots: + slots.append("latin") + return list(dict.fromkeys(declared[slot] for slot in slots if slot in declared)) def iter_text_frames(shapes): """Shape text frames plus every table cell's text frame (a graphic frame @@ -286,9 +307,9 @@ for i, slide in enumerate(prs.slides): holder = getattr(frame, "_parent", None) # the shape for ordinary frames for paragraph in frame.paragraphs: for run in paragraph.runs: - explicit = explicit_run_face(run, run.text) + explicit = explicit_run_faces(run, run.text) if explicit: - face, source = explicit, "run script face (a:latin/a:ea/a:cs)" + face, source = explicit, "run script faces (a:latin/a:ea/a:cs)" elif run.font.name: face, source = run.font.name, "run latin" elif paragraph.font.name: diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md index 519234b..72e76fd 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md @@ -51,6 +51,21 @@ def non_cell_references(workbook): refs.append(("chart series", f"{owner} chart {index}", element.text)) return refs +def cell_formula_references(workbook): + """Inventory ordinary, array, and data-table formulas before row/column moves.""" + refs = [] + for sheet in workbook.worksheets: + for row in sheet.iter_rows(): + for cell in row: + if cell.data_type == "f": + value = cell.value + refs.append(( + "cell formula", + f"{sheet.title}!{cell.coordinate}", + getattr(value, "text", None) or str(value), + )) + return refs + # Address cells directly; check the header to confirm column meaning first ws["D2"] = "=C2*1.08" # real formula ws["E2"] = date(2025, 9, 30) @@ -58,14 +73,15 @@ ws["E2"].number_format = "yyyy-mm-dd" ws["F2"] = 1234.5 ws["F2"].number_format = "#,##0.00" -# Insert/delete does not adjust formulas or non-cell dependencies. Fail closed until every -# reported reference that can intersect the shifted region has an explicit rewrite plan. -references_before = non_cell_references(wb) +# Insert/delete does not adjust formulas or non-cell dependencies. Snapshot both inventories +# before moving anything; fail closed until every reference that can intersect row 5 or below +# has an explicit old -> new rewrite plan. +references_before = cell_formula_references(wb) + non_cell_references(wb) if references_before: for reference in references_before: print("structural-edit dependency:", reference) raise RuntimeError( - "insert_rows is unsafe until chart/name/table/filter/validation/format references are audited" + "insert_rows is unsafe until cell formulas and non-cell references are audited" ) ws.insert_rows(5) @@ -81,14 +97,10 @@ for row in summary["A1:B1"]: for cell in row: cell.font = header_font -# References to the shifted region may live on any sheet; inspect every formula, then rerun -# non_cell_references and verify every planned rewrite before saving. -for formula_ws in wb.worksheets: - for row in formula_ws.iter_rows(): - for cell in row: - if cell.data_type == "f": - formula = getattr(cell.value, "text", None) or str(cell.value) - print(f"{formula_ws.title}!{cell.coordinate}: {formula}") +# After applying the planned rewrites, rerun both inventories and compare them with the +# pre-edit snapshot before saving. A post-edit listing alone cannot reveal a stale formula. +print("cell formulas after planned rewrites:", cell_formula_references(wb)) +print("non-cell references after planned rewrites:", non_cell_references(wb)) wb.save("input-edited.xlsx") ``` diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md index 3978007..1a5080a 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md @@ -25,7 +25,9 @@ for sheet_name in value_wb.sheetnames: value_ws.reset_dimensions() formula_ws.reset_dimensions() print(f"--- {sheet_name} --- implausible dimension {declared!r}; reset, real extent:") - print(f"--- {sheet_name} --- dims:", value_ws.calculate_dimension()) + # reset_dimensions() makes a read-only worksheet unsized; force=True scans + # the stream to rebuild its bounds instead of raising "Worksheet is unsized". + print(f"--- {sheet_name} --- dims:", value_ws.calculate_dimension(force=True)) rows = value_ws.iter_rows(values_only=True) header = next(rows, None) diff --git a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py index fc6d64e..041379b 100644 --- a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py @@ -152,7 +152,13 @@ def extract_slide_content(slide): shapes = list(iter_shapes(slide.shapes)) text = [sh.text_frame.text for sh in shapes if sh.has_text_frame and sh.text_frame.text] tables = [ - [[cell.text for cell in row.cells] for row in sh.table.rows] + [[{ + "text": cell.text, + "is_merge_origin": cell.is_merge_origin, + "is_spanned": cell.is_spanned, + "span_width": cell.span_width, + "span_height": cell.span_height, + } for cell in row.cells] for row in sh.table.rows] for sh in shapes if sh.has_table ] charts = [] @@ -196,7 +202,8 @@ def extract_slide_content(slide): content = extract_slide_content(Presentation("input.pptx").slides[0]) check("content inventory emits body text", any("old wording" in value for value in content["text"]), content) -check("content inventory emits table cell text", content["tables"][0][0][1] == "old cell text", content["tables"]) +check("content inventory emits table cell text", + content["tables"][0][0][1]["text"] == "old cell text", content["tables"]) check( "content inventory emits chart title, categories, series, and values", content["charts"][0]["title"] == "Units by region" @@ -207,6 +214,22 @@ def extract_slide_content(slide): ) check("content inventory emits notes text", "regional split" in content["notes"], content["notes"]) +merge_prs = Presentation() +merge_slide = merge_prs.slides.add_slide(merge_prs.slide_layouts[6]) +merge_shape = merge_slide.shapes.add_table(2, 3, 0, 0, 4000000, 2000000) +merge_origin = merge_shape.table.cell(0, 0) +merge_origin.text = "Merged heading" +merge_origin.merge(merge_shape.table.cell(0, 1)) +merged_inventory = extract_slide_content(merge_slide)["tables"][0][0] +check( + "table inventory preserves merge origin and span metadata", + merged_inventory[0]["is_merge_origin"] + and merged_inventory[0]["span_width"] == 2 + and merged_inventory[0]["span_height"] == 1 + and merged_inventory[1]["is_spanned"], + merged_inventory, +) + # XY scatter and bubble plots do not have category/value-series semantics. xy_prs = Presentation() xy_slide = xy_prs.slides.add_slide(xy_prs.slide_layouts[6]) @@ -587,23 +610,28 @@ def cached_numeric_points(series_element, element_name): from pptx.oxml import parse_xml as pptx_parse_xml2 from pptx.oxml.ns import qn -def explicit_run_face(run, text, script_tags_fn): +def explicit_run_faces(run, text, script_tags_fn): rPr = run._r.find(qn("a:rPr")) if rPr is None: - return None + return [] declared = {} for slot, tag in (("latin", "a:latin"), ("eastAsia", "a:ea"), ("complexScript", "a:cs")): node = rPr.find(qn(tag)) if node is not None and node.get("typeface"): declared[slot] = node.get("typeface") if not declared: - return None + return [] tags = script_tags_fn(text) - if any(tag in ("Hans", "Hant", "Jpan", "Hang") for tag in tags) and "eastAsia" in declared: - return declared["eastAsia"] - if any(tag in ("Arab", "Hebr", "Deva") for tag in tags) and "complexScript" in declared: - return declared["complexScript"] - return declared.get("latin") + slots = [] + if any(ch.isascii() and ch.isalnum() for ch in text): + slots.append("latin") + if any(tag in ("Hans", "Hant", "Jpan", "Hang") for tag in tags): + slots.append("eastAsia") + if any(tag in ("Arab", "Hebr", "Deva") for tag in tags): + slots.append("complexScript") + if not slots: + slots.append("latin") + return list(dict.fromkeys(declared[slot] for slot in slots if slot in declared)) run_xml = ( '' @@ -614,7 +642,9 @@ def explicit_run_face(run, text, script_tags_fn): class FakeRun: _r = dual_run check("a CJK run with latin+ea declared resolves to the eastAsia face", - explicit_run_face(FakeRun(), "\u6d4b\u8bd5", script_tags) == "SimSun") + explicit_run_faces(FakeRun(), "\u6d4b\u8bd5", script_tags) == ["SimSun"]) +check("a mixed Latin+CJK run reports both applicable declared faces", + explicit_run_faces(FakeRun(), "Q3 \u6d4b\u8bd5", script_tags) == ["Arial", "SimSun"]) check("run.font.name alone would report only the Latin face (negative control)", dual_run.find(qn("a:rPr")).find(qn("a:latin")).get("typeface") == "Arial") diff --git a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py index 99ac13b..9ef4f8c 100644 --- a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py @@ -266,6 +266,21 @@ def non_cell_references(workbook): return refs +def cell_formula_references(workbook): + refs = [] + for sheet in workbook.worksheets: + for row in sheet.iter_rows(): + for cell in row: + if cell.data_type == "f": + value = cell.value + refs.append(( + "cell formula", + f"{sheet.title}!{cell.coordinate}", + getattr(value, "text", None) or str(value), + )) + return refs + + class LegacyDefinedNames: """Minimal openpyxl 3.0-style DefinedNameList surface.""" definedName = [DefinedName("LegacyRange", attr_text="'Legacy'!$A$1")] @@ -289,6 +304,7 @@ class LegacyWorkbook: audit_ws.append(["Value"]) audit_ws.append([1]) audit_ws.append([2]) +audit_ws["C1"] = "=SUM(A2:A3)" audit_wb.defined_names.add(DefinedName("AuditRange", attr_text="'Audit'!$A$2:$A$3")) audit_ws.add_table(Table(displayName="AuditTable", ref="A1:A3")) audit_ws.merge_cells("B2:B3") @@ -304,6 +320,7 @@ class LegacyWorkbook: chart.add_data(Reference(audit_ws, min_col=1, min_row=1, max_row=3), titles_from_data=True) audit_ws.add_chart(chart, "C1") reference_kinds = {kind for kind, _, _ in non_cell_references(audit_wb)} +formula_references = cell_formula_references(audit_wb) check( "structural audit covers names, tables, filters, validation, formatting, and charts", {"defined name", "table", "merged range", "auto filter", "print area", @@ -312,6 +329,25 @@ class LegacyWorkbook: "conditional formatting formula", "chart series"} <= reference_kinds, reference_kinds, ) +check( + "structural audit snapshots ordinary cell formulas before row insertion", + formula_references == [("cell formula", "Audit!C1", "=SUM(A2:A3)")], + formula_references, +) + +stale_wb = openpyxl.Workbook() +stale_ws = stale_wb.active +stale_ws["A5"], stale_ws["A6"] = 10, 20 +stale_ws["B1"] = "=SUM(A5:A6)" +stale_formula_before = cell_formula_references(stale_wb) +stale_ws.insert_rows(5) +check( + "insert_rows leaves intersecting formulas stale (negative control)", + stale_formula_before == [("cell formula", "Sheet!B1", "=SUM(A5:A6)")] + and stale_ws["B1"].value == "=SUM(A5:A6)" + and (stale_ws["A6"].value, stale_ws["A7"].value) == (10, 20), + (stale_formula_before, stale_ws["B1"].value), +) # and the edit itself still works after the warning path wb2 = openpyxl.load_workbook("plain.xlsx") @@ -454,8 +490,11 @@ def markers_in(data): dim_ws_ro.max_row == 1, dim_ws_ro.max_row) streamed_before_reset = [row for row in dim_ws_ro.iter_rows(values_only=True)] dim_ws_ro.reset_dimensions() +forced_extent = dim_ws_ro.calculate_dimension(force=True) streamed_after_reset = [row for row in dim_ws_ro.iter_rows(values_only=True)] dim_value.close() +check("forced dimension calculation sizes a reset read-only sheet", + forced_extent == "A1:B3", forced_extent) check("reset_dimensions restores the real extent", len(streamed_after_reset) == 3 and streamed_after_reset[2] == (3, 4), (len(streamed_before_reset), streamed_after_reset[-1])) From 98ce01efff4ca38152524af82fc76ade0cca4523 Mon Sep 17 00:00:00 2001 From: Hylouis233 Date: Sun, 16 Aug 2026 19:23:10 +0800 Subject: [PATCH 14/34] fix(document-skills): close delayed review findings --- .../document-skills/skills/pdf/SKILL.md | 12 ++- .../skills/pptx/references/analyze.md | 94 +++++++++++-------- .../skills/xlsx/references/edit.md | 47 ++++++++-- .../document-skills/tests/pdf_fixture.py | 15 ++- .../document-skills/tests/pptx_fixture.py | 72 ++++++++++---- .../document-skills/tests/xlsx_fixture.py | 53 ++++++++++- 6 files changed, 221 insertions(+), 72 deletions(-) diff --git a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md index 57a2b52..4db7a7a 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md @@ -118,10 +118,18 @@ if overflow_doc.needs_pass: overflow_doc.authenticate(password) overflow_pages = [] for page in overflow_doc: - clip = fitz.Rect(-2000, -2000, page.rect.width + 2000, page.rect.height + 2000) + # get_text() block coordinates are unrotated even when /Rotate is 90/270; + # page.rect uses rotated dimensions. Compare against an unrotated crop-box + # extent so valid high-y portrait text is not flagged on a rotated page. + page_box = fitz.Rect(0, 0, page.cropbox.width, page.cropbox.height) + clip = fitz.Rect( + page_box.x0 - 2000, page_box.y0 - 2000, + page_box.x1 + 2000, page_box.y1 + 2000, + ) text_blocks = [b for b in page.get_text("blocks", clip=clip) if b[6] == 0] beyond_box = any( - b[0] < -0.5 or b[1] < -0.5 or b[2] > page.rect.width + 0.5 or b[3] > page.rect.height + 0.5 + b[0] < page_box.x0 - 0.5 or b[1] < page_box.y0 - 0.5 + or b[2] > page_box.x1 + 0.5 or b[3] > page_box.y1 + 0.5 for b in text_blocks ) if beyond_box: diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md index c9e06e4..2f4f631 100644 --- a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md @@ -53,7 +53,7 @@ for i, slide in enumerate(prs.slides): chart = sh.chart chart_title = ( chart.chart_title.text_frame.text - if chart.has_title and chart.chart_title.has_text_frame else "" + if chart.has_title else "" ) plots = [] for plot in chart.plots: @@ -251,42 +251,66 @@ def script_tags(text): tags.append("Deva") return list(dict.fromkeys(tags)) -def theme_candidates(role_fonts, text): - faces = [role_fonts["latin"]] +def required_font_slots(text): tags = script_tags(text) + slots = [] + if any(ch.isascii() and ch.isalnum() for ch in text): + slots.append("latin") if any(tag in ("Hans", "Hant", "Jpan", "Hang") for tag in tags): - faces.append(role_fonts["eastAsia"]) + slots.append("eastAsia") if any(tag in ("Arab", "Hebr", "Deva") for tag in tags): - faces.append(role_fonts["complexScript"]) - faces.extend(role_fonts["scripts"].get(tag, "") for tag in tags) - return [face for face in dict.fromkeys(faces) if face] + slots.append("complexScript") + return slots or ["latin"] + +def theme_candidates_by_slot(role_fonts, text): + tags = script_tags(text) + east_tags = {"Hans", "Hant", "Jpan", "Hang"} + complex_tags = {"Arab", "Hebr", "Deva"} + candidates = {} + for slot in required_font_slots(text): + slot_tags = [ + tag for tag in tags + if (slot == "eastAsia" and tag in east_tags) + or (slot == "complexScript" and tag in complex_tags) + or (slot == "latin" and tag not in east_tags | complex_tags) + ] + faces = [role_fonts[slot]] + faces.extend(role_fonts["scripts"].get(tag, "") for tag in slot_tags) + candidates[slot] = [face for face in dict.fromkeys(faces) if face] + return candidates # 2. Per run: explicit value, else paragraph defaults, else report as inherited. # run.font.name exposes only the LATIN typeface; a run that also declares a:ea # or a:cs must be resolved per the scripts present in its own text first. -def explicit_run_faces(run, text): - """Distinct direct a:latin/a:ea/a:cs faces applicable to scripts in this run.""" +def explicit_run_faces(run): + """Direct a:latin/a:ea/a:cs faces keyed by slot; missing slots stay missing.""" rPr = run._r.find(qn("a:rPr")) if rPr is None: - return [] + return {} declared = {} for slot, tag in (("latin", "a:latin"), ("eastAsia", "a:ea"), ("complexScript", "a:cs")): node = rPr.find(qn(tag)) if node is not None and node.get("typeface"): declared[slot] = node.get("typeface") - if not declared: - return [] - tags = script_tags(text) - slots = [] - if any(ch.isascii() and ch.isalnum() for ch in text): - slots.append("latin") - if any(tag in ("Hans", "Hant", "Jpan", "Hang") for tag in tags): - slots.append("eastAsia") - if any(tag in ("Arab", "Hebr", "Deva") for tag in tags): - slots.append("complexScript") - if not slots: - slots.append("latin") - return list(dict.fromkeys(declared[slot] for slot in slots if slot in declared)) + return declared + +def resolve_run_faces(run, paragraph, role_fonts): + """Resolve every required script slot without letting one direct slot hide another.""" + direct = explicit_run_faces(run) + inherited = theme_candidates_by_slot(role_fonts, run.text) + resolved = [] + for slot in required_font_slots(run.text): + if slot in direct: + faces, source = [direct[slot]], "run direct" + elif slot == "latin" and run.font.name: + faces, source = [run.font.name], "run latin" + elif slot == "latin" and paragraph.font.name: + faces, source = [paragraph.font.name], "paragraph defaults (latin)" + else: + faces = inherited.get(slot, []) or ["(unresolved inherited face)"] + source = "theme candidates (verify placeholder chain/locale)" + resolved.append({"slot": slot, "faces": faces, "source": source}) + return resolved def iter_text_frames(shapes): """Shape text frames plus every table cell's text frame (a graphic frame @@ -307,24 +331,12 @@ for i, slide in enumerate(prs.slides): holder = getattr(frame, "_parent", None) # the shape for ordinary frames for paragraph in frame.paragraphs: for run in paragraph.runs: - explicit = explicit_run_faces(run, run.text) - if explicit: - face, source = explicit, "run script faces (a:latin/a:ea/a:cs)" - elif run.font.name: - face, source = run.font.name, "run latin" - elif paragraph.font.name: - # paragraph.font is the LATIN slot of a:pPr/a:defRPr only; - # script-specific paragraph defaults are not modeled by - # python-pptx and need raw XML for full coverage. - face, source = paragraph.font.name, "paragraph defaults (latin)" - else: - role = "major" if ( - title_shape is not None and - getattr(holder, "_element", None) is title_shape._element - ) else "minor" - face = theme_candidates(theme_fonts[role], run.text) - source = f"inherited {role} theme candidates (verify placeholder chain/locale)" - print(i, repr(run.text[:20]), "font:", ascii(face), "source:", source) + role = "major" if ( + title_shape is not None and + getattr(holder, "_element", None) is title_shape._element + ) else "minor" + resolved = resolve_run_faces(run, paragraph, theme_fonts[role]) + print(i, repr(run.text[:20]), f"{role} font slots:", ascii(resolved)) ``` python-pptx does not evaluate the full placeholder -> layout -> master inheritance chain; when diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md index 72e76fd..0702534 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md @@ -3,7 +3,9 @@ ```python import openpyxl from datetime import date +from openpyxl.formula import Tokenizer from openpyxl.styles import Font +from openpyxl.utils.cell import range_boundaries wb = openpyxl.load_workbook("input.xlsx") # NOT data_only: that would drop all formulas ws = wb["Data"] @@ -61,11 +63,36 @@ def cell_formula_references(workbook): value = cell.value refs.append(( "cell formula", - f"{sheet.title}!{cell.coordinate}", + sheet.title, + cell.coordinate, getattr(value, "text", None) or str(value), )) return refs +def formula_may_intersect_rows(owner_sheet, formula, shifted_sheet, start_row): + """Fail closed unless every range token is provably above/outside the shifted rows.""" + if not isinstance(formula, str) or not formula.startswith("="): + return True + for token in Tokenizer(formula).items: + if token.type != "OPERAND" or token.subtype != "RANGE": + continue + reference = token.value + target_sheet = owner_sheet + if "!" in reference: + qualifier, reference = reference.rsplit("!", 1) + if "[" in qualifier or ":" in qualifier: # external or 3-D reference + return True + target_sheet = qualifier.strip("'").replace("''", "'") + if target_sheet != shifted_sheet: + continue + try: + _, min_row, _, max_row = range_boundaries(reference.replace("$", "")) + except ValueError: # named/dynamic reference: require a manual rewrite plan + return True + if min_row is None or max_row is None or max_row >= start_row: + return True + return False + # Address cells directly; check the header to confirm column meaning first ws["D2"] = "=C2*1.08" # real formula ws["E2"] = date(2025, 9, 30) @@ -73,12 +100,17 @@ ws["E2"].number_format = "yyyy-mm-dd" ws["F2"] = 1234.5 ws["F2"].number_format = "#,##0.00" -# Insert/delete does not adjust formulas or non-cell dependencies. Snapshot both inventories -# before moving anything; fail closed until every reference that can intersect row 5 or below -# has an explicit old -> new rewrite plan. -references_before = cell_formula_references(wb) + non_cell_references(wb) -if references_before: - for reference in references_before: +# Insert/delete does not adjust formulas or non-cell dependencies. Snapshot both inventories, +# but block only formula ranges that may intersect the shifted rows (plus the conservative +# non-cell inventory) so an audited formula such as D2 = C2*1.08 can proceed. +cell_formulas_before = cell_formula_references(wb) +intersecting_formulas = [ + reference for reference in cell_formulas_before + if formula_may_intersect_rows(reference[1], reference[3], ws.title, 5) +] +unaudited_references = intersecting_formulas + non_cell_references(wb) +if unaudited_references: + for reference in unaudited_references: print("structural-edit dependency:", reference) raise RuntimeError( "insert_rows is unsafe until cell formulas and non-cell references are audited" @@ -99,6 +131,7 @@ for row in summary["A1:B1"]: # After applying the planned rewrites, rerun both inventories and compare them with the # pre-edit snapshot before saving. A post-edit listing alone cannot reveal a stale formula. +print("cell formulas before:", cell_formulas_before) print("cell formulas after planned rewrites:", cell_formula_references(wb)) print("non-cell references after planned rewrites:", non_cell_references(wb)) diff --git a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py index 41e885b..19d9a5e 100644 --- a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py @@ -365,15 +365,26 @@ def overflow_pages(path): pages = [] for page in doc: # Plain block extraction drops fully off-page text; enlarge the clip. - clip = fitz.Rect(-2000, -2000, page.rect.width + 2000, page.rect.height + 2000) + page_box = fitz.Rect(0, 0, page.cropbox.width, page.cropbox.height) + clip = fitz.Rect( + page_box.x0 - 2000, page_box.y0 - 2000, + page_box.x1 + 2000, page_box.y1 + 2000, + ) blocks = [b for b in page.get_text("blocks", clip=clip) if b[6] == 0] - if any(b[0] < -0.5 or b[1] < -0.5 or b[2] > page.rect.width + 0.5 or b[3] > page.rect.height + 0.5 + if any(b[0] < page_box.x0 - 0.5 or b[1] < page_box.y0 - 0.5 + or b[2] > page_box.x1 + 0.5 or b[3] > page_box.y1 + 0.5 for b in blocks): pages.append(page.number + 1) doc.close() return pages check("in-bounds PDF reports no overflow pages", overflow_pages("overflow.pdf") == []) +rotated_doc = fitz.open("overflow.pdf") +rotated_doc[0].set_rotation(90) +rotated_doc.save("overflow-rotated.pdf") +rotated_doc.close() +check("rotated in-bounds text uses the unrotated crop-box coordinate space", + overflow_pages("overflow-rotated.pdf") == []) check("off-page text is detected by the overflow check (negative control)", overflow_pages("overflow-bad.pdf") == [1]) check("off-page text still extracts, so extraction alone cannot catch it", diff --git a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py index 041379b..650f25d 100644 --- a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py @@ -168,7 +168,7 @@ def extract_slide_content(slide): chart = sh.chart chart_title = ( chart.chart_title.text_frame.text - if chart.has_title and chart.chart_title.has_text_frame else "" + if chart.has_title else "" ) plots = [] for plot in chart.plots: @@ -610,17 +610,7 @@ def cached_numeric_points(series_element, element_name): from pptx.oxml import parse_xml as pptx_parse_xml2 from pptx.oxml.ns import qn -def explicit_run_faces(run, text, script_tags_fn): - rPr = run._r.find(qn("a:rPr")) - if rPr is None: - return [] - declared = {} - for slot, tag in (("latin", "a:latin"), ("eastAsia", "a:ea"), ("complexScript", "a:cs")): - node = rPr.find(qn(tag)) - if node is not None and node.get("typeface"): - declared[slot] = node.get("typeface") - if not declared: - return [] +def required_font_slots(text, script_tags_fn): tags = script_tags_fn(text) slots = [] if any(ch.isascii() and ch.isalnum() for ch in text): @@ -629,9 +619,35 @@ def explicit_run_faces(run, text, script_tags_fn): slots.append("eastAsia") if any(tag in ("Arab", "Hebr", "Deva") for tag in tags): slots.append("complexScript") - if not slots: - slots.append("latin") - return list(dict.fromkeys(declared[slot] for slot in slots if slot in declared)) + return slots or ["latin"] + +def explicit_run_faces(run): + rPr = run._r.find(qn("a:rPr")) + if rPr is None: + return {} + declared = {} + for slot, tag in (("latin", "a:latin"), ("eastAsia", "a:ea"), ("complexScript", "a:cs")): + node = rPr.find(qn(tag)) + if node is not None and node.get("typeface"): + declared[slot] = node.get("typeface") + return declared + +def resolve_faces(run, text, role_fonts, script_tags_fn): + direct = explicit_run_faces(run) + tags = script_tags_fn(text) + result = {} + for slot in required_font_slots(text, script_tags_fn): + if slot in direct: + result[slot] = [direct[slot]] + continue + relevant = { + "eastAsia": {"Hans", "Hant", "Jpan", "Hang"}, + "complexScript": {"Arab", "Hebr", "Deva"}, + }.get(slot, set()) + candidates = [role_fonts[slot]] + candidates.extend(role_fonts["scripts"].get(tag, "") for tag in tags if tag in relevant) + result[slot] = [face for face in dict.fromkeys(candidates) if face] + return result run_xml = ( '' @@ -642,9 +658,31 @@ def explicit_run_faces(run, text, script_tags_fn): class FakeRun: _r = dual_run check("a CJK run with latin+ea declared resolves to the eastAsia face", - explicit_run_faces(FakeRun(), "\u6d4b\u8bd5", script_tags) == ["SimSun"]) + resolve_faces(FakeRun(), "\u6d4b\u8bd5", { + "latin": "Theme Latin", "eastAsia": "Theme East", "complexScript": "Theme CS", + "scripts": {}, + }, script_tags) == {"eastAsia": ["SimSun"]}) check("a mixed Latin+CJK run reports both applicable declared faces", - explicit_run_faces(FakeRun(), "Q3 \u6d4b\u8bd5", script_tags) == ["Arial", "SimSun"]) + resolve_faces(FakeRun(), "Q3 \u6d4b\u8bd5", { + "latin": "Theme Latin", "eastAsia": "Theme East", "complexScript": "Theme CS", + "scripts": {}, + }, script_tags) == {"latin": ["Arial"], "eastAsia": ["SimSun"]}) + +latin_only_xml = ( + '' + 'Q3 \u6d4b\u8bd5' +) +latin_only_run = pptx_parse_xml2(latin_only_xml) +class LatinOnlyRun: + _r = latin_only_run +partial_resolution = resolve_faces(LatinOnlyRun(), "Q3 \u6d4b\u8bd5", { + "latin": "Theme Latin", "eastAsia": "Theme East", "complexScript": "Theme CS", + "scripts": {"Hans": "Theme Hans"}, +}, script_tags) +check("a direct Latin face does not suppress inherited CJK candidates", + partial_resolution["latin"] == ["Arial"] + and partial_resolution["eastAsia"] == ["Theme East", "Theme Hans"], + partial_resolution) check("run.font.name alone would report only the Latin face (negative control)", dual_run.find(qn("a:rPr")).find(qn("a:latin")).get("typeface") == "Arial") diff --git a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py index 9ef4f8c..98cf727 100644 --- a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py @@ -218,7 +218,9 @@ def sheet_ref(sheet): # ---- edit.md structural audit includes non-cell dependencies ------------------ from openpyxl.chart import BarChart, Reference +from openpyxl.formula import Tokenizer from openpyxl.formatting.rule import CellIsRule, FormulaRule +from openpyxl.utils.cell import range_boundaries from openpyxl.workbook.defined_name import DefinedName from openpyxl.worksheet.datavalidation import DataValidation from openpyxl.worksheet.table import Table @@ -275,12 +277,37 @@ def cell_formula_references(workbook): value = cell.value refs.append(( "cell formula", - f"{sheet.title}!{cell.coordinate}", + sheet.title, + cell.coordinate, getattr(value, "text", None) or str(value), )) return refs +def formula_may_intersect_rows(owner_sheet, formula, shifted_sheet, start_row): + if not isinstance(formula, str) or not formula.startswith("="): + return True + for token in Tokenizer(formula).items: + if token.type != "OPERAND" or token.subtype != "RANGE": + continue + reference = token.value + target_sheet = owner_sheet + if "!" in reference: + qualifier, reference = reference.rsplit("!", 1) + if "[" in qualifier or ":" in qualifier: + return True + target_sheet = qualifier.strip("'").replace("''", "'") + if target_sheet != shifted_sheet: + continue + try: + _, min_row, _, max_row = range_boundaries(reference.replace("$", "")) + except ValueError: + return True + if min_row is None or max_row is None or max_row >= start_row: + return True + return False + + class LegacyDefinedNames: """Minimal openpyxl 3.0-style DefinedNameList surface.""" definedName = [DefinedName("LegacyRange", attr_text="'Legacy'!$A$1")] @@ -331,9 +358,13 @@ class LegacyWorkbook: ) check( "structural audit snapshots ordinary cell formulas before row insertion", - formula_references == [("cell formula", "Audit!C1", "=SUM(A2:A3)")], + formula_references == [("cell formula", "Audit", "C1", "=SUM(A2:A3)")], formula_references, ) +check("intersecting formula ranges are blocked before row insertion", + formula_may_intersect_rows("Audit", "=SUM(A2:A3)", "Audit", 3)) +check("audited formula ranges above the insertion can proceed", + not formula_may_intersect_rows("Data", "=C2*1.08", "Data", 5)) stale_wb = openpyxl.Workbook() stale_ws = stale_wb.active @@ -343,12 +374,28 @@ class LegacyWorkbook: stale_ws.insert_rows(5) check( "insert_rows leaves intersecting formulas stale (negative control)", - stale_formula_before == [("cell formula", "Sheet!B1", "=SUM(A5:A6)")] + stale_formula_before == [("cell formula", "Sheet", "B1", "=SUM(A5:A6)")] and stale_ws["B1"].value == "=SUM(A5:A6)" and (stale_ws["A6"].value, stale_ws["A7"].value) == (10, 20), (stale_formula_before, stale_ws["B1"].value), ) +safe_wb = openpyxl.Workbook() +safe_ws = safe_wb.active +safe_ws.title = "Data" +safe_ws["C2"], safe_ws["D2"] = 100, "=C2*1.08" +safe_before = cell_formula_references(safe_wb) +safe_dependencies = [ + reference for reference in safe_before + if formula_may_intersect_rows(reference[1], reference[3], "Data", 5) +] +if not safe_dependencies: + safe_ws.insert_rows(5) + safe_wb.save("audited-structural-edit.xlsx") +safe_reopened = openpyxl.load_workbook("audited-structural-edit.xlsx", data_only=False) +check("audited non-intersecting formula path reaches save", + safe_reopened["Data"]["D2"].value == "=C2*1.08") + # and the edit itself still works after the warning path wb2 = openpyxl.load_workbook("plain.xlsx") wb2["Data"]["B2"] = "=B2*1" # formula stays a formula From fbef6032cac075bd836e599862ae30c5696d8d48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=98=E6=89=AC?= Date: Sun, 16 Aug 2026 14:05:57 +0800 Subject: [PATCH 15/34] fix(document-skills): close latest review gaps --- .../skills/docx/references/cjk.md | 42 +- .../skills/docx/references/edit.md | 12 +- .../skills/docx/references/read.md | 106 ++-- .../document-skills/skills/pdf/SKILL.md | 2 +- .../skills/pdf/references/transform.md | 24 +- .../skills/pptx/references/analyze.md | 248 +++++----- .../skills/pptx/references/edit.md | 6 +- .../document-skills/skills/xlsx/SKILL.md | 14 +- .../skills/xlsx/references/edit.md | 42 +- .../skills/xlsx/references/read.md | 17 +- .../document-skills/tests/docx_fixture.py | 250 +++++++--- .../document-skills/tests/pdf_fixture.py | 90 +++- .../document-skills/tests/pptx_fixture.py | 467 +++++++++++++----- .../document-skills/tests/xlsx_fixture.py | 114 ++++- 14 files changed, 988 insertions(+), 446 deletions(-) diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md b/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md index 26fee57..61bf74d 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md @@ -24,7 +24,9 @@ style.element.get_or_add_rPr().get_or_add_rFonts().set(qn("w:eastAsia"), EAST_AS ``` Convention for mixed-script body text: CJK glyphs from the east-asian face, digits and Latin -from a Latin face (Times New Roman or Arial). Both slots set = deterministic rendering.## Chinese font-size table (字号) +from a Latin face (Times New Roman or Arial). Both slots set = deterministic rendering. + +## Chinese font-size table (字号) Word's Chinese UI names map to point sizes; scripts must use the points: @@ -101,14 +103,23 @@ section.left_margin, section.right_margin = Cm(2.8), Cm(2.6) from fontTools.ttLib import TTFont from docx.oxml.ns import qn from docx.text.paragraph import Paragraph + from docx.text.run import Run doc = Document("output.docx") normal_style = doc.styles["Normal"] + def paragraph_runs(paragraph): + def walk(element): + for child in element.iterchildren(): + if child.tag == qn("w:r"): + yield Run(child, paragraph) + elif child.tag != qn("w:p"): + yield from walk(child) + yield from walk(paragraph._p) def xml_runs(element, parent): - # Covers direct paragraphs, tables, nested tables, and block content controls. + # Covers paragraphs in tables/block controls and runs in inline controls/hyperlinks. for paragraph_element in element.iter(qn("w:p")): - yield from Paragraph(paragraph_element, parent).runs + yield from paragraph_runs(Paragraph(paragraph_element, parent)) emitted_runs = list(xml_runs(doc.element.body, doc)) for section in doc.sections: @@ -140,15 +151,24 @@ section.left_margin, section.right_margin = Cm(2.8), Cm(2.6) raise LookupError(f"no resolved {slot} face for run {run.text!r}; resolve theme defaults") def font_slot(character): + # This is the ordinary non-cs/rtl mapping. Resolve effective w:cs/w:rtl and special + # w:rFonts@w:hint overrides separately before relying on a slot for those runs. codepoint = ord(character) - # Word renders Korean through the East Asian slot as well: Hangul Jamo, - # compatibility Jamo, extended-A, and the syllable blocks. - return "eastAsia" if ( - 0x1100 <= codepoint <= 0x11FF or 0x2E80 <= codepoint <= 0x9FFF - or 0x3130 <= codepoint <= 0x318F or 0xA960 <= codepoint <= 0xA97F - or 0xAC00 <= codepoint <= 0xD7FF or 0xF900 <= codepoint <= 0xFAFF - or 0x20000 <= codepoint <= 0x3134F - ) else ("ascii" if codepoint < 128 else "hAnsi") + uses_east_asian_slot = ( + 0x1100 <= codepoint <= 0x11FF # Hangul Jamo + or 0x2E80 <= codepoint <= 0x9FFF # CJK radicals, kana, bopomofo, ideographs + or 0x3130 <= codepoint <= 0x318F # Hangul compatibility Jamo + or 0xA000 <= codepoint <= 0xA4CF # Yi syllables and radicals + or 0xA960 <= codepoint <= 0xA97F # Hangul Jamo Extended-A + or 0xAC00 <= codepoint <= 0xD7FF # Hangul syllables and Extended-B + or 0xF900 <= codepoint <= 0xFAFF # CJK compatibility ideographs + or 0xFE30 <= codepoint <= 0xFE6F # CJK compatibility and small forms + or 0xFF00 <= codepoint <= 0xFFEF # fullwidth and halfwidth forms + or 0x20000 <= codepoint <= 0x3134F # CJK unified ideograph extensions + ) + return "eastAsia" if uses_east_asian_slot else ( + "ascii" if codepoint < 128 else "hAnsi" + ) # Resolve installed files by exact face name first; do not pool their cmaps. # TTC collections need the face's fontNumber; ordinary TTF files use -1. diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/edit.md b/plugins/Hylouis233/document-skills/skills/docx/references/edit.md index 592b742..b167def 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/edit.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/edit.md @@ -13,17 +13,17 @@ from docx import Document from docx.oxml.ns import qn SAFE_RUN_CHILDREN = { - qn("w:rPr"), qn("w:t"), qn("w:tab"), qn("w:br"), qn("w:cr"), + qn("w:rPr"), qn("w:t"), qn("w:tab"), qn("w:cr"), } def unsafe_run_content(run): unsafe = [] for child in run._r: # Assigning run.text can reconstruct text, tabs, and ordinary line breaks only. - typed_break = child.tag == qn("w:br") and child.get(qn("w:type")) not in ( - None, "textWrapping", + ordinary_break = child.tag == qn("w:br") and dict(child.attrib) in ( + {}, {qn("w:type"): "textWrapping"}, ) - if child.tag not in SAFE_RUN_CHILDREN or typed_break: + if child.tag not in SAFE_RUN_CHILDREN and not ordinary_break: unsafe.append(child.tag.rsplit("}", 1)[-1]) return unsafe @@ -103,8 +103,8 @@ doc.save("input.edited.docx") The replacement text inherits the first matched run's formatting. Unmatched text before and after it stays in its original runs, so its formatting is preserved. The routine fails before -making changes if any matched run contains a drawing, field, reference, or typed page/column -break that `run.text` would destroy. Use raw OOXML for those cases and for tracked changes or +making changes if any matched run contains a drawing, field, reference, typed page/column break, +or a text-wrapping break with `w:clear` that `run.text` would destroy. Use raw OOXML for those cases and for tracked changes or other content that `paragraph.runs` does not expose. ## Tier 2 - raw OOXML surgery (only when Tier 1 cannot express it) diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/read.md b/plugins/Hylouis233/document-skills/skills/docx/references/read.md index 31cb092..53cada3 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/read.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/read.md @@ -18,29 +18,43 @@ from docx import Document from docx.oxml.ns import qn from docx.table import Table from docx.text.paragraph import Paragraph +from docx.text.run import Run -def iter_body_blocks(parent, document): - """Yield ("p", paragraph) / ("table", table) blocks, recursing into block content controls.""" - for child in parent.iterchildren(): +def iter_part_blocks(root, parent): + """Yield each paragraph/table once, descending through block content controls.""" + for child in root.iterchildren(): if child.tag == qn("w:p"): - yield ("p", Paragraph(child, document)) + yield "paragraph", Paragraph(child, parent) elif child.tag == qn("w:tbl"): - yield ("table", Table(child, document)) - elif child.tag == qn("w:sdt"): - content = child.find(qn("w:sdtContent")) - if content is not None: - yield from iter_body_blocks(content, document) - -def tc_text(tc): + yield "table", Table(child, parent) + else: + yield from iter_part_blocks(child, parent) + +def iter_paragraph_runs(paragraph): + """Include runs wrapped by hyperlinks, fields, revisions, or inline content controls.""" + def walk(element): + for child in element.iterchildren(): + if child.tag == qn("w:r"): + yield Run(child, paragraph) + elif child.tag != qn("w:p"): # nested text-box paragraphs are yielded separately + yield from walk(child) + yield from walk(paragraph._p) + +def paragraph_text(paragraph): + return "".join(run.text for run in iter_paragraph_runs(paragraph)) + +def tc_text(tc, parent): """Cell text rebuilt per paragraph, keeping tabs and breaks visible. Joining only the w:t descendants concatenates separate paragraphs ("First" + "Second" -> "FirstSecond") and loses separators entirely. """ paragraphs = [] - for p in tc.iter(qn("w:p")): + for kind, block in iter_part_blocks(tc, parent): + if kind != "paragraph": # nested tables are represented recursively, not duplicated here + continue pieces = [] - for node in p.iter(): + for node in block._p.iter(): if node.tag == qn("w:t"): pieces.append(node.text or "") elif node.tag == qn("w:tab"): @@ -50,47 +64,53 @@ def tc_text(tc): paragraphs.append("".join(pieces)) return " / ".join(paragraphs) -def table_matrix(table): - """Rows of cell text with merge structure preserved. - - python-docx's row.cells repeats a merge-origin cell across every grid - position it spans, which hides the real structure. Walk the actual w:tc - elements and annotate gridSpan/vMerge instead. - """ +def table_content(table): rows = [] for row in table.rows: - cells = [] - for tc in row._tr.tc_lst: - tc_pr = tc.find(qn("w:tcPr")) - grid_span = tc_pr.find(qn("w:gridSpan")) if tc_pr is not None else None - v_merge = tc_pr.find(qn("w:vMerge")) if tc_pr is not None else None - note = "" - if grid_span is not None: - note += "(span {})".format(grid_span.get(qn("w:val"))) - if v_merge is not None: - note += "(vmerge start)" if v_merge.get(qn("w:val")) == "restart" else "(vmerge cont.)" - text = tc_text(tc) - cells.append(text + note if note else text) - rows.append(cells) + rendered_cells = [] + column = 0 + # row.cells repeats a merge-origin proxy for every grid position it spans. + # Walk physical w:tc elements and expose the merge structure instead. + for cell_element in row._tr.tc_lst: + cell_properties = cell_element.find(qn("w:tcPr")) + grid_span = None if cell_properties is None else cell_properties.find(qn("w:gridSpan")) + colspan = 1 if grid_span is None else int(grid_span.get(qn("w:val"), "1")) + vertical = None if cell_properties is None else cell_properties.find(qn("w:vMerge")) + vertical_merge = None if vertical is None else vertical.get(qn("w:val"), "continue") + nested_tables = [] + for kind, block in iter_part_blocks(cell_element, table): + if kind == "table": + nested_tables.append(table_content(block)) + rendered_cells.append({ + "column": column, "colspan": colspan, + "vMerge": vertical_merge, + "text": tc_text(cell_element, table), + "tables": nested_tables, + }) + column += colspan + rows.append(rendered_cells) return rows doc = Document("input.docx") content_controls = list(doc.element.body.iter(qn("w:sdt"))) -print("block content controls:", len(content_controls)) -for kind, block in iter_body_blocks(doc.element.body, doc): - if kind == "p": - print("p", block.style.name, "|", block.text) +blocks = list(iter_part_blocks(doc.element.body, doc)) +print("content controls:", len(content_controls), "top-level blocks:", len(blocks)) +for kind, block in blocks: + if kind == "paragraph": + print(block.style.name, "|", paragraph_text(block)) else: - print("table", len(block.rows), "x", len(block.columns), table_matrix(block)) + print("table |", table_content(block)) ``` Notes: -- `doc.paragraphs` includes only direct body paragraphs and `doc.tables` only top-level - tables; both omit content nested in block content controls (`w:sdt`). The walker above - yields paragraphs and tables inside `w:sdtContent` too, and reports the content-control - count. Tables nested inside table cells still require walking `cell.tables`; text boxes, - headers, footers, and footnotes still require their own collections +- `doc.paragraphs` includes only direct body paragraphs, `doc.tables` includes only direct body + tables, and `Paragraph.runs` omits runs wrapped by inline content controls and other containers. + Use the XML-backed traversal above and report the content-control count. It stops descending + when a table is yielded, so table text is not also emitted as prose; `table_content()` handles + nested tables recursively and emits `column`, `colspan`, and `vMerge` metadata for physical + cells instead of duplicating merge-origin text through `row.cells`. + Text boxes, headers, footers, and footnotes still require their own collections (`section.header/.footer`) or raw XML. - For revision/comment metadata, inspect the XML parts directly: `word/comments.xml`, `w:ins`/`w:del` elements in `word/document.xml`. diff --git a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md index 4db7a7a..d4cef0c 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md @@ -14,7 +14,7 @@ Route by job, use one tool per job, verify the artifact. python -c "import reportlab, pypdf, fitz; print('reportlab', reportlab.Version, '| pypdf ok | pymupdf ok')" ``` -- `reportlab` - creation (preferred: real text, real structure). +- `reportlab` - creation (preferred: selectable text, controlled layout). - `pypdf` - page-level transforms: split, merge, rotate, encrypt, form fields. - `pymupdf` (`fitz`) - extraction (text with coordinates, tables, images), rasterization, page-level inspection. diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md b/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md index 6b45905..7804888 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md @@ -41,25 +41,29 @@ Watermark / stamp by merging a stamp page onto each page: from pypdf import Transformation stamp = open_pdf("watermark.pdf").pages[0] +stamp.transfer_rotation_to_content() stamp_text = (stamp.extract_text() or "").strip() reader = open_pdf("input.pdf") -expected_sizes = [tuple(float(value) for value in page.mediabox) for page in reader.pages] expected_fields = reader.get_fields() or {} writer = PdfWriter() writer.append(reader) # clone pages plus catalog entries such as /AcroForm # A plain merge_page() overlays the stamp in its own coordinates, so on a page # with different dimensions, origin, or rotation the stamp can be clipped or -# land entirely off-page. Scale each copy to fit its destination and center it. -sw, sh = float(stamp.mediabox.width), float(stamp.mediabox.height) +# land entirely off-page. Normalize /Rotate into content, then scale each copy +# to the visible destination box and include both boxes' non-zero origins. +stamp_box = stamp.cropbox +sw, sh = float(stamp_box.width), float(stamp_box.height) for page in writer.pages: - dw, dh = float(page.mediabox.width), float(page.mediabox.height) + page.transfer_rotation_to_content() + destination = page.cropbox + dw, dh = float(destination.width), float(destination.height) scale = min(dw / sw, dh / sh) - tx = (dw - sw * scale) / 2 - float(stamp.mediabox.left) * scale - ty = (dh - sh * scale) / 2 - float(stamp.mediabox.bottom) * scale + tx = float(destination.left) + (dw - sw * scale) / 2 - float(stamp_box.left) * scale + ty = float(destination.bottom) + (dh - sh * scale) / 2 - float(stamp_box.bottom) * scale page.merge_transformed_page(stamp, Transformation().scale(scale).translate(tx, ty)) -# Rotated destination pages (/Rotate != 0) apply the transformation in -# unrotated page space: verify rotated pages visually after stamping. + +expected_sizes = [tuple(float(value) for value in page.mediabox) for page in writer.pages] with open("watermarked.pdf", "wb") as f: writer.write(f) @@ -119,6 +123,10 @@ Encryption and forms: ## Rules - Always write a new file; transformation in place risks losing the original on a bad write. +- Open every source through `open_pdf` above before touching `.pages`; this authenticates + encrypted inputs and fails with a clear password error instead of failing mid-transform. +- Merge complete inputs with `writer.append(..., import_outline=True)` so their outlines and + named destinations are imported and remapped; page-by-page `add_page` is for intentional splits. - After writing, re-open with `PdfReader("output.pdf")` and verify page count and page sizes. - `rotate` is cumulative on already-rotated pages - read `/Rotate` first if the source was scanned sideways. diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md index 2f4f631..e19dcaa 100644 --- a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md @@ -15,19 +15,37 @@ def iter_shapes(shapes): else: yield shape -def cached_numeric_points(series, element_name): - """Read cached numeric points python-pptx does not expose for XY/bubble axes. - - Returns (idx, value) pairs: a series with blank points omits those - entries while later points keep their original idx, so values extracted - without their indices cannot be paired across the x/y/bubble axes. - """ +def cached_numeric_points(source): + """Return indexed cached points, or None when the cache metadata is unavailable.""" + if source is None: + return None + point_counts = source.xpath("./c:numRef/c:numCache/c:ptCount | ./c:numLit/c:ptCount") + if not point_counts: + return None + points = source.xpath("./c:numRef/c:numCache/c:pt | ./c:numLit/c:pt") return [ - (int(pt.get("idx")), pt.find(qn("c:v")).text) - for pt in series._element.xpath(f"./c:{element_name}//c:pt") - if pt.get("idx") is not None and pt.find(qn("c:v")) is not None + (int(point.get("idx")), float(value.text)) + for point in points + if point.get("idx") is not None and (value := point.find(qn("c:v"))) is not None ] +def series_content(series): + x_source = getattr(series._element, "xVal", None) + if x_source is None: # category/value chart + return {"name": series.name, "values": list(series.values)} + x_points = cached_numeric_points(x_source) + y_points = cached_numeric_points(getattr(series._element, "yVal", None)) + if x_points is None or y_points is None: + return {"name": series.name, "points": None, "cache_status": "unavailable"} + content = {"name": series.name, "x_points": x_points, "y_points": y_points} + size_source = getattr(series._element, "bubbleSize", None) + if size_source is not None: + bubble_points = cached_numeric_points(size_source) + if bubble_points is None: + return {"name": series.name, "points": None, "cache_status": "unavailable"} + content["bubble_points"] = bubble_points + return content + prs = Presentation("input.pptx") print("slide size:", prs.slide_width, prs.slide_height) for i, slide in enumerate(prs.slides): @@ -57,31 +75,13 @@ for i, slide in enumerate(prs.slides): ) plots = [] for plot in chart.plots: - plot_kind = type(plot).__name__ - if plot_kind in {"XyPlot", "BubblePlot"}: - series = [] - for item in plot.series: - values = { - "name": item.name, - # Pair x/y/bubble entries by idx; a missing idx marks a - # blank point and must not shift the pairing. - "x_points": cached_numeric_points(item, "xVal"), - "y_points": cached_numeric_points(item, "yVal"), - } - if plot_kind == "BubblePlot": - values["bubble_points"] = cached_numeric_points(item, "bubbleSize") - series.append(values) - plots.append({"kind": plot_kind, "series": series}) - else: - categories = [ - [str(level) for level in label] - for label in plot.categories.flattened_labels - ] - series = [ - {"name": item.name, "values": list(item.values)} - for item in plot.series - ] - plots.append({"kind": plot_kind, "categories": categories, "series": series}) + items = list(plot.series) + has_xy_values = any(getattr(item._element, "xVal", None) is not None for item in items) + categories = [] if has_xy_values else [ + [str(level) for level in label] for label in plot.categories.flattened_labels + ] + series = [series_content(item) for item in items] + plots.append({"kind": type(plot).__name__, "categories": categories, "series": series}) charts.append({"title": chart_title, "plots": plots}) pictures = [sh.name for sh in shapes if sh.shape_type == MSO_SHAPE_TYPE.PICTURE] @@ -106,7 +106,7 @@ layouts where the title placeholder is missing.) | Text is clipped or overflows its box | render every slide and inspect right/left and bottom/vertical fit; shape bounds do not measure laid-out text | reflow, resize the box, or reduce text/font size, then render again | | Everything shifted | slide size changed between sources | normalize slide size or re-layout on the target size | | Fonts look wrong elsewhere | non-embedded fonts (pptx rarely embeds) | report every effective font via the resolution chain below, not just explicit `run.font.name` values | -| File will not open | broken ZIP / part mismatch | run the bounded package health check below before parsing every XML part | +| File will not open | broken ZIP / part mismatch | run the bounded package check below before parsing parts | | Pictures blank | media parts missing or rels broken | verify `ppt/media/*` present and slide rels reference them | ## Bounded package health check @@ -118,7 +118,6 @@ must not be the first check because it expands every member, including an archiv import zipfile from lxml import etree -path = "input.pptx" MAX_XML_PART = 20 * 1024 * 1024 MAX_ENTRY = 100 * 1024 * 1024 MAX_TOTAL_UNCOMPRESSED = 500 * 1024 * 1024 @@ -138,7 +137,7 @@ safe_xml_parser = etree.XMLParser( recover=False, ) -with zipfile.ZipFile(path) as archive: +with zipfile.ZipFile("input.pptx") as archive: infos = archive.infolist() names = {info.filename for info in infos} require(len(names) == len(infos), "duplicate archive member names are unsafe") @@ -231,86 +230,106 @@ def theme_faces_for_slide(slide): } return cache_key, theme_cache[cache_key] +EAST_ASIAN_SCRIPTS = {"Hans", "Hant", "Jpan", "Hang"} +COMPLEX_SCRIPTS = {"Arab", "Hebr", "Deva", "Beng", "Taml", "Thai"} + +def character_tags(character): + codepoint = ord(character) + if 0x3040 <= codepoint <= 0x30FF: + return ["Jpan"] + if 0x1100 <= codepoint <= 0x11FF or 0xAC00 <= codepoint <= 0xD7AF: + return ["Hang"] + if 0x2F00 <= codepoint <= 0x9FFF: + return ["Hans", "Hant", "Jpan", "Hang"] # locale disambiguates Han + for tag, start, end in ( + ("Cyrl", 0x0400, 0x052F), ("Hebr", 0x0590, 0x05FF), + ("Arab", 0x0600, 0x06FF), ("Deva", 0x0900, 0x097F), + ("Beng", 0x0980, 0x09FF), ("Taml", 0x0B80, 0x0BFF), + ("Thai", 0x0E00, 0x0E7F), + ): + if start <= codepoint <= end: + return [tag] + return [] + def script_tags(text): tags = [] for character in text: - codepoint = ord(character) - if 0x3040 <= codepoint <= 0x30FF: - tags.append("Jpan") - elif 0xAC00 <= codepoint <= 0xD7AF: - tags.append("Hang") - elif 0x2E80 <= codepoint <= 0x9FFF: - tags.extend(("Hans", "Hant", "Jpan", "Hang")) # locale disambiguates Han - elif 0x0400 <= codepoint <= 0x052F: - tags.append("Cyrl") - elif 0x0590 <= codepoint <= 0x05FF: - tags.append("Hebr") - elif 0x0600 <= codepoint <= 0x06FF: - tags.append("Arab") - elif 0x0900 <= codepoint <= 0x097F: - tags.append("Deva") + tags.extend(character_tags(character)) return list(dict.fromkeys(tags)) -def required_font_slots(text): - tags = script_tags(text) +def required_slots(text): slots = [] - if any(ch.isascii() and ch.isalnum() for ch in text): - slots.append("latin") - if any(tag in ("Hans", "Hant", "Jpan", "Hang") for tag in tags): - slots.append("eastAsia") - if any(tag in ("Arab", "Hebr", "Deva") for tag in tags): - slots.append("complexScript") + for character in text: + tags = set(character_tags(character)) + slot = "eastAsia" if tags & EAST_ASIAN_SCRIPTS else ( + "complexScript" if tags & COMPLEX_SCRIPTS else "latin" + ) + if slot not in slots: + slots.append(slot) return slots or ["latin"] -def theme_candidates_by_slot(role_fonts, text): - tags = script_tags(text) - east_tags = {"Hans", "Hant", "Jpan", "Hang"} - complex_tags = {"Arab", "Hebr", "Deva"} - candidates = {} - for slot in required_font_slots(text): +def raw_font_slots(rpr): + slots = {} + if rpr is None: + return slots + for slot, tag in (("latin", "a:latin"), ("eastAsia", "a:ea"), + ("complexScript", "a:cs")): + child = rpr.find(qn(tag)) + if child is not None and child.get("typeface"): + slots[slot] = child.get("typeface") + return slots + +THEME_TOKENS = { + "+mj-lt": ("major", "latin"), "+mj-ea": ("major", "eastAsia"), + "+mj-cs": ("major", "complexScript"), "+mn-lt": ("minor", "latin"), + "+mn-ea": ("minor", "eastAsia"), "+mn-cs": ("minor", "complexScript"), +} + +def expand_theme_token(face, theme_fonts): + role_slot = THEME_TOKENS.get(face) + return theme_fonts[role_slot[0]][role_slot[1]] if role_slot else face + +def font_candidates(run, paragraph, theme_fonts, role): + run_slots = raw_font_slots(run._r.rPr) + ppr = paragraph._p.pPr + paragraph_slots = raw_font_slots(None if ppr is None else ppr.defRPr) + tags = script_tags(run.text) + candidates = [] + for slot in required_slots(run.text): + fallback_role = role + explicit_source = None + face = run_slots.get(slot) + if face: + explicit_source = "run" + else: + face = paragraph_slots.get(slot) + if face: + explicit_source = "paragraph defaults" + if face: + resolved = expand_theme_token(face, theme_fonts) + if resolved: + candidates.append((slot, resolved, explicit_source)) + continue + # A generic +mj/+mn slot can be empty while the same role has a matching + # script-specific theme face. Preserve the token's role for that fallback. + if face in THEME_TOKENS: + fallback_role = THEME_TOKENS[face][0] + else: + continue + role_fonts = theme_fonts[fallback_role] slot_tags = [ tag for tag in tags - if (slot == "eastAsia" and tag in east_tags) - or (slot == "complexScript" and tag in complex_tags) - or (slot == "latin" and tag not in east_tags | complex_tags) + if (slot == "eastAsia" and tag in EAST_ASIAN_SCRIPTS) + or (slot == "complexScript" and tag in COMPLEX_SCRIPTS) + or (slot == "latin" and tag not in EAST_ASIAN_SCRIPTS | COMPLEX_SCRIPTS) ] - faces = [role_fonts[slot]] - faces.extend(role_fonts["scripts"].get(tag, "") for tag in slot_tags) - candidates[slot] = [face for face in dict.fromkeys(faces) if face] - return candidates - -# 2. Per run: explicit value, else paragraph defaults, else report as inherited. -# run.font.name exposes only the LATIN typeface; a run that also declares a:ea -# or a:cs must be resolved per the scripts present in its own text first. -def explicit_run_faces(run): - """Direct a:latin/a:ea/a:cs faces keyed by slot; missing slots stay missing.""" - rPr = run._r.find(qn("a:rPr")) - if rPr is None: - return {} - declared = {} - for slot, tag in (("latin", "a:latin"), ("eastAsia", "a:ea"), ("complexScript", "a:cs")): - node = rPr.find(qn(tag)) - if node is not None and node.get("typeface"): - declared[slot] = node.get("typeface") - return declared - -def resolve_run_faces(run, paragraph, role_fonts): - """Resolve every required script slot without letting one direct slot hide another.""" - direct = explicit_run_faces(run) - inherited = theme_candidates_by_slot(role_fonts, run.text) - resolved = [] - for slot in required_font_slots(run.text): - if slot in direct: - faces, source = [direct[slot]], "run direct" - elif slot == "latin" and run.font.name: - faces, source = [run.font.name], "run latin" - elif slot == "latin" and paragraph.font.name: - faces, source = [paragraph.font.name], "paragraph defaults (latin)" - else: - faces = inherited.get(slot, []) or ["(unresolved inherited face)"] - source = "theme candidates (verify placeholder chain/locale)" - resolved.append({"slot": slot, "faces": faces, "source": source}) - return resolved + candidates.extend( + (slot, role_fonts["scripts"][tag], f"{fallback_role} theme script {tag}") + for tag in slot_tags if role_fonts["scripts"].get(tag) + ) + if role_fonts.get(slot): + candidates.append((slot, role_fonts[slot], f"{fallback_role} theme {slot}")) + return list(dict.fromkeys(candidates)) def iter_text_frames(shapes): """Shape text frames plus every table cell's text frame (a graphic frame @@ -335,15 +354,16 @@ for i, slide in enumerate(prs.slides): title_shape is not None and getattr(holder, "_element", None) is title_shape._element ) else "minor" - resolved = resolve_run_faces(run, paragraph, theme_fonts[role]) - print(i, repr(run.text[:20]), f"{role} font slots:", ascii(resolved)) + faces = font_candidates(run, paragraph, theme_fonts, role) + print(i, repr(run.text[:20]), "font candidates:", ascii(faces)) ``` -python-pptx does not evaluate the full placeholder -> layout -> master inheritance chain; when -a run reports inherited, list the script-aware theme candidates above and, if the exact face -matters, check the layout and master placeholder of the same index for explicit ``, -``, ``, and script-specific `` mappings. Han text also needs the deck locale -to distinguish Hans, Hant, Japanese, and Korean theme mappings. +`run.font.name` exposes only the Latin slot, so it must not short-circuit inspection of explicit +``/`` faces. python-pptx also does not evaluate the full placeholder -> layout -> +master inheritance chain; the output above remains a candidate list. Check matching layout/master +placeholders when the exact face matters. Han text needs the deck locale to distinguish Hans, +Hant, Japanese, and Korean mappings; add other script ranges when the task uses them rather than +claiming the Latin fallback is definitive. ## Text-fit verification diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md b/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md index ff4d335..56f9f4a 100644 --- a/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md @@ -11,6 +11,7 @@ prs = Presentation("input.pptx") old, new = "old wording", "new wording" slide_index = None # Set this and shape_name when repeated text is expected. shape_name = None +target_location = None # e.g. "Table 1/table[0,1]" for duplicate table text def iter_shapes(shapes, path=""): """Yield (path, shape) for every shape, recursing into groups so text inside @@ -38,6 +39,8 @@ for i, slide in enumerate(prs.slides): if shape_name is not None and shape.name != shape_name: continue for location, text_frame in iter_text_targets(path, shape): + if target_location is not None and location != target_location: + continue if old in text_frame.text: candidates.append((i, location, text_frame)) @@ -67,7 +70,8 @@ prs.save("input-edited.pptx") template, masters, notes, and animations. Edit in place, save to a new path. 2. Address shapes by slide index + shape name or matched text, and **assert exactly one match**. The locator must search both shape text frames and every table cell, retaining a stable - `/table[row,column]` suffix. If copy repeats, set both selectors rather than choosing one. + `/table[row,column]` suffix. If copy repeats inside one table, set `target_location` as well as + the slide/shape selectors rather than choosing one. 3. For formatted text, change `run.text` only when the target is wholly inside one run. Assigning `paragraph.text` or `text_frame.text` rebuilds runs and can discard run formatting and links. If the target spans runs, stop and make an explicitly reviewed run/XML edit. diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md b/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md index 6dbfc49..8483c87 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md @@ -67,6 +67,18 @@ expected_number_formats = { # "Sales": {"D2": "#,##0.00", "E2": "yyyy-mm-dd"}, } wb = openpyxl.load_workbook(output_path) + +def formula_text(value): + if isinstance(value, str): + return value + if text := getattr(value, "text", None): + return text + fields = ("ref", "r1", "r2", "dt2D", "dtr", "ca", "del1", "del2") + details = ", ".join( + f"{name}={getattr(value, name)!r}" for name in fields if hasattr(value, name) + ) + return f"{type(value).__name__}({details})" + print("sheets:", wb.sheetnames) missing = set(expected_sheets) - set(wb.sheetnames) assert not missing, f"missing expected sheets: {sorted(missing)}" @@ -76,7 +88,7 @@ print("calcMode:", getattr(calc, "calcMode", None), for ws in wb.worksheets: print(f"{ws.title} dims:", ws.dimensions) formulas = [ - (c.coordinate, getattr(c.value, "text", None) or str(c.value)) + (c.coordinate, formula_text(c.value)) for row in ws.iter_rows() for c in row if c.data_type == "f" ] print(f"{ws.title} formula cells:", formulas[:10]) diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md index 0702534..3f9dabc 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md @@ -10,20 +10,34 @@ from openpyxl.utils.cell import range_boundaries wb = openpyxl.load_workbook("input.xlsx") # NOT data_only: that would drop all formulas ws = wb["Data"] -def non_cell_references(workbook): - """Inventory ranges/formulas that insert_rows/delete_rows will not rewrite.""" - refs = [] - defined_names = workbook.defined_names - # openpyxl 3.1 exposes a dict-like mapping; 3.0 uses DefinedNameList. - defined_name_items = ( - defined_names.values() - if hasattr(defined_names, "values") - else defined_names.definedName +def formula_text(value): + if isinstance(value, str): + return value + if text := getattr(value, "text", None): + return text + fields = ("ref", "r1", "r2", "dt2D", "dtr", "ca", "del1", "del2") + details = ", ".join( + f"{name}={getattr(value, name)!r}" for name in fields if hasattr(value, name) ) - for item in defined_name_items: + return f"{type(value).__name__}({details})" + +def defined_name_values(workbook): + names = workbook.defined_names + # openpyxl 3.1 uses DefinedNameDict; 3.0 uses DefinedNameList. Support both. + return names.values() if hasattr(names, "values") else names.definedName + +def structural_references(workbook): + """Inventory formulas/ranges that insert_rows/delete_rows will not rewrite.""" + refs = [] + for item in defined_name_values(workbook): refs.append(("defined name", item.name, item.attr_text)) for sheet in workbook.worksheets: owner = sheet.title + for row in sheet.iter_rows(): + for cell in row: + if cell.data_type == "f": + refs.append(("cell formula", f"{owner}!{cell.coordinate}", + formula_text(cell.value))) for table in sheet.tables.values(): refs.append(("table", owner + "!" + table.name, table.ref)) for merged_range in sheet.merged_cells.ranges: @@ -53,6 +67,10 @@ def non_cell_references(workbook): refs.append(("chart series", f"{owner} chart {index}", element.text)) return refs +def non_cell_references(workbook): + return [reference for reference in structural_references(workbook) + if reference[0] != "cell formula"] + def cell_formula_references(workbook): """Inventory ordinary, array, and data-table formulas before row/column moves.""" refs = [] @@ -65,7 +83,7 @@ def cell_formula_references(workbook): "cell formula", sheet.title, cell.coordinate, - getattr(value, "text", None) or str(value), + formula_text(value), )) return refs @@ -129,7 +147,7 @@ for row in summary["A1:B1"]: for cell in row: cell.font = header_font -# After applying the planned rewrites, rerun both inventories and compare them with the +# After applying the planned rewrites, rerun the same inventory and compare it with the # pre-edit snapshot before saving. A post-edit listing alone cannot reveal a stale formula. print("cell formulas before:", cell_formulas_before) print("cell formulas after planned rewrites:", cell_formula_references(wb)) diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md index 1a5080a..d8267bc 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md @@ -7,10 +7,17 @@ formula_wb = openpyxl.load_workbook("input.xlsx", read_only=True, data_only=Fals value_wb = openpyxl.load_workbook("input.xlsx", read_only=True, data_only=True) print("sheets:", value_wb.sheetnames) -def formula_text(cell): - value = cell.value - # ArrayFormula/DataTableFormula are objects in current openpyxl, not strings. - return getattr(value, "text", None) or str(value) +def formula_text(value): + if isinstance(value, str): + return value + if text := getattr(value, "text", None): # ArrayFormula + return text + # DataTableFormula has no .text; render stable, useful attributes, not an address repr. + fields = ("ref", "r1", "r2", "dt2D", "dtr", "ca", "del1", "del2") + details = ", ".join( + f"{name}={getattr(value, name)!r}" for name in fields if hasattr(value, name) + ) + return f"{type(value).__name__}({details})" # Profile EVERY sheet by default; only narrow when the task names a specific sheet. for sheet_name in value_wb.sheetnames: @@ -43,7 +50,7 @@ for sheet_name in value_wb.sheetnames: missing_cache_count += 1 if missing_cache_count <= 10: print("formula without cached value:", formula_cell.coordinate, - formula_text(formula_cell)) + formula_text(formula_cell.value)) print("formulas without cached values:", missing_cache_count) formula_wb.close() value_wb.close() diff --git a/plugins/Hylouis233/document-skills/tests/docx_fixture.py b/plugins/Hylouis233/document-skills/tests/docx_fixture.py index 43de2bc..320ddd1 100644 --- a/plugins/Hylouis233/document-skills/tests/docx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/docx_fixture.py @@ -17,6 +17,7 @@ from docx.oxml.ns import qn from docx.table import Table from docx.text.paragraph import Paragraph +from docx.text.run import Run from lxml import etree failures = [] @@ -50,31 +51,48 @@ def check(name, cond, extra=""): MAX_COMPRESSION_RATIO = 200 +def require(condition, message): + if not condition: + raise ValueError(message) + + def validate_docx_package(path): with zipfile.ZipFile(path) as archive: infos = archive.infolist() names = {info.filename for info in infos} - assert len(names) == len(infos) - assert "[Content_Types].xml" in names and "word/document.xml" in names - assert sum(info.file_size for info in infos) <= MAX_TOTAL_UNCOMPRESSED + require(len(names) == len(infos), "duplicate archive member names are unsafe") + require( + "[Content_Types].xml" in names and "word/document.xml" in names, + "required DOCX package parts are missing", + ) + require( + sum(info.file_size for info in infos) <= MAX_TOTAL_UNCOMPRESSED, + "declared archive size exceeds the review limit", + ) actual_total = 0 for info in infos: - assert info.file_size <= MAX_ENTRY - assert info.file_size / max(info.compress_size, 1) <= MAX_COMPRESSION_RATIO + require(info.file_size <= MAX_ENTRY, f"oversized part: {info.filename}") + require( + info.file_size / max(info.compress_size, 1) <= MAX_COMPRESSION_RATIO, + f"suspicious compression ratio: {info.filename}", + ) is_xml = info.filename.endswith((".xml", ".rels")) if is_xml: - assert info.file_size <= MAX_XML_PART + require(info.file_size <= MAX_XML_PART, f"oversized XML part: {info.filename}") chunks = [] actual_size = 0 with archive.open(info) as stream: while chunk := stream.read(64 * 1024): actual_size += len(chunk) actual_total += len(chunk) - assert actual_size <= MAX_ENTRY - assert actual_total <= MAX_TOTAL_UNCOMPRESSED + require(actual_size <= MAX_ENTRY, f"part exceeded read limit: {info.filename}") + require( + actual_total <= MAX_TOTAL_UNCOMPRESSED, + "archive exceeded total read limit", + ) if is_xml: chunks.append(chunk) - assert actual_size == info.file_size + require(actual_size == info.file_size, f"size mismatch: {info.filename}") if is_xml: etree.fromstring(b"".join(chunks), parser=safe_xml_parser) @@ -98,71 +116,94 @@ def validate_docx_package(path): except (AssertionError, ValueError): archive_bomb_rejected = True check("suspicious compression ratio is rejected before XML expansion", archive_bomb_rejected) +check( + "archive safety checks remain active under optimized Python", + __debug__ or archive_bomb_rejected, +) -# ---- read.md includes block paragraphs nested in content controls -------------- -def iter_body_paragraphs(parent, document): - for child in parent.iterchildren(): +# ---- read.md includes block/inline content controls and controlled tables ------- +def iter_part_blocks(root, parent): + for child in root.iterchildren(): if child.tag == qn("w:p"): - yield Paragraph(child, document) - elif child.tag == qn("w:sdt"): - content = child.find(qn("w:sdtContent")) - if content is not None: - yield from iter_body_paragraphs(content, document) + yield "paragraph", Paragraph(child, parent) + elif child.tag == qn("w:tbl"): + yield "table", Table(child, parent) + else: + yield from iter_part_blocks(child, parent) -def iter_body_blocks(parent, document): - for child in parent.iterchildren(): - if child.tag == qn("w:p"): - yield ("p", Paragraph(child, document)) - elif child.tag == qn("w:tbl"): - yield ("table", Table(child, document)) - elif child.tag == qn("w:sdt"): - content = child.find(qn("w:sdtContent")) - if content is not None: - yield from iter_body_blocks(content, document) +def iter_paragraph_runs(paragraph): + def walk(element): + for child in element.iterchildren(): + if child.tag == qn("w:r"): + yield Run(child, paragraph) + elif child.tag != qn("w:p"): + yield from walk(child) + yield from walk(paragraph._p) + + +def paragraph_text(paragraph): + return "".join(run.text for run in iter_paragraph_runs(paragraph)) + + +def table_content(table): + rows = [] + for row in table.rows: + rendered_cells = [] + for cell in row.cells: + items = [] + for kind, block in iter_part_blocks(cell._tc, cell): + items.append(paragraph_text(block) if kind == "paragraph" else table_content(block)) + rendered_cells.append(items) + rows.append(rendered_cells) + return rows + + +def wrap_in_sdt(element): + sdt = OxmlElement("w:sdt") + content = OxmlElement("w:sdtContent") + element.getparent().replace(element, sdt) + content.append(element) + sdt.append(content) sdt_doc = Document() sdt_doc.add_paragraph("direct paragraph") sdt_paragraph = sdt_doc.add_paragraph("inside content control") -sdt = OxmlElement("w:sdt") -sdt_content = OxmlElement("w:sdtContent") -sdt_paragraph._p.getparent().replace(sdt_paragraph._p, sdt) -sdt_content.append(sdt_paragraph._p) -sdt.append(sdt_content) -sdt_doc.add_paragraph("trailing paragraph") +wrap_in_sdt(sdt_paragraph._p) +controlled_table = sdt_doc.add_table(rows=1, cols=1) +controlled_table.cell(0, 0).text = "table한" +nested_table = controlled_table.cell(0, 0).add_table(rows=1, cols=1) +nested_table.cell(0, 0).text = "nested한" +wrap_in_sdt(controlled_table._tbl) +inline_paragraph = sdt_doc.add_paragraph("before-") +inline_run = inline_paragraph.add_run("inline한") +inline_paragraph.add_run("-after") +wrap_in_sdt(inline_run._r) sdt_doc.save("content-control.docx") sdt_reopened = Document("content-control.docx") check("doc.paragraphs omits block content-control text (negative control)", "inside content control" not in [paragraph.text for paragraph in sdt_reopened.paragraphs]) -walked_text = [paragraph.text for paragraph in iter_body_paragraphs(sdt_reopened.element.body, sdt_reopened)] +walked_blocks = list(iter_part_blocks(sdt_reopened.element.body, sdt_reopened)) +check("doc.tables omits a table wrapped by a block content control (negative control)", + len(sdt_reopened.tables) == 0) +check("block walker preserves document order across content controls", + [kind for kind, _ in walked_blocks] + == ["paragraph", "paragraph", "table", "paragraph"], walked_blocks) +walked_paragraphs = [block for kind, block in walked_blocks if kind == "paragraph"] +walked_text = [paragraph_text(paragraph) for paragraph in walked_paragraphs] check("content-control traversal emits the nested paragraph", "inside content control" in walked_text, walked_text) - -# A table placed inside w:sdtContent is invisible to doc.tables; the block walker reaches it. -table_doc = Document() -table_doc.add_paragraph("before the control") -sdt_table = table_doc.add_table(rows=2, cols=2) -sdt_table.cell(0, 0).text = "ctrl-a1" -sdt_table.cell(1, 1).text = "ctrl-b2" -table_sdt = OxmlElement("w:sdt") -table_sdt_content = OxmlElement("w:sdtContent") -sdt_table._tbl.getparent().replace(sdt_table._tbl, table_sdt) -table_sdt_content.append(sdt_table._tbl) -table_sdt.append(table_sdt_content) -table_doc.add_paragraph("after the control") -table_doc.save("content-control-table.docx") -table_reopened = Document("content-control-table.docx") -check("doc.tables omits content-control tables (negative control)", len(table_reopened.tables) == 0) -blocks = list(iter_body_blocks(table_reopened.element.body, table_reopened)) -kinds = [kind for kind, _ in blocks] -nested_tables = [block for kind, block in blocks if kind == "table"] -check("block walker keeps document order across the control", - kinds == ["p", "table", "p"], kinds) -check("block walker surfaces the content-control table", len(nested_tables) == 1) -if nested_tables: - check("content-control table cells are readable", - nested_tables[0].cell(0, 0).text == "ctrl-a1" and nested_tables[0].cell(1, 1).text == "ctrl-b2", - [[cell.text for cell in row.cells] for row in nested_tables[0].rows]) +check("content-control traversal emits inline run text", + "before-inline한-after" in walked_text, walked_text) +walked_tables = [block for kind, block in walked_blocks if kind == "table"] +rendered_tables = [table_content(table) for table in walked_tables] +check("content-control traversal emits a wrapped table", + len(walked_tables) == 1 and "table한" in str(rendered_tables), rendered_tables) +all_emitted = walked_text + [str(table) for table in rendered_tables] +check("table text is emitted exactly once, not again as prose", + sum(item.count("table한") for item in all_emitted) == 1, all_emitted) +check("nested-table text is emitted exactly once", + sum(item.count("nested한") for item in all_emitted) == 1, all_emitted) # Per-run glyph validation must not let a different referenced font hide a missing glyph. fixture_cmaps = {"CJK Face": {ord("漢")}, "Latin Face": {ord("A")}} @@ -175,57 +216,90 @@ def iter_body_blocks(parent, document): check("per-run cmap check identifies the actual missing glyph", per_run_missing == [("Latin Face", "漢")], per_run_missing) -# Header/footer runs do not expose run.part.document; resolve Normal from doc once. -header_doc = Document() -normal_style = header_doc.styles["Normal"] -normal_rfonts = normal_style._element.get_or_add_rPr().get_or_add_rFonts() -normal_rfonts.set(qn("w:eastAsia"), "Header CJK Face") -header_run = header_doc.sections[0].header.paragraphs[0].add_run("漢") +def font_slot(character): + codepoint = ord(character) + uses_east_asian_slot = ( + 0x1100 <= codepoint <= 0x11FF + or 0x2F00 <= codepoint <= 0x9FFF + or 0xA000 <= codepoint <= 0xA4CF + or 0xA960 <= codepoint <= 0xA97F + or 0xAC00 <= codepoint <= 0xD7FF + or 0xF900 <= codepoint <= 0xFAFF + or 0xFE30 <= codepoint <= 0xFE6F + or 0xFF00 <= codepoint <= 0xFFEF + or 0x20000 <= codepoint <= 0x3134F + ) + return "eastAsia" if uses_east_asian_slot else ("ascii" if codepoint < 128 else "hAnsi") + + +check("Han ideographs use the east-Asian font slot", font_slot("漢") == "eastAsia") +check("Hangul syllables use the east-Asian font slot", font_slot("한") == "eastAsia") +check("Hangul Jamo use the east-Asian font slot", font_slot("ᄒ") == "eastAsia") +check("Yi syllables use the east-Asian font slot", font_slot("ꀀ") == "eastAsia") +check("CJK compatibility forms use the east-Asian font slot", font_slot("︰") == "eastAsia") +check("ASCII text keeps the ascii font slot", font_slot("A") == "ascii") + +# Header/footer parts have no `.document`; effective styles close over the document. +font_doc = Document() +normal_style = font_doc.styles["Normal"] +normal_fonts = normal_style.element.get_or_add_rPr().get_or_add_rFonts() +normal_fonts.set(qn("w:ascii"), "Latin Face") +normal_fonts.set(qn("w:hAnsi"), "Latin Face") +normal_fonts.set(qn("w:eastAsia"), "CJK Face") +header_run = font_doc.sections[0].header.paragraphs[0].add_run("页眉") +footer_run = font_doc.sections[0].footer.paragraphs[0].add_run("页脚") -def fixture_face_from_rpr(rpr, slot): +def face_from_rpr(rpr, slot): if rpr is None: return None rfonts = rpr.find(qn("w:rFonts")) return None if rfonts is None else rfonts.get(qn("w:" + slot)) -def fixture_style_faces(style, slot): +def style_faces(style, slot): while style is not None: - face = fixture_face_from_rpr(style.element.find(qn("w:rPr")), slot) + face = face_from_rpr(style.element.find(qn("w:rPr")), slot) if face: yield face style = style.base_style -def fixture_effective_face(run, slot): - direct = fixture_face_from_rpr(run._r.find(qn("w:rPr")), slot) +def effective_face(run, slot): + direct = face_from_rpr(run._r.find(qn("w:rPr")), slot) if direct: return direct for style in (run.style, run._parent.style, normal_style): - if face := next(fixture_style_faces(style, slot), None): + if face := next(style_faces(style, slot), None): return face raise LookupError(slot) check("header part has no document back-reference (negative control)", not hasattr(header_run.part, "document")) -check("header glyph validation resolves document Normal style", - fixture_effective_face(header_run, "eastAsia") == "Header CJK Face") +check("header runs resolve Normal from the owning document", + effective_face(header_run, "eastAsia") == "CJK Face") +check("footer runs resolve Normal from the owning document", + effective_face(footer_run, "eastAsia") == "CJK Face") +inline_paragraph_reopened = next( + paragraph for paragraph in walked_paragraphs if "before-" in paragraph_text(paragraph) +) +check("inline content-control runs are included in glyph traversal", + any(run.text == "inline한" for run in iter_paragraph_runs(inline_paragraph_reopened))) # ---- edit.md guarded cross-run replacement ------------------------------------ SAFE_RUN_CHILDREN = { - qn("w:rPr"), qn("w:t"), qn("w:tab"), qn("w:br"), qn("w:cr"), + qn("w:rPr"), qn("w:t"), qn("w:tab"), qn("w:cr"), } def unsafe_run_content(run): unsafe = [] for child in run._r: - typed_break = child.tag == qn("w:br") and child.get(qn("w:type")) not in ( - None, "textWrapping", + ordinary_break = child.tag == qn("w:br") and dict(child.attrib) in ( + {}, {qn("w:type"): "textWrapping"}, ) - if child.tag not in SAFE_RUN_CHILDREN or typed_break: + if child.tag not in SAFE_RUN_CHILDREN and not ordinary_break: unsafe.append(child.tag.rsplit("}", 1)[-1]) return unsafe @@ -304,6 +378,23 @@ def replace_across_runs(paragraph, old, new): guard_run.text == "TBD" and len(guard_run._r.findall(qn("w:drawing"))) == 1, ) +clear_doc = Document() +clear_paragraph = clear_doc.add_paragraph() +clear_run = clear_paragraph.add_run("TBD") +clear_break = OxmlElement("w:br") +clear_break.set(qn("w:type"), "textWrapping") +clear_break.set(qn("w:clear"), "left") +clear_run._r.append(clear_break) +clear_before = etree.tostring(clear_run._r) +try: + replace_across_runs(clear_paragraph, "TBD", "done") + clear_break_rejected = False +except ValueError: + clear_break_rejected = True +check("replacement rejects a wrapping break with clear semantics", clear_break_rejected) +check("rejected clear-break replacement is atomic", + etree.tostring(clear_run._r) == clear_before) + def list_number_num_id(doc): """The numId that the ListNumber style binds to in this document part.""" @@ -503,6 +594,9 @@ def font_slot(character): check("Hangul syllables use the East Asian slot", font_slot("한") == "eastAsia") check("Hangul jamo use the East Asian slot", font_slot("ᄀ") == "eastAsia") check("compatibility jamo use the East Asian slot", font_slot("ㄱ") == "eastAsia") +check("Hangul extended-A uses the East Asian slot", font_slot(chr(0xA960)) == "eastAsia") +check("Hangul extended-B uses the East Asian slot", font_slot(chr(0xD7B0)) == "eastAsia") +check("U+2E80 CJK radical uses the East Asian slot", font_slot(chr(0x2E80)) == "eastAsia") check("Latin stays in the ascii slot", font_slot("A") == "ascii") check("non-CJK fullwidth-range-adjacent Latin-1 stays hAnsi", font_slot("é") == "hAnsi") diff --git a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py index 19d9a5e..58043af 100644 --- a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py @@ -22,6 +22,14 @@ def check(name, cond, extra=""): failures.append(name) +def open_pdf(path, password=None): + reader = pypdf.PdfReader(path) + if reader.is_encrypted: + if not password or reader.decrypt(password) == 0: + raise RuntimeError(f"valid password required for {path}") + return reader + + # ---- build a 2-page A4 PDF with one AcroForm text field on page 1 ------------- c = canvas.Canvas("form.pdf", pagesize=A4) c.setFont("Helvetica", 16) @@ -55,6 +63,15 @@ def check(name, cond, extra=""): check("encrypted fixture is detected before page access", probe.is_encrypted) encrypted_r = pypdf.PdfReader("encrypted.pdf", password="fixture-password") check("password-authenticated postcheck can access every page", len(encrypted_r.pages) == 2) +try: + open_pdf("encrypted.pdf") + transform_rejected_missing_password = False +except RuntimeError: + transform_rejected_missing_password = True +check("PDF transforms reject encrypted input without a password", + transform_rejected_missing_password) +check("PDF transforms authenticate before page access", + len(open_pdf("encrypted.pdf", "fixture-password").pages) == 2) encrypted_extract = fitz.open("encrypted.pdf") check("PyMuPDF extraction detects that authentication is required", encrypted_extract.needs_pass) check("PyMuPDF rejects the wrong extraction password", encrypted_extract.authenticate("wrong") == 0) @@ -162,7 +179,7 @@ def widget_count(page): ) # ---- transform.md watermark snippet ------------------------------------------- -from pypdf import PdfReader as R2 +from pypdf import PdfReader as R2, Transformation stamp_src = canvas.Canvas("stamp.pdf", pagesize=A4) stamp_src.setFont("Helvetica", 40) @@ -171,14 +188,27 @@ def widget_count(page): stamp_src.save() stamp = R2("stamp.pdf").pages[0] +stamp.transfer_rotation_to_content() stamp_text = (stamp.extract_text() or "").strip() reader = R2("form.pdf") -expected_sizes = [(round(float(p.mediabox.width), 2), round(float(p.mediabox.height), 2)) for p in reader.pages] expected_fields = reader.get_fields() or {} writer = PdfWriter() writer.append(reader) +stamp_box = stamp.cropbox for page in writer.pages: - page.merge_page(stamp) + page.transfer_rotation_to_content() + destination = page.cropbox + scale = min(float(destination.width) / float(stamp_box.width), + float(destination.height) / float(stamp_box.height)) + tx = (float(destination.left) + + (float(destination.width) - float(stamp_box.width) * scale) / 2 + - float(stamp_box.left) * scale) + ty = (float(destination.bottom) + + (float(destination.height) - float(stamp_box.height) * scale) / 2 + - float(stamp_box.bottom) * scale) + page.merge_transformed_page(stamp, Transformation().scale(scale).translate(tx, ty)) +expected_sizes = [(round(float(p.mediabox.width), 2), round(float(p.mediabox.height), 2)) + for p in writer.pages] with open("watermarked.pdf", "wb") as f: writer.write(f) @@ -192,6 +222,28 @@ def widget_count(page): set(expected_fields) <= set(verify.get_fields() or {}), verify.get_fields()) check("stamp text present on every page", all(stamp_text in (p.extract_text() or "") for p in verify.pages)) +# ---- inspect.md blank-page predicate includes widgets and annotations ----------- +form_only_canvas = canvas.Canvas("form-only.pdf", pagesize=A4) +form_only_canvas.acroForm.textfield( + name="widget_only", x=72, y=740, width=260, height=20, borderWidth=0 +) +form_only_canvas.showPage() +form_only_canvas.save() +form_only_doc = fitz.open("form-only.pdf") +form_only_page = form_only_doc[0] +form_only_widgets = list(form_only_page.widgets() or ()) +form_only_annotations = list(form_only_page.annots() or ()) +form_only_is_blank = ( + not form_only_page.get_text().strip() + and not form_only_page.get_images() + and not form_only_page.get_drawings() + and not form_only_page.get_links() + and not form_only_widgets + and not form_only_annotations +) +check("form-only page exposes a widget", len(form_only_widgets) == 1) +check("form-only page is not classified as blank", not form_only_is_blank) + # ---- extract.md CMYK conversion snippet --------------------------------------- pix = fitz.Pixmap(fitz.csCMYK, fitz.IRect(0, 0, 24, 24)) # CMYK pixmap like a CMYK PDF image converted = fitz.Pixmap(fitz.csRGB, pix) if pix.colorspace not in (fitz.csGRAY, fitz.csRGB) else pix @@ -286,13 +338,19 @@ def widget_count(page): check("find_tables extracts data rows", extracted_rows[2] == ["South", "340"], extracted_rows) -# ---- transform.md: stamps are scaled to each destination page -------------------- -from pypdf import Transformation +# ---- transform.md: stamps fit non-zero-origin and rotated destination pages ------ mixed_writer = PdfWriter() mixed_writer.append(open_pdf("form.pdf")) small_source = PdfWriter() small_source.add_blank_page(width=200, height=300) +offset_page = small_source.add_blank_page(width=200, height=300) +offset_page.mediabox.lower_left = (100, 200) +offset_page.mediabox.upper_right = (300, 500) +offset_page.cropbox.lower_left = (100, 200) +offset_page.cropbox.upper_right = (300, 500) +rotated_page = small_source.add_blank_page(width=240, height=160) +rotated_page.rotate(90) mixed_writer.append(small_source) with open("mixed.pdf", "wb") as f: mixed_writer.write(f) @@ -328,12 +386,16 @@ def stamp_bboxes(path, page_number): scaled_writer = PdfWriter() scaled_writer.append(open_pdf("mixed.pdf")) stamp_page = R2("stamp.pdf").pages[0] -sw2, sh2 = float(stamp_page.mediabox.width), float(stamp_page.mediabox.height) +stamp_page.transfer_rotation_to_content() +stamp_box2 = stamp_page.cropbox +sw2, sh2 = float(stamp_box2.width), float(stamp_box2.height) for page in scaled_writer.pages: - dw, dh = float(page.mediabox.width), float(page.mediabox.height) + page.transfer_rotation_to_content() + destination = page.cropbox + dw, dh = float(destination.width), float(destination.height) scale = min(dw / sw2, dh / sh2) - tx = (dw - sw2 * scale) / 2 - float(stamp_page.mediabox.left) * scale - ty = (dh - sh2 * scale) / 2 - float(stamp_page.mediabox.bottom) * scale + tx = float(destination.left) + (dw - sw2 * scale) / 2 - float(stamp_box2.left) * scale + ty = float(destination.bottom) + (dh - sh2 * scale) / 2 - float(stamp_box2.bottom) * scale page.merge_transformed_page(stamp_page, Transformation().scale(scale).translate(tx, ty)) with open("scaled-stamped.pdf", "wb") as f: scaled_writer.write(f) @@ -344,8 +406,14 @@ def stamp_bboxes(path, page_number): check("scaled stamp lands inside the mixed-size small page", bool(scaled_spans) and all(bbox[1] < 300 and bbox[3] <= 300.5 for bbox in scaled_spans), scaled_spans) -check("mixed-size pages keep their original dimensions", - [round(float(p.mediabox.width)) for p in scaled_check.pages] == [595, 595, 200]) +offset_spans = stamp_bboxes("scaled-stamped.pdf", 3) +rotated_spans = stamp_bboxes("scaled-stamped.pdf", 4) +check("scaled stamp lands inside the non-zero-origin page", bool(offset_spans), offset_spans) +check("scaled stamp lands inside the normalized 90-degree page", bool(rotated_spans), rotated_spans) +check("mixed-size pages keep their visible dimensions after rotation normalization", + [(round(float(p.mediabox.width)), round(float(p.mediabox.height))) + for p in scaled_check.pages] + == [(595, 842), (595, 842), (200, 300), (200, 300), (160, 240)]) # ---- SKILL.md overflow check: off-page text is a defect ------------------------- diff --git a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py index 650f25d..e616a4a 100644 --- a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py @@ -5,7 +5,9 @@ # python pptx_fixture.py (deps: python-pptx) # python xlsx_fixture.py (deps: openpyxl) # python docx_fixture.py (deps: python-docx, pymupdf, soffice on PATH) +import copy import sys +import xml.etree.ElementTree as ET import zipfile from lxml import etree @@ -61,10 +63,101 @@ def check(name, cond, extra=""): ).chart chart.has_title = True chart.chart_title.text_frame.text = "Units by region" + +xy_data = XyChartData() +xy_series = xy_data.add_series("Trend") +xy_series.add_data_point(1, 2) +xy_series.add_data_point(3, 4) +xy_chart = slide.shapes.add_chart( + XL_CHART_TYPE.XY_SCATTER, Inches(7.2), Inches(4.7), Inches(2), Inches(2), xy_data, +).chart +xy_chart.has_title = True +xy_chart.chart_title.text_frame.text = "XY trend" + +bubble_data = BubbleChartData() +bubble_series = bubble_data.add_series("Risk") +bubble_series.add_data_point(3, 4, 5) +bubble_chart = slide.shapes.add_chart( + XL_CHART_TYPE.BUBBLE, Inches(5), Inches(4.7), Inches(2), Inches(2), bubble_data, +).chart +bubble_chart.has_title = True +bubble_chart.chart_title.text_frame.text = "Bubble risk" slide.notes_slide.notes_text_frame.text = "Speaker note: explain the regional split." prs.save("input.pptx") +# ---- analyze.md bounded package check rejects archive bombs before expansion --- +MAX_XML_PART = 20 * 1024 * 1024 +MAX_ENTRY = 100 * 1024 * 1024 +MAX_TOTAL_UNCOMPRESSED = 500 * 1024 * 1024 +MAX_COMPRESSION_RATIO = 200 +safe_xml_parser = etree.XMLParser( + load_dtd=False, + resolve_entities=False, + no_network=True, + huge_tree=False, + recover=False, +) + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def validate_pptx_package(path): + with zipfile.ZipFile(path) as archive: + infos = archive.infolist() + names = {info.filename for info in infos} + require(len(names) == len(infos), "duplicate archive member names are unsafe") + require("[Content_Types].xml" in names and "ppt/presentation.xml" in names, + "required PPTX package parts are missing") + require(sum(info.file_size for info in infos) <= MAX_TOTAL_UNCOMPRESSED, + "declared archive size exceeds the review limit") + actual_total = 0 + for info in infos: + require(info.file_size <= MAX_ENTRY, f"oversized part: {info.filename}") + require(info.file_size / max(info.compress_size, 1) <= MAX_COMPRESSION_RATIO, + f"suspicious compression ratio: {info.filename}") + is_xml = info.filename.endswith((".xml", ".rels")) + if is_xml: + require(info.file_size <= MAX_XML_PART, f"oversized XML part: {info.filename}") + chunks = [] + actual_size = 0 + with archive.open(info) as stream: + while chunk := stream.read(64 * 1024): + actual_size += len(chunk) + actual_total += len(chunk) + require(actual_size <= MAX_ENTRY, + f"part exceeded read limit: {info.filename}") + require(actual_total <= MAX_TOTAL_UNCOMPRESSED, + "archive exceeded total read limit") + if is_xml: + chunks.append(chunk) + require(actual_size == info.file_size, f"size mismatch: {info.filename}") + if is_xml: + etree.fromstring(b"".join(chunks), parser=safe_xml_parser) + + +try: + validate_pptx_package("input.pptx") + healthy_pptx_passed = True +except Exception: + healthy_pptx_passed = False +check("bounded PPTX check accepts an ordinary deck", healthy_pptx_passed) +with zipfile.ZipFile("compressed-bomb.pptx", "w", zipfile.ZIP_DEFLATED) as archive: + archive.writestr("[Content_Types].xml", "") + archive.writestr( + "ppt/presentation.xml", + '' + (" " * 2_000_000) + "", + ) +try: + validate_pptx_package("compressed-bomb.pptx") + pptx_bomb_rejected = False +except ValueError: + pptx_bomb_rejected = True +check("PPTX compression bomb is rejected before XML expansion", pptx_bomb_rejected) + # ---- edit.md snippet: single-shape run replace keeps styling and hyperlink ----- prs = Presentation("input.pptx") old, new = "old wording", "new wording" @@ -114,6 +207,8 @@ def check(name, cond, extra=""): # ---- analyze.md snippet: grouped-shape walker ---------------------------------- from pptx.oxml import parse_xml +from pptx.oxml.ns import qn +from pptx.oxml.xmlchemy import OxmlElement prs6 = Presentation() slide6 = prs6.slides.add_slide(prs6.slide_layouts[5]) @@ -141,13 +236,38 @@ def iter_shapes(shapes): yield shape -def cached_numeric_values(series, element_name): +def cached_numeric_points(source): + if source is None: + return None + point_counts = source.xpath("./c:numRef/c:numCache/c:ptCount | ./c:numLit/c:ptCount") + if not point_counts: + return None + points = source.xpath("./c:numRef/c:numCache/c:pt | ./c:numLit/c:pt") return [ - node.text for node in - series._element.xpath(f"./c:{element_name}//c:pt/c:v") + (int(point.get("idx")), float(value.text)) + for point in points + if point.get("idx") is not None and (value := point.find(qn("c:v"))) is not None ] +def series_content(series): + x_source = getattr(series._element, "xVal", None) + if x_source is None: + return {"name": series.name, "values": list(series.values)} + x_points = cached_numeric_points(x_source) + y_points = cached_numeric_points(getattr(series._element, "yVal", None)) + if x_points is None or y_points is None: + return {"name": series.name, "points": None, "cache_status": "unavailable"} + content = {"name": series.name, "x_points": x_points, "y_points": y_points} + size_source = getattr(series._element, "bubbleSize", None) + if size_source is not None: + bubble_points = cached_numeric_points(size_source) + if bubble_points is None: + return {"name": series.name, "points": None, "cache_status": "unavailable"} + content["bubble_points"] = bubble_points + return content + + def extract_slide_content(slide): shapes = list(iter_shapes(slide.shapes)) text = [sh.text_frame.text for sh in shapes if sh.has_text_frame and sh.text_frame.text] @@ -172,29 +292,13 @@ def extract_slide_content(slide): ) plots = [] for plot in chart.plots: - plot_kind = type(plot).__name__ - if plot_kind in {"XyPlot", "BubblePlot"}: - series = [] - for item in plot.series: - values = { - "name": item.name, - "x_values": cached_numeric_values(item, "xVal"), - "y_values": cached_numeric_values(item, "yVal"), - } - if plot_kind == "BubblePlot": - values["bubble_sizes"] = cached_numeric_values(item, "bubbleSize") - series.append(values) - plots.append({"kind": plot_kind, "series": series}) - else: - categories = [ - [str(level) for level in label] - for label in plot.categories.flattened_labels - ] - series = [ - {"name": item.name, "values": list(item.values)} - for item in plot.series - ] - plots.append({"kind": plot_kind, "categories": categories, "series": series}) + items = list(plot.series) + has_xy_values = any(getattr(item._element, "xVal", None) is not None for item in items) + categories = [] if has_xy_values else [ + [str(level) for level in label] for label in plot.categories.flattened_labels + ] + series = [series_content(item) for item in items] + plots.append({"kind": type(plot).__name__, "categories": categories, "series": series}) charts.append({"title": chart_title, "plots": plots}) notes = slide.notes_slide.notes_text_frame.text if slide.has_notes_slide else "" return {"text": text, "tables": tables, "charts": charts, "notes": notes} @@ -212,6 +316,17 @@ def extract_slide_content(slide): == [{"name": "Units", "values": [120.0, 80.0]}], content["charts"], ) +chart_by_title = {item["title"]: item for item in content["charts"]} +check("content inventory emits XY x/y points", + chart_by_title["XY trend"]["plots"][0]["series"][0]["x_points"] + == [(0, 1.0), (1, 3.0)] + and chart_by_title["XY trend"]["plots"][0]["series"][0]["y_points"] + == [(0, 2.0), (1, 4.0)]) +check("content inventory emits bubble x/y/size points", + chart_by_title["Bubble risk"]["plots"][0]["series"][0]["x_points"] == [(0, 3.0)] + and chart_by_title["Bubble risk"]["plots"][0]["series"][0]["y_points"] == [(0, 4.0)] + and chart_by_title["Bubble risk"]["plots"][0]["series"][0]["bubble_points"] + == [(0, 5.0)]) check("content inventory emits notes text", "regional split" in content["notes"], content["notes"]) merge_prs = Presentation() @@ -255,8 +370,8 @@ def extract_slide_content(slide): "scatter inventory emits x and y caches without category access", any( plot["kind"] == "XyPlot" - and plot["series"][0]["x_values"] == ["1", "3"] - and plot["series"][0]["y_values"] == ["2", "4"] + and plot["series"][0]["x_points"] == [(0, 1.0), (1, 3.0)] + and plot["series"][0]["y_points"] == [(0, 2.0), (1, 4.0)] for plot in xy_plots ), xy_plots, @@ -265,14 +380,33 @@ def extract_slide_content(slide): "bubble inventory emits x, y, and bubble-size caches", any( plot["kind"] == "BubblePlot" - and plot["series"][0]["x_values"] == ["5"] - and plot["series"][0]["y_values"] == ["6"] - and plot["series"][0]["bubble_sizes"] == ["7"] + and plot["series"][0]["x_points"] == [(0, 5.0)] + and plot["series"][0]["y_points"] == [(0, 6.0)] + and plot["series"][0]["bubble_points"] == [(0, 7.0)] for plot in xy_plots ), xy_plots, ) +# Missing caches and missing ptCount are unavailable, not a proven empty series. +xy_plot = next(plot for chart in Presentation("xy-bubble.pptx").slides[0].shapes + if chart.has_chart for plot in chart.chart.plots + if type(plot).__name__ == "XyPlot") +xy_item = list(xy_plot.series)[0] +x_source = xy_item._element.xVal +missing_cache = copy.deepcopy(x_source) +missing_cache_ref = missing_cache.find(qn("c:numRef")) +missing_cache_ref.remove(missing_cache_ref.find(qn("c:numCache"))) +check("numeric cache without numCache reports unavailable", cached_numeric_points(missing_cache) is None) +missing_count = copy.deepcopy(x_source) +missing_count_cache = missing_count.find(qn("c:numRef") + "/" + qn("c:numCache")) +missing_count_cache.remove(missing_count_cache.find(qn("c:ptCount"))) +check("numeric cache without ptCount reports unavailable", cached_numeric_points(missing_count) is None) +actual_cache = x_source.xpath("./c:numRef/c:numCache")[0] +x_source.xpath("./c:numRef")[0].remove(actual_cache) +check("XY series with an unavailable cache is explicit", + series_content(xy_item).get("cache_status") == "unavailable", series_content(xy_item)) + sp_element = slide6.shapes[-1]._element # the textbox; layout 5 still carries a Title placeholder group_element = parse_xml(GRP) @@ -348,8 +482,25 @@ def iter_text_targets(path, shape): check("table-cell locator retains row and column", table_candidates[0][1].endswith("/table[0,1]"), table_candidates[0][1]) +duplicate_prs = Presentation("input.pptx") +duplicate_shape = next(shape for shape in duplicate_prs.slides[0].shapes if shape.has_table) +duplicate_shape.table.cell(1, 0).text = old_cell +all_table_candidates = [ + (i, location, text_frame) + for i, slide_item in enumerate(duplicate_prs.slides) + for path, shape in iter_shapes_with_path(slide_item.shapes) + for location, text_frame in iter_text_targets(path, shape) + if old_cell in text_frame.text +] +target_location = f"{duplicate_shape.name}/table[0,1]" +selected_table_candidates = [ + candidate for candidate in all_table_candidates if candidate[1] == target_location +] +check("duplicate table text requires a location selector", len(all_table_candidates) == 2) +check("table location selector chooses one row/column", + len(selected_table_candidates) == 1, [item[1] for item in all_table_candidates]) + # ---- analyze.md snippet: per-master, script-aware theme font resolution -------- -import xml.etree.ElementTree as ET prs7 = Presentation("input.pptx") theme_cache = {} @@ -389,36 +540,111 @@ def theme_faces_for_slide(slide): return cache_key, theme_cache[cache_key] +EAST_ASIAN_SCRIPTS = {"Hans", "Hant", "Jpan", "Hang"} +COMPLEX_SCRIPTS = {"Arab", "Hebr", "Deva", "Beng", "Taml", "Thai"} + + +def character_tags(character): + codepoint = ord(character) + if 0x3040 <= codepoint <= 0x30FF: + return ["Jpan"] + if 0x1100 <= codepoint <= 0x11FF or 0xAC00 <= codepoint <= 0xD7AF: + return ["Hang"] + if 0x2F00 <= codepoint <= 0x9FFF: + return ["Hans", "Hant", "Jpan", "Hang"] + for tag, start, end in ( + ("Cyrl", 0x0400, 0x052F), ("Hebr", 0x0590, 0x05FF), + ("Arab", 0x0600, 0x06FF), ("Deva", 0x0900, 0x097F), + ("Beng", 0x0980, 0x09FF), ("Taml", 0x0B80, 0x0BFF), + ("Thai", 0x0E00, 0x0E7F), + ): + if start <= codepoint <= end: + return [tag] + return [] + + def script_tags(text): tags = [] for character in text: - codepoint = ord(character) - if 0x3040 <= codepoint <= 0x30FF: - tags.append("Jpan") - elif 0xAC00 <= codepoint <= 0xD7AF: - tags.append("Hang") - elif 0x2E80 <= codepoint <= 0x9FFF: - tags.extend(("Hans", "Hant", "Jpan", "Hang")) - elif 0x0400 <= codepoint <= 0x052F: - tags.append("Cyrl") - elif 0x0590 <= codepoint <= 0x05FF: - tags.append("Hebr") - elif 0x0600 <= codepoint <= 0x06FF: - tags.append("Arab") - elif 0x0900 <= codepoint <= 0x097F: - tags.append("Deva") + tags.extend(character_tags(character)) return list(dict.fromkeys(tags)) -def theme_candidates(role_fonts, text): - faces = [role_fonts["latin"]] - tags = script_tags(text) - if any(tag in ("Hans", "Hant", "Jpan", "Hang") for tag in tags): - faces.append(role_fonts["eastAsia"]) - if any(tag in ("Arab", "Hebr", "Deva") for tag in tags): - faces.append(role_fonts["complexScript"]) - faces.extend(role_fonts["scripts"].get(tag, "") for tag in tags) - return [face for face in dict.fromkeys(faces) if face] +def required_slots(text): + slots = [] + for character in text: + tags = set(character_tags(character)) + slot = "eastAsia" if tags & EAST_ASIAN_SCRIPTS else ( + "complexScript" if tags & COMPLEX_SCRIPTS else "latin" + ) + if slot not in slots: + slots.append(slot) + return slots or ["latin"] + + +def raw_font_slots(rpr): + slots = {} + if rpr is None: + return slots + for slot, tag in (("latin", "a:latin"), ("eastAsia", "a:ea"), + ("complexScript", "a:cs")): + child = rpr.find(qn(tag)) + if child is not None and child.get("typeface"): + slots[slot] = child.get("typeface") + return slots + + +THEME_TOKENS = { + "+mj-lt": ("major", "latin"), "+mj-ea": ("major", "eastAsia"), + "+mj-cs": ("major", "complexScript"), "+mn-lt": ("minor", "latin"), + "+mn-ea": ("minor", "eastAsia"), "+mn-cs": ("minor", "complexScript"), +} + + +def expand_theme_token(face, theme_fonts): + role_slot = THEME_TOKENS.get(face) + return theme_fonts[role_slot[0]][role_slot[1]] if role_slot else face + + +def font_candidates(run, paragraph, theme_fonts, role): + run_slots = raw_font_slots(run._r.rPr) + ppr = paragraph._p.pPr + paragraph_slots = raw_font_slots(None if ppr is None else ppr.defRPr) + tags = script_tags(run.text) + candidates = [] + for slot in required_slots(run.text): + fallback_role = role + explicit_source = None + face = run_slots.get(slot) + if face: + explicit_source = "run" + else: + face = paragraph_slots.get(slot) + if face: + explicit_source = "paragraph defaults" + if face: + resolved = expand_theme_token(face, theme_fonts) + if resolved: + candidates.append((slot, resolved, explicit_source)) + continue + if face in THEME_TOKENS: + fallback_role = THEME_TOKENS[face][0] + else: + continue + role_fonts = theme_fonts[fallback_role] + slot_tags = [ + tag for tag in tags + if (slot == "eastAsia" and tag in EAST_ASIAN_SCRIPTS) + or (slot == "complexScript" and tag in COMPLEX_SCRIPTS) + or (slot == "latin" and tag not in EAST_ASIAN_SCRIPTS | COMPLEX_SCRIPTS) + ] + candidates.extend( + (slot, role_fonts["scripts"][tag], f"{fallback_role} theme script {tag}") + for tag in slot_tags if role_fonts["scripts"].get(tag) + ) + if role_fonts.get(slot): + candidates.append((slot, role_fonts[slot], f"{fallback_role} theme {slot}")) + return list(dict.fromkeys(candidates)) master_name, theme_fonts = theme_faces_for_slide(prs7.slides[0]) @@ -475,19 +701,11 @@ def stub_slide(name, major, minor, **theme_options): _, script_fonts = theme_faces_for_slide(stub_slide( "/ppt/slideMasters/scripts.xml", "Latin Theme", "Latin Body", east_asian="East Asian Theme", complex_script="Complex Script Theme", - scripts={"Hans": "Simplified Chinese Theme", "Cyrl": "Cyrillic Theme"}, + scripts={ + "Hans": "Simplified Chinese Theme", "Cyrl": "Cyrillic Theme", + "Thai": "Thai Theme", + }, )) -check( - "CJK inherited-font triage includes east-Asian and script-specific faces", - {"East Asian Theme", "Simplified Chinese Theme"} - <= set(theme_candidates(script_fonts["major"], "汉字")), - theme_candidates(script_fonts["major"], "汉字"), -) -check( - "non-Latin inherited-font triage uses script mappings", - "Cyrillic Theme" in theme_candidates(script_fonts["minor"], "текст"), - theme_candidates(script_fonts["minor"], "текст"), -) check( "unstyled runs report as inherited, not as a concrete face", @@ -502,78 +720,57 @@ def stub_slide(name, major, minor, **theme_options): explicit_run = font_paragraph.add_run() explicit_run.text = "run override" explicit_run.font.name = "Run Face" -detected_faces = [] -for run in font_paragraph.runs: - if run.font.name: - detected_faces.append((run.font.name, "run")) - elif font_paragraph.font.name: - detected_faces.append((font_paragraph.font.name, "paragraph defaults")) -check("font triage reports the run face and source", ("Run Face", "run") in detected_faces, detected_faces) +cjk_theme_run = font_paragraph.add_run() +cjk_theme_run.text = "汉字" +thai_theme_run = font_paragraph.add_run() +thai_theme_run.text = "ไทย" +mixed_run = font_paragraph.add_run() +mixed_run.text = "A汉ก" +mixed_run.font.name = "Latin Explicit" +for tag, face in (("a:ea", "East Explicit"), ("a:cs", "Complex Explicit")): + node = OxmlElement(tag) + node.set("typeface", face) + mixed_run._r.get_or_add_rPr().append(node) +token_run = font_paragraph.add_run() +token_run.text = "汉" +token_ea = OxmlElement("a:ea") +token_ea.set("typeface", "+mj-ea") +token_run._r.get_or_add_rPr().append(token_ea) + +token_fallback_fonts = { + "major": {"latin": "Major Latin", "eastAsia": "", "complexScript": "", + "scripts": {"Hans": "Major Hans"}}, + "minor": {"latin": "Minor Latin", "eastAsia": "Minor East", "complexScript": "", + "scripts": {"Hans": "Minor Hans"}}, +} + +detected_faces = { + run.text: font_candidates(run, font_paragraph, script_fonts, "minor") + for run in font_paragraph.runs +} +check("font triage reports the run face and source", + ("latin", "Run Face", "run") in detected_faces["run override"], detected_faces) check( "font triage reports the paragraph face and source", - ("Paragraph Face", "paragraph defaults") in detected_faces, + ("latin", "Paragraph Face", "paragraph defaults") + in detected_faces["paragraph default"], detected_faces, ) - -# ---- analyze.md bounded ZIP/XML health check ---------------------------------- -MAX_XML_PART = 20 * 1024 * 1024 -MAX_ENTRY = 100 * 1024 * 1024 -MAX_TOTAL_UNCOMPRESSED = 500 * 1024 * 1024 -MAX_COMPRESSION_RATIO = 200 -safe_xml_parser = etree.XMLParser( - load_dtd=False, - resolve_entities=False, - no_network=True, - huge_tree=False, - recover=False, +check( + "CJK inherited-font triage includes east-Asian and script-specific faces", + {"East Asian Theme", "Simplified Chinese Theme"} + <= {face for _, face, _ in detected_faces["汉字"]}, detected_faces["汉字"], ) - - -def validate_pptx_package(path): - with zipfile.ZipFile(path) as archive: - infos = archive.infolist() - names = {info.filename for info in infos} - assert len(names) == len(infos) - assert "[Content_Types].xml" in names and "ppt/presentation.xml" in names - assert sum(info.file_size for info in infos) <= MAX_TOTAL_UNCOMPRESSED - actual_total = 0 - for info in infos: - assert info.file_size <= MAX_ENTRY - assert info.file_size / max(info.compress_size, 1) <= MAX_COMPRESSION_RATIO - is_xml = info.filename.endswith((".xml", ".rels")) - if is_xml: - assert info.file_size <= MAX_XML_PART - chunks = [] - actual_size = 0 - with archive.open(info) as stream: - while chunk := stream.read(64 * 1024): - actual_size += len(chunk) - actual_total += len(chunk) - assert actual_size <= MAX_ENTRY - assert actual_total <= MAX_TOTAL_UNCOMPRESSED - if is_xml: - chunks.append(chunk) - assert actual_size == info.file_size - if is_xml: - etree.fromstring(b"".join(chunks), parser=safe_xml_parser) - - -try: - validate_pptx_package("input.pptx") - healthy_package_passed = True -except Exception: - healthy_package_passed = False -check("bounded package health check accepts an ordinary PPTX", healthy_package_passed) - -with zipfile.ZipFile("compressed-bomb.pptx", "w", zipfile.ZIP_DEFLATED) as archive: - archive.writestr("[Content_Types].xml", "") - archive.writestr("ppt/presentation.xml", "" + (" " * 2_000_000) + "") -try: - validate_pptx_package("compressed-bomb.pptx") - archive_bomb_rejected = False -except (AssertionError, ValueError): - archive_bomb_rejected = True -check("PPTX compression bomb is rejected before XML expansion", archive_bomb_rejected) +check("Thai inherited-font triage uses its script mapping", + "Thai Theme" in {face for _, face, _ in detected_faces["ไทย"]}, detected_faces["ไทย"]) +check("explicit Latin/East-Asian/complex-script slots resolve independently", + {(slot, face) for slot, face, _ in detected_faces["A汉ก"]} + == {("latin", "Latin Explicit"), ("eastAsia", "East Explicit"), + ("complexScript", "Complex Explicit")}, detected_faces["A汉ก"]) +check("empty +mj-ea generic face falls back to the major script mapping", + ("eastAsia", "Major Hans", "major theme script Hans") + in font_candidates(token_run, font_paragraph, token_fallback_fonts, "minor"), + font_candidates(token_run, font_paragraph, token_fallback_fonts, "minor")) # ---- analyze.md snippet: sparse cache points keep their idx ---------------------- diff --git a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py index 98cf727..948fb82 100644 --- a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py @@ -219,25 +219,41 @@ def sheet_ref(sheet): # ---- edit.md structural audit includes non-cell dependencies ------------------ from openpyxl.chart import BarChart, Reference from openpyxl.formula import Tokenizer -from openpyxl.formatting.rule import CellIsRule, FormulaRule +from openpyxl.formatting.rule import CellIsRule, ColorScaleRule, DataBarRule, FormulaRule from openpyxl.utils.cell import range_boundaries from openpyxl.workbook.defined_name import DefinedName from openpyxl.worksheet.datavalidation import DataValidation from openpyxl.worksheet.table import Table -def non_cell_references(workbook): - refs = [] - defined_names = workbook.defined_names - defined_name_items = ( - defined_names.values() - if hasattr(defined_names, "values") - else defined_names.definedName +def formula_text(value): + if isinstance(value, str): + return value + if text := getattr(value, "text", None): + return text + fields = ("ref", "r1", "r2", "dt2D", "dtr", "ca", "del1", "del2") + details = ", ".join( + f"{name}={getattr(value, name)!r}" for name in fields if hasattr(value, name) ) - for item in defined_name_items: + return f"{type(value).__name__}({details})" + + +def defined_name_values(workbook): + names = workbook.defined_names + return names.values() if hasattr(names, "values") else names.definedName + + +def structural_references(workbook): + refs = [] + for item in defined_name_values(workbook): refs.append(("defined name", item.name, item.attr_text)) for sheet in workbook.worksheets: owner = sheet.title + for row in sheet.iter_rows(): + for cell in row: + if cell.data_type == "f": + refs.append(("cell formula", f"{owner}!{cell.coordinate}", + formula_text(cell.value))) for table in sheet.tables.values(): refs.append(("table", owner + "!" + table.name, table.ref)) for merged_range in sheet.merged_cells.ranges: @@ -279,7 +295,7 @@ def cell_formula_references(workbook): "cell formula", sheet.title, cell.coordinate, - getattr(value, "text", None) or str(value), + formula_text(value), )) return refs @@ -307,7 +323,6 @@ def formula_may_intersect_rows(owner_sheet, formula, shifted_sheet, start_row): return True return False - class LegacyDefinedNames: """Minimal openpyxl 3.0-style DefinedNameList surface.""" definedName = [DefinedName("LegacyRange", attr_text="'Legacy'!$A$1")] @@ -320,7 +335,7 @@ class LegacyWorkbook: check( "structural audit supports openpyxl 3.0 DefinedNameList", - non_cell_references(LegacyWorkbook()) + structural_references(LegacyWorkbook()) == [("defined name", "LegacyRange", "'Legacy'!$A$1")], ) @@ -346,11 +361,12 @@ class LegacyWorkbook: chart = BarChart() chart.add_data(Reference(audit_ws, min_col=1, min_row=1, max_row=3), titles_from_data=True) audit_ws.add_chart(chart, "C1") -reference_kinds = {kind for kind, _, _ in non_cell_references(audit_wb)} +audit_references = structural_references(audit_wb) +reference_kinds = {kind for kind, _, _ in audit_references} formula_references = cell_formula_references(audit_wb) check( "structural audit covers names, tables, filters, validation, formatting, and charts", - {"defined name", "table", "merged range", "auto filter", "print area", + {"defined name", "cell formula", "table", "merged range", "auto filter", "print area", "print title rows", "print title columns", "data validation range", "data validation formula", "conditional formatting range", "conditional formatting formula", "chart series"} <= reference_kinds, @@ -396,6 +412,18 @@ class LegacyWorkbook: check("audited non-intersecting formula path reaches save", safe_reopened["Data"]["D2"].value == "=C2*1.08") +legacy_name = DefinedName("LegacyName", attr_text="Audit!$A$1") +legacy_names = type("LegacyDefinedNames", (), {"definedName": [legacy_name]})() +legacy_workbook = type("LegacyWorkbook", (), {"defined_names": legacy_names})() +check("defined-name adapter supports openpyxl 3.0 collections", + list(defined_name_values(legacy_workbook)) == [legacy_name]) + +formula_only = openpyxl.Workbook() +formula_only.active["A1"] = "=Data!A5" +check("structural guard sees formulas before row insertion", + any(reference[0] == "cell formula" + for reference in structural_references(formula_only))) + # and the edit itself still works after the warning path wb2 = openpyxl.load_workbook("plain.xlsx") wb2["Data"]["B2"] = "=B2*1" # formula stays a formula @@ -412,7 +440,7 @@ class LegacyWorkbook: check("task-specific number format mapping is verified", format_matches) # ---- read.md snippet: multi-sheet profiles cover every sheet ---------------------- -from openpyxl.worksheet.formula import ArrayFormula +from openpyxl.worksheet.formula import ArrayFormula, DataTableFormula wb_h = openpyxl.Workbook() wb_h.active.title = "First" @@ -421,11 +449,12 @@ class LegacyWorkbook: second = wb_h.create_sheet("Second") second["A1"] = "plain" second["A2"] = "=2+2" +second["B1"] = DataTableFormula(ref="B1:B2", r1="C1") wb_h.save("multi.xlsx") formula_wb = openpyxl.load_workbook("multi.xlsx", read_only=True, data_only=False) value_wb = openpyxl.load_workbook("multi.xlsx", read_only=True, data_only=True) profiled = list(value_wb.sheetnames) -uncached = {(sn, fc.coordinate, getattr(fc.value, "text", None) or str(fc.value)) +uncached = {(sn, fc.coordinate, formula_text(fc.value)) for sn in profiled for frow, vrow in zip(formula_wb[sn].iter_rows(), value_wb[sn].iter_rows()) for fc, vc in zip(frow, vrow) @@ -433,6 +462,10 @@ class LegacyWorkbook: check("multi-sheet profile iterates every sheet", profiled == ["First", "Second"], profiled) check("uncached formulas found on both sheets", {item[0] for item in uncached} == {"First", "Second"}, uncached) check("array-formula objects are detected by data_type", ("First", "A2", "=ROW(A2:A3)") in uncached, uncached) +data_table_entry = next(item for item in uncached if item[:2] == ("Second", "B1")) +check("data-table formulas have stable diagnostic text", + data_table_entry[2].startswith("DataTableFormula(ref='B1:B2', r1='C1'") + and "0x" not in data_table_entry[2], data_table_entry) # csv.md: value export uses the cached-value workbook and reports every missing cache. missing_caches = [] @@ -505,12 +538,53 @@ def markers_in(data): not any(marker in custom_prefix_sheet for marker in legacy_prefix_markers)) # ---- formatting.md guard: header-only sheets skip conditional formatting --------- +def add_demo_formatting(sheet): + last = sheet.max_row + if last < 2: + return 0 + sheet.conditional_formatting.add( + f"D2:D{last}", CellIsRule(operator="lessThan", formula=["0"]), + ) + sheet.conditional_formatting.add( + f"A2:F{last}", FormulaRule(formula=["$D2<0"]), + ) + sheet.conditional_formatting.add( + f"C2:C{last}", + ColorScaleRule(start_type="min", start_color="FFFFFF", + end_type="max", end_color="63BE7B"), + ) + sheet.conditional_formatting.add( + f"E2:E{last}", DataBarRule(start_type="min", end_type="max", color="638EC6"), + ) + return 4 + + header_wb = openpyxl.Workbook() header_ws = header_wb.active -header_ws.append(["Qty", "Note"]) -guard_last = header_ws.max_row -if guard_last < 2: - pass # guarded route: skip building rules +header_ws.append(["A", "B", "C", "D", "E", "F"]) +try: + header_ws.conditional_formatting.add( + "D2:D1", CellIsRule(operator="lessThan", formula=["0"]), + ) + inverted_range_rejected = False +except (TypeError, ValueError): + inverted_range_rejected = True +check("unguarded header-only range is rejected (negative control)", inverted_range_rejected) +check("header-only guard skips all four formatting rules", add_demo_formatting(header_ws) == 0) +header_wb.save("header-only-formatting.xlsx") +header_reopened = openpyxl.load_workbook("header-only-formatting.xlsx") +check("header-only workbook saves and reopens with no conditional formatting", + len(header_reopened.active.conditional_formatting) == 0) + +data_wb = openpyxl.Workbook() +data_ws = data_wb.active +data_ws.append(["A", "B", "C", "D", "E", "F"]) +data_ws.append([1, 2, 3, -1, 5, 6]) +check("data rows receive all four formatting rules", add_demo_formatting(data_ws) == 4) +data_wb.save("data-formatting.xlsx") +data_reopened = openpyxl.load_workbook("data-formatting.xlsx") +check("all four formatting rules survive save/reopen", + len(data_reopened.active.conditional_formatting) == 4) # ---- read.md: implausible is reset before streaming ------------------ From 53d3aac41cdf7020dc18cf711cdd534454e0eb63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=98=E6=89=AC?= Date: Sun, 16 Aug 2026 17:46:16 +0800 Subject: [PATCH 16/34] fix(document-skills): preserve review edge cases --- .../skills/pptx/references/analyze.md | 23 ++++--- .../skills/xlsx/references/read.md | 7 ++- .../document-skills/tests/pptx_fixture.py | 63 +++++++++++++------ .../document-skills/tests/xlsx_fixture.py | 16 +++-- 4 files changed, 75 insertions(+), 34 deletions(-) diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md index e19dcaa..20a7c74 100644 --- a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md @@ -15,6 +15,21 @@ def iter_shapes(shapes): else: yield shape +def table_cells(table): + """Inventory the grid without repeating a merged cell's text in covered slots.""" + return [[ + { + "row": row_index, + "column": column_index, + "text": None if cell.is_spanned else cell.text, + "is_merge_origin": cell.is_merge_origin, + "is_spanned": cell.is_spanned, + "span_width": cell.span_width, + "span_height": cell.span_height, + } + for column_index, cell in enumerate(row.cells) + ] for row_index, row in enumerate(table.rows)] + def cached_numeric_points(source): """Return indexed cached points, or None when the cache metadata is unavailable.""" if source is None: @@ -55,13 +70,7 @@ for i, slide in enumerate(prs.slides): shapes = list(iter_shapes(slide.shapes)) # flattened; groups are common in template decks text = [sh.text_frame.text for sh in shapes if sh.has_text_frame and sh.text_frame.text] tables = [ - [[{ - "text": cell.text, - "is_merge_origin": cell.is_merge_origin, - "is_spanned": cell.is_spanned, - "span_width": cell.span_width, - "span_height": cell.span_height, - } for cell in row.cells] for row in sh.table.rows] + table_cells(sh.table) for sh in shapes if sh.has_table ] charts = [] diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md index d8267bc..3f3d68b 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md @@ -31,10 +31,11 @@ for sheet_name in value_wb.sheetnames: if value_ws.max_row in (None, 0) or declared in ("A1:A1", "A1"): value_ws.reset_dimensions() formula_ws.reset_dimensions() + discovered = value_ws.calculate_dimension(force=True) print(f"--- {sheet_name} --- implausible dimension {declared!r}; reset, real extent:") - # reset_dimensions() makes a read-only worksheet unsized; force=True scans - # the stream to rebuild its bounds instead of raising "Worksheet is unsized". - print(f"--- {sheet_name} --- dims:", value_ws.calculate_dimension(force=True)) + else: + discovered = declared + print(f"--- {sheet_name} --- dims:", discovered) rows = value_ws.iter_rows(values_only=True) header = next(rows, None) diff --git a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py index e616a4a..3fac806 100644 --- a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py @@ -236,6 +236,21 @@ def iter_shapes(shapes): yield shape +def table_cells(table): + return [[ + { + "row": row_index, + "column": column_index, + "text": None if cell.is_spanned else cell.text, + "is_merge_origin": cell.is_merge_origin, + "is_spanned": cell.is_spanned, + "span_width": cell.span_width, + "span_height": cell.span_height, + } + for column_index, cell in enumerate(row.cells) + ] for row_index, row in enumerate(table.rows)] + + def cached_numeric_points(source): if source is None: return None @@ -272,13 +287,7 @@ def extract_slide_content(slide): shapes = list(iter_shapes(slide.shapes)) text = [sh.text_frame.text for sh in shapes if sh.has_text_frame and sh.text_frame.text] tables = [ - [[{ - "text": cell.text, - "is_merge_origin": cell.is_merge_origin, - "is_spanned": cell.is_spanned, - "span_width": cell.span_width, - "span_height": cell.span_height, - } for cell in row.cells] for row in sh.table.rows] + table_cells(sh.table) for sh in shapes if sh.has_table ] charts = [] @@ -329,19 +338,35 @@ def extract_slide_content(slide): == [(0, 5.0)]) check("content inventory emits notes text", "regional split" in content["notes"], content["notes"]) -merge_prs = Presentation() -merge_slide = merge_prs.slides.add_slide(merge_prs.slide_layouts[6]) -merge_shape = merge_slide.shapes.add_table(2, 3, 0, 0, 4000000, 2000000) -merge_origin = merge_shape.table.cell(0, 0) -merge_origin.text = "Merged heading" -merge_origin.merge(merge_shape.table.cell(0, 1)) -merged_inventory = extract_slide_content(merge_slide)["tables"][0][0] +merged_prs = Presentation() +merged_slide = merged_prs.slides.add_slide(merged_prs.slide_layouts[6]) +merged_table = merged_slide.shapes.add_table( + 3, 3, Inches(1), Inches(1), Inches(6), Inches(3) +).table +merged_table.cell(0, 0).text = "Merged heading" +merged_table.cell(0, 0).merge(merged_table.cell(1, 1)) +merged_table.cell(2, 0).text = "ordinary cell" +merged_prs.save("merged-table.pptx") +merged_inventory = extract_slide_content( + Presentation("merged-table.pptx").slides[0] +)["tables"][0] +merged_origin = merged_inventory[0][0] +covered_slots = [merged_inventory[0][1], merged_inventory[1][0], merged_inventory[1][1]] +check( + "table inventory records a merged origin and its row/column spans", + merged_origin["is_merge_origin"] + and not merged_origin["is_spanned"] + and merged_origin["span_height"] == 2 + and merged_origin["span_width"] == 2, + merged_origin, +) check( - "table inventory preserves merge origin and span metadata", - merged_inventory[0]["is_merge_origin"] - and merged_inventory[0]["span_width"] == 2 - and merged_inventory[0]["span_height"] == 1 - and merged_inventory[1]["is_spanned"], + "table inventory marks covered slots without repeating merged text", + all(item["is_spanned"] and item["text"] is None for item in covered_slots) + and sum( + item["text"] == "Merged heading" + for row in merged_inventory for item in row + ) == 1, merged_inventory, ) diff --git a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py index 948fb82..6da3d62 100644 --- a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py @@ -421,8 +421,9 @@ class LegacyWorkbook: formula_only = openpyxl.Workbook() formula_only.active["A1"] = "=Data!A5" check("structural guard sees formulas before row insertion", - any(reference[0] == "cell formula" - for reference in structural_references(formula_only))) + structural_references(formula_only) + == [("cell formula", "Sheet!A1", "=Data!A5")], + structural_references(formula_only)) # and the edit itself still works after the warning path wb2 = openpyxl.load_workbook("plain.xlsx") @@ -611,11 +612,16 @@ def add_demo_formatting(sheet): dim_ws_ro.max_row == 1, dim_ws_ro.max_row) streamed_before_reset = [row for row in dim_ws_ro.iter_rows(values_only=True)] dim_ws_ro.reset_dimensions() -forced_extent = dim_ws_ro.calculate_dimension(force=True) +try: + dim_ws_ro.calculate_dimension() + unforced_dimension_rejected = False +except ValueError: + unforced_dimension_rejected = True +forced_dimension = dim_ws_ro.calculate_dimension(force=True) streamed_after_reset = [row for row in dim_ws_ro.iter_rows(values_only=True)] dim_value.close() -check("forced dimension calculation sizes a reset read-only sheet", - forced_extent == "A1:B3", forced_extent) +check("reset dimensions require an explicit force scan before reporting extent", + unforced_dimension_rejected and forced_dimension == "A1:B3", forced_dimension) check("reset_dimensions restores the real extent", len(streamed_after_reset) == 3 and streamed_after_reset[2] == (3, 4), (len(streamed_before_reset), streamed_after_reset[-1])) From f66fc8215709dcc1e2f5470cd1967d7c52eda0fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=98=E6=89=AC?= Date: Sun, 16 Aug 2026 19:39:51 +0800 Subject: [PATCH 17/34] fix(document-skills): address latest review edge cases --- .../skills/pdf/references/transform.md | 28 +++- .../document-skills/skills/pptx/SKILL.md | 34 ++++- .../skills/pptx/references/edit.md | 18 ++- .../skills/xlsx/references/edit.md | 4 +- .../skills/xlsx/references/read.md | 45 +++++-- .../document-skills/tests/docx_fixture.py | 18 +++ .../document-skills/tests/pdf_fixture.py | 127 +++++++++++++++++- .../document-skills/tests/pptx_fixture.py | 96 ++++++++++++- .../document-skills/tests/xlsx_fixture.py | 96 +++++++++++-- 9 files changed, 417 insertions(+), 49 deletions(-) diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md b/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md index 7804888..597c80c 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md @@ -100,12 +100,28 @@ Encryption and forms: writer = PdfWriter() writer.append(reader) # clones every page AND the catalog /AcroForm into the writer - # update fields on the writer's page copies; a field widget can sit on any page, - # so pass the writer page that actually carries the field you are filling - writer.update_page_form_field_values( - writer.pages[0], - {"applicant_name": "Ada Byron"}, - ) + def widget_field_name(widget): + """Resolve /T on a widget or its parent field dictionary.""" + while widget is not None: + if widget.get("/T") is not None: + return str(widget["/T"]) + parent = widget.get("/Parent") + widget = None if parent is None else parent.get_object() + return None + + field_name = "applicant_name" + target_pages = [ + page for page in writer.pages + if any( + (widget := ref.get_object()).get("/Subtype") == "/Widget" + and widget_field_name(widget) == field_name + for ref in (page.get("/Annots") or []) + ) + ] + if not target_pages: + raise KeyError(f"no widget page found for field {field_name!r}") + for page in target_pages: + writer.update_page_form_field_values(page, {field_name: "Ada Byron"}) with open("filled.pdf", "wb") as f: writer.write(f) diff --git a/plugins/Hylouis233/document-skills/skills/pptx/SKILL.md b/plugins/Hylouis233/document-skills/skills/pptx/SKILL.md index 6432d81..d20cb09 100644 --- a/plugins/Hylouis233/document-skills/skills/pptx/SKILL.md +++ b/plugins/Hylouis233/document-skills/skills/pptx/SKILL.md @@ -45,11 +45,41 @@ python -c "import pptx; print('python-pptx ok')" ```python from pptx import Presentation +from pptx.enum.shapes import MSO_SHAPE_TYPE + +def iter_text_frames(shapes, path=""): + """Yield every logical text frame, including group members and table cells.""" + for shape in shapes: + here = f"{path}/{shape.name}" if path else shape.name + if shape.shape_type == MSO_SHAPE_TYPE.GROUP: + yield from iter_text_frames(shape.shapes, here) + continue + if shape.has_text_frame: + yield here, shape.text_frame + if shape.has_table: + for row_index, row in enumerate(shape.table.rows): + for column_index, cell in enumerate(row.cells): + if cell.is_spanned: # a covered merge slot is intentionally empty + continue + yield f"{here}/table[{row_index},{column_index}]", cell.text_frame + prs = Presentation("output.pptx") print("slides:", len(prs.slides)) for i, slide in enumerate(prs.slides): - texts = [sh.text_frame.text for sh in slide.shapes if sh.has_text_frame] - print(i, len(slide.shapes), texts[:3]) + inventory = [(location, frame.text) for location, frame in iter_text_frames(slide.shapes)] + print(i, len(slide.shapes), inventory) + +# Populate this only with locations intentionally left blank by the task. +intentionally_empty_locations = set() +unexpected_empty_locations = [ + (slide_index, location) + for slide_index, slide in enumerate(prs.slides) + for location, frame in iter_text_frames(slide.shapes) + if not frame.text.strip() + and (slide_index, location) not in intentionally_empty_locations +] +if unexpected_empty_locations: + raise ValueError(f"unexpected empty text frames: {unexpected_empty_locations}") ``` Confirm: slide count matches the outline; every slide has its intended title text; no text diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md b/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md index 56f9f4a..c6fcf74 100644 --- a/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md @@ -13,6 +13,10 @@ slide_index = None # Set this and shape_name when repeated text shape_name = None target_location = None # e.g. "Table 1/table[0,1]" for duplicate table text +def require(condition, message): + if not condition: + raise ValueError(message) + def iter_shapes(shapes, path=""): """Yield (path, shape) for every shape, recursing into groups so text inside grouped artwork is reachable; the path keeps the uniqueness check readable.""" @@ -45,20 +49,20 @@ for i, slide in enumerate(prs.slides): candidates.append((i, location, text_frame)) locations = [(i, location) for i, location, _ in candidates] -assert len(candidates) == 1, f"expected one matching text target, found {locations}" +require(len(candidates) == 1, f"expected one matching text target, found {locations}") _, _, tf = candidates[0] # Replace inside one existing run so its formatting and hyperlink are retained. -assert tf.text.count(old) == 1, "target occurs more than once in the selected shape" +require(tf.text.count(old) == 1, "target occurs more than once in the selected shape") run_hits = [ run for paragraph in tf.paragraphs for run in paragraph.runs if old in run.text ] -assert len(run_hits) == 1 and run_hits[0].text.count(old) == 1, ( +require(len(run_hits) == 1 and run_hits[0].text.count(old) == 1, ( "target is duplicated or split across runs; report it instead of flattening the paragraph" -) +)) run_hits[0].text = run_hits[0].text.replace(old, new, 1) prs.save("input-edited.pptx") @@ -68,7 +72,8 @@ prs.save("input-edited.pptx") 1. **Never rebuild the file to make a small change.** Rewriting slides from scratch loses the template, masters, notes, and animations. Edit in place, save to a new path. -2. Address shapes by slide index + shape name or matched text, and **assert exactly one match**. +2. Address shapes by slide index + shape name or matched text, and **require exactly one match** + with an explicit exception (never a Python `assert`, which `python -O` removes). The locator must search both shape text frames and every table cell, retaining a stable `/table[row,column]` suffix. If copy repeats inside one table, set `target_location` as well as the slide/shape selectors rather than choosing one. @@ -83,7 +88,8 @@ prs.save("input-edited.pptx") ```python cell = table.cell(2, 1) hits = [run for p in cell.text_frame.paragraphs for run in p.runs if old in run.text] - assert len(hits) == 1, "target is duplicated or split across runs in this cell" + if len(hits) != 1 or hits[0].text.count(old) != 1: + raise ValueError("target is duplicated or split across runs in this cell") hits[0].text = hits[0].text.replace(old, new, 1) ``` 5. Chart data: `chart.replace_data(CategoryChartData(...))` updates the embedded workbook and diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md index 3f9dabc..d058b92 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md @@ -101,7 +101,9 @@ def formula_may_intersect_rows(owner_sheet, formula, shifted_sheet, start_row): if "[" in qualifier or ":" in qualifier: # external or 3-D reference return True target_sheet = qualifier.strip("'").replace("''", "'") - if target_sheet != shifted_sheet: + # Excel worksheet names are case-insensitive even though their spelling + # is preserved in formulas and workbook metadata. + if target_sheet.casefold() != shifted_sheet.casefold(): continue try: _, min_row, _, max_row = range_boundaries(reference.replace("$", "")) diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md index 3f3d68b..b130433 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md @@ -2,6 +2,7 @@ ```python import openpyxl +from openpyxl.utils import get_column_letter formula_wb = openpyxl.load_workbook("input.xlsx", read_only=True, data_only=False) value_wb = openpyxl.load_workbook("input.xlsx", read_only=True, data_only=True) @@ -19,22 +20,44 @@ def formula_text(value): ) return f"{type(value).__name__}({details})" +def discover_dimension(worksheet): + """Scan an untrusted read-only stream without relying on its .""" + worksheet.reset_dimensions() + min_row = min_column = max_row = max_column = None + for row in worksheet.iter_rows(): + for cell in row: + row_index = getattr(cell, "row", None) # EmptyCell has no coordinates + column_index = getattr(cell, "column", None) + if row_index is None or column_index is None: + continue + min_row = row_index if min_row is None else min(min_row, row_index) + min_column = column_index if min_column is None else min(min_column, column_index) + max_row = row_index if max_row is None else max(max_row, row_index) + max_column = column_index if max_column is None else max(max_column, column_index) + if max_row is None: + return "A1:A1" + return ( + f"{get_column_letter(min_column)}{min_row}:" + f"{get_column_letter(max_column)}{max_row}" + ) + # Profile EVERY sheet by default; only narrow when the task names a specific sheet. for sheet_name in value_wb.sheetnames: formula_ws = formula_wb[sheet_name] value_ws = value_wb[sheet_name] - # Read-only iteration is bounded by the sheet's metadata. - # Non-Excel producers write wrong dimensions, which silently truncates the - # stream; when the declared extent looks implausible, reset it and let - # openpyxl discover the real used range. + # Read-only iteration is bounded by the sheet's metadata. A + # non-Excel producer can declare a plausible but truncated range (for + # example A1:B2 while data continues below it), so treat that metadata as + # untrusted: reset both streams and discover the real bounds before reading. declared = value_ws.calculate_dimension() - if value_ws.max_row in (None, 0) or declared in ("A1:A1", "A1"): - value_ws.reset_dimensions() - formula_ws.reset_dimensions() - discovered = value_ws.calculate_dimension(force=True) - print(f"--- {sheet_name} --- implausible dimension {declared!r}; reset, real extent:") - else: - discovered = declared + discovered = discover_dimension(value_ws) + formula_discovered = discover_dimension(formula_ws) + if formula_discovered != discovered: + raise ValueError( + f"formula/value stream dimensions disagree: {formula_discovered} != {discovered}" + ) + if discovered != declared: + print(f"--- {sheet_name} --- declared {declared!r}; discovered real extent:") print(f"--- {sheet_name} --- dims:", discovered) rows = value_ws.iter_rows(values_only=True) diff --git a/plugins/Hylouis233/document-skills/tests/docx_fixture.py b/plugins/Hylouis233/document-skills/tests/docx_fixture.py index 320ddd1..21927e1 100644 --- a/plugins/Hylouis233/document-skills/tests/docx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/docx_fixture.py @@ -287,6 +287,24 @@ def effective_face(run, slot): check("inline content-control runs are included in glyph traversal", any(run.text == "inline한" for run in iter_paragraph_runs(inline_paragraph_reopened))) +hyperlink_paragraph = font_doc.add_paragraph() +hyperlink = OxmlElement("w:hyperlink") +hyperlink.set(qn("w:anchor"), "fixture-target") +hyperlink_run = OxmlElement("w:r") +hyperlink_text = OxmlElement("w:t") +hyperlink_text.text = "链接漢" +hyperlink_run.append(hyperlink_text) +hyperlink.append(hyperlink_run) +hyperlink_paragraph._p.append(hyperlink) +walked_hyperlink_runs = list(iter_paragraph_runs(hyperlink_paragraph)) +check("Paragraph.runs omits hyperlink runs (negative control)", + all(run.text != "链接漢" for run in hyperlink_paragraph.runs)) +check("glyph traversal includes CJK text nested in a hyperlink", + [run.text for run in walked_hyperlink_runs] == ["链接漢"], + [run.text for run in walked_hyperlink_runs]) +check("hyperlink CJK text resolves through the east-Asian font slot", + effective_face(walked_hyperlink_runs[0], "eastAsia") == "CJK Face") + # ---- edit.md guarded cross-run replacement ------------------------------------ SAFE_RUN_CHILDREN = { qn("w:rPr"), qn("w:t"), qn("w:tab"), qn("w:cr"), diff --git a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py index 58043af..70db7aa 100644 --- a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py @@ -159,6 +159,67 @@ def widget_count(page): value = str((check_r.get_fields() or {}).get("applicant_name", {}).get("/V", "")) check("field value round-trips", value.strip("/") == "Ada Byron", repr(value)) +# A widget can live on any page; locate its annotation instead of assuming page 1. +page2_form = canvas.Canvas("form-page2.pdf", pagesize=A4) +page2_form.drawString(72, 780, "Cover page") +page2_form.showPage() +page2_form.drawString(72, 780, "Form page") +page2_form.acroForm.textfield( + name="applicant_name", x=72, y=740, width=260, height=20, borderWidth=0, +) +page2_form.showPage() +page2_form.save() + +wrong_page_writer = PdfWriter() +wrong_page_writer.append(PdfReader("form-page2.pdf")) +wrong_page_writer.update_page_form_field_values( + wrong_page_writer.pages[0], {"applicant_name": "Wrong page"}, +) +with open("form-page2-wrong.pdf", "wb") as f: + wrong_page_writer.write(f) +wrong_value = str( + (PdfReader("form-page2-wrong.pdf").get_fields() or {}) + .get("applicant_name", {}).get("/V", "") +) +check("hard-coded first-page form fill misses a page-2 widget (negative control)", + wrong_value.strip("/") != "Wrong page", repr(wrong_value)) + + +def widget_field_name(widget): + while widget is not None: + if widget.get("/T") is not None: + return str(widget["/T"]) + parent = widget.get("/Parent") + widget = None if parent is None else parent.get_object() + return None + + +page2_writer = PdfWriter() +page2_writer.append(PdfReader("form-page2.pdf")) +field_name = "applicant_name" +target_pages = [ + page for page in page2_writer.pages + if any( + (widget := ref.get_object()).get("/Subtype") == "/Widget" + and widget_field_name(widget) == field_name + for ref in (page.get("/Annots") or []) + ) +] +for target_page in target_pages: + page2_writer.update_page_form_field_values( + target_page, {field_name: "Ada on page 2"}, + ) +with open("form-page2-filled.pdf", "wb") as f: + page2_writer.write(f) +page2_value = str( + (PdfReader("form-page2-filled.pdf").get_fields() or {}) + .get(field_name, {}).get("/V", "") +) +check("form fill locates the widget page before updating", + len(target_pages) == 1 and target_pages[0] is page2_writer.pages[1], len(target_pages)) +check("page-2 field value round-trips", page2_value.strip("/") == "Ada on page 2", + repr(page2_value)) + # ---- transform.md merge imports outline navigation ---------------------------- appendix_writer = PdfWriter() appendix_writer.add_blank_page(width=200, height=300) @@ -351,6 +412,9 @@ def widget_count(page): offset_page.cropbox.upper_right = (300, 500) rotated_page = small_source.add_blank_page(width=240, height=160) rotated_page.rotate(90) +cropped_page = small_source.add_blank_page(width=400, height=500) +cropped_page.cropbox.lower_left = (250, 300) +cropped_page.cropbox.upper_right = (390, 480) mixed_writer.append(small_source) with open("mixed.pdf", "wb") as f: mixed_writer.write(f) @@ -383,6 +447,28 @@ def stamp_bboxes(path, page_number): plain_spans == [] and "DRAFT" in (R2("plain-stamped.pdf").pages[2].extract_text() or ""), plain_spans) +# Negative control for crop-box fitting: media-box centering puts the stamp +# outside this page's small, offset visible region. +media_fit_writer = PdfWriter() +media_fit_writer.append(open_pdf("mixed.pdf"), pages=(5, 6)) +media_fit_page = media_fit_writer.pages[0] +media_destination = media_fit_page.mediabox +media_scale = min(float(media_destination.width) / float(stamp_box.width), + float(media_destination.height) / float(stamp_box.height)) +media_tx = (float(media_destination.left) + + (float(media_destination.width) - float(stamp_box.width) * media_scale) / 2 + - float(stamp_box.left) * media_scale) +media_ty = (float(media_destination.bottom) + + (float(media_destination.height) - float(stamp_box.height) * media_scale) / 2 + - float(stamp_box.bottom) * media_scale) +media_fit_page.merge_transformed_page( + stamp, Transformation().scale(media_scale).translate(media_tx, media_ty), +) +with open("media-fit-cropped.pdf", "wb") as f: + media_fit_writer.write(f) +check("media-box fitting misses an offset crop region (negative control)", + stamp_bboxes("media-fit-cropped.pdf", 0) == []) + scaled_writer = PdfWriter() scaled_writer.append(open_pdf("mixed.pdf")) stamp_page = R2("stamp.pdf").pages[0] @@ -408,12 +494,22 @@ def stamp_bboxes(path, page_number): scaled_spans) offset_spans = stamp_bboxes("scaled-stamped.pdf", 3) rotated_spans = stamp_bboxes("scaled-stamped.pdf", 4) +cropped_spans = stamp_bboxes("scaled-stamped.pdf", 5) check("scaled stamp lands inside the non-zero-origin page", bool(offset_spans), offset_spans) check("scaled stamp lands inside the normalized 90-degree page", bool(rotated_spans), rotated_spans) +cropped_rect = fitz.open("scaled-stamped.pdf")[5].rect +check("scaled stamp lands inside the offset visible crop box", + bool(cropped_spans) + and all( + bbox[0] >= -0.5 and bbox[1] >= -0.5 + and bbox[2] <= cropped_rect.width + 0.5 + and bbox[3] <= cropped_rect.height + 0.5 + for bbox in cropped_spans + ), cropped_spans) check("mixed-size pages keep their visible dimensions after rotation normalization", [(round(float(p.mediabox.width)), round(float(p.mediabox.height))) for p in scaled_check.pages] - == [(595, 842), (595, 842), (200, 300), (200, 300), (160, 240)]) + == [(595, 842), (595, 842), (200, 300), (200, 300), (160, 240), (400, 500)]) # ---- SKILL.md overflow check: off-page text is a defect ------------------------- @@ -428,6 +524,17 @@ def stamp_bboxes(path, page_number): overflow_bad.showPage() overflow_bad.save() +rotated_source = canvas.Canvas("overflow-rotated-source.pdf", pagesize=A4) +rotated_source.setFont("Helvetica", 16) +rotated_source.drawString(72, 30, "valid near the unrotated page bottom") +rotated_source.showPage() +rotated_source.save() +rotated_writer = PdfWriter() +rotated_writer.append(R2("overflow-rotated-source.pdf")) +rotated_writer.pages[0].rotate(90) +with open("overflow-rotated.pdf", "wb") as f: + rotated_writer.write(f) + def overflow_pages(path): doc = fitz.open(path) pages = [] @@ -447,14 +554,20 @@ def overflow_pages(path): return pages check("in-bounds PDF reports no overflow pages", overflow_pages("overflow.pdf") == []) -rotated_doc = fitz.open("overflow.pdf") -rotated_doc[0].set_rotation(90) -rotated_doc.save("overflow-rotated.pdf") -rotated_doc.close() -check("rotated in-bounds text uses the unrotated crop-box coordinate space", - overflow_pages("overflow-rotated.pdf") == []) check("off-page text is detected by the overflow check (negative control)", overflow_pages("overflow-bad.pdf") == [1]) +rotated_probe = fitz.open("overflow-rotated.pdf")[0] +rotated_blocks = rotated_probe.get_text( + "blocks", clip=fitz.Rect(-2000, -2000, 2000, 3000), +) +rotation_blind_flag = any( + block[2] > rotated_probe.rect.width + 0.5 + or block[3] > rotated_probe.rect.height + 0.5 + for block in rotated_blocks if block[6] == 0 +) +check("rotated rect comparison falsely flags valid text (negative control)", rotation_blind_flag) +check("overflow check compares rotated pages in unrotated coordinates", + overflow_pages("overflow-rotated.pdf") == []) check("off-page text still extracts, so extraction alone cannot catch it", "drawn far below" in (pypdf.PdfReader("overflow-bad.pdf").pages[0].extract_text() or "")) diff --git a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py index 3fac806..9d7b8a9 100644 --- a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py @@ -166,11 +166,11 @@ def validate_pptx_package(path): for shape in s.shapes: if shape.has_text_frame and old in shape.text_frame.text: candidates.append((i, shape.name, shape)) -assert len(candidates) == 1 +require(len(candidates) == 1, "expected exactly one text target") _, _, target_shape = candidates[0] tf = target_shape.text_frame run_hits = [run for par in tf.paragraphs for run in par.runs if old in run.text] -assert len(run_hits) == 1 +require(len(run_hits) == 1, "target is duplicated or split across runs") run_hits[0].text = run_hits[0].text.replace(old, new, 1) edited_link = [run for par in tf.paragraphs for run in par.runs if run.hyperlink.address] @@ -185,7 +185,7 @@ def validate_pptx_package(path): tbl = next(sh for sh in prs2.slides[0].shapes if sh.has_table).table cell = tbl.cell(0, 1) hits = [run for par in cell.text_frame.paragraphs for run in par.runs if old_cell in run.text] -assert len(hits) == 1, "target is duplicated or split across runs in this cell" +require(len(hits) == 1, "target is duplicated or split across runs in this cell") hits[0].text = hits[0].text.replace(old_cell, new_cell, 1) prs2.save("cell-edited.pptx") @@ -479,6 +479,37 @@ def iter_text_targets(path, shape): yield f"{path}/table[{row_index},{column_index}]", cell.text_frame +def iter_postcheck_text_frames(shapes, path=""): + for shape in shapes: + here = f"{path}/{shape.name}" if path else shape.name + if shape.shape_type == MSO_SHAPE_TYPE.GROUP: + yield from iter_postcheck_text_frames(shape.shapes, here) + continue + if shape.has_text_frame: + yield here, shape.text_frame + if shape.has_table: + for row_index, row in enumerate(shape.table.rows): + for column_index, cell in enumerate(row.cells): + if not cell.is_spanned: + yield f"{here}/table[{row_index},{column_index}]", cell.text_frame + + +group_postcheck = { + location: frame.text for location, frame in iter_postcheck_text_frames(slide6.shapes) +} +table_postcheck = { + location: frame.text + for location, frame in iter_postcheck_text_frames(Presentation("input.pptx").slides[0].shapes) +} +check("mandatory postcheck reaches grouped text", + "nested member" in group_postcheck.values(), group_postcheck) +check("mandatory postcheck reaches table-cell text", + "old cell text" in table_postcheck.values(), table_postcheck) +check("mandatory postcheck inventories unexpected empty table cells", + any(location.endswith("/table[0,0]") and not text.strip() + for location, text in table_postcheck.items()), table_postcheck) + + candidates = [ (i, location, text_frame) for i, s in enumerate(prs6.slides) @@ -524,6 +555,44 @@ def iter_text_targets(path, shape): check("duplicate table text requires a location selector", len(all_table_candidates) == 2) check("table location selector chooses one row/column", len(selected_table_candidates) == 1, [item[1] for item in all_table_candidates]) +try: + require(len(all_table_candidates) == 1, "target is not unique") + optimized_duplicate_guard_rejected = False +except ValueError: + optimized_duplicate_guard_rejected = True +check("explicit uniqueness guard rejects duplicates even under python -O", + optimized_duplicate_guard_rejected) + +split_prs = Presentation() +split_slide = split_prs.slides.add_slide(split_prs.slide_layouts[6]) +split_frame = split_slide.shapes.add_textbox( + Inches(1), Inches(1), Inches(4), Inches(1) +).text_frame +split_frame.paragraphs[0].add_run().text = "old " +split_frame.paragraphs[0].add_run().text = "wording" +split_hits = [ + run for paragraph in split_frame.paragraphs for run in paragraph.runs + if old in run.text +] +try: + require(len(split_hits) == 1 and split_hits[0].text.count(old) == 1, + "target is split across runs") + optimized_run_guard_rejected = False +except ValueError: + optimized_run_guard_rejected = True +check("explicit run-boundary guard survives python -O", optimized_run_guard_rejected) + +repeated_frame = split_slide.shapes.add_textbox( + Inches(1), Inches(2), Inches(4), Inches(1) +).text_frame +repeated_frame.text = f"{old} / {old}" +try: + require(repeated_frame.text.count(old) == 1, + "target occurs more than once in the selected shape") + optimized_repeated_guard_rejected = False +except ValueError: + optimized_repeated_guard_rejected = True +check("explicit repeated-text guard survives python -O", optimized_repeated_guard_rejected) # ---- analyze.md snippet: per-master, script-aware theme font resolution -------- @@ -762,6 +831,17 @@ def stub_slide(name, major, minor, **theme_options): token_ea.set("typeface", "+mj-ea") token_run._r.get_or_add_rPr().append(token_ea) +partial_box = prs7.slides[0].shapes.add_textbox(Inches(5), Inches(4), Inches(4), Inches(1)) +partial_paragraph = partial_box.text_frame.paragraphs[0] +latin_only_run = partial_paragraph.add_run() +latin_only_run.text = "A汉" +latin_only_run.font.name = "Latin Only" +east_only_run = partial_paragraph.add_run() +east_only_run.text = "A汉" +east_only = OxmlElement("a:ea") +east_only.set("typeface", "East Only") +east_only_run._r.get_or_add_rPr().append(east_only) + token_fallback_fonts = { "major": {"latin": "Major Latin", "eastAsia": "", "complexScript": "", "scripts": {"Hans": "Major Hans"}}, @@ -773,6 +853,8 @@ def stub_slide(name, major, minor, **theme_options): run.text: font_candidates(run, font_paragraph, script_fonts, "minor") for run in font_paragraph.runs } +latin_only_faces = font_candidates(latin_only_run, partial_paragraph, script_fonts, "minor") +east_only_faces = font_candidates(east_only_run, partial_paragraph, script_fonts, "minor") check("font triage reports the run face and source", ("latin", "Run Face", "run") in detected_faces["run override"], detected_faces) check( @@ -792,6 +874,14 @@ def stub_slide(name, major, minor, **theme_options): {(slot, face) for slot, face, _ in detected_faces["A汉ก"]} == {("latin", "Latin Explicit"), ("eastAsia", "East Explicit"), ("complexScript", "Complex Explicit")}, detected_faces["A汉ก"]) +check("mixed run combines direct Latin with inherited east-Asian faces", + {("latin", "Latin Only"), ("eastAsia", "East Asian Theme")} + <= {(slot, face) for slot, face, _ in latin_only_faces}, + latin_only_faces) +check("mixed run combines inherited Latin with a direct east-Asian face", + {("latin", "Latin Body"), ("eastAsia", "East Only")} + <= {(slot, face) for slot, face, _ in east_only_faces}, + east_only_faces) check("empty +mj-ea generic face falls back to the major script mapping", ("eastAsia", "Major Hans", "major theme script Hans") in font_candidates(token_run, font_paragraph, token_fallback_fonts, "minor"), diff --git a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py index 6da3d62..d824d0b 100644 --- a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py @@ -425,6 +425,44 @@ class LegacyWorkbook: == [("cell formula", "Sheet!A1", "=Data!A5")], structural_references(formula_only)) +audited_wb = openpyxl.Workbook() +audited_ws = audited_wb.active +audited_ws.title = "Data" +audited_ws["B2"] = "=A2*2" +audited_ws["A5"] = "shift me" +audited_references_before = structural_references(audited_wb) +audited_unchanged = {("cell formula", "Data!B2", "=A2*2")} +unaudited_references = [ + reference for reference in audited_references_before + if reference not in audited_unchanged +] +if not unaudited_references: + audited_ws.insert_rows(5) + audited_ws["D2"] = "=C2*1.08" + audited_wb.save("audited-edit.xlsx") +audited_reopened = openpyxl.load_workbook("audited-edit.xlsx", data_only=False) +check("exact dependency allowlist lets an audited edit reach save", + audited_references_before == [("cell formula", "Data!B2", "=A2*2")] + and unaudited_references == [] + and audited_reopened["Data"]["A6"].value == "shift me" + and audited_reopened["Data"]["B2"].value == "=A2*2" + and audited_reopened["Data"]["D2"].value == "=C2*1.08") + +unsafe_wb = openpyxl.Workbook() +unsafe_ws = unsafe_wb.active +unsafe_ws.title = "Data" +unsafe_ws["A5"], unsafe_ws["A6"] = 10, 20 +unsafe_ws["B1"] = "=SUM(A5:A6)" +unsafe_dependencies = [ + reference for reference in structural_references(unsafe_wb) + if reference not in audited_unchanged +] +unsafe_blocked_before_insert = bool(unsafe_dependencies) +check("intersecting formula not on the exact allowlist is blocked before insertion", + unsafe_blocked_before_insert + and unsafe_ws["A5"].value == 10 and unsafe_ws["A6"].value == 20, + unsafe_dependencies) + # and the edit itself still works after the warning path wb2 = openpyxl.load_workbook("plain.xlsx") wb2["Data"]["B2"] = "=B2*1" # formula stays a formula @@ -594,38 +632,70 @@ def add_demo_formatting(sheet): dim_ws.append(["h1", "h2"]) dim_ws.append([1, 2]) dim_ws.append([3, 4]) +dim_ws["C4"] = "=SUM(A2:B3)" dim_wb.save("dimension.xlsx") # Corrupt the sheet's dimension metadata the way non-Excel producers do. import zipfile as dim_zip with dim_zip.ZipFile("dimension.xlsx") as archive: members = {name: archive.read(name) for name in archive.namelist()} members["xl/worksheets/sheet1.xml"] = members["xl/worksheets/sheet1.xml"].replace( - b"", b"" + b"", b"" ) with dim_zip.ZipFile("dimension.xlsx", "w") as archive: for name, data in members.items(): archive.writestr(name, data) +from openpyxl.utils import get_column_letter + + +def discover_dimension(worksheet): + worksheet.reset_dimensions() + min_row = min_column = max_row = max_column = None + for row in worksheet.iter_rows(): + for cell in row: + row_index = getattr(cell, "row", None) + column_index = getattr(cell, "column", None) + if row_index is None or column_index is None: + continue + min_row = row_index if min_row is None else min(min_row, row_index) + min_column = column_index if min_column is None else min(min_column, column_index) + max_row = row_index if max_row is None else max(max_row, row_index) + max_column = column_index if max_column is None else max(max_column, column_index) + if max_row is None: + return "A1:A1" + return (f"{get_column_letter(min_column)}{min_row}:" + f"{get_column_letter(max_column)}{max_row}") + + dim_value = openpyxl.load_workbook("dimension.xlsx", read_only=True, data_only=True) +dim_formula = openpyxl.load_workbook("dimension.xlsx", read_only=True, data_only=False) dim_ws_ro = dim_value.active -check("corrupted dimension truncates streaming (negative control)", - dim_ws_ro.max_row == 1, dim_ws_ro.max_row) +check("plausible but truncated dimension limits streaming (negative control)", + dim_ws_ro.calculate_dimension() == "A1:B2" and dim_ws_ro.max_row == 2, + (dim_ws_ro.calculate_dimension(), dim_ws_ro.max_row)) streamed_before_reset = [row for row in dim_ws_ro.iter_rows(values_only=True)] -dim_ws_ro.reset_dimensions() -try: - dim_ws_ro.calculate_dimension() - unforced_dimension_rejected = False -except ValueError: - unforced_dimension_rejected = True -forced_dimension = dim_ws_ro.calculate_dimension(force=True) +discovered_value_dimension = discover_dimension(dim_ws_ro) +discovered_formula_dimension = discover_dimension(dim_formula.active) streamed_after_reset = [row for row in dim_ws_ro.iter_rows(values_only=True)] dim_value.close() -check("reset dimensions require an explicit force scan before reporting extent", - unforced_dimension_rejected and forced_dimension == "A1:B3", forced_dimension) +dim_formula.close() +check("dimension scan recovers plausible row/column truncation in both workbook views", + discovered_value_dimension == "A1:C4" + and discovered_formula_dimension == "A1:C4", + (discovered_value_dimension, discovered_formula_dimension)) check("reset_dimensions restores the real extent", - len(streamed_after_reset) == 3 and streamed_after_reset[2] == (3, 4), + len(streamed_before_reset) == 2 + and len(streamed_after_reset) == 4 + and streamed_after_reset[2][:2] == (3, 4), (len(streamed_before_reset), streamed_after_reset[-1])) +empty_dimension_wb = openpyxl.Workbook() +empty_dimension_wb.save("empty-dimension.xlsx") +empty_dimension_ro = openpyxl.load_workbook("empty-dimension.xlsx", read_only=True) +check("dimension scan handles an actually empty worksheet", + discover_dimension(empty_dimension_ro.active) == "A1:A1") +empty_dimension_ro.close() + print("\n" + ("ALL XLSX FIXTURES PASSED" if not failures else f"{len(failures)} FAILURES: {failures}")) sys.exit(0 if not failures else 1) From c6ad3690a7527e46ed9f009fb7c0db6d53936a65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=98=E6=89=AC?= Date: Sun, 16 Aug 2026 20:22:49 +0800 Subject: [PATCH 18/34] fix(document-skills): cover inline and placeholder edge cases --- .../skills/docx/references/edit.md | 15 ++++- .../skills/pptx/references/analyze.md | 18 ++++- .../document-skills/tests/docx_fixture.py | 35 ++++++++++ .../document-skills/tests/pptx_fixture.py | 66 ++++++++++++++++++- .../document-skills/tests/xlsx_fixture.py | 6 +- 5 files changed, 133 insertions(+), 7 deletions(-) diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/edit.md b/plugins/Hylouis233/document-skills/skills/docx/references/edit.md index b167def..d2497f8 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/edit.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/edit.md @@ -15,6 +15,7 @@ from docx.oxml.ns import qn SAFE_RUN_CHILDREN = { qn("w:rPr"), qn("w:t"), qn("w:tab"), qn("w:cr"), } +MODELED_PARAGRAPH_CHILDREN = {qn("w:pPr"), qn("w:r")} def unsafe_run_content(run): unsafe = [] @@ -32,6 +33,14 @@ def replace_across_runs(paragraph, old, new): if not old: raise ValueError("old must not be empty") + unmodeled = [ + child.tag.rsplit("}", 1)[-1] + for child in paragraph._p + if child.tag not in MODELED_PARAGRAPH_CHILDREN + ] + if unmodeled: + raise ValueError(f"paragraph contains unmodeled inline containers: {unmodeled}") + runs = list(paragraph.runs) text = "".join(run.text for run in runs) starts = [] @@ -103,9 +112,9 @@ doc.save("input.edited.docx") The replacement text inherits the first matched run's formatting. Unmatched text before and after it stays in its original runs, so its formatting is preserved. The routine fails before -making changes if any matched run contains a drawing, field, reference, typed page/column break, -or a text-wrapping break with `w:clear` that `run.text` would destroy. Use raw OOXML for those cases and for tracked changes or -other content that `paragraph.runs` does not expose. +making changes if the paragraph contains an inline container that `paragraph.runs` does not expose, +or if any matched run contains a drawing, field, reference, typed page/column break, or a +text-wrapping break with `w:clear` that `run.text` would destroy. Use raw OOXML for those cases. ## Tier 2 - raw OOXML surgery (only when Tier 1 cannot express it) diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md index 20a7c74..841f637 100644 --- a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md @@ -30,6 +30,19 @@ def table_cells(table): for column_index, cell in enumerate(row.cells) ] for row_index, row in enumerate(table.rows)] +def picture_content(shape): + """Inventory ordinary pictures and populated picture placeholders alike.""" + try: + image = shape.image + except (AttributeError, ValueError): + return None + return { + "name": shape.name, + "filename": image.filename, + "extension": image.ext, + "bytes": len(image.blob), + } + def cached_numeric_points(source): """Return indexed cached points, or None when the cache metadata is unavailable.""" if source is None: @@ -92,7 +105,10 @@ for i, slide in enumerate(prs.slides): series = [series_content(item) for item in items] plots.append({"kind": type(plot).__name__, "categories": categories, "series": series}) charts.append({"title": chart_title, "plots": plots}) - pictures = [sh.name for sh in shapes if sh.shape_type == MSO_SHAPE_TYPE.PICTURE] + pictures = [ + content for sh in shapes + if (content := picture_content(sh)) is not None + ] # These full values - not only counts or lengths - are the evidence for summaries and # repurposing. Keep notes verbatim so markdown output can preserve them as blockquotes. diff --git a/plugins/Hylouis233/document-skills/tests/docx_fixture.py b/plugins/Hylouis233/document-skills/tests/docx_fixture.py index 21927e1..72dd664 100644 --- a/plugins/Hylouis233/document-skills/tests/docx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/docx_fixture.py @@ -309,6 +309,7 @@ def effective_face(run, slot): SAFE_RUN_CHILDREN = { qn("w:rPr"), qn("w:t"), qn("w:tab"), qn("w:cr"), } +MODELED_PARAGRAPH_CHILDREN = {qn("w:pPr"), qn("w:r")} def unsafe_run_content(run): @@ -325,6 +326,13 @@ def unsafe_run_content(run): def replace_across_runs(paragraph, old, new): if not old: raise ValueError("old must not be empty") + unmodeled = [ + child.tag.rsplit("}", 1)[-1] + for child in paragraph._p + if child.tag not in MODELED_PARAGRAPH_CHILDREN + ] + if unmodeled: + raise ValueError(f"paragraph contains unmodeled inline containers: {unmodeled}") runs = list(paragraph.runs) text = "".join(run.text for run in runs) starts = [] @@ -413,6 +421,33 @@ def replace_across_runs(paragraph, old, new): check("rejected clear-break replacement is atomic", etree.tostring(clear_run._r) == clear_before) +container_doc = Document() +container_paragraph = container_doc.add_paragraph() +container_paragraph.add_run("T") +container_hyperlink = OxmlElement("w:hyperlink") +container_hyperlink.set(qn("w:anchor"), "fixture-target") +container_link_run = OxmlElement("w:r") +container_link_text = OxmlElement("w:t") +container_link_text.text = "link" +container_link_run.append(container_link_text) +container_hyperlink.append(container_link_run) +container_paragraph._p.append(container_hyperlink) +container_paragraph.add_run("BD") +container_before = etree.tostring(container_paragraph._p) +check("Paragraph.runs can manufacture a false match across a hyperlink (negative control)", + "".join(run.text for run in container_paragraph.runs) == "TBD" + and paragraph_text(container_paragraph) == "TlinkBD") +try: + replace_across_runs(container_paragraph, "TBD", "Done") + rejected_inline_container = False +except ValueError: + rejected_inline_container = True +check("replacement rejects unmodeled inline containers before matching", + rejected_inline_container) +check("rejected inline-container replacement is atomic", + etree.tostring(container_paragraph._p) == container_before + and paragraph_text(container_paragraph) == "TlinkBD") + def list_number_num_id(doc): """The numId that the ListNumber style binds to in this document part.""" diff --git a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py index 9d7b8a9..9e7483a 100644 --- a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py @@ -5,6 +5,7 @@ # python pptx_fixture.py (deps: python-pptx) # python xlsx_fixture.py (deps: openpyxl) # python docx_fixture.py (deps: python-docx, pymupdf, soffice on PATH) +import base64 import copy import sys import xml.etree.ElementTree as ET @@ -15,7 +16,7 @@ from pptx.chart.data import BubbleChartData, ChartData, XyChartData from pptx.dml.color import RGBColor from pptx.enum.chart import XL_CHART_TYPE -from pptx.enum.shapes import MSO_SHAPE_TYPE +from pptx.enum.shapes import MSO_SHAPE_TYPE, PP_PLACEHOLDER from pptx.opc.constants import RELATIONSHIP_TYPE as RT from pptx.util import Inches, Pt @@ -251,6 +252,19 @@ def table_cells(table): ] for row_index, row in enumerate(table.rows)] +def picture_content(shape): + try: + image = shape.image + except (AttributeError, ValueError): + return None + return { + "name": shape.name, + "filename": image.filename, + "extension": image.ext, + "bytes": len(image.blob), + } + + def cached_numeric_points(source): if source is None: return None @@ -310,7 +324,12 @@ def extract_slide_content(slide): plots.append({"kind": type(plot).__name__, "categories": categories, "series": series}) charts.append({"title": chart_title, "plots": plots}) notes = slide.notes_slide.notes_text_frame.text if slide.has_notes_slide else "" - return {"text": text, "tables": tables, "charts": charts, "notes": notes} + pictures = [ + info for shape in shapes + if (info := picture_content(shape)) is not None + ] + return {"text": text, "tables": tables, "charts": charts, + "pictures": pictures, "notes": notes} content = extract_slide_content(Presentation("input.pptx").slides[0]) @@ -338,6 +357,40 @@ def extract_slide_content(slide): == [(0, 5.0)]) check("content inventory emits notes text", "regional split" in content["notes"], content["notes"]) +placeholder_png = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9ZlRYAAAAASUVORK5CYII=" +) +with open("placeholder-picture.png", "wb") as stream: + stream.write(placeholder_png) +placeholder_prs = Presentation() +picture_layout = next( + layout for layout in placeholder_prs.slide_layouts + if any( + shape.placeholder_format.type == PP_PLACEHOLDER.PICTURE + for shape in layout.placeholders + ) +) +placeholder_slide = placeholder_prs.slides.add_slide(picture_layout) +picture_placeholder = next( + shape for shape in placeholder_slide.placeholders + if shape.placeholder_format.type == PP_PLACEHOLDER.PICTURE +) +placeholder_picture = picture_placeholder.insert_picture("placeholder-picture.png") +placeholder_prs.save("picture-placeholder.pptx") +placeholder_reopened = Presentation("picture-placeholder.pptx").slides[0] +reopened_picture = next( + shape for shape in placeholder_reopened.placeholders if hasattr(shape, "image") +) +placeholder_inventory = extract_slide_content(placeholder_reopened)["pictures"] +check("picture placeholder remains a placeholder after image insertion", + reopened_picture.shape_type == MSO_SHAPE_TYPE.PLACEHOLDER, + reopened_picture.shape_type) +check("picture inventory includes populated picture placeholders and image metadata", + len(placeholder_inventory) == 1 + and placeholder_inventory[0]["extension"] == "png" + and placeholder_inventory[0]["bytes"] == len(placeholder_png), + placeholder_inventory) + merged_prs = Presentation() merged_slide = merged_prs.slides.add_slide(merged_prs.slide_layouts[6]) merged_table = merged_slide.shapes.add_table( @@ -841,6 +894,8 @@ def stub_slide(name, major, minor, **theme_options): east_only = OxmlElement("a:ea") east_only.set("typeface", "East Only") east_only_run._r.get_or_add_rPr().append(east_only) +cyrillic_cjk_run = partial_paragraph.add_run() +cyrillic_cjk_run.text = "Тест 汉" token_fallback_fonts = { "major": {"latin": "Major Latin", "eastAsia": "", "complexScript": "", @@ -855,6 +910,9 @@ def stub_slide(name, major, minor, **theme_options): } latin_only_faces = font_candidates(latin_only_run, partial_paragraph, script_fonts, "minor") east_only_faces = font_candidates(east_only_run, partial_paragraph, script_fonts, "minor") +cyrillic_cjk_faces = font_candidates( + cyrillic_cjk_run, partial_paragraph, script_fonts, "minor" +) check("font triage reports the run face and source", ("latin", "Run Face", "run") in detected_faces["run override"], detected_faces) check( @@ -882,6 +940,10 @@ def stub_slide(name, major, minor, **theme_options): {("latin", "Latin Body"), ("eastAsia", "East Only")} <= {(slot, face) for slot, face, _ in east_only_faces}, east_only_faces) +check("Cyrillic plus CJK requires both Latin and east-Asian theme slots", + {("latin", "Cyrillic Theme"), ("eastAsia", "Simplified Chinese Theme")} + <= {(slot, face) for slot, face, _ in cyrillic_cjk_faces}, + cyrillic_cjk_faces) check("empty +mj-ea generic face falls back to the major script mapping", ("eastAsia", "Major Hans", "major theme script Hans") in font_candidates(token_run, font_paragraph, token_fallback_fonts, "minor"), diff --git a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py index d824d0b..1a5bdc3 100644 --- a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py @@ -313,7 +313,7 @@ def formula_may_intersect_rows(owner_sheet, formula, shifted_sheet, start_row): if "[" in qualifier or ":" in qualifier: return True target_sheet = qualifier.strip("'").replace("''", "'") - if target_sheet != shifted_sheet: + if target_sheet.casefold() != shifted_sheet.casefold(): continue try: _, min_row, _, max_row = range_boundaries(reference.replace("$", "")) @@ -381,6 +381,10 @@ class LegacyWorkbook: formula_may_intersect_rows("Audit", "=SUM(A2:A3)", "Audit", 3)) check("audited formula ranges above the insertion can proceed", not formula_may_intersect_rows("Data", "=C2*1.08", "Data", 5)) +check("sheet qualifiers are matched case-insensitively", + formula_may_intersect_rows("Summary", "=SUM(data!A5:A6)", "Data", 5)) +check("a genuinely different sheet remains outside the shifted rows", + not formula_may_intersect_rows("Summary", "=SUM(Archive!A5:A6)", "Data", 5)) stale_wb = openpyxl.Workbook() stale_ws = stale_wb.active From 5b895451c50257be98dc265d810b1269516e1f6f Mon Sep 17 00:00:00 2001 From: Hylouis233 Date: Sun, 16 Aug 2026 21:43:22 +0800 Subject: [PATCH 19/34] fix(document-skills): address latest review findings --- .../skills/docx/references/read.md | 19 +++++++++- .../document-skills/skills/pdf/SKILL.md | 14 +++++--- .../skills/pdf/references/transform.md | 8 +++-- .../skills/pptx/references/analyze.md | 17 ++++++++- .../skills/xlsx/references/edit.md | 12 ++++++- .../document-skills/tests/docx_fixture.py | 36 +++++++++++++++++-- .../document-skills/tests/pdf_fixture.py | 28 ++++++++++++--- .../document-skills/tests/pptx_fixture.py | 24 ++++++++++++- .../document-skills/tests/xlsx_fixture.py | 14 +++++++- 9 files changed, 153 insertions(+), 19 deletions(-) diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/read.md b/plugins/Hylouis233/document-skills/skills/docx/references/read.md index 53cada3..ebf9045 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/read.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/read.md @@ -27,6 +27,17 @@ def iter_part_blocks(root, parent): yield "paragraph", Paragraph(child, parent) elif child.tag == qn("w:tbl"): yield "table", Table(child, parent) + elif child.tag == qn("w:altChunk"): + relationship_id = child.get(qn("r:id")) + part = getattr(parent, "part", None) + relationship = None if part is None else part.rels.get(relationship_id) + yield "unreadable", { + "kind": "altChunk", + "relationship_id": relationship_id, + "target": None if relationship is None else relationship.target_ref, + "content_type": None if relationship is None or relationship.is_external + else relationship.target_part.content_type, + } else: yield from iter_part_blocks(child, parent) @@ -98,8 +109,10 @@ print("content controls:", len(content_controls), "top-level blocks:", len(block for kind, block in blocks: if kind == "paragraph": print(block.style.name, "|", paragraph_text(block)) - else: + elif kind == "table": print("table |", table_content(block)) + else: + print("unreadable |", block) ``` Notes: @@ -110,6 +123,10 @@ Notes: when a table is yielded, so table text is not also emitted as prose; `table_content()` handles nested tables recursively and emits `column`, `colspan`, and `vMerge` metadata for physical cells instead of duplicating merge-origin text through `row.cells`. + Imported `w:altChunk` HTML/RTF/document parts are not modeled by python-docx; the traversal + reports their relationship target and content type as `unreadable` instead of silently + presenting an incomplete extraction. Convert them with a trusted office renderer before + claiming their content was read. Text boxes, headers, footers, and footnotes still require their own collections (`section.header/.footer`) or raw XML. - For revision/comment metadata, inspect the XML parts directly: `word/comments.xml`, diff --git a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md index d4cef0c..1f5d39c 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md @@ -61,6 +61,12 @@ if r.is_encrypted: if not password: raise RuntimeError("set PDF_PASSWORD so the encrypted output can be postchecked") r = pypdf.PdfReader(output_path, password=password) # wrong passwords fail here + +def require(condition, message): + # Mandatory verification must remain active under python -O. + if not condition: + raise ValueError(message) + page_count = len(r.pages) page_texts = { number: (page.extract_text() or "").strip() @@ -82,13 +88,13 @@ missing_text_pages = [ and not text and widget_count(r.pages[number - 1]) == 0 ] -assert not missing_text_pages, f"pages without extractable text: {missing_text_pages}" +require(not missing_text_pages, f"pages without extractable text: {missing_text_pages}") # Add task-specific checks when exact copy matters, for example # {1: ("Report title",), 2: ("Conclusion",)}. Per-page text presence is enforced above. expected_strings_by_page = {} for page_number, expected_strings in expected_strings_by_page.items(): missing = [value for value in expected_strings if value not in page_texts[page_number]] - assert not missing, f"page {page_number} is missing {missing}" + require(not missing, f"page {page_number} is missing {missing}") # Compare width/height to the exact size used at creation with a small point tolerance. # ReportLab A4 is about (595.2756, 841.8898), not the rounded prose value (595.27, 841.89). expected_page_size = (595.2756, 841.8898) # replace for Letter or a task-specific size @@ -103,7 +109,7 @@ size_mismatches = [ if any(abs(value - expected) > page_size_tolerance for value, expected in zip(actual, expected_page_size)) ] -assert not size_mismatches, f"unexpected page sizes: {size_mismatches}" +require(not size_mismatches, f"unexpected page sizes: {size_mismatches}") # Overflow is a defect (shared rule 4): text that runs past the page box is # clipped or off-page even though every check above still passes. Plain block @@ -135,7 +141,7 @@ for page in overflow_doc: if beyond_box: overflow_pages.append(page.number + 1) overflow_doc.close() -assert not overflow_pages, f"text blocks extend past the page box on pages: {overflow_pages}" +require(not overflow_pages, f"text blocks extend past the page box on pages: {overflow_pages}") # The page box is the hard bound. When the task declares specific margins, # additionally check key blocks against them (or render and inspect visually) - # content inside the box but past a declared margin is a softer, task-specific diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md b/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md index 597c80c..ba04dd3 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md @@ -50,12 +50,14 @@ writer.append(reader) # clone pages plus catalog entries such as # A plain merge_page() overlays the stamp in its own coordinates, so on a page # with different dimensions, origin, or rotation the stamp can be clipped or -# land entirely off-page. Normalize /Rotate into content, then scale each copy -# to the visible destination box and include both boxes' non-zero origins. +# land entirely off-page. Normalize only the stamp, then scale each copy to the +# destination crop box and include both boxes' non-zero origins. Keep the +# destination page's /Rotate value intact: transfer_rotation_to_content() does +# not transform annotation rectangles, so normalizing a page that has widgets, +# links, or other annotations can displace its interactive geometry. stamp_box = stamp.cropbox sw, sh = float(stamp_box.width), float(stamp_box.height) for page in writer.pages: - page.transfer_rotation_to_content() destination = page.cropbox dw, dh = float(destination.width), float(destination.height) scale = min(dw / sw, dh / sh) diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md index 841f637..f2604bb 100644 --- a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md @@ -74,6 +74,17 @@ def series_content(series): content["bubble_points"] = bubble_points return content +def chart_axis_titles(chart): + """Return titles for axes the chart actually exposes (pie charts have none).""" + titles = {} + for label, attribute in (("category", "category_axis"), ("value", "value_axis")): + try: + axis = getattr(chart, attribute) + except (AttributeError, ValueError): + continue + titles[label] = axis.axis_title.text_frame.text if axis.has_title else "" + return titles + prs = Presentation("input.pptx") print("slide size:", prs.slide_width, prs.slide_height) for i, slide in enumerate(prs.slides): @@ -104,7 +115,11 @@ for i, slide in enumerate(prs.slides): ] series = [series_content(item) for item in items] plots.append({"kind": type(plot).__name__, "categories": categories, "series": series}) - charts.append({"title": chart_title, "plots": plots}) + charts.append({ + "title": chart_title, + "axis_titles": chart_axis_titles(chart), + "plots": plots, + }) pictures = [ content for sh in shapes if (content := picture_content(sh)) is not None diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md index d058b92..bb17015 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md @@ -91,7 +91,17 @@ def formula_may_intersect_rows(owner_sheet, formula, shifted_sheet, start_row): """Fail closed unless every range token is provably above/outside the shifted rows.""" if not isinstance(formula, str) or not formula.startswith("="): return True - for token in Tokenizer(formula).items: + tokens = Tokenizer(formula).items + # These functions can manufacture references from strings or numeric + # offsets that the RANGE-token audit below cannot see or rewrite safely. + unmodeled_reference_functions = {"indirect", "offset", "address"} + if any( + token.type == "FUNC" and token.subtype == "OPEN" + and token.value.rstrip("(").casefold() in unmodeled_reference_functions + for token in tokens + ): + return True + for token in tokens: if token.type != "OPERAND" or token.subtype != "RANGE": continue reference = token.value diff --git a/plugins/Hylouis233/document-skills/tests/docx_fixture.py b/plugins/Hylouis233/document-skills/tests/docx_fixture.py index 72dd664..70aeb2f 100644 --- a/plugins/Hylouis233/document-skills/tests/docx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/docx_fixture.py @@ -128,6 +128,17 @@ def iter_part_blocks(root, parent): yield "paragraph", Paragraph(child, parent) elif child.tag == qn("w:tbl"): yield "table", Table(child, parent) + elif child.tag == qn("w:altChunk"): + relationship_id = child.get(qn("r:id")) + part = getattr(parent, "part", None) + relationship = None if part is None else part.rels.get(relationship_id) + yield "unreadable", { + "kind": "altChunk", + "relationship_id": relationship_id, + "target": None if relationship is None else relationship.target_ref, + "content_type": None if relationship is None or relationship.is_external + else relationship.target_part.content_type, + } else: yield from iter_part_blocks(child, parent) @@ -153,7 +164,12 @@ def table_content(table): for cell in row.cells: items = [] for kind, block in iter_part_blocks(cell._tc, cell): - items.append(paragraph_text(block) if kind == "paragraph" else table_content(block)) + if kind == "paragraph": + items.append(paragraph_text(block)) + elif kind == "table": + items.append(table_content(block)) + else: + items.append(block) rendered_cells.append(items) rows.append(rendered_cells) return rows @@ -180,6 +196,14 @@ def wrap_in_sdt(element): inline_run = inline_paragraph.add_run("inline한") inline_paragraph.add_run("-after") wrap_in_sdt(inline_run._r) +chunk_relationship = sdt_doc.part.relate_to( + "https://example.invalid/imported.html", + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/aFChunk", + is_external=True, +) +alt_chunk = OxmlElement("w:altChunk") +alt_chunk.set(qn("r:id"), chunk_relationship) +sdt_doc.element.body.insert(len(sdt_doc.element.body) - 1, alt_chunk) sdt_doc.save("content-control.docx") sdt_reopened = Document("content-control.docx") check("doc.paragraphs omits block content-control text (negative control)", @@ -189,7 +213,7 @@ def wrap_in_sdt(element): len(sdt_reopened.tables) == 0) check("block walker preserves document order across content controls", [kind for kind, _ in walked_blocks] - == ["paragraph", "paragraph", "table", "paragraph"], walked_blocks) + == ["paragraph", "paragraph", "table", "paragraph", "unreadable"], walked_blocks) walked_paragraphs = [block for kind, block in walked_blocks if kind == "paragraph"] walked_text = [paragraph_text(paragraph) for paragraph in walked_paragraphs] check("content-control traversal emits the nested paragraph", "inside content control" in walked_text, walked_text) @@ -204,6 +228,14 @@ def wrap_in_sdt(element): sum(item.count("table한") for item in all_emitted) == 1, all_emitted) check("nested-table text is emitted exactly once", sum(item.count("nested한") for item in all_emitted) == 1, all_emitted) +unreadable_parts = [block for kind, block in walked_blocks if kind == "unreadable"] +check("altChunk content is reported instead of silently omitted", + unreadable_parts == [{ + "kind": "altChunk", + "relationship_id": chunk_relationship, + "target": "https://example.invalid/imported.html", + "content_type": None, + }], unreadable_parts) # Per-run glyph validation must not let a different referenced font hide a missing glyph. fixture_cmaps = {"CJK Face": {ord("漢")}, "Latin Face": {ord("A")}} diff --git a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py index 70db7aa..a5d1b5d 100644 --- a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py @@ -257,7 +257,6 @@ def widget_field_name(widget): writer.append(reader) stamp_box = stamp.cropbox for page in writer.pages: - page.transfer_rotation_to_content() destination = page.cropbox scale = min(float(destination.width) / float(stamp_box.width), float(destination.height) / float(stamp_box.height)) @@ -412,6 +411,14 @@ def widget_field_name(widget): offset_page.cropbox.upper_right = (300, 500) rotated_page = small_source.add_blank_page(width=240, height=160) rotated_page.rotate(90) +from pypdf.generic import ArrayObject, DictionaryObject, FloatObject, NameObject +rotated_link = DictionaryObject({ + NameObject("/Type"): NameObject("/Annot"), + NameObject("/Subtype"): NameObject("/Link"), + NameObject("/Rect"): ArrayObject([FloatObject(value) for value in (20, 30, 100, 60)]), + NameObject("/Border"): ArrayObject([FloatObject(0), FloatObject(0), FloatObject(0)]), +}) +rotated_page[NameObject("/Annots")] = ArrayObject([small_source._add_object(rotated_link)]) cropped_page = small_source.add_blank_page(width=400, height=500) cropped_page.cropbox.lower_left = (250, 300) cropped_page.cropbox.upper_right = (390, 480) @@ -419,6 +426,12 @@ def widget_field_name(widget): with open("mixed.pdf", "wb") as f: mixed_writer.write(f) +def first_annotation_rect(page): + return tuple(float(value) for value in page["/Annots"][0].get_object()["/Rect"]) + +mixed_rotated_page = R2("mixed.pdf").pages[4] +rotated_geometry_before = (mixed_rotated_page.rotation, first_annotation_rect(mixed_rotated_page)) + # Negative control: a plain merge keeps the A4 stamp's coordinates, so the text # lands outside the small page and cannot be extracted. plain_writer = PdfWriter() @@ -476,7 +489,6 @@ def stamp_bboxes(path, page_number): stamp_box2 = stamp_page.cropbox sw2, sh2 = float(stamp_box2.width), float(stamp_box2.height) for page in scaled_writer.pages: - page.transfer_rotation_to_content() destination = page.cropbox dw, dh = float(destination.width), float(destination.height) scale = min(dw / sw2, dh / sh2) @@ -496,7 +508,7 @@ def stamp_bboxes(path, page_number): rotated_spans = stamp_bboxes("scaled-stamped.pdf", 4) cropped_spans = stamp_bboxes("scaled-stamped.pdf", 5) check("scaled stamp lands inside the non-zero-origin page", bool(offset_spans), offset_spans) -check("scaled stamp lands inside the normalized 90-degree page", bool(rotated_spans), rotated_spans) +check("scaled stamp lands inside the rotated page without normalizing it", bool(rotated_spans), rotated_spans) cropped_rect = fitz.open("scaled-stamped.pdf")[5].rect check("scaled stamp lands inside the offset visible crop box", bool(cropped_spans) @@ -506,10 +518,16 @@ def stamp_bboxes(path, page_number): and bbox[3] <= cropped_rect.height + 0.5 for bbox in cropped_spans ), cropped_spans) -check("mixed-size pages keep their visible dimensions after rotation normalization", +check("mixed-size pages keep their original media boxes", [(round(float(p.mediabox.width)), round(float(p.mediabox.height))) for p in scaled_check.pages] - == [(595, 842), (595, 842), (200, 300), (200, 300), (160, 240), (400, 500)]) + == [(595, 842), (595, 842), (200, 300), (200, 300), (240, 160), (400, 500)]) +scaled_rotated_page = scaled_check.pages[4] +check("watermarking preserves rotated-page annotation geometry", + (scaled_rotated_page.rotation, first_annotation_rect(scaled_rotated_page)) + == rotated_geometry_before, + ((scaled_rotated_page.rotation, first_annotation_rect(scaled_rotated_page)), + rotated_geometry_before)) # ---- SKILL.md overflow check: off-page text is a defect ------------------------- diff --git a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py index 9e7483a..de04829 100644 --- a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py @@ -64,6 +64,10 @@ def check(name, cond, extra=""): ).chart chart.has_title = True chart.chart_title.text_frame.text = "Units by region" +chart.category_axis.has_title = True +chart.category_axis.axis_title.text_frame.text = "Region" +chart.value_axis.has_title = True +chart.value_axis.axis_title.text_frame.text = "Units sold" xy_data = XyChartData() xy_series = xy_data.add_series("Trend") @@ -297,6 +301,17 @@ def series_content(series): return content +def chart_axis_titles(chart): + titles = {} + for label, attribute in (("category", "category_axis"), ("value", "value_axis")): + try: + axis = getattr(chart, attribute) + except (AttributeError, ValueError): + continue + titles[label] = axis.axis_title.text_frame.text if axis.has_title else "" + return titles + + def extract_slide_content(slide): shapes = list(iter_shapes(slide.shapes)) text = [sh.text_frame.text for sh in shapes if sh.has_text_frame and sh.text_frame.text] @@ -322,7 +337,11 @@ def extract_slide_content(slide): ] series = [series_content(item) for item in items] plots.append({"kind": type(plot).__name__, "categories": categories, "series": series}) - charts.append({"title": chart_title, "plots": plots}) + charts.append({ + "title": chart_title, + "axis_titles": chart_axis_titles(chart), + "plots": plots, + }) notes = slide.notes_slide.notes_text_frame.text if slide.has_notes_slide else "" pictures = [ info for shape in shapes @@ -344,6 +363,9 @@ def extract_slide_content(slide): == [{"name": "Units", "values": [120.0, 80.0]}], content["charts"], ) +check("content inventory emits category and value axis titles", + content["charts"][0]["axis_titles"] + == {"category": "Region", "value": "Units sold"}, content["charts"][0]) chart_by_title = {item["title"]: item for item in content["charts"]} check("content inventory emits XY x/y points", chart_by_title["XY trend"]["plots"][0]["series"][0]["x_points"] diff --git a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py index 1a5bdc3..caf5b69 100644 --- a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py @@ -303,7 +303,15 @@ def cell_formula_references(workbook): def formula_may_intersect_rows(owner_sheet, formula, shifted_sheet, start_row): if not isinstance(formula, str) or not formula.startswith("="): return True - for token in Tokenizer(formula).items: + tokens = Tokenizer(formula).items + unmodeled_reference_functions = {"indirect", "offset", "address"} + if any( + token.type == "FUNC" and token.subtype == "OPEN" + and token.value.rstrip("(").casefold() in unmodeled_reference_functions + for token in tokens + ): + return True + for token in tokens: if token.type != "OPERAND" or token.subtype != "RANGE": continue reference = token.value @@ -385,6 +393,10 @@ class LegacyWorkbook: formula_may_intersect_rows("Summary", "=SUM(data!A5:A6)", "Data", 5)) check("a genuinely different sheet remains outside the shifted rows", not formula_may_intersect_rows("Summary", "=SUM(Archive!A5:A6)", "Data", 5)) +check("INDIRECT string references require a manual structural rewrite plan", + formula_may_intersect_rows("Data", '=SUM(INDIRECT("A5:A6"))', "Data", 5)) +check("OFFSET numeric row references require a manual structural rewrite plan", + formula_may_intersect_rows("Data", "=SUM(OFFSET(A1,4,0,2,1))", "Data", 5)) stale_wb = openpyxl.Workbook() stale_ws = stale_wb.active From 88a3723709dc4de75b1d2805f010306e065a4589 Mon Sep 17 00:00:00 2001 From: Hylouis233 Date: Sun, 16 Aug 2026 23:29:54 +0800 Subject: [PATCH 20/34] fix(document-skills): satisfy maintainer verification gate --- .github/workflows/ci.yml | 7 ++ plugins/Hylouis233/document-skills/README.md | 5 +- .../skills/docx/references/cjk.md | 28 ++++++++ .../skills/docx/references/read.md | 19 ++++-- .../skills/pdf/references/inspect.md | 30 +++++++- .../skills/pdf/references/transform.md | 23 +++++-- .../skills/pptx/references/analyze.md | 29 ++++++++ .../document-skills/skills/xlsx/SKILL.md | 27 +++++++- .../skills/xlsx/references/edit.md | 22 +++++- .../document-skills/tests/docx_fixture.py | 68 ++++++++++++++++++- .../document-skills/tests/fixtures.test.mjs | 34 ++++++++++ .../document-skills/tests/pdf_fixture.py | 37 ++++++++++ .../document-skills/tests/pptx_fixture.py | 60 +++++++++++++++- .../tests/requirements-fixtures.txt | 8 +++ .../document-skills/tests/xlsx_fixture.py | 34 +++++++++- test/hosted-plugins.test.mjs | 2 +- 16 files changed, 410 insertions(+), 23 deletions(-) create mode 100644 plugins/Hylouis233/document-skills/tests/fixtures.test.mjs create mode 100644 plugins/Hylouis233/document-skills/tests/requirements-fixtures.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b2c256a..d946dd4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,5 +17,12 @@ jobs: with: node-version: 22 cache: npm + - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + with: + python-version: '3.13' + cache: pip + cache-dependency-path: plugins/Hylouis233/document-skills/tests/requirements-fixtures.txt + - run: sudo apt-get update && sudo apt-get install -y --no-install-recommends libreoffice-writer + - run: python -m pip install -r plugins/Hylouis233/document-skills/tests/requirements-fixtures.txt - run: npm ci - run: npm run check diff --git a/plugins/Hylouis233/document-skills/README.md b/plugins/Hylouis233/document-skills/README.md index c0ffb91..dbd2265 100644 --- a/plugins/Hylouis233/document-skills/README.md +++ b/plugins/Hylouis233/document-skills/README.md @@ -84,7 +84,10 @@ encrypted extraction, CMYK conversion, soft masks; PPTX run-preserving edits, ta actual content extraction, grouped shapes, script-aware per-master themes; XLSX dialect sniffing, array formulas, streamed extension checks, structural-reference audits; DOCX bounded package review, content controls, per-run glyph checks, guarded replacement, and a LibreOffice-rendered numbering -restart). Each script is self-contained and exits non-zero on failure. +restart). Each script is self-contained and exits non-zero on failure. `npm run check` executes +all four through `tests/fixtures.test.mjs`; install the pinned Python environment with +`python -m pip install -r tests/requirements-fixtures.txt` and make `soffice` available first. +The repository CI installs those dependencies and LibreOffice Writer before running the same gate. ## Requirements diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md b/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md index 61bf74d..a2a68d8 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md @@ -100,6 +100,7 @@ section.left_margin, section.right_margin = Cm(2.8), Cm(2.6) ```python from docx import Document + from docx.enum.style import WD_STYLE_TYPE from fontTools.ttLib import TTFont from docx.oxml.ns import qn from docx.text.paragraph import Paragraph @@ -139,10 +140,37 @@ section.left_margin, section.right_margin = Cm(2.8), Cm(2.6) yield face style = style.base_style + def table_style_faces(run, slot): + """Find table/conditional style faces that python-docx does not cascade for runs.""" + element = run._r + while element is not None and element.tag != qn("w:tbl"): + element = element.getparent() + if element is None: + return [] + table_properties = element.find(qn("w:tblPr")) + table_style = None if table_properties is None else table_properties.find(qn("w:tblStyle")) + style_id = None if table_style is None else table_style.get(qn("w:val")) + style = None if not style_id else doc.styles.get_by_id(style_id, WD_STYLE_TYPE.TABLE) + faces = [] + while style is not None: + rprs = [style.element.find(qn("w:rPr"))] + rprs.extend( + region.find(qn("w:rPr")) + for region in style.element.findall(qn("w:tblStylePr")) + ) + faces.extend(face for rpr in rprs if (face := face_from_rpr(rpr, slot))) + style = style.base_style + return list(dict.fromkeys(faces)) + def effective_face(run, slot): direct = face_from_rpr(run._r.find(qn("w:rPr")), slot) if direct: return direct + if faces := table_style_faces(run, slot): + raise LookupError( + f"table style can override {slot} with {faces}; resolve the applicable " + "tblStylePr region or require rendered inspection" + ) # Header/footer runs belong to a HeaderPart/FooterPart, which has no .document. # Resolve the owning document's Normal style once rather than via run.part. for style in (run.style, run._parent.style, normal_style): diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/read.md b/plugins/Hylouis233/document-skills/skills/docx/references/read.md index ebf9045..09148e4 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/read.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/read.md @@ -79,7 +79,12 @@ def table_content(table): rows = [] for row in table.rows: rendered_cells = [] - column = 0 + row_properties = row._tr.find(qn("w:trPr")) + grid_before_node = None if row_properties is None else row_properties.find(qn("w:gridBefore")) + grid_after_node = None if row_properties is None else row_properties.find(qn("w:gridAfter")) + grid_before = 0 if grid_before_node is None else int(grid_before_node.get(qn("w:val"), "0")) + grid_after = 0 if grid_after_node is None else int(grid_after_node.get(qn("w:val"), "0")) + column = grid_before # row.cells repeats a merge-origin proxy for every grid position it spans. # Walk physical w:tc elements and expose the merge structure instead. for cell_element in row._tr.tc_lst: @@ -99,7 +104,11 @@ def table_content(table): "tables": nested_tables, }) column += colspan - rows.append(rendered_cells) + rows.append({ + "grid_before": grid_before, + "cells": rendered_cells, + "grid_after": grid_after, + }) return rows doc = Document("input.docx") @@ -121,8 +130,10 @@ Notes: tables, and `Paragraph.runs` omits runs wrapped by inline content controls and other containers. Use the XML-backed traversal above and report the content-control count. It stops descending when a table is yielded, so table text is not also emitted as prose; `table_content()` handles - nested tables recursively and emits `column`, `colspan`, and `vMerge` metadata for physical - cells instead of duplicating merge-origin text through `row.cells`. + nested tables recursively and emits `grid_before`, `grid_after`, `column`, `colspan`, and + `vMerge` metadata for physical cells instead of duplicating merge-origin text through + `row.cells`. The row-level grid omissions are required for nonuniform tables whose cells do + not start in logical column zero or do not extend to the final grid column. Imported `w:altChunk` HTML/RTF/document parts are not modeled by python-docx; the traversal reports their relationship target and content type as `unreadable` instead of silently presenting an incomplete extraction. Convert them with a trusted office renderer before diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md b/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md index 3602695..983bf7b 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md @@ -10,6 +10,30 @@ if doc.needs_pass: if doc.authenticate(password) <= 0: raise RuntimeError("Encrypted PDF: set a valid PDF_PASSWORD before inspection") +def font_inventory(document, page): + """Distinguish fonts with extractable programs from referenced-only faces.""" + fonts = [] + for entry in page.get_fonts(full=True): + xref, extension, font_type, base_name, resource_name, encoding = entry[:6] + embedded_bytes = 0 + if xref > 0: + try: + extracted = document.extract_font(xref) + embedded_bytes = len(extracted[3] or b"") + except (RuntimeError, ValueError): + embedded_bytes = 0 + fonts.append({ + "xref": xref, + "base_name": base_name, + "resource_name": resource_name, + "type": font_type, + "encoding": encoding, + "extension": extension, + "embedded": embedded_bytes > 0, + "embedded_bytes": embedded_bytes, + }) + return fonts + print("pages:", doc.page_count) print("password_protected:", doc.needs_pass, "| still_encrypted:", doc.is_encrypted, "| pdf:", doc.is_pdf) @@ -22,6 +46,7 @@ for page in doc: "images:", len(page.get_images()), "drawings:", len(drawings), "widgets:", len(widgets), "annotations:", len(annotations), "links:", len(links)) + print(" fonts:", font_inventory(doc, page)) ``` ## Checks worth automating @@ -30,8 +55,9 @@ for page in doc: links are all absent. Interactive form fields are widgets rather than page text, so the three-content-stream predicate alone would misclassify a usable form page as blank. A blank page after generation usually means an overflowing flowable created it. -- **Font inventory**: `page.get_fonts()` lists embedded names - needed when the user reports - "looks different on machine X". +- **Font inventory**: `page.get_fonts()` lists referenced fonts, including non-embedded base + fonts. Use `doc.extract_font(xref)` as above and report `embedded` separately; a referenced + face with no extractable program may be substituted on another machine. - **Page size consistency**: mixed `page.rect` sizes in one file break duplex printing; report it rather than silently normalizing. - **Damage**: `fitz.open` on a corrupt file raises or yields garbage - pair with diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md b/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md index ba04dd3..96d0726 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md @@ -40,6 +40,10 @@ Watermark / stamp by merging a stamp page onto each page: ```python from pypdf import Transformation +def require(condition, message): + if not condition: + raise ValueError(message) + stamp = open_pdf("watermark.pdf").pages[0] stamp.transfer_rotation_to_content() stamp_text = (stamp.extract_text() or "").strip() @@ -71,12 +75,17 @@ with open("watermarked.pdf", "wb") as f: writer.write(f) check = PdfReader("watermarked.pdf") -assert len(check.pages) == len(expected_sizes) -assert [tuple(float(value) for value in page.mediabox) for page in check.pages] == expected_sizes +require(len(check.pages) == len(expected_sizes), "watermarking changed the page count") +require( + [tuple(float(value) for value in page.mediabox) for page in check.pages] == expected_sizes, + "watermarking changed page geometry", +) if expected_fields: - assert set(expected_fields) <= set(check.get_fields() or {}), "watermarking dropped form fields" + require(set(expected_fields) <= set(check.get_fields() or {}), + "watermarking dropped form fields") if stamp_text: - assert all(stamp_text in (page.extract_text() or "") for page in check.pages) + require(all(stamp_text in (page.extract_text() or "") for page in check.pages), + "watermark text is missing from one or more pages") ``` If the stamp is graphical, render every output page and visually confirm that it is present; @@ -97,7 +106,8 @@ Encryption and forms: reader = open_pdf("form.pdf") fields = reader.get_fields() or {} - assert fields, "this PDF has no AcroForm form fields" + if not fields: + raise ValueError("this PDF has no AcroForm form fields") writer = PdfWriter() writer.append(reader) # clones every page AND the catalog /AcroForm into the writer @@ -130,7 +140,8 @@ Encryption and forms: check = PdfReader("filled.pdf") value = str((check.get_fields() or {}).get("applicant_name", {}).get("/V", "")) - assert value.strip("/") == "Ada Byron" + if value.strip("/") != "Ada Byron": + raise ValueError(f"filled field did not round-trip: {value!r}") ``` A freshly constructed `PdfWriter` is empty: `append` (or `clone_document_from_reader`) must diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md index f2604bb..c762193 100644 --- a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md @@ -6,6 +6,10 @@ from pptx import Presentation from pptx.enum.shapes import MSO_SHAPE_TYPE from pptx.oxml.ns import qn +from lxml import etree + +DIAGRAM_NS = "http://schemas.openxmlformats.org/drawingml/2006/diagram" +SAFE_XML = etree.XMLParser(load_dtd=False, resolve_entities=False, no_network=True) def iter_shapes(shapes): """Walk shapes recursively so content nested inside group shapes is counted too.""" @@ -85,6 +89,26 @@ def chart_axis_titles(chart): titles[label] = axis.axis_title.text_frame.text if axis.has_title else "" return titles +def smartart_content(shape): + """Extract SmartArt data-part labels, or report why the diagram is unreadable.""" + graphic_data = shape._element.find(".//" + qn("a:graphicData")) + if graphic_data is None or graphic_data.get("uri") != DIAGRAM_NS: + return None + rel_ids = graphic_data.find(f".//{{{DIAGRAM_NS}}}relIds") + relationship_id = None if rel_ids is None else rel_ids.get(qn("r:dm")) + if not relationship_id: + return {"name": shape.name, "status": "unreadable", "reason": "missing data relationship"} + try: + data_part = shape.part.related_part(relationship_id) + except (KeyError, ValueError): + return {"name": shape.name, "status": "unreadable", "reason": relationship_id} + try: + root = etree.fromstring(data_part.blob, parser=SAFE_XML) + except etree.XMLSyntaxError as error: + return {"name": shape.name, "status": "unreadable", "reason": str(error)} + labels = [node.text for node in root.iter(qn("a:t")) if node.text] + return {"name": shape.name, "status": "ok", "text": labels} + prs = Presentation("input.pptx") print("slide size:", prs.slide_width, prs.slide_height) for i, slide in enumerate(prs.slides): @@ -124,6 +148,10 @@ for i, slide in enumerate(prs.slides): content for sh in shapes if (content := picture_content(sh)) is not None ] + smartart = [ + content for sh in shapes + if (content := smartart_content(sh)) is not None + ] # These full values - not only counts or lengths - are the evidence for summaries and # repurposing. Keep notes verbatim so markdown output can preserve them as blockquotes. @@ -132,6 +160,7 @@ for i, slide in enumerate(prs.slides): print(" tables:", tables) print(" charts:", charts) print(" pictures:", pictures) + print(" smartart:", smartart) print(" notes:", notes) ``` diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md b/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md index 8483c87..43f2af9 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md @@ -66,8 +66,16 @@ output_path, *expected_sheets = sys.argv[1:] expected_number_formats = { # "Sales": {"D2": "#,##0.00", "E2": "yyyy-mm-dd"}, } +# Populate every formula the task intends to create or preserve. +expected_formulas = { + # "Sales": {"D2": "=C2*1.08"}, +} wb = openpyxl.load_workbook(output_path) +def require(condition, message): + if not condition: + raise ValueError(message) + def formula_text(value): if isinstance(value, str): return value @@ -81,10 +89,16 @@ def formula_text(value): print("sheets:", wb.sheetnames) missing = set(expected_sheets) - set(wb.sheetnames) -assert not missing, f"missing expected sheets: {sorted(missing)}" +require(not missing, f"missing expected sheets: {sorted(missing)}") calc = wb.calculation print("calcMode:", getattr(calc, "calcMode", None), "fullCalcOnLoad:", getattr(calc, "fullCalcOnLoad", None)) +if any(expected_formulas.values()): + require( + getattr(calc, "fullCalcOnLoad", False) is True + or getattr(calc, "calcMode", None) == "auto", + "formula output is not configured to recalculate in spreadsheet viewers", + ) for ws in wb.worksheets: print(f"{ws.title} dims:", ws.dimensions) formulas = [ @@ -92,11 +106,18 @@ for ws in wb.worksheets: for row in ws.iter_rows() for c in row if c.data_type == "f" ] print(f"{ws.title} formula cells:", formulas[:10]) + actual_formulas = dict(formulas) + for coordinate, expected_formula in expected_formulas.get(ws.title, {}).items(): + require( + actual_formulas.get(coordinate) == expected_formula, + f"{ws.title}!{coordinate}: expected formula {expected_formula!r}, " + f"got {actual_formulas.get(coordinate)!r}", + ) for coordinate, expected_format in expected_number_formats.get(ws.title, {}).items(): actual_format = ws[coordinate].number_format - assert actual_format == expected_format, ( + require(actual_format == expected_format, ( f"{ws.title}!{coordinate}: expected format {expected_format!r}, got {actual_format!r}" - ) + )) wb.close() ``` diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md index bb17015..90c894f 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md @@ -5,7 +5,7 @@ import openpyxl from datetime import date from openpyxl.formula import Tokenizer from openpyxl.styles import Font -from openpyxl.utils.cell import range_boundaries +from openpyxl.utils.cell import coordinate_to_tuple, range_boundaries wb = openpyxl.load_workbook("input.xlsx") # NOT data_only: that would drop all formulas ws = wb["Data"] @@ -26,6 +26,18 @@ def defined_name_values(workbook): # openpyxl 3.1 uses DefinedNameDict; 3.0 uses DefinedNameList. Support both. return names.values() if hasattr(names, "values") else names.definedName +def drawing_anchor_rows(drawing): + """Return 1-based rows occupied by a string/one-cell/two-cell anchor.""" + anchor = drawing.anchor + if isinstance(anchor, str): + return (coordinate_to_tuple(anchor)[0],) + rows = [] + if marker := getattr(anchor, "_from", None): + rows.append(marker.row + 1) + if marker := getattr(anchor, "to", None): + rows.append(marker.row + 1) + return tuple(rows) + def structural_references(workbook): """Inventory formulas/ranges that insert_rows/delete_rows will not rewrite.""" refs = [] @@ -62,9 +74,12 @@ def structural_references(workbook): for formula in getattr(rule, "formula", ()): refs.append(("conditional formatting formula", owner, str(formula))) for index, chart in enumerate(sheet._charts, start=1): + refs.append(("drawing anchor", f"{owner} chart {index}", drawing_anchor_rows(chart))) for element in chart._write().iter(): if element.tag.rsplit("}", 1)[-1] == "f" and element.text: refs.append(("chart series", f"{owner} chart {index}", element.text)) + for index, image in enumerate(sheet._images, start=1): + refs.append(("drawing anchor", f"{owner} image {index}", drawing_anchor_rows(image))) return refs def non_cell_references(workbook): @@ -165,6 +180,11 @@ print("cell formulas before:", cell_formulas_before) print("cell formulas after planned rewrites:", cell_formula_references(wb)) print("non-cell references after planned rewrites:", non_cell_references(wb)) +# Formula caches are not calculated by openpyxl. Force spreadsheet viewers to +# recalculate every edited formula instead of preserving a manual/stale input mode. +wb.calculation.fullCalcOnLoad = True +wb.calculation.forceFullCalc = True +wb.calculation.calcMode = "auto" wb.save("input-edited.xlsx") ``` diff --git a/plugins/Hylouis233/document-skills/tests/docx_fixture.py b/plugins/Hylouis233/document-skills/tests/docx_fixture.py index 70aeb2f..7fb4f49 100644 --- a/plugins/Hylouis233/document-skills/tests/docx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/docx_fixture.py @@ -12,6 +12,7 @@ import fitz from docx import Document +from docx.enum.style import WD_STYLE_TYPE from docx.opc.constants import RELATIONSHIP_TYPE as RT from docx.oxml import OxmlElement from docx.oxml.ns import qn @@ -161,6 +162,12 @@ def table_content(table): rows = [] for row in table.rows: rendered_cells = [] + row_properties = row._tr.find(qn("w:trPr")) + grid_before_node = None if row_properties is None else row_properties.find(qn("w:gridBefore")) + grid_after_node = None if row_properties is None else row_properties.find(qn("w:gridAfter")) + grid_before = 0 if grid_before_node is None else int(grid_before_node.get(qn("w:val"), "0")) + grid_after = 0 if grid_after_node is None else int(grid_after_node.get(qn("w:val"), "0")) + column = grid_before for cell in row.cells: items = [] for kind, block in iter_part_blocks(cell._tc, cell): @@ -170,8 +177,11 @@ def table_content(table): items.append(table_content(block)) else: items.append(block) - rendered_cells.append(items) - rows.append(rendered_cells) + rendered_cells.append({"column": column, "items": items}) + column += 1 + rows.append({ + "grid_before": grid_before, "cells": rendered_cells, "grid_after": grid_after, + }) return rows @@ -189,6 +199,13 @@ def wrap_in_sdt(element): wrap_in_sdt(sdt_paragraph._p) controlled_table = sdt_doc.add_table(rows=1, cols=1) controlled_table.cell(0, 0).text = "table한" +row_properties = controlled_table.rows[0]._tr.get_or_add_trPr() +grid_before = OxmlElement("w:gridBefore") +grid_before.set(qn("w:val"), "1") +row_properties.append(grid_before) +grid_after = OxmlElement("w:gridAfter") +grid_after.set(qn("w:val"), "2") +row_properties.append(grid_after) nested_table = controlled_table.cell(0, 0).add_table(rows=1, cols=1) nested_table.cell(0, 0).text = "nested한" wrap_in_sdt(controlled_table._tbl) @@ -228,6 +245,11 @@ def wrap_in_sdt(element): sum(item.count("table한") for item in all_emitted) == 1, all_emitted) check("nested-table text is emitted exactly once", sum(item.count("nested한") for item in all_emitted) == 1, all_emitted) +check("nonuniform table rows preserve leading/trailing grid omissions", + rendered_tables[0][0]["grid_before"] == 1 + and rendered_tables[0][0]["cells"][0]["column"] == 1 + and rendered_tables[0][0]["grid_after"] == 2, + rendered_tables[0]) unreadable_parts = [block for kind, block in walked_blocks if kind == "unreadable"] check("altChunk content is reported instead of silently omitted", unreadable_parts == [{ @@ -297,10 +319,32 @@ def style_faces(style, slot): style = style.base_style +def table_style_faces(run, slot): + element = run._r + while element is not None and element.tag != qn("w:tbl"): + element = element.getparent() + if element is None: + return [] + table_properties = element.find(qn("w:tblPr")) + table_style = None if table_properties is None else table_properties.find(qn("w:tblStyle")) + style_id = None if table_style is None else table_style.get(qn("w:val")) + style = None if not style_id else font_doc.styles.get_by_id(style_id, WD_STYLE_TYPE.TABLE) + faces = [] + while style is not None: + rprs = [style.element.find(qn("w:rPr"))] + rprs.extend(region.find(qn("w:rPr")) + for region in style.element.findall(qn("w:tblStylePr"))) + faces.extend(face for rpr in rprs if (face := face_from_rpr(rpr, slot))) + style = style.base_style + return list(dict.fromkeys(faces)) + + def effective_face(run, slot): direct = face_from_rpr(run._r.find(qn("w:rPr")), slot) if direct: return direct + if faces := table_style_faces(run, slot): + raise LookupError(f"conditional table style face requires rendered resolution: {faces}") for style in (run.style, run._parent.style, normal_style): if face := next(style_faces(style, slot), None): return face @@ -337,6 +381,26 @@ def effective_face(run, slot): check("hyperlink CJK text resolves through the east-Asian font slot", effective_face(walked_hyperlink_runs[0], "eastAsia") == "CJK Face") +conditional_style = font_doc.styles.add_style("Conditional CJK Table", WD_STYLE_TYPE.TABLE) +first_row = OxmlElement("w:tblStylePr") +first_row.set(qn("w:type"), "firstRow") +conditional_rpr = OxmlElement("w:rPr") +conditional_fonts = OxmlElement("w:rFonts") +conditional_fonts.set(qn("w:eastAsia"), "Conditional CJK Face") +conditional_rpr.append(conditional_fonts) +first_row.append(conditional_rpr) +conditional_style.element.append(first_row) +font_table = font_doc.add_table(rows=1, cols=1) +font_table.style = conditional_style +conditional_run = font_table.cell(0, 0).paragraphs[0].add_run("漢") +try: + effective_face(conditional_run, "eastAsia") + conditional_style_rejected = False +except LookupError: + conditional_style_rejected = True +check("glyph audit fails closed for conditional table-style fonts", + conditional_style_rejected) + # ---- edit.md guarded cross-run replacement ------------------------------------ SAFE_RUN_CHILDREN = { qn("w:rPr"), qn("w:t"), qn("w:tab"), qn("w:cr"), diff --git a/plugins/Hylouis233/document-skills/tests/fixtures.test.mjs b/plugins/Hylouis233/document-skills/tests/fixtures.test.mjs new file mode 100644 index 0000000..e3ffa77 --- /dev/null +++ b/plugins/Hylouis233/document-skills/tests/fixtures.test.mjs @@ -0,0 +1,34 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { mkdtemp, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { promisify } from "node:util"; +import { fileURLToPath } from "node:url"; + +const execFileAsync = promisify(execFile); +const testsRoot = path.dirname(fileURLToPath(import.meta.url)); +const python = process.env.PYTHON || (process.platform === "win32" ? "python" : "python3"); + +test("document-skills Python fixtures", { timeout: 12 * 60_000 }, async (context) => { + for (const name of ["xlsx", "pptx", "pdf", "docx"]) { + await context.test(name + " fixture", { timeout: 3 * 60_000 }, async () => { + const scratch = await mkdtemp(path.join(os.tmpdir(), "document-skills-" + name + "-")); + try { + await execFileAsync(python, [path.join(testsRoot, name + "_fixture.py")], { + cwd: scratch, + encoding: "utf8", + maxBuffer: 8 * 1024 * 1024, + timeout: 170_000, + windowsHide: true, + }); + } catch (error) { + const output = String(error.stdout ?? "") + "\n" + String(error.stderr ?? ""); + assert.fail(name + " fixture failed:\n" + output.slice(-16_000)); + } finally { + await rm(scratch, { recursive: true, force: true }); + } + }); + } +}); diff --git a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py index a5d1b5d..60c64f6 100644 --- a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py @@ -10,7 +10,10 @@ import fitz import pypdf +import reportlab from reportlab.lib.pagesizes import A4 +from reportlab.pdfbase import pdfmetrics +from reportlab.pdfbase.ttfonts import TTFont from reportlab.pdfgen import canvas failures = [] @@ -53,6 +56,40 @@ def open_pdf(path, password=None): page_sizes, ) +# ---- inspect.md distinguishes referenced-only and embedded fonts --------------- +pdfmetrics.registerFont(TTFont("FixtureVera", os.path.join( + os.path.dirname(reportlab.__file__), "fonts", "Vera.ttf", +))) +font_canvas = canvas.Canvas("font-inventory.pdf", pagesize=A4) +font_canvas.setFont("Helvetica", 12) # standard PDF face, normally referenced only +font_canvas.drawString(72, 780, "Referenced Helvetica") +font_canvas.setFont("FixtureVera", 12) +font_canvas.drawString(72, 750, "Embedded Vera") +font_canvas.save() + +def font_inventory(document, page): + fonts = [] + for entry in page.get_fonts(full=True): + xref, extension, font_type, base_name, resource_name, encoding = entry[:6] + embedded_bytes = 0 + if xref > 0: + try: + embedded_bytes = len((document.extract_font(xref)[3] or b"")) + except (RuntimeError, ValueError): + embedded_bytes = 0 + fonts.append({ + "base_name": base_name, "embedded": embedded_bytes > 0, + "embedded_bytes": embedded_bytes, + }) + return fonts + +font_doc = fitz.open("font-inventory.pdf") +fonts = font_inventory(font_doc, font_doc[0]) +check("font inventory labels the embedded TrueType face", + any("Vera" in item["base_name"] and item["embedded"] for item in fonts), fonts) +check("font inventory labels referenced-only Helvetica as non-embedded", + any("Helvetica" in item["base_name"] and not item["embedded"] for item in fonts), fonts) + # ---- SKILL.md postcheck: encrypted output is reopened with its password ------- encrypted_writer = pypdf.PdfWriter() encrypted_writer.append(r) diff --git a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py index de04829..e96a0c9 100644 --- a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py @@ -312,6 +312,30 @@ def chart_axis_titles(chart): return titles +DIAGRAM_NS = "http://schemas.openxmlformats.org/drawingml/2006/diagram" +SAFE_DIAGRAM_XML = etree.XMLParser(load_dtd=False, resolve_entities=False, no_network=True) + + +def smartart_content(shape): + graphic_data = shape._element.find(".//" + qn("a:graphicData")) + if graphic_data is None or graphic_data.get("uri") != DIAGRAM_NS: + return None + rel_ids = graphic_data.find(f".//{{{DIAGRAM_NS}}}relIds") + relationship_id = None if rel_ids is None else rel_ids.get(qn("r:dm")) + if not relationship_id: + return {"name": shape.name, "status": "unreadable", "reason": "missing data relationship"} + try: + data_part = shape.part.related_part(relationship_id) + except (KeyError, ValueError): + return {"name": shape.name, "status": "unreadable", "reason": relationship_id} + try: + root = etree.fromstring(data_part.blob, parser=SAFE_DIAGRAM_XML) + except etree.XMLSyntaxError as error: + return {"name": shape.name, "status": "unreadable", "reason": str(error)} + labels = [node.text for node in root.iter(qn("a:t")) if node.text] + return {"name": shape.name, "status": "ok", "text": labels} + + def extract_slide_content(slide): shapes = list(iter_shapes(slide.shapes)) text = [sh.text_frame.text for sh in shapes if sh.has_text_frame and sh.text_frame.text] @@ -347,8 +371,12 @@ def extract_slide_content(slide): info for shape in shapes if (info := picture_content(shape)) is not None ] + smartart = [ + info for shape in shapes + if (info := smartart_content(shape)) is not None + ] return {"text": text, "tables": tables, "charts": charts, - "pictures": pictures, "notes": notes} + "pictures": pictures, "smartart": smartart, "notes": notes} content = extract_slide_content(Presentation("input.pptx").slides[0]) @@ -379,6 +407,36 @@ def extract_slide_content(slide): == [(0, 5.0)]) check("content inventory emits notes text", "regional split" in content["notes"], content["notes"]) +diagram_frame = etree.fromstring(f''' + +'''.encode()) +diagram_data = f''' + Revenue +'''.encode() +diagram_part = type("DiagramPart", (), {"blob": diagram_data})() +diagram_owner = type("SlidePart", (), { + "related_part": lambda self, relationship_id: {"rIdSmart": diagram_part}[relationship_id], +})() +diagram_shape = type("SmartArtShape", (), { + "name": "SmartArt 1", "_element": diagram_frame, "part": diagram_owner, +})() +check("SmartArt inventory extracts text from the diagram data part", + smartart_content(diagram_shape) + == {"name": "SmartArt 1", "status": "ok", "text": ["Revenue"]}) +missing_diagram_shape = type("SmartArtShape", (), { + "name": "Broken SmartArt", "_element": diagram_frame, + "part": type("SlidePart", (), { + "related_part": lambda self, relationship_id: {}[relationship_id], + })(), +})() +check("SmartArt inventory reports an unresolved diagram relationship", + smartart_content(missing_diagram_shape)["status"] == "unreadable") + placeholder_png = base64.b64decode( "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9ZlRYAAAAASUVORK5CYII=" ) diff --git a/plugins/Hylouis233/document-skills/tests/requirements-fixtures.txt b/plugins/Hylouis233/document-skills/tests/requirements-fixtures.txt new file mode 100644 index 0000000..fb549f2 --- /dev/null +++ b/plugins/Hylouis233/document-skills/tests/requirements-fixtures.txt @@ -0,0 +1,8 @@ +lxml==6.0.2 +openpyxl==3.1.5 +Pillow==11.3.0 +pymupdf==1.27.2.2 +pypdf==6.9.2 +python-docx==1.2.0 +python-pptx==1.0.2 +reportlab==4.4.10 diff --git a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py index caf5b69..7aaac53 100644 --- a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py @@ -5,6 +5,7 @@ # python pptx_fixture.py (deps: python-pptx) # python xlsx_fixture.py (deps: openpyxl) # python docx_fixture.py (deps: python-docx, pymupdf, soffice on PATH) +import base64 import csv import sys import zipfile @@ -218,9 +219,10 @@ def sheet_ref(sheet): # ---- edit.md structural audit includes non-cell dependencies ------------------ from openpyxl.chart import BarChart, Reference +from openpyxl.drawing.image import Image from openpyxl.formula import Tokenizer from openpyxl.formatting.rule import CellIsRule, ColorScaleRule, DataBarRule, FormulaRule -from openpyxl.utils.cell import range_boundaries +from openpyxl.utils.cell import coordinate_to_tuple, range_boundaries from openpyxl.workbook.defined_name import DefinedName from openpyxl.worksheet.datavalidation import DataValidation from openpyxl.worksheet.table import Table @@ -243,6 +245,18 @@ def defined_name_values(workbook): return names.values() if hasattr(names, "values") else names.definedName +def drawing_anchor_rows(drawing): + anchor = drawing.anchor + if isinstance(anchor, str): + return (coordinate_to_tuple(anchor)[0],) + rows = [] + if marker := getattr(anchor, "_from", None): + rows.append(marker.row + 1) + if marker := getattr(anchor, "to", None): + rows.append(marker.row + 1) + return tuple(rows) + + def structural_references(workbook): refs = [] for item in defined_name_values(workbook): @@ -278,9 +292,12 @@ def structural_references(workbook): for formula in getattr(rule, "formula", ()): refs.append(("conditional formatting formula", owner, str(formula))) for index, chart in enumerate(sheet._charts, start=1): + refs.append(("drawing anchor", f"{owner} chart {index}", drawing_anchor_rows(chart))) for element in chart._write().iter(): if element.tag.rsplit("}", 1)[-1] == "f" and element.text: refs.append(("chart series", f"{owner} chart {index}", element.text)) + for index, image in enumerate(sheet._images, start=1): + refs.append(("drawing anchor", f"{owner} image {index}", drawing_anchor_rows(image))) return refs @@ -369,6 +386,11 @@ class LegacyWorkbook: chart = BarChart() chart.add_data(Reference(audit_ws, min_col=1, min_row=1, max_row=3), titles_from_data=True) audit_ws.add_chart(chart, "C1") +with open("anchor.png", "wb") as stream: + stream.write(base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9ZlRYAAAAASUVORK5CYII=" + )) +audit_ws.add_image(Image("anchor.png"), "E5") audit_references = structural_references(audit_wb) reference_kinds = {kind for kind, _, _ in audit_references} formula_references = cell_formula_references(audit_wb) @@ -377,9 +399,11 @@ class LegacyWorkbook: {"defined name", "cell formula", "table", "merged range", "auto filter", "print area", "print title rows", "print title columns", "data validation range", "data validation formula", "conditional formatting range", - "conditional formatting formula", "chart series"} <= reference_kinds, + "conditional formatting formula", "chart series", "drawing anchor"} <= reference_kinds, reference_kinds, ) +check("drawing audit records an image anchored at the insertion row", + ("drawing anchor", "Audit image 1", (5,)) in audit_references, audit_references) check( "structural audit snapshots ordinary cell formulas before row insertion", formula_references == [("cell formula", "Audit", "C1", "=SUM(A2:A3)")], @@ -423,10 +447,16 @@ class LegacyWorkbook: ] if not safe_dependencies: safe_ws.insert_rows(5) + safe_wb.calculation.fullCalcOnLoad = True + safe_wb.calculation.forceFullCalc = True + safe_wb.calculation.calcMode = "auto" safe_wb.save("audited-structural-edit.xlsx") safe_reopened = openpyxl.load_workbook("audited-structural-edit.xlsx", data_only=False) check("audited non-intersecting formula path reaches save", safe_reopened["Data"]["D2"].value == "=C2*1.08") +check("formula edit forces recalculation after save", + safe_reopened.calculation.fullCalcOnLoad is True + and safe_reopened.calculation.calcMode == "auto") legacy_name = DefinedName("LegacyName", attr_text="Audit!$A$1") legacy_names = type("LegacyDefinedNames", (), {"definedName": [legacy_name]})() diff --git a/test/hosted-plugins.test.mjs b/test/hosted-plugins.test.mjs index 4e58656..400ac3f 100644 --- a/test/hosted-plugins.test.mjs +++ b/test/hosted-plugins.test.mjs @@ -36,7 +36,7 @@ test('contributor can scaffold a hosted Skill plugin with one command', async (c assert.match(readme, /# Hello World/u); assert.match(license, /Apache License/u); assert.match(skill, /^---\nname: hello-world\n/mu); - assert.match(stdout, /plugins\/alice\/hello-world/u); + assert.match(stdout, /plugins[\\/]alice[\\/]hello-world/u); }); test('hosted Plugin is valid when its package and contribution docs are complete', async (context) => { From beade83f436f3d53197e6ba7fb9191c1cbda35dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=98=E6=89=AC?= Date: Sun, 16 Aug 2026 23:31:09 +0800 Subject: [PATCH 21/34] fix(document-skills): cover remaining review edge cases --- .../skills/docx/references/read.md | 7 +- .../skills/pdf/references/transform.md | 74 +++++++++-- .../skills/xlsx/references/edit.md | 4 +- .../document-skills/tests/docx_fixture.py | 36 ++++-- .../document-skills/tests/pdf_fixture.py | 121 +++++++++++++++--- .../document-skills/tests/xlsx_fixture.py | 12 +- 6 files changed, 211 insertions(+), 43 deletions(-) diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/read.md b/plugins/Hylouis233/document-skills/skills/docx/references/read.md index 09148e4..ba2c9c9 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/read.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/read.md @@ -94,14 +94,18 @@ def table_content(table): vertical = None if cell_properties is None else cell_properties.find(qn("w:vMerge")) vertical_merge = None if vertical is None else vertical.get(qn("w:val"), "continue") nested_tables = [] + unreadable = [] for kind, block in iter_part_blocks(cell_element, table): if kind == "table": nested_tables.append(table_content(block)) + elif kind == "unreadable": + unreadable.append(block) rendered_cells.append({ "column": column, "colspan": colspan, "vMerge": vertical_merge, "text": tc_text(cell_element, table), "tables": nested_tables, + "unreadable": unreadable, }) column += colspan rows.append({ @@ -137,7 +141,8 @@ Notes: Imported `w:altChunk` HTML/RTF/document parts are not modeled by python-docx; the traversal reports their relationship target and content type as `unreadable` instead of silently presenting an incomplete extraction. Convert them with a trusted office renderer before - claiming their content was read. + claiming their content was read. Each table-cell record carries its own `unreadable` list so + an import nested in a cell is not lost from the report. Text boxes, headers, footers, and footnotes still require their own collections (`section.header/.footer`) or raw XML. - For revision/comment metadata, inspect the XML parts directly: `word/comments.xml`, diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md b/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md index 96d0726..bbc017e 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md @@ -39,6 +39,63 @@ Watermark / stamp by merging a stamp page onto each page: ```python from pypdf import Transformation +from pypdf.generic import RectangleObject + +def rotation_transfer(page): + """Map default user space to the visible coordinates used after /Rotate.""" + media = RectangleObject(page.mediabox) + transform = ( + Transformation() + .translate( + -float(media.left + media.width / 2), + -float(media.bottom + media.height / 2), + ) + .rotate(-page.rotation) + ) + corners = [ + transform.apply_on(point) + for point in (media.lower_left, media.lower_right, media.upper_left, media.upper_right) + ] + return transform.translate( + -min(point[0] for point in corners), + -min(point[1] for point in corners), + ) + +def inverse_transformation(transform): + a, b, c, d, e, f = map(float, transform.ctm) + determinant = a * d - b * c + return Transformation(( + d / determinant, -b / determinant, + -c / determinant, a / determinant, + (c * f - d * e) / determinant, + (b * e - a * f) / determinant, + )) + +def transformed_rectangle(rectangle, transform): + rectangle = RectangleObject(rectangle) + corners = [ + transform.apply_on(point) + for point in ( + rectangle.lower_left, rectangle.lower_right, + rectangle.upper_left, rectangle.upper_right, + ) + ] + return RectangleObject(( + min(point[0] for point in corners), min(point[1] for point in corners), + max(point[0] for point in corners), max(point[1] for point in corners), + )) + +def stamp_placement(page, stamp_box): + """Fit in visible space, then map the stamp back without changing the destination page.""" + to_visual = rotation_transfer(page) + destination = transformed_rectangle(page.cropbox, to_visual) + sw, sh = float(stamp_box.width), float(stamp_box.height) + dw, dh = float(destination.width), float(destination.height) + scale = min(dw / sw, dh / sh) + tx = float(destination.left) + (dw - sw * scale) / 2 - float(stamp_box.left) * scale + ty = float(destination.bottom) + (dh - sh * scale) / 2 - float(stamp_box.bottom) * scale + visible_placement = Transformation().scale(scale).translate(tx, ty) + return visible_placement.transform(inverse_transformation(to_visual)) def require(condition, message): if not condition: @@ -54,20 +111,13 @@ writer.append(reader) # clone pages plus catalog entries such as # A plain merge_page() overlays the stamp in its own coordinates, so on a page # with different dimensions, origin, or rotation the stamp can be clipped or -# land entirely off-page. Normalize only the stamp, then scale each copy to the -# destination crop box and include both boxes' non-zero origins. Keep the -# destination page's /Rotate value intact: transfer_rotation_to_content() does -# not transform annotation rectangles, so normalizing a page that has widgets, -# links, or other annotations can displace its interactive geometry. +# land entirely off-page. Normalize only the stamp, fit each copy in the visible +# (rotation-aware) crop box, then map that placement back into default user space. +# This keeps the destination /Rotate, boxes, annotation /Rect values, and widget +# appearance streams intact while keeping the watermark horizontal to the viewer. stamp_box = stamp.cropbox -sw, sh = float(stamp_box.width), float(stamp_box.height) for page in writer.pages: - destination = page.cropbox - dw, dh = float(destination.width), float(destination.height) - scale = min(dw / sw, dh / sh) - tx = float(destination.left) + (dw - sw * scale) / 2 - float(stamp_box.left) * scale - ty = float(destination.bottom) + (dh - sh * scale) / 2 - float(stamp_box.bottom) * scale - page.merge_transformed_page(stamp, Transformation().scale(scale).translate(tx, ty)) + page.merge_transformed_page(stamp, stamp_placement(page, stamp_box)) expected_sizes = [tuple(float(value) for value in page.mediabox) for page in writer.pages] diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md index 90c894f..fdcabda 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md @@ -112,7 +112,9 @@ def formula_may_intersect_rows(owner_sheet, formula, shifted_sheet, start_row): unmodeled_reference_functions = {"indirect", "offset", "address"} if any( token.type == "FUNC" and token.subtype == "OPEN" - and token.value.rstrip("(").casefold() in unmodeled_reference_functions + and token.value.rstrip("(").rsplit(":", 1)[-1] + .lstrip("@").rsplit(".", 1)[-1].casefold() + in unmodeled_reference_functions for token in tokens ): return True diff --git a/plugins/Hylouis233/document-skills/tests/docx_fixture.py b/plugins/Hylouis233/document-skills/tests/docx_fixture.py index 7fb4f49..5bc1a4c 100644 --- a/plugins/Hylouis233/document-skills/tests/docx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/docx_fixture.py @@ -14,6 +14,8 @@ from docx import Document from docx.enum.style import WD_STYLE_TYPE from docx.opc.constants import RELATIONSHIP_TYPE as RT +from docx.opc.packuri import PackURI +from docx.opc.part import Part from docx.oxml import OxmlElement from docx.oxml.ns import qn from docx.table import Table @@ -213,14 +215,19 @@ def wrap_in_sdt(element): inline_run = inline_paragraph.add_run("inline한") inline_paragraph.add_run("-after") wrap_in_sdt(inline_run._r) -chunk_relationship = sdt_doc.part.relate_to( - "https://example.invalid/imported.html", - "http://schemas.openxmlformats.org/officeDocument/2006/relationships/aFChunk", - is_external=True, +chunk_part = Part( + PackURI("/word/altChunk1.html"), "text/html", + b"IMPORTED ALTCHUNK TEXT", sdt_doc.part.package, ) +chunk_relationship = sdt_doc.part.relate_to(chunk_part, RT.A_F_CHUNK) alt_chunk = OxmlElement("w:altChunk") alt_chunk.set(qn("r:id"), chunk_relationship) sdt_doc.element.body.insert(len(sdt_doc.element.body) - 1, alt_chunk) +cell_alt_chunk = OxmlElement("w:altChunk") +cell_alt_chunk.set(qn("r:id"), chunk_relationship) +controlled_table.cell(0, 0)._tc.insert( + len(controlled_table.cell(0, 0)._tc) - 1, cell_alt_chunk +) sdt_doc.save("content-control.docx") sdt_reopened = Document("content-control.docx") check("doc.paragraphs omits block content-control text (negative control)", @@ -251,13 +258,22 @@ def wrap_in_sdt(element): and rendered_tables[0][0]["grid_after"] == 2, rendered_tables[0]) unreadable_parts = [block for kind, block in walked_blocks if kind == "unreadable"] +expected_alt_chunk = { + "kind": "altChunk", + "relationship_id": chunk_relationship, + "target": "altChunk1.html", + "content_type": "text/html", +} check("altChunk content is reported instead of silently omitted", - unreadable_parts == [{ - "kind": "altChunk", - "relationship_id": chunk_relationship, - "target": "https://example.invalid/imported.html", - "content_type": None, - }], unreadable_parts) + unreadable_parts == [expected_alt_chunk], unreadable_parts) +cell_unreadable = [ + item for item in rendered_tables[0][0]["cells"][0]["items"] + if isinstance(item, dict) and item.get("kind") == "altChunk" +] +check("altChunk content inside a table cell is also reported", + cell_unreadable == [expected_alt_chunk], rendered_tables) +check("altChunk payload is not misrepresented as extracted paragraph text", + all("IMPORTED ALTCHUNK TEXT" not in text for text in walked_text), walked_text) # Per-run glyph validation must not let a different referenced font hide a missing glyph. fixture_cmaps = {"CJK Face": {ord("漢")}, "Latin Face": {ord("A")}} diff --git a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py index 60c64f6..f6156de 100644 --- a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py @@ -278,6 +278,65 @@ def widget_field_name(widget): # ---- transform.md watermark snippet ------------------------------------------- from pypdf import PdfReader as R2, Transformation +from pypdf.generic import RectangleObject + + +def rotation_transfer(page): + media = RectangleObject(page.mediabox) + transform = ( + Transformation() + .translate( + -float(media.left + media.width / 2), + -float(media.bottom + media.height / 2), + ) + .rotate(-page.rotation) + ) + corners = [ + transform.apply_on(point) + for point in (media.lower_left, media.lower_right, media.upper_left, media.upper_right) + ] + return transform.translate( + -min(point[0] for point in corners), + -min(point[1] for point in corners), + ) + + +def inverse_transformation(transform): + a, b, c, d, e, f = map(float, transform.ctm) + determinant = a * d - b * c + return Transformation(( + d / determinant, -b / determinant, + -c / determinant, a / determinant, + (c * f - d * e) / determinant, + (b * e - a * f) / determinant, + )) + + +def transformed_rectangle(rectangle, transform): + rectangle = RectangleObject(rectangle) + corners = [ + transform.apply_on(point) + for point in ( + rectangle.lower_left, rectangle.lower_right, + rectangle.upper_left, rectangle.upper_right, + ) + ] + return RectangleObject(( + min(point[0] for point in corners), min(point[1] for point in corners), + max(point[0] for point in corners), max(point[1] for point in corners), + )) + + +def stamp_placement(page, stamp_box): + to_visual = rotation_transfer(page) + destination = transformed_rectangle(page.cropbox, to_visual) + sw, sh = float(stamp_box.width), float(stamp_box.height) + dw, dh = float(destination.width), float(destination.height) + scale = min(dw / sw, dh / sh) + tx = float(destination.left) + (dw - sw * scale) / 2 - float(stamp_box.left) * scale + ty = float(destination.bottom) + (dh - sh * scale) / 2 - float(stamp_box.bottom) * scale + visible_placement = Transformation().scale(scale).translate(tx, ty) + return visible_placement.transform(inverse_transformation(to_visual)) stamp_src = canvas.Canvas("stamp.pdf", pagesize=A4) stamp_src.setFont("Helvetica", 40) @@ -294,16 +353,7 @@ def widget_field_name(widget): writer.append(reader) stamp_box = stamp.cropbox for page in writer.pages: - destination = page.cropbox - scale = min(float(destination.width) / float(stamp_box.width), - float(destination.height) / float(stamp_box.height)) - tx = (float(destination.left) - + (float(destination.width) - float(stamp_box.width) * scale) / 2 - - float(stamp_box.left) * scale) - ty = (float(destination.bottom) - + (float(destination.height) - float(stamp_box.height) * scale) / 2 - - float(stamp_box.bottom) * scale) - page.merge_transformed_page(stamp, Transformation().scale(scale).translate(tx, ty)) + page.merge_transformed_page(stamp, stamp_placement(page, stamp_box)) expected_sizes = [(round(float(p.mediabox.width), 2), round(float(p.mediabox.height), 2)) for p in writer.pages] with open("watermarked.pdf", "wb") as f: @@ -448,12 +498,16 @@ def widget_field_name(widget): offset_page.cropbox.upper_right = (300, 500) rotated_page = small_source.add_blank_page(width=240, height=160) rotated_page.rotate(90) -from pypdf.generic import ArrayObject, DictionaryObject, FloatObject, NameObject +from pypdf.generic import ArrayObject, DictionaryObject, FloatObject, NameObject, TextStringObject rotated_link = DictionaryObject({ NameObject("/Type"): NameObject("/Annot"), NameObject("/Subtype"): NameObject("/Link"), NameObject("/Rect"): ArrayObject([FloatObject(value) for value in (20, 30, 100, 60)]), NameObject("/Border"): ArrayObject([FloatObject(0), FloatObject(0), FloatObject(0)]), + NameObject("/A"): DictionaryObject({ + NameObject("/S"): NameObject("/URI"), + NameObject("/URI"): TextStringObject("https://example.invalid/rotated-link"), + }), }) rotated_page[NameObject("/Annots")] = ArrayObject([small_source._add_object(rotated_link)]) cropped_page = small_source.add_blank_page(width=400, height=500) @@ -488,6 +542,15 @@ def stamp_bboxes(path, page_number): spans.append(span["bbox"]) return spans +def stamp_line_directions(path, page_number): + doc = fitz.open(path) + return [ + line["dir"] + for block in doc[page_number].get_text("dict")["blocks"] + for line in block.get("lines", []) + if any("DRAFT" in span["text"] for span in line["spans"]) + ] + # The plain merge keeps the A4 stamp's coordinates, so on the 200x300 page the # stamp is far outside the box: PyMuPDF's positioned extraction sees no span at # all, while pypdf's plain extractor still returns the text - proof that text @@ -526,12 +589,7 @@ def stamp_bboxes(path, page_number): stamp_box2 = stamp_page.cropbox sw2, sh2 = float(stamp_box2.width), float(stamp_box2.height) for page in scaled_writer.pages: - destination = page.cropbox - dw, dh = float(destination.width), float(destination.height) - scale = min(dw / sw2, dh / sh2) - tx = float(destination.left) + (dw - sw2 * scale) / 2 - float(stamp_box2.left) * scale - ty = float(destination.bottom) + (dh - sh2 * scale) / 2 - float(stamp_box2.bottom) * scale - page.merge_transformed_page(stamp_page, Transformation().scale(scale).translate(tx, ty)) + page.merge_transformed_page(stamp_page, stamp_placement(page, stamp_box2)) with open("scaled-stamped.pdf", "wb") as f: scaled_writer.write(f) scaled_check = R2("scaled-stamped.pdf") @@ -545,7 +603,24 @@ def stamp_bboxes(path, page_number): rotated_spans = stamp_bboxes("scaled-stamped.pdf", 4) cropped_spans = stamp_bboxes("scaled-stamped.pdf", 5) check("scaled stamp lands inside the non-zero-origin page", bool(offset_spans), offset_spans) -check("scaled stamp lands inside the rotated page without normalizing it", bool(rotated_spans), rotated_spans) +rotated_fitz_page = fitz.open("scaled-stamped.pdf")[4] +rotated_visible_spans = [ + fitz.Rect(bbox) * rotated_fitz_page.rotation_matrix for bbox in rotated_spans +] +rotated_visible_directions = [ + ( + direction[0] * rotated_fitz_page.rotation_matrix.a + + direction[1] * rotated_fitz_page.rotation_matrix.c, + direction[0] * rotated_fitz_page.rotation_matrix.b + + direction[1] * rotated_fitz_page.rotation_matrix.d, + ) + for direction in stamp_line_directions("scaled-stamped.pdf", 4) +] +check("scaled stamp stays horizontal in the rotated page's visible space", + bool(rotated_spans) + and all(rect.width > rect.height for rect in rotated_visible_spans) + and all(dx > 0.9 and abs(dy) < 0.1 for dx, dy in rotated_visible_directions), + (rotated_visible_spans, rotated_visible_directions)) cropped_rect = fitz.open("scaled-stamped.pdf")[5].rect check("scaled stamp lands inside the offset visible crop box", bool(cropped_spans) @@ -565,6 +640,16 @@ def stamp_bboxes(path, page_number): == rotated_geometry_before, ((scaled_rotated_page.rotation, first_annotation_rect(scaled_rotated_page)), rotated_geometry_before)) +mixed_link_before = tuple( + round(float(value), 4) + for value in fitz.open("mixed.pdf")[4].get_links()[0]["from"] +) +mixed_link_after = tuple( + round(float(value), 4) + for value in fitz.open("scaled-stamped.pdf")[4].get_links()[0]["from"] +) +check("watermarking preserves the rotated link's visible hit rectangle", + mixed_link_after == mixed_link_before, (mixed_link_before, mixed_link_after)) # ---- SKILL.md overflow check: off-page text is a defect ------------------------- diff --git a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py index 7aaac53..54ba259 100644 --- a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py @@ -324,7 +324,9 @@ def formula_may_intersect_rows(owner_sheet, formula, shifted_sheet, start_row): unmodeled_reference_functions = {"indirect", "offset", "address"} if any( token.type == "FUNC" and token.subtype == "OPEN" - and token.value.rstrip("(").casefold() in unmodeled_reference_functions + and token.value.rstrip("(").rsplit(":", 1)[-1] + .lstrip("@").rsplit(".", 1)[-1].casefold() + in unmodeled_reference_functions for token in tokens ): return True @@ -421,6 +423,14 @@ class LegacyWorkbook: formula_may_intersect_rows("Data", '=SUM(INDIRECT("A5:A6"))', "Data", 5)) check("OFFSET numeric row references require a manual structural rewrite plan", formula_may_intersect_rows("Data", "=SUM(OFFSET(A1,4,0,2,1))", "Data", 5)) +check("implicit-intersection INDIRECT references require a manual rewrite plan", + formula_may_intersect_rows("Data", '=@INDIRECT("A5:A6")', "Data", 5)) +check("OFFSET used by the range operator requires a manual rewrite plan", + formula_may_intersect_rows("Data", "=SUM(A1:OFFSET(A1,5,0))", "Data", 5)) +check("INDIRECT text inside a string does not create a dynamic reference", + not formula_may_intersect_rows( + "Data", '=IF(A1="INDIRECT(A5:A6)",1,0)', "Data", 5 + )) stale_wb = openpyxl.Workbook() stale_ws = stale_wb.active From 032a97a94c34a78e5426d6b05b018ca61b2d2924 Mon Sep 17 00:00:00 2001 From: Hylouis233 Date: Sun, 16 Aug 2026 23:49:31 +0800 Subject: [PATCH 22/34] fix(document-skills): address post-gate review findings --- .../skills/docx/references/cjk.md | 6 ++- .../skills/docx/references/review.md | 6 +++ .../document-skills/skills/pdf/SKILL.md | 10 +++-- .../skills/pdf/references/transform.md | 16 +++++--- .../skills/xlsx/references/csv.md | 21 ++++++++-- .../skills/xlsx/references/formatting.md | 17 +++++++- .../document-skills/tests/docx_fixture.py | 19 +++++++++ .../document-skills/tests/pdf_fixture.py | 40 ++++++++++++++++++- .../document-skills/tests/xlsx_fixture.py | 34 +++++++++++++++- 9 files changed, 150 insertions(+), 19 deletions(-) diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md b/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md index a2a68d8..346099a 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md @@ -222,8 +222,10 @@ section.left_margin, section.right_margin = Cm(2.8), Cm(2.6) cmap = cmaps[font_spec] if ord(character) not in cmap: missing.append((run_index, character, face)) - assert not unresolved, f"font files not resolved per run: {unresolved}" - assert not missing, f"glyph missing from the run's effective font: {missing}" + if unresolved: + raise RuntimeError(f"font files not resolved per run: {unresolved}") + if missing: + raise RuntimeError(f"glyph missing from the run's effective font: {missing}") ``` When font files cannot be located, rasterize the rendered pages with PyMuPDF and inspect diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/review.md b/plugins/Hylouis233/document-skills/skills/docx/references/review.md index 8e5c1c8..10e2487 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/review.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/review.md @@ -14,6 +14,7 @@ Fonts render differently on another machine | non-embedded fonts | expected; rep ```python import zipfile +from pathlib import Path from lxml import etree path = "input.docx" @@ -21,6 +22,8 @@ MAX_XML_PART = 20 * 1024 * 1024 MAX_ENTRY = 100 * 1024 * 1024 MAX_TOTAL_UNCOMPRESSED = 500 * 1024 * 1024 MAX_COMPRESSION_RATIO = 200 +MAX_COMPRESSED_FILE = 200 * 1024 * 1024 +MAX_MEMBERS = 10_000 # Security limits must survive `python -O` (which strips assert statements), # so every check raises explicitly instead of asserting. @@ -35,8 +38,11 @@ safe_xml_parser = etree.XMLParser( huge_tree=False, recover=False, ) +require(Path(path).stat().st_size <= MAX_COMPRESSED_FILE, + "compressed DOCX file size above limit") with zipfile.ZipFile(path) as z: infos = z.infolist() + require(len(infos) <= MAX_MEMBERS, "too many archive members") names = {info.filename for info in infos} require(len(names) == len(infos), "duplicate archive member names are unsafe") require("[Content_Types].xml" in names and "word/document.xml" in names, diff --git a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md index 1f5d39c..325c690 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md @@ -58,9 +58,13 @@ output_path = "output.pdf" password = os.environ.get("PDF_PASSWORD") r = pypdf.PdfReader(output_path) if r.is_encrypted: - if not password: - raise RuntimeError("set PDF_PASSWORD so the encrypted output can be postchecked") - r = pypdf.PdfReader(output_path, password=password) # wrong passwords fail here + # Permission-encrypted PDFs commonly use an empty user password and open normally. + # Authenticate that case before requiring an operator-supplied password. + if r.decrypt("") == 0: + if not password: + raise RuntimeError("set PDF_PASSWORD so the encrypted output can be postchecked") + if r.decrypt(password) == 0: + raise RuntimeError("PDF_PASSWORD could not decrypt the output") def require(condition, message): # Mandatory verification must remain active under python -O. diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md b/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md index bbc017e..a6ab33b 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md @@ -163,15 +163,21 @@ Encryption and forms: writer.append(reader) # clones every page AND the catalog /AcroForm into the writer def widget_field_name(widget): - """Resolve /T on a widget or its parent field dictionary.""" + """Resolve the fully qualified /T name from every field ancestor.""" + parts = [] + seen = set() while widget is not None: + object_id = id(widget) + if object_id in seen: + raise ValueError("cycle in AcroForm field parent chain") + seen.add(object_id) if widget.get("/T") is not None: - return str(widget["/T"]) + parts.append(str(widget["/T"])) parent = widget.get("/Parent") widget = None if parent is None else parent.get_object() - return None + return ".".join(reversed(parts)) or None - field_name = "applicant_name" + field_name = "application.applicant_name" # use the key exposed by get_fields() target_pages = [ page for page in writer.pages if any( @@ -189,7 +195,7 @@ Encryption and forms: writer.write(f) check = PdfReader("filled.pdf") - value = str((check.get_fields() or {}).get("applicant_name", {}).get("/V", "")) + value = str((check.get_fields() or {}).get(field_name, {}).get("/V", "")) if value.strip("/") != "Ada Byron": raise ValueError(f"filled field did not round-trip: {value!r}") ``` diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md index e37000d..9d0f369 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md @@ -67,6 +67,18 @@ with open("output.csv", "w", newline="", encoding="utf-8") as f: formula_wb = openpyxl.load_workbook("input.xlsx", read_only=True, data_only=False) value_wb = openpyxl.load_workbook("input.xlsx", read_only=True, data_only=True) formula_ws, value_ws = formula_wb["Data"], value_wb["Data"] + # Do not trust a producer's cached declaration. In read-only mode a + # plausible-but-truncated dimension otherwise hides populated cells from iter_rows(). + formula_ws.reset_dimensions() + value_ws.reset_dimensions() + + def spreadsheet_safe_csv_value(value): + # Spreadsheet-targeted CSV is the safe default. A leading apostrophe prevents + # Excel/LibreOffice from evaluating literal text as a formula when opened. + if isinstance(value, str) and value.startswith(("=", "+", "-", "@")): + return "'" + value + return value + missing_caches = [] output_path = Path("output.csv") temporary_path = output_path.with_suffix(output_path.suffix + ".tmp") @@ -76,7 +88,7 @@ with open("output.csv", "w", newline="", encoding="utf-8") as f: for formula_cell, value_cell in zip(formula_row, value_row): if formula_cell.data_type == "f" and value_cell.value is None: missing_caches.append(formula_cell.coordinate) - writer.writerow([cell.value for cell in value_row]) + writer.writerow([spreadsheet_safe_csv_value(cell.value) for cell in value_row]) formula_wb.close() value_wb.close() if missing_caches: @@ -86,8 +98,11 @@ with open("output.csv", "w", newline="", encoding="utf-8") as f: ``` Format numbers yourself only if the user needs a fixed display format; otherwise write raw - cached values and say so. Export formula text from the `data_only=False` workbook only when - the user explicitly requests formulas rather than displayed values. + cached values and say so. The snippet neutralizes formula-like literal text because its CSV is + intended for spreadsheet applications. Preserve such prefixes unchanged only in an explicitly + requested raw-data export, and warn that opening that raw CSV in a spreadsheet is unsafe. + Export formula text from the `data_only=False` workbook only when the user explicitly requests + formulas rather than displayed values. - Large CSV -> keep it CSV or move to SQLite/Parquet; loading it all into one sheet to "preserve" it usually exceeds limits and helps nobody. diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/formatting.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/formatting.md index 69742bd..b560f1d 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/formatting.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/formatting.md @@ -11,7 +11,13 @@ from openpyxl.styles import Font, PatternFill red_fill = PatternFill(start_color="FFC7CE", end_color="FFC7CE", fill_type="solid") red_font = Font(color="9C0006") -last = ws.max_row # real data boundary, not the column +# Derive the boundary from values in the rule/table source columns. max_row can +# include a styled or formerly cleared cell near Excel's row limit. +last = next( + (row for row in range(ws.max_row, 1, -1) + if any(ws.cell(row, column).value is not None for column in range(1, 7))), + 1, +) # With only a header row (max_row == 1) every range below would be inverted # ("D2:D1"); openpyxl rejects those ranges, so guard before building rules. @@ -50,7 +56,14 @@ A real Table gives filter UI, banded styling, and structured references: ```python from openpyxl.worksheet.table import Table, TableStyleInfo -tbl = Table(displayName="TData", ref=f"A1:F{ws.max_row}") # name has no spaces +last = next( + (row for row in range(ws.max_row, 1, -1) + if any(ws.cell(row, column).value is not None for column in range(1, 7))), + 1, +) +if last < 2: + raise ValueError("cannot create a data table without populated rows") +tbl = Table(displayName="TData", ref=f"A1:F{last}") # name has no spaces tbl.tableStyleInfo = TableStyleInfo(name="TableStyleMedium9", showRowStripes=True) ws.add_table(tbl) ``` diff --git a/plugins/Hylouis233/document-skills/tests/docx_fixture.py b/plugins/Hylouis233/document-skills/tests/docx_fixture.py index 5bc1a4c..cf057a7 100644 --- a/plugins/Hylouis233/document-skills/tests/docx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/docx_fixture.py @@ -9,6 +9,7 @@ import subprocess import sys import zipfile +from pathlib import Path import fitz from docx import Document @@ -52,6 +53,8 @@ def check(name, cond, extra=""): MAX_ENTRY = 100 * 1024 * 1024 MAX_TOTAL_UNCOMPRESSED = 500 * 1024 * 1024 MAX_COMPRESSION_RATIO = 200 +MAX_COMPRESSED_FILE = 200 * 1024 * 1024 +MAX_MEMBERS = 10_000 def require(condition, message): @@ -60,8 +63,11 @@ def require(condition, message): def validate_docx_package(path): + require(Path(path).stat().st_size <= MAX_COMPRESSED_FILE, + "compressed DOCX file size above limit") with zipfile.ZipFile(path) as archive: infos = archive.infolist() + require(len(infos) <= MAX_MEMBERS, "too many archive members") names = {info.filename for info in infos} require(len(names) == len(infos), "duplicate archive member names are unsafe") require( @@ -124,6 +130,19 @@ def validate_docx_package(path): __debug__ or archive_bomb_rejected, ) +with zipfile.ZipFile("too-many-members.docx", "w", zipfile.ZIP_STORED) as archive: + archive.writestr("[Content_Types].xml", "") + archive.writestr("word/document.xml", "") + for index in range(MAX_MEMBERS - 1): + archive.writestr(f"custom/empty-{index}", b"") +try: + validate_docx_package("too-many-members.docx") + excessive_member_count_rejected = False +except ValueError: + excessive_member_count_rejected = True +check("bounded package health check rejects excessive member counts", + excessive_member_count_rejected) + # ---- read.md includes block/inline content controls and controlled tables ------- def iter_part_blocks(root, parent): for child in root.iterchildren(): diff --git a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py index f6156de..8336b0d 100644 --- a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py @@ -100,6 +100,19 @@ def font_inventory(document, page): check("encrypted fixture is detected before page access", probe.is_encrypted) encrypted_r = pypdf.PdfReader("encrypted.pdf", password="fixture-password") check("password-authenticated postcheck can access every page", len(encrypted_r.pages) == 2) + +blank_password_writer = pypdf.PdfWriter() +blank_password_writer.append(r) +blank_password_writer.encrypt("", owner_password="fixture-owner-password") +with open("blank-password-encrypted.pdf", "wb") as f: + blank_password_writer.write(f) +blank_password_reader = pypdf.PdfReader("blank-password-encrypted.pdf") +if blank_password_reader.is_encrypted and blank_password_reader.decrypt("") == 0: + blank_password_opened = False +else: + blank_password_opened = len(blank_password_reader.pages) == 2 +check("postcheck accepts permission encryption with an empty user password", + blank_password_opened) try: open_pdf("encrypted.pdf") transform_rejected_missing_password = False @@ -223,12 +236,35 @@ def widget_count(page): def widget_field_name(widget): + parts = [] + seen = set() while widget is not None: + object_id = id(widget) + if object_id in seen: + raise ValueError("cycle in AcroForm field parent chain") + seen.add(object_id) if widget.get("/T") is not None: - return str(widget["/T"]) + parts.append(str(widget["/T"])) parent = widget.get("/Parent") widget = None if parent is None else parent.get_object() - return None + return ".".join(reversed(parts)) or None + + +class DirectObjectReference: + def __init__(self, value): + self.value = value + + def get_object(self): + return self.value + + +field_parent = {"/T": "application"} +hierarchical_widget = { + "/T": "applicant_name", + "/Parent": DirectObjectReference(field_parent), +} +check("widget lookup resolves a fully qualified hierarchical field name", + widget_field_name(hierarchical_widget) == "application.applicant_name") page2_writer = PdfWriter() diff --git a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py index 54ba259..5667ec2 100644 --- a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py @@ -563,6 +563,15 @@ class LegacyWorkbook: and "0x" not in data_table_entry[2], data_table_entry) # csv.md: value export uses the cached-value workbook and reports every missing cache. +formula_wb["First"].reset_dimensions() +value_wb["First"].reset_dimensions() + +def spreadsheet_safe_csv_value(value): + if isinstance(value, str) and value.startswith(("=", "+", "-", "@")): + return "'" + value + return value + + missing_caches = [] with open("formula-values.csv", "w", newline="", encoding="utf-8") as output: writer = csv.writer(output) @@ -570,12 +579,24 @@ class LegacyWorkbook: for formula_cell, value_cell in zip(formula_row, value_row): if formula_cell.data_type == "f" and value_cell.value is None: missing_caches.append(formula_cell.coordinate) - writer.writerow([cell.value for cell in value_row]) + writer.writerow([spreadsheet_safe_csv_value(cell.value) for cell in value_row]) with open("formula-values.csv", newline="", encoding="utf-8") as exported: exported_values = [value for row in csv.reader(exported) for value in row] check("XLSX-to-CSV reports formulas with no cached value", set(missing_caches) >= {"A1", "A2"}, missing_caches) check("XLSX-to-CSV does not leak formula strings into value output", not any(value.startswith("=") for value in exported_values), exported_values) + +formula_like_literals = ["=1+1", "+cmd", "-2+3", "@SUM(A1:A2)", -5] +with open("formula-like-literals.csv", "w", newline="", encoding="utf-8") as output: + csv.writer(output).writerow([ + spreadsheet_safe_csv_value(value) for value in formula_like_literals + ]) +with open("formula-like-literals.csv", newline="", encoding="utf-8") as exported: + safe_literals = next(csv.reader(exported)) +check("spreadsheet-targeted CSV neutralizes every formula-like text prefix", + safe_literals[:4] == ["'=1+1", "'+cmd", "'-2+3", "'@SUM(A1:A2)"], safe_literals) +check("CSV neutralization leaves numeric negative values numeric", + safe_literals[4] == "-5", safe_literals) formula_wb.close() value_wb.close() @@ -634,7 +655,11 @@ def markers_in(data): # ---- formatting.md guard: header-only sheets skip conditional formatting --------- def add_demo_formatting(sheet): - last = sheet.max_row + last = next( + (row for row in range(sheet.max_row, 1, -1) + if any(sheet.cell(row, column).value is not None for column in range(1, 7))), + 1, + ) if last < 2: return 0 sheet.conditional_formatting.add( @@ -675,7 +700,12 @@ def add_demo_formatting(sheet): data_ws = data_wb.active data_ws.append(["A", "B", "C", "D", "E", "F"]) data_ws.append([1, 2, 3, -1, 5, 6]) +data_ws["F10000"].number_format = "0.00" # inflate max_row without adding data check("data rows receive all four formatting rules", add_demo_formatting(data_ws) == 4) +formatting_ranges = [str(item.sqref) for item in data_ws.conditional_formatting] +check("conditional formatting ignores a styled ghost row", + all("10000" not in item and item.endswith("2") for item in formatting_ranges), + formatting_ranges) data_wb.save("data-formatting.xlsx") data_reopened = openpyxl.load_workbook("data-formatting.xlsx") check("all four formatting rules survive save/reopen", From 6b799c57912f64b320bb186a1bbfb032295c4c28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=98=E6=89=AC?= Date: Sun, 16 Aug 2026 23:53:27 +0800 Subject: [PATCH 23/34] fix(document-skills): close post-push review gaps --- .../skills/docx/references/cjk.md | 5 +- .../skills/docx/references/review.md | 10 +- .../document-skills/skills/pdf/SKILL.md | 3 +- .../skills/pdf/references/transform.md | 13 +- .../skills/xlsx/references/csv.md | 56 +++++--- .../skills/xlsx/references/formatting.md | 37 +++--- .../document-skills/tests/docx_fixture.py | 60 +++++++-- .../document-skills/tests/pdf_fixture.py | 103 ++++++++++----- .../document-skills/tests/xlsx_fixture.py | 120 +++++++++++++----- 9 files changed, 284 insertions(+), 123 deletions(-) diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md b/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md index 346099a..ea90d16 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md @@ -222,10 +222,11 @@ section.left_margin, section.right_margin = Cm(2.8), Cm(2.6) cmap = cmaps[font_spec] if ord(character) not in cmap: missing.append((run_index, character, face)) + # These are mandatory audit gates, so raise explicitly: `python -O` removes asserts. if unresolved: - raise RuntimeError(f"font files not resolved per run: {unresolved}") + raise ValueError(f"font files not resolved per run: {unresolved}") if missing: - raise RuntimeError(f"glyph missing from the run's effective font: {missing}") + raise ValueError(f"glyph missing from the run's effective font: {missing}") ``` When font files cannot be located, rasterize the rendered pages with PyMuPDF and inspect diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/review.md b/plugins/Hylouis233/document-skills/skills/docx/references/review.md index 10e2487..24fe912 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/review.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/review.md @@ -18,12 +18,12 @@ from pathlib import Path from lxml import etree path = "input.docx" +MAX_ARCHIVE_BYTES = 200 * 1024 * 1024 +MAX_MEMBERS = 10_000 MAX_XML_PART = 20 * 1024 * 1024 MAX_ENTRY = 100 * 1024 * 1024 MAX_TOTAL_UNCOMPRESSED = 500 * 1024 * 1024 MAX_COMPRESSION_RATIO = 200 -MAX_COMPRESSED_FILE = 200 * 1024 * 1024 -MAX_MEMBERS = 10_000 # Security limits must survive `python -O` (which strips assert statements), # so every check raises explicitly instead of asserting. @@ -38,11 +38,13 @@ safe_xml_parser = etree.XMLParser( huge_tree=False, recover=False, ) -require(Path(path).stat().st_size <= MAX_COMPRESSED_FILE, +# Check the package itself before ZipFile materializes its central directory. +require(Path(path).stat().st_size <= MAX_ARCHIVE_BYTES, "compressed DOCX file size above limit") with zipfile.ZipFile(path) as z: infos = z.infolist() - require(len(infos) <= MAX_MEMBERS, "too many archive members") + # Check the count before building sets, summing sizes, or opening any member. + require(len(infos) <= MAX_MEMBERS, "archive member count above limit") names = {info.filename for info in infos} require(len(names) == len(infos), "duplicate archive member names are unsafe") require("[Content_Types].xml" in names and "word/document.xml" in names, diff --git a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md index 325c690..55ce728 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md @@ -58,8 +58,7 @@ output_path = "output.pdf" password = os.environ.get("PDF_PASSWORD") r = pypdf.PdfReader(output_path) if r.is_encrypted: - # Permission-encrypted PDFs commonly use an empty user password and open normally. - # Authenticate that case before requiring an operator-supplied password. + # Permission-encrypted PDFs commonly have an empty user password and open normally. if r.decrypt("") == 0: if not password: raise RuntimeError("set PDF_PASSWORD so the encrypted output can be postchecked") diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md b/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md index a6ab33b..45d2d3e 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md @@ -163,7 +163,7 @@ Encryption and forms: writer.append(reader) # clones every page AND the catalog /AcroForm into the writer def widget_field_name(widget): - """Resolve the fully qualified /T name from every field ancestor.""" + """Resolve the fully qualified /T from a widget and its field ancestors.""" parts = [] seen = set() while widget is not None: @@ -171,13 +171,14 @@ Encryption and forms: if object_id in seen: raise ValueError("cycle in AcroForm field parent chain") seen.add(object_id) - if widget.get("/T") is not None: - parts.append(str(widget["/T"])) + partial_name = widget.get("/T") + if partial_name is not None: + parts.append(str(partial_name)) parent = widget.get("/Parent") widget = None if parent is None else parent.get_object() - return ".".join(reversed(parts)) or None + return ".".join(reversed(parts)) if parts else None - field_name = "application.applicant_name" # use the key exposed by get_fields() + field_name = "applicant_name" # use the exact simple/qualified key exposed by get_fields() target_pages = [ page for page in writer.pages if any( @@ -195,7 +196,7 @@ Encryption and forms: writer.write(f) check = PdfReader("filled.pdf") - value = str((check.get_fields() or {}).get(field_name, {}).get("/V", "")) + value = str((check.get_fields() or {}).get("applicant_name", {}).get("/V", "")) if value.strip("/") != "Ada Byron": raise ValueError(f"filled field did not round-trip: {value!r}") ``` diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md index 9d0f369..8a8a3d2 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md @@ -36,12 +36,31 @@ with open("input.csv", newline="", encoding="utf-8-sig") as f: # utf-8-sig str ```python import csv +FORMULA_PREFIXES = ("=", "+", "-", "@") + +def spreadsheet_csv_field(value, *, mode="safe"): + if mode not in {"safe", "raw"}: + raise ValueError("mode must be 'safe' or 'raw'") + if mode == "safe" and isinstance(value, str) and value.startswith(FORMULA_PREFIXES): + return "'" + value + return value + with open("output.csv", "w", newline="", encoding="utf-8") as f: writer = csv.writer(f) - writer.writerow(["Region", "Units"]) - writer.writerow(["EU", 120]) + rows = [["Region", "Units", "Note"], ["EU", 120, "=2+2"]] + writer.writerows([spreadsheet_csv_field(value) for value in row] for row in rows) ``` +Use `mode="safe"` (the default above) when the CSV will be opened in Excel, LibreOffice, +Google Sheets, or another spreadsheet application. It neutralizes literal text beginning with +`=`, `+`, `-`, or `@` by prefixing an apostrophe, so the application does not interpret the +field as a formula. Numeric values, including negative numbers represented as numbers, are not +changed. This protection deliberately changes those serialized string values. + +Use `mode="raw"` only when the user explicitly requires byte-for-value interchange with a +trusted machine consumer. Raw mode preserves the exact strings and provides **no spreadsheet +formula-injection protection**; do not present a raw export as safe to open in a spreadsheet. + ## Converting - CSV -> XLSX: read with `csv`, write with openpyxl; convert values to real types on the way @@ -64,21 +83,23 @@ with open("output.csv", "w", newline="", encoding="utf-8") as f: import openpyxl from pathlib import Path + FORMULA_PREFIXES = ("=", "+", "-", "@") + + def spreadsheet_csv_field(value, *, mode="safe"): + if mode not in {"safe", "raw"}: + raise ValueError("mode must be 'safe' or 'raw'") + if mode == "safe" and isinstance(value, str) and value.startswith(FORMULA_PREFIXES): + return "'" + value + return value + + export_mode = "safe" # use "raw" only for explicitly requested trusted machine interchange formula_wb = openpyxl.load_workbook("input.xlsx", read_only=True, data_only=False) value_wb = openpyxl.load_workbook("input.xlsx", read_only=True, data_only=True) formula_ws, value_ws = formula_wb["Data"], value_wb["Data"] - # Do not trust a producer's cached declaration. In read-only mode a - # plausible-but-truncated dimension otherwise hides populated cells from iter_rows(). + # Producer-written metadata can look plausible while truncating real cells. + # Reset both paired streams before their first iter_rows() call. formula_ws.reset_dimensions() value_ws.reset_dimensions() - - def spreadsheet_safe_csv_value(value): - # Spreadsheet-targeted CSV is the safe default. A leading apostrophe prevents - # Excel/LibreOffice from evaluating literal text as a formula when opened. - if isinstance(value, str) and value.startswith(("=", "+", "-", "@")): - return "'" + value - return value - missing_caches = [] output_path = Path("output.csv") temporary_path = output_path.with_suffix(output_path.suffix + ".tmp") @@ -88,7 +109,9 @@ with open("output.csv", "w", newline="", encoding="utf-8") as f: for formula_cell, value_cell in zip(formula_row, value_row): if formula_cell.data_type == "f" and value_cell.value is None: missing_caches.append(formula_cell.coordinate) - writer.writerow([spreadsheet_safe_csv_value(cell.value) for cell in value_row]) + writer.writerow([ + spreadsheet_csv_field(cell.value, mode=export_mode) for cell in value_row + ]) formula_wb.close() value_wb.close() if missing_caches: @@ -98,11 +121,8 @@ with open("output.csv", "w", newline="", encoding="utf-8") as f: ``` Format numbers yourself only if the user needs a fixed display format; otherwise write raw - cached values and say so. The snippet neutralizes formula-like literal text because its CSV is - intended for spreadsheet applications. Preserve such prefixes unchanged only in an explicitly - requested raw-data export, and warn that opening that raw CSV in a spreadsheet is unsafe. - Export formula text from the `data_only=False` workbook only when the user explicitly requests - formulas rather than displayed values. + cached values and say so. Export formula text from the `data_only=False` workbook only when + the user explicitly requests formulas rather than displayed values. - Large CSV -> keep it CSV or move to SQLite/Parquet; loading it all into one sheet to "preserve" it usually exceeds limits and helps nobody. diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/formatting.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/formatting.md index b560f1d..baff9fc 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/formatting.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/formatting.md @@ -11,18 +11,25 @@ from openpyxl.styles import Font, PatternFill red_fill = PatternFill(start_color="FFC7CE", end_color="FFC7CE", fill_type="solid") red_font = Font(color="9C0006") -# Derive the boundary from values in the rule/table source columns. max_row can -# include a styled or formerly cleared cell near Excel's row limit. -last = next( - (row for row in range(ws.max_row, 1, -1) - if any(ws.cell(row, column).value is not None for column in range(1, 7))), - 1, -) - -# With only a header row (max_row == 1) every range below would be inverted + +def last_populated_row(sheet, *, first_data_row=2, min_col=1, max_col=6): + # Conditional formatting uses a normal in-memory Worksheet. Its sparse cell store avoids + # iterating/materializing every row up to an inflated max_row. + populated_rows = ( + cell.row for cell in sheet._cells.values() + if first_data_row <= cell.row + and min_col <= cell.column <= max_col + and cell.value is not None + ) + return max(populated_rows, default=first_data_row - 1) + +# Do not use ws.max_row: a styled but empty cell can inflate it far below the data. +last = last_populated_row(ws) + +# With only a header row every range below would be inverted # ("D2:D1"); openpyxl rejects those ranges, so guard before building rules. if last < 2: - print(f"skipping conditional formatting: no data rows below the header (max_row={last})") + print("skipping conditional formatting: no populated data rows below the header") else: # value-based rule ws.conditional_formatting.add( @@ -44,6 +51,10 @@ else: ) ``` +Restrict the scan to the columns that define the data region. If the data is already a declared +Table, its ref is authoritative instead: use +`openpyxl.utils.cell.range_boundaries(ws.tables["TData"].ref)[3]` for `last`. + - FormulaRule formulas are US-locale and relative to the range's top-left cell - `$D2` (lock column, free row) is what makes the whole-row pattern work. - Multiple rules on one range evaluate by priority; if exactly one should apply, set @@ -56,11 +67,7 @@ A real Table gives filter UI, banded styling, and structured references: ```python from openpyxl.worksheet.table import Table, TableStyleInfo -last = next( - (row for row in range(ws.max_row, 1, -1) - if any(ws.cell(row, column).value is not None for column in range(1, 7))), - 1, -) +last = last_populated_row(ws) if last < 2: raise ValueError("cannot create a data table without populated rows") tbl = Table(displayName="TData", ref=f"A1:F{last}") # name has no spaces diff --git a/plugins/Hylouis233/document-skills/tests/docx_fixture.py b/plugins/Hylouis233/document-skills/tests/docx_fixture.py index cf057a7..ce03234 100644 --- a/plugins/Hylouis233/document-skills/tests/docx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/docx_fixture.py @@ -49,12 +49,12 @@ def check(name, cond, extra=""): check("DOCX XML parser leaves external entities unresolved", parsed.text is None and len(parsed) == 1) # ---- review.md health check rejects archive bombs before expanding parts ------- +MAX_ARCHIVE_BYTES = 200 * 1024 * 1024 +MAX_MEMBERS = 10_000 MAX_XML_PART = 20 * 1024 * 1024 MAX_ENTRY = 100 * 1024 * 1024 MAX_TOTAL_UNCOMPRESSED = 500 * 1024 * 1024 MAX_COMPRESSION_RATIO = 200 -MAX_COMPRESSED_FILE = 200 * 1024 * 1024 -MAX_MEMBERS = 10_000 def require(condition, message): @@ -63,11 +63,13 @@ def require(condition, message): def validate_docx_package(path): - require(Path(path).stat().st_size <= MAX_COMPRESSED_FILE, - "compressed DOCX file size above limit") + require( + Path(path).stat().st_size <= MAX_ARCHIVE_BYTES, + "compressed DOCX file size above limit", + ) with zipfile.ZipFile(path) as archive: infos = archive.infolist() - require(len(infos) <= MAX_MEMBERS, "too many archive members") + require(len(infos) <= MAX_MEMBERS, "archive member count above limit") names = {info.filename for info in infos} require(len(names) == len(infos), "duplicate archive member names are unsafe") require( @@ -130,18 +132,20 @@ def validate_docx_package(path): __debug__ or archive_bomb_rejected, ) -with zipfile.ZipFile("too-many-members.docx", "w", zipfile.ZIP_STORED) as archive: +with zipfile.ZipFile("many-members.docx", "w", zipfile.ZIP_STORED) as archive: archive.writestr("[Content_Types].xml", "") archive.writestr("word/document.xml", "") - for index in range(MAX_MEMBERS - 1): - archive.writestr(f"custom/empty-{index}", b"") + for member_index in range(MAX_MEMBERS - 1): + archive.writestr(f"word/zero-{member_index:05d}.bin", b"") try: - validate_docx_package("too-many-members.docx") - excessive_member_count_rejected = False -except ValueError: - excessive_member_count_rejected = True -check("bounded package health check rejects excessive member counts", - excessive_member_count_rejected) + validate_docx_package("many-members.docx") + many_members_rejected = False +except ValueError as exc: + many_members_rejected = str(exc) == "archive member count above limit" +check( + "member-count gate rejects 10,001 distinct zero-byte archive members before traversal", + many_members_rejected, +) # ---- read.md includes block/inline content controls and controlled tables ------- def iter_part_blocks(root, parent): @@ -305,6 +309,34 @@ def wrap_in_sdt(element): check("per-run cmap check identifies the actual missing glyph", per_run_missing == [("Latin Face", "漢")], per_run_missing) + +def require_clean_cjk_glyph_audit(unresolved, missing): + if unresolved: + raise ValueError(f"font files not resolved per run: {unresolved}") + if missing: + raise ValueError(f"glyph missing from the run's effective font: {missing}") + + +try: + require_clean_cjk_glyph_audit([(0, "漢", "Unresolved Face")], []) + unresolved_face_rejected = False +except ValueError: + unresolved_face_rejected = True +check( + "mandatory CJK audit rejects unresolved font files under optimized Python", + unresolved_face_rejected, +) + +try: + require_clean_cjk_glyph_audit([], [(0, "漢", "Missing Glyph Face")]) + missing_glyph_rejected = False +except ValueError: + missing_glyph_rejected = True +check( + "mandatory CJK audit rejects missing glyphs under optimized Python", + missing_glyph_rejected, +) + def font_slot(character): codepoint = ord(character) uses_east_asian_slot = ( diff --git a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py index 8336b0d..bbe653f 100644 --- a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py @@ -98,21 +98,29 @@ def font_inventory(document, page): encrypted_writer.write(f) probe = pypdf.PdfReader("encrypted.pdf") check("encrypted fixture is detected before page access", probe.is_encrypted) -encrypted_r = pypdf.PdfReader("encrypted.pdf", password="fixture-password") + + +def postcheck_reader(path, password=None): + reader = pypdf.PdfReader(path) + if reader.is_encrypted and reader.decrypt("") == 0: + if password is None or reader.decrypt(password) == 0: + raise RuntimeError(f"valid password required to postcheck {path}") + return reader + + +encrypted_r = postcheck_reader("encrypted.pdf", "fixture-password") check("password-authenticated postcheck can access every page", len(encrypted_r.pages) == 2) blank_password_writer = pypdf.PdfWriter() blank_password_writer.append(r) blank_password_writer.encrypt("", owner_password="fixture-owner-password") -with open("blank-password-encrypted.pdf", "wb") as f: +with open("blank-user-password.pdf", "wb") as f: blank_password_writer.write(f) -blank_password_reader = pypdf.PdfReader("blank-password-encrypted.pdf") -if blank_password_reader.is_encrypted and blank_password_reader.decrypt("") == 0: - blank_password_opened = False -else: - blank_password_opened = len(blank_password_reader.pages) == 2 -check("postcheck accepts permission encryption with an empty user password", - blank_password_opened) +blank_password_probe = pypdf.PdfReader("blank-user-password.pdf") +check("blank-user-password fixture still reports encryption", blank_password_probe.is_encrypted) +blank_password_r = postcheck_reader("blank-user-password.pdf") +check("postcheck tries the empty user password before requiring PDF_PASSWORD", + len(blank_password_r.pages) == 2) try: open_pdf("encrypted.pdf") transform_rejected_missing_password = False @@ -243,28 +251,12 @@ def widget_field_name(widget): if object_id in seen: raise ValueError("cycle in AcroForm field parent chain") seen.add(object_id) - if widget.get("/T") is not None: - parts.append(str(widget["/T"])) + partial_name = widget.get("/T") + if partial_name is not None: + parts.append(str(partial_name)) parent = widget.get("/Parent") widget = None if parent is None else parent.get_object() - return ".".join(reversed(parts)) or None - - -class DirectObjectReference: - def __init__(self, value): - self.value = value - - def get_object(self): - return self.value - - -field_parent = {"/T": "application"} -hierarchical_widget = { - "/T": "applicant_name", - "/Parent": DirectObjectReference(field_parent), -} -check("widget lookup resolves a fully qualified hierarchical field name", - widget_field_name(hierarchical_widget) == "application.applicant_name") + return ".".join(reversed(parts)) if parts else None page2_writer = PdfWriter() @@ -293,6 +285,59 @@ def get_object(self): check("page-2 field value round-trips", page2_value.strip("/") == "Ada on page 2", repr(page2_value)) +# A hierarchical field stores one partial /T at each level. Build a non-terminal +# `application` parent around the page-2 widget and address the terminal field by +# the fully qualified name returned by get_fields(). +from pypdf.generic import ArrayObject, DictionaryObject, NameObject, TextStringObject + +hierarchy_writer = PdfWriter() +hierarchy_writer.append(PdfReader("form-page2.pdf")) +hierarchy_widget_ref = hierarchy_writer.pages[1]["/Annots"][0] +hierarchy_widget = hierarchy_widget_ref.get_object() +hierarchy_parent = DictionaryObject({ + NameObject("/T"): TextStringObject("application"), + NameObject("/Kids"): ArrayObject([hierarchy_widget_ref]), +}) +hierarchy_parent_ref = hierarchy_writer._add_object(hierarchy_parent) +hierarchy_widget[NameObject("/Parent")] = hierarchy_parent_ref +hierarchy_acroform = hierarchy_writer._root_object["/AcroForm"] +hierarchy_acroform[NameObject("/Fields")] = ArrayObject([hierarchy_parent_ref]) +with open("hierarchical-form.pdf", "wb") as f: + hierarchy_writer.write(f) + +hierarchy_reader = PdfReader("hierarchical-form.pdf") +hierarchical_field_name = "application.applicant_name" +check("get_fields exposes the fully qualified hierarchical field name", + hierarchical_field_name in (hierarchy_reader.get_fields() or {}), + list((hierarchy_reader.get_fields() or {}).keys())) + +hierarchy_fill_writer = PdfWriter() +hierarchy_fill_writer.append(hierarchy_reader) +hierarchy_target_pages = [ + page for page in hierarchy_fill_writer.pages + if any( + (widget := ref.get_object()).get("/Subtype") == "/Widget" + and widget_field_name(widget) == hierarchical_field_name + for ref in (page.get("/Annots") or []) + ) +] +for target_page in hierarchy_target_pages: + hierarchy_fill_writer.update_page_form_field_values( + target_page, {hierarchical_field_name: "Ada Hierarchical"}, + ) +with open("hierarchical-form-filled.pdf", "wb") as f: + hierarchy_fill_writer.write(f) +hierarchy_value = str( + (PdfReader("hierarchical-form-filled.pdf").get_fields() or {}) + .get(hierarchical_field_name, {}).get("/V", "") +) +check("qualified field lookup locates the hierarchical widget page", + len(hierarchy_target_pages) == 1 + and hierarchy_target_pages[0] is hierarchy_fill_writer.pages[1], + len(hierarchy_target_pages)) +check("hierarchical field value round-trips", + hierarchy_value.strip("/") == "Ada Hierarchical", repr(hierarchy_value)) + # ---- transform.md merge imports outline navigation ---------------------------- appendix_writer = PdfWriter() appendix_writer.add_blank_page(width=200, height=300) diff --git a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py index 5667ec2..b7a5d14 100644 --- a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py @@ -57,6 +57,36 @@ def check(name, cond, extra=""): unsafe_wb.active["A1"] = formula_looking check("plain assignment is proven unsafe (negative control)", unsafe_wb.active["A1"].data_type == "f") +CSV_FORMULA_PREFIXES = ("=", "+", "-", "@") + + +def spreadsheet_csv_field(value, *, mode="safe"): + if mode not in {"safe", "raw"}: + raise ValueError("mode must be 'safe' or 'raw'") + if mode == "safe" and isinstance(value, str) and value.startswith(CSV_FORMULA_PREFIXES): + return "'" + value + return value + + +formula_like_fields = ["=1+1", "+SUM(A1:A2)", "-2+3", "@cmd", "plain", "-7", -7] +with open("spreadsheet-safe.csv", "w", newline="", encoding="utf-8") as output: + csv.writer(output).writerow([spreadsheet_csv_field(value) for value in formula_like_fields]) +with open("spreadsheet-safe.csv", newline="", encoding="utf-8") as exported: + safe_fields = next(csv.reader(exported)) +check("spreadsheet-safe CSV neutralizes all four formula prefixes", + safe_fields[:4] == ["'=1+1", "'+SUM(A1:A2)", "'-2+3", "'@cmd"], safe_fields) +check("safe CSV preserves benign text and numeric values", + safe_fields[4:] == ["plain", "'-7", "-7"], safe_fields[4:]) +check("raw CSV mode preserves exact formula-like literal strings", + [spreadsheet_csv_field(value, mode="raw") for value in formula_like_fields[:4]] + == formula_like_fields[:4]) +try: + spreadsheet_csv_field("=1+1", mode="unknown") + invalid_csv_mode_rejected = False +except ValueError: + invalid_csv_mode_rejected = True +check("CSV export rejects an ambiguous safety mode", invalid_csv_mode_rejected) + # ---- edit.md snippet: round_trip_changes detects dropped parts AND stripped extensions ---- wb = openpyxl.Workbook() @@ -541,6 +571,11 @@ class LegacyWorkbook: wb_h.active.title = "First" wb_h.active["A1"] = "=1+1" wb_h.active["A2"] = ArrayFormula("A2:A3", "=ROW(A2:A3)") +wb_h.active["C1"] = "=literal" +wb_h.active["C1"].data_type = "s" +wb_h.active["D1"] = "+literal" +wb_h.active["E1"] = "-literal" +wb_h.active["F1"] = "@literal" second = wb_h.create_sheet("Second") second["A1"] = "plain" second["A2"] = "=2+2" @@ -561,42 +596,34 @@ class LegacyWorkbook: check("data-table formulas have stable diagnostic text", data_table_entry[2].startswith("DataTableFormula(ref='B1:B2', r1='C1'") and "0x" not in data_table_entry[2], data_table_entry) +formula_wb.close() +value_wb.close() # csv.md: value export uses the cached-value workbook and reports every missing cache. -formula_wb["First"].reset_dimensions() -value_wb["First"].reset_dimensions() - -def spreadsheet_safe_csv_value(value): - if isinstance(value, str) and value.startswith(("=", "+", "-", "@")): - return "'" + value - return value - - +formula_wb = openpyxl.load_workbook("multi.xlsx", read_only=True, data_only=False) +value_wb = openpyxl.load_workbook("multi.xlsx", read_only=True, data_only=True) +formula_ws, value_ws = formula_wb["First"], value_wb["First"] +formula_ws.reset_dimensions() +value_ws.reset_dimensions() missing_caches = [] +export_mode = "safe" with open("formula-values.csv", "w", newline="", encoding="utf-8") as output: writer = csv.writer(output) - for formula_row, value_row in zip(formula_wb["First"].iter_rows(), value_wb["First"].iter_rows()): + for formula_row, value_row in zip(formula_ws.iter_rows(), value_ws.iter_rows()): for formula_cell, value_cell in zip(formula_row, value_row): if formula_cell.data_type == "f" and value_cell.value is None: missing_caches.append(formula_cell.coordinate) - writer.writerow([spreadsheet_safe_csv_value(cell.value) for cell in value_row]) + writer.writerow([ + spreadsheet_csv_field(cell.value, mode=export_mode) for cell in value_row + ]) with open("formula-values.csv", newline="", encoding="utf-8") as exported: exported_values = [value for row in csv.reader(exported) for value in row] check("XLSX-to-CSV reports formulas with no cached value", set(missing_caches) >= {"A1", "A2"}, missing_caches) check("XLSX-to-CSV does not leak formula strings into value output", not any(value.startswith("=") for value in exported_values), exported_values) - -formula_like_literals = ["=1+1", "+cmd", "-2+3", "@SUM(A1:A2)", -5] -with open("formula-like-literals.csv", "w", newline="", encoding="utf-8") as output: - csv.writer(output).writerow([ - spreadsheet_safe_csv_value(value) for value in formula_like_literals - ]) -with open("formula-like-literals.csv", newline="", encoding="utf-8") as exported: - safe_literals = next(csv.reader(exported)) -check("spreadsheet-targeted CSV neutralizes every formula-like text prefix", - safe_literals[:4] == ["'=1+1", "'+cmd", "'-2+3", "'@SUM(A1:A2)"], safe_literals) -check("CSV neutralization leaves numeric negative values numeric", - safe_literals[4] == "-5", safe_literals) +check("XLSX-to-CSV safe mode neutralizes cached literal text for spreadsheet consumers", + {"'=literal", "'+literal", "'-literal", "'@literal"} <= set(exported_values), + exported_values) formula_wb.close() value_wb.close() @@ -654,12 +681,18 @@ def markers_in(data): not any(marker in custom_prefix_sheet for marker in legacy_prefix_markers)) # ---- formatting.md guard: header-only sheets skip conditional formatting --------- -def add_demo_formatting(sheet): - last = next( - (row for row in range(sheet.max_row, 1, -1) - if any(sheet.cell(row, column).value is not None for column in range(1, 7))), - 1, +def last_populated_row(sheet, *, first_data_row=2, min_col=1, max_col=6): + populated_rows = ( + cell.row for cell in sheet._cells.values() + if first_data_row <= cell.row + and min_col <= cell.column <= max_col + and cell.value is not None ) + return max(populated_rows, default=first_data_row - 1) + + +def add_demo_formatting(sheet): + last = last_populated_row(sheet) if last < 2: return 0 sheet.conditional_formatting.add( @@ -682,6 +715,7 @@ def add_demo_formatting(sheet): header_wb = openpyxl.Workbook() header_ws = header_wb.active header_ws.append(["A", "B", "C", "D", "E", "F"]) +header_ws["F100"].number_format = "0.00" # styled empty cell inflates max_row try: header_ws.conditional_formatting.add( "D2:D1", CellIsRule(operator="lessThan", formula=["0"]), @@ -690,6 +724,8 @@ def add_demo_formatting(sheet): except (TypeError, ValueError): inverted_range_rejected = True check("unguarded header-only range is rejected (negative control)", inverted_range_rejected) +check("style-only ghost row inflates max_row (negative control)", header_ws.max_row == 100) +check("populated-row scan ignores a style-only ghost row", last_populated_row(header_ws) == 1) check("header-only guard skips all four formatting rules", add_demo_formatting(header_ws) == 0) header_wb.save("header-only-formatting.xlsx") header_reopened = openpyxl.load_workbook("header-only-formatting.xlsx") @@ -700,12 +736,11 @@ def add_demo_formatting(sheet): data_ws = data_wb.active data_ws.append(["A", "B", "C", "D", "E", "F"]) data_ws.append([1, 2, 3, -1, 5, 6]) -data_ws["F10000"].number_format = "0.00" # inflate max_row without adding data +data_ws["F100"].number_format = "0.00" check("data rows receive all four formatting rules", add_demo_formatting(data_ws) == 4) -formatting_ranges = [str(item.sqref) for item in data_ws.conditional_formatting] -check("conditional formatting ignores a styled ghost row", - all("10000" not in item and item.endswith("2") for item in formatting_ranges), - formatting_ranges) +data_ranges = {str(item.sqref) for item in data_ws.conditional_formatting} +check("conditional formatting stops at the last populated row despite ghost styles", + data_ranges == {"D2", "A2:F2", "C2", "E2"}, data_ranges) data_wb.save("data-formatting.xlsx") data_reopened = openpyxl.load_workbook("data-formatting.xlsx") check("all four formatting rules survive save/reopen", @@ -775,6 +810,25 @@ def discover_dimension(worksheet): and streamed_after_reset[2][:2] == (3, 4), (len(streamed_before_reset), streamed_after_reset[-1])) +csv_formula_wb = openpyxl.load_workbook("dimension.xlsx", read_only=True, data_only=False) +csv_value_wb = openpyxl.load_workbook("dimension.xlsx", read_only=True, data_only=True) +csv_formula_ws, csv_value_ws = csv_formula_wb.active, csv_value_wb.active +check("both CSV source streams initially trust the truncated dimension (negative control)", + csv_formula_ws.calculate_dimension() == "A1:B2" + and csv_value_ws.calculate_dimension() == "A1:B2") +csv_formula_ws.reset_dimensions() +csv_value_ws.reset_dimensions() +csv_stream_rows = list(zip(csv_formula_ws.iter_rows(), csv_value_ws.iter_rows())) +check("CSV export resets both paired streams before iterating", + len(csv_stream_rows) == 4 + and len(csv_stream_rows[-1][0]) == 3 + and len(csv_stream_rows[-1][1]) == 3 + and csv_stream_rows[-1][0][2].data_type == "f" + and csv_stream_rows[-1][1][2].value is None, + [(len(formula_row), len(value_row)) for formula_row, value_row in csv_stream_rows]) +csv_formula_wb.close() +csv_value_wb.close() + empty_dimension_wb = openpyxl.Workbook() empty_dimension_wb.save("empty-dimension.xlsx") empty_dimension_ro = openpyxl.load_workbook("empty-dimension.xlsx", read_only=True) From 5791d58e1018d98346da06ebf0e5698c146c4669 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=98=E6=89=AC?= Date: Mon, 17 Aug 2026 00:08:28 +0800 Subject: [PATCH 24/34] fix(document-skills): handle latest format edge cases --- .../skills/docx/references/create.md | 19 +- .../skills/docx/references/read.md | 21 ++- .../document-skills/skills/pdf/SKILL.md | 47 +++-- .../skills/pdf/references/inspect.md | 25 ++- .../skills/pptx/references/analyze.md | 11 +- .../skills/pptx/references/create.md | 22 ++- .../skills/xlsx/references/edit.md | 109 +++++++----- .../document-skills/tests/docx_fixture.py | 75 ++++++-- .../document-skills/tests/pdf_fixture.py | 129 ++++++++++++-- .../document-skills/tests/pptx_fixture.py | 121 +++++++++++++ .../document-skills/tests/xlsx_fixture.py | 166 ++++++++++++------ 11 files changed, 598 insertions(+), 147 deletions(-) diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/create.md b/plugins/Hylouis233/document-skills/skills/docx/references/create.md index 628e15b..d195ccc 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/create.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/create.md @@ -84,12 +84,23 @@ doc.save("report.docx") clone = copy.deepcopy(source) new_id = max(int(n.get(qn("w:numId"))) for n in numbering.findall(qn("w:num"))) + 1 clone.set(qn("w:numId"), str(new_id)) - override = OxmlElement("w:lvlOverride") - override.set(qn("w:ilvl"), "0") + level_zero_overrides = [ + item for item in clone.findall(qn("w:lvlOverride")) + if item.get(qn("w:ilvl")) == "0" + ] + if len(level_zero_overrides) > 1: + raise ValueError("base numbering has duplicate level-zero overrides") + if level_zero_overrides: + override = level_zero_overrides[0] + for old_start in override.findall(qn("w:startOverride")): + override.remove(old_start) + else: + override = OxmlElement("w:lvlOverride") + override.set(qn("w:ilvl"), "0") + clone.append(override) start = OxmlElement("w:startOverride") start.set(qn("w:val"), "1") - override.append(start) - clone.append(override) + override.insert(0, start) # startOverride precedes an optional embedded w:lvl numbering.append(clone) return new_id diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/read.md b/plugins/Hylouis233/document-skills/skills/docx/references/read.md index ba2c9c9..ae14edf 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/read.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/read.md @@ -51,8 +51,22 @@ def iter_paragraph_runs(paragraph): yield from walk(child) yield from walk(paragraph._p) +def legacy_symbol_record(symbol): + """Report font-specific w:sym content without guessing at a Unicode mapping.""" + font = symbol.get(qn("w:font")) + character = symbol.get(qn("w:char")) + return f"[unreadable legacy symbol font={font!r} char={character!r}]" + +def run_text(run): + pieces = [] + for child in run._r.xpath( + "w:br | w:cr | w:noBreakHyphen | w:ptab | w:t | w:tab | w:sym" + ): + pieces.append(legacy_symbol_record(child) if child.tag == qn("w:sym") else str(child)) + return "".join(pieces) + def paragraph_text(paragraph): - return "".join(run.text for run in iter_paragraph_runs(paragraph)) + return "".join(run_text(run) for run in iter_paragraph_runs(paragraph)) def tc_text(tc, parent): """Cell text rebuilt per paragraph, keeping tabs and breaks visible. @@ -72,6 +86,8 @@ def tc_text(tc, parent): pieces.append("") elif node.tag in (qn("w:br"), qn("w:cr")): pieces.append("
") + elif node.tag == qn("w:sym"): + pieces.append(legacy_symbol_record(node)) paragraphs.append("".join(pieces)) return " / ".join(paragraphs) @@ -143,6 +159,9 @@ Notes: presenting an incomplete extraction. Convert them with a trusted office renderer before claiming their content was read. Each table-cell record carries its own `unreadable` list so an import nested in a cell is not lost from the report. + A legacy `w:sym` code is specific to its symbol font, so the helpers preserve its position + as an explicit unreadable record containing the font and character code rather than silently + dropping visible content or guessing a Unicode character. Text boxes, headers, footers, and footnotes still require their own collections (`section.header/.footer`) or raw XML. - For revision/comment metadata, inspect the XML parts directly: `word/comments.xml`, diff --git a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md index 55ce728..6295da2 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md @@ -114,10 +114,10 @@ size_mismatches = [ ] require(not size_mismatches, f"unexpected page sizes: {size_mismatches}") -# Overflow is a defect (shared rule 4): text that runs past the page box is -# clipped or off-page even though every check above still passes. Plain block -# extraction silently drops fully off-page text, so extract through an -# explicitly enlarged clip rectangle and compare the block boxes to the page. +# Overflow is a defect (shared rule 4): text, images, or vector drawings that run +# past the page box are clipped even though every check above still passes. Plain +# text extraction silently drops fully off-page text, so use an enlarged clip for +# text and inspect the placement boxes reported for every image and drawing. import fitz overflow_doc = fitz.open(output_path) @@ -127,24 +127,45 @@ if overflow_doc.needs_pass: overflow_doc.authenticate(password) overflow_pages = [] for page in overflow_doc: - # get_text() block coordinates are unrotated even when /Rotate is 90/270; + # These APIs report unrotated coordinates even when /Rotate is 90/270; # page.rect uses rotated dimensions. Compare against an unrotated crop-box - # extent so valid high-y portrait text is not flagged on a rotated page. + # extent so valid high-y portrait content is not flagged on a rotated page. page_box = fitz.Rect(0, 0, page.cropbox.width, page.cropbox.height) clip = fitz.Rect( page_box.x0 - 2000, page_box.y0 - 2000, page_box.x1 + 2000, page_box.y1 + 2000, ) - text_blocks = [b for b in page.get_text("blocks", clip=clip) if b[6] == 0] + text_rects = [ + fitz.Rect(block[:4]) + for block in page.get_text("blocks", clip=clip) + if block[6] == 0 + ] + image_rects = [ + fitz.Rect(0, 0, 1, 1) * fitz.Matrix(*image["transform"]) + for image in page.get_image_info() + ] + drawing_rects = [] + for drawing in page.get_drawings(): + rect = fitz.Rect(drawing["rect"]) + # Path rectangles exclude stroke thickness. Expand stroked paths so a + # line centered on the page edge cannot hide half its width off-page. + stroke_pad = ( + float(drawing.get("width") or 0) / 2 + if "s" in drawing.get("type", "") else 0 + ) + drawing_rects.append(fitz.Rect( + rect.x0 - stroke_pad, rect.y0 - stroke_pad, + rect.x1 + stroke_pad, rect.y1 + stroke_pad, + )) beyond_box = any( - b[0] < page_box.x0 - 0.5 or b[1] < page_box.y0 - 0.5 - or b[2] > page_box.x1 + 0.5 or b[3] > page_box.y1 + 0.5 - for b in text_blocks + rect.x0 < page_box.x0 - 0.5 or rect.y0 < page_box.y0 - 0.5 + or rect.x1 > page_box.x1 + 0.5 or rect.y1 > page_box.y1 + 0.5 + for rect in text_rects + image_rects + drawing_rects ) if beyond_box: overflow_pages.append(page.number + 1) overflow_doc.close() -require(not overflow_pages, f"text blocks extend past the page box on pages: {overflow_pages}") +require(not overflow_pages, f"content extends past the page box on pages: {overflow_pages}") # The page box is the hard bound. When the task declares specific margins, # additionally check key blocks against them (or render and inspect visually) - # content inside the box but past a declared margin is a softer, task-specific @@ -155,6 +176,6 @@ Confirm: page count matches the request; every page except those explicitly list `intentionally_raster_only_pages` has extractable text or at least one form widget (a pure interactive page); each requested key string is listed in `expected_strings_by_page` and extracts on the correct page; every value in `page_sizes` is the declared size within -`page_size_tolerance`; and no page's text blocks extend past the page box (the overflow -check). Report all five. For pixel-sensitive work, render every applicable page +`page_size_tolerance`; and no page's text, image placements, or vector drawings extend past +the page box (the overflow check). Report all five. For pixel-sensitive work, render every applicable page with PyMuPDF at 100 dpi and check that each render is non-blank (mean pixel value). diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md b/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md index 983bf7b..6204185 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md @@ -38,23 +38,32 @@ print("pages:", doc.page_count) print("password_protected:", doc.needs_pass, "| still_encrypted:", doc.is_encrypted, "| pdf:", doc.is_pdf) for page in doc: + blocks = page.get_text("dict")["blocks"] + image_blocks = [block for block in blocks if block["type"] == 1] drawings = page.get_drawings() widgets = list(page.widgets() or ()) annotations = list(page.annots() or ()) links = page.get_links() + is_blank = not ( + page.get_text().strip() or page.get_images() or image_blocks or drawings + or widgets or annotations or links + ) print(page.number, page.rect, "text_len:", len(page.get_text()), - "images:", len(page.get_images()), "drawings:", len(drawings), + "resource_images:", len(page.get_images()), + "image_blocks:", len(image_blocks), "drawings:", len(drawings), "widgets:", len(widgets), "annotations:", len(annotations), - "links:", len(links)) + "links:", len(links), "blank:", is_blank) print(" fonts:", font_inventory(doc, page)) ``` ## Checks worth automating -- **Blank page detection**: flag only when text, images, drawings, widgets, annotations, and - links are all absent. Interactive form fields are widgets rather than page text, so the - three-content-stream predicate alone would misclassify a usable form page as blank. A blank - page after generation usually means an overflowing flowable created it. +- **Blank page detection**: flag only when text, resource images, type-1 image blocks, drawings, + widgets, annotations, and links are all absent. `page.get_images()` lists image XObjects but + misses images embedded inline in the content stream; type-1 blocks from `get_text("dict")` + cover both inline and XObject image placements. Interactive form fields are widgets rather + than page text, so a three-content-stream predicate alone would misclassify a usable form + page as blank. A blank page after generation usually means an overflowing flowable created it. - **Font inventory**: `page.get_fonts()` lists referenced fonts, including non-embedded base fonts. Use `doc.extract_font(xref)` as above and report `embedded` separately; a referenced face with no extractable program may be substituted on another machine. @@ -63,5 +72,5 @@ for page in doc: - **Damage**: `fitz.open` on a corrupt file raises or yields garbage - pair with `pypdf.PdfReader` cross-check when provenance is unknown. -Report findings as a table (page, size, text chars, images, drawings, widgets, annotations, -links) - it is what every downstream decision hangs off. +Report findings as a table (page, size, text chars, resource images, image blocks, drawings, +widgets, annotations, links, blank) - it is what every downstream decision hangs off. diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md index c762193..109789e 100644 --- a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md @@ -185,8 +185,12 @@ must not be the first check because it expands every member, including an archiv ```python import zipfile +from pathlib import Path from lxml import etree +path = Path("input.pptx") +MAX_ARCHIVE_BYTES = 200 * 1024 * 1024 +MAX_MEMBERS = 10_000 MAX_XML_PART = 20 * 1024 * 1024 MAX_ENTRY = 100 * 1024 * 1024 MAX_TOTAL_UNCOMPRESSED = 500 * 1024 * 1024 @@ -206,8 +210,13 @@ safe_xml_parser = etree.XMLParser( recover=False, ) -with zipfile.ZipFile("input.pptx") as archive: +# Bound the package itself before ZipFile materializes its central directory. +require(path.stat().st_size <= MAX_ARCHIVE_BYTES, + "compressed PPTX file size above limit") +with zipfile.ZipFile(path) as archive: infos = archive.infolist() + # Check the count before building sets, summing sizes, or opening any member. + require(len(infos) <= MAX_MEMBERS, "archive member count above limit") names = {info.filename for info in infos} require(len(names) == len(infos), "duplicate archive member names are unsafe") require("[Content_Types].xml" in names and "ppt/presentation.xml" in names, diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/create.md b/plugins/Hylouis233/document-skills/skills/pptx/references/create.md index 080544b..04909d7 100644 --- a/plugins/Hylouis233/document-skills/skills/pptx/references/create.md +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/create.md @@ -26,7 +26,15 @@ def placeholder_of_type(slide, *types): ph for ph in slide.placeholders if ph.placeholder_format.type in types ] - assert len(matches) == 1, f"expected one placeholder of {types}, found {len(matches)}" + if len(matches) != 1: + available = [ + f"{ph.name} ({ph.placeholder_format.type})" + for ph in slide.placeholders + ] + raise ValueError( + f"expected exactly one placeholder of {types}, found {len(matches)}; " + f"available placeholders: {available or 'none'}" + ) return matches[0] # P1 title slide @@ -48,14 +56,22 @@ for i, line in enumerate(lines): # P3 table slide s = add_slide(title_only_layout) s.shapes.title.text = "Regional service health" -rows, cols = 4, 3 +hdr = ["Region", "Error rate", "P99 latency"] +body = [ + ["Americas", "0.08%", "182 ms"], + ["Europe", "0.05%", "164 ms"], + ["Asia Pacific", "0.11%", "213 ms"], +] +rows, cols = 1 + len(body), len(hdr) tbl_shape = s.shapes.add_table(rows, cols, Inches(0.5), Inches(1.5), Inches(9), Inches(3.5)) table = tbl_shape.table -hdr = ["Region", "Error rate", "P99 latency"] for j, text in enumerate(hdr): cell = table.cell(0, j); cell.text = text for par in cell.text_frame.paragraphs: for run in par.runs: run.font.bold = True +for i, row in enumerate(body, start=1): + for j, text in enumerate(row): + table.cell(i, j).text = text # P4 chart slide (real chart part, not a picture) from pptx.chart.data import CategoryChartData diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md index fdcabda..dd6b0af 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md @@ -193,81 +193,102 @@ wb.save("input-edited.xlsx") - Before editing an unknown workbook, detect what an openpyxl load/save round trip silently changes. Dropped parts (slicers, pivot caches, power-query connections) are only half the risk: features stored *inside* a retained part, such as `x14` extension lists in - `xl/worksheets/sheet1.xml`, can be stripped while the archive member name stays. Compare - part contents too, not just names, and report both lists before overwriting the file: + `xl/worksheets/sheet1.xml`, can be stripped while the archive member name stays. Inventory + every worksheet extension record separately; a per-part set of coarse markers cannot detect + one lost `` when another record keeps the same URI/namespace markers alive: ```python import zipfile + from collections import Counter from tempfile import TemporaryFile + from xml.etree import ElementTree as ET - # XML prefixes are arbitrary - a valid workbook may bind the x14 namespace to - # "sx" or the markup-compatibility namespace to anything. Match the namespace - # URIs (and the prefix-independent local name extLst), never prefixes. - EXTENSION_MARKERS = { - "extLst": b"extLst", - "x14 namespace": b"http://schemas.microsoft.com/office/spreadsheetml/2009/9/main", - "markup compatibility": b"http://schemas.openxmlformats.org/markup-compatibility/2006", - } + SHEET_NS = "http://schemas.openxmlformats.org/spreadsheetml/2006/main" + EXT_TAG = f"{{{SHEET_NS}}}ext" - def scan_extension_markers(archive, info): - """Stream one XML part and retain marker names only, never the full payload.""" - found = set() - overlap = max(len(marker) for marker in EXTENSION_MARKERS.values()) - 1 - tail = b"" + def normalized_text(value): + return (value or "").strip() + + def normalized_element(element): + """Prefix/attribute-order independent, but sensitive to child values and order.""" + return ( + element.tag, # ElementTree expands prefixes to URI names + tuple(sorted(element.attrib.items())), + normalized_text(element.text), + tuple((normalized_element(child), normalized_text(child.tail)) + for child in element), + ) + + def worksheet_extension_records(archive, info): + """Return a multiset of (ext URI, normalized child content) for one worksheet.""" + records = Counter() + extension_depth = 0 with archive.open(info) as stream: - while chunk := stream.read(64 * 1024): - window = tail + chunk - found.update( - label for label, marker in EXTENSION_MARKERS.items() if marker in window - ) - if len(found) == len(EXTENSION_MARKERS): - break - tail = window[-overlap:] - return found + for event, element in ET.iterparse(stream, events=("start", "end")): + if event == "start": + if extension_depth: + extension_depth += 1 + elif element.tag == EXT_TAG: + extension_depth = 1 + elif extension_depth: + if extension_depth == 1: + children = tuple( + (normalized_element(child), normalized_text(child.tail)) + for child in element + ) + records[(element.attrib.get("uri", ""), children)] += 1 + element.clear() + extension_depth = 0 + else: + extension_depth -= 1 + else: + element.clear() # keep large non-extension sheets streaming + return records def archive_inventory(source): with zipfile.ZipFile(source) as archive: names = set(archive.namelist()) - markers = {} + extensions = {} for info in archive.infolist(): - if info.filename.endswith((".xml", ".rels")): - found = scan_extension_markers(archive, info) - if found: - markers[info.filename] = found - return names, markers + if (info.filename.startswith("xl/worksheets/") + and info.filename.endswith(".xml")): + extensions[info.filename] = worksheet_extension_records(archive, info) + return names, extensions - def stripped_extension_markers(before, after, common_names): - """Return every (part, marker) that was present before and absent after.""" - return sorted( - (name, label) - for name in common_names - for label in before.get(name, set()) - after.get(name, set()) - ) + def stripped_extension_records(before, after, common_names): + """Return each lost (worksheet, URI, normalized children), including duplicates.""" + stripped = [] + for name in sorted(common_names): + for (uri, children), count in (before.get(name, Counter()) + - after.get(name, Counter())).items(): + stripped.extend((name, uri, children) for _ in range(count)) + return sorted(stripped, key=repr) def round_trip_changes(path, **load_options): - before_names, before_markers = archive_inventory(path) + before_names, before_extensions = archive_inventory(path) wb = openpyxl.load_workbook(path, **load_options) # same options as the real edit with TemporaryFile() as output: wb.save(output) output.seek(0) - after_names, after_markers = archive_inventory(output) + after_names, after_extensions = archive_inventory(output) wb.close() dropped = sorted(before_names - after_names) - stripped_extensions = stripped_extension_markers( - before_markers, after_markers, before_names & after_names + stripped_extensions = stripped_extension_records( + before_extensions, after_extensions, before_names & after_names ) return dropped, stripped_extensions dropped, stripped = round_trip_changes("input.xlsx") if dropped or stripped: print("WARNING: saving with openpyxl will drop:", dropped) - print("WARNING: saving with openpyxl will strip (part, marker):", stripped) + print("WARNING: saving with openpyxl will strip (worksheet, URI, content):", stripped) # report to the user and get confirmation before the first save ``` (openpyxl re-serializes every sheet it touches, so byte-identity of sheets is not a - meaningful check; comparing each marker independently is what catches partial loss when, for - example, `` survives but its `x14:` content does not.) + meaningful check. Expanded XML names normalize arbitrary prefixes, sorted attributes normalize + serialization order, and the per-worksheet multiset still detects one missing record when an + identical URI or even an identical duplicate record survives.) ## Rules diff --git a/plugins/Hylouis233/document-skills/tests/docx_fixture.py b/plugins/Hylouis233/document-skills/tests/docx_fixture.py index ce03234..e8380d7 100644 --- a/plugins/Hylouis233/document-skills/tests/docx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/docx_fixture.py @@ -179,8 +179,23 @@ def walk(element): yield from walk(paragraph._p) +def legacy_symbol_record(symbol): + font = symbol.get(qn("w:font")) + character = symbol.get(qn("w:char")) + return f"[unreadable legacy symbol font={font!r} char={character!r}]" + + +def run_text(run): + pieces = [] + for child in run._r.xpath( + "w:br | w:cr | w:noBreakHyphen | w:ptab | w:t | w:tab | w:sym" + ): + pieces.append(legacy_symbol_record(child) if child.tag == qn("w:sym") else str(child)) + return "".join(pieces) + + def paragraph_text(paragraph): - return "".join(run.text for run in iter_paragraph_runs(paragraph)) + return "".join(run_text(run) for run in iter_paragraph_runs(paragraph)) def table_content(table): @@ -238,6 +253,13 @@ def wrap_in_sdt(element): inline_run = inline_paragraph.add_run("inline한") inline_paragraph.add_run("-after") wrap_in_sdt(inline_run._r) +inline_paragraph.add_run("-legacy-") +legacy_symbol_run = inline_paragraph.add_run() +legacy_symbol = OxmlElement("w:sym") +legacy_symbol.set(qn("w:font"), "Wingdings") +legacy_symbol.set(qn("w:char"), "F052") +legacy_symbol_run._r.append(legacy_symbol) +inline_paragraph.add_run("-visible") chunk_part = Part( PackURI("/word/altChunk1.html"), "text/html", b"IMPORTED ALTCHUNK TEXT", sdt_doc.part.package, @@ -265,7 +287,15 @@ def wrap_in_sdt(element): walked_text = [paragraph_text(paragraph) for paragraph in walked_paragraphs] check("content-control traversal emits the nested paragraph", "inside content control" in walked_text, walked_text) check("content-control traversal emits inline run text", - "before-inline한-after" in walked_text, walked_text) + any("before-inline한-after" in text for text in walked_text), walked_text) +legacy_symbol_marker = "[unreadable legacy symbol font='Wingdings' char='F052']" +legacy_paragraph = next(paragraph for paragraph in walked_paragraphs + if "-legacy-" in paragraph_text(paragraph)) +check("Run.text silently omits visible legacy w:sym content (negative control)", + legacy_symbol_marker not in "".join(run.text for run in iter_paragraph_runs(legacy_paragraph))) +check("paragraph extraction preserves the legacy symbol's position, font, and character code", + f"-legacy-{legacy_symbol_marker}-visible" in paragraph_text(legacy_paragraph), + paragraph_text(legacy_paragraph)) walked_tables = [block for kind, block in walked_blocks if kind == "table"] rendered_tables = [table_content(table) for table in walked_tables] check("content-control traversal emits a wrapped table", @@ -634,12 +664,23 @@ def new_restart_num_id(doc, base_num_id): clone = copy.deepcopy(source) new_id = max(int(n.get(qn("w:numId"))) for n in numbering.findall(qn("w:num"))) + 1 clone.set(qn("w:numId"), str(new_id)) - override = OxmlElement("w:lvlOverride") - override.set(qn("w:ilvl"), "0") + level_zero_overrides = [ + item for item in clone.findall(qn("w:lvlOverride")) + if item.get(qn("w:ilvl")) == "0" + ] + if len(level_zero_overrides) > 1: + raise ValueError("base numbering has duplicate level-zero overrides") + if level_zero_overrides: + override = level_zero_overrides[0] + for old_start in override.findall(qn("w:startOverride")): + override.remove(old_start) + else: + override = OxmlElement("w:lvlOverride") + override.set(qn("w:ilvl"), "0") + clone.append(override) start = OxmlElement("w:startOverride") start.set(qn("w:val"), "1") - override.append(start) - clone.append(override) + override.insert(0, start) numbering.append(clone) return new_id @@ -675,6 +716,15 @@ def numbered_paragraph(doc, text, num_id): doc_b.add_paragraph(item, style="List Number") doc_b.add_paragraph("Second list, restarted") base_id = list_number_num_id(doc_b) +numbering_b = doc_b.part.part_related_by(RT.NUMBERING).element +base_entry = next(n for n in numbering_b.findall(qn("w:num")) + if n.get(qn("w:numId")) == str(base_id)) +preexisting_override = OxmlElement("w:lvlOverride") +preexisting_override.set(qn("w:ilvl"), "0") +preexisting_start = OxmlElement("w:startOverride") +preexisting_start.set(qn("w:val"), "7") +preexisting_override.append(preexisting_start) +base_entry.append(preexisting_override) restart_id = new_restart_num_id(doc_b, base_id) for item in ("four", "five", "six"): numbered_paragraph(doc_b, item, restart_id) @@ -685,10 +735,15 @@ def numbered_paragraph(doc, text, num_id): nums_b = numbering_b.findall(qn("w:num")) check("cloned num entry exists", any(n.get(qn("w:numId")) == str(restart_id) for n in nums_b)) clone_entry = next(n for n in nums_b if n.get(qn("w:numId")) == str(restart_id)) -override = clone_entry.find(qn("w:lvlOverride")) -check("clone carries startOverride=1 at ilvl 0", - override is not None and override.get(qn("w:ilvl")) == "0" - and override.find(qn("w:startOverride")).get(qn("w:val")) == "1") +level_zero_overrides = [item for item in clone_entry.findall(qn("w:lvlOverride")) + if item.get(qn("w:ilvl")) == "0"] +start_overrides = ([] if not level_zero_overrides else + level_zero_overrides[0].findall(qn("w:startOverride"))) +check("clone replaces a preexisting ilvl=0 override without creating a duplicate", + len(level_zero_overrides) == 1 and len(start_overrides) == 1 + and start_overrides[0].get(qn("w:val")) == "1") +check("cloning leaves the base numbering override unchanged", + preexisting_start.get(qn("w:val")) == "7") second_list_num_ids = [ p._p.find(qn("w:pPr") + "/" + qn("w:numPr") + "/" + qn("w:numId")).get(qn("w:val")) for p in doc_b.paragraphs[-3:] diff --git a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py index bbe653f..ae67627 100644 --- a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py @@ -461,17 +461,45 @@ def stamp_placement(page, stamp_box): form_only_page = form_only_doc[0] form_only_widgets = list(form_only_page.widgets() or ()) form_only_annotations = list(form_only_page.annots() or ()) -form_only_is_blank = ( - not form_only_page.get_text().strip() - and not form_only_page.get_images() - and not form_only_page.get_drawings() - and not form_only_page.get_links() - and not form_only_widgets - and not form_only_annotations -) + + +def inspected_page_is_blank(page): + image_blocks = [ + block for block in page.get_text("dict")["blocks"] + if block["type"] == 1 + ] + return not ( + page.get_text().strip() or page.get_images() or image_blocks + or page.get_drawings() or list(page.widgets() or ()) + or list(page.annots() or ()) or page.get_links() + ) + + +form_only_is_blank = inspected_page_is_blank(form_only_page) check("form-only page exposes a widget", len(form_only_widgets) == 1) check("form-only page is not classified as blank", not form_only_is_blank) +# ReportLab's drawInlineImage emits BI/ID/EI content rather than an image XObject. +inline_source = fitz.Pixmap(fitz.csRGB, fitz.IRect(0, 0, 12, 12)) +inline_source.clear_with(96) +inline_source.save("inline-only-source.png") +inline_canvas = canvas.Canvas("inline-only.pdf", pagesize=A4) +inline_canvas.drawInlineImage("inline-only-source.png", 72, 700, width=80, height=80) +inline_canvas.showPage() +inline_canvas.save() +inline_doc = fitz.open("inline-only.pdf") +inline_page = inline_doc[0] +inline_blocks = [ + block for block in inline_page.get_text("dict")["blocks"] + if block["type"] == 1 +] +check("inline-only image is absent from the XObject inventory", + inline_page.get_images() == [], inline_page.get_images()) +check("inline-only image appears as a type-1 text-dictionary block", + len(inline_blocks) == 1, inline_blocks) +check("inline-only image page is not classified as blank", + not inspected_page_is_blank(inline_page)) + # ---- extract.md CMYK conversion snippet --------------------------------------- pix = fitz.Pixmap(fitz.csCMYK, fitz.IRect(0, 0, 24, 24)) # CMYK pixmap like a CMYK PDF image converted = fitz.Pixmap(fitz.csRGB, pix) if pix.colorspace not in (fitz.csGRAY, fitz.csRGB) else pix @@ -733,7 +761,7 @@ def stamp_line_directions(path, page_number): mixed_link_after == mixed_link_before, (mixed_link_before, mixed_link_after)) -# ---- SKILL.md overflow check: off-page text is a defect ------------------------- +# ---- SKILL.md overflow check: off-page text and graphics are defects ------------ overflow_ok = canvas.Canvas("overflow.pdf", pagesize=A4) overflow_ok.setFont("Helvetica", 16) overflow_ok.drawString(72, 780, "fits on page") @@ -756,6 +784,51 @@ def stamp_line_directions(path, page_number): with open("overflow-rotated.pdf", "wb") as f: rotated_writer.write(f) +graphics_ok = canvas.Canvas("overflow-graphics-ok.pdf", pagesize=A4) +graphics_ok.drawInlineImage("inline-only-source.png", 72, 700, width=80, height=80) +graphics_ok.setLineWidth(2) +graphics_ok.rect(72, 600, 100, 50, stroke=1, fill=0) +graphics_ok.showPage() +graphics_ok.save() + +image_bad = canvas.Canvas("overflow-image-bad.pdf", pagesize=A4) +image_bad.drawInlineImage("inline-only-source.png", 560, 700, width=80, height=80) +image_bad.showPage() +image_bad.save() + +drawing_bad = canvas.Canvas("overflow-drawing-bad.pdf", pagesize=A4) +drawing_bad.setLineWidth(2) +drawing_bad.rect(560, 600, 80, 50, stroke=1, fill=0) +drawing_bad.showPage() +drawing_bad.save() + +rotated_graphics_source = canvas.Canvas("overflow-graphics-rotated-source.pdf", pagesize=A4) +rotated_graphics_source.drawInlineImage( + "inline-only-source.png", 500, 700, width=80, height=80, +) +rotated_graphics_source.setLineWidth(2) +rotated_graphics_source.rect(20, 20, 80, 40, stroke=1, fill=0) +rotated_graphics_source.showPage() +rotated_graphics_source.save() +rotated_graphics_writer = PdfWriter() +rotated_graphics_writer.append(R2("overflow-graphics-rotated-source.pdf")) +rotated_graphics_writer.pages[0].rotate(90) +with open("overflow-graphics-rotated.pdf", "wb") as f: + rotated_graphics_writer.write(f) + + +def drawing_bounds(drawing): + rect = fitz.Rect(drawing["rect"]) + stroke_pad = ( + float(drawing.get("width") or 0) / 2 + if "s" in drawing.get("type", "") else 0 + ) + return fitz.Rect( + rect.x0 - stroke_pad, rect.y0 - stroke_pad, + rect.x1 + stroke_pad, rect.y1 + stroke_pad, + ) + + def overflow_pages(path): doc = fitz.open(path) pages = [] @@ -766,10 +839,19 @@ def overflow_pages(path): page_box.x0 - 2000, page_box.y0 - 2000, page_box.x1 + 2000, page_box.y1 + 2000, ) - blocks = [b for b in page.get_text("blocks", clip=clip) if b[6] == 0] - if any(b[0] < page_box.x0 - 0.5 or b[1] < page_box.y0 - 0.5 - or b[2] > page_box.x1 + 0.5 or b[3] > page_box.y1 + 0.5 - for b in blocks): + text_rects = [ + fitz.Rect(block[:4]) + for block in page.get_text("blocks", clip=clip) + if block[6] == 0 + ] + image_rects = [ + fitz.Rect(0, 0, 1, 1) * fitz.Matrix(*image["transform"]) + for image in page.get_image_info() + ] + drawing_rects = [drawing_bounds(drawing) for drawing in page.get_drawings()] + if any(rect.x0 < page_box.x0 - 0.5 or rect.y0 < page_box.y0 - 0.5 + or rect.x1 > page_box.x1 + 0.5 or rect.y1 > page_box.y1 + 0.5 + for rect in text_rects + image_rects + drawing_rects): pages.append(page.number + 1) doc.close() return pages @@ -777,6 +859,12 @@ def overflow_pages(path): check("in-bounds PDF reports no overflow pages", overflow_pages("overflow.pdf") == []) check("off-page text is detected by the overflow check (negative control)", overflow_pages("overflow-bad.pdf") == [1]) +check("in-bounds image and vector drawing pass the overflow check", + overflow_pages("overflow-graphics-ok.pdf") == []) +check("out-of-bounds image placement is detected", + overflow_pages("overflow-image-bad.pdf") == [1]) +check("out-of-bounds vector drawing is detected", + overflow_pages("overflow-drawing-bad.pdf") == [1]) rotated_probe = fitz.open("overflow-rotated.pdf")[0] rotated_blocks = rotated_probe.get_text( "blocks", clip=fitz.Rect(-2000, -2000, 2000, 3000), @@ -789,6 +877,21 @@ def overflow_pages(path): check("rotated rect comparison falsely flags valid text (negative control)", rotation_blind_flag) check("overflow check compares rotated pages in unrotated coordinates", overflow_pages("overflow-rotated.pdf") == []) +rotated_graphics_probe = fitz.open("overflow-graphics-rotated.pdf")[0] +rotated_graphic_rects = ( + [fitz.Rect(0, 0, 1, 1) * fitz.Matrix(*image["transform"]) + for image in rotated_graphics_probe.get_image_info()] + + [drawing_bounds(drawing) for drawing in rotated_graphics_probe.get_drawings()] +) +rotation_blind_graphics_flag = any( + rect.x1 > rotated_graphics_probe.rect.width + 0.5 + or rect.y1 > rotated_graphics_probe.rect.height + 0.5 + for rect in rotated_graphic_rects +) +check("rotated rect comparison falsely flags valid graphics (negative control)", + rotation_blind_graphics_flag, rotated_graphic_rects) +check("graphic overflow check preserves rotated coordinate correctness", + overflow_pages("overflow-graphics-rotated.pdf") == []) check("off-page text still extracts, so extraction alone cannot catch it", "drawn far below" in (pypdf.PdfReader("overflow-bad.pdf").pages[0].extract_text() or "")) diff --git a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py index e96a0c9..2b58bcd 100644 --- a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py @@ -10,6 +10,7 @@ import sys import xml.etree.ElementTree as ET import zipfile +from pathlib import Path from lxml import etree from pptx import Presentation @@ -91,7 +92,91 @@ def check(name, cond, extra=""): prs.save("input.pptx") +# ---- create.md skeleton: explicit placeholder selection and populated table ---- +def placeholder_of_type(slide, *types): + matches = [ + ph for ph in slide.placeholders + if ph.placeholder_format.type in types + ] + if len(matches) != 1: + available = [ + f"{ph.name} ({ph.placeholder_format.type})" + for ph in slide.placeholders + ] + raise ValueError( + f"expected exactly one placeholder of {types}, found {len(matches)}; " + f"available placeholders: {available or 'none'}" + ) + return matches[0] + + +selector_prs = Presentation() +no_object_slide = selector_prs.slides.add_slide(selector_prs.slide_layouts[6]) +try: + placeholder_of_type(no_object_slide, PP_PLACEHOLDER.OBJECT) + missing_placeholder_message = None +except ValueError as exc: + missing_placeholder_message = str(exc) +check( + "placeholder selector explicitly rejects zero matches", + missing_placeholder_message is not None + and "found 0" in missing_placeholder_message + and "available placeholders: none" in missing_placeholder_message, + missing_placeholder_message, +) + +two_object_slide = selector_prs.slides.add_slide(selector_prs.slide_layouts[3]) +try: + placeholder_of_type(two_object_slide, PP_PLACEHOLDER.OBJECT) + ambiguous_placeholder_message = None +except ValueError as exc: + ambiguous_placeholder_message = str(exc) +check( + "placeholder selector explicitly rejects multiple matches", + ambiguous_placeholder_message is not None + and "found 2" in ambiguous_placeholder_message + and "Content Placeholder 2" in ambiguous_placeholder_message + and "Content Placeholder 3" in ambiguous_placeholder_message, + ambiguous_placeholder_message, +) + +skeleton_prs = Presentation() +skeleton_slide = skeleton_prs.slides.add_slide(skeleton_prs.slide_layouts[5]) +skeleton_slide.shapes.title.text = "Regional service health" +skeleton_headers = ["Region", "Error rate", "P99 latency"] +skeleton_body = [ + ["Americas", "0.08%", "182 ms"], + ["Europe", "0.05%", "164 ms"], + ["Asia Pacific", "0.11%", "213 ms"], +] +skeleton_table = skeleton_slide.shapes.add_table( + 1 + len(skeleton_body), len(skeleton_headers), + Inches(0.5), Inches(1.5), Inches(9), Inches(3.5), +).table +for column_index, text in enumerate(skeleton_headers): + skeleton_table.cell(0, column_index).text = text +for row_index, row in enumerate(skeleton_body, start=1): + for column_index, text in enumerate(row): + skeleton_table.cell(row_index, column_index).text = text +skeleton_prs.save("create-skeleton.pptx") +skeleton_reopened = Presentation("create-skeleton.pptx") +skeleton_reopened_table = next( + shape.table for shape in skeleton_reopened.slides[0].shapes if shape.has_table +) +skeleton_values = [ + [cell.text.strip() for cell in row.cells] + for row in skeleton_reopened_table.rows +] +check( + "creation skeleton table has no unexpected empty cells", + skeleton_values == [skeleton_headers, *skeleton_body] + and all(text for row in skeleton_values for text in row), + skeleton_values, +) + # ---- analyze.md bounded package check rejects archive bombs before expansion --- +MAX_ARCHIVE_BYTES = 200 * 1024 * 1024 +MAX_MEMBERS = 10_000 MAX_XML_PART = 20 * 1024 * 1024 MAX_ENTRY = 100 * 1024 * 1024 MAX_TOTAL_UNCOMPRESSED = 500 * 1024 * 1024 @@ -111,8 +196,13 @@ def require(condition, message): def validate_pptx_package(path): + require( + Path(path).stat().st_size <= MAX_ARCHIVE_BYTES, + "compressed PPTX file size above limit", + ) with zipfile.ZipFile(path) as archive: infos = archive.infolist() + require(len(infos) <= MAX_MEMBERS, "archive member count above limit") names = {info.filename for info in infos} require(len(names) == len(infos), "duplicate archive member names are unsafe") require("[Content_Types].xml" in names and "ppt/presentation.xml" in names, @@ -150,6 +240,37 @@ def validate_pptx_package(path): except Exception: healthy_pptx_passed = False check("bounded PPTX check accepts an ordinary deck", healthy_pptx_passed) + +Path("oversized-before-open.pptx").write_bytes(b"not a ZIP package") +original_archive_limit = MAX_ARCHIVE_BYTES +MAX_ARCHIVE_BYTES = 0 +try: + validate_pptx_package("oversized-before-open.pptx") + compressed_size_rejected_before_open = False +except ValueError as exc: + compressed_size_rejected_before_open = ( + str(exc) == "compressed PPTX file size above limit" + ) +finally: + MAX_ARCHIVE_BYTES = original_archive_limit +check( + "compressed PPTX size is bounded before ZipFile opens the package", + compressed_size_rejected_before_open, +) + +with zipfile.ZipFile("too-many-members.pptx", "w", zipfile.ZIP_STORED) as archive: + for member_index in range(MAX_MEMBERS + 1): + archive.writestr(f"zero-{member_index:05d}.bin", b"") +try: + validate_pptx_package("too-many-members.pptx") + many_members_rejected = False +except ValueError as exc: + many_members_rejected = str(exc) == "archive member count above limit" +check( + "member-count gate rejects 10,001 zero-byte members before traversal", + many_members_rejected, +) + with zipfile.ZipFile("compressed-bomb.pptx", "w", zipfile.ZIP_DEFLATED) as archive: archive.writestr("[Content_Types].xml", "") archive.writestr( diff --git a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py index b7a5d14..e617899 100644 --- a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py @@ -88,12 +88,15 @@ def spreadsheet_csv_field(value, *, mode="safe"): check("CSV export rejects an ambiguous safety mode", invalid_csv_mode_rejected) # ---- edit.md snippet: round_trip_changes detects dropped parts AND stripped extensions ---- +from collections import Counter +from xml.etree import ElementTree as ET wb = openpyxl.Workbook() ws = wb.active ws.title = "Data" ws.append(["Region", "Units"]) ws.append(["EU", 120]) +wb.create_sheet("Keep")["A1"] = "keep" wb.save("plain.xlsx") with zipfile.ZipFile("plain.xlsx") as zin: @@ -103,89 +106,152 @@ def spreadsheet_csv_field(value, *, mode="safe"): payload["xl/slicers/slicer1.xml"] = b"" payload["xl/media/large.bin"] = b"x14:" * 32_768 # binary payload must never be marker-scanned -# simulate an in-part x14 extension that openpyxl will strip while keeping the part name -EXT = (b"" - b"") +# Two records deliberately share a URI/namespace but carry different child content. A coarse +# presence set cannot see one disappear while the other survives. +SHEET_NS = "http://schemas.openxmlformats.org/spreadsheetml/2006/main" +X14_NS = "http://schemas.microsoft.com/office/spreadsheetml/2009/9/main" +EXT_URI = "{00000000-0000-0000-0000-000000000000}" +KEEP_EXT = ( + f'' +).encode() +KEEP_EXT_ALT_PREFIX = ( + f'' +).encode() +DROP_EXT = ( + f'' +).encode() +EXT_LIST_BOTH = b"" + KEEP_EXT + DROP_EXT + b"" +EXT_LIST_KEEP = b"" + KEEP_EXT_ALT_PREFIX + b"" +EXT_LIST_DROP = b"" + DROP_EXT + b"" payload["xl/worksheets/sheet1.xml"] = payload["xl/worksheets/sheet1.xml"].replace( - b"", EXT + b"") + b"", EXT_LIST_BOTH + b"") +payload["xl/worksheets/sheet2.xml"] = payload["xl/worksheets/sheet2.xml"].replace( + b"", EXT_LIST_DROP + b"") with zipfile.ZipFile("extended.xlsx", "w", zipfile.ZIP_DEFLATED) as zout: for name, data in payload.items(): zout.writestr(name, data) -EXTENSION_MARKERS = { - "extLst": b" Date: Mon, 17 Aug 2026 00:30:23 +0800 Subject: [PATCH 25/34] fix(document-skills): harden final verification paths --- .../document-skills/skills/pdf/SKILL.md | 10 +- .../skills/pdf/references/extract.md | 13 +- .../skills/pdf/references/inspect.md | 25 +- .../skills/pptx/references/analyze.md | 3 + .../skills/xlsx/references/csv.md | 79 +++++- .../skills/xlsx/references/read.md | 25 +- .../document-skills/tests/pdf_fixture.py | 102 ++++++- .../document-skills/tests/pptx_fixture.py | 58 ++++ .../document-skills/tests/xlsx_fixture.py | 250 +++++++++++++++++- 9 files changed, 528 insertions(+), 37 deletions(-) diff --git a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md index 6295da2..3726ea1 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md @@ -55,6 +55,7 @@ import os import pypdf output_path = "output.pdf" +expected_page_count = 1 # set this from the task; do not derive it from the output password = os.environ.get("PDF_PASSWORD") r = pypdf.PdfReader(output_path) if r.is_encrypted: @@ -71,6 +72,10 @@ def require(condition, message): raise ValueError(message) page_count = len(r.pages) +require( + page_count == expected_page_count, + f"expected {expected_page_count} pages, got {page_count}", +) page_texts = { number: (page.extract_text() or "").strip() for number, page in enumerate(r.pages, start=1) @@ -121,10 +126,11 @@ require(not size_mismatches, f"unexpected page sizes: {size_mismatches}") import fitz overflow_doc = fitz.open(output_path) -if overflow_doc.needs_pass: +if overflow_doc.needs_pass and overflow_doc.authenticate("") <= 0: if not password: raise RuntimeError("set PDF_PASSWORD so the encrypted output can be overflow-checked") - overflow_doc.authenticate(password) + if overflow_doc.authenticate(password) <= 0: + raise RuntimeError("PDF_PASSWORD could not decrypt the output for overflow checking") overflow_pages = [] for page in overflow_doc: # These APIs report unrotated coordinates even when /Rotate is 90/270; diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md b/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md index 6eb5629..dcba4b2 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md @@ -62,12 +62,15 @@ for page in doc: pix = base pix.save(f"img-p{page_number}-{i}.png") - # Inline images live in the page content stream, not the XObject table, so - # get_images() never lists them. Enumerate image blocks from the dict pass - # and report both lists: the xref loop is authoritative for XObjects, the - # block pass catches inline placements. + # Type-1 blocks include both XObjects and inline images. An xref of zero + # identifies true inline content; positive xrefs were already exported by + # the XObject loop above and must not be written a second time. + image_xrefs = { + image["number"]: image["xref"] + for image in page.get_image_info(xrefs=True) + } for b in page.get_text("dict")["blocks"]: - if b["type"] != 1: + if b["type"] != 1 or image_xrefs.get(b["number"]) != 0: continue ext = b.get("ext") or "png" with open(f"img-p{page_number}-inline-{b['number']}.{ext}", "wb") as fh: diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md b/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md index 6204185..1200b00 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md @@ -37,7 +37,16 @@ def font_inventory(document, page): print("pages:", doc.page_count) print("password_protected:", doc.needs_pass, "| still_encrypted:", doc.is_encrypted, "| pdf:", doc.is_pdf) +page_geometry = [] for page in doc: + media_size = (round(page.mediabox.width, 2), round(page.mediabox.height, 2)) + crop_size = (round(page.cropbox.width, 2), round(page.cropbox.height, 2)) + page_geometry.append({ + "page": page.number + 1, + "media_size": media_size, + "crop_size": crop_size, + "rotation": page.rotation, + }) blocks = page.get_text("dict")["blocks"] image_blocks = [block for block in blocks if block["type"] == 1] drawings = page.get_drawings() @@ -48,12 +57,15 @@ for page in doc: page.get_text().strip() or page.get_images() or image_blocks or drawings or widgets or annotations or links ) - print(page.number, page.rect, "text_len:", len(page.get_text()), + print(page.number + 1, "media_size:", media_size, "crop_size:", crop_size, + "rotation:", page.rotation, "text_len:", len(page.get_text()), "resource_images:", len(page.get_images()), "image_blocks:", len(image_blocks), "drawings:", len(drawings), "widgets:", len(widgets), "annotations:", len(annotations), "links:", len(links), "blank:", is_blank) print(" fonts:", font_inventory(doc, page)) +print("media_size_consistent:", len({row["media_size"] for row in page_geometry}) <= 1) +print("crop_size_consistent:", len({row["crop_size"] for row in page_geometry}) <= 1) ``` ## Checks worth automating @@ -67,10 +79,13 @@ for page in doc: - **Font inventory**: `page.get_fonts()` lists referenced fonts, including non-embedded base fonts. Use `doc.extract_font(xref)` as above and report `embedded` separately; a referenced face with no extractable program may be substituted on another machine. -- **Page size consistency**: mixed `page.rect` sizes in one file break duplex printing; report - it rather than silently normalizing. +- **Page size consistency**: compare unrotated `(width, height)` pairs from `page.mediabox` and + `page.cropbox`, and report `page.rotation` separately. Do not compare `page.rect`: it applies + `/Rotate`, so otherwise identical paper appears to swap width and height at 90 or 270 degrees. + Report genuinely mixed media or crop sizes rather than silently normalizing them. - **Damage**: `fitz.open` on a corrupt file raises or yields garbage - pair with `pypdf.PdfReader` cross-check when provenance is unknown. -Report findings as a table (page, size, text chars, resource images, image blocks, drawings, -widgets, annotations, links, blank) - it is what every downstream decision hangs off. +Report findings as a table (page, media size, crop size, rotation, text chars, resource images, +image blocks, drawings, widgets, annotations, links, blank) - it is what every downstream +decision hangs off. diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md index 109789e..18efabd 100644 --- a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md @@ -64,6 +64,9 @@ def cached_numeric_points(source): def series_content(series): x_source = getattr(series._element, "xVal", None) if x_source is None: # category/value chart + value_source = getattr(series._element, "val", None) + if cached_numeric_points(value_source) is None: + return {"name": series.name, "values": None, "cache_status": "unavailable"} return {"name": series.name, "values": list(series.values)} x_points = cached_numeric_points(x_source) y_points = cached_numeric_points(getattr(series._element, "yVal", None)) diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md index 8a8a3d2..03f3f6c 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md @@ -75,15 +75,28 @@ formula-injection protection**; do not present a raw export as safe to open in a cell.data_type = "s" # openpyxl otherwise promotes it to an XLSX formula ``` - XLSX -> CSV: use a separate `data_only=True` read so formulas export the cached values users - see, not formula strings. Pair it with a formula-preserving read and report missing caches; - cached values can also be stale until a spreadsheet application recalculates the workbook: + see, not formula strings. Pair it with a formula-preserving read and report missing caches. + `openpyxl` exposes both a missing cache and a present empty-string cache as `None`, so consult + the worksheet XML. A nonempty `` is cached; an empty `` is a valid displayed blank only + when the formula cell explicitly has string-result type `t="str"`. A missing `` or the bare + empty `` that openpyxl writes for an uncalculated formula must fail closed. Cached values + can still be stale until a spreadsheet application recalculates the workbook: ```python import csv import openpyxl + import posixpath + import zipfile from pathlib import Path + from xml.etree import ElementTree as ET FORMULA_PREFIXES = ("=", "+", "-", "@") + MAIN_NS = "http://schemas.openxmlformats.org/spreadsheetml/2006/main" + DOC_REL_NS = "http://schemas.openxmlformats.org/officeDocument/2006/relationships" + PKG_REL_NS = "http://schemas.openxmlformats.org/package/2006/relationships" + CELL_TAG = f"{{{MAIN_NS}}}c" + FORMULA_TAG = f"{{{MAIN_NS}}}f" + VALUE_TAG = f"{{{MAIN_NS}}}v" def spreadsheet_csv_field(value, *, mode="safe"): if mode not in {"safe", "raw"}: @@ -92,10 +105,63 @@ formula-injection protection**; do not present a raw export as safe to open in a return "'" + value return value + def worksheet_part(archive, sheet_name): + workbook = ET.fromstring(archive.read("xl/workbook.xml")) + sheet = next( + item for item in workbook.iter(f"{{{MAIN_NS}}}sheet") + if item.attrib["name"] == sheet_name + ) + relationship_id = sheet.attrib[f"{{{DOC_REL_NS}}}id"] + relationships = ET.fromstring(archive.read("xl/_rels/workbook.xml.rels")) + target = next( + item.attrib["Target"] for item in relationships.iter(f"{{{PKG_REL_NS}}}Relationship") + if item.attrib["Id"] == relationship_id + ) + if target.startswith("/"): + return target.lstrip("/") + return posixpath.normpath(posixpath.join("xl", target)) + + def cached_formula_coordinates(archive, part): + """Find formula cells with a nonempty cache or an explicitly typed empty-string cache.""" + cached = set() + coordinate = None + cell_type = None + has_formula = value_seen = False + value_text = None + with archive.open(part) as source: + for event, element in ET.iterparse(source, events=("start", "end")): + if event == "start" and element.tag == CELL_TAG: + coordinate = element.attrib["r"] + cell_type = element.attrib.get("t") + has_formula = value_seen = False + value_text = None + elif event == "end" and coordinate is not None: + if element.tag == FORMULA_TAG: + has_formula = True + elif element.tag == VALUE_TAG: + value_seen = True + value_text = element.text + elif element.tag == CELL_TAG: + valid_cache = value_seen and ( + value_text not in (None, "") or cell_type == "str" + ) + if has_formula and valid_cache: + cached.add(coordinate) + coordinate = None + element.clear() + elif event == "end": + element.clear() + return cached + + input_path = "input.xlsx" + sheet_name = "Data" export_mode = "safe" # use "raw" only for explicitly requested trusted machine interchange - formula_wb = openpyxl.load_workbook("input.xlsx", read_only=True, data_only=False) - value_wb = openpyxl.load_workbook("input.xlsx", read_only=True, data_only=True) - formula_ws, value_ws = formula_wb["Data"], value_wb["Data"] + with zipfile.ZipFile(input_path) as archive: + part = worksheet_part(archive, sheet_name) + cached_formula_cells = cached_formula_coordinates(archive, part) + formula_wb = openpyxl.load_workbook(input_path, read_only=True, data_only=False) + value_wb = openpyxl.load_workbook(input_path, read_only=True, data_only=True) + formula_ws, value_ws = formula_wb[sheet_name], value_wb[sheet_name] # Producer-written metadata can look plausible while truncating real cells. # Reset both paired streams before their first iter_rows() call. formula_ws.reset_dimensions() @@ -107,7 +173,8 @@ formula-injection protection**; do not present a raw export as safe to open in a writer = csv.writer(output) for formula_row, value_row in zip(formula_ws.iter_rows(), value_ws.iter_rows()): for formula_cell, value_cell in zip(formula_row, value_row): - if formula_cell.data_type == "f" and value_cell.value is None: + if (formula_cell.data_type == "f" and value_cell.value is None + and formula_cell.coordinate not in cached_formula_cells): missing_caches.append(formula_cell.coordinate) writer.writerow([ spreadsheet_csv_field(cell.value, mode=export_mode) for cell in value_row diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md index b130433..e6c2b9c 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md @@ -35,11 +35,12 @@ def discover_dimension(worksheet): max_row = row_index if max_row is None else max(max_row, row_index) max_column = column_index if max_column is None else max(max_column, column_index) if max_row is None: - return "A1:A1" - return ( + return "A1:A1", None + extent = ( f"{get_column_letter(min_column)}{min_row}:" f"{get_column_letter(max_column)}{max_row}" ) + return extent, min_row # Profile EVERY sheet by default; only narrow when the task names a specific sheet. for sheet_name in value_wb.sheetnames: @@ -50,18 +51,23 @@ for sheet_name in value_wb.sheetnames: # example A1:B2 while data continues below it), so treat that metadata as # untrusted: reset both streams and discover the real bounds before reading. declared = value_ws.calculate_dimension() - discovered = discover_dimension(value_ws) - formula_discovered = discover_dimension(formula_ws) - if formula_discovered != discovered: + discovered, first_populated_row = discover_dimension(value_ws) + formula_discovered, formula_first_row = discover_dimension(formula_ws) + if (formula_discovered, formula_first_row) != (discovered, first_populated_row): raise ValueError( - f"formula/value stream dimensions disagree: {formula_discovered} != {discovered}" + "formula/value stream bounds disagree: " + f"{(formula_discovered, formula_first_row)} != " + f"{(discovered, first_populated_row)}" ) if discovered != declared: print(f"--- {sheet_name} --- declared {declared!r}; discovered real extent:") print(f"--- {sheet_name} --- dims:", discovered) - rows = value_ws.iter_rows(values_only=True) - header = next(rows, None) + if first_populated_row is None: + rows, header = iter(()), None + else: + rows = value_ws.iter_rows(min_row=first_populated_row, values_only=True) + header = next(rows, None) print("header:", header) for i, row in enumerate(rows): if i >= 5: break @@ -84,7 +90,8 @@ value_wb.close() - First pass always: sheet names, per-sheet dimensions, header row, 5 sample rows. Report those before any analysis. Multi-sheet workbooks report every sheet - a profile that - silently covers only `sheetnames[0]` is incomplete. + silently covers only `sheetnames[0]` is incomplete. Begin the header/sample iterator at the + discovered first populated row; leading blank rows are not a header. - `read_only=True` streams large files; you lose random access (`ws["B2"]` works but is slow in read_only mode - iterate instead). - `data_only=True` gives cached values. A file saved by a library (never opened in Excel) diff --git a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py index ae67627..9ceb1a7 100644 --- a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py @@ -56,6 +56,23 @@ def open_pdf(path, password=None): page_sizes, ) + +def verify_page_count(reader, expected_page_count): + page_count = len(reader.pages) + if page_count != expected_page_count: + raise ValueError(f"expected {expected_page_count} pages, got {page_count}") + return page_count + + +check("postcheck accepts the requested page count", verify_page_count(r, 2) == 2) +try: + verify_page_count(r, 1) + page_count_mismatch = "" +except ValueError as exc: + page_count_mismatch = str(exc) +check("postcheck rejects a page-count mismatch even under python -O", + page_count_mismatch == "expected 1 pages, got 2", page_count_mismatch) + # ---- inspect.md distinguishes referenced-only and embedded fonts --------------- pdfmetrics.registerFont(TTFont("FixtureVera", os.path.join( os.path.dirname(reportlab.__file__), "fonts", "Vera.ttf", @@ -479,6 +496,33 @@ def inspected_page_is_blank(page): check("form-only page exposes a widget", len(form_only_widgets) == 1) check("form-only page is not classified as blank", not form_only_is_blank) +# Identical media/crop geometry remains consistent when one page has /Rotate 90. +mixed_rotation_writer = PdfWriter() +mixed_rotation_writer.append(PdfReader("form.pdf")) +mixed_rotation_writer.pages[1].rotate(90) +with open("inspect-mixed-rotation.pdf", "wb") as f: + mixed_rotation_writer.write(f) +mixed_rotation_doc = fitz.open("inspect-mixed-rotation.pdf") +mixed_media_sizes = { + (round(page.mediabox.width, 2), round(page.mediabox.height, 2)) + for page in mixed_rotation_doc +} +mixed_crop_sizes = { + (round(page.cropbox.width, 2), round(page.cropbox.height, 2)) + for page in mixed_rotation_doc +} +mixed_display_sizes = { + (round(page.rect.width, 2), round(page.rect.height, 2)) + for page in mixed_rotation_doc +} +check("page.rect falsely reports mixed sizes for identical rotated media (negative control)", + len(mixed_display_sizes) == 2, mixed_display_sizes) +check("unrotated media and crop sizes stay consistent across mixed rotation", + len(mixed_media_sizes) == 1 and len(mixed_crop_sizes) == 1, + (mixed_media_sizes, mixed_crop_sizes)) +check("inspection reports mixed rotation separately from paper size", + [page.rotation for page in mixed_rotation_doc] == [0, 90]) + # ReportLab's drawInlineImage emits BI/ID/EI content rather than an image XObject. inline_source = fitz.Pixmap(fitz.csRGB, fitz.IRect(0, 0, 12, 12)) inline_source.clear_with(96) @@ -500,6 +544,55 @@ def inspected_page_is_blank(page): check("inline-only image page is not classified as blank", not inspected_page_is_blank(inline_page)) +# The extraction XObject pass and inline pass are disjoint: type-1 blocks with +# positive xrefs belong to the first pass, while true inline blocks have xref 0. +xobject_canvas = canvas.Canvas("xobject-image.pdf", pagesize=A4) +xobject_canvas.drawImage("inline-only-source.png", 72, 700, width=80, height=80) +xobject_canvas.showPage() +xobject_canvas.save() + + +def extract_images_without_duplicates(document, page, prefix): + outputs = [] + for i, info in enumerate(page.get_images(full=True), start=1): + pix = fitz.Pixmap(document, info[0]) + output = f"{prefix}-{i}.png" + pix.save(output) + outputs.append(output) + image_xrefs = { + image["number"]: image["xref"] + for image in page.get_image_info(xrefs=True) + } + for block in page.get_text("dict")["blocks"]: + if block["type"] != 1 or image_xrefs.get(block["number"]) != 0: + continue + ext = block.get("ext") or "png" + output = f"{prefix}-inline-{block['number']}.{ext}" + with open(output, "wb") as fh: + fh.write(block["image"]) + outputs.append(output) + return outputs + + +xobject_doc = fitz.open("xobject-image.pdf") +xobject_blocks = [ + block for block in xobject_doc[0].get_text("dict")["blocks"] + if block["type"] == 1 +] +xobject_info = xobject_doc[0].get_image_info(xrefs=True) +xobject_outputs = extract_images_without_duplicates(xobject_doc, xobject_doc[0], "xobject-export") +check("ordinary XObject image info exposes a positive xref", + len(xobject_blocks) == 1 and len(xobject_info) == 1 + and xobject_info[0]["xref"] > 0, + xobject_info) +check("ordinary XObject is exported once, not duplicated by the inline pass", + len(xobject_outputs) == 1 and "-inline-" not in xobject_outputs[0], xobject_outputs) +inline_outputs = extract_images_without_duplicates(inline_doc, inline_page, "inline-export") +check("true inline image is exported by the xref-zero second pass", + len(inline_outputs) == 1 and "-inline-" in inline_outputs[0] + and os.path.getsize(inline_outputs[0]) > 0, + inline_outputs) + # ---- extract.md CMYK conversion snippet --------------------------------------- pix = fitz.Pixmap(fitz.csCMYK, fitz.IRect(0, 0, 24, 24)) # CMYK pixmap like a CMYK PDF image converted = fitz.Pixmap(fitz.csRGB, pix) if pix.colorspace not in (fitz.csGRAY, fitz.csRGB) else pix @@ -829,8 +922,13 @@ def drawing_bounds(drawing): ) -def overflow_pages(path): +def overflow_pages(path, password=None): doc = fitz.open(path) + if doc.needs_pass and doc.authenticate("") <= 0: + if not password: + raise RuntimeError(f"valid password required to overflow-check {path}") + if doc.authenticate(password) <= 0: + raise RuntimeError(f"password could not decrypt {path} for overflow checking") pages = [] for page in doc: # Plain block extraction drops fully off-page text; enlarge the clip. @@ -857,6 +955,8 @@ def overflow_pages(path): return pages check("in-bounds PDF reports no overflow pages", overflow_pages("overflow.pdf") == []) +check("blank-user-password encrypted PDF passes the independent overflow check", + overflow_pages("blank-user-password.pdf") == []) check("off-page text is detected by the overflow check (negative control)", overflow_pages("overflow-bad.pdf") == [1]) check("in-bounds image and vector drawing pass the overflow check", diff --git a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py index 2b58bcd..40b19be 100644 --- a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py @@ -407,6 +407,9 @@ def cached_numeric_points(source): def series_content(series): x_source = getattr(series._element, "xVal", None) if x_source is None: + value_source = getattr(series._element, "val", None) + if cached_numeric_points(value_source) is None: + return {"name": series.name, "values": None, "cache_status": "unavailable"} return {"name": series.name, "values": list(series.values)} x_points = cached_numeric_points(x_source) y_points = cached_numeric_points(getattr(series._element, "yVal", None)) @@ -528,6 +531,61 @@ def extract_slide_content(slide): == [(0, 5.0)]) check("content inventory emits notes text", "regional split" in content["notes"], content["notes"]) +# Category/value series can retain an external workbook formula without a numCache. +category_prs = Presentation() +category_slide = category_prs.slides.add_slide(category_prs.slide_layouts[6]) +category_data = ChartData() +category_data.categories = ["A", "B"] +category_data.add_series("Missing cache", (1, 2)) +category_data.add_series("Cached series", (3, 4)) +category_slide.shapes.add_chart( + XL_CHART_TYPE.COLUMN_CLUSTERED, + Inches(0.5), Inches(0.5), Inches(6), Inches(4), category_data, +) +category_prs.save("category-cache-source.pptx") + +category_mutated = Presentation("category-cache-source.pptx") +category_chart = next( + shape.chart for shape in category_mutated.slides[0].shapes if shape.has_chart +) +category_series = list(category_chart.plots[0].series) +missing_value_ref = category_series[0]._element.val.find(qn("c:numRef")) +missing_formula_before = missing_value_ref.find(qn("c:f")).text +missing_value_ref.remove(missing_value_ref.find(qn("c:numCache"))) +category_mutated.save("category-cacheless.pptx") + +category_reopened = Presentation("category-cacheless.pptx") +category_reopened_chart = next( + shape.chart for shape in category_reopened.slides[0].shapes if shape.has_chart +) +category_reopened_series = list(category_reopened_chart.plots[0].series) +reopened_value_ref = category_reopened_series[0]._element.val.find(qn("c:numRef")) +reopened_formula = reopened_value_ref.find(qn("c:f")) +with zipfile.ZipFile("category-cacheless.pptx") as category_archive: + embedded_workbook_remains = any( + name.startswith("ppt/embeddings/") for name in category_archive.namelist() + ) +check( + "cacheless category series retains its formula and embedded workbook", + reopened_formula is not None + and reopened_formula.text == missing_formula_before + and reopened_value_ref.find(qn("c:numCache")) is None + and embedded_workbook_remains, +) +category_inventory = extract_slide_content(category_reopened.slides[0]) +category_inventory_series = category_inventory["charts"][0]["plots"][0]["series"] +check( + "cacheless category series is unavailable without aborting inventory", + category_inventory_series[0] + == {"name": "Missing cache", "values": None, "cache_status": "unavailable"}, + category_inventory_series, +) +check( + "cached category series still reports its values after a cacheless sibling", + category_inventory_series[1] == {"name": "Cached series", "values": [3.0, 4.0]}, + category_inventory_series, +) + diagram_frame = etree.fromstring(f''' for uncached formulas. Remove those elements explicitly so +# this fixture represents a truly absent cache rather than a cached displayed blank. +with zipfile.ZipFile("multi.xlsx") as archive: + multi_members = {name: archive.read(name) for name in archive.namelist()} +for name in [item for item in multi_members if item.startswith("xl/worksheets/") + and item.endswith(".xml")]: + root = ET.fromstring(multi_members[name]) + for cell in root.iter(CELL_TAG): + if cell.find(FORMULA_TAG) is not None: + cached_value = cell.find(VALUE_TAG) + if cached_value is not None: + cell.remove(cached_value) + multi_members[name] = ET.tostring(root, encoding="utf-8", xml_declaration=True) +with zipfile.ZipFile("multi.xlsx", "w", zipfile.ZIP_DEFLATED) as archive: + for name, data in multi_members.items(): + archive.writestr(name, data) formula_wb = openpyxl.load_workbook("multi.xlsx", read_only=True, data_only=False) value_wb = openpyxl.load_workbook("multi.xlsx", read_only=True, data_only=True) profiled = list(value_wb.sheetnames) @@ -666,6 +692,58 @@ class LegacyWorkbook: value_wb.close() # csv.md: value export uses the cached-value workbook and reports every missing cache. +def worksheet_part(archive, sheet_name): + workbook = ET.fromstring(archive.read("xl/workbook.xml")) + sheet = next( + item for item in workbook.iter(f"{{{MAIN_NS}}}sheet") + if item.attrib["name"] == sheet_name + ) + relationship_id = sheet.attrib[f"{{{DOC_REL_NS}}}id"] + relationships = ET.fromstring(archive.read("xl/_rels/workbook.xml.rels")) + target = next( + item.attrib["Target"] for item in relationships.iter(f"{{{PKG_REL_NS}}}Relationship") + if item.attrib["Id"] == relationship_id + ) + if target.startswith("/"): + return target.lstrip("/") + return posixpath.normpath(posixpath.join("xl", target)) + + +def cached_formula_coordinates(archive, part): + cached = set() + coordinate = None + cell_type = None + has_formula = value_seen = False + value_text = None + with archive.open(part) as source: + for event, element in ET.iterparse(source, events=("start", "end")): + if event == "start" and element.tag == CELL_TAG: + coordinate = element.attrib["r"] + cell_type = element.attrib.get("t") + has_formula = value_seen = False + value_text = None + elif event == "end" and coordinate is not None: + if element.tag == FORMULA_TAG: + has_formula = True + elif element.tag == VALUE_TAG: + value_seen = True + value_text = element.text + elif element.tag == CELL_TAG: + valid_cache = value_seen and ( + value_text not in (None, "") or cell_type == "str" + ) + if has_formula and valid_cache: + cached.add(coordinate) + coordinate = None + element.clear() + elif event == "end": + element.clear() + return cached + + +with zipfile.ZipFile("multi.xlsx") as archive: + multi_part = worksheet_part(archive, "First") + cached_formula_cells = cached_formula_coordinates(archive, multi_part) formula_wb = openpyxl.load_workbook("multi.xlsx", read_only=True, data_only=False) value_wb = openpyxl.load_workbook("multi.xlsx", read_only=True, data_only=True) formula_ws, value_ws = formula_wb["First"], value_wb["First"] @@ -677,7 +755,8 @@ class LegacyWorkbook: writer = csv.writer(output) for formula_row, value_row in zip(formula_ws.iter_rows(), value_ws.iter_rows()): for formula_cell, value_cell in zip(formula_row, value_row): - if formula_cell.data_type == "f" and value_cell.value is None: + if (formula_cell.data_type == "f" and value_cell.value is None + and formula_cell.coordinate not in cached_formula_cells): missing_caches.append(formula_cell.coordinate) writer.writerow([ spreadsheet_csv_field(cell.value, mode=export_mode) for cell in value_row @@ -693,6 +772,126 @@ class LegacyWorkbook: formula_wb.close() value_wb.close() + +def write_formula_cache_fixture(path, cache_kind): + workbook = openpyxl.Workbook() + sheet = workbook.active + sheet.title = "Data" + sheet["A1"] = ('=IF(TRUE,"","x")' if cache_kind == "empty-string" else "=1+1") + workbook.save(path) + with zipfile.ZipFile(path) as archive: + members = {name: archive.read(name) for name in archive.namelist()} + root = ET.fromstring(members["xl/worksheets/sheet1.xml"]) + cell = next(item for item in root.iter(CELL_TAG) if item.attrib["r"] == "A1") + cached_value = cell.find(VALUE_TAG) + if cache_kind == "nonempty": + cell.attrib.pop("t", None) + if cached_value is None: + cached_value = ET.SubElement(cell, VALUE_TAG) + cached_value.text = "2" + elif cache_kind == "empty-string": + cell.set("t", "str") + if cached_value is None: + cached_value = ET.SubElement(cell, VALUE_TAG) + cached_value.text = None + elif cache_kind == "bare-empty": + cell.attrib.pop("t", None) + if cached_value is None: + cached_value = ET.SubElement(cell, VALUE_TAG) + cached_value.text = None + elif cache_kind == "absent": + cell.attrib.pop("t", None) + if cached_value is not None: + cell.remove(cached_value) + else: + raise ValueError(cache_kind) + members["xl/worksheets/sheet1.xml"] = ET.tostring( + root, encoding="utf-8", xml_declaration=True, + ) + with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as archive: + for name, data in members.items(): + archive.writestr(name, data) + + +def export_formula_values(source_path, output_path): + sheet_name = "Data" + with zipfile.ZipFile(source_path) as archive: + part = worksheet_part(archive, sheet_name) + cached_cells = cached_formula_coordinates(archive, part) + formula_book = openpyxl.load_workbook(source_path, read_only=True, data_only=False) + value_book = openpyxl.load_workbook(source_path, read_only=True, data_only=True) + formula_sheet, value_sheet = formula_book[sheet_name], value_book[sheet_name] + formula_sheet.reset_dimensions() + value_sheet.reset_dimensions() + missing = [] + destination = Path(output_path) + temporary = destination.with_suffix(destination.suffix + ".tmp") + with temporary.open("w", newline="", encoding="utf-8") as output: + writer = csv.writer(output) + for formula_row, value_row in zip(formula_sheet.iter_rows(), value_sheet.iter_rows()): + for formula_cell, value_cell in zip(formula_row, value_row): + if (formula_cell.data_type == "f" and value_cell.value is None + and formula_cell.coordinate not in cached_cells): + missing.append(formula_cell.coordinate) + writer.writerow([spreadsheet_csv_field(cell.value) for cell in value_row]) + formula_book.close() + value_book.close() + if missing: + temporary.unlink(missing_ok=True) + raise RuntimeError(f"formula cells have no cached value: {missing}") + temporary.replace(destination) + + +write_formula_cache_fixture("cached-value.xlsx", "nonempty") +write_formula_cache_fixture("cached-empty.xlsx", "empty-string") +write_formula_cache_fixture("bare-empty-cache.xlsx", "bare-empty") +write_formula_cache_fixture("missing-cache.xlsx", "absent") +with zipfile.ZipFile("cached-value.xlsx") as archive: + cached_value_cells = cached_formula_coordinates(archive, worksheet_part(archive, "Data")) +with zipfile.ZipFile("cached-empty.xlsx") as archive: + cached_empty_cells = cached_formula_coordinates(archive, worksheet_part(archive, "Data")) +with zipfile.ZipFile("bare-empty-cache.xlsx") as archive: + bare_empty_cells = cached_formula_coordinates(archive, worksheet_part(archive, "Data")) +with zipfile.ZipFile("missing-cache.xlsx") as archive: + truly_missing_cells = cached_formula_coordinates(archive, worksheet_part(archive, "Data")) +check("XML cache inventory accepts nonempty and typed empty-string caches", + cached_value_cells == cached_empty_cells == {"A1"}, + (cached_value_cells, cached_empty_cells)) +check("XML cache inventory rejects bare empty and an absent cache", + bare_empty_cells == truly_missing_cells == set(), + (bare_empty_cells, truly_missing_cells)) +export_formula_values("cached-value.xlsx", "cached-value.csv") +with open("cached-value.csv", newline="", encoding="utf-8") as exported: + cached_value_rows = list(csv.reader(exported)) +check("nonempty cached formula result exports its displayed value", + cached_value_rows == [["2"]], cached_value_rows) +export_formula_values("cached-empty.xlsx", "cached-empty.csv") +with open("cached-empty.csv", newline="", encoding="utf-8") as exported: + cached_empty_rows = list(csv.reader(exported)) +check("cached empty-string formula result exports as a displayed blank", + cached_empty_rows == [[""]], cached_empty_rows) +Path("missing-cache.csv").write_text("sentinel\n", encoding="utf-8") +try: + export_formula_values("missing-cache.xlsx", "missing-cache.csv") + missing_cache_rejected = False +except RuntimeError as error: + missing_cache_rejected = "A1" in str(error) +check("formula with no XML cache element is rejected", missing_cache_rejected) +check("failed cache audit preserves the prior destination and removes its temporary file", + Path("missing-cache.csv").read_text(encoding="utf-8") == "sentinel\n" + and not Path("missing-cache.csv.tmp").exists()) +Path("bare-empty-cache.csv").write_text("sentinel\n", encoding="utf-8") +try: + export_formula_values("bare-empty-cache.xlsx", "bare-empty-cache.csv") + bare_empty_rejected = False +except RuntimeError as error: + bare_empty_rejected = "A1" in str(error) +check("untyped bare empty from an uncalculated formula is rejected", + bare_empty_rejected) +check("bare-empty rejection preserves destination and removes temporary output", + Path("bare-empty-cache.csv").read_text(encoding="utf-8") == "sentinel\n" + and not Path("bare-empty-cache.csv.tmp").exists()) + # SKILL.md contract: fullCalcOnLoad makes viewers recalculate even in manual calc mode. calc_wb = openpyxl.Workbook() calc_ws = calc_wb.active @@ -849,9 +1048,10 @@ def discover_dimension(worksheet): max_row = row_index if max_row is None else max(max_row, row_index) max_column = column_index if max_column is None else max(max_column, column_index) if max_row is None: - return "A1:A1" - return (f"{get_column_letter(min_column)}{min_row}:" - f"{get_column_letter(max_column)}{max_row}") + return "A1:A1", None + extent = (f"{get_column_letter(min_column)}{min_row}:" + f"{get_column_letter(max_column)}{max_row}") + return extent, min_row dim_value = openpyxl.load_workbook("dimension.xlsx", read_only=True, data_only=True) @@ -861,15 +1061,17 @@ def discover_dimension(worksheet): dim_ws_ro.calculate_dimension() == "A1:B2" and dim_ws_ro.max_row == 2, (dim_ws_ro.calculate_dimension(), dim_ws_ro.max_row)) streamed_before_reset = [row for row in dim_ws_ro.iter_rows(values_only=True)] -discovered_value_dimension = discover_dimension(dim_ws_ro) -discovered_formula_dimension = discover_dimension(dim_formula.active) +discovered_value_dimension, discovered_value_first_row = discover_dimension(dim_ws_ro) +discovered_formula_dimension, discovered_formula_first_row = discover_dimension(dim_formula.active) streamed_after_reset = [row for row in dim_ws_ro.iter_rows(values_only=True)] dim_value.close() dim_formula.close() check("dimension scan recovers plausible row/column truncation in both workbook views", discovered_value_dimension == "A1:C4" - and discovered_formula_dimension == "A1:C4", - (discovered_value_dimension, discovered_formula_dimension)) + and discovered_formula_dimension == "A1:C4" + and discovered_value_first_row == discovered_formula_first_row == 1, + ((discovered_value_dimension, discovered_value_first_row), + (discovered_formula_dimension, discovered_formula_first_row))) check("reset_dimensions restores the real extent", len(streamed_before_reset) == 2 and len(streamed_after_reset) == 4 @@ -895,11 +1097,41 @@ def discover_dimension(worksheet): csv_formula_wb.close() csv_value_wb.close() +offset_wb = openpyxl.Workbook() +offset_ws = offset_wb.active +offset_ws["A7"], offset_ws["B7"] = "Region", "Units" +offset_ws["A8"], offset_ws["B8"] = "EU", 120 +offset_wb.save("leading-blank-rows.xlsx") +offset_formula_wb = openpyxl.load_workbook( + "leading-blank-rows.xlsx", read_only=True, data_only=False, +) +offset_value_wb = openpyxl.load_workbook( + "leading-blank-rows.xlsx", read_only=True, data_only=True, +) +offset_formula_extent, offset_formula_first = discover_dimension(offset_formula_wb.active) +offset_value_extent, offset_value_first = discover_dimension(offset_value_wb.active) +offset_rows = offset_value_wb.active.iter_rows( + min_row=offset_value_first, values_only=True, +) +offset_header = next(offset_rows, None) +offset_sample = next(offset_rows, None) +check("dimension discovery retains the first populated row in both read-only views", + (offset_formula_extent, offset_formula_first) == ("A7:B8", 7) + and (offset_value_extent, offset_value_first) == ("A7:B8", 7), + ((offset_formula_extent, offset_formula_first), + (offset_value_extent, offset_value_first))) +check("header sampling skips six leading blank rows", + offset_header[:2] == ("Region", "Units") + and offset_sample[:2] == ("EU", 120), + (offset_header, offset_sample)) +offset_formula_wb.close() +offset_value_wb.close() + empty_dimension_wb = openpyxl.Workbook() empty_dimension_wb.save("empty-dimension.xlsx") empty_dimension_ro = openpyxl.load_workbook("empty-dimension.xlsx", read_only=True) check("dimension scan handles an actually empty worksheet", - discover_dimension(empty_dimension_ro.active) == "A1:A1") + discover_dimension(empty_dimension_ro.active) == ("A1:A1", None)) empty_dimension_ro.close() From 943c278e7bdf46a77303c033777c88505e6f6648 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=98=E6=89=AC?= Date: Mon, 17 Aug 2026 06:14:54 +0800 Subject: [PATCH 26/34] fix(document-skills): close latest review gaps --- .../skills/docx/references/cjk.md | 19 +- .../skills/docx/references/edit.md | 49 +++--- .../document-skills/skills/pdf/SKILL.md | 6 + .../skills/pdf/references/extract.md | 2 +- .../skills/pdf/references/transform.md | 2 +- .../skills/pptx/references/analyze.md | 53 +++++- .../skills/xlsx/references/csv.md | 20 ++- .../skills/xlsx/references/formatting.md | 80 ++++++++- .../skills/xlsx/references/read.md | 87 ++++++++-- .../document-skills/tests/docx_fixture.py | 92 +++++++++- .../document-skills/tests/fixtures.test.mjs | 10 +- .../document-skills/tests/pdf_fixture.py | 46 +++++ .../document-skills/tests/pptx_fixture.py | 93 +++++++++- .../document-skills/tests/xlsx_fixture.py | 164 ++++++++++++++++-- 14 files changed, 658 insertions(+), 65 deletions(-) diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md b/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md index ea90d16..d239222 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md @@ -131,7 +131,24 @@ section.left_margin, section.right_margin = Cm(2.8), Cm(2.6) if rpr is None: return None rfonts = rpr.find(qn("w:rFonts")) - return None if rfonts is None else rfonts.get(qn("w:" + slot)) + if rfonts is None: + return None + theme_attribute = { + "ascii": "asciiTheme", "hAnsi": "hAnsiTheme", + "eastAsia": "eastAsiaTheme", "cs": "cstheme", + }[slot] + theme_token = rfonts.get(qn("w:" + theme_attribute)) + if theme_token is not None: + # A direct theme declaration must not fall through to an inherited literal face. + # Resolve the document theme/fontScheme (including themeFontLang/script mapping) + # before auditing this run. Both a literal and theme attribute for one slot are + # ambiguous across consumers, so fail closed in that case too. + literal = rfonts.get(qn("w:" + slot)) + raise LookupError( + f"unresolved direct {theme_attribute}={theme_token!r}" + + (f" alongside {slot}={literal!r}" if literal is not None else "") + ) + return rfonts.get(qn("w:" + slot)) def style_faces(style, slot): while style is not None: diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/edit.md b/plugins/Hylouis233/document-skills/skills/docx/references/edit.md index d2497f8..df40ec1 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/edit.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/edit.md @@ -122,36 +122,45 @@ For field codes, sectPr surgery, tracked changes, or parts python-docx does not Rules that keep the archive valid: 1. Operate on a **copy** of the file. -2. Unzip preserving structure: `python -m zipfile -e input.docx work/` or use `zipfile` in - Python with `ZIP_DEFLATED` on repack. +2. Extract into a **new empty temporary directory for every input**. Never reuse a fixed `work/` + directory: members absent from the next DOCX would remain there and be repacked as stale or + confidential content. 3. Parse XML with `lxml`/`xml.etree` - never string replace. Text lives in `w:t` inside runs (`w:r`) inside paragraphs (`w:p`); a logical sentence can span several runs. 4. Repack with `[Content_Types].xml` first and stored/deflated entries only: ```python from pathlib import Path +from tempfile import TemporaryDirectory from zipfile import ZIP_DEFLATED, ZipFile -src = Path("work") -content_types = src / "[Content_Types].xml" -if not content_types.is_file(): - raise FileNotFoundError(content_types) - -files = sorted( - (path for path in src.rglob("*") if path.is_file() and path != content_types), - key=lambda path: path.relative_to(src).as_posix(), -) - -with ZipFile( - "output.docx", "w", compression=ZIP_DEFLATED, strict_timestamps=False -) as archive: - archive.write(content_types, "[Content_Types].xml") - for path in files: - archive.write(path, path.relative_to(src).as_posix()) +input_path = Path("input.docx") +output_path = Path("output.docx").resolve() # keep output outside the temporary tree +with TemporaryDirectory(prefix="docx-edit-") as scratch: + src = Path(scratch) + with ZipFile(input_path) as archive: + archive.extractall(src) + + # Apply the required XML edits under `src` here, using an XML parser. + content_types = src / "[Content_Types].xml" + if not content_types.is_file(): + raise FileNotFoundError(content_types) + files = sorted( + (path for path in src.rglob("*") if path.is_file() and path != content_types), + key=lambda path: path.relative_to(src).as_posix(), + ) + + with ZipFile( + output_path, "w", compression=ZIP_DEFLATED, strict_timestamps=False + ) as archive: + archive.write(content_types, "[Content_Types].xml") + for path in files: + archive.write(path, path.relative_to(src).as_posix()) ``` - This writes relative POSIX archive names, does not add directory entries, and excludes - `[Content_Types].xml` from the remaining files so it cannot be added twice. + The temporary directory is deleted after repacking. This writes relative POSIX archive names, + does not add directory entries, and excludes `[Content_Types].xml` from the remaining files so + it cannot be added twice. 5. If you touched part names or added parts, update `[Content_Types].xml` and `word/_rels/document.xml.rels` consistently - a mismatch here is the classic silent corrupt. diff --git a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md index 3726ea1..402a1b5 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md @@ -47,6 +47,12 @@ file is not. 4. **Overflow is a defect**: content that spills past the last page or the margin must be detected in postcheck and fixed (shrink, paginate, or cut), never shipped. 5. Write output to a new path; keep inputs untouched unless in-place was requested. +6. **Accessibility claims require tagged output.** ReportLab does not produce a tagged PDF/UA + document, so do not describe ordinary ReportLab output as accessible or screen-reader-ready. + When PDF/UA or assistive-technology compatibility is required, use a toolchain that can create + and validate tags, reading order, structure roles, language, and alternative text. If that + toolchain is unavailable, stop and report the limitation; offer an accessible DOCX or HTML + deliverable instead of silently returning an untagged PDF. ## Step 3 - Postcheck (mandatory) diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md b/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md index dcba4b2..0e8cd21 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/extract.md @@ -5,7 +5,7 @@ import os import fitz # pymupdf doc = fitz.open("input.pdf") -if doc.needs_pass: +if doc.needs_pass and doc.authenticate("") <= 0: password = os.environ.get("PDF_PASSWORD") if not password or not doc.authenticate(password): raise RuntimeError("set PDF_PASSWORD to the correct password before extracting") diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md b/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md index 45d2d3e..ef1d7bf 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/transform.md @@ -196,7 +196,7 @@ Encryption and forms: writer.write(f) check = PdfReader("filled.pdf") - value = str((check.get_fields() or {}).get("applicant_name", {}).get("/V", "")) + value = str((check.get_fields() or {}).get(field_name, {}).get("/V", "")) if value.strip("/") != "Ada Byron": raise ValueError(f"filled field did not round-trip: {value!r}") ``` diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md index 18efabd..ced6018 100644 --- a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md @@ -61,6 +61,20 @@ def cached_numeric_points(source): if point.get("idx") is not None and (value := point.find(qn("c:v"))) is not None ] +def cached_category_labels(plot): + """Return flattened labels, or None when a formula has no category cache.""" + category_nodes = plot._element.xpath("./c:ser[1]/c:cat") + if not category_nodes: + return [] + cache_payload = category_nodes[0].xpath( + "./c:strRef/c:strCache/c:ptCount | ./c:numRef/c:numCache/c:ptCount | " + "./c:multiLvlStrRef/c:multiLvlStrCache/c:ptCount | " + "./c:strLit/c:ptCount | ./c:numLit/c:ptCount" + ) + if not cache_payload: + return None + return [[str(level) for level in label] for label in plot.categories.flattened_labels] + def series_content(series): x_source = getattr(series._element, "xVal", None) if x_source is None: # category/value chart @@ -137,11 +151,17 @@ for i, slide in enumerate(prs.slides): for plot in chart.plots: items = list(plot.series) has_xy_values = any(getattr(item._element, "xVal", None) is not None for item in items) - categories = [] if has_xy_values else [ - [str(level) for level in label] for label in plot.categories.flattened_labels - ] + categories = [] if has_xy_values else cached_category_labels(plot) series = [series_content(item) for item in items] - plots.append({"kind": type(plot).__name__, "categories": categories, "series": series}) + plots.append({ + "kind": type(plot).__name__, + "categories": categories, + "category_cache_status": ( + "not-applicable" if has_xy_values else + "unavailable" if categories is None else "available" + ), + "series": series, + }) charts.append({ "title": chart_title, "axis_titles": chart_axis_titles(chart), @@ -281,6 +301,7 @@ prs = Presentation("deck.pptx") # multiple masters with different themes, so the first /ppt/theme/* part is not a safe default. theme_cache = {} DRAWINGML = {"a": "http://schemas.openxmlformats.org/drawingml/2006/main"} +DIAGRAM_NS = "http://schemas.openxmlformats.org/drawingml/2006/diagram" def read_theme_role(root, role): node = root.find(f".//a:{role}Font", DRAWINGML) @@ -423,11 +444,22 @@ def iter_text_frames(shapes): for cell in row.cells: yield cell.text_frame +def unresolved_graphic_font_regions(shapes): + """Mark text systems whose full font cascade python-pptx does not expose.""" + for shape in shapes: + if getattr(shape, "has_chart", False): + yield {"shape": shape.name, "kind": "chart"} + graphic_data = shape._element.find(".//" + qn("a:graphicData")) + if graphic_data is not None and graphic_data.get("uri") == DIAGRAM_NS: + yield {"shape": shape.name, "kind": "SmartArt"} + +unresolved_fonts = [] for i, slide in enumerate(prs.slides): master_name, theme_fonts = theme_faces_for_slide(slide) print(i, "master:", master_name, "theme:", ascii(theme_fonts)) title_shape = slide.shapes.title - for frame in iter_text_frames(iter_shapes(slide.shapes)): + shapes = list(iter_shapes(slide.shapes)) + for frame in iter_text_frames(shapes): holder = getattr(frame, "_parent", None) # the shape for ordinary frames for paragraph in frame.paragraphs: for run in paragraph.runs: @@ -437,6 +469,14 @@ for i, slide in enumerate(prs.slides): ) else "minor" faces = font_candidates(run, paragraph, theme_fonts, role) print(i, repr(run.text[:20]), "font candidates:", ascii(faces)) + unresolved_fonts.extend( + {"slide": i + 1, **record} for record in unresolved_graphic_font_regions(shapes) + ) +if unresolved_fonts: + raise LookupError( + "font audit cannot verify chart/SmartArt text properties: " + f"{unresolved_fonts}; inspect those package parts or render with production fonts" + ) ``` `run.font.name` exposes only the Latin slot, so it must not short-circuit inspection of explicit @@ -445,6 +485,9 @@ master inheritance chain; the output above remains a candidate list. Check match placeholders when the exact face matters. Han text needs the deck locale to distinguish Hans, Hant, Japanese, and Korean mappings; add other script ranges when the task uses them rather than claiming the Latin fallback is definitive. +Chart titles, legends, tick labels, data labels, and SmartArt nodes use additional DrawingML +font cascades across chart/diagram parts. The guard above deliberately fails closed until those +parts are audited directly or the deck is rendered and inspected with the production fonts. ## Text-fit verification diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md index 03f3f6c..fc8b97e 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md @@ -35,6 +35,7 @@ with open("input.csv", newline="", encoding="utf-8-sig") as f: # utf-8-sig str ```python import csv +from pathlib import Path FORMULA_PREFIXES = ("=", "+", "-", "@") @@ -45,8 +46,15 @@ def spreadsheet_csv_field(value, *, mode="safe"): return "'" + value return value -with open("output.csv", "w", newline="", encoding="utf-8") as f: - writer = csv.writer(f) +def delimiter_for(path): + delimiter = {".csv": ",", ".tsv": "\t"}.get(Path(path).suffix.lower()) + if delimiter is None: + raise ValueError("output must use a .csv or .tsv extension") + return delimiter + +output_path = Path("output.csv") # use .tsv when tab-separated output was requested +with output_path.open("w", newline="", encoding="utf-8") as f: + writer = csv.writer(f, delimiter=delimiter_for(output_path)) rows = [["Region", "Units", "Note"], ["EU", 120, "=2+2"]] writer.writerows([spreadsheet_csv_field(value) for value in row] for row in rows) ``` @@ -105,6 +113,12 @@ formula-injection protection**; do not present a raw export as safe to open in a return "'" + value return value + def delimiter_for(path): + delimiter = {".csv": ",", ".tsv": "\t"}.get(Path(path).suffix.lower()) + if delimiter is None: + raise ValueError("output must use a .csv or .tsv extension") + return delimiter + def worksheet_part(archive, sheet_name): workbook = ET.fromstring(archive.read("xl/workbook.xml")) sheet = next( @@ -170,7 +184,7 @@ formula-injection protection**; do not present a raw export as safe to open in a output_path = Path("output.csv") temporary_path = output_path.with_suffix(output_path.suffix + ".tmp") with temporary_path.open("w", newline="", encoding="utf-8") as output: - writer = csv.writer(output) + writer = csv.writer(output, delimiter=delimiter_for(output_path)) for formula_row, value_row in zip(formula_ws.iter_rows(), value_ws.iter_rows()): for formula_cell, value_cell in zip(formula_row, value_row): if (formula_cell.data_type == "f" and value_cell.value is None diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/formatting.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/formatting.md index baff9fc..2b1cc79 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/formatting.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/formatting.md @@ -90,8 +90,60 @@ one you took: hard-coding `Data!` breaks on any workbook whose sheet is named differently: ```python - ws2 = wb.create_sheet("ByRegion") - ws2.append(["Region", "Units", "Revenue"]) + import openpyxl + import posixpath + import zipfile + from xml.etree import ElementTree as ET + + MAIN_NS = "http://schemas.openxmlformats.org/spreadsheetml/2006/main" + DOC_REL_NS = "http://schemas.openxmlformats.org/officeDocument/2006/relationships" + PKG_REL_NS = "http://schemas.openxmlformats.org/package/2006/relationships" + + def cached_formula_coordinates_for_sheet(path, sheet_name): + """Distinguish a typed cached blank from a formula with no cached result.""" + with zipfile.ZipFile(path) as archive: + workbook = ET.fromstring(archive.read("xl/workbook.xml")) + sheet = next( + item for item in workbook.iter(f"{{{MAIN_NS}}}sheet") + if item.attrib["name"] == sheet_name + ) + relationship_id = sheet.attrib[f"{{{DOC_REL_NS}}}id"] + relationships = ET.fromstring(archive.read("xl/_rels/workbook.xml.rels")) + target = next( + item.attrib["Target"] + for item in relationships.iter(f"{{{PKG_REL_NS}}}Relationship") + if item.attrib["Id"] == relationship_id + ) + part = target.lstrip("/") if target.startswith("/") else posixpath.normpath( + posixpath.join("xl", target) + ) + cached = set() + coordinate = cell_type = value_text = None + has_formula = value_seen = False + with archive.open(part) as source: + for event, element in ET.iterparse(source, events=("start", "end")): + if event == "start" and element.tag == f"{{{MAIN_NS}}}c": + coordinate = element.attrib["r"] + cell_type = element.attrib.get("t") + has_formula = value_seen = False + value_text = None + elif event == "end" and coordinate is not None: + if element.tag == f"{{{MAIN_NS}}}f": + has_formula = True + elif element.tag == f"{{{MAIN_NS}}}v": + value_seen = True + value_text = element.text + elif element.tag == f"{{{MAIN_NS}}}c": + valid_cache = value_seen and ( + value_text not in (None, "") or cell_type == "str" + ) + if has_formula and valid_cache: + cached.add(coordinate) + coordinate = None + element.clear() + elif event == "end": + element.clear() + return cached def sheet_ref(sheet): # Always quote: valid titles such as Q1-Data are ambiguous when left bare. @@ -100,15 +152,37 @@ one you took: return f"'{escaped}'!" src = sheet_ref(ws) # e.g. "'Sales'!" or "'Raw Data'!" + source_path = "input.xlsx" # the same original path used to load formula-preserving `wb` + cached_region_formulas = cached_formula_coordinates_for_sheet(source_path, ws.title) + value_wb = openpyxl.load_workbook(source_path, read_only=True, data_only=True) + value_ws = value_wb[ws.title] + value_ws.reset_dimensions() regions = [] seen_region_keys = set() - for (region,) in ws.iter_rows(min_row=2, min_col=1, max_col=1, values_only=True): + missing_region_caches = [] + source_rows = ws.iter_rows(min_row=2, min_col=1, max_col=1) + value_rows = value_ws.iter_rows(min_row=2, min_col=1, max_col=1) + for source_row, value_row in zip(source_rows, value_rows): + source_cell, value_cell = source_row[0], value_row[0] + region = value_cell.value if source_cell.data_type == "f" else source_cell.value + if (source_cell.data_type == "f" and region is None + and source_cell.coordinate not in cached_region_formulas): + missing_region_caches.append(source_cell.coordinate) + continue if region is None or region == "": # keep valid falsey values: 0 and False continue key = (type(region), region) # do not collapse False and numeric 0 if key not in seen_region_keys: seen_region_keys.add(key) regions.append(region) # stable source order; no mixed-type sort + value_wb.close() + if missing_region_caches: + raise RuntimeError( + f"aggregation keys have no cached value: {missing_region_caches}" + ) + + ws2 = wb.create_sheet("ByRegion") + ws2.append(["Region", "Units", "Revenue"]) for i, region in enumerate(regions, start=2): ws2.cell(row=i, column=1, value=region) ws2.cell(row=i, column=2, value=f"=SUMIF({src}A:A,A{i},{src}C:C)") diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md index e6c2b9c..17426e9 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md @@ -2,10 +2,72 @@ ```python import openpyxl +import posixpath +import zipfile from openpyxl.utils import get_column_letter +from xml.etree import ElementTree as ET -formula_wb = openpyxl.load_workbook("input.xlsx", read_only=True, data_only=False) -value_wb = openpyxl.load_workbook("input.xlsx", read_only=True, data_only=True) +MAIN_NS = "http://schemas.openxmlformats.org/spreadsheetml/2006/main" +DOC_REL_NS = "http://schemas.openxmlformats.org/officeDocument/2006/relationships" +PKG_REL_NS = "http://schemas.openxmlformats.org/package/2006/relationships" +CELL_TAG = f"{{{MAIN_NS}}}c" +FORMULA_TAG = f"{{{MAIN_NS}}}f" +VALUE_TAG = f"{{{MAIN_NS}}}v" + +def worksheet_part(archive, sheet_name): + workbook = ET.fromstring(archive.read("xl/workbook.xml")) + sheet = next( + item for item in workbook.iter(f"{{{MAIN_NS}}}sheet") + if item.attrib["name"] == sheet_name + ) + relationship_id = sheet.attrib[f"{{{DOC_REL_NS}}}id"] + relationships = ET.fromstring(archive.read("xl/_rels/workbook.xml.rels")) + target = next( + item.attrib["Target"] for item in relationships.iter(f"{{{PKG_REL_NS}}}Relationship") + if item.attrib["Id"] == relationship_id + ) + return target.lstrip("/") if target.startswith("/") else posixpath.normpath( + posixpath.join("xl", target) + ) + +def cached_formula_coordinates(archive, part): + """Find nonempty caches plus explicitly typed empty-string caches.""" + cached = set() + coordinate = cell_type = value_text = None + has_formula = value_seen = False + with archive.open(part) as source: + for event, element in ET.iterparse(source, events=("start", "end")): + if event == "start" and element.tag == CELL_TAG: + coordinate = element.attrib["r"] + cell_type = element.attrib.get("t") + has_formula = value_seen = False + value_text = None + elif event == "end" and coordinate is not None: + if element.tag == FORMULA_TAG: + has_formula = True + elif element.tag == VALUE_TAG: + value_seen = True + value_text = element.text + elif element.tag == CELL_TAG: + valid_cache = value_seen and ( + value_text not in (None, "") or cell_type == "str" + ) + if has_formula and valid_cache: + cached.add(coordinate) + coordinate = None + element.clear() + elif event == "end": + element.clear() + return cached + +input_path = "input.xlsx" +formula_wb = openpyxl.load_workbook(input_path, read_only=True, data_only=False) +value_wb = openpyxl.load_workbook(input_path, read_only=True, data_only=True) +with zipfile.ZipFile(input_path) as archive: + cached_formulas = { + sheet_name: cached_formula_coordinates(archive, worksheet_part(archive, sheet_name)) + for sheet_name in formula_wb.sheetnames + } print("sheets:", value_wb.sheetnames) def formula_text(value): @@ -26,6 +88,10 @@ def discover_dimension(worksheet): min_row = min_column = max_row = max_column = None for row in worksheet.iter_rows(): for cell in row: + # A styled-but-empty cell has coordinates in a read-only stream but is not data. + # Formula cells remain part of the logical range even when their cache is missing. + if getattr(cell, "value", None) is None and getattr(cell, "data_type", None) != "f": + continue row_index = getattr(cell, "row", None) # EmptyCell has no coordinates column_index = getattr(cell, "column", None) if row_index is None or column_index is None: @@ -50,15 +116,11 @@ for sheet_name in value_wb.sheetnames: # non-Excel producer can declare a plausible but truncated range (for # example A1:B2 while data continues below it), so treat that metadata as # untrusted: reset both streams and discover the real bounds before reading. - declared = value_ws.calculate_dimension() - discovered, first_populated_row = discover_dimension(value_ws) - formula_discovered, formula_first_row = discover_dimension(formula_ws) - if (formula_discovered, formula_first_row) != (discovered, first_populated_row): - raise ValueError( - "formula/value stream bounds disagree: " - f"{(formula_discovered, formula_first_row)} != " - f"{(discovered, first_populated_row)}" - ) + declared = formula_ws.calculate_dimension() + discovered, first_populated_row = discover_dimension(formula_ws) + # The value stream cannot distinguish a missing formula cache from a displayed blank, + # so it follows the logical bounds discovered from the formula-preserving stream. + value_ws.reset_dimensions() if discovered != declared: print(f"--- {sheet_name} --- declared {declared!r}; discovered real extent:") print(f"--- {sheet_name} --- dims:", discovered) @@ -76,7 +138,8 @@ for sheet_name in value_wb.sheetnames: missing_cache_count = 0 for formula_row, value_row in zip(formula_ws.iter_rows(), value_ws.iter_rows()): for formula_cell, value_cell in zip(formula_row, value_row): - if formula_cell.data_type == "f" and value_cell.value is None: + if (formula_cell.data_type == "f" and value_cell.value is None + and formula_cell.coordinate not in cached_formulas[sheet_name]): missing_cache_count += 1 if missing_cache_count <= 10: print("formula without cached value:", formula_cell.coordinate, diff --git a/plugins/Hylouis233/document-skills/tests/docx_fixture.py b/plugins/Hylouis233/document-skills/tests/docx_fixture.py index e8380d7..355e5b0 100644 --- a/plugins/Hylouis233/document-skills/tests/docx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/docx_fixture.py @@ -10,6 +10,7 @@ import sys import zipfile from pathlib import Path +from tempfile import TemporaryDirectory import fitz from docx import Document @@ -405,7 +406,20 @@ def face_from_rpr(rpr, slot): if rpr is None: return None rfonts = rpr.find(qn("w:rFonts")) - return None if rfonts is None else rfonts.get(qn("w:" + slot)) + if rfonts is None: + return None + theme_attribute = { + "ascii": "asciiTheme", "hAnsi": "hAnsiTheme", + "eastAsia": "eastAsiaTheme", "cs": "cstheme", + }[slot] + theme_token = rfonts.get(qn("w:" + theme_attribute)) + if theme_token is not None: + literal = rfonts.get(qn("w:" + slot)) + raise LookupError( + f"unresolved direct {theme_attribute}={theme_token!r}" + + (f" alongside {slot}={literal!r}" if literal is not None else "") + ) + return rfonts.get(qn("w:" + slot)) def style_faces(style, slot): @@ -448,6 +462,35 @@ def effective_face(run, slot): raise LookupError(slot) +theme_attribute_by_slot = { + "ascii": "asciiTheme", "hAnsi": "hAnsiTheme", + "eastAsia": "eastAsiaTheme", "cs": "cstheme", +} +for slot, theme_attribute in theme_attribute_by_slot.items(): + themed_run = font_doc.add_paragraph().add_run("漢" if slot == "eastAsia" else "A") + themed_fonts = themed_run._r.get_or_add_rPr().get_or_add_rFonts() + themed_fonts.set(qn("w:" + theme_attribute), "majorEastAsia" if slot == "eastAsia" else "minorAscii") + try: + effective_face(themed_run, slot) + direct_theme_rejected = False + except LookupError: + direct_theme_rejected = True + check(f"direct {theme_attribute} does not fall through to an inherited literal face", + direct_theme_rejected) + +ambiguous_theme_run = font_doc.add_paragraph().add_run("漢") +ambiguous_fonts = ambiguous_theme_run._r.get_or_add_rPr().get_or_add_rFonts() +ambiguous_fonts.set(qn("w:eastAsia"), "Literal Face") +ambiguous_fonts.set(qn("w:eastAsiaTheme"), "majorEastAsia") +try: + effective_face(ambiguous_theme_run, "eastAsia") + literal_and_theme_rejected = False +except LookupError: + literal_and_theme_rejected = True +check("same-slot literal plus theme font declaration fails closed", + literal_and_theme_rejected) + + check("header part has no document back-reference (negative control)", not hasattr(header_run.part, "document")) check("header runs resolve Normal from the owning document", @@ -908,5 +951,52 @@ def tc_text(tc): check("tc_text keeps tabs visible", "after tab" in extracted, extracted) +# ---- edit.md raw OOXML repack: every input gets a fresh extraction tree --------- +media_doc = Document() +media_doc.add_paragraph("first document") +media_doc.add_picture("inline-icon.png") +media_doc.save("repack-with-media.docx") +plain_doc = Document() +plain_doc.add_paragraph("second document") +plain_doc.save("repack-without-media.docx") + + +def repack_tree(source, output, extraction_root): + with zipfile.ZipFile(source) as archive: + archive.extractall(extraction_root) + content_types = Path(extraction_root) / "[Content_Types].xml" + files = sorted( + (path for path in Path(extraction_root).rglob("*") + if path.is_file() and path != content_types), + key=lambda path: path.relative_to(extraction_root).as_posix(), + ) + with zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED, strict_timestamps=False) as archive: + archive.write(content_types, "[Content_Types].xml") + for path in files: + archive.write(path, path.relative_to(extraction_root).as_posix()) + + +reused_root = Path("reused-docx-work") +reused_root.mkdir() +repack_tree("repack-with-media.docx", "reused-first.docx", reused_root) +repack_tree("repack-without-media.docx", "reused-second.docx", reused_root) +with zipfile.ZipFile("reused-second.docx") as archive: + reused_names = archive.namelist() +check("reusing an extraction tree demonstrably leaks a prior document's media", + any(name.startswith("word/media/") for name in reused_names), reused_names) + +with TemporaryDirectory(prefix="docx-edit-") as scratch: + repack_tree("repack-without-media.docx", "fresh-second.docx", Path(scratch)) +with zipfile.ZipFile("fresh-second.docx") as archive: + fresh_names = archive.namelist() +check("fresh extraction excludes media absent from the current input", + not any(name.startswith("word/media/") for name in fresh_names), fresh_names) +check("fresh repack writes Content_Types exactly once and first", + fresh_names[0] == "[Content_Types].xml" + and fresh_names.count("[Content_Types].xml") == 1, fresh_names[:3]) +check("freshly repacked DOCX reopens with only the current document's content", + Document("fresh-second.docx").paragraphs[0].text == "second document") + + print("\n" + ("ALL DOCX FIXTURES PASSED" if not failures else f"{len(failures)} FAILURES: {failures}")) sys.exit(0 if not failures else 1) diff --git a/plugins/Hylouis233/document-skills/tests/fixtures.test.mjs b/plugins/Hylouis233/document-skills/tests/fixtures.test.mjs index e3ffa77..7a763b7 100644 --- a/plugins/Hylouis233/document-skills/tests/fixtures.test.mjs +++ b/plugins/Hylouis233/document-skills/tests/fixtures.test.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { execFile } from "node:child_process"; -import { mkdtemp, rm } from "node:fs/promises"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import test from "node:test"; @@ -11,6 +11,14 @@ const execFileAsync = promisify(execFile); const testsRoot = path.dirname(fileURLToPath(import.meta.url)); const python = process.env.PYTHON || (process.platform === "win32" ? "python" : "python3"); +test("PDF runtime skill refuses unsupported accessibility claims", async () => { + const skill = await readFile(path.join(testsRoot, "..", "skills", "pdf", "SKILL.md"), "utf8"); + assert.match(skill, /ReportLab does not produce a tagged PDF\/UA\s+document/); + assert.match(skill, /do not describe ordinary ReportLab output as accessible or screen-reader-ready/); + assert.match(skill, /stop and report the limitation/); + assert.match(skill, /accessible DOCX or HTML/); +}); + test("document-skills Python fixtures", { timeout: 12 * 60_000 }, async (context) => { for (const name of ["xlsx", "pptx", "pdf", "docx"]) { await context.test(name + " fixture", { timeout: 3 * 60_000 }, async () => { diff --git a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py index 9ceb1a7..3fc3d5c 100644 --- a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py @@ -33,6 +33,30 @@ def open_pdf(path, password=None): return reader +def authenticate_for_extraction(document, password=None): + """Mirror extract.md: try the valid blank user password before requiring env input.""" + if document.needs_pass and document.authenticate("") <= 0: + if not password or document.authenticate(password) <= 0: + raise RuntimeError("set PDF_PASSWORD to the correct password before extracting") + + +class BlankPasswordDocument: + needs_pass = True + + def __init__(self): + self.attempts = [] + + def authenticate(self, password): + self.attempts.append(password) + return 2 if password == "" else 0 + + +blank_password_stub = BlankPasswordDocument() +authenticate_for_extraction(blank_password_stub) +check("PDF extraction authenticates the blank password before requiring environment input", + blank_password_stub.attempts == [""], blank_password_stub.attempts) + + # ---- build a 2-page A4 PDF with one AcroForm text field on page 1 ------------- c = canvas.Canvas("form.pdf", pagesize=A4) c.setFont("Helvetica", 16) @@ -43,6 +67,22 @@ def open_pdf(path, password=None): c.drawString(72, 780, "Second page content") c.save() +# Current PyMuPDF opens a permission-encrypted PDF with an empty user password directly +# (`needs_pass == 0`). Keep this real-file proof in addition to the branch-order stub above. +blank_user_writer = pypdf.PdfWriter() +blank_user_writer.append(pypdf.PdfReader("form.pdf")) +blank_user_writer.encrypt(user_password="", owner_password="fixture-owner") +with open("blank-user-password.pdf", "wb") as output: + blank_user_writer.write(output) +blank_user_doc = fitz.open("blank-user-password.pdf") +authenticate_for_extraction(blank_user_doc) +check("blank-user permission-encrypted PDF extracts without PDF_PASSWORD", + "Application form" in blank_user_doc[0].get_text(), blank_user_doc.needs_pass) +blank_user_doc.close() + +check("ordinary ReportLab output is not a tagged PDF/UA document", + "/StructTreeRoot" not in pypdf.PdfReader("form.pdf").trailer["/Root"]) + # ---- SKILL.md postcheck snippet: width/height pairs from the 4-coordinate box -- r = pypdf.PdfReader("form.pdf") page_sizes = [ @@ -348,12 +388,18 @@ def widget_field_name(widget): (PdfReader("hierarchical-form-filled.pdf").get_fields() or {}) .get(hierarchical_field_name, {}).get("/V", "") ) +hierarchy_literal_value = str( + (PdfReader("hierarchical-form-filled.pdf").get_fields() or {}) + .get("applicant_name", {}).get("/V", "") +) check("qualified field lookup locates the hierarchical widget page", len(hierarchy_target_pages) == 1 and hierarchy_target_pages[0] is hierarchy_fill_writer.pages[1], len(hierarchy_target_pages)) check("hierarchical field value round-trips", hierarchy_value.strip("/") == "Ada Hierarchical", repr(hierarchy_value)) +check("literal applicant_name lookup is a proven false negative for a qualified field", + hierarchy_literal_value == "", repr(hierarchy_literal_value)) # ---- transform.md merge imports outline navigation ---------------------------- appendix_writer = PdfWriter() diff --git a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py index 40b19be..fc0a8d3 100644 --- a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py @@ -404,6 +404,20 @@ def cached_numeric_points(source): ] +def cached_category_labels(plot): + category_nodes = plot._element.xpath("./c:ser[1]/c:cat") + if not category_nodes: + return [] + cache_payload = category_nodes[0].xpath( + "./c:strRef/c:strCache/c:ptCount | ./c:numRef/c:numCache/c:ptCount | " + "./c:multiLvlStrRef/c:multiLvlStrCache/c:ptCount | " + "./c:strLit/c:ptCount | ./c:numLit/c:ptCount" + ) + if not cache_payload: + return None + return [[str(level) for level in label] for label in plot.categories.flattened_labels] + + def series_content(series): x_source = getattr(series._element, "xVal", None) if x_source is None: @@ -480,11 +494,17 @@ def extract_slide_content(slide): for plot in chart.plots: items = list(plot.series) has_xy_values = any(getattr(item._element, "xVal", None) is not None for item in items) - categories = [] if has_xy_values else [ - [str(level) for level in label] for label in plot.categories.flattened_labels - ] + categories = [] if has_xy_values else cached_category_labels(plot) series = [series_content(item) for item in items] - plots.append({"kind": type(plot).__name__, "categories": categories, "series": series}) + plots.append({ + "kind": type(plot).__name__, + "categories": categories, + "category_cache_status": ( + "not-applicable" if has_xy_values else + "unavailable" if categories is None else "available" + ), + "series": series, + }) charts.append({ "title": chart_title, "axis_titles": chart_axis_titles(chart), @@ -586,6 +606,39 @@ def extract_slide_content(slide): category_inventory_series, ) +category_labels_mutated = Presentation("category-cache-source.pptx") +category_labels_chart = next( + shape.chart for shape in category_labels_mutated.slides[0].shapes if shape.has_chart +) +category_ref = category_labels_chart.plots[0]._element.xpath( + "./c:ser[1]/c:cat/c:strRef" +)[0] +category_formula_before = category_ref.find(qn("c:f")).text +category_ref.remove(category_ref.find(qn("c:strCache"))) +category_labels_mutated.save("category-label-cacheless.pptx") +category_labels_reopened = Presentation("category-label-cacheless.pptx") +category_labels_chart = next( + shape.chart for shape in category_labels_reopened.slides[0].shapes if shape.has_chart +) +category_ref = category_labels_chart.plots[0]._element.xpath( + "./c:ser[1]/c:cat/c:strRef" +)[0] +with zipfile.ZipFile("category-label-cacheless.pptx") as category_archive: + category_workbook_remains = any( + name.startswith("ppt/embeddings/") for name in category_archive.namelist() + ) +category_labels_inventory = extract_slide_content(category_labels_reopened.slides[0]) +category_plot_inventory = category_labels_inventory["charts"][0]["plots"][0] +check("cacheless categories retain their worksheet formula and embedded workbook", + category_ref.find(qn("c:f")).text == category_formula_before + and category_ref.find(qn("c:strCache")) is None + and category_workbook_remains) +check("missing category cache is reported without aborting later chart inventory", + category_plot_inventory["categories"] is None + and category_plot_inventory["category_cache_status"] == "unavailable" + and category_plot_inventory["series"][1]["values"] == [3.0, 4.0], + category_plot_inventory) + diagram_frame = etree.fromstring(f''' and an absent cache", bare_empty_cells == truly_missing_cells == set(), (bare_empty_cells, truly_missing_cells)) + + +def profile_missing_formula_caches(source_path): + with zipfile.ZipFile(source_path) as archive: + cached_cells = cached_formula_coordinates(archive, worksheet_part(archive, "Data")) + formula_book = openpyxl.load_workbook(source_path, read_only=True, data_only=False) + value_book = openpyxl.load_workbook(source_path, read_only=True, data_only=True) + formula_sheet, value_sheet = formula_book["Data"], value_book["Data"] + formula_sheet.reset_dimensions() + value_sheet.reset_dimensions() + missing = [ + formula_cell.coordinate + for formula_row, value_row in zip(formula_sheet.iter_rows(), value_sheet.iter_rows()) + for formula_cell, value_cell in zip(formula_row, value_row) + if (formula_cell.data_type == "f" and value_cell.value is None + and formula_cell.coordinate not in cached_cells) + ] + formula_book.close() + value_book.close() + return missing + + +check("workbook profiling accepts nonempty and typed blank formula caches", + profile_missing_formula_caches("cached-value.xlsx") == [] + and profile_missing_formula_caches("cached-empty.xlsx") == []) +check("workbook profiling reports absent and untyped empty formula caches", + profile_missing_formula_caches("missing-cache.xlsx") == ["A1"] + and profile_missing_formula_caches("bare-empty-cache.xlsx") == ["A1"]) + + +def write_region_key_fixture(path, cache_kind): + workbook = openpyxl.Workbook() + sheet = workbook.active + sheet.title = "Data" + sheet.append(["Region", "Units", "Revenue", "Source Region"]) + sheet.append(["=D2", 2, 4, "EU"]) + workbook.save(path) + with zipfile.ZipFile(path) as archive: + members = {name: archive.read(name) for name in archive.namelist()} + root = ET.fromstring(members["xl/worksheets/sheet1.xml"]) + cell = next(item for item in root.iter(CELL_TAG) if item.attrib["r"] == "A2") + value = cell.find(VALUE_TAG) + if cache_kind == "nonempty": + cell.set("t", "str") + if value is None: + value = ET.SubElement(cell, VALUE_TAG) + value.text = "EU" + elif cache_kind == "empty-string": + cell.set("t", "str") + if value is None: + value = ET.SubElement(cell, VALUE_TAG) + value.text = None + elif cache_kind == "absent": + cell.attrib.pop("t", None) + if value is not None: + cell.remove(value) + else: + raise ValueError(cache_kind) + members["xl/worksheets/sheet1.xml"] = ET.tostring( + root, encoding="utf-8", xml_declaration=True, + ) + with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as archive: + for name, data in members.items(): + archive.writestr(name, data) + + +def aggregation_regions(source_path): + formula_book = openpyxl.load_workbook(source_path, data_only=False) + source_sheet = formula_book["Data"] + with zipfile.ZipFile(source_path) as archive: + cached_cells = cached_formula_coordinates(archive, worksheet_part(archive, "Data")) + value_book = openpyxl.load_workbook(source_path, read_only=True, data_only=True) + value_sheet = value_book["Data"] + value_sheet.reset_dimensions() + regions = [] + missing = [] + for source_row, value_row in zip( + source_sheet.iter_rows(min_row=2, min_col=1, max_col=1), + value_sheet.iter_rows(min_row=2, min_col=1, max_col=1), + ): + source_cell, value_cell = source_row[0], value_row[0] + region = value_cell.value if source_cell.data_type == "f" else source_cell.value + if (source_cell.data_type == "f" and region is None + and source_cell.coordinate not in cached_cells): + missing.append(source_cell.coordinate) + continue + if region is not None and region != "": + regions.append(region) + value_book.close() + formula_book.close() + if missing: + raise RuntimeError(f"aggregation keys have no cached value: {missing}") + return regions + + +write_region_key_fixture("region-key-cached.xlsx", "nonempty") +write_region_key_fixture("region-key-empty.xlsx", "empty-string") +write_region_key_fixture("region-key-missing.xlsx", "absent") +cached_regions = aggregation_regions("region-key-cached.xlsx") +aggregate_book = openpyxl.Workbook() +aggregate_book.active["A1"] = cached_regions[0] +check("formula-backed aggregation key is copied from its cached displayed value", + aggregate_book.active["A1"].value == "EU" + and aggregate_book.active["A1"].data_type != "f") +check("typed cached blank aggregation keys are skipped", + aggregation_regions("region-key-empty.xlsx") == []) +try: + aggregation_regions("region-key-missing.xlsx") + missing_region_cache_rejected = False +except RuntimeError as error: + missing_region_cache_rejected = "A2" in str(error) +check("aggregation fails closed when a formula key has no cached value", + missing_region_cache_rejected) + export_formula_values("cached-value.xlsx", "cached-value.csv") with open("cached-value.csv", newline="", encoding="utf-8") as exported: cached_value_rows = list(csv.reader(exported)) @@ -1039,6 +1174,8 @@ def discover_dimension(worksheet): min_row = min_column = max_row = max_column = None for row in worksheet.iter_rows(): for cell in row: + if getattr(cell, "value", None) is None and getattr(cell, "data_type", None) != "f": + continue row_index = getattr(cell, "row", None) column_index = getattr(cell, "column", None) if row_index is None or column_index is None: @@ -1066,10 +1203,11 @@ def discover_dimension(worksheet): streamed_after_reset = [row for row in dim_ws_ro.iter_rows(values_only=True)] dim_value.close() dim_formula.close() -check("dimension scan recovers plausible row/column truncation in both workbook views", - discovered_value_dimension == "A1:C4" - and discovered_formula_dimension == "A1:C4" - and discovered_value_first_row == discovered_formula_first_row == 1, +check("formula-preserving dimension scan retains uncached formulas in the logical range", + discovered_formula_dimension == "A1:C4" + and discovered_formula_first_row == 1 + and discovered_value_dimension == "A1:B3" + and discovered_value_first_row == 1, ((discovered_value_dimension, discovered_value_first_row), (discovered_formula_dimension, discovered_formula_first_row))) check("reset_dimensions restores the real extent", @@ -1101,6 +1239,8 @@ def discover_dimension(worksheet): offset_ws = offset_wb.active offset_ws["A7"], offset_ws["B7"] = "Region", "Units" offset_ws["A8"], offset_ws["B8"] = "EU", 120 +offset_ws["A2"].number_format = "0.00" # styled but empty: not part of the data range +offset_ws["C8"] = "=SUM(B8)" # uncached formula: remains part of the range offset_wb.save("leading-blank-rows.xlsx") offset_formula_wb = openpyxl.load_workbook( "leading-blank-rows.xlsx", read_only=True, data_only=False, @@ -1109,17 +1249,15 @@ def discover_dimension(worksheet): "leading-blank-rows.xlsx", read_only=True, data_only=True, ) offset_formula_extent, offset_formula_first = discover_dimension(offset_formula_wb.active) -offset_value_extent, offset_value_first = discover_dimension(offset_value_wb.active) +offset_value_wb.active.reset_dimensions() offset_rows = offset_value_wb.active.iter_rows( - min_row=offset_value_first, values_only=True, + min_row=offset_formula_first, values_only=True, ) offset_header = next(offset_rows, None) offset_sample = next(offset_rows, None) -check("dimension discovery retains the first populated row in both read-only views", - (offset_formula_extent, offset_formula_first) == ("A7:B8", 7) - and (offset_value_extent, offset_value_first) == ("A7:B8", 7), - ((offset_formula_extent, offset_formula_first), - (offset_value_extent, offset_value_first))) +check("styled empty cells do not move the formula-preserving logical range", + (offset_formula_extent, offset_formula_first) == ("A7:C8", 7), + (offset_formula_extent, offset_formula_first)) check("header sampling skips six leading blank rows", offset_header[:2] == ("Region", "Units") and offset_sample[:2] == ("EU", 120), From e6a148b2b063ce387961b4772e278dff8f0eae7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=98=E6=89=AC?= Date: Mon, 17 Aug 2026 17:19:38 +0800 Subject: [PATCH 27/34] fix(document-skills): harden review edge cases --- .../skills/docx/references/edit.md | 60 +++++- .../document-skills/skills/pdf/SKILL.md | 81 ++++++-- .../skills/pptx/references/analyze.md | 35 +++- .../document-skills/skills/xlsx/SKILL.md | 17 +- .../document-skills/tests/docx_fixture.py | 57 ++++++ .../document-skills/tests/pdf_fixture.py | 182 +++++++++++++++++- .../document-skills/tests/pptx_fixture.py | 118 +++++++++++- .../document-skills/tests/xlsx_fixture.py | 38 ++++ 8 files changed, 545 insertions(+), 43 deletions(-) diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/edit.md b/plugins/Hylouis233/document-skills/skills/docx/references/edit.md index df40ec1..e2cc759 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/edit.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/edit.md @@ -122,26 +122,78 @@ For field codes, sectPr surgery, tracked changes, or parts python-docx does not Rules that keep the archive valid: 1. Operate on a **copy** of the file. -2. Extract into a **new empty temporary directory for every input**. Never reuse a fixed `work/` - directory: members absent from the next DOCX would remain there and be repacked as stale or - confidential content. +2. Run the bounded package validator below **before extracting anything**, then extract into a + **new empty temporary directory for every input**. Never reuse a fixed `work/` directory: + members absent from the next DOCX would remain there and be repacked as stale or confidential + content. 3. Parse XML with `lxml`/`xml.etree` - never string replace. Text lives in `w:t` inside runs (`w:r`) inside paragraphs (`w:p`); a logical sentence can span several runs. 4. Repack with `[Content_Types].xml` first and stored/deflated entries only: ```python +import os from pathlib import Path from tempfile import TemporaryDirectory from zipfile import ZIP_DEFLATED, ZipFile +from lxml import etree + +MAX_ARCHIVE_BYTES = 200 * 1024 * 1024 +MAX_MEMBERS = 10_000 +MAX_XML_PART = 20 * 1024 * 1024 +MAX_ENTRY = 100 * 1024 * 1024 +MAX_TOTAL_UNCOMPRESSED = 500 * 1024 * 1024 +MAX_COMPRESSION_RATIO = 200 + +def require(condition, message): + if not condition: # remains active under python -O + raise ValueError(message) + +safe_xml_parser = etree.XMLParser( + load_dtd=False, resolve_entities=False, no_network=True, + huge_tree=False, recover=False, +) + +def validate_docx_archive(archive): + """Enforce archive bounds and CRC before this same handle is extracted.""" + require(os.fstat(archive.fp.fileno()).st_size <= MAX_ARCHIVE_BYTES, + "compressed DOCX file size above limit") + infos = archive.infolist() + require(len(infos) <= MAX_MEMBERS, "archive member count above limit") + names = {info.filename for info in infos} + require(len(names) == len(infos), "duplicate archive member names are unsafe") + require("[Content_Types].xml" in names and "word/document.xml" in names, + "required DOCX package parts are missing") + require(sum(info.file_size for info in infos) <= MAX_TOTAL_UNCOMPRESSED, + "declared archive size exceeds the edit limit") + actual_total = 0 + for info in infos: + require(info.file_size <= MAX_ENTRY, f"oversized part: {info.filename}") + require( + info.file_size / max(info.compress_size, 1) <= MAX_COMPRESSION_RATIO, + f"suspicious compression ratio: {info.filename}", + ) + if info.filename.endswith((".xml", ".rels")): + require(info.file_size <= MAX_XML_PART, f"oversized XML part: {info.filename}") + actual_size = 0 + with archive.open(info) as stream: # streams data and verifies its CRC + while chunk := stream.read(64 * 1024): + actual_size += len(chunk) + actual_total += len(chunk) + require(actual_size <= MAX_ENTRY, f"part exceeded read limit: {info.filename}") + require(actual_total <= MAX_TOTAL_UNCOMPRESSED, + "archive exceeded total read limit") + require(actual_size == info.file_size, f"size mismatch: {info.filename}") input_path = Path("input.docx") output_path = Path("output.docx").resolve() # keep output outside the temporary tree with TemporaryDirectory(prefix="docx-edit-") as scratch: src = Path(scratch) with ZipFile(input_path) as archive: + validate_docx_archive(archive) # same open handle; prevents validate/extract swap archive.extractall(src) - # Apply the required XML edits under `src` here, using an XML parser. + # Apply the required XML edits under `src` here with `safe_xml_parser`. + # Archive bounds are a safety gate, not a semantic XML-validity gate: Tier 2 may repair XML. content_types = src / "[Content_Types].xml" if not content_types.is_file(): raise FileNotFoundError(content_types) diff --git a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md index 402a1b5..1df5122 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md @@ -58,6 +58,7 @@ file is not. ```python import os +import fitz import pypdf output_path = "output.pdf" @@ -82,16 +83,73 @@ require( page_count == expected_page_count, f"expected {expected_page_count} pages, got {page_count}", ) +render_doc = fitz.open(output_path) +if render_doc.needs_pass and render_doc.authenticate("") <= 0: + if not password: + raise RuntimeError("set PDF_PASSWORD so encrypted widgets can be rendered") + if render_doc.authenticate(password) <= 0: + raise RuntimeError("PDF_PASSWORD could not decrypt the output for widget rendering") page_texts = { number: (page.extract_text() or "").strip() for number, page in enumerate(r.pages, start=1) } -def widget_count(page): - # Interactive-only pages (pure AcroForm screens) extract no text yet are valid output. +def normalized_box(box): + x0, y0, x1, y1 = (float(value) for value in box) + left, right = sorted((x0, x1)) + bottom, top = sorted((y0, y1)) + return left, bottom, right, top + +def normalized_size(box): + left, bottom, right, top = normalized_box(box) + return right - left, top - bottom + +def widget_count(page, rendered_page): + # Interactive-only pages can be valid, but hidden metadata widgets are not visible content. + non_viewable_flags = 1 | 2 | 32 # Invisible, Hidden, NoView annotation flags count = 0 for ref in page.get("/Annots") or []: - if ref.get_object().get("/Subtype") == "/Widget": + widget = ref.get_object() + if widget.get("/Subtype") != "/Widget": + continue + flags = int(widget.get("/F", 0)) + rectangle = widget.get("/Rect") + if flags & non_viewable_flags or rectangle is None: + continue + left, bottom, right, top = normalized_box(rectangle) + crop_left, crop_bottom, crop_right, crop_top = normalized_box(page.cropbox) + media_left, media_bottom, media_right, media_top = normalized_box(page.mediabox) + visible_left = max(crop_left, media_left) + visible_bottom = max(crop_bottom, media_bottom) + visible_right = min(crop_right, media_right) + visible_top = min(crop_top, media_top) + intersects_visible_page = ( + min(right, visible_right) > max(left, visible_left) + and min(top, visible_top) > max(bottom, visible_bottom) + ) + if right <= left or top <= bottom or not intersects_visible_page: + continue + # Do not require /AP: viewers may synthesize it from field defaults. Instead, + # render annotations on and off and require this widget region to change visibly. + xref = getattr(ref, "idnum", None) + if xref is None: + continue + try: + rendered_widget = rendered_page.load_widget(xref) + clip = ( + rendered_widget.rect * rendered_page.rotation_matrix + ) & rendered_page.rect + if clip.is_empty: + continue + with_widget = rendered_page.get_pixmap( + matrix=fitz.Matrix(2, 2), clip=clip, alpha=False, annots=True, + ) + without_widgets = rendered_page.get_pixmap( + matrix=fitz.Matrix(2, 2), clip=clip, alpha=False, annots=False, + ) + except (RuntimeError, ValueError): + continue + if with_widget.samples != without_widgets.samples: count += 1 return count @@ -100,7 +158,7 @@ missing_text_pages = [ number for number, text in page_texts.items() if number not in intentionally_raster_only_pages and not text - and widget_count(r.pages[number - 1]) == 0 + and widget_count(r.pages[number - 1], render_doc[number - 1]) == 0 ] require(not missing_text_pages, f"pages without extractable text: {missing_text_pages}") # Add task-specific checks when exact copy matters, for example @@ -114,8 +172,7 @@ for page_number, expected_strings in expected_strings_by_page.items(): expected_page_size = (595.2756, 841.8898) # replace for Letter or a task-specific size page_size_tolerance = 0.5 page_sizes = [ - (float(page.mediabox.width), float(page.mediabox.height)) - for page in r.pages + normalized_size(page.mediabox) for page in r.pages ] size_mismatches = [ (number, actual) @@ -129,20 +186,14 @@ require(not size_mismatches, f"unexpected page sizes: {size_mismatches}") # past the page box are clipped even though every check above still passes. Plain # text extraction silently drops fully off-page text, so use an enlarged clip for # text and inspect the placement boxes reported for every image and drawing. -import fitz - -overflow_doc = fitz.open(output_path) -if overflow_doc.needs_pass and overflow_doc.authenticate("") <= 0: - if not password: - raise RuntimeError("set PDF_PASSWORD so the encrypted output can be overflow-checked") - if overflow_doc.authenticate(password) <= 0: - raise RuntimeError("PDF_PASSWORD could not decrypt the output for overflow checking") +overflow_doc = render_doc overflow_pages = [] for page in overflow_doc: # These APIs report unrotated coordinates even when /Rotate is 90/270; # page.rect uses rotated dimensions. Compare against an unrotated crop-box # extent so valid high-y portrait content is not flagged on a rotated page. - page_box = fitz.Rect(0, 0, page.cropbox.width, page.cropbox.height) + crop_left, crop_bottom, crop_right, crop_top = normalized_box(page.cropbox) + page_box = fitz.Rect(0, 0, crop_right - crop_left, crop_top - crop_bottom) clip = fitz.Rect( page_box.x0 - 2000, page_box.y0 - 2000, page_box.x1 + 2000, page_box.y1 + 2000, diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md index ced6018..a81c3fb 100644 --- a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md @@ -75,23 +75,48 @@ def cached_category_labels(plot): return None return [[str(level) for level in label] for label in plot.categories.flattened_labels] +def series_name_content(series): + """Return a cached/literal title, or mark a worksheet-backed title unavailable.""" + titles = series._element.xpath("./c:tx") + if not titles: + return {"name": ""} + literal = titles[0].find(qn("c:v")) + if literal is not None: + return {"name": literal.text or ""} + reference = titles[0].find(qn("c:strRef")) + if reference is None: + return {"name": None, "name_cache_status": "unavailable"} + cache = reference.find(qn("c:strCache")) + if cache is None: + return {"name": None, "name_cache_status": "unavailable"} + point_count = cache.find(qn("c:ptCount")) + points = cache.findall(qn("c:pt")) + if (point_count is None or point_count.get("val") != "1" + or len(points) != 1 or points[0].get("idx") != "0"): + return {"name": None, "name_cache_status": "unavailable"} + value = points[0].find(qn("c:v")) + if value is None: + return {"name": None, "name_cache_status": "unavailable"} + return {"name": value.text or ""} + def series_content(series): + name_content = series_name_content(series) x_source = getattr(series._element, "xVal", None) if x_source is None: # category/value chart value_source = getattr(series._element, "val", None) if cached_numeric_points(value_source) is None: - return {"name": series.name, "values": None, "cache_status": "unavailable"} - return {"name": series.name, "values": list(series.values)} + return {**name_content, "values": None, "cache_status": "unavailable"} + return {**name_content, "values": list(series.values)} x_points = cached_numeric_points(x_source) y_points = cached_numeric_points(getattr(series._element, "yVal", None)) if x_points is None or y_points is None: - return {"name": series.name, "points": None, "cache_status": "unavailable"} - content = {"name": series.name, "x_points": x_points, "y_points": y_points} + return {**name_content, "points": None, "cache_status": "unavailable"} + content = {**name_content, "x_points": x_points, "y_points": y_points} size_source = getattr(series._element, "bubbleSize", None) if size_source is not None: bubble_points = cached_numeric_points(size_source) if bubble_points is None: - return {"name": series.name, "points": None, "cache_status": "unavailable"} + return {**name_content, "points": None, "cache_status": "unavailable"} content["bubble_points"] = bubble_points return content diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md b/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md index 43f2af9..68913a6 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md @@ -101,18 +101,19 @@ if any(expected_formulas.values()): ) for ws in wb.worksheets: print(f"{ws.title} dims:", ws.dimensions) - formulas = [ - (c.coordinate, formula_text(c.value)) - for row in ws.iter_rows() for c in row if c.data_type == "f" - ] - print(f"{ws.title} formula cells:", formulas[:10]) - actual_formulas = dict(formulas) + # `expected_formulas` is the task contract, so verify those coordinates directly. + # Never call unbounded iter_rows(): one styled extreme cell can make the rectangle huge. + actual_formulas = {} for coordinate, expected_formula in expected_formulas.get(ws.title, {}).items(): + cell = ws[coordinate] + actual_formula = formula_text(cell.value) if cell.data_type == "f" else None + actual_formulas[coordinate] = actual_formula require( - actual_formulas.get(coordinate) == expected_formula, + actual_formula == expected_formula, f"{ws.title}!{coordinate}: expected formula {expected_formula!r}, " - f"got {actual_formulas.get(coordinate)!r}", + f"got {actual_formula!r}", ) + print(f"{ws.title} expected formula cells:", list(actual_formulas.items())[:10]) for coordinate, expected_format in expected_number_formats.get(ws.title, {}).items(): actual_format = ws[coordinate].number_format require(actual_format == expected_format, ( diff --git a/plugins/Hylouis233/document-skills/tests/docx_fixture.py b/plugins/Hylouis233/document-skills/tests/docx_fixture.py index 355e5b0..9ae101d 100644 --- a/plugins/Hylouis233/document-skills/tests/docx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/docx_fixture.py @@ -6,6 +6,7 @@ # python xlsx_fixture.py (deps: openpyxl) # python docx_fixture.py (deps: python-docx, pymupdf, soffice on PATH) import copy +import os import subprocess import sys import zipfile @@ -961,8 +962,38 @@ def tc_text(tc): plain_doc.save("repack-without-media.docx") +def validate_docx_archive_bounds(archive): + require(os.fstat(archive.fp.fileno()).st_size <= MAX_ARCHIVE_BYTES, + "compressed DOCX file size above limit") + infos = archive.infolist() + require(len(infos) <= MAX_MEMBERS, "archive member count above limit") + names = {info.filename for info in infos} + require(len(names) == len(infos), "duplicate archive member names are unsafe") + require("[Content_Types].xml" in names and "word/document.xml" in names, + "required DOCX package parts are missing") + require(sum(info.file_size for info in infos) <= MAX_TOTAL_UNCOMPRESSED, + "declared archive size exceeds the edit limit") + actual_total = 0 + for info in infos: + require(info.file_size <= MAX_ENTRY, f"oversized part: {info.filename}") + require(info.file_size / max(info.compress_size, 1) <= MAX_COMPRESSION_RATIO, + f"suspicious compression ratio: {info.filename}") + if info.filename.endswith((".xml", ".rels")): + require(info.file_size <= MAX_XML_PART, f"oversized XML part: {info.filename}") + actual_size = 0 + with archive.open(info) as stream: + while chunk := stream.read(64 * 1024): + actual_size += len(chunk) + actual_total += len(chunk) + require(actual_size <= MAX_ENTRY, f"part exceeded read limit: {info.filename}") + require(actual_total <= MAX_TOTAL_UNCOMPRESSED, + "archive exceeded total read limit") + require(actual_size == info.file_size, f"size mismatch: {info.filename}") + + def repack_tree(source, output, extraction_root): with zipfile.ZipFile(source) as archive: + validate_docx_archive_bounds(archive) archive.extractall(extraction_root) content_types = Path(extraction_root) / "[Content_Types].xml" files = sorted( @@ -976,6 +1007,32 @@ def repack_tree(source, output, extraction_root): archive.write(path, path.relative_to(extraction_root).as_posix()) +bomb_extract_root = Path("bomb-extract") +bomb_extract_root.mkdir() +try: + repack_tree("compressed-bomb.docx", "bomb-output.docx", bomb_extract_root) + edit_bomb_rejected_before_extract = False +except ValueError as error: + edit_bomb_rejected_before_extract = ( + str(error) == "suspicious compression ratio: word/document.xml" + and + not any(bomb_extract_root.iterdir()) and not Path("bomb-output.docx").exists() + ) +check("Tier 2 edit rejects an archive bomb before extracting any member", + edit_bomb_rejected_before_extract) +check("Tier 2 pre-extract bounds remain active under optimized Python", + __debug__ or edit_bomb_rejected_before_extract) + +with zipfile.ZipFile("malformed-for-repair.docx", "w", zipfile.ZIP_STORED) as archive: + archive.writestr("[Content_Types].xml", "") + archive.writestr("word/document.xml", " max(left, visible_left) + and min(top, visible_top) > max(bottom, visible_bottom) + ) + if right <= left or top <= bottom or not intersects_visible_page: + continue + xref = getattr(ref, "idnum", None) + if xref is None: + continue + try: + rendered_widget = rendered_page.load_widget(xref) + clip = ( + rendered_widget.rect * rendered_page.rotation_matrix + ) & rendered_page.rect + if clip.is_empty: + continue + with_widget = rendered_page.get_pixmap( + matrix=fitz.Matrix(2, 2), clip=clip, alpha=False, annots=True, + ) + without_widgets = rendered_page.get_pixmap( + matrix=fitz.Matrix(2, 2), clip=clip, alpha=False, annots=False, + ) + except (RuntimeError, ValueError): + continue + if with_widget.samples != without_widgets.samples: count += 1 return count widget_postcheck = pypdf.PdfReader("widget-only.pdf") +widget_render = fitz.open("widget-only.pdf") widget_text = (widget_postcheck.pages[0].extract_text() or "").strip() check("widget-only page extracts no text", widget_text == "", repr(widget_text)) -check("postcheck counts the widget annotation", widget_count(widget_postcheck.pages[0]) == 1) +check("postcheck counts the visibly rendered widget annotation", + widget_count(widget_postcheck.pages[0], widget_render[0]) == 1) check("widget-only page passes the text postcheck via the widget exemption", - bool(widget_text) or widget_count(widget_postcheck.pages[0]) > 0) + bool(widget_text) or widget_count(widget_postcheck.pages[0], widget_render[0]) > 0) + +from pypdf.generic import ArrayObject, FloatObject, NameObject, NumberObject + +for label, flag in (("invisible", 1), ("hidden", 2), ("no-view", 32)): + hidden_writer = pypdf.PdfWriter() + hidden_writer.append(widget_postcheck) + hidden_widget = hidden_writer.pages[0]["/Annots"][0].get_object() + hidden_widget[NameObject("/F")] = NumberObject(flag) + hidden_path = f"widget-{label}.pdf" + with open(hidden_path, "wb") as output: + hidden_writer.write(output) + hidden_page = pypdf.PdfReader(hidden_path).pages[0] + hidden_render = fitz.open(hidden_path) + check(f"{label} widget does not exempt an otherwise blank page", + widget_count(hidden_page, hidden_render[0]) == 0 + and not bool((hidden_page.extract_text() or "").strip())) + +appearance_writer = pypdf.PdfWriter() +appearance_writer.append(widget_postcheck) +appearance_widget = appearance_writer.pages[0]["/Annots"][0].get_object() +del appearance_widget[NameObject("/AP")] +with open("widget-no-appearance.pdf", "wb") as output: + appearance_writer.write(output) +appearance_page = pypdf.PdfReader("widget-no-appearance.pdf").pages[0] +appearance_render = fitz.open("widget-no-appearance.pdf") +check("visible widget without /AP can still be viewer-generated and render visibly", + widget_count(appearance_page, appearance_render[0]) == 1) + +blank_appearance_writer = pypdf.PdfWriter() +blank_appearance_writer.append(widget_postcheck) +blank_appearance_widget = blank_appearance_writer.pages[0]["/Annots"][0].get_object() +for key in ("/AP", "/MK", "/BS", "/DA", "/V", "/DV"): + blank_appearance_widget.pop(NameObject(key), None) +blank_acroform = blank_appearance_writer._root_object["/AcroForm"].get_object() +for key in ("/DA", "/DR", "/NeedAppearances"): + blank_acroform.pop(NameObject(key), None) +with open("widget-blank-appearance.pdf", "wb") as output: + blank_appearance_writer.write(output) +blank_appearance_page = pypdf.PdfReader("widget-blank-appearance.pdf").pages[0] +blank_appearance_render = fitz.open("widget-blank-appearance.pdf") +check("widget with no renderable appearance does not exempt a white page", + widget_count(blank_appearance_page, blank_appearance_render[0]) == 0 + and not any(value != 255 for value in + blank_appearance_render[0].get_pixmap(alpha=False).samples)) + +for label, rectangle in ( + ("zero-area", [72, 740, 72, 760]), + ("off-page", [1000, 1000, 1100, 1100]), +): + geometry_writer = pypdf.PdfWriter() + geometry_writer.append(widget_postcheck) + geometry_widget = geometry_writer.pages[0]["/Annots"][0].get_object() + geometry_widget[NameObject("/Rect")] = ArrayObject([ + FloatObject(value) for value in rectangle + ]) + geometry_path = f"widget-{label}.pdf" + with open(geometry_path, "wb") as output: + geometry_writer.write(output) + geometry_page = pypdf.PdfReader(geometry_path).pages[0] + geometry_render = fitz.open(geometry_path) + check(f"{label} widget does not exempt an otherwise blank page", + widget_count(geometry_page, geometry_render[0]) == 0) + +reversed_writer = pypdf.PdfWriter() +reversed_writer.append(widget_postcheck) +reversed_page = reversed_writer.pages[0] +reversed_widget = reversed_page["/Annots"][0].get_object() +reversed_widget[NameObject("/Rect")] = ArrayObject([ + FloatObject(332), FloatObject(760), FloatObject(72), FloatObject(740), +]) +with open("widget-reversed-rect.pdf", "wb") as output: + reversed_writer.write(output) +reversed_reader_page = pypdf.PdfReader("widget-reversed-rect.pdf").pages[0] +reversed_render = fitz.open("widget-reversed-rect.pdf") +check("legal reversed widget rectangle is normalized and remains visible", + widget_count(reversed_reader_page, reversed_render[0]) == 1) + +reversed_boxes_writer = pypdf.PdfWriter() +reversed_boxes_writer.append(widget_postcheck) +reversed_boxes_page = reversed_boxes_writer.pages[0] +reversed_page_box = ArrayObject([ + FloatObject(A4[0]), FloatObject(A4[1]), FloatObject(0), FloatObject(0), +]) +reversed_boxes_page[NameObject("/MediaBox")] = reversed_page_box +reversed_boxes_page[NameObject("/CropBox")] = ArrayObject(reversed_page_box) +with open("widget-reversed-page-boxes.pdf", "wb") as output: + reversed_boxes_writer.write(output) +reversed_boxes_reader_page = pypdf.PdfReader("widget-reversed-page-boxes.pdf").pages[0] +reversed_boxes_render = fitz.open("widget-reversed-page-boxes.pdf") +check("legal reversed page boxes are normalized before widget intersection", + widget_count(reversed_boxes_reader_page, reversed_boxes_render[0]) == 1 + and all( + abs(actual - expected) < A4_TOLERANCE + for actual, expected in zip( + normalized_size(reversed_boxes_reader_page.mediabox), A4, + ) + )) + +rotated_widget_writer = pypdf.PdfWriter() +rotated_widget_writer.append(widget_postcheck) +rotated_widget_page = rotated_widget_writer.pages[0] +rotated_widget_page.rotate(90) +rotated_widget = rotated_widget_page["/Annots"][0].get_object() +rotated_widget[NameObject("/Rect")] = ArrayObject([ + FloatObject(72), FloatObject(72), FloatObject(332), FloatObject(92), +]) +with open("widget-rotated.pdf", "wb") as output: + rotated_widget_writer.write(output) +rotated_widget_reader_page = pypdf.PdfReader("widget-rotated.pdf").pages[0] +rotated_widget_render = fitz.open("widget-rotated.pdf") +check("visible widget on a rotated page is clipped in rotated coordinates", + widget_count(rotated_widget_reader_page, rotated_widget_render[0]) == 1) blank_writer = pypdf.PdfWriter() blank_writer.add_blank_page(width=200, height=300) with open("blank.pdf", "wb") as f: blank_writer.write(f) blank_r = pypdf.PdfReader("blank.pdf") +blank_render = fitz.open("blank.pdf") check("a truly blank page still fails the text postcheck", - not (bool((blank_r.pages[0].extract_text() or "").strip()) or widget_count(blank_r.pages[0]) > 0)) + not (bool((blank_r.pages[0].extract_text() or "").strip()) + or widget_count(blank_r.pages[0], blank_render[0]) > 0)) # ---- transform.md AcroForm snippet: clone into writer, fill on writer pages ---- from pypdf import PdfReader, PdfWriter @@ -978,7 +1145,8 @@ def overflow_pages(path, password=None): pages = [] for page in doc: # Plain block extraction drops fully off-page text; enlarge the clip. - page_box = fitz.Rect(0, 0, page.cropbox.width, page.cropbox.height) + crop_left, crop_bottom, crop_right, crop_top = normalized_box(page.cropbox) + page_box = fitz.Rect(0, 0, crop_right - crop_left, crop_top - crop_bottom) clip = fitz.Rect( page_box.x0 - 2000, page_box.y0 - 2000, page_box.x1 + 2000, page_box.y1 + 2000, @@ -1003,6 +1171,8 @@ def overflow_pages(path, password=None): check("in-bounds PDF reports no overflow pages", overflow_pages("overflow.pdf") == []) check("blank-user-password encrypted PDF passes the independent overflow check", overflow_pages("blank-user-password.pdf") == []) +check("reversed page boxes remain valid through the overflow postcheck", + overflow_pages("widget-reversed-page-boxes.pdf") == []) check("off-page text is detected by the overflow check (negative control)", overflow_pages("overflow-bad.pdf") == [1]) check("in-bounds image and vector drawing pass the overflow check", diff --git a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py index fc0a8d3..ed4f49e 100644 --- a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py @@ -418,23 +418,48 @@ def cached_category_labels(plot): return [[str(level) for level in label] for label in plot.categories.flattened_labels] +def series_name_content(series): + titles = series._element.xpath("./c:tx") + if not titles: + return {"name": ""} + literal = titles[0].find(qn("c:v")) + if literal is not None: + return {"name": literal.text or ""} + reference = titles[0].find(qn("c:strRef")) + if reference is None: + return {"name": None, "name_cache_status": "unavailable"} + cache = reference.find(qn("c:strCache")) + if cache is None: + return {"name": None, "name_cache_status": "unavailable"} + point_count = cache.find(qn("c:ptCount")) + points = cache.findall(qn("c:pt")) + if (point_count is None or point_count.get("val") != "1" + or len(points) != 1 or points[0].get("idx") != "0"): + return {"name": None, "name_cache_status": "unavailable"} + value = points[0].find(qn("c:v")) + if value is None: + return {"name": None, "name_cache_status": "unavailable"} + return {"name": value.text or ""} + + def series_content(series): + name_content = series_name_content(series) x_source = getattr(series._element, "xVal", None) if x_source is None: value_source = getattr(series._element, "val", None) if cached_numeric_points(value_source) is None: - return {"name": series.name, "values": None, "cache_status": "unavailable"} - return {"name": series.name, "values": list(series.values)} + return {**name_content, "values": None, "cache_status": "unavailable"} + return {**name_content, "values": list(series.values)} x_points = cached_numeric_points(x_source) y_points = cached_numeric_points(getattr(series._element, "yVal", None)) if x_points is None or y_points is None: - return {"name": series.name, "points": None, "cache_status": "unavailable"} - content = {"name": series.name, "x_points": x_points, "y_points": y_points} + return {**name_content, "points": None, "cache_status": "unavailable"} + content = {**name_content, "x_points": x_points, "y_points": y_points} size_source = getattr(series._element, "bubbleSize", None) if size_source is not None: bubble_points = cached_numeric_points(size_source) if bubble_points is None: - return {"name": series.name, "points": None, "cache_status": "unavailable"} + return {**name_content, "points": None, "cache_status": "unavailable"} content["bubble_points"] = bubble_points return content @@ -564,6 +589,89 @@ def extract_slide_content(slide): ) category_prs.save("category-cache-source.pptx") +series_name_mutated = Presentation("category-cache-source.pptx") +series_name_chart = next( + shape.chart for shape in series_name_mutated.slides[0].shapes if shape.has_chart +) +series_name_ref = list(series_name_chart.plots[0].series)[0]._element.xpath( + "./c:tx/c:strRef" +)[0] +series_name_formula = series_name_ref.find(qn("c:f")).text +series_name_ref.remove(series_name_ref.find(qn("c:strCache"))) +series_name_mutated.save("series-name-cacheless.pptx") +series_name_reopened = Presentation("series-name-cacheless.pptx") +series_name_chart = next( + shape.chart for shape in series_name_reopened.slides[0].shapes if shape.has_chart +) +series_name_series = list(series_name_chart.plots[0].series)[0] +series_name_result = series_content(series_name_series) +check("cacheless series title retains its worksheet formula", + series_name_series._element.xpath("./c:tx/c:strRef/c:f")[0].text + == series_name_formula) +check("cacheless series title is unavailable without aborting value inventory", + series_name_result == { + "name": None, "name_cache_status": "unavailable", "values": [1.0, 2.0], + }, series_name_result) +series_name_inventory = extract_slide_content(series_name_reopened.slides[0]) +series_name_inventory_items = series_name_inventory["charts"][0]["plots"][0]["series"] +check("deck inventory continues through cached siblings after an unavailable series title", + series_name_inventory_items[0] == series_name_result + and series_name_inventory_items[1] + == {"name": "Cached series", "values": [3.0, 4.0]}, + series_name_inventory_items) + + +class RaisingNameSeries: + def __init__(self, element, values): + self._element = element + self.values = values + + @property + def name(self): + raise RuntimeError("series.name must not be accessed") + + +raising_name_result = series_content(RaisingNameSeries( + series_name_series._element, list(series_name_series.values), +)) +check("cacheless name path never touches the python-pptx name property", + raising_name_result == series_name_result, raising_name_result) + +source_name_series = list(series_name_chart.plots[0].series)[1] +malformed_name_element = copy.deepcopy(source_name_series._element) +malformed_cache = malformed_name_element.find(qn("c:tx")).find( + qn("c:strRef") +).find(qn("c:strCache")) +for point in malformed_cache.findall(qn("c:pt")): + malformed_cache.remove(point) +malformed_name_result = series_content(RaisingNameSeries( + malformed_name_element, list(source_name_series.values), +)) +check("incomplete series-name cache is unavailable while values remain readable", + malformed_name_result == { + "name": None, "name_cache_status": "unavailable", "values": [3.0, 4.0], + }, malformed_name_result) + +literal_name_element = copy.deepcopy(source_name_series._element) +literal_tx = literal_name_element.find(qn("c:tx")) +literal_tx.remove(literal_tx.find(qn("c:strRef"))) +literal_value = OxmlElement("c:v") +literal_value.text = "Literal series" +literal_tx.append(literal_value) +literal_name_result = series_content(RaisingNameSeries( + literal_name_element, list(source_name_series.values), +)) +check("literal series title is read directly without the name property", + literal_name_result["name"] == "Literal series", literal_name_result) + +untitled_element = copy.deepcopy(source_name_series._element) +untitled_element.remove(untitled_element.find(qn("c:tx"))) +untitled_result = series_content(RaisingNameSeries( + untitled_element, list(source_name_series.values), +)) +check("series without a title remains a readable unnamed series", + untitled_result["name"] == "", untitled_result) + category_mutated = Presentation("category-cache-source.pptx") category_chart = next( shape.chart for shape in category_mutated.slides[0].shapes if shape.has_chart diff --git a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py index 6fdcfee..3fb54ad 100644 --- a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py @@ -1053,6 +1053,44 @@ def aggregation_regions(source_path): calc_reopened.active["A3"].value == "=SUM(A1:A2)", calc_reopened.active["A3"].value) calc_reopened.close() +# ---- SKILL.md postcheck: formula inventory stays sparse at worksheet limits ---- +formula_bound_wb = openpyxl.Workbook() +formula_bound_ws = formula_bound_wb.active +formula_bound_ws["D2"] = "=1+1" +formula_bound_ws["XFD1048576"].number_format = "0.00" # styled but empty extreme cell +formula_bound_wb.save("formula-bound.xlsx") +formula_bound_reopened = openpyxl.load_workbook("formula-bound.xlsx", data_only=False) +formula_bound_ws = formula_bound_reopened.active + + +def expected_formula_inventory(sheet, expected): + actual = {} + for coordinate, expected_formula in expected.items(): + cell = sheet[coordinate] + actual_formula = formula_text(cell.value) if cell.data_type == "f" else None + if actual_formula != expected_formula: + raise ValueError( + f"{coordinate}: expected {expected_formula!r}, got {actual_formula!r}" + ) + actual[coordinate] = actual_formula + return actual + + +check("extreme styled cell inflates the rectangular worksheet bounds (negative control)", + formula_bound_ws.max_row == 1_048_576 and formula_bound_ws.max_column == 16_384, + (formula_bound_ws.max_row, formula_bound_ws.max_column)) +original_iter_rows = formula_bound_ws.iter_rows +formula_bound_ws.iter_rows = lambda *args, **kwargs: (_ for _ in ()).throw( + RuntimeError("unbounded iter_rows must not run") +) +try: + bounded_formulas = expected_formula_inventory(formula_bound_ws, {"D2": "=1+1"}) +finally: + formula_bound_ws.iter_rows = original_iter_rows +check("formula postcheck uses bounded public coordinate lookups", + bounded_formulas == {"D2": "=1+1"}, bounded_formulas) +formula_bound_reopened.close() + # ---- edit.md snippet: extension detection is prefix-independent ------------------ X14_URI = b"http://schemas.microsoft.com/office/spreadsheetml/2009/9/main" From 88a48952a2090c7ce761d1ce1abee05113b8265a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=98=E6=89=AC?= Date: Mon, 17 Aug 2026 21:01:00 +0800 Subject: [PATCH 28/34] fix(document-skills): close remaining review edge cases --- .../skills/docx/references/edit.md | 78 ++- .../skills/pptx/references/analyze.md | 296 ++++++-- .../skills/xlsx/references/csv.md | 30 +- .../document-skills/tests/docx_fixture.py | 220 +++++- .../document-skills/tests/pptx_fixture.py | 652 ++++++++++++++++-- .../document-skills/tests/xlsx_fixture.py | 47 +- 6 files changed, 1174 insertions(+), 149 deletions(-) diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/edit.md b/plugins/Hylouis233/document-skills/skills/docx/references/edit.md index e2cc759..4e90b56 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/edit.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/edit.md @@ -132,7 +132,8 @@ Rules that keep the archive valid: ```python import os -from pathlib import Path +import unicodedata +from pathlib import Path, PurePosixPath from tempfile import TemporaryDirectory from zipfile import ZIP_DEFLATED, ZipFile from lxml import etree @@ -143,11 +144,85 @@ MAX_XML_PART = 20 * 1024 * 1024 MAX_ENTRY = 100 * 1024 * 1024 MAX_TOTAL_UNCOMPRESSED = 500 * 1024 * 1024 MAX_COMPRESSION_RATIO = 200 +MAX_MEMBER_COMPONENT_BYTES = 255 +MAX_MEMBER_COMPONENT_UTF16_UNITS = 255 +MAX_MEMBER_PATH_BYTES = 1024 +MAX_MEMBER_PATH_UTF16_UNITS = 240 +MAX_MEMBER_COMPONENTS = 64 def require(condition, message): if not condition: # remains active under python -O raise ValueError(message) +WINDOWS_DEVICE_NAMES = { + "con", "prn", "aux", "nul", "conin$", "conout$", + *(f"com{suffix}" for suffix in "123456789¹²³"), + *(f"lpt{suffix}" for suffix in "123456789¹²³"), +} + +def extraction_key(name): + """Return a component key only for portable, canonical member names.""" + is_directory = name.endswith("/") + path = name[:-1] if is_directory else name + require(path and not name.startswith("/") and "\\" not in name, + f"non-canonical archive member path: {name}") + parts = path.split("/") + require(len(parts) <= MAX_MEMBER_COMPONENTS, + "archive member depth exceeds portable extraction limit") + require( + all( + part not in {"", ".", ".."} + and not any(character in ':<>|"?*' for character in part) + and not any(ord(character) < 32 for character in part) + and not part.endswith((".", " ")) + and unicodedata.normalize("NFC", part) == part + and part.partition(".")[0].rstrip(" ").casefold() not in WINDOWS_DEVICE_NAMES + for part in parts + ), + f"non-canonical archive member path: {name}", + ) + require( + all( + len(part.encode("utf-8")) <= MAX_MEMBER_COMPONENT_BYTES + and len(part.encode("utf-16-le")) // 2 <= MAX_MEMBER_COMPONENT_UTF16_UNITS + for part in parts + ), + "archive member component exceeds portable extraction limit", + ) + require(len(path.encode("utf-8")) <= MAX_MEMBER_PATH_BYTES + and len(path.encode("utf-16-le")) // 2 <= MAX_MEMBER_PATH_UTF16_UNITS, + "archive member path exceeds portable extraction limit") + canonical = PurePosixPath(*parts).as_posix() + require(canonical == path, f"non-canonical archive member path: {name}") + key = tuple(unicodedata.normalize("NFC", part.casefold()) for part in parts) + return key, tuple(parts), is_directory + +def validate_extraction_paths(infos): + """Reject aliases and file/directory collisions with a linear component trie.""" + root = {"children": {}, "member": False, "file": False, "spelling": None} + for info in infos: + key, spellings, is_directory = extraction_key(info.filename) + node = root + for normalized, spelling in zip(key, spellings): + require(not node["file"], + "archive file and directory paths collide after extraction") + child = node["children"].get(normalized) + if child is None: + child = { + "children": {}, "member": False, "file": False, + "spelling": spelling, + } + node["children"][normalized] = child + else: + require(child["spelling"] == spelling, + "archive member path spelling collides after extraction") + node = child + require(not node["member"], "archive member paths collide after extraction") + require(is_directory or not node["children"], + "archive file and directory paths collide after extraction") + node["member"] = True + node["file"] = not is_directory + safe_xml_parser = etree.XMLParser( load_dtd=False, resolve_entities=False, no_network=True, huge_tree=False, recover=False, @@ -161,6 +236,7 @@ def validate_docx_archive(archive): require(len(infos) <= MAX_MEMBERS, "archive member count above limit") names = {info.filename for info in infos} require(len(names) == len(infos), "duplicate archive member names are unsafe") + validate_extraction_paths(infos) require("[Content_Types].xml" in names and "word/document.xml" in names, "required DOCX package parts are missing") require(sum(info.file_size for info in infos) <= MAX_TOTAL_UNCOMPRESSED, diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md index a81c3fb..6965875 100644 --- a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md @@ -47,33 +47,153 @@ def picture_content(shape): "bytes": len(image.blob), } -def cached_numeric_points(source): - """Return indexed cached points, or None when the cache metadata is unavailable.""" +MAX_CHART_POINTS = 100_000 +MAX_CATEGORY_LEVELS = 16 +CATEGORY_SOURCE_NAMES = {"strRef", "numRef", "multiLvlStrRef", "strLit", "numLit"} + +def cache_point_count(cache): + point_counts = cache.findall(qn("c:ptCount")) + if len(point_counts) != 1: + return None + try: + point_count = int(point_counts[0].get("val")) + except (TypeError, ValueError): + return None + return point_count if 0 <= point_count <= MAX_CHART_POINTS else None + +def consume_point_budget(point_budget, count): + if count > MAX_CHART_POINTS: + return False + if point_budget is None: + return True + if count > point_budget["remaining"]: + return False + point_budget["remaining"] -= count + return True + +def cached_numeric_points( + source, *, fill_missing=False, include_count=False, point_budget=None +): + """Return bounded indexed cached points, or None when metadata is unavailable.""" if source is None: return None - point_counts = source.xpath("./c:numRef/c:numCache/c:ptCount | ./c:numLit/c:ptCount") - if not point_counts: + caches = source.xpath("./c:numRef/c:numCache | ./c:numLit") + if len(caches) != 1 or (point_count := cache_point_count(caches[0])) is None: return None - points = source.xpath("./c:numRef/c:numCache/c:pt | ./c:numLit/c:pt") - return [ - (int(point.get("idx")), float(value.text)) - for point in points - if point.get("idx") is not None and (value := point.find(qn("c:v"))) is not None + cached = {} + for point in caches[0].findall(qn("c:pt")): + value = point.find(qn("c:v")) + try: + index = int(point.get("idx")) + except (TypeError, ValueError): + return None + if value is None or not 0 <= index < point_count or index in cached: + return None + try: + parsed = float(value.text) if value.text not in (None, "") else value.text + except (TypeError, ValueError): + parsed = value.text # preserve #N/A and other error markers + cached[index] = parsed + if fill_missing: + if not consume_point_budget(point_budget, point_count): + return None + points = [(index, cached.get(index)) for index in range(point_count)] + else: + points = sorted(cached.items()) + return {"point_count": point_count, "points": points} if include_count else points + +def cached_text_point_map(container, point_count): + values = {} + for point in container.findall(qn("c:pt")): + try: + index = int(point.get("idx")) + except (TypeError, ValueError): + return None + value = point.find(qn("c:v")) + if value is None or not 0 <= index < point_count or index in values: + return None + values[index] = value.text or "" + return values + +def category_source(series): + category = series._element.find(qn("c:cat")) + if category is None: + return None, None + sources = [ + child for child in category + if isinstance(child.tag, str) + and etree.QName(child).localname in CATEGORY_SOURCE_NAMES ] + return (category, sources[0]) if len(sources) == 1 else (category, None) -def cached_category_labels(plot): - """Return flattened labels, or None when a formula has no category cache.""" - category_nodes = plot._element.xpath("./c:ser[1]/c:cat") - if not category_nodes: +def cached_category_labels(series, point_budget=None): + """Return this series' flattened labels, or None when its cache is unavailable.""" + category, source = category_source(series) + if category is None: return [] - cache_payload = category_nodes[0].xpath( - "./c:strRef/c:strCache/c:ptCount | ./c:numRef/c:numCache/c:ptCount | " - "./c:multiLvlStrRef/c:multiLvlStrCache/c:ptCount | " - "./c:strLit/c:ptCount | ./c:numLit/c:ptCount" - ) - if not cache_payload: + if source is None: + return None + source_name = etree.QName(source).localname + + if source_name == "multiLvlStrRef": + caches = source.findall(qn("c:multiLvlStrCache")) + if len(caches) != 1 or (count := cache_point_count(caches[0])) is None: + return None + level_nodes = caches[0].findall(qn("c:lvl")) + if len(level_nodes) > MAX_CATEGORY_LEVELS or (count and not level_nodes): + return None + levels = [cached_text_point_map(level, count) for level in level_nodes] + if any(points is None for points in levels): + return None + if not consume_point_budget(point_budget, count * max(len(level_nodes), 1)): + return None + expanded_levels = [] + for level_index, points in enumerate(levels): + if level_index == 0: + expanded_levels.append([points.get(index, "") for index in range(count)]) + continue + ordered = sorted(points.items()) + if not ordered: + expanded_levels.append([""] * count) + continue + cursor = 0 + expanded = [] + for leaf_index in range(count): + while cursor + 1 < len(ordered) and ordered[cursor + 1][0] <= leaf_index: + cursor += 1 + expanded.append(ordered[cursor][1]) + expanded_levels.append(expanded) + return [ + [level[index] for level in reversed(expanded_levels)] + for index in range(count) + ] + + cache_name = {"strRef": "c:strCache", "numRef": "c:numCache"}.get(source_name) + if cache_name is None: + cache = source + else: + caches = source.findall(qn(cache_name)) + if len(caches) != 1: + return None + cache = caches[0] + if (count := cache_point_count(cache)) is None: + return None + values = cached_text_point_map(cache, count) + if values is None: + return None + if not consume_point_budget(point_budget, count): return None - return [[str(level) for level in label] for label in plot.categories.flattened_labels] + return [[values.get(index, "")] for index in range(count)] + +def category_content(series, point_budget=None): + labels = cached_category_labels(series, point_budget) + _, source = category_source(series) + formula = None if source is None else source.find(qn("c:f")) + return { + "categories": labels, + "category_formula": formula.text if formula is not None else None, + "category_cache_status": "unavailable" if labels is None else "available", + } def series_name_content(series): """Return a cached/literal title, or mark a worksheet-backed title unavailable.""" @@ -99,37 +219,84 @@ def series_name_content(series): return {"name": None, "name_cache_status": "unavailable"} return {"name": value.text or ""} -def series_content(series): +def series_content(series, *, include_categories=False, point_budget=None): + if point_budget is None: + point_budget = {"remaining": MAX_CHART_POINTS} name_content = series_name_content(series) + content = { + **name_content, + **(category_content(series, point_budget) if include_categories else {}), + } x_source = getattr(series._element, "xVal", None) if x_source is None: # category/value chart value_source = getattr(series._element, "val", None) - if cached_numeric_points(value_source) is None: - return {**name_content, "values": None, "cache_status": "unavailable"} - return {**name_content, "values": list(series.values)} - x_points = cached_numeric_points(x_source) - y_points = cached_numeric_points(getattr(series._element, "yVal", None)) - if x_points is None or y_points is None: - return {**name_content, "points": None, "cache_status": "unavailable"} - content = {**name_content, "x_points": x_points, "y_points": y_points} + value_cache = cached_numeric_points( + value_source, fill_missing=True, include_count=True, + point_budget=point_budget, + ) + if value_cache is None: + return {**content, "values": None, "cache_status": "unavailable"} + return {**content, "values": [value for _, value in value_cache["points"]]} + x_cache = cached_numeric_points(x_source, include_count=True) + y_cache = cached_numeric_points( + getattr(series._element, "yVal", None), include_count=True + ) + if x_cache is None or y_cache is None: + return {**content, "points": None, "cache_status": "unavailable"} + content.update({ + "x_points": x_cache["points"], "x_point_count": x_cache["point_count"], + "y_points": y_cache["points"], "y_point_count": y_cache["point_count"], + }) size_source = getattr(series._element, "bubbleSize", None) if size_source is not None: - bubble_points = cached_numeric_points(size_source) - if bubble_points is None: - return {**name_content, "points": None, "cache_status": "unavailable"} - content["bubble_points"] = bubble_points + bubble_cache = cached_numeric_points(size_source, include_count=True) + if bubble_cache is None: + return {**content, "points": None, "cache_status": "unavailable"} + content.update({ + "bubble_points": bubble_cache["points"], + "bubble_point_count": bubble_cache["point_count"], + }) return content -def chart_axis_titles(chart): - """Return titles for axes the chart actually exposes (pie charts have none).""" - titles = {} - for label, attribute in (("category", "category_axis"), ("value", "value_axis")): - try: - axis = getattr(chart, attribute) - except (AttributeError, ValueError): - continue - titles[label] = axis.axis_title.text_frame.text if axis.has_title else "" - return titles +def chart_axis_text(axis): + title = axis.find(qn("c:title")) + if title is None: + return "" + rich = title.xpath("./c:tx/c:rich") + references = title.xpath("./c:tx/c:strRef") + if len(rich) + len(references) != 1: + return None + if rich: + return "\n".join(paragraph.text for paragraph in rich[0].findall(qn("a:p"))) + caches = references[0].findall(qn("c:strCache")) + if (len(caches) != 1 or (count := cache_point_count(caches[0])) is None + or count != 1): + return None + values = cached_text_point_map(caches[0], count) + return None if values is None else "\n".join(values.get(index, "") for index in range(count)) + +def chart_axes(chart): + """Return every category, date, value, and series axis in document order.""" + axes = chart._element.xpath( + "./c:chart/c:plotArea/c:catAx | ./c:chart/c:plotArea/c:dateAx | " + "./c:chart/c:plotArea/c:valAx | ./c:chart/c:plotArea/c:serAx" + ) + return [ + { + "kind": etree.QName(axis).localname, + "id": axis.find(qn("c:axId")).get("val") if axis.find(qn("c:axId")) is not None else None, + "position": ( + axis.find(qn("c:axPos")).get("val") + if axis.find(qn("c:axPos")) is not None else None + ), + "cross_axis_id": ( + axis.find(qn("c:crossAx")).get("val") + if axis.find(qn("c:crossAx")) is not None else None + ), + "title": chart_axis_text(axis), + } + for axis in axes + ] def smartart_content(shape): """Extract SmartArt data-part labels, or report why the diagram is unreadable.""" @@ -153,6 +320,7 @@ def smartart_content(shape): prs = Presentation("input.pptx") print("slide size:", prs.slide_width, prs.slide_height) +point_budget = {"remaining": MAX_CHART_POINTS} for i, slide in enumerate(prs.slides): layout = slide.slide_layout.name title = slide.shapes.title.text_frame.text if slide.shapes.title is not None else "" @@ -176,20 +344,19 @@ for i, slide in enumerate(prs.slides): for plot in chart.plots: items = list(plot.series) has_xy_values = any(getattr(item._element, "xVal", None) is not None for item in items) - categories = [] if has_xy_values else cached_category_labels(plot) - series = [series_content(item) for item in items] + series = [ + series_content( + item, include_categories=not has_xy_values, point_budget=point_budget + ) + for item in items + ] plots.append({ "kind": type(plot).__name__, - "categories": categories, - "category_cache_status": ( - "not-applicable" if has_xy_values else - "unavailable" if categories is None else "available" - ), "series": series, }) charts.append({ "title": chart_title, - "axis_titles": chart_axis_titles(chart), + "axes": chart_axes(chart), "plots": plots, }) pictures = [ @@ -357,16 +524,30 @@ def theme_faces_for_slide(slide): } return cache_key, theme_cache[cache_key] -EAST_ASIAN_SCRIPTS = {"Hans", "Hant", "Jpan", "Hang"} +EAST_ASIAN_SCRIPTS = {"Hans", "Hant", "Jpan", "Hang", "Bopo"} COMPLEX_SCRIPTS = {"Arab", "Hebr", "Deva", "Beng", "Taml", "Thai"} +HAN_RANGES = ( + (0x2E80, 0x2E99), (0x2E9B, 0x2EF3), (0x2F00, 0x2FD5), + (0x3005, 0x3005), (0x3007, 0x3007), (0x3021, 0x3029), (0x3038, 0x303B), + (0x3400, 0x4DBF), (0x4E00, 0x9FFF), + (0xF900, 0xFA6D), (0xFA70, 0xFAD9), + (0x16FE2, 0x16FE3), (0x16FF0, 0x16FF6), + (0x20000, 0x2A6DF), (0x2A700, 0x2B81D), (0x2B820, 0x2CEAD), + (0x2CEB0, 0x2EBE0), (0x2EBF0, 0x2EE5D), (0x2F800, 0x2FA1D), + (0x30000, 0x3134A), (0x31350, 0x33479), +) def character_tags(character): codepoint = ord(character) - if 0x3040 <= codepoint <= 0x30FF: + if 0x3040 <= codepoint <= 0x30FF or 0x31F0 <= codepoint <= 0x31FF: return ["Jpan"] - if 0x1100 <= codepoint <= 0x11FF or 0xAC00 <= codepoint <= 0xD7AF: + if (0x1100 <= codepoint <= 0x11FF or 0x3130 <= codepoint <= 0x318F + or 0xA960 <= codepoint <= 0xA97F or 0xAC00 <= codepoint <= 0xD7AF + or 0xD7B0 <= codepoint <= 0xD7FF): return ["Hang"] - if 0x2F00 <= codepoint <= 0x9FFF: + if 0x3100 <= codepoint <= 0x312F or 0x31A0 <= codepoint <= 0x31BF: + return ["Bopo"] + if any(start <= codepoint <= end for start, end in HAN_RANGES): return ["Hans", "Hant", "Jpan", "Hang"] # locale disambiguates Han for tag, start, end in ( ("Cyrl", 0x0400, 0x052F), ("Hebr", 0x0590, 0x05FF), @@ -508,8 +689,9 @@ if unresolved_fonts: ``/`` faces. python-pptx also does not evaluate the full placeholder -> layout -> master inheritance chain; the output above remains a candidate list. Check matching layout/master placeholders when the exact face matters. Han text needs the deck locale to distinguish Hans, -Hant, Japanese, and Korean mappings; add other script ranges when the task uses them rather than -claiming the Latin fallback is definitive. +Hant, Japanese, and Korean mappings. The ranges above follow Unicode 17 `Script=Han`, including +BMP compatibility ideographs and supplementary-plane Extension J; add other script ranges when +the task uses them rather than claiming the Latin fallback is definitive. Chart titles, legends, tick labels, data labels, and SmartArt nodes use additional DrawingML font cascades across chart/diagram parts. The guard above deliberately fails closed until those parts are audited directly or the deck is rendered and inspected with the production fonts. diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md index fc8b97e..3546957 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md @@ -37,12 +37,18 @@ with open("input.csv", newline="", encoding="utf-8-sig") as f: # utf-8-sig str import csv from pathlib import Path -FORMULA_PREFIXES = ("=", "+", "-", "@") +FORMULA_OPERATORS = ("=", "+", "-", "@", "＝", "＋", "－", "＠") + +def begins_spreadsheet_formula(value): + index = 0 + while index < len(value) and (ord(value[index]) <= 0x20 or value[index] == "\ufeff"): + index += 1 + return value.startswith(FORMULA_OPERATORS, index) def spreadsheet_csv_field(value, *, mode="safe"): if mode not in {"safe", "raw"}: raise ValueError("mode must be 'safe' or 'raw'") - if mode == "safe" and isinstance(value, str) and value.startswith(FORMULA_PREFIXES): + if mode == "safe" and isinstance(value, str) and begins_spreadsheet_formula(value): return "'" + value return value @@ -60,10 +66,12 @@ with output_path.open("w", newline="", encoding="utf-8") as f: ``` Use `mode="safe"` (the default above) when the CSV will be opened in Excel, LibreOffice, -Google Sheets, or another spreadsheet application. It neutralizes literal text beginning with -`=`, `+`, `-`, or `@` by prefixing an apostrophe, so the application does not interpret the -field as a formula. Numeric values, including negative numbers represented as numbers, are not -changed. This protection deliberately changes those serialized string values. +Google Sheets, or another spreadsheet application. It scans past leading C0 controls/spaces and +BOMs, then prefixes the **complete original field** with an apostrophe when the next character is +`=`, `+`, `-`, `@`, or the corresponding fullwidth operator. Importers can strip or ignore leading +tab/CR/LF/BOM characters before formula detection, and CSV quoting does not neutralize them. +Benign control-prefixed text is preserved. Numeric values, including negative numbers represented +as numbers, are not changed. This protection deliberately changes formula-like serialized strings. Use `mode="raw"` only when the user explicitly requires byte-for-value interchange with a trusted machine consumer. Raw mode preserves the exact strings and provides **no spreadsheet @@ -98,7 +106,7 @@ formula-injection protection**; do not present a raw export as safe to open in a from pathlib import Path from xml.etree import ElementTree as ET - FORMULA_PREFIXES = ("=", "+", "-", "@") + FORMULA_OPERATORS = ("=", "+", "-", "@", "＝", "＋", "－", "＠") MAIN_NS = "http://schemas.openxmlformats.org/spreadsheetml/2006/main" DOC_REL_NS = "http://schemas.openxmlformats.org/officeDocument/2006/relationships" PKG_REL_NS = "http://schemas.openxmlformats.org/package/2006/relationships" @@ -106,10 +114,16 @@ formula-injection protection**; do not present a raw export as safe to open in a FORMULA_TAG = f"{{{MAIN_NS}}}f" VALUE_TAG = f"{{{MAIN_NS}}}v" + def begins_spreadsheet_formula(value): + index = 0 + while index < len(value) and (ord(value[index]) <= 0x20 or value[index] == "\ufeff"): + index += 1 + return value.startswith(FORMULA_OPERATORS, index) + def spreadsheet_csv_field(value, *, mode="safe"): if mode not in {"safe", "raw"}: raise ValueError("mode must be 'safe' or 'raw'") - if mode == "safe" and isinstance(value, str) and value.startswith(FORMULA_PREFIXES): + if mode == "safe" and isinstance(value, str) and begins_spreadsheet_formula(value): return "'" + value return value diff --git a/plugins/Hylouis233/document-skills/tests/docx_fixture.py b/plugins/Hylouis233/document-skills/tests/docx_fixture.py index 9ae101d..cf3a30b 100644 --- a/plugins/Hylouis233/document-skills/tests/docx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/docx_fixture.py @@ -9,8 +9,9 @@ import os import subprocess import sys +import unicodedata import zipfile -from pathlib import Path +from pathlib import Path, PurePosixPath from tempfile import TemporaryDirectory import fitz @@ -57,6 +58,11 @@ def check(name, cond, extra=""): MAX_ENTRY = 100 * 1024 * 1024 MAX_TOTAL_UNCOMPRESSED = 500 * 1024 * 1024 MAX_COMPRESSION_RATIO = 200 +MAX_MEMBER_COMPONENT_BYTES = 255 +MAX_MEMBER_COMPONENT_UTF16_UNITS = 255 +MAX_MEMBER_PATH_BYTES = 1024 +MAX_MEMBER_PATH_UTF16_UNITS = 240 +MAX_MEMBER_COMPONENTS = 64 def require(condition, message): @@ -962,6 +968,76 @@ def tc_text(tc): plain_doc.save("repack-without-media.docx") +WINDOWS_DEVICE_NAMES = { + "con", "prn", "aux", "nul", "conin$", "conout$", + *(f"com{suffix}" for suffix in "123456789¹²³"), + *(f"lpt{suffix}" for suffix in "123456789¹²³"), +} + + +def extraction_key(name): + is_directory = name.endswith("/") + path = name[:-1] if is_directory else name + require(path and not name.startswith("/") and "\\" not in name, + f"non-canonical archive member path: {name}") + parts = path.split("/") + require(len(parts) <= MAX_MEMBER_COMPONENTS, + "archive member depth exceeds portable extraction limit") + require( + all( + part not in {"", ".", ".."} + and not any(character in ':<>|"?*' for character in part) + and not any(ord(character) < 32 for character in part) + and not part.endswith((".", " ")) + and unicodedata.normalize("NFC", part) == part + and part.partition(".")[0].rstrip(" ").casefold() not in WINDOWS_DEVICE_NAMES + for part in parts + ), + f"non-canonical archive member path: {name}", + ) + require( + all( + len(part.encode("utf-8")) <= MAX_MEMBER_COMPONENT_BYTES + and len(part.encode("utf-16-le")) // 2 <= MAX_MEMBER_COMPONENT_UTF16_UNITS + for part in parts + ), + "archive member component exceeds portable extraction limit", + ) + require(len(path.encode("utf-8")) <= MAX_MEMBER_PATH_BYTES + and len(path.encode("utf-16-le")) // 2 <= MAX_MEMBER_PATH_UTF16_UNITS, + "archive member path exceeds portable extraction limit") + canonical = PurePosixPath(*parts).as_posix() + require(canonical == path, f"non-canonical archive member path: {name}") + key = tuple(unicodedata.normalize("NFC", part.casefold()) for part in parts) + return key, tuple(parts), is_directory + + +def validate_extraction_paths(infos): + root = {"children": {}, "member": False, "file": False, "spelling": None} + for info in infos: + key, spellings, is_directory = extraction_key(info.filename) + node = root + for normalized, spelling in zip(key, spellings): + require(not node["file"], + "archive file and directory paths collide after extraction") + child = node["children"].get(normalized) + if child is None: + child = { + "children": {}, "member": False, "file": False, + "spelling": spelling, + } + node["children"][normalized] = child + else: + require(child["spelling"] == spelling, + "archive member path spelling collides after extraction") + node = child + require(not node["member"], "archive member paths collide after extraction") + require(is_directory or not node["children"], + "archive file and directory paths collide after extraction") + node["member"] = True + node["file"] = not is_directory + + def validate_docx_archive_bounds(archive): require(os.fstat(archive.fp.fileno()).st_size <= MAX_ARCHIVE_BYTES, "compressed DOCX file size above limit") @@ -969,6 +1045,7 @@ def validate_docx_archive_bounds(archive): require(len(infos) <= MAX_MEMBERS, "archive member count above limit") names = {info.filename for info in infos} require(len(names) == len(infos), "duplicate archive member names are unsafe") + validate_extraction_paths(infos) require("[Content_Types].xml" in names and "word/document.xml" in names, "required DOCX package parts are missing") require(sum(info.file_size for info in infos) <= MAX_TOTAL_UNCOMPRESSED, @@ -1023,6 +1100,147 @@ def repack_tree(source, output, extraction_root): check("Tier 2 pre-extract bounds remain active under optimized Python", __debug__ or edit_bomb_rejected_before_extract) +with zipfile.ZipFile("noncanonical-member.docx", "w", zipfile.ZIP_STORED) as archive: + archive.writestr("[Content_Types].xml", "") + archive.writestr("word/document.xml", "") + archive.writestr("../word/document.xml", "") +with TemporaryDirectory(prefix="docx-noncanonical-") as scratch: + noncanonical_root = Path(scratch) + try: + repack_tree("noncanonical-member.docx", "noncanonical-output.docx", + noncanonical_root) + noncanonical_rejected = False + except ValueError as error: + noncanonical_rejected = ( + str(error) == "non-canonical archive member path: ../word/document.xml" + and not any(noncanonical_root.iterdir()) + and not Path("noncanonical-output.docx").exists() + ) +check("Tier 2 rejects traversal aliases before they can overwrite a validated part", + noncanonical_rejected) +check("non-canonical member rejection remains active under optimized Python", + __debug__ or noncanonical_rejected) + +with zipfile.ZipFile("normalized-collision.docx", "w", zipfile.ZIP_STORED) as archive: + archive.writestr("[Content_Types].xml", "") + archive.writestr("word/document.xml", "") + archive.writestr("custom", b"file") + archive.writestr("custom/", b"") +with TemporaryDirectory(prefix="docx-collision-") as scratch: + collision_root = Path(scratch) + try: + repack_tree("normalized-collision.docx", "collision-output.docx", collision_root) + normalized_collision_rejected = False + except ValueError as error: + normalized_collision_rejected = ( + str(error) == "archive member paths collide after extraction" + and not any(collision_root.iterdir()) + and not Path("collision-output.docx").exists() + ) +check("Tier 2 rejects distinct names that normalize to one extraction path", + normalized_collision_rejected) + +with zipfile.ZipFile("prefix-collision.docx", "w", zipfile.ZIP_STORED) as archive: + archive.writestr("[Content_Types].xml", "") + archive.writestr("word/document.xml", "") + archive.writestr("custom", b"file") + archive.writestr("custom/child.bin", b"child") +with TemporaryDirectory(prefix="docx-prefix-collision-") as scratch: + prefix_collision_root = Path(scratch) + try: + repack_tree("prefix-collision.docx", "prefix-collision-output.docx", + prefix_collision_root) + prefix_collision_rejected = False + except ValueError as error: + prefix_collision_rejected = ( + str(error) == "archive file and directory paths collide after extraction" + and not any(prefix_collision_root.iterdir()) + and not Path("prefix-collision-output.docx").exists() + ) +check("Tier 2 rejects a file path that is also an extracted directory prefix", + prefix_collision_rejected) + +with zipfile.ZipFile("case-collision.docx", "w", zipfile.ZIP_STORED) as archive: + archive.writestr("[Content_Types].xml", "") + archive.writestr("word/document.xml", "") + archive.writestr("WORD/document.xml", "") +with TemporaryDirectory(prefix="docx-case-collision-") as scratch: + case_collision_root = Path(scratch) + try: + repack_tree("case-collision.docx", "case-collision-output.docx", + case_collision_root) + case_collision_rejected = False + except ValueError as error: + case_collision_rejected = ( + str(error) == "archive member path spelling collides after extraction" + and not any(case_collision_root.iterdir()) + and not Path("case-collision-output.docx").exists() + ) +check("Tier 2 rejects case aliases before Windows extraction can overwrite a part", + case_collision_rejected) + +with zipfile.ZipFile("prefix-spelling-collision.docx", "w", zipfile.ZIP_STORED) as archive: + archive.writestr("[Content_Types].xml", "") + archive.writestr("word/document.xml", "") + archive.writestr("WORD/styles.xml", "") +with TemporaryDirectory(prefix="docx-prefix-spelling-") as scratch: + prefix_spelling_root = Path(scratch) + try: + repack_tree("prefix-spelling-collision.docx", "prefix-spelling-output.docx", + prefix_spelling_root) + prefix_spelling_rejected = False + except ValueError as error: + prefix_spelling_rejected = ( + str(error) == "archive member path spelling collides after extraction" + and not any(prefix_spelling_root.iterdir()) + and not Path("prefix-spelling-output.docx").exists() + ) +check("Tier 2 rejects case aliases in a shared directory prefix", + prefix_spelling_rejected) + +with zipfile.ZipFile("overlong-member.docx", "w", zipfile.ZIP_STORED) as archive: + archive.writestr("[Content_Types].xml", "") + archive.writestr("word/document.xml", "") + archive.writestr("word/" + "a" * 256, b"oversized component") +with TemporaryDirectory(prefix="docx-overlong-member-") as scratch: + overlong_root = Path(scratch) + try: + repack_tree("overlong-member.docx", "overlong-output.docx", overlong_root) + overlong_member_rejected = False + except ValueError as error: + overlong_member_rejected = ( + str(error) == "archive member component exceeds portable extraction limit" + and not any(overlong_root.iterdir()) + and not Path("overlong-output.docx").exists() + ) +check("Tier 2 rejects an unportable component before partial Windows extraction", + overlong_member_rejected) + +portable_path_limits_rejected = [] +for unsafe_name in ( + "/".join(["a"] * (MAX_MEMBER_COMPONENTS + 1)), + "/".join(["a" * 80] * 4), +): + try: + extraction_key(unsafe_name) + except ValueError: + portable_path_limits_rejected.append(unsafe_name) +check("portable extraction bounds total path length and component depth", + len(portable_path_limits_rejected) == 2) + +portable_name_rejections = [] +for unsafe_name in ( + "/word/document.xml", "word\\document.xml", "word/con.xml", "word/COM¹.xml", + "word/LPT².txt", "word/CONIN$.xml", "word/CONOUT$.xml", "word/NUL .xml", + "word/trailing. ", "word/cafe\u0301.xml", "word/control\x01.xml", +): + try: + extraction_key(unsafe_name) + except ValueError: + portable_name_rejections.append(unsafe_name) +check("portable extraction rejects absolute, alternate, device, non-NFC, and control names", + len(portable_name_rejections) == 11, portable_name_rejections) + with zipfile.ZipFile("malformed-for-repair.docx", "w", zipfile.ZIP_STORED) as archive: archive.writestr("[Content_Types].xml", "") archive.writestr("word/document.xml", " MAX_CHART_POINTS: + return False + if point_budget is None: + return True + if count > point_budget["remaining"]: + return False + point_budget["remaining"] -= count + return True + + +def cached_numeric_points( + source, *, fill_missing=False, include_count=False, point_budget=None +): if source is None: return None - point_counts = source.xpath("./c:numRef/c:numCache/c:ptCount | ./c:numLit/c:ptCount") - if not point_counts: + caches = source.xpath("./c:numRef/c:numCache | ./c:numLit") + if len(caches) != 1 or (point_count := cache_point_count(caches[0])) is None: return None - points = source.xpath("./c:numRef/c:numCache/c:pt | ./c:numLit/c:pt") - return [ - (int(point.get("idx")), float(value.text)) - for point in points - if point.get("idx") is not None and (value := point.find(qn("c:v"))) is not None + cached = {} + for point in caches[0].findall(qn("c:pt")): + value = point.find(qn("c:v")) + try: + index = int(point.get("idx")) + except (TypeError, ValueError): + return None + if value is None or not 0 <= index < point_count or index in cached: + return None + try: + parsed = float(value.text) if value.text not in (None, "") else value.text + except (TypeError, ValueError): + parsed = value.text + cached[index] = parsed + if fill_missing: + if not consume_point_budget(point_budget, point_count): + return None + points = [(index, cached.get(index)) for index in range(point_count)] + else: + points = sorted(cached.items()) + return {"point_count": point_count, "points": points} if include_count else points + + +def cached_text_point_map(container, point_count): + values = {} + for point in container.findall(qn("c:pt")): + try: + index = int(point.get("idx")) + except (TypeError, ValueError): + return None + value = point.find(qn("c:v")) + if value is None or not 0 <= index < point_count or index in values: + return None + values[index] = value.text or "" + return values + + +def category_source(series): + category = series._element.find(qn("c:cat")) + if category is None: + return None, None + sources = [ + child for child in category + if isinstance(child.tag, str) + and etree.QName(child).localname in CATEGORY_SOURCE_NAMES ] + return (category, sources[0]) if len(sources) == 1 else (category, None) -def cached_category_labels(plot): - category_nodes = plot._element.xpath("./c:ser[1]/c:cat") - if not category_nodes: +def cached_category_labels(series, point_budget=None): + category, source = category_source(series) + if category is None: return [] - cache_payload = category_nodes[0].xpath( - "./c:strRef/c:strCache/c:ptCount | ./c:numRef/c:numCache/c:ptCount | " - "./c:multiLvlStrRef/c:multiLvlStrCache/c:ptCount | " - "./c:strLit/c:ptCount | ./c:numLit/c:ptCount" - ) - if not cache_payload: + if source is None: return None - return [[str(level) for level in label] for label in plot.categories.flattened_labels] + source_name = etree.QName(source).localname + + if source_name == "multiLvlStrRef": + caches = source.findall(qn("c:multiLvlStrCache")) + if len(caches) != 1 or (count := cache_point_count(caches[0])) is None: + return None + level_nodes = caches[0].findall(qn("c:lvl")) + if len(level_nodes) > MAX_CATEGORY_LEVELS or (count and not level_nodes): + return None + levels = [cached_text_point_map(level, count) for level in level_nodes] + if any(points is None for points in levels): + return None + if not consume_point_budget(point_budget, count * max(len(level_nodes), 1)): + return None + expanded_levels = [] + for level_index, points in enumerate(levels): + if level_index == 0: + expanded_levels.append([points.get(index, "") for index in range(count)]) + continue + ordered = sorted(points.items()) + if not ordered: + expanded_levels.append([""] * count) + continue + cursor = 0 + expanded = [] + for leaf_index in range(count): + while cursor + 1 < len(ordered) and ordered[cursor + 1][0] <= leaf_index: + cursor += 1 + expanded.append(ordered[cursor][1]) + expanded_levels.append(expanded) + return [ + [level[index] for level in reversed(expanded_levels)] + for index in range(count) + ] + + cache_name = {"strRef": "c:strCache", "numRef": "c:numCache"}.get(source_name) + if cache_name is None: + cache = source + else: + caches = source.findall(qn(cache_name)) + if len(caches) != 1: + return None + cache = caches[0] + if (count := cache_point_count(cache)) is None: + return None + values = cached_text_point_map(cache, count) + if values is None: + return None + if not consume_point_budget(point_budget, count): + return None + return [[values.get(index, "")] for index in range(count)] + + +def category_content(series, point_budget=None): + labels = cached_category_labels(series, point_budget) + _, source = category_source(series) + formula = None if source is None else source.find(qn("c:f")) + return { + "categories": labels, + "category_formula": formula.text if formula is not None else None, + "category_cache_status": "unavailable" if labels is None else "available", + } def series_name_content(series): @@ -442,37 +568,85 @@ def series_name_content(series): return {"name": value.text or ""} -def series_content(series): +def series_content(series, *, include_categories=False, point_budget=None): + if point_budget is None: + point_budget = {"remaining": MAX_CHART_POINTS} name_content = series_name_content(series) + content = { + **name_content, + **(category_content(series, point_budget) if include_categories else {}), + } x_source = getattr(series._element, "xVal", None) if x_source is None: value_source = getattr(series._element, "val", None) - if cached_numeric_points(value_source) is None: - return {**name_content, "values": None, "cache_status": "unavailable"} - return {**name_content, "values": list(series.values)} - x_points = cached_numeric_points(x_source) - y_points = cached_numeric_points(getattr(series._element, "yVal", None)) - if x_points is None or y_points is None: - return {**name_content, "points": None, "cache_status": "unavailable"} - content = {**name_content, "x_points": x_points, "y_points": y_points} + value_cache = cached_numeric_points( + value_source, fill_missing=True, include_count=True, + point_budget=point_budget, + ) + if value_cache is None: + return {**content, "values": None, "cache_status": "unavailable"} + return {**content, "values": [value for _, value in value_cache["points"]]} + x_cache = cached_numeric_points(x_source, include_count=True) + y_cache = cached_numeric_points( + getattr(series._element, "yVal", None), include_count=True + ) + if x_cache is None or y_cache is None: + return {**content, "points": None, "cache_status": "unavailable"} + content.update({ + "x_points": x_cache["points"], "x_point_count": x_cache["point_count"], + "y_points": y_cache["points"], "y_point_count": y_cache["point_count"], + }) size_source = getattr(series._element, "bubbleSize", None) if size_source is not None: - bubble_points = cached_numeric_points(size_source) - if bubble_points is None: - return {**name_content, "points": None, "cache_status": "unavailable"} - content["bubble_points"] = bubble_points + bubble_cache = cached_numeric_points(size_source, include_count=True) + if bubble_cache is None: + return {**content, "points": None, "cache_status": "unavailable"} + content.update({ + "bubble_points": bubble_cache["points"], + "bubble_point_count": bubble_cache["point_count"], + }) return content -def chart_axis_titles(chart): - titles = {} - for label, attribute in (("category", "category_axis"), ("value", "value_axis")): - try: - axis = getattr(chart, attribute) - except (AttributeError, ValueError): - continue - titles[label] = axis.axis_title.text_frame.text if axis.has_title else "" - return titles +def chart_axis_text(axis): + title = axis.find(qn("c:title")) + if title is None: + return "" + rich = title.xpath("./c:tx/c:rich") + references = title.xpath("./c:tx/c:strRef") + if len(rich) + len(references) != 1: + return None + if rich: + return "\n".join(paragraph.text for paragraph in rich[0].findall(qn("a:p"))) + caches = references[0].findall(qn("c:strCache")) + if (len(caches) != 1 or (count := cache_point_count(caches[0])) is None + or count != 1): + return None + values = cached_text_point_map(caches[0], count) + return None if values is None else "\n".join(values.get(index, "") for index in range(count)) + + +def chart_axes(chart): + axes = chart._element.xpath( + "./c:chart/c:plotArea/c:catAx | ./c:chart/c:plotArea/c:dateAx | " + "./c:chart/c:plotArea/c:valAx | ./c:chart/c:plotArea/c:serAx" + ) + return [ + { + "kind": etree.QName(axis).localname, + "id": axis.find(qn("c:axId")).get("val") if axis.find(qn("c:axId")) is not None else None, + "position": ( + axis.find(qn("c:axPos")).get("val") + if axis.find(qn("c:axPos")) is not None else None + ), + "cross_axis_id": ( + axis.find(qn("c:crossAx")).get("val") + if axis.find(qn("c:crossAx")) is not None else None + ), + "title": chart_axis_text(axis), + } + for axis in axes + ] DIAGRAM_NS = "http://schemas.openxmlformats.org/drawingml/2006/diagram" @@ -499,8 +673,10 @@ def smartart_content(shape): return {"name": shape.name, "status": "ok", "text": labels} -def extract_slide_content(slide): +def extract_slide_content(slide, point_budget=None): shapes = list(iter_shapes(slide.shapes)) + if point_budget is None: + point_budget = {"remaining": MAX_CHART_POINTS} text = [sh.text_frame.text for sh in shapes if sh.has_text_frame and sh.text_frame.text] tables = [ table_cells(sh.table) @@ -519,20 +695,19 @@ def extract_slide_content(slide): for plot in chart.plots: items = list(plot.series) has_xy_values = any(getattr(item._element, "xVal", None) is not None for item in items) - categories = [] if has_xy_values else cached_category_labels(plot) - series = [series_content(item) for item in items] + series = [ + series_content( + item, include_categories=not has_xy_values, point_budget=point_budget + ) + for item in items + ] plots.append({ "kind": type(plot).__name__, - "categories": categories, - "category_cache_status": ( - "not-applicable" if has_xy_values else - "unavailable" if categories is None else "available" - ), "series": series, }) charts.append({ "title": chart_title, - "axis_titles": chart_axis_titles(chart), + "axes": chart_axes(chart), "plots": plots, }) notes = slide.notes_slide.notes_text_frame.text if slide.has_notes_slide else "" @@ -555,25 +730,132 @@ def extract_slide_content(slide): check( "content inventory emits chart title, categories, series, and values", content["charts"][0]["title"] == "Units by region" - and content["charts"][0]["plots"][0]["categories"] == [["EU"], ["US"]] - and content["charts"][0]["plots"][0]["series"] - == [{"name": "Units", "values": [120.0, 80.0]}], + and content["charts"][0]["plots"][0]["series"][0]["categories"] + == [["EU"], ["US"]] + and content["charts"][0]["plots"][0]["series"][0]["category_cache_status"] + == "available" + and content["charts"][0]["plots"][0]["series"][0]["values"] == [120.0, 80.0], content["charts"], ) -check("content inventory emits category and value axis titles", - content["charts"][0]["axis_titles"] - == {"category": "Region", "value": "Units sold"}, content["charts"][0]) +check("content inventory emits raw category and value axis metadata", + {axis["title"] for axis in content["charts"][0]["axes"]} + == {"Region", "Units sold"} + and all(axis["id"] and axis["position"] and axis["cross_axis_id"] + for axis in content["charts"][0]["axes"]), content["charts"][0]) + +secondary_axis_prs = Presentation("input.pptx") +secondary_axis_chart = next( + shape.chart for shape in secondary_axis_prs.slides[0].shapes + if shape.has_chart and shape.chart.has_title + and shape.chart.chart_title.text_frame.text == "Units by region" +) +plot_area = secondary_axis_chart._element.xpath("./c:chart/c:plotArea")[0] +primary_category_axis = plot_area.find(qn("c:catAx")) +primary_value_axis = plot_area.find(qn("c:valAx")) +primary_category_id = primary_category_axis.find(qn("c:axId")).get("val") +primary_value_id = primary_value_axis.find(qn("c:axId")).get("val") +secondary_category_id, secondary_value_id = "91000001", "91000002" + +secondary_plot = copy.deepcopy(plot_area.xpath("./c:barChart")[0]) +for axis_id in secondary_plot.findall(qn("c:axId")): + axis_id.set("val", ( + secondary_category_id if axis_id.get("val") == primary_category_id + else secondary_value_id + )) +first_axis_index = min(plot_area.index(primary_category_axis), plot_area.index(primary_value_axis)) +plot_area.insert(first_axis_index, secondary_plot) + +secondary_category_axis = copy.deepcopy(primary_category_axis) +secondary_category_axis.find(qn("c:axId")).set("val", secondary_category_id) +secondary_category_axis.find(qn("c:axPos")).set("val", "t") +secondary_category_axis.find(qn("c:crossAx")).set("val", secondary_value_id) +secondary_category_text = secondary_category_axis.xpath("./c:title//a:t")[0] +secondary_category_text.text = "Secondary" +secondary_category_run = secondary_category_text.getparent() +secondary_category_break = OxmlElement("a:br") +secondary_category_tail = OxmlElement("a:r") +secondary_category_tail_text = OxmlElement("a:t") +secondary_category_tail_text.text = "region" +secondary_category_tail.append(secondary_category_tail_text) +secondary_category_run.addnext(secondary_category_break) +secondary_category_break.addnext(secondary_category_tail) + +secondary_value_axis = copy.deepcopy(primary_value_axis) +secondary_value_axis.find(qn("c:axId")).set("val", secondary_value_id) +secondary_value_axis.find(qn("c:axPos")).set("val", "r") +secondary_value_axis.find(qn("c:crossAx")).set("val", secondary_category_id) +secondary_title_tx = secondary_value_axis.find(qn("c:title")).find(qn("c:tx")) +for child in list(secondary_title_tx): + secondary_title_tx.remove(child) +secondary_title_ref = OxmlElement("c:strRef") +secondary_title_formula = OxmlElement("c:f") +secondary_title_formula.text = "Sheet1!$F$1" +secondary_title_cache = OxmlElement("c:strCache") +secondary_title_count = OxmlElement("c:ptCount") +secondary_title_count.set("val", "1") +secondary_title_point = OxmlElement("c:pt") +secondary_title_point.set("idx", "0") +secondary_title_value = OxmlElement("c:v") +secondary_title_value.text = "Percent" +secondary_title_point.append(secondary_title_value) +secondary_title_cache.extend([secondary_title_count, secondary_title_point]) +secondary_title_ref.extend([secondary_title_formula, secondary_title_cache]) +secondary_title_tx.append(secondary_title_ref) +empty_secondary_axis = copy.deepcopy(secondary_value_axis) +empty_secondary_axis.xpath("./c:title/c:tx/c:strRef/c:strCache/c:pt/c:v")[0].text = None +check("raw worksheet-backed axis title preserves an explicit empty cache value", + chart_axis_text(empty_secondary_axis) == "", chart_axis_text(empty_secondary_axis)) +multi_point_axis = copy.deepcopy(secondary_value_axis) +multi_point_axis.xpath("./c:title/c:tx/c:strRef/c:strCache/c:ptCount")[0].set("val", "2") +check("worksheet-backed axis title rejects a non-scalar cache before expansion", + chart_axis_text(multi_point_axis) is None) + +plot_area.extend([secondary_category_axis, secondary_value_axis]) +secondary_axis_prs.save("secondary-axes.pptx") +secondary_axis_reopened = Presentation("secondary-axes.pptx") +secondary_axis_content = extract_slide_content( + secondary_axis_reopened.slides[0] +)["charts"][0] +secondary_axes = {axis["title"]: axis for axis in secondary_axis_content["axes"]} +secondary_axis_reopened_chart = next( + shape.chart for shape in secondary_axis_reopened.slides[0].shapes + if shape.has_chart and shape.chart.has_title + and shape.chart.chart_title.text_frame.text == "Units by region" +) +secondary_title_refs = secondary_axis_reopened_chart._element.xpath( + f'./c:chart/c:plotArea/c:valAx[c:axId[@val="{secondary_value_id}"]]' + '/c:title/c:tx/c:strRef' +) +check("combination-chart inventory includes all primary and secondary axes", + len(secondary_axis_content["plots"]) == 2 + and set(secondary_axes) == {"Region", "Units sold", "Secondary\vregion", "Percent"} + and secondary_axes["Secondary\vregion"] == { + "kind": "catAx", "id": secondary_category_id, "position": "t", + "cross_axis_id": secondary_value_id, "title": "Secondary\vregion", + } + and secondary_axes["Percent"] == { + "kind": "valAx", "id": secondary_value_id, "position": "r", + "cross_axis_id": secondary_category_id, "title": "Percent", + }, secondary_axis_content) +check("raw axis-title inventory preserves a worksheet-backed strRef", + len(secondary_title_refs) == 1 + and secondary_title_refs[0].find(qn("c:f")).text == "Sheet1!$F$1", + secondary_title_refs) chart_by_title = {item["title"]: item for item in content["charts"]} check("content inventory emits XY x/y points", chart_by_title["XY trend"]["plots"][0]["series"][0]["x_points"] == [(0, 1.0), (1, 3.0)] + and chart_by_title["XY trend"]["plots"][0]["series"][0]["x_point_count"] == 2 and chart_by_title["XY trend"]["plots"][0]["series"][0]["y_points"] - == [(0, 2.0), (1, 4.0)]) + == [(0, 2.0), (1, 4.0)] + and chart_by_title["XY trend"]["plots"][0]["series"][0]["y_point_count"] == 2) check("content inventory emits bubble x/y/size points", chart_by_title["Bubble risk"]["plots"][0]["series"][0]["x_points"] == [(0, 3.0)] and chart_by_title["Bubble risk"]["plots"][0]["series"][0]["y_points"] == [(0, 4.0)] and chart_by_title["Bubble risk"]["plots"][0]["series"][0]["bubble_points"] - == [(0, 5.0)]) + == [(0, 5.0)] + and chart_by_title["Bubble risk"]["plots"][0]["series"][0]["bubble_point_count"] + == 1) check("content inventory emits notes text", "regional split" in content["notes"], content["notes"]) # Category/value series can retain an external workbook formula without a numCache. @@ -589,6 +871,20 @@ def extract_slide_content(slide): ) category_prs.save("category-cache-source.pptx") +shared_deck_budget = {"remaining": 10} +budget_source = Presentation("category-cache-source.pptx").slides[0] +first_budgeted_slide = extract_slide_content(budget_source, shared_deck_budget) +second_budgeted_slide = extract_slide_content(budget_source, shared_deck_budget) +first_budgeted_items = first_budgeted_slide["charts"][0]["plots"][0]["series"] +second_budgeted_items = second_budgeted_slide["charts"][0]["plots"][0]["series"] +check("one shared logical-point budget bounds aggregate inventory across slides", + all(item["values"] is not None for item in first_budgeted_items) + and second_budgeted_items[0]["categories"] == [["A"], ["B"]] + and second_budgeted_items[0]["values"] is None + and second_budgeted_items[1]["categories"] is None + and shared_deck_budget["remaining"] == 0, + (first_budgeted_items, second_budgeted_items, shared_deck_budget)) + series_name_mutated = Presentation("category-cache-source.pptx") series_name_chart = next( shape.chart for shape in series_name_mutated.slides[0].shapes if shape.has_chart @@ -615,9 +911,10 @@ def extract_slide_content(slide): series_name_inventory = extract_slide_content(series_name_reopened.slides[0]) series_name_inventory_items = series_name_inventory["charts"][0]["plots"][0]["series"] check("deck inventory continues through cached siblings after an unavailable series title", - series_name_inventory_items[0] == series_name_result - and series_name_inventory_items[1] - == {"name": "Cached series", "values": [3.0, 4.0]}, + series_name_inventory_items[0]["name"] is None + and series_name_inventory_items[0]["values"] == [1.0, 2.0] + and series_name_inventory_items[1]["name"] == "Cached series" + and series_name_inventory_items[1]["values"] == [3.0, 4.0], series_name_inventory_items) @@ -704,16 +1001,149 @@ def name(self): category_inventory_series = category_inventory["charts"][0]["plots"][0]["series"] check( "cacheless category series is unavailable without aborting inventory", - category_inventory_series[0] - == {"name": "Missing cache", "values": None, "cache_status": "unavailable"}, + category_inventory_series[0]["name"] == "Missing cache" + and category_inventory_series[0]["values"] is None + and category_inventory_series[0]["cache_status"] == "unavailable", category_inventory_series, ) check( "cached category series still reports its values after a cacheless sibling", - category_inventory_series[1] == {"name": "Cached series", "values": [3.0, 4.0]}, + category_inventory_series[1]["name"] == "Cached series" + and category_inventory_series[1]["values"] == [3.0, 4.0], category_inventory_series, ) +marker_element = copy.deepcopy(category_series[1]._element) +marker_values = marker_element.xpath("./c:val/c:numRef/c:numCache/c:pt") +marker_values[0].find(qn("c:v")).text = "#N/A" +marker_values[1].find(qn("c:v")).text = None +marker_series = type("CachedMarkerSeries", (), {"_element": marker_element})() +marker_result = series_content(marker_series, include_categories=True) +check("category series preserves #N/A and blank cache markers without series.values", + marker_result["values"] == ["#N/A", None], marker_result) +omitted_marker_element = copy.deepcopy(marker_element) +omitted_marker_cache = omitted_marker_element.xpath("./c:val/c:numRef/c:numCache")[0] +omitted_marker_cache.remove(omitted_marker_cache.findall(qn("c:pt"))[1]) +omitted_marker_series = type( + "SparseCategorySeries", (), {"_element": omitted_marker_element} +)() +omitted_marker_result = series_content(omitted_marker_series, include_categories=True) +check("category series densifies an omitted sparse point as blank", + omitted_marker_result["values"] == ["#N/A", None], omitted_marker_result) + +divergent_prs = Presentation("category-cache-source.pptx") +divergent_chart = next( + shape.chart for shape in divergent_prs.slides[0].shapes if shape.has_chart +) +divergent_series = list(divergent_chart.plots[0].series) +second_category = divergent_series[1]._element.find(qn("c:cat")) +second_category.find(qn("c:strRef")).find(qn("c:f")).text = "Sheet1!$D$2:$D$3" +second_points = second_category.xpath("./c:strRef/c:strCache/c:pt") +second_points[0].find(qn("c:v")).text = "North" +second_points[1].find(qn("c:v")).text = "South" +divergent_inventory = extract_slide_content(divergent_prs.slides[0]) +divergent_items = divergent_inventory["charts"][0]["plots"][0]["series"] +check("each category series retains its own formula and cached labels", + divergent_items[0]["categories"] == [["A"], ["B"]] + and divergent_items[1]["categories"] == [["North"], ["South"]] + and divergent_items[0]["category_formula"] != divergent_items[1]["category_formula"], + divergent_items) +commented_category_element = copy.deepcopy(divergent_series[0]._element) +commented_category_element.find(qn("c:cat")).insert(0, etree.Comment("source follows")) +commented_category_series = type( + "CommentedCategorySeries", (), {"_element": commented_category_element} +)() +check("category source selection ignores preserved XML comments", + category_content(commented_category_series)["categories"] == [["A"], ["B"]]) +malformed_category_element = copy.deepcopy(divergent_series[1]._element) +malformed_category_cache = malformed_category_element.xpath( + "./c:cat/c:strRef/c:strCache" +)[0] +malformed_category_cache.append(copy.deepcopy( + malformed_category_cache.findall(qn("c:pt"))[0] +)) +malformed_category_series = type( + "MalformedCategorySeries", (), {"_element": malformed_category_element} +)() +malformed_category_result = category_content(malformed_category_series) +check("duplicate category cache indexes fail closed for only that series", + malformed_category_result["categories"] is None + and malformed_category_result["category_cache_status"] == "unavailable", + malformed_category_result) + +duplicate_count_element = copy.deepcopy(divergent_series[1]._element) +duplicate_count_cache = duplicate_count_element.xpath("./c:cat/c:strRef/c:strCache")[0] +duplicate_count_cache.insert(1, copy.deepcopy(duplicate_count_cache.find(qn("c:ptCount")))) +duplicate_count_series = type( + "DuplicateCategoryCountSeries", (), {"_element": duplicate_count_element} +)() +duplicate_count_budget = {"remaining": 2} +duplicate_count_result = category_content(duplicate_count_series, duplicate_count_budget) +valid_after_malformed = category_content(divergent_series[0], duplicate_count_budget) +check("duplicate category ptCount fails closed without starving a valid sibling", + duplicate_count_result["categories"] is None + and valid_after_malformed["categories"] == [["A"], ["B"]] + and duplicate_count_budget["remaining"] == 0, + (duplicate_count_result, valid_after_malformed, duplicate_count_budget)) + +choice_conflict_element = copy.deepcopy(divergent_series[1]._element) +choice_conflict_category = choice_conflict_element.find(qn("c:cat")) +choice_conflict_category.append(parse_xml( + '' + 'Sheet1!$E$2:$E$3' + 'X' + 'Y' + '' +)) +choice_conflict_series = type( + "ChoiceConflictSeries", (), {"_element": choice_conflict_element} +)() +choice_conflict_result = category_content(choice_conflict_series) +check("conflicting category source choices fail closed without mixing formula and labels", + choice_conflict_result["categories"] is None + and choice_conflict_result["category_formula"] is None + and choice_conflict_result["category_cache_status"] == "unavailable", + choice_conflict_result) + +huge_category_element = copy.deepcopy(divergent_series[1]._element) +huge_category_element.xpath("./c:cat/c:strRef/c:strCache/c:ptCount")[0].set( + "val", "4294967295" +) +huge_category_series = type( + "HugeCategoryCountSeries", (), {"_element": huge_category_element} +)() +check("oversized logical category counts fail before allocation", + category_content(huge_category_series)["categories"] is None) + +aggregate_budget = {"remaining": 3} +first_budgeted_categories = category_content(divergent_series[0], aggregate_budget) +second_budgeted_categories = category_content(divergent_series[1], aggregate_budget) +check("shared category budget bounds aggregate dense inventory", + first_budgeted_categories["category_cache_status"] == "available" + and second_budgeted_categories["category_cache_status"] == "unavailable" + and aggregate_budget["remaining"] == 1, + (first_budgeted_categories, second_budgeted_categories, aggregate_budget)) + +multi_level_element = copy.deepcopy(divergent_series[0]._element) +multi_level_category = multi_level_element.find(qn("c:cat")) +multi_level_category.remove(next(iter(multi_level_category))) +multi_level_category.append(parse_xml( + '' + 'Sheet1!$A$2:$A$5' + 'AB' + 'CD' + 'G1' + 'G2' + '' +)) +multi_level_series = type("MultiLevelSeries", (), {"_element": multi_level_element})() +multi_level_result = category_content(multi_level_series) +check("multi-level categories flatten parent-to-child with linear carry-forward", + multi_level_result["categories"] + == [["G1", "A"], ["G1", "B"], ["G2", "C"], ["G2", "D"]] + and multi_level_result["category_formula"] == "Sheet1!$A$2:$A$5", + multi_level_result) + category_labels_mutated = Presentation("category-cache-source.pptx") category_labels_chart = next( shape.chart for shape in category_labels_mutated.slides[0].shapes if shape.has_chart @@ -742,8 +1172,9 @@ def name(self): and category_ref.find(qn("c:strCache")) is None and category_workbook_remains) check("missing category cache is reported without aborting later chart inventory", - category_plot_inventory["categories"] is None - and category_plot_inventory["category_cache_status"] == "unavailable" + category_plot_inventory["series"][0]["categories"] is None + and category_plot_inventory["series"][0]["category_cache_status"] == "unavailable" + and category_plot_inventory["series"][1]["categories"] == [["A"], ["B"]] and category_plot_inventory["series"][1]["values"] == [3.0, 4.0], category_plot_inventory) @@ -900,6 +1331,39 @@ def name(self): missing_count_cache = missing_count.find(qn("c:numRef") + "/" + qn("c:numCache")) missing_count_cache.remove(missing_count_cache.find(qn("c:ptCount"))) check("numeric cache without ptCount reports unavailable", cached_numeric_points(missing_count) is None) +marker_source = copy.deepcopy(x_source) +marker_points = marker_source.xpath("./c:numRef/c:numCache/c:pt") +marker_points[0].find(qn("c:v")).text = "#N/A" +marker_points[1].find(qn("c:v")).text = None +check("numeric cache preserves #N/A and explicit blank markers", + cached_numeric_points(marker_source) == [(0, "#N/A"), (1, None)], + cached_numeric_points(marker_source)) +sparse_source = copy.deepcopy(x_source) +sparse_cache = sparse_source.xpath("./c:numRef/c:numCache")[0] +sparse_cache.find(qn("c:ptCount")).set("val", "3") +sparse_cache.remove(sparse_cache.findall(qn("c:pt"))[1]) +check("XY numeric cache retains only present sparse indexed points", + cached_numeric_points(sparse_source) == [(0, 1.0)], + cached_numeric_points(sparse_source)) +check("sparse XY cache reports ptCount without densifying trailing blanks", + cached_numeric_points(sparse_source, include_count=True) + == {"point_count": 3, "points": [(0, 1.0)]}, + cached_numeric_points(sparse_source, include_count=True)) +check("category numeric cache can densify sparse positions as blanks", + cached_numeric_points(sparse_source, fill_missing=True) + == [(0, 1.0), (1, None), (2, None)], + cached_numeric_points(sparse_source, fill_missing=True)) +duplicate_source = copy.deepcopy(x_source) +duplicate_cache = duplicate_source.xpath("./c:numRef/c:numCache")[0] +duplicate_cache.append(copy.deepcopy(duplicate_cache.findall(qn("c:pt"))[0])) +check("duplicate numeric cache indexes fail closed", + cached_numeric_points(duplicate_source) is None) +huge_count_source = copy.deepcopy(x_source) +huge_count_source.xpath("./c:numRef/c:numCache/c:ptCount")[0].set( + "val", "4294967295" +) +check("oversized logical numeric counts fail before allocation", + cached_numeric_points(huge_count_source, fill_missing=True) is None) actual_cache = x_source.xpath("./c:numRef/c:numCache")[0] x_source.xpath("./c:numRef")[0].remove(actual_cache) check("XY series with an unavailable cache is explicit", @@ -1107,17 +1571,31 @@ def theme_faces_for_slide(slide): return cache_key, theme_cache[cache_key] -EAST_ASIAN_SCRIPTS = {"Hans", "Hant", "Jpan", "Hang"} +EAST_ASIAN_SCRIPTS = {"Hans", "Hant", "Jpan", "Hang", "Bopo"} COMPLEX_SCRIPTS = {"Arab", "Hebr", "Deva", "Beng", "Taml", "Thai"} +HAN_RANGES = ( + (0x2E80, 0x2E99), (0x2E9B, 0x2EF3), (0x2F00, 0x2FD5), + (0x3005, 0x3005), (0x3007, 0x3007), (0x3021, 0x3029), (0x3038, 0x303B), + (0x3400, 0x4DBF), (0x4E00, 0x9FFF), + (0xF900, 0xFA6D), (0xFA70, 0xFAD9), + (0x16FE2, 0x16FE3), (0x16FF0, 0x16FF6), + (0x20000, 0x2A6DF), (0x2A700, 0x2B81D), (0x2B820, 0x2CEAD), + (0x2CEB0, 0x2EBE0), (0x2EBF0, 0x2EE5D), (0x2F800, 0x2FA1D), + (0x30000, 0x3134A), (0x31350, 0x33479), +) def character_tags(character): codepoint = ord(character) - if 0x3040 <= codepoint <= 0x30FF: + if 0x3040 <= codepoint <= 0x30FF or 0x31F0 <= codepoint <= 0x31FF: return ["Jpan"] - if 0x1100 <= codepoint <= 0x11FF or 0xAC00 <= codepoint <= 0xD7AF: + if (0x1100 <= codepoint <= 0x11FF or 0x3130 <= codepoint <= 0x318F + or 0xA960 <= codepoint <= 0xA97F or 0xAC00 <= codepoint <= 0xD7AF + or 0xD7B0 <= codepoint <= 0xD7FF): return ["Hang"] - if 0x2F00 <= codepoint <= 0x9FFF: + if 0x3100 <= codepoint <= 0x312F or 0x31A0 <= codepoint <= 0x31BF: + return ["Bopo"] + if any(start <= codepoint <= end for start, end in HAN_RANGES): return ["Hans", "Hant", "Jpan", "Hang"] for tag, start, end in ( ("Cyrl", 0x0400, 0x052F), ("Hebr", 0x0590, 0x05FF), @@ -1149,6 +1627,42 @@ def required_slots(text): return slots or ["latin"] +han_examples = [ + chr(codepoint) + for start, end in HAN_RANGES + for codepoint in {start, end} +] +han_gap_examples = [ + chr(codepoint) for codepoint in ( + 0x2E9A, 0x2EF4, 0x2FD6, 0x3006, 0x3105, 0x3131, 0x4DC0, + 0xFA6E, 0xFA6F, 0xFADA, 0x16FE1, 0x16FE4, 0x16FEF, 0x16FF7, + 0x2A6E0, 0x2B81E, 0x2CEAE, 0x2EBE1, 0x2EE5E, 0x2FA1E, + 0x3134B, 0x3347A, 0x33480, + ) +] +han_tags = {"Hans", "Hant", "Jpan", "Hang"} +supplementary_han_classified = all( + set(character_tags(character)) == han_tags + and required_slots(character) == ["eastAsia"] + for character in han_examples +) +han_gaps_stay_latin = all( + set(character_tags(character)) != han_tags + for character in han_gap_examples +) +check("compatibility and supplementary-plane Han use the east-Asian slot", + supplementary_han_classified) +check("Unicode 17 Han gaps and neighboring scripts are not mislabeled as Han", + han_gaps_stay_latin) +check("Bopomofo and compatibility Jamo keep distinct East-Asian script tags", + character_tags("ㄅ") == ["Bopo"] and required_slots("ㄅ") == ["eastAsia"] + and character_tags("ㄱ") == ["Hang"] and required_slots("ㄱ") == ["eastAsia"]) +check("Katakana phonetic extensions remain Japanese after narrowing Han ranges", + character_tags("ㇰ") == ["Jpan"] and required_slots("ㇰ") == ["eastAsia"]) +check("supplementary Han range checks remain active under optimized Python", + __debug__ or (supplementary_han_classified and han_gaps_stay_latin)) + + def raw_font_slots(rpr): slots = {} if rpr is None: diff --git a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py index 3fb54ad..52b678f 100644 --- a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py @@ -58,13 +58,20 @@ def check(name, cond, extra=""): unsafe_wb.active["A1"] = formula_looking check("plain assignment is proven unsafe (negative control)", unsafe_wb.active["A1"].data_type == "f") -CSV_FORMULA_PREFIXES = ("=", "+", "-", "@") +CSV_FORMULA_OPERATORS = ("=", "+", "-", "@", "＝", "＋", "－", "＠") + + +def begins_spreadsheet_formula(value): + index = 0 + while index < len(value) and (ord(value[index]) <= 0x20 or value[index] == "\ufeff"): + index += 1 + return value.startswith(CSV_FORMULA_OPERATORS, index) def spreadsheet_csv_field(value, *, mode="safe"): if mode not in {"safe", "raw"}: raise ValueError("mode must be 'safe' or 'raw'") - if mode == "safe" and isinstance(value, str) and value.startswith(CSV_FORMULA_PREFIXES): + if mode == "safe" and isinstance(value, str) and begins_spreadsheet_formula(value): return "'" + value return value @@ -76,18 +83,27 @@ def delimiter_for(path): return delimiter -formula_like_fields = ["=1+1", "+SUM(A1:A2)", "-2+3", "@cmd", "plain", "-7", -7] +formula_like_fields = [ + "\ufeff=1+1", "=1+1", "+SUM(A1:A2)", "-2+3", "@cmd", + "\t=1+1", "\r@cmd", "\n-2+3", "\x00\t +SUM(A1:A2)", + "＝1+1", "＋SUM(A1:A2)", "－2+3", "＠cmd", "-7", +] +benign_fields = ["plain", -7, "\tplain", "\rplain", "\nplain", "\ufeffplain", "\x00 plain"] with open("spreadsheet-safe.csv", "w", newline="", encoding="utf-8") as output: - csv.writer(output).writerow([spreadsheet_csv_field(value) for value in formula_like_fields]) -with open("spreadsheet-safe.csv", newline="", encoding="utf-8") as exported: + csv.writer(output).writerow([ + spreadsheet_csv_field(value) for value in formula_like_fields + benign_fields + ]) +with open("spreadsheet-safe.csv", newline="", encoding="utf-8-sig") as exported: safe_fields = next(csv.reader(exported)) -check("spreadsheet-safe CSV neutralizes all four formula prefixes", - safe_fields[:4] == ["'=1+1", "'+SUM(A1:A2)", "'-2+3", "'@cmd"], safe_fields) +check("spreadsheet-safe CSV neutralizes operator, control, BOM, and fullwidth prefixes", + safe_fields[:len(formula_like_fields)] + == ["'" + value for value in formula_like_fields], safe_fields) check("safe CSV preserves benign text and numeric values", - safe_fields[4:] == ["plain", "'-7", "-7"], safe_fields[4:]) + safe_fields[len(formula_like_fields):] + == [str(value) for value in benign_fields], safe_fields) check("raw CSV mode preserves exact formula-like literal strings", - [spreadsheet_csv_field(value, mode="raw") for value in formula_like_fields[:4]] - == formula_like_fields[:4]) + [spreadsheet_csv_field(value, mode="raw") for value in formula_like_fields] + == formula_like_fields) try: spreadsheet_csv_field("=1+1", mode="unknown") invalid_csv_mode_rejected = False @@ -95,13 +111,18 @@ def delimiter_for(path): invalid_csv_mode_rejected = True check("CSV export rejects an ambiguous safety mode", invalid_csv_mode_rejected) -tabular_rows = [["Region", "Units", "Note"], ["EU, West", 120, "contains\ttab"]] +tabular_rows = [["Region", "Units", "Note"], ["EU, West", 120, "\t=1+1"]] +safe_tabular_rows = [ + [spreadsheet_csv_field(value) for value in row] for row in tabular_rows +] with open("spreadsheet-safe.TSV", "w", newline="", encoding="utf-8") as output: - csv.writer(output, delimiter=delimiter_for("spreadsheet-safe.TSV")).writerows(tabular_rows) + csv.writer(output, delimiter=delimiter_for("spreadsheet-safe.TSV")).writerows(safe_tabular_rows) with open("spreadsheet-safe.TSV", newline="", encoding="utf-8") as exported: tsv_rows = list(csv.reader(exported, delimiter="\t")) check("TSV export selects a tab delimiter case-insensitively", - tsv_rows == [[str(value) for value in row] for row in tabular_rows], tsv_rows) + tsv_rows == [[str(value) for value in row] for row in safe_tabular_rows], tsv_rows) +check("TSV safe mode neutralizes a control-prefixed formula after round-trip", + tsv_rows[1][2] == "'\t=1+1", tsv_rows) check("CSV export retains its comma delimiter", delimiter_for("output.csv") == ",") try: delimiter_for("output.txt") From de322191992df75f5bf845a74961d55b6bde5e55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=98=E6=89=AC?= Date: Mon, 17 Aug 2026 22:47:30 +0800 Subject: [PATCH 29/34] fix(document-skills): address latest review findings --- .../skills/docx/references/read.md | 2 + .../document-skills/skills/pdf/SKILL.md | 6 +- .../skills/pdf/references/inspect.md | 106 ++- .../skills/pptx/references/analyze.md | 314 +++++--- .../skills/xlsx/references/create.md | 4 +- .../skills/xlsx/references/edit.md | 37 +- .../document-skills/tests/docx_fixture.py | 7 + .../document-skills/tests/pdf_fixture.py | 195 ++++- .../document-skills/tests/pptx_fixture.py | 747 ++++++++++++++++-- .../document-skills/tests/xlsx_fixture.py | 89 ++- 10 files changed, 1305 insertions(+), 202 deletions(-) diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/read.md b/plugins/Hylouis233/document-skills/skills/docx/references/read.md index ae14edf..5dfe8d5 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/read.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/read.md @@ -86,6 +86,8 @@ def tc_text(tc, parent): pieces.append("") elif node.tag in (qn("w:br"), qn("w:cr")): pieces.append("
") + elif node.tag == qn("w:noBreakHyphen"): + pieces.append("-") elif node.tag == qn("w:sym"): pieces.append(legacy_symbol_record(node)) paragraphs.append("".join(pieces)) diff --git a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md index 1df5122..f877fd7 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/SKILL.md @@ -194,13 +194,9 @@ for page in overflow_doc: # extent so valid high-y portrait content is not flagged on a rotated page. crop_left, crop_bottom, crop_right, crop_top = normalized_box(page.cropbox) page_box = fitz.Rect(0, 0, crop_right - crop_left, crop_top - crop_bottom) - clip = fitz.Rect( - page_box.x0 - 2000, page_box.y0 - 2000, - page_box.x1 + 2000, page_box.y1 + 2000, - ) text_rects = [ fitz.Rect(block[:4]) - for block in page.get_text("blocks", clip=clip) + for block in page.get_text("blocks", clip=fitz.INFINITE_RECT()) if block[6] == 0 ] image_rects = [ diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md b/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md index 1200b00..acbb562 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md @@ -1,6 +1,7 @@ # Inspect a PDF (PyMuPDF) ```python +import math import os import fitz @@ -34,6 +35,94 @@ def font_inventory(document, page): }) return fonts +NON_VIEWABLE_ANNOTATION_FLAGS = ( + fitz.PDF_ANNOT_IS_INVISIBLE | fitz.PDF_ANNOT_IS_HIDDEN | fitz.PDF_ANNOT_IS_NO_VIEW +) + +def annotation_flags(page, item): + flags = getattr(item, "flags", None) + if flags is not None: + return int(flags) + xref = getattr(item, "xref", 0) + if not xref: + return 0 + value_type, value = page.parent.xref_get_key(xref, "F") + try: + return int(value) if value_type == "int" else 0 + except (TypeError, ValueError): + return 0 + +def visible_clip(page, rectangle, *, already_rotated=False): + try: + rectangle = fitz.Rect(rectangle) + except (TypeError, ValueError): + return None + if not all(math.isfinite(value) for value in rectangle): + return None + rectangle.normalize() + if rectangle.is_empty or rectangle.is_infinite: + return None + rotated = rectangle if already_rotated else rectangle * page.rotation_matrix + clip = rotated & page.rect + return None if clip.is_empty else clip + +def rendered_interactives(page, items): + rendered = [] + visibility_unknown = False + for item in items: + if annotation_flags(page, item) & NON_VIEWABLE_ANNOTATION_FLAGS: + continue + clip = visible_clip(page, item.rect) + if clip is None: + continue + try: + with_annotations = page.get_pixmap( + matrix=fitz.Matrix(2, 2), clip=clip, alpha=False, annots=True, + ) + without_annotations = page.get_pixmap( + matrix=fitz.Matrix(2, 2), clip=clip, alpha=False, annots=False, + ) + except (RuntimeError, ValueError): + visibility_unknown = True + continue + if with_annotations.samples != without_annotations.samples: + rendered.append(item) + return rendered, visibility_unknown + +def page_links(page): + links = [] + link = page.first_link + while link is not None: + links.append(link) + link = link.next + return links + +def link_has_target(link): + destination = getattr(link, "dest", None) + return bool(getattr(link, "uri", None)) or ( + destination is not None and getattr(destination, "page", -1) >= 0 + ) + +def viewable_interactives(page): + widgets, widget_visibility_unknown = rendered_interactives( + page, list(page.widgets() or ()) + ) + annotations, annotation_visibility_unknown = rendered_interactives( + page, list(page.annots() or ()) + ) + # Link hit rectangles are useful without a painted appearance. PyMuPDF reports + # them in rotated page coordinates already, unlike widget / annotation rects. + links = [ + link for link in page_links(page) + if link_has_target(link) + and not annotation_flags(page, link) & NON_VIEWABLE_ANNOTATION_FLAGS + and visible_clip(page, link.rect, already_rotated=True) is not None + ] + return ( + widgets, annotations, links, + widget_visibility_unknown or annotation_visibility_unknown, + ) + print("pages:", doc.page_count) print("password_protected:", doc.needs_pass, "| still_encrypted:", doc.is_encrypted, "| pdf:", doc.is_pdf) @@ -50,19 +139,18 @@ for page in doc: blocks = page.get_text("dict")["blocks"] image_blocks = [block for block in blocks if block["type"] == 1] drawings = page.get_drawings() - widgets = list(page.widgets() or ()) - annotations = list(page.annots() or ()) - links = page.get_links() + widgets, annotations, links, interaction_visibility_unknown = viewable_interactives(page) is_blank = not ( page.get_text().strip() or page.get_images() or image_blocks or drawings - or widgets or annotations or links + or widgets or annotations or links or interaction_visibility_unknown ) print(page.number + 1, "media_size:", media_size, "crop_size:", crop_size, "rotation:", page.rotation, "text_len:", len(page.get_text()), "resource_images:", len(page.get_images()), "image_blocks:", len(image_blocks), "drawings:", len(drawings), "widgets:", len(widgets), "annotations:", len(annotations), - "links:", len(links), "blank:", is_blank) + "links:", len(links), "interaction_visibility_unknown:", + interaction_visibility_unknown, "blank:", is_blank) print(" fonts:", font_inventory(doc, page)) print("media_size_consistent:", len({row["media_size"] for row in page_geometry}) <= 1) print("crop_size_consistent:", len({row["crop_size"] for row in page_geometry}) <= 1) @@ -71,7 +159,9 @@ print("crop_size_consistent:", len({row["crop_size"] for row in page_geometry}) ## Checks worth automating - **Blank page detection**: flag only when text, resource images, type-1 image blocks, drawings, - widgets, annotations, and links are all absent. `page.get_images()` lists image XObjects but + and viewable widgets, annotations, and links are all absent. Ignore interactive objects carrying + invisible, hidden, or no-view flags, as well as empty, off-page, or unrendered appearances. + `page.get_images()` lists image XObjects but misses images embedded inline in the content stream; type-1 blocks from `get_text("dict")` cover both inline and XObject image placements. Interactive form fields are widgets rather than page text, so a three-content-stream predicate alone would misclassify a usable form @@ -87,5 +177,5 @@ print("crop_size_consistent:", len({row["crop_size"] for row in page_geometry}) `pypdf.PdfReader` cross-check when provenance is unknown. Report findings as a table (page, media size, crop size, rotation, text chars, resource images, -image blocks, drawings, widgets, annotations, links, blank) - it is what every downstream -decision hangs off. +image blocks, drawings, widgets, annotations, links, interaction visibility unknown, blank) - it +is what every downstream decision hangs off. diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md index 6965875..08a46fa 100644 --- a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md @@ -7,9 +7,133 @@ from pptx import Presentation from pptx.enum.shapes import MSO_SHAPE_TYPE from pptx.oxml.ns import qn from lxml import etree +from pathlib import Path +import zipfile DIAGRAM_NS = "http://schemas.openxmlformats.org/drawingml/2006/diagram" -SAFE_XML = etree.XMLParser(load_dtd=False, resolve_entities=False, no_network=True) +CONTENT_TYPES_NS = "http://schemas.openxmlformats.org/package/2006/content-types" +SAFE_XML = etree.XMLParser( + load_dtd=False, resolve_entities=False, no_network=True, + huge_tree=False, recover=False, +) +MAX_ARCHIVE_BYTES = 200 * 1024 * 1024 +MAX_MEMBERS = 10_000 +MAX_XML_PART = 20 * 1024 * 1024 +MAX_ENTRY = 100 * 1024 * 1024 +MAX_TOTAL_UNCOMPRESSED = 500 * 1024 * 1024 +MAX_COMPRESSION_RATIO = 200 + +def require(condition, message): + """Security gates must remain active under python -O.""" + if not condition: + raise ValueError(message) + +def validate_pptx_package(source): + """Validate the same seekable handle that python-pptx will parse.""" + source.seek(0, 2) + compressed_size = source.tell() + source.seek(0) + require(compressed_size <= MAX_ARCHIVE_BYTES, "compressed PPTX file size above limit") + with zipfile.ZipFile(source) as archive: + infos = archive.infolist() + require(len(infos) <= MAX_MEMBERS, "archive member count above limit") + names = {info.filename for info in infos} + require(len(names) == len(infos), "duplicate archive member names are unsafe") + require("[Content_Types].xml" in names and "ppt/presentation.xml" in names, + "missing required OPC members") + require(sum(info.file_size for info in infos) <= MAX_TOTAL_UNCOMPRESSED, + "declared total uncompressed size above limit") + + content_types_info = next( + info for info in infos if info.filename == "[Content_Types].xml" + ) + require(content_types_info.file_size <= MAX_XML_PART, + "oversized XML part: [Content_Types].xml") + require( + content_types_info.file_size / max(content_types_info.compress_size, 1) + <= MAX_COMPRESSION_RATIO, + "suspicious compression ratio: [Content_Types].xml", + ) + content_type_chunks = [] + content_type_size = 0 + with archive.open(content_types_info) as stream: + while chunk := stream.read(64 * 1024): + content_type_size += len(chunk) + require(content_type_size <= MAX_XML_PART, + "part exceeded read limit: [Content_Types].xml") + content_type_chunks.append(chunk) + require(content_type_size == content_types_info.file_size, + "size mismatch: [Content_Types].xml") + content_types_blob = b"".join(content_type_chunks) + content_types_root = etree.fromstring(content_types_blob, parser=SAFE_XML) + require(content_types_root.tag == f"{{{CONTENT_TYPES_NS}}}Types", + "invalid content-types root") + default_types = {} + override_types = {} + for declaration in content_types_root: + if declaration.tag == f"{{{CONTENT_TYPES_NS}}}Default": + key = (declaration.get("Extension") or "").casefold() + target = default_types + elif declaration.tag == f"{{{CONTENT_TYPES_NS}}}Override": + part_name = declaration.get("PartName") or "" + require(part_name.startswith("/"), "invalid content-type part name") + key = part_name[1:].casefold() + target = override_types + else: + continue + content_type = declaration.get("ContentType") or "" + require(key and content_type and key not in target, + "invalid or duplicate content-type declaration") + target[key] = content_type.partition(";")[0].strip().casefold() + + actual_total = 0 + for info in infos: + require(info.file_size <= MAX_ENTRY, f"oversized part: {info.filename}") + ratio = info.file_size / max(info.compress_size, 1) + require(ratio <= MAX_COMPRESSION_RATIO, + f"suspicious compression ratio: {info.filename}") + extension = info.filename.rpartition(".")[2].casefold() + content_type = override_types.get( + info.filename.casefold(), default_types.get(extension, "") + ) + is_xml = ( + info.filename.casefold().endswith((".xml", ".rels")) + or content_type in {"application/xml", "text/xml"} + or content_type.endswith("+xml") + ) + if is_xml: + require(info.file_size <= MAX_XML_PART, f"oversized XML part: {info.filename}") + + chunks = [] + actual_size = 0 + if info.filename == "[Content_Types].xml": + chunks = [content_types_blob] + actual_size = len(content_types_blob) + actual_total += actual_size + else: + with archive.open(info) as stream: + while chunk := stream.read(64 * 1024): + actual_size += len(chunk) + actual_total += len(chunk) + require(actual_size <= MAX_ENTRY, + f"part exceeded read limit: {info.filename}") + require(actual_total <= MAX_TOTAL_UNCOMPRESSED, + "archive exceeded total read limit") + if is_xml: + chunks.append(chunk) + require(actual_total <= MAX_TOTAL_UNCOMPRESSED, + "archive exceeded total read limit") + require(actual_size == info.file_size, f"size mismatch: {info.filename}") + if is_xml: + etree.fromstring(b"".join(chunks), parser=SAFE_XML) + source.seek(0) + +def open_validated_presentation(path): + """Preflight and parse one immutable open-file identity, then release the handle.""" + with Path(path).open("rb") as source: + validate_pptx_package(source) + source.seek(0) + return Presentation(source) def iter_shapes(shapes): """Walk shapes recursively so content nested inside group shapes is counted too.""" @@ -19,6 +143,56 @@ def iter_shapes(shapes): else: yield shape +OOXML_TRUE = {"1", "true"} +OOXML_FALSE = {"0", "false"} + +def ooxml_bool(element, attribute, default): + value = element.get(attribute) + if value is None: + return default + value = value.strip(" \t\r\n") + if value in OOXML_TRUE: + return True + if value in OOXML_FALSE: + return False + raise ValueError(f"invalid OOXML boolean {attribute}={value!r}") + +def shape_is_hidden(shape): + properties = shape._element.find(".//" + qn("p:cNvPr")) + return properties is not None and ooxml_bool(properties, "hidden", False) + +def layer_text_content(shapes, source, *, inherited): + """Collect visible text from one layer; inherited placeholders are template prompts.""" + records = [] + for shape in shapes: + if shape_is_hidden(shape): + continue + if shape.shape_type == MSO_SHAPE_TYPE.GROUP: + records.extend(layer_text_content(shape.shapes, source, inherited=inherited)) + continue + if inherited and shape.is_placeholder: + continue + if shape.has_text_frame and shape.text_frame.text: + records.append({ + "source": source, + "shape": shape.name, + "text": shape.text_frame.text, + }) + return records + +def slide_text_content(slide): + """Inventory rendered text layers in master -> layout -> slide order.""" + records = [] + if ooxml_bool(slide._element, "showMasterSp", True): + layout = slide.slide_layout + if ooxml_bool(layout._element, "showMasterSp", True): + records.extend(layer_text_content( + layout.slide_master.shapes, "master", inherited=True, + )) + records.extend(layer_text_content(layout.shapes, "layout", inherited=True)) + records.extend(layer_text_content(slide.shapes, "slide", inherited=False)) + return records + def table_cells(table): """Inventory the grid without repeating a merged cell's text in covered slots.""" return [[ @@ -109,10 +283,11 @@ def cached_text_point_map(container, point_count): index = int(point.get("idx")) except (TypeError, ValueError): return None - value = point.find(qn("c:v")) - if value is None or not 0 <= index < point_count or index in values: + value_nodes = point.findall(qn("c:v")) + if (len(value_nodes) != 1 or not 0 <= index < point_count + or index in values): return None - values[index] = value.text or "" + values[index] = value_nodes[0].text or "" return values def category_source(series): @@ -258,22 +433,39 @@ def series_content(series, *, include_categories=False, point_budget=None): }) return content -def chart_axis_text(axis): - title = axis.find(qn("c:title")) +def title_text(title): + """Read a chart/axis title without invoking python-pptx's mutating text accessor.""" if title is None: return "" - rich = title.xpath("./c:tx/c:rich") - references = title.xpath("./c:tx/c:strRef") - if len(rich) + len(references) != 1: + text_nodes = title.findall(qn("c:tx")) + if len(text_nodes) != 1: + return None + sources = [child for child in text_nodes[0] if isinstance(child.tag, str)] + if len(sources) != 1: + return None + if sources[0].tag == qn("c:rich"): + paragraphs = sources[0].findall(qn("a:p")) + return None if not paragraphs else "\n".join( + paragraph.text for paragraph in paragraphs + ) + if sources[0].tag != qn("c:strRef"): return None - if rich: - return "\n".join(paragraph.text for paragraph in rich[0].findall(qn("a:p"))) - caches = references[0].findall(qn("c:strCache")) + formulas = sources[0].findall(qn("c:f")) + if len(formulas) != 1 or not formulas[0].text: + return None + caches = sources[0].findall(qn("c:strCache")) if (len(caches) != 1 or (count := cache_point_count(caches[0])) is None or count != 1): return None values = cached_text_point_map(caches[0], count) - return None if values is None else "\n".join(values.get(index, "") for index in range(count)) + return None if values is None or set(values) != {0} else values[0] + +def chart_axis_text(axis): + return title_text(axis.find(qn("c:title"))) + +def chart_title_text(chart): + titles = chart._element.xpath("./c:chart/c:title") + return title_text(titles[0]) if len(titles) == 1 else ("" if not titles else None) def chart_axes(chart): """Return every category, date, value, and series axis in document order.""" @@ -318,7 +510,7 @@ def smartart_content(shape): labels = [node.text for node in root.iter(qn("a:t")) if node.text] return {"name": shape.name, "status": "ok", "text": labels} -prs = Presentation("input.pptx") +prs = open_validated_presentation("input.pptx") print("slide size:", prs.slide_width, prs.slide_height) point_budget = {"remaining": MAX_CHART_POINTS} for i, slide in enumerate(prs.slides): @@ -326,7 +518,7 @@ for i, slide in enumerate(prs.slides): title = slide.shapes.title.text_frame.text if slide.shapes.title is not None else "" notes = slide.notes_slide.notes_text_frame.text if slide.has_notes_slide else "" shapes = list(iter_shapes(slide.shapes)) # flattened; groups are common in template decks - text = [sh.text_frame.text for sh in shapes if sh.has_text_frame and sh.text_frame.text] + text = slide_text_content(slide) tables = [ table_cells(sh.table) for sh in shapes if sh.has_table @@ -336,10 +528,7 @@ for i, slide in enumerate(prs.slides): if not sh.has_chart: continue chart = sh.chart - chart_title = ( - chart.chart_title.text_frame.text - if chart.has_title else "" - ) + chart_title = chart_title_text(chart) plots = [] for plot in chart.plots: items = list(plot.series) @@ -379,8 +568,10 @@ for i, slide in enumerate(prs.slides): print(" notes:", notes) ``` -(Simplify the title lookup to `slide.shapes.title` when present; the defensive loop is for -layouts where the title placeholder is missing.) +Inherited layout/master placeholders are deliberately excluded: their stored text is a template +prompt, not rendered slide copy, and slide placeholders already contribute their instantiated +text. Footer/date/slide-number fields and occlusion still require a rendered-slide check when +pixel-level visibility matters. ## Triage: deck renders wrong @@ -395,74 +586,10 @@ layouts where the title placeholder is missing.) ## Bounded package health check -Inspect declared sizes and compression ratios before decompressing anything. `ZipFile.testzip()` -must not be the first check because it expands every member, including an archive bomb. - -```python -import zipfile -from pathlib import Path -from lxml import etree - -path = Path("input.pptx") -MAX_ARCHIVE_BYTES = 200 * 1024 * 1024 -MAX_MEMBERS = 10_000 -MAX_XML_PART = 20 * 1024 * 1024 -MAX_ENTRY = 100 * 1024 * 1024 -MAX_TOTAL_UNCOMPRESSED = 500 * 1024 * 1024 -MAX_COMPRESSION_RATIO = 200 - -# Security limits must survive `python -O` (which strips assert statements), -# so every check raises explicitly instead of asserting. -def require(condition, message): - if not condition: - raise ValueError(message) - -safe_xml_parser = etree.XMLParser( - load_dtd=False, - resolve_entities=False, - no_network=True, - huge_tree=False, - recover=False, -) - -# Bound the package itself before ZipFile materializes its central directory. -require(path.stat().st_size <= MAX_ARCHIVE_BYTES, - "compressed PPTX file size above limit") -with zipfile.ZipFile(path) as archive: - infos = archive.infolist() - # Check the count before building sets, summing sizes, or opening any member. - require(len(infos) <= MAX_MEMBERS, "archive member count above limit") - names = {info.filename for info in infos} - require(len(names) == len(infos), "duplicate archive member names are unsafe") - require("[Content_Types].xml" in names and "ppt/presentation.xml" in names, - "missing required OPC members") - require(sum(info.file_size for info in infos) <= MAX_TOTAL_UNCOMPRESSED, - "declared total uncompressed size above limit") - - actual_total = 0 - for info in infos: - require(info.file_size <= MAX_ENTRY, f"oversized part: {info.filename}") - ratio = info.file_size / max(info.compress_size, 1) - require(ratio <= MAX_COMPRESSION_RATIO, f"suspicious compression ratio: {info.filename}") - is_xml = info.filename.endswith((".xml", ".rels")) - if is_xml: - require(info.file_size <= MAX_XML_PART, f"oversized XML part: {info.filename}") - - chunks = [] - actual_size = 0 - # Streaming to EOF verifies decompression and CRC only after metadata limits pass. - with archive.open(info) as stream: - while chunk := stream.read(64 * 1024): - actual_size += len(chunk) - actual_total += len(chunk) - require(actual_size <= MAX_ENTRY, f"part exceeded read limit: {info.filename}") - require(actual_total <= MAX_TOTAL_UNCOMPRESSED, "archive exceeded total read limit") - if is_xml: - chunks.append(chunk) - require(actual_size == info.file_size, f"size mismatch: {info.filename}") - if is_xml: - etree.fromstring(b"".join(chunks), parser=safe_xml_parser) -``` +The content-inventory block performs this check before its first `Presentation()` call and parses +the same open handle it validated. Do not replace that order with `Presentation(path)` followed by +a later check: package parsing already expands ZIP members. Likewise, `ZipFile.testzip()` must not +be the first check because it expands every member, including an archive bomb. These are conservative triage defaults, not PPTX format limits. Raise a limit only for an explicitly trusted large deck, and retain the per-member and streaming checks. @@ -470,25 +597,24 @@ explicitly trusted large deck, and retain the per-member and streaming checks. ## Font triage with inheritance Most template decks set no explicit `run.font.name`; the effective face is inherited from the -placeholder, layout, master, or theme. Resolve what you can and name the fallback explicitly: +placeholder, layout, master, or theme. This block is a continuation of the content-inventory +session above: reuse its already validated `prs` object rather than reopening the path. Resolve +what you can and name the fallback explicitly: ```python import xml.etree.ElementTree as ET -from pptx import Presentation from pptx.enum.shapes import MSO_SHAPE_TYPE from pptx.opc.constants import RELATIONSHIP_TYPE as RT from pptx.oxml.ns import qn def iter_shapes(shapes): - """Self-contained recursive walker for this independently runnable block.""" + """Local recursive walker for the font-triage continuation.""" for shape in shapes: if shape.shape_type == MSO_SHAPE_TYPE.GROUP: yield from iter_shapes(shape.shapes) else: yield shape -prs = Presentation("deck.pptx") - # 1. Resolve the theme related to each slide's own layout/master. A package can contain # multiple masters with different themes, so the first /ppt/theme/* part is not a safe default. theme_cache = {} diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/create.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/create.md index 5658b82..6f3220d 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/create.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/create.md @@ -47,7 +47,9 @@ ws.freeze_panes = "A2" # Native chart bound to the sheet data chart = BarChart() chart.type = "col" -chart.title = "Revenue by product" +# The source has one bar per sales row (including two separate Widget rows), not +# a product aggregate, so keep the title explicit about that granularity. +chart.title = "Revenue by transaction row" chart.y_axis.title = "Revenue" data = Reference(ws, min_col=5, min_row=1, max_row=last) # includes header for series name cats = Reference(ws, min_col=2, min_row=2, max_row=last) diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md index dd6b0af..4bb8a92 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md @@ -38,6 +38,17 @@ def drawing_anchor_rows(drawing): rows.append(marker.row + 1) return tuple(rows) +def sparse_formula_cells(sheet): + """Walk instantiated formula cells only; worksheet bounds can span the full grid.""" + if not hasattr(sheet, "_cells"): + raise RuntimeError("structural edits require a normal writable Worksheet") + return ( + cell for cell in sorted( + sheet._cells.values(), key=lambda cell: (cell.row, cell.column) + ) + if cell.data_type == "f" + ) + def structural_references(workbook): """Inventory formulas/ranges that insert_rows/delete_rows will not rewrite.""" refs = [] @@ -45,11 +56,9 @@ def structural_references(workbook): refs.append(("defined name", item.name, item.attr_text)) for sheet in workbook.worksheets: owner = sheet.title - for row in sheet.iter_rows(): - for cell in row: - if cell.data_type == "f": - refs.append(("cell formula", f"{owner}!{cell.coordinate}", - formula_text(cell.value))) + for cell in sparse_formula_cells(sheet): + refs.append(("cell formula", f"{owner}!{cell.coordinate}", + formula_text(cell.value))) for table in sheet.tables.values(): refs.append(("table", owner + "!" + table.name, table.ref)) for merged_range in sheet.merged_cells.ranges: @@ -90,16 +99,14 @@ def cell_formula_references(workbook): """Inventory ordinary, array, and data-table formulas before row/column moves.""" refs = [] for sheet in workbook.worksheets: - for row in sheet.iter_rows(): - for cell in row: - if cell.data_type == "f": - value = cell.value - refs.append(( - "cell formula", - sheet.title, - cell.coordinate, - formula_text(value), - )) + for cell in sparse_formula_cells(sheet): + value = cell.value + refs.append(( + "cell formula", + sheet.title, + cell.coordinate, + formula_text(value), + )) return refs def formula_may_intersect_rows(owner_sheet, formula, shifted_sheet, start_row): diff --git a/plugins/Hylouis233/document-skills/tests/docx_fixture.py b/plugins/Hylouis233/document-skills/tests/docx_fixture.py index cf3a30b..2a599a5 100644 --- a/plugins/Hylouis233/document-skills/tests/docx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/docx_fixture.py @@ -935,6 +935,8 @@ def tc_text(tc): pieces.append("") elif node.tag in (qn("w:br"), qn("w:cr")): pieces.append("
") + elif node.tag == qn("w:noBreakHyphen"): + pieces.append("-") paragraphs.append("".join(pieces)) return " / ".join(paragraphs) @@ -947,6 +949,9 @@ def tc_text(tc): tab_element = OxmlElement("w:tab") run_with_tab._r.append(tab_element) run_with_tab.add_text("after tab") +hyphen_run = cell.paragraphs[0].add_run("non") +hyphen_run._r.append(OxmlElement("w:noBreakHyphen")) +hyphen_run.add_text("breaking") cells_doc.save("cell-paragraphs.docx") cells_reopened = Document("cell-paragraphs.docx") cells_tc = cells_reopened.tables[0].rows[0]._tr.tc_lst[0] @@ -956,6 +961,8 @@ def tc_text(tc): "Firstafter tab" in joined_raw, joined_raw) check("tc_text preserves the paragraph boundary", " / Second" in extracted, extracted) check("tc_text keeps tabs visible", "after tab" in extracted, extracted) +check("tc_text preserves a table-cell nonbreaking hyphen", + "non-breaking" in extracted, extracted) # ---- edit.md raw OOXML repack: every input gets a fresh extraction tree --------- diff --git a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py index fcac288..0f106b1 100644 --- a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py @@ -5,6 +5,7 @@ # python pptx_fixture.py (deps: python-pptx) # python xlsx_fixture.py (deps: openpyxl) # python docx_fixture.py (deps: python-docx, pymupdf, soffice on PATH) +import math import os import sys @@ -218,14 +219,130 @@ def open_pdf(path): widget_canvas.save() widget_doc = fitz.open("widget-only.pdf") widget_page = widget_doc[0] -widgets = list(widget_page.widgets() or ()) -annotations = list(widget_page.annots() or ()) -links = widget_page.get_links() + +NON_VIEWABLE_ANNOTATION_FLAGS = ( + fitz.PDF_ANNOT_IS_INVISIBLE | fitz.PDF_ANNOT_IS_HIDDEN | fitz.PDF_ANNOT_IS_NO_VIEW +) + + +def annotation_flags(page, item): + flags = getattr(item, "flags", None) + if flags is not None: + return int(flags) + xref = getattr(item, "xref", 0) + if not xref: + return 0 + value_type, value = page.parent.xref_get_key(xref, "F") + try: + return int(value) if value_type == "int" else 0 + except (TypeError, ValueError): + return 0 + + +def visible_clip(page, rectangle, *, already_rotated=False): + try: + rectangle = fitz.Rect(rectangle) + except (TypeError, ValueError): + return None + if not all(math.isfinite(value) for value in rectangle): + return None + rectangle.normalize() + if rectangle.is_empty or rectangle.is_infinite: + return None + rotated = rectangle if already_rotated else rectangle * page.rotation_matrix + clip = rotated & page.rect + return None if clip.is_empty else clip + + +def rendered_interactives(page, items): + rendered = [] + visibility_unknown = False + for item in items: + if annotation_flags(page, item) & NON_VIEWABLE_ANNOTATION_FLAGS: + continue + clip = visible_clip(page, item.rect) + if clip is None: + continue + try: + with_annotations = page.get_pixmap( + matrix=fitz.Matrix(2, 2), clip=clip, alpha=False, annots=True, + ) + without_annotations = page.get_pixmap( + matrix=fitz.Matrix(2, 2), clip=clip, alpha=False, annots=False, + ) + except (RuntimeError, ValueError): + visibility_unknown = True + continue + if with_annotations.samples != without_annotations.samples: + rendered.append(item) + return rendered, visibility_unknown + + +def page_links(page): + links = [] + link = page.first_link + while link is not None: + links.append(link) + link = link.next + return links + + +def link_has_target(link): + destination = getattr(link, "dest", None) + return bool(getattr(link, "uri", None)) or ( + destination is not None and getattr(destination, "page", -1) >= 0 + ) + + +def viewable_interactives(page): + widgets, widget_visibility_unknown = rendered_interactives( + page, list(page.widgets() or ()) + ) + annotations, annotation_visibility_unknown = rendered_interactives( + page, list(page.annots() or ()) + ) + links = [ + link for link in page_links(page) + if link_has_target(link) + and not annotation_flags(page, link) & NON_VIEWABLE_ANNOTATION_FLAGS + and visible_clip(page, link.rect, already_rotated=True) is not None + ] + return ( + widgets, annotations, links, + widget_visibility_unknown or annotation_visibility_unknown, + ) + + +class RenderFailurePage: + rect = fitz.Rect(0, 0, 200, 300) + rotation_matrix = fitz.Identity + + def get_pixmap(self, **kwargs): + raise RuntimeError("fixture render failure") + + +class VisibleInteractiveProbe: + flags = 0 + rect = fitz.Rect(20, 20, 80, 40) + + +failed_render_items, failed_render_unknown = rendered_interactives( + RenderFailurePage(), [VisibleInteractiveProbe()] +) +failed_render_blank = not (failed_render_items or failed_render_unknown) +check("interactive render failures keep blank-page classification fail closed", + not failed_render_items and failed_render_unknown and not failed_render_blank, + (failed_render_items, failed_render_unknown, failed_render_blank)) + + +widgets, annotations, links, interaction_visibility_unknown = viewable_interactives( + widget_page +) blank = ( not widget_page.get_text().strip() and not widget_page.get_images() and not widget_page.get_drawings() - and not widgets and not annotations and not links + and not widgets and not annotations and not links and not interaction_visibility_unknown ) check("widget-only form page exposes a widget", len(widgets) == 1, len(widgets)) check("widget-aware blank-page predicate keeps form page", not blank) @@ -312,7 +429,8 @@ def widget_count(page, rendered_page): hidden_render = fitz.open(hidden_path) check(f"{label} widget does not exempt an otherwise blank page", widget_count(hidden_page, hidden_render[0]) == 0 - and not bool((hidden_page.extract_text() or "").strip())) + and not bool((hidden_page.extract_text() or "").strip()) + and not any(viewable_interactives(hidden_render[0])[:3])) appearance_writer = pypdf.PdfWriter() appearance_writer.append(widget_postcheck) @@ -358,7 +476,49 @@ def widget_count(page, rendered_page): geometry_page = pypdf.PdfReader(geometry_path).pages[0] geometry_render = fitz.open(geometry_path) check(f"{label} widget does not exempt an otherwise blank page", - widget_count(geometry_page, geometry_render[0]) == 0) + widget_count(geometry_page, geometry_render[0]) == 0 + and not any(viewable_interactives(geometry_render[0])[:3])) + +interaction_doc = fitz.open() +interaction_page = interaction_doc.new_page(width=200, height=300) +hidden_annotation = interaction_page.add_text_annot((40, 40), "hidden note") +hidden_annotation.set_flags(fitz.PDF_ANNOT_IS_HIDDEN) +hidden_annotation.update() +interaction_page.insert_link({ + "kind": fitz.LINK_URI, + "from": fitz.Rect(500, 500, 600, 520), + "uri": "https://example.invalid", +}) +interaction_doc.save("non-viewable-interactives.pdf") +interaction_doc.close() +interaction_reopened = fitz.open("non-viewable-interactives.pdf") +filtered_widgets, filtered_annotations, filtered_links, filtered_unknown = ( + viewable_interactives(interaction_reopened[0]) +) +check("hidden annotations and off-page links do not exempt a blank page", + not filtered_widgets and not filtered_annotations and not filtered_links + and not filtered_unknown, + (filtered_widgets, filtered_annotations, filtered_links, filtered_unknown)) + +visible_interaction_doc = fitz.open() +visible_interaction_page = visible_interaction_doc.new_page(width=200, height=300) +visible_annotation = visible_interaction_page.add_text_annot((40, 40), "visible note") +visible_annotation.update() +visible_interaction_page.insert_link({ + "kind": fitz.LINK_URI, + "from": fitz.Rect(40, 80, 140, 100), + "uri": "https://example.invalid", +}) +visible_interaction_doc.save("visible-interactives.pdf") +visible_interaction_doc.close() +visible_interaction_reopened = fitz.open("visible-interactives.pdf") +visible_widgets, visible_annotations, visible_links, visible_unknown = ( + viewable_interactives(visible_interaction_reopened[0]) +) +check("visible annotations and usable links keep an interactive page nonblank", + not visible_widgets and len(visible_annotations) == 1 and len(visible_links) == 1 + and not visible_unknown, + (visible_widgets, visible_annotations, visible_links, visible_unknown)) reversed_writer = pypdf.PdfWriter() reversed_writer.append(widget_postcheck) @@ -1078,6 +1238,11 @@ def stamp_line_directions(path, page_number): overflow_bad.drawString(72, -200, "drawn far below the page box") overflow_bad.showPage() overflow_bad.save() +overflow_very_far = canvas.Canvas("overflow-very-far.pdf", pagesize=A4) +overflow_very_far.setFont("Helvetica", 16) +overflow_very_far.drawString(72, -5000, "drawn beyond the old finite search window") +overflow_very_far.showPage() +overflow_very_far.save() rotated_source = canvas.Canvas("overflow-rotated-source.pdf", pagesize=A4) rotated_source.setFont("Helvetica", 16) @@ -1144,16 +1309,12 @@ def overflow_pages(path, password=None): raise RuntimeError(f"password could not decrypt {path} for overflow checking") pages = [] for page in doc: - # Plain block extraction drops fully off-page text; enlarge the clip. + # Plain block extraction drops fully off-page text; disable clipping. crop_left, crop_bottom, crop_right, crop_top = normalized_box(page.cropbox) page_box = fitz.Rect(0, 0, crop_right - crop_left, crop_top - crop_bottom) - clip = fitz.Rect( - page_box.x0 - 2000, page_box.y0 - 2000, - page_box.x1 + 2000, page_box.y1 + 2000, - ) text_rects = [ fitz.Rect(block[:4]) - for block in page.get_text("blocks", clip=clip) + for block in page.get_text("blocks", clip=fitz.INFINITE_RECT()) if block[6] == 0 ] image_rects = [ @@ -1175,6 +1336,16 @@ def overflow_pages(path, password=None): overflow_pages("widget-reversed-page-boxes.pdf") == []) check("off-page text is detected by the overflow check (negative control)", overflow_pages("overflow-bad.pdf") == [1]) +very_far_probe = fitz.open("overflow-very-far.pdf")[0] +old_finite_clip = fitz.Rect( + very_far_probe.rect.x0 - 2000, very_far_probe.rect.y0 - 2000, + very_far_probe.rect.x1 + 2000, very_far_probe.rect.y1 + 2000, +) +check("the old finite clip misses text positioned more than 2,000 points away (negative control)", + not [block for block in very_far_probe.get_text("blocks", clip=old_finite_clip) + if block[6] == 0]) +check("the unbounded overflow check detects very distant positioned text", + overflow_pages("overflow-very-far.pdf") == [1]) check("in-bounds image and vector drawing pass the overflow check", overflow_pages("overflow-graphics-ok.pdf") == []) check("out-of-bounds image placement is detected", diff --git a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py index f1c5607..12c0f47 100644 --- a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py @@ -181,7 +181,8 @@ def placeholder_of_type(slide, *types): MAX_ENTRY = 100 * 1024 * 1024 MAX_TOTAL_UNCOMPRESSED = 500 * 1024 * 1024 MAX_COMPRESSION_RATIO = 200 -safe_xml_parser = etree.XMLParser( +CONTENT_TYPES_NS = "http://schemas.openxmlformats.org/package/2006/content-types" +SAFE_XML = etree.XMLParser( load_dtd=False, resolve_entities=False, no_network=True, @@ -195,47 +196,116 @@ def require(condition, message): raise ValueError(message) -def validate_pptx_package(path): - require( - Path(path).stat().st_size <= MAX_ARCHIVE_BYTES, - "compressed PPTX file size above limit", - ) - with zipfile.ZipFile(path) as archive: +def validate_pptx_package(source): + source.seek(0, 2) + compressed_size = source.tell() + source.seek(0) + require(compressed_size <= MAX_ARCHIVE_BYTES, + "compressed PPTX file size above limit") + with zipfile.ZipFile(source) as archive: infos = archive.infolist() require(len(infos) <= MAX_MEMBERS, "archive member count above limit") names = {info.filename for info in infos} require(len(names) == len(infos), "duplicate archive member names are unsafe") require("[Content_Types].xml" in names and "ppt/presentation.xml" in names, - "required PPTX package parts are missing") + "missing required OPC members") require(sum(info.file_size for info in infos) <= MAX_TOTAL_UNCOMPRESSED, - "declared archive size exceeds the review limit") + "declared total uncompressed size above limit") + + content_types_info = next( + info for info in infos if info.filename == "[Content_Types].xml" + ) + require(content_types_info.file_size <= MAX_XML_PART, + "oversized XML part: [Content_Types].xml") + require( + content_types_info.file_size / max(content_types_info.compress_size, 1) + <= MAX_COMPRESSION_RATIO, + "suspicious compression ratio: [Content_Types].xml", + ) + content_type_chunks = [] + content_type_size = 0 + with archive.open(content_types_info) as stream: + while chunk := stream.read(64 * 1024): + content_type_size += len(chunk) + require(content_type_size <= MAX_XML_PART, + "part exceeded read limit: [Content_Types].xml") + content_type_chunks.append(chunk) + require(content_type_size == content_types_info.file_size, + "size mismatch: [Content_Types].xml") + content_types_blob = b"".join(content_type_chunks) + content_types_root = etree.fromstring(content_types_blob, parser=SAFE_XML) + require(content_types_root.tag == f"{{{CONTENT_TYPES_NS}}}Types", + "invalid content-types root") + default_types = {} + override_types = {} + for declaration in content_types_root: + if declaration.tag == f"{{{CONTENT_TYPES_NS}}}Default": + key = (declaration.get("Extension") or "").casefold() + target = default_types + elif declaration.tag == f"{{{CONTENT_TYPES_NS}}}Override": + part_name = declaration.get("PartName") or "" + require(part_name.startswith("/"), "invalid content-type part name") + key = part_name[1:].casefold() + target = override_types + else: + continue + content_type = declaration.get("ContentType") or "" + require(key and content_type and key not in target, + "invalid or duplicate content-type declaration") + target[key] = content_type.partition(";")[0].strip().casefold() + actual_total = 0 for info in infos: require(info.file_size <= MAX_ENTRY, f"oversized part: {info.filename}") - require(info.file_size / max(info.compress_size, 1) <= MAX_COMPRESSION_RATIO, + ratio = info.file_size / max(info.compress_size, 1) + require(ratio <= MAX_COMPRESSION_RATIO, f"suspicious compression ratio: {info.filename}") - is_xml = info.filename.endswith((".xml", ".rels")) + extension = info.filename.rpartition(".")[2].casefold() + content_type = override_types.get( + info.filename.casefold(), default_types.get(extension, "") + ) + is_xml = ( + info.filename.casefold().endswith((".xml", ".rels")) + or content_type in {"application/xml", "text/xml"} + or content_type.endswith("+xml") + ) if is_xml: require(info.file_size <= MAX_XML_PART, f"oversized XML part: {info.filename}") chunks = [] actual_size = 0 - with archive.open(info) as stream: - while chunk := stream.read(64 * 1024): - actual_size += len(chunk) - actual_total += len(chunk) - require(actual_size <= MAX_ENTRY, - f"part exceeded read limit: {info.filename}") - require(actual_total <= MAX_TOTAL_UNCOMPRESSED, - "archive exceeded total read limit") - if is_xml: - chunks.append(chunk) + if info.filename == "[Content_Types].xml": + chunks = [content_types_blob] + actual_size = len(content_types_blob) + actual_total += actual_size + else: + with archive.open(info) as stream: + while chunk := stream.read(64 * 1024): + actual_size += len(chunk) + actual_total += len(chunk) + require(actual_size <= MAX_ENTRY, + f"part exceeded read limit: {info.filename}") + require(actual_total <= MAX_TOTAL_UNCOMPRESSED, + "archive exceeded total read limit") + if is_xml: + chunks.append(chunk) + require(actual_total <= MAX_TOTAL_UNCOMPRESSED, + "archive exceeded total read limit") require(actual_size == info.file_size, f"size mismatch: {info.filename}") if is_xml: - etree.fromstring(b"".join(chunks), parser=safe_xml_parser) + etree.fromstring(b"".join(chunks), parser=SAFE_XML) + source.seek(0) + + +def open_validated_presentation(path): + with Path(path).open("rb") as source: + validate_pptx_package(source) + source.seek(0) + return Presentation(source) try: - validate_pptx_package("input.pptx") + with Path("input.pptx").open("rb") as source: + validate_pptx_package(source) healthy_pptx_passed = True except Exception: healthy_pptx_passed = False @@ -245,7 +315,8 @@ def validate_pptx_package(path): original_archive_limit = MAX_ARCHIVE_BYTES MAX_ARCHIVE_BYTES = 0 try: - validate_pptx_package("oversized-before-open.pptx") + with Path("oversized-before-open.pptx").open("rb") as source: + validate_pptx_package(source) compressed_size_rejected_before_open = False except ValueError as exc: compressed_size_rejected_before_open = ( @@ -262,7 +333,8 @@ def validate_pptx_package(path): for member_index in range(MAX_MEMBERS + 1): archive.writestr(f"zero-{member_index:05d}.bin", b"") try: - validate_pptx_package("too-many-members.pptx") + with Path("too-many-members.pptx").open("rb") as source: + validate_pptx_package(source) many_members_rejected = False except ValueError as exc: many_members_rejected = str(exc) == "archive member count above limit" @@ -271,18 +343,232 @@ def validate_pptx_package(path): many_members_rejected, ) +with zipfile.ZipFile("input.pptx") as source_archive: + compressed_bomb_payload = { + info.filename: source_archive.read(info) for info in source_archive.infolist() + } +compressed_bomb_member = next( + name for name in compressed_bomb_payload + if not name.casefold().endswith((".xml", ".rels")) +) +compressed_bomb_payload[compressed_bomb_member] = b"x" * 2_000_000 with zipfile.ZipFile("compressed-bomb.pptx", "w", zipfile.ZIP_DEFLATED) as archive: - archive.writestr("[Content_Types].xml", "") - archive.writestr( - "ppt/presentation.xml", - '' + (" " * 2_000_000) + "", - ) + for member_name, member_data in compressed_bomb_payload.items(): + archive.writestr(member_name, member_data) try: - validate_pptx_package("compressed-bomb.pptx") + with Path("compressed-bomb.pptx").open("rb") as source: + validate_pptx_package(source) pptx_bomb_rejected = False +except ValueError as error: + pptx_bomb_rejected = ( + str(error) == f"suspicious compression ratio: {compressed_bomb_member}" + ) +check("valid-manifest PPTX compression bomb reaches the ratio gate", + pptx_bomb_rejected, compressed_bomb_member) + +with zipfile.ZipFile("input.pptx") as source_archive: + uppercase_xml_payload = { + info.filename: source_archive.read(info) for info in source_archive.infolist() + } +uppercase_slide = uppercase_xml_payload.pop("ppt/slides/slide1.xml") +uppercase_xml_limit = 64 * 1024 +uppercase_xml_payload["ppt/slides/slide1.XML"] = uppercase_slide.replace( + b"", b" " * uppercase_xml_limit + b"", 1, +) +uppercase_xml_payload["[Content_Types].xml"] = uppercase_xml_payload[ + "[Content_Types].xml" +].replace(b"/ppt/slides/slide1.xml", b"/ppt/slides/slide1.XML") +uppercase_xml_payload["ppt/_rels/presentation.xml.rels"] = uppercase_xml_payload[ + "ppt/_rels/presentation.xml.rels" +].replace(b"slides/slide1.xml", b"slides/slide1.XML") +with zipfile.ZipFile("uppercase-xml-part.pptx", "w", zipfile.ZIP_STORED) as archive: + for member_name, member_data in uppercase_xml_payload.items(): + archive.writestr(member_name, member_data) +uppercase_route_parses = len(Presentation("uppercase-xml-part.pptx").slides) == 1 +original_xml_limit = MAX_XML_PART +MAX_XML_PART = uppercase_xml_limit +try: + with Path("uppercase-xml-part.pptx").open("rb") as source: + validate_pptx_package(source) + uppercase_oversized_xml_rejected = False +except ValueError as error: + uppercase_oversized_xml_rejected = ( + str(error) == "oversized XML part: ppt/slides/slide1.XML" + ) +finally: + MAX_XML_PART = original_xml_limit +check("uppercase XML part names cannot bypass the XML size and parser gate", + uppercase_route_parses and uppercase_oversized_xml_rejected) + +with zipfile.ZipFile("input.pptx") as source_archive: + typed_xml_payload = { + info.filename: source_archive.read(info) for info in source_archive.infolist() + } +typed_slide = typed_xml_payload.pop("ppt/slides/slide1.xml") +typed_xml_payload["ppt/slides/Slide1.DaT"] = typed_slide.replace( + b"", b" " * uppercase_xml_limit + b"", 1, +) +typed_xml_payload["[Content_Types].xml"] = typed_xml_payload[ + "[Content_Types].xml" +].replace(b"/ppt/slides/slide1.xml", b"/PPT/SLIDES/SLIDE1.DAT") +typed_xml_payload["ppt/_rels/presentation.xml.rels"] = typed_xml_payload[ + "ppt/_rels/presentation.xml.rels" +].replace(b"slides/slide1.xml", b"slides/Slide1.DaT") +with zipfile.ZipFile("content-typed-xml-part.pptx", "w", zipfile.ZIP_STORED) as archive: + for member_name, member_data in typed_xml_payload.items(): + archive.writestr(member_name, member_data) +typed_xml_route_parses = len(Presentation("content-typed-xml-part.pptx").slides) == 1 +original_xml_limit = MAX_XML_PART +MAX_XML_PART = uppercase_xml_limit +try: + with Path("content-typed-xml-part.pptx").open("rb") as source: + validate_pptx_package(source) + typed_oversized_xml_rejected = False +except ValueError as error: + typed_oversized_xml_rejected = ( + str(error) == "oversized XML part: ppt/slides/Slide1.DaT" + ) +finally: + MAX_XML_PART = original_xml_limit +check("case-insensitive XML content types cannot bypass bounds with an arbitrary extension", + typed_xml_route_parses and typed_oversized_xml_rejected) + +mixed_case_xml_rejections = [] +for mixed_case_name in ("custom.XmL", "custom.ReLs"): + with zipfile.ZipFile("input.pptx") as source_archive: + mixed_case_payload = { + info.filename: source_archive.read(info) for info in source_archive.infolist() + } + mixed_case_payload[mixed_case_name] = ( + b"" + b" " * uppercase_xml_limit + b"" + ) + mixed_case_declaration = ( + f'' + ).encode() + mixed_case_payload["[Content_Types].xml"] = mixed_case_payload[ + "[Content_Types].xml" + ].replace(b"", mixed_case_declaration + b"", 1) + mixed_case_path = "mixed-case-" + mixed_case_name.replace(".", "-") + ".pptx" + with zipfile.ZipFile(mixed_case_path, "w", zipfile.ZIP_STORED) as archive: + for member_name, member_data in mixed_case_payload.items(): + archive.writestr(member_name, member_data) + original_xml_limit = MAX_XML_PART + MAX_XML_PART = uppercase_xml_limit + try: + with Path(mixed_case_path).open("rb") as source: + validate_pptx_package(source) + except ValueError as error: + if str(error) == f"oversized XML part: {mixed_case_name}": + mixed_case_xml_rejections.append(mixed_case_name) + finally: + MAX_XML_PART = original_xml_limit +check("mixed-case XML and relationship suffixes keep XML bounds", + mixed_case_xml_rejections == ["custom.XmL", "custom.ReLs"], + mixed_case_xml_rejections) + +with zipfile.ZipFile("input.pptx") as source_archive: + default_xml_payload = { + info.filename: source_archive.read(info) for info in source_archive.infolist() + } +default_xml_payload["custom.payload"] = ( + b"" + b" " * uppercase_xml_limit + b"" +) +default_xml_declaration = ( + b'' +) +default_xml_payload["[Content_Types].xml"] = default_xml_payload[ + "[Content_Types].xml" +].replace(b"", default_xml_declaration + b"", 1) +with zipfile.ZipFile("default-content-type-xml.pptx", "w", zipfile.ZIP_STORED) as archive: + for member_name, member_data in default_xml_payload.items(): + archive.writestr(member_name, member_data) +original_xml_limit = MAX_XML_PART +MAX_XML_PART = uppercase_xml_limit +try: + with Path("default-content-type-xml.pptx").open("rb") as source: + validate_pptx_package(source) + default_typed_xml_rejected = False +except ValueError as error: + default_typed_xml_rejected = str(error) == "oversized XML part: custom.payload" +finally: + MAX_XML_PART = original_xml_limit +check("default +xml content types apply XML bounds to arbitrary extensions", + default_typed_xml_rejected) + +duplicate_override_payload = dict(typed_xml_payload) +duplicate_override = ( + b'' +) +duplicate_override_payload["[Content_Types].xml"] = duplicate_override_payload[ + "[Content_Types].xml" +].replace(b"", duplicate_override + b"", 1) +with zipfile.ZipFile("duplicate-case-override.pptx", "w", zipfile.ZIP_STORED) as archive: + for member_name, member_data in duplicate_override_payload.items(): + archive.writestr(member_name, member_data) +try: + with Path("duplicate-case-override.pptx").open("rb") as source: + validate_pptx_package(source) + duplicate_case_override_rejected = False +except ValueError as error: + duplicate_case_override_rejected = ( + str(error) == "invalid or duplicate content-type declaration" + ) +check("case-insensitive duplicate content-type overrides fail closed", + duplicate_case_override_rejected) + +presentation_loader_calls = [] +real_presentation_loader = Presentation + + +def unexpected_presentation_loader(source): + presentation_loader_calls.append(source) + return real_presentation_loader(source) + + +Presentation = unexpected_presentation_loader +try: + open_validated_presentation("compressed-bomb.pptx") + bomb_rejected_before_presentation = False except ValueError: - pptx_bomb_rejected = True -check("PPTX compression bomb is rejected before XML expansion", pptx_bomb_rejected) + bomb_rejected_before_presentation = not presentation_loader_calls +finally: + Presentation = real_presentation_loader +check("content inventory rejects a package bomb before Presentation parses it", + bomb_rejected_before_presentation, len(presentation_loader_calls)) + +validated_sources = [] +parsed_sources = [] +parsed_source_open_states = [] +real_package_validator = validate_pptx_package +real_presentation_loader = Presentation + + +def tracking_package_validator(source): + validated_sources.append(source) + return real_package_validator(source) + + +def tracking_presentation_loader(source): + parsed_sources.append(source) + parsed_source_open_states.append(not source.closed) + return real_presentation_loader(source) + + +validate_pptx_package = tracking_package_validator +Presentation = tracking_presentation_loader +try: + same_handle_prs = open_validated_presentation("input.pptx") +finally: + validate_pptx_package = real_package_validator + Presentation = real_presentation_loader +check("validated presentation parses the exact same open handle before closing it", + len(same_handle_prs.slides) == 1 + and len(validated_sources) == len(parsed_sources) == 1 + and validated_sources[0] is parsed_sources[0] + and parsed_source_open_states == [True] + and validated_sources[0].closed) # ---- edit.md snippet: single-shape run replace keeps styling and hyperlink ----- prs = Presentation("input.pptx") @@ -362,6 +648,56 @@ def iter_shapes(shapes): yield shape +OOXML_TRUE = {"1", "true"} +OOXML_FALSE = {"0", "false"} + + +def ooxml_bool(element, attribute, default): + value = element.get(attribute) + if value is None: + return default + value = value.strip(" \t\r\n") + if value in OOXML_TRUE: + return True + if value in OOXML_FALSE: + return False + raise ValueError(f"invalid OOXML boolean {attribute}={value!r}") + + +def shape_is_hidden(shape): + properties = shape._element.find(".//" + qn("p:cNvPr")) + return properties is not None and ooxml_bool(properties, "hidden", False) + + +def layer_text_content(shapes, source, *, inherited): + records = [] + for shape in shapes: + if shape_is_hidden(shape): + continue + if shape.shape_type == MSO_SHAPE_TYPE.GROUP: + records.extend(layer_text_content(shape.shapes, source, inherited=inherited)) + continue + if inherited and shape.is_placeholder: + continue + if shape.has_text_frame and shape.text_frame.text: + records.append({"source": source, "shape": shape.name, + "text": shape.text_frame.text}) + return records + + +def slide_text_content(slide): + records = [] + if ooxml_bool(slide._element, "showMasterSp", True): + layout = slide.slide_layout + if ooxml_bool(layout._element, "showMasterSp", True): + records.extend(layer_text_content( + layout.slide_master.shapes, "master", inherited=True, + )) + records.extend(layer_text_content(layout.shapes, "layout", inherited=True)) + records.extend(layer_text_content(slide.shapes, "slide", inherited=False)) + return records + + def table_cells(table): return [[ { @@ -455,10 +791,11 @@ def cached_text_point_map(container, point_count): index = int(point.get("idx")) except (TypeError, ValueError): return None - value = point.find(qn("c:v")) - if value is None or not 0 <= index < point_count or index in values: + value_nodes = point.findall(qn("c:v")) + if (len(value_nodes) != 1 or not 0 <= index < point_count + or index in values): return None - values[index] = value.text or "" + values[index] = value_nodes[0].text or "" return values @@ -608,22 +945,40 @@ def series_content(series, *, include_categories=False, point_budget=None): return content -def chart_axis_text(axis): - title = axis.find(qn("c:title")) +def title_text(title): if title is None: return "" - rich = title.xpath("./c:tx/c:rich") - references = title.xpath("./c:tx/c:strRef") - if len(rich) + len(references) != 1: + text_nodes = title.findall(qn("c:tx")) + if len(text_nodes) != 1: + return None + sources = [child for child in text_nodes[0] if isinstance(child.tag, str)] + if len(sources) != 1: return None - if rich: - return "\n".join(paragraph.text for paragraph in rich[0].findall(qn("a:p"))) - caches = references[0].findall(qn("c:strCache")) + if sources[0].tag == qn("c:rich"): + paragraphs = sources[0].findall(qn("a:p")) + return None if not paragraphs else "\n".join( + paragraph.text for paragraph in paragraphs + ) + if sources[0].tag != qn("c:strRef"): + return None + formulas = sources[0].findall(qn("c:f")) + if len(formulas) != 1 or not formulas[0].text: + return None + caches = sources[0].findall(qn("c:strCache")) if (len(caches) != 1 or (count := cache_point_count(caches[0])) is None or count != 1): return None values = cached_text_point_map(caches[0], count) - return None if values is None else "\n".join(values.get(index, "") for index in range(count)) + return None if values is None or set(values) != {0} else values[0] + + +def chart_axis_text(axis): + return title_text(axis.find(qn("c:title"))) + + +def chart_title_text(chart): + titles = chart._element.xpath("./c:chart/c:title") + return title_text(titles[0]) if len(titles) == 1 else ("" if not titles else None) def chart_axes(chart): @@ -677,7 +1032,7 @@ def extract_slide_content(slide, point_budget=None): shapes = list(iter_shapes(slide.shapes)) if point_budget is None: point_budget = {"remaining": MAX_CHART_POINTS} - text = [sh.text_frame.text for sh in shapes if sh.has_text_frame and sh.text_frame.text] + text = slide_text_content(slide) tables = [ table_cells(sh.table) for sh in shapes if sh.has_table @@ -687,10 +1042,7 @@ def extract_slide_content(slide, point_budget=None): if not sh.has_chart: continue chart = sh.chart - chart_title = ( - chart.chart_title.text_frame.text - if chart.has_title else "" - ) + chart_title = chart_title_text(chart) plots = [] for plot in chart.plots: items = list(plot.series) @@ -723,8 +1075,9 @@ def extract_slide_content(slide, point_budget=None): "pictures": pictures, "smartart": smartart, "notes": notes} -content = extract_slide_content(Presentation("input.pptx").slides[0]) -check("content inventory emits body text", any("old wording" in value for value in content["text"]), content) +content = extract_slide_content(open_validated_presentation("input.pptx").slides[0]) +check("content inventory emits body text", + any("old wording" in value["text"] for value in content["text"]), content) check("content inventory emits table cell text", content["tables"][0][0][1]["text"] == "old cell text", content["tables"]) check( @@ -743,6 +1096,296 @@ def extract_slide_content(slide, point_budget=None): and all(axis["id"] and axis["position"] and axis["cross_axis_id"] for axis in content["charts"][0]["axes"]), content["charts"][0]) +chart_title_prs = Presentation("input.pptx") +chart_title_chart = next( + shape.chart for shape in chart_title_prs.slides[0].shapes if shape.has_chart +) +chart_title_element = chart_title_chart._element.xpath("./c:chart/c:title")[0] +chart_title_tx = chart_title_element.find(qn("c:tx")) +for child in list(chart_title_tx): + chart_title_tx.remove(child) +chart_title_ref = OxmlElement("c:strRef") +chart_title_formula = OxmlElement("c:f") +chart_title_formula.text = "Sheet1!$G$1" +chart_title_cache = OxmlElement("c:strCache") +chart_title_count = OxmlElement("c:ptCount") +chart_title_count.set("val", "1") +chart_title_point = OxmlElement("c:pt") +chart_title_point.set("idx", "0") +chart_title_value = OxmlElement("c:v") +chart_title_value.text = "Cached chart title" +chart_title_point.append(chart_title_value) +chart_title_cache.extend([chart_title_count, chart_title_point]) +chart_title_ref.extend([chart_title_formula, chart_title_cache]) +chart_title_tx.append(chart_title_ref) +chart_title_prs.save("chart-title-strref.pptx") + +chart_title_reopened = Presentation("chart-title-strref.pptx") +raw_title_chart = next( + shape.chart for shape in chart_title_reopened.slides[0].shapes if shape.has_chart +) +raw_title_element = raw_title_chart._element.xpath("./c:chart/c:title")[0] +title_xml_before = etree.tostring(raw_title_element) +cached_chart_title = chart_title_text(raw_title_chart) +title_xml_after = etree.tostring(raw_title_element) +check("raw chart title reads a worksheet cache without mutating strRef", + cached_chart_title == "Cached chart title" + and title_xml_before == title_xml_after + and raw_title_element.xpath("./c:tx/c:strRef/c:f")[0].text == "Sheet1!$G$1" + and not raw_title_element.xpath("./c:tx/c:rich"), + cached_chart_title) +inventory_title_xml_before = etree.tostring(raw_title_element) +cached_title_inventory = extract_slide_content(chart_title_reopened.slides[0]) +inventory_title_xml_after = etree.tostring(raw_title_element) +check("full content inventory keeps a worksheet-backed chart title and its XML intact", + cached_title_inventory["charts"][0]["title"] == "Cached chart title" + and inventory_title_xml_before == inventory_title_xml_after + and raw_title_element.xpath("./c:tx/c:strRef/c:f")[0].text == "Sheet1!$G$1" + and not raw_title_element.xpath("./c:tx/c:rich"), + cached_title_inventory["charts"][0]) + +cacheless_chart_title = copy.deepcopy(raw_title_element) +cacheless_chart_title.xpath("./c:tx/c:strRef")[0].remove( + cacheless_chart_title.xpath("./c:tx/c:strRef/c:strCache")[0] +) +empty_chart_title = copy.deepcopy(raw_title_element) +empty_chart_title.xpath("./c:tx/c:strRef/c:strCache/c:pt/c:v")[0].text = None +ambiguous_chart_title = copy.deepcopy(raw_title_element) +ambiguous_chart_title.find(qn("c:tx")).append(OxmlElement("c:v")) +literal_only_chart_title = copy.deepcopy(raw_title_element) +literal_only_tx = literal_only_chart_title.find(qn("c:tx")) +literal_only_tx.remove(literal_only_tx.find(qn("c:strRef"))) +literal_only_value = OxmlElement("c:v") +literal_only_value.text = "invalid literal title" +literal_only_tx.append(literal_only_value) +foreign_namespace_title = copy.deepcopy(raw_title_element) +foreign_namespace_tx = foreign_namespace_title.find(qn("c:tx")) +foreign_namespace_ref = foreign_namespace_tx.find(qn("c:strRef")) +foreign_namespace_ref.tag = "{urn:foreign-chart-title}strRef" +empty_rich_title = copy.deepcopy(raw_title_element) +empty_rich_tx = empty_rich_title.find(qn("c:tx")) +empty_rich_tx.remove(empty_rich_tx.find(qn("c:strRef"))) +empty_rich_tx.append(OxmlElement("c:rich")) +duplicate_choice_title = copy.deepcopy(raw_title_element) +duplicate_choice_tx = duplicate_choice_title.find(qn("c:tx")) +duplicate_choice_tx.append(copy.deepcopy(duplicate_choice_tx.find(qn("c:strRef")))) +duplicate_cache_title = copy.deepcopy(raw_title_element) +duplicate_cache_ref = duplicate_cache_title.xpath("./c:tx/c:strRef")[0] +duplicate_cache_ref.append(copy.deepcopy(duplicate_cache_ref.find(qn("c:strCache")))) +missing_formula_title = copy.deepcopy(raw_title_element) +missing_formula_ref = missing_formula_title.xpath("./c:tx/c:strRef")[0] +missing_formula_ref.remove(missing_formula_ref.find(qn("c:f"))) +duplicate_formula_title = copy.deepcopy(raw_title_element) +duplicate_formula_ref = duplicate_formula_title.xpath("./c:tx/c:strRef")[0] +duplicate_formula_ref.append(copy.deepcopy(duplicate_formula_ref.find(qn("c:f")))) +missing_count_title = copy.deepcopy(raw_title_element) +missing_count_cache = missing_count_title.xpath("./c:tx/c:strRef/c:strCache")[0] +missing_count_cache.remove(missing_count_cache.find(qn("c:ptCount"))) +missing_point_title = copy.deepcopy(raw_title_element) +missing_point_cache = missing_point_title.xpath("./c:tx/c:strRef/c:strCache")[0] +missing_point_cache.remove(missing_point_cache.find(qn("c:pt"))) +missing_value_title = copy.deepcopy(raw_title_element) +missing_value_point = missing_value_title.xpath("./c:tx/c:strRef/c:strCache/c:pt")[0] +missing_value_point.remove(missing_value_point.find(qn("c:v"))) +duplicate_value_title = copy.deepcopy(raw_title_element) +duplicate_value_point = duplicate_value_title.xpath("./c:tx/c:strRef/c:strCache/c:pt")[0] +duplicate_value = OxmlElement("c:v") +duplicate_value.text = "second value" +duplicate_value_point.append(duplicate_value) + +rich_paragraph_title = copy.deepcopy(raw_title_element) +rich_paragraph_tx = rich_paragraph_title.find(qn("c:tx")) +rich_paragraph_tx.remove(rich_paragraph_tx.find(qn("c:strRef"))) +rich_title = OxmlElement("c:rich") +for text_value in ("First paragraph", "Second paragraph"): + paragraph = OxmlElement("a:p") + run = OxmlElement("a:r") + text_node = OxmlElement("a:t") + text_node.text = text_value + run.append(text_node) + paragraph.append(run) + rich_title.append(paragraph) +rich_paragraph_tx.append(rich_title) +check("chart title cache parser distinguishes unavailable, empty, and ambiguous choices", + title_text(cacheless_chart_title) is None + and title_text(empty_chart_title) == "" + and title_text(ambiguous_chart_title) is None + and title_text(literal_only_chart_title) is None + and title_text(foreign_namespace_title) is None + and title_text(empty_rich_title) is None + and title_text(duplicate_choice_title) is None + and title_text(duplicate_cache_title) is None + and title_text(missing_formula_title) is None + and title_text(duplicate_formula_title) is None + and title_text(missing_count_title) is None + and title_text(missing_point_title) is None + and title_text(missing_value_title) is None + and title_text(duplicate_value_title) is None + and title_text(rich_paragraph_title) == "First paragraph\nSecond paragraph") + + +def install_inherited_text(target_shapes, scratch_slide, text, *, grouped=False, hidden=False): + if grouped: + source_shape = scratch_slide.shapes.add_group_shape() + group_texts = (text,) if isinstance(text, str) else tuple(text) + for index, text_value in enumerate(group_texts): + child = source_shape.shapes.add_textbox( + Inches(1), Inches(1 + index), Inches(5), Inches(0.5) + ) + child.text = text_value + else: + source_shape = scratch_slide.shapes.add_textbox( + Inches(1), Inches(1), Inches(5), Inches(0.5) + ) + source_shape.text = text + copied = copy.deepcopy(source_shape._element) + next_shape_id = target_shapes._next_shape_id + for non_visual_properties in copied.iter(qn("p:cNvPr")): + non_visual_properties.set("id", str(next_shape_id)) + next_shape_id += 1 + if hidden: + copied.find(".//" + qn("p:cNvPr")).set("hidden", "1") + target_shapes._spTree.insert_element_before(copied, "p:extLst") + scratch_slide.shapes._spTree.remove(source_shape._element) + + +inheritance_prs = Presentation() +inheritance_layout = inheritance_prs.slide_layouts[5] +inheritance_slide = inheritance_prs.slides.add_slide(inheritance_layout) +inheritance_slide.shapes.title.text = "Actual slide title" +inheritance_layout_title = next( + shape for shape in inheritance_layout.placeholders + if shape.placeholder_format.type in (PP_PLACEHOLDER.TITLE, PP_PLACEHOLDER.CENTER_TITLE) +) +inheritance_layout_title.text = "TEMPLATE PLACEHOLDER PROMPT" +inheritance_master_placeholder = next(iter(inheritance_layout.slide_master.placeholders)) +inheritance_master_placeholder.text = "MASTER TEMPLATE PLACEHOLDER PROMPT" +install_inherited_text( + inheritance_layout.slide_master.shapes, inheritance_slide, "Master disclaimer" +) +install_inherited_text(inheritance_layout.shapes, inheritance_slide, "Layout disclaimer") +install_inherited_text( + inheritance_layout.shapes, inheritance_slide, + ("Grouped layout first", "Grouped layout second"), grouped=True, +) +install_inherited_text( + inheritance_layout.slide_master.shapes, inheritance_slide, + "Hidden master copy", hidden=True, +) +install_inherited_text( + inheritance_layout.shapes, inheritance_slide, + "Hidden grouped layout copy", grouped=True, hidden=True, +) +install_inherited_text( + inheritance_layout.shapes, inheritance_slide, "Whitespace-hidden layout copy" +) +whitespace_hidden_shape = next( + shape for shape in inheritance_layout.shapes + if shape.has_text_frame and shape.text_frame.text == "Whitespace-hidden layout copy" +) +whitespace_hidden_shape._element.find(".//" + qn("p:cNvPr")).set( + "hidden", " \ttrue\r\n" +) +hidden_slide_shape = inheritance_slide.shapes.add_textbox( + Inches(1), Inches(6), Inches(5), Inches(0.5) +) +hidden_slide_shape.text = "Hidden slide copy" +hidden_slide_shape._element.find(".//" + qn("p:cNvPr")).set("hidden", "1") +inheritance_prs.save("inherited-text.pptx") + +inherited_slide = open_validated_presentation("inherited-text.pptx").slides[0] +inherited_text = extract_slide_content(inherited_slide)["text"] +inherited_pairs = [(item["source"], item["text"]) for item in inherited_text] +inherited_master_ids = [ + node.get("id") for node in inherited_slide.slide_layout.slide_master._element.xpath( + ".//p:cNvPr" + ) +] +inherited_layout_ids = [ + node.get("id") for node in inherited_slide.slide_layout._element.xpath(".//p:cNvPr") +] +check("inherited-text fixture keeps unique non-visual shape ids per part", + len(inherited_master_ids) == len(set(inherited_master_ids)) + and len(inherited_layout_ids) == len(set(inherited_layout_ids)), + (inherited_master_ids, inherited_layout_ids)) +check("content inventory includes visible master and layout text with provenance", + inherited_pairs == [ + ("master", "Master disclaimer"), + ("layout", "Layout disclaimer"), + ("layout", "Grouped layout first"), + ("layout", "Grouped layout second"), + ("slide", "Actual slide title"), + ], + inherited_pairs) + +slide_hidden_prs = Presentation("inherited-text.pptx") +slide_hidden_prs.slides[0]._element.set("showMasterSp", "0") +slide_hidden_prs.save("slide-hides-inherited-text.pptx") +slide_hidden_pairs = [ + (item["source"], item["text"]) + for item in extract_slide_content( + open_validated_presentation("slide-hides-inherited-text.pptx").slides[0] + )["text"] +] +check("slide showMasterSp=false hides both layout and master copy", + ("slide", "Actual slide title") in slide_hidden_pairs + and not any(source in {"layout", "master"} for source, _ in slide_hidden_pairs), + slide_hidden_pairs) + +layout_hidden_prs = Presentation("inherited-text.pptx") +layout_hidden_prs.slides[0].slide_layout._element.set("showMasterSp", "false") +layout_hidden_prs.save("layout-hides-master-text.pptx") +layout_hidden_pairs = [ + (item["source"], item["text"]) + for item in extract_slide_content( + open_validated_presentation("layout-hides-master-text.pptx").slides[0] + )["text"] +] +check("layout showMasterSp=false hides only master copy", + ("layout", "Layout disclaimer") in layout_hidden_pairs + and ("slide", "Actual slide title") in layout_hidden_pairs + and not any(source == "master" for source, _ in layout_hidden_pairs), + layout_hidden_pairs) + +whitespace_false_prs = Presentation("inherited-text.pptx") +whitespace_false_prs.slides[0]._element.set("showMasterSp", " \t0\r\n") +whitespace_false_prs.save("whitespace-false-inherited-text.pptx") +whitespace_false_text = extract_slide_content( + open_validated_presentation("whitespace-false-inherited-text.pptx").slides[0] +)["text"] +check("xsd whitespace around false hides inherited layers", + all(item["source"] == "slide" for item in whitespace_false_text), + whitespace_false_text) + +whitespace_true_prs = Presentation("inherited-text.pptx") +whitespace_true_prs.slides[0]._element.set("showMasterSp", "\ttrue\n") +whitespace_true_prs.save("whitespace-true-inherited-text.pptx") +whitespace_true_pairs = [ + (item["source"], item["text"]) + for item in extract_slide_content( + open_validated_presentation("whitespace-true-inherited-text.pptx").slides[0] + )["text"] +] +check("xsd whitespace around true preserves inherited layers", + ("master", "Master disclaimer") in whitespace_true_pairs + and ("layout", "Layout disclaimer") in whitespace_true_pairs, + whitespace_true_pairs) + +invalid_visibility_values = ( + "maybe", "on", "off", "yes", "no", "TRUE", "False", "true false", +) +invalid_visibility_rejections = [] +for invalid_value in invalid_visibility_values: + invalid_visibility_slide = Presentation("inherited-text.pptx").slides[0] + invalid_visibility_slide._element.set("showMasterSp", invalid_value) + try: + extract_slide_content(invalid_visibility_slide) + except ValueError: + invalid_visibility_rejections.append(invalid_value) +check("non-xsd showMasterSp values fail closed under optimized Python", + invalid_visibility_rejections == list(invalid_visibility_values), + invalid_visibility_rejections) + secondary_axis_prs = Presentation("input.pptx") secondary_axis_chart = next( shape.chart for shape in secondary_axis_prs.slides[0].shapes diff --git a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py index 52b678f..bded508 100644 --- a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py @@ -368,6 +368,34 @@ def sheet_ref(sheet): from openpyxl.worksheet.table import Table +# ---- create.md chart: duplicate products are described as row-level sales ---- +row_chart_wb = openpyxl.Workbook() +row_chart_ws = row_chart_wb.active +row_chart_ws.title = "Sales" +row_chart_ws.append(["Region", "Product", "Revenue"]) +for row in (("EU", "Widget", 1140), ("EU", "Gadget", 1680), ("US", "Widget", 1900)): + row_chart_ws.append(row) +row_chart = BarChart() +row_chart.type = "col" +row_chart.title = "Revenue by transaction row" +row_chart.add_data(Reference(row_chart_ws, min_col=3, min_row=1, max_row=4), + titles_from_data=True) +row_chart.set_categories(Reference(row_chart_ws, min_col=2, min_row=2, max_row=4)) +row_chart_ws.add_chart(row_chart, "E2") +row_chart_wb.save("row-level-sales-chart.xlsx") +row_chart_reopened = openpyxl.load_workbook("row-level-sales-chart.xlsx", data_only=False) +reopened_row_chart = row_chart_reopened["Sales"]._charts[0] +row_chart_title = reopened_row_chart.title.tx.rich.p[0].r[0].t +check("duplicate product labels are explicitly charted per transaction row", + [row_chart_reopened["Sales"][f"B{row}"].value for row in range(2, 5)].count("Widget") == 2 + and row_chart_title == "Revenue by transaction row" + and reopened_row_chart.ser[0].cat.numRef.f == "'Sales'!$B$2:$B$4" + and reopened_row_chart.ser[0].val.numRef.f == "'Sales'!$C$2:$C$4" + and reopened_row_chart.ser[0].tx.strRef.f == "'Sales'!C1", + row_chart_title) +row_chart_reopened.close() + + def formula_text(value): if isinstance(value, str): return value @@ -397,17 +425,26 @@ def drawing_anchor_rows(drawing): return tuple(rows) +def sparse_formula_cells(sheet): + if not hasattr(sheet, "_cells"): + raise RuntimeError("structural edits require a normal writable Worksheet") + return ( + cell for cell in sorted( + sheet._cells.values(), key=lambda cell: (cell.row, cell.column) + ) + if cell.data_type == "f" + ) + + def structural_references(workbook): refs = [] for item in defined_name_values(workbook): refs.append(("defined name", item.name, item.attr_text)) for sheet in workbook.worksheets: owner = sheet.title - for row in sheet.iter_rows(): - for cell in row: - if cell.data_type == "f": - refs.append(("cell formula", f"{owner}!{cell.coordinate}", - formula_text(cell.value))) + for cell in sparse_formula_cells(sheet): + refs.append(("cell formula", f"{owner}!{cell.coordinate}", + formula_text(cell.value))) for table in sheet.tables.values(): refs.append(("table", owner + "!" + table.name, table.ref)) for merged_range in sheet.merged_cells.ranges: @@ -444,16 +481,14 @@ def structural_references(workbook): def cell_formula_references(workbook): refs = [] for sheet in workbook.worksheets: - for row in sheet.iter_rows(): - for cell in row: - if cell.data_type == "f": - value = cell.value - refs.append(( - "cell formula", - sheet.title, - cell.coordinate, - formula_text(value), - )) + for cell in sparse_formula_cells(sheet): + value = cell.value + refs.append(( + "cell formula", + sheet.title, + cell.coordinate, + formula_text(value), + )) return refs @@ -551,6 +586,30 @@ class LegacyWorkbook: formula_references == [("cell formula", "Audit", "C1", "=SUM(A2:A3)")], formula_references, ) + +sparse_scan_wb = openpyxl.Workbook() +sparse_scan_ws = sparse_scan_wb.active +sparse_scan_ws.title = "Sparse" +sparse_scan_ws["D2"] = "=1+1" +sparse_scan_ws["XFD1048576"].number_format = "0.00" +sparse_scan_wb.save("sparse-structural-scan.xlsx") +sparse_scan_wb.close() +sparse_scan_wb = openpyxl.load_workbook("sparse-structural-scan.xlsx", data_only=False) +sparse_scan_ws = sparse_scan_wb["Sparse"] +original_sparse_iter_rows = sparse_scan_ws.iter_rows +sparse_scan_ws.iter_rows = lambda *args, **kwargs: (_ for _ in ()).throw( + RuntimeError("rectangular formula scan must not run") +) +try: + sparse_structural = structural_references(sparse_scan_wb) + sparse_formulas = cell_formula_references(sparse_scan_wb) +finally: + sparse_scan_ws.iter_rows = original_sparse_iter_rows +check("structural formula inventory walks sparse cells at worksheet limits", + ("cell formula", "Sparse!D2", "=1+1") in sparse_structural + and sparse_formulas == [("cell formula", "Sparse", "D2", "=1+1")], + (sparse_structural, sparse_formulas)) +sparse_scan_wb.close() check("intersecting formula ranges are blocked before row insertion", formula_may_intersect_rows("Audit", "=SUM(A2:A3)", "Audit", 3)) check("audited formula ranges above the insertion can proceed", From 93d0ace6ed50b33cd1074b4706a7ca87611f4fac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=98=E6=89=AC?= Date: Tue, 18 Aug 2026 00:04:20 +0800 Subject: [PATCH 30/34] fix(document-skills): harden content inventory routes --- .../skills/pdf/references/inspect.md | 46 ++- .../skills/pptx/references/analyze.md | 77 +++-- .../skills/pptx/references/edit.md | 138 +++++++- .../document-skills/tests/pdf_fixture.py | 126 +++++++- .../document-skills/tests/pptx_fixture.py | 300 +++++++++++++++++- 5 files changed, 618 insertions(+), 69 deletions(-) diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md b/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md index acbb562..b5d33a2 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md @@ -12,10 +12,18 @@ if doc.needs_pass: raise RuntimeError("Encrypted PDF: set a valid PDF_PASSWORD before inspection") def font_inventory(document, page): - """Distinguish fonts with extractable programs from referenced-only faces.""" + """Distinguish font files, self-contained Type3 glyphs, and referenced-only faces.""" fonts = [] for entry in page.get_fonts(full=True): xref, extension, font_type, base_name, resource_name, encoding = entry[:6] + is_type3 = font_type.replace(" ", "").casefold() == "type3" + type3_self_contained = False + if is_type3 and xref > 0: + try: + charprocs_type, _ = document.xref_get_key(xref, "CharProcs") + type3_self_contained = charprocs_type in {"dict", "xref"} + except (RuntimeError, ValueError): + type3_self_contained = False embedded_bytes = 0 if xref > 0: try: @@ -30,8 +38,13 @@ def font_inventory(document, page): "type": font_type, "encoding": encoding, "extension": extension, - "embedded": embedded_bytes > 0, + "embedded": type3_self_contained or embedded_bytes > 0, "embedded_bytes": embedded_bytes, + "self_contained": type3_self_contained, + "program_source": ( + "type3-charprocs" if type3_self_contained else + ("font-file" if embedded_bytes else None) + ), }) return fonts @@ -136,18 +149,20 @@ for page in doc: "crop_size": crop_size, "rotation": page.rotation, }) - blocks = page.get_text("dict")["blocks"] - image_blocks = [block for block in blocks if block["type"] == 1] + visible_images = [ + info for info in page.get_image_info() + if visible_clip(page, info.get("bbox")) is not None + ] drawings = page.get_drawings() widgets, annotations, links, interaction_visibility_unknown = viewable_interactives(page) is_blank = not ( - page.get_text().strip() or page.get_images() or image_blocks or drawings + page.get_text().strip() or visible_images or drawings or widgets or annotations or links or interaction_visibility_unknown ) print(page.number + 1, "media_size:", media_size, "crop_size:", crop_size, "rotation:", page.rotation, "text_len:", len(page.get_text()), "resource_images:", len(page.get_images()), - "image_blocks:", len(image_blocks), "drawings:", len(drawings), + "visible_image_placements:", len(visible_images), "drawings:", len(drawings), "widgets:", len(widgets), "annotations:", len(annotations), "links:", len(links), "interaction_visibility_unknown:", interaction_visibility_unknown, "blank:", is_blank) @@ -158,17 +173,21 @@ print("crop_size_consistent:", len({row["crop_size"] for row in page_geometry}) ## Checks worth automating -- **Blank page detection**: flag only when text, resource images, type-1 image blocks, drawings, +- **Blank page detection**: flag only when text, visible painted image placements, drawings, and viewable widgets, annotations, and links are all absent. Ignore interactive objects carrying invisible, hidden, or no-view flags, as well as empty, off-page, or unrendered appearances. - `page.get_images()` lists image XObjects but - misses images embedded inline in the content stream; type-1 blocks from `get_text("dict")` - cover both inline and XObject image placements. Interactive form fields are widgets rather + `page.get_images()` lists every image XObject resource, including unused resources, and also + misses images embedded inline in the content stream. `page.get_image_info()` reports painted + inline and XObject placements without loading their bytes; retain only finite, nonempty placements + that intersect the rotated page via `visible_clip()`. Interactive form fields are widgets rather than page text, so a three-content-stream predicate alone would misclassify a usable form page as blank. A blank page after generation usually means an overflowing flowable created it. - **Font inventory**: `page.get_fonts()` lists referenced fonts, including non-embedded base - fonts. Use `doc.extract_font(xref)` as above and report `embedded` separately; a referenced - face with no extractable program may be substituted on another machine. + fonts. Use `doc.extract_font(xref)` as above for conventional font files. Type3 fonts are a + separate self-contained case: when the font dictionary has a `/CharProcs` dictionary, its glyph + programs are PDF content streams, so report `self_contained=True` and + `program_source="type3-charprocs"` even when no conventional font-file bytes extract. + Other referenced faces with no extractable program may be substituted on another machine. - **Page size consistency**: compare unrotated `(width, height)` pairs from `page.mediabox` and `page.cropbox`, and report `page.rotation` separately. Do not compare `page.rect`: it applies `/Rotate`, so otherwise identical paper appears to swap width and height at 90 or 270 degrees. @@ -177,5 +196,6 @@ print("crop_size_consistent:", len({row["crop_size"] for row in page_geometry}) `pypdf.PdfReader` cross-check when provenance is unknown. Report findings as a table (page, media size, crop size, rotation, text chars, resource images, -image blocks, drawings, widgets, annotations, links, interaction visibility unknown, blank) - it +visible image placements, drawings, widgets, annotations, links, interaction +visibility unknown, blank) - it is what every downstream decision hangs off. diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md index 08a46fa..71171dc 100644 --- a/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/analyze.md @@ -135,14 +135,6 @@ def open_validated_presentation(path): source.seek(0) return Presentation(source) -def iter_shapes(shapes): - """Walk shapes recursively so content nested inside group shapes is counted too.""" - for shape in shapes: - if shape.shape_type == MSO_SHAPE_TYPE.GROUP: - yield from iter_shapes(shape.shapes) - else: - yield shape - OOXML_TRUE = {"1", "true"} OOXML_FALSE = {"0", "false"} @@ -161,6 +153,16 @@ def shape_is_hidden(shape): properties = shape._element.find(".//" + qn("p:cNvPr")) return properties is not None and ooxml_bool(properties, "hidden", False) +def iter_shapes(shapes): + """Yield visible leaf shapes, propagating a hidden group's visibility to its children.""" + for shape in shapes: + if shape_is_hidden(shape): + continue + if shape.shape_type == MSO_SHAPE_TYPE.GROUP: + yield from iter_shapes(shape.shapes) + else: + yield shape + def layer_text_content(shapes, source, *, inherited): """Collect visible text from one layer; inherited placeholders are template prompts.""" records = [] @@ -210,16 +212,27 @@ def table_cells(table): def picture_content(shape): """Inventory ordinary pictures and populated picture placeholders alike.""" + if getattr(type(shape), "image", None) is None: + return None try: image = shape.image - except (AttributeError, ValueError): - return None - return { - "name": shape.name, - "filename": image.filename, - "extension": image.ext, - "bytes": len(image.blob), - } + return { + "name": shape.name, + "filename": image.filename, + "extension": image.ext, + "bytes": len(image.blob), + } + except (AttributeError, KeyError, OSError, ValueError): + blips = shape._element.xpath(".//a:blip") + relationship_id = ( + blips[0].get(qn("r:embed")) if len(blips) == 1 else None + ) + return { + "name": shape.name, + "status": "unreadable", + "relationship_id": relationship_id, + "reason": "missing or invalid image relationship or payload", + } MAX_CHART_POINTS = 100_000 MAX_CATEGORY_LEVELS = 16 @@ -254,6 +267,8 @@ def cached_numeric_points( caches = source.xpath("./c:numRef/c:numCache | ./c:numLit") if len(caches) != 1 or (point_count := cache_point_count(caches[0])) is None: return None + if not consume_point_budget(point_budget, point_count): + return None cached = {} for point in caches[0].findall(qn("c:pt")): value = point.find(qn("c:v")) @@ -269,8 +284,6 @@ def cached_numeric_points( parsed = value.text # preserve #N/A and other error markers cached[index] = parsed if fill_missing: - if not consume_point_budget(point_budget, point_count): - return None points = [(index, cached.get(index)) for index in range(point_count)] else: points = sorted(cached.items()) @@ -412,9 +425,12 @@ def series_content(series, *, include_categories=False, point_budget=None): if value_cache is None: return {**content, "values": None, "cache_status": "unavailable"} return {**content, "values": [value for _, value in value_cache["points"]]} - x_cache = cached_numeric_points(x_source, include_count=True) + x_cache = cached_numeric_points( + x_source, include_count=True, point_budget=point_budget + ) y_cache = cached_numeric_points( - getattr(series._element, "yVal", None), include_count=True + getattr(series._element, "yVal", None), include_count=True, + point_budget=point_budget, ) if x_cache is None or y_cache is None: return {**content, "points": None, "cache_status": "unavailable"} @@ -424,7 +440,9 @@ def series_content(series, *, include_categories=False, point_budget=None): }) size_source = getattr(series._element, "bubbleSize", None) if size_source is not None: - bubble_cache = cached_numeric_points(size_source, include_count=True) + bubble_cache = cached_numeric_points( + size_source, include_count=True, point_budget=point_budget + ) if bubble_cache is None: return {**content, "points": None, "cache_status": "unavailable"} content.update({ @@ -515,9 +533,13 @@ print("slide size:", prs.slide_width, prs.slide_height) point_budget = {"remaining": MAX_CHART_POINTS} for i, slide in enumerate(prs.slides): layout = slide.slide_layout.name - title = slide.shapes.title.text_frame.text if slide.shapes.title is not None else "" + title_shape = slide.shapes.title + title = ( + title_shape.text_frame.text + if title_shape is not None and not shape_is_hidden(title_shape) else "" + ) notes = slide.notes_slide.notes_text_frame.text if slide.has_notes_slide else "" - shapes = list(iter_shapes(slide.shapes)) # flattened; groups are common in template decks + shapes = list(iter_shapes(slide.shapes)) # flattened and visible; hidden groups hide children text = slide_text_content(slide) tables = [ table_cells(sh.table) @@ -603,17 +625,10 @@ what you can and name the fallback explicitly: ```python import xml.etree.ElementTree as ET -from pptx.enum.shapes import MSO_SHAPE_TYPE from pptx.opc.constants import RELATIONSHIP_TYPE as RT from pptx.oxml.ns import qn -def iter_shapes(shapes): - """Local recursive walker for the font-triage continuation.""" - for shape in shapes: - if shape.shape_type == MSO_SHAPE_TYPE.GROUP: - yield from iter_shapes(shape.shapes) - else: - yield shape +# Reuse the visibility-aware iter_shapes() walker from the content inventory above. # 1. Resolve the theme related to each slide's own layout/master. A package can contain # multiple masters with different themes, so the first /ppt/theme/* part is not a safe default. diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md b/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md index c6fcf74..9f6820d 100644 --- a/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md @@ -2,21 +2,149 @@ ## Locate by content, then edit narrowly +The self-contained route below uses the same bounded `validate_pptx_package()` and +`open_validated_presentation()` preflight as `analyze.md`. Keep both copies in sync. The loader +validates and parses one open file identity; do not replace it with `Presentation(path)`, which +decompresses the package before the bounds run. + ```python +import zipfile +from pathlib import Path + +from lxml import etree from pptx import Presentation from pptx.enum.shapes import MSO_SHAPE_TYPE -prs = Presentation("input.pptx") +CONTENT_TYPES_NS = "http://schemas.openxmlformats.org/package/2006/content-types" +SAFE_XML = etree.XMLParser( + load_dtd=False, resolve_entities=False, no_network=True, + huge_tree=False, recover=False, +) +MAX_ARCHIVE_BYTES = 200 * 1024 * 1024 +MAX_MEMBERS = 10_000 +MAX_XML_PART = 20 * 1024 * 1024 +MAX_ENTRY = 100 * 1024 * 1024 +MAX_TOTAL_UNCOMPRESSED = 500 * 1024 * 1024 +MAX_COMPRESSION_RATIO = 200 + +def require(condition, message): + """Security and edit gates must remain active under python -O.""" + if not condition: + raise ValueError(message) + +def validate_pptx_package(source): + """Apply the same bounded OPC preflight used by the analysis route.""" + source.seek(0, 2) + compressed_size = source.tell() + source.seek(0) + require(compressed_size <= MAX_ARCHIVE_BYTES, "compressed PPTX file size above limit") + with zipfile.ZipFile(source) as archive: + infos = archive.infolist() + require(len(infos) <= MAX_MEMBERS, "archive member count above limit") + names = {info.filename for info in infos} + require(len(names) == len(infos), "duplicate archive member names are unsafe") + require("[Content_Types].xml" in names and "ppt/presentation.xml" in names, + "missing required OPC members") + require(sum(info.file_size for info in infos) <= MAX_TOTAL_UNCOMPRESSED, + "declared total uncompressed size above limit") + + content_types_info = next( + info for info in infos if info.filename == "[Content_Types].xml" + ) + require(content_types_info.file_size <= MAX_XML_PART, + "oversized XML part: [Content_Types].xml") + require( + content_types_info.file_size / max(content_types_info.compress_size, 1) + <= MAX_COMPRESSION_RATIO, + "suspicious compression ratio: [Content_Types].xml", + ) + content_type_chunks = [] + content_type_size = 0 + with archive.open(content_types_info) as stream: + while chunk := stream.read(64 * 1024): + content_type_size += len(chunk) + require(content_type_size <= MAX_XML_PART, + "part exceeded read limit: [Content_Types].xml") + content_type_chunks.append(chunk) + require(content_type_size == content_types_info.file_size, + "size mismatch: [Content_Types].xml") + content_types_blob = b"".join(content_type_chunks) + content_types_root = etree.fromstring(content_types_blob, parser=SAFE_XML) + require(content_types_root.tag == f"{{{CONTENT_TYPES_NS}}}Types", + "invalid content-types root") + default_types = {} + override_types = {} + for declaration in content_types_root: + if declaration.tag == f"{{{CONTENT_TYPES_NS}}}Default": + key = (declaration.get("Extension") or "").casefold() + target = default_types + elif declaration.tag == f"{{{CONTENT_TYPES_NS}}}Override": + part_name = declaration.get("PartName") or "" + require(part_name.startswith("/"), "invalid content-type part name") + key = part_name[1:].casefold() + target = override_types + else: + continue + content_type = declaration.get("ContentType") or "" + require(key and content_type and key not in target, + "invalid or duplicate content-type declaration") + target[key] = content_type.partition(";")[0].strip().casefold() + + actual_total = 0 + for info in infos: + require(info.file_size <= MAX_ENTRY, f"oversized part: {info.filename}") + ratio = info.file_size / max(info.compress_size, 1) + require(ratio <= MAX_COMPRESSION_RATIO, + f"suspicious compression ratio: {info.filename}") + extension = info.filename.rpartition(".")[2].casefold() + content_type = override_types.get( + info.filename.casefold(), default_types.get(extension, "") + ) + is_xml = ( + info.filename.casefold().endswith((".xml", ".rels")) + or content_type in {"application/xml", "text/xml"} + or content_type.endswith("+xml") + ) + if is_xml: + require(info.file_size <= MAX_XML_PART, f"oversized XML part: {info.filename}") + chunks = [] + actual_size = 0 + if info.filename == "[Content_Types].xml": + chunks = [content_types_blob] + actual_size = len(content_types_blob) + actual_total += actual_size + else: + with archive.open(info) as stream: + while chunk := stream.read(64 * 1024): + actual_size += len(chunk) + actual_total += len(chunk) + require(actual_size <= MAX_ENTRY, + f"part exceeded read limit: {info.filename}") + require(actual_total <= MAX_TOTAL_UNCOMPRESSED, + "archive exceeded total read limit") + if is_xml: + chunks.append(chunk) + require(actual_total <= MAX_TOTAL_UNCOMPRESSED, + "archive exceeded total read limit") + require(actual_size == info.file_size, f"size mismatch: {info.filename}") + if is_xml: + etree.fromstring(b"".join(chunks), parser=SAFE_XML) + source.seek(0) + +def open_validated_presentation(path): + """Preflight and parse one immutable open-file identity, then release the handle.""" + with Path(path).open("rb") as source: + validate_pptx_package(source) + source.seek(0) + return Presentation(source) + +prs = open_validated_presentation("input.pptx") old, new = "old wording", "new wording" slide_index = None # Set this and shape_name when repeated text is expected. shape_name = None target_location = None # e.g. "Table 1/table[0,1]" for duplicate table text -def require(condition, message): - if not condition: - raise ValueError(message) - def iter_shapes(shapes, path=""): """Yield (path, shape) for every shape, recursing into groups so text inside grouped artwork is reachable; the path keeps the uniqueness check readable.""" diff --git a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py index 0f106b1..e94e5df 100644 --- a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py @@ -12,6 +12,10 @@ import fitz import pypdf import reportlab +from pypdf.generic import ( + ArrayObject, DecodedStreamObject, DictionaryObject, FloatObject, + NameObject, NumberObject, +) from reportlab.lib.pagesizes import A4 from reportlab.pdfbase import pdfmetrics from reportlab.pdfbase.ttfonts import TTFont @@ -129,6 +133,14 @@ def font_inventory(document, page): fonts = [] for entry in page.get_fonts(full=True): xref, extension, font_type, base_name, resource_name, encoding = entry[:6] + is_type3 = font_type.replace(" ", "").casefold() == "type3" + type3_self_contained = False + if is_type3 and xref > 0: + try: + charprocs_type, _ = document.xref_get_key(xref, "CharProcs") + type3_self_contained = charprocs_type in {"dict", "xref"} + except (RuntimeError, ValueError): + type3_self_contained = False embedded_bytes = 0 if xref > 0: try: @@ -136,8 +148,14 @@ def font_inventory(document, page): except (RuntimeError, ValueError): embedded_bytes = 0 fonts.append({ - "base_name": base_name, "embedded": embedded_bytes > 0, + "base_name": base_name, "type": font_type, + "embedded": type3_self_contained or embedded_bytes > 0, "embedded_bytes": embedded_bytes, + "self_contained": type3_self_contained, + "program_source": ( + "type3-charprocs" if type3_self_contained else + ("font-file" if embedded_bytes else None) + ), }) return fonts @@ -148,6 +166,67 @@ def font_inventory(document, page): check("font inventory labels referenced-only Helvetica as non-embedded", any("Helvetica" in item["base_name"] and not item["embedded"] for item in fonts), fonts) +# A Type3 font stores each glyph as PDF content in /CharProcs rather than as an +# extractable conventional font file. +type3_writer = pypdf.PdfWriter() +type3_page = type3_writer.add_blank_page(width=612, height=792) +type3_glyph = DecodedStreamObject() +type3_glyph.set_data(b"500 0 0 0 500 700 d1 0 0 500 700 re f") +type3_glyph_ref = type3_writer._add_object(type3_glyph) +type3_charprocs_ref = type3_writer._add_object(DictionaryObject({ + NameObject("/A"): type3_glyph_ref, +})) +type3_encoding = DictionaryObject({ + NameObject("/Type"): NameObject("/Encoding"), + NameObject("/Differences"): ArrayObject([ + NumberObject(65), NameObject("/A"), + ]), +}) +type3_font = DictionaryObject({ + NameObject("/Type"): NameObject("/Font"), + NameObject("/Subtype"): NameObject("/Type3"), + NameObject("/Name"): NameObject("/FType3"), + NameObject("/FontBBox"): ArrayObject([ + NumberObject(0), NumberObject(0), NumberObject(500), NumberObject(700), + ]), + NameObject("/FontMatrix"): ArrayObject([ + FloatObject(0.001), NumberObject(0), NumberObject(0), + FloatObject(0.001), NumberObject(0), NumberObject(0), + ]), + NameObject("/CharProcs"): type3_charprocs_ref, + NameObject("/Encoding"): type3_encoding, + NameObject("/FirstChar"): NumberObject(65), + NameObject("/LastChar"): NumberObject(65), + NameObject("/Widths"): ArrayObject([NumberObject(500)]), + NameObject("/Resources"): DictionaryObject(), +}) +type3_font_ref = type3_writer._add_object(type3_font) +type3_page[NameObject("/Resources")] = DictionaryObject({ + NameObject("/Font"): DictionaryObject({NameObject("/FType3"): type3_font_ref}), +}) +type3_content = DecodedStreamObject() +type3_content.set_data(b"BT /FType3 72 Tf 72 700 Td (A) Tj ET") +type3_page[NameObject("/Contents")] = type3_writer._add_object(type3_content) +with open("type3-font.pdf", "wb") as output: + type3_writer.write(output) +type3_doc = fitz.open("type3-font.pdf") +type3_entry = next(item for item in type3_doc[0].get_fonts(full=True) + if item[2].replace(" ", "").casefold() == "type3") +type3_extracted_bytes = type3_doc.extract_font(type3_entry[0])[3] or b"" +type3_fonts = font_inventory(type3_doc, type3_doc[0]) +type3_record = next(item for item in type3_fonts + if item["type"].replace(" ", "").casefold() == "type3") +check("Type3 negative control has no conventional extractable font-file bytes", + type3_extracted_bytes == b"", len(type3_extracted_bytes)) +check("Type3 glyph program renders and extracts its encoded character", + type3_doc[0].get_text().strip() == "A" + and min(type3_doc[0].get_pixmap(alpha=False).samples) < 250) +check("font inventory classifies Type3 CharProcs as self-contained content", + type3_record["embedded"] and type3_record["self_contained"] + and type3_record["embedded_bytes"] == 0 + and type3_record["program_source"] == "type3-charprocs", + type3_record) + # ---- SKILL.md postcheck: encrypted output is reopened with its password ------- encrypted_writer = pypdf.PdfWriter() encrypted_writer.append(r) @@ -854,12 +933,12 @@ def stamp_placement(page, stamp_box): def inspected_page_is_blank(page): - image_blocks = [ - block for block in page.get_text("dict")["blocks"] - if block["type"] == 1 + visible_images = [ + info for info in page.get_image_info() + if visible_clip(page, info.get("bbox")) is not None ] return not ( - page.get_text().strip() or page.get_images() or image_blocks + page.get_text().strip() or visible_images or page.get_drawings() or list(page.widgets() or ()) or list(page.annots() or ()) or page.get_links() ) @@ -924,6 +1003,43 @@ def inspected_page_is_blank(page): xobject_canvas.showPage() xobject_canvas.save() +# Editing can remove the only Do operation while leaving the image in /Resources. +# The resource inventory is then nonempty, but no image is painted. +unused_image_writer = pypdf.PdfWriter() +unused_image_writer.append(pypdf.PdfReader("xobject-image.pdf")) +empty_page_content = DecodedStreamObject() +empty_page_content.set_data(b"") +unused_image_writer.pages[0][NameObject("/Contents")] = ( + unused_image_writer._add_object(empty_page_content) +) +with open("unused-image-resource.pdf", "wb") as output: + unused_image_writer.write(output) +unused_image_doc = fitz.open("unused-image-resource.pdf") +unused_image_page = unused_image_doc[0] +check("unused image fixture retains an XObject resource (negative control)", + bool(unused_image_page.get_images()), unused_image_page.get_images()) +check("unused image resource has no painted placement", + unused_image_page.get_image_info(xrefs=True) == [], + unused_image_page.get_image_info(xrefs=True)) +check("unused image resource does not exempt an otherwise blank rendered page", + inspected_page_is_blank(unused_image_page) + and min(unused_image_page.get_pixmap(alpha=False).samples) == 255) + +# Placement geometry, rather than mere presence in the content stream, decides visibility. +placement_canvas = canvas.Canvas("image-placement-visibility.pdf", pagesize=(200, 200)) +placement_canvas.drawImage("inline-only-source.png", -40, 80, width=80, height=80) +placement_canvas.showPage() +placement_canvas.drawImage("inline-only-source.png", 250, 80, width=80, height=80) +placement_canvas.showPage() +placement_canvas.save() +placement_doc = fitz.open("image-placement-visibility.pdf") +check("partly intersecting image placement keeps a page nonblank", + not inspected_page_is_blank(placement_doc[0]), + placement_doc[0].get_image_info(xrefs=True)) +check("fully off-page image placement does not exempt a blank page", + inspected_page_is_blank(placement_doc[1]), + placement_doc[1].get_image_info(xrefs=True)) + def extract_images_without_duplicates(document, page, prefix): outputs = [] diff --git a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py index 12c0f47..914b9e3 100644 --- a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py @@ -570,8 +570,28 @@ def tracking_presentation_loader(source): and parsed_source_open_states == [True] and validated_sources[0].closed) -# ---- edit.md snippet: single-shape run replace keeps styling and hyperlink ----- -prs = Presentation("input.pptx") +# ---- edit.md snippet: validated single-shape run replace keeps styling/link ---- +edit_loader_calls = [] +real_presentation_loader = Presentation + + +def unexpected_edit_loader(source): + edit_loader_calls.append(source) + return real_presentation_loader(source) + + +Presentation = unexpected_edit_loader +try: + open_validated_presentation("compressed-bomb.pptx") + edit_bomb_rejected_before_parse = False +except ValueError: + edit_bomb_rejected_before_parse = not edit_loader_calls +finally: + Presentation = real_presentation_loader +check("edit route rejects a package bomb before Presentation parses it", + edit_bomb_rejected_before_parse, edit_loader_calls) + +prs = open_validated_presentation("input.pptx") old, new = "old wording", "new wording" candidates = [] for i, s in enumerate(prs.slides): @@ -642,6 +662,8 @@ def tracking_presentation_loader(source): def iter_shapes(shapes): for shape in shapes: + if shape_is_hidden(shape): + continue if shape.shape_type == MSO_SHAPE_TYPE.GROUP: yield from iter_shapes(shape.shapes) else: @@ -714,16 +736,27 @@ def table_cells(table): def picture_content(shape): + if getattr(type(shape), "image", None) is None: + return None try: image = shape.image - except (AttributeError, ValueError): - return None - return { - "name": shape.name, - "filename": image.filename, - "extension": image.ext, - "bytes": len(image.blob), - } + return { + "name": shape.name, + "filename": image.filename, + "extension": image.ext, + "bytes": len(image.blob), + } + except (AttributeError, KeyError, OSError, ValueError): + blips = shape._element.xpath(".//a:blip") + relationship_id = ( + blips[0].get(qn("r:embed")) if len(blips) == 1 else None + ) + return { + "name": shape.name, + "status": "unreadable", + "relationship_id": relationship_id, + "reason": "missing or invalid image relationship or payload", + } MAX_CHART_POINTS = 100_000 @@ -761,6 +794,8 @@ def cached_numeric_points( caches = source.xpath("./c:numRef/c:numCache | ./c:numLit") if len(caches) != 1 or (point_count := cache_point_count(caches[0])) is None: return None + if not consume_point_budget(point_budget, point_count): + return None cached = {} for point in caches[0].findall(qn("c:pt")): value = point.find(qn("c:v")) @@ -776,8 +811,6 @@ def cached_numeric_points( parsed = value.text cached[index] = parsed if fill_missing: - if not consume_point_budget(point_budget, point_count): - return None points = [(index, cached.get(index)) for index in range(point_count)] else: points = sorted(cached.items()) @@ -923,9 +956,12 @@ def series_content(series, *, include_categories=False, point_budget=None): if value_cache is None: return {**content, "values": None, "cache_status": "unavailable"} return {**content, "values": [value for _, value in value_cache["points"]]} - x_cache = cached_numeric_points(x_source, include_count=True) + x_cache = cached_numeric_points( + x_source, include_count=True, point_budget=point_budget + ) y_cache = cached_numeric_points( - getattr(series._element, "yVal", None), include_count=True + getattr(series._element, "yVal", None), include_count=True, + point_budget=point_budget, ) if x_cache is None or y_cache is None: return {**content, "points": None, "cache_status": "unavailable"} @@ -935,7 +971,9 @@ def series_content(series, *, include_categories=False, point_budget=None): }) size_source = getattr(series._element, "bubbleSize", None) if size_source is not None: - bubble_cache = cached_numeric_points(size_source, include_count=True) + bubble_cache = cached_numeric_points( + size_source, include_count=True, point_budget=point_budget + ) if bubble_cache is None: return {**content, "points": None, "cache_status": "unavailable"} content.update({ @@ -1885,6 +1923,179 @@ def name(self): and placeholder_inventory[0]["bytes"] == len(placeholder_png), placeholder_inventory) +# A dangling r:embed must produce an unreadable record and allow later slides to inventory. +broken_picture_source = Presentation("picture-placeholder.pptx") +later_picture_slide = broken_picture_source.slides.add_slide( + broken_picture_source.slide_layouts[6] +) +later_picture_slide.shapes.add_picture( + "placeholder-picture.png", Inches(1), Inches(1), Inches(1), Inches(1) +) +broken_picture_source.save("broken-picture-source.pptx") +with zipfile.ZipFile("broken-picture-source.pptx") as source_archive: + broken_picture_payload = { + info.filename: source_archive.read(info) for info in source_archive.infolist() + } +broken_slide_root = etree.fromstring(broken_picture_payload["ppt/slides/slide1.xml"]) +broken_blips = broken_slide_root.findall(".//" + qn("a:blip")) +require(len(broken_blips) == 1, "broken-picture fixture expected one slide-1 image") +broken_relationship_id = broken_blips[0].get(qn("r:embed")) +relationships_path = "ppt/slides/_rels/slide1.xml.rels" +relationships_root = etree.fromstring(broken_picture_payload[relationships_path]) +package_relationship_ns = "http://schemas.openxmlformats.org/package/2006/relationships" +broken_relationships = [ + relation for relation in relationships_root.findall( + f"{{{package_relationship_ns}}}Relationship" + ) + if relation.get("Id") == broken_relationship_id +] +require(len(broken_relationships) == 1, + "broken-picture fixture expected one matching relationship") +relationships_root.remove(broken_relationships[0]) +broken_picture_payload[relationships_path] = etree.tostring( + relationships_root, xml_declaration=True, encoding="UTF-8", standalone=True, +) +with zipfile.ZipFile("broken-picture-relationship.pptx", "w", zipfile.ZIP_DEFLATED) as archive: + for member_name, member_data in broken_picture_payload.items(): + archive.writestr(member_name, member_data) +broken_picture_deck = open_validated_presentation("broken-picture-relationship.pptx") +broken_picture_inventory = [ + extract_slide_content(slide)["pictures"] for slide in broken_picture_deck.slides +] +check("broken picture relationship is explicit instead of aborting inventory", + broken_picture_inventory[0] == [{ + "name": reopened_picture.name, + "status": "unreadable", + "relationship_id": broken_relationship_id, + "reason": "missing or invalid image relationship or payload", + }], broken_picture_inventory[0]) +check("picture inventory continues to a healthy later slide after a broken relationship", + len(broken_picture_inventory[1]) == 1 + and broken_picture_inventory[1][0]["extension"] == "png", + broken_picture_inventory[1]) + +# Directly hidden leaf shapes and children of a hidden group must be absent from every +# non-text inventory, while visible siblings after them remain discoverable. +hidden_inventory_prs = Presentation() +hidden_inventory_slide = hidden_inventory_prs.slides.add_slide( + hidden_inventory_prs.slide_layouts[6] +) + +def set_shape_hidden(shape): + properties = shape._element.find(".//" + qn("p:cNvPr")) + require(properties is not None, f"shape {shape.name!r} has no cNvPr") + properties.set("hidden", "1") + +hidden_table_shape = hidden_inventory_slide.shapes.add_table( + 1, 1, Inches(0.2), Inches(0.2), Inches(1), Inches(0.5) +) +hidden_table_shape.table.cell(0, 0).text = "secret table" +set_shape_hidden(hidden_table_shape) + +hidden_chart_data = ChartData() +hidden_chart_data.categories = ["secret"] +hidden_chart_data.add_series("secret", (9,)) +hidden_chart_shape = hidden_inventory_slide.shapes.add_chart( + XL_CHART_TYPE.COLUMN_CLUSTERED, + Inches(1.3), Inches(0.2), Inches(1.5), Inches(1), hidden_chart_data, +) +set_shape_hidden(hidden_chart_shape) + +hidden_picture_shape = hidden_inventory_slide.shapes.add_picture( + "placeholder-picture.png", Inches(3), Inches(0.2), Inches(1), Inches(1) +) +set_shape_hidden(hidden_picture_shape) + +def smartart_frame(shape_id, name, relationship_id, *, hidden=False): + hidden_attribute = ' hidden="1"' if hidden else "" + return etree.fromstring(f''' + + + + + + + + + '''.encode()) + +hidden_inventory_slide.shapes._spTree.append( + smartart_frame(910, "Hidden SmartArt", "rIdHiddenSmart", hidden=True) +) + +nested_hidden_table = hidden_inventory_slide.shapes.add_table( + 1, 1, Inches(4.2), Inches(0.2), Inches(1), Inches(0.5) +) +nested_hidden_table.table.cell(0, 0).text = "secret grouped table" +nested_hidden_element = nested_hidden_table._element +hidden_group_element = parse_xml(GRP) +hidden_group_element.find(".//" + qn("p:cNvPr")).set("hidden", "1") +nested_hidden_element.getparent().replace(nested_hidden_element, hidden_group_element) +hidden_group_element.append(nested_hidden_element) + +visible_table_shape = hidden_inventory_slide.shapes.add_table( + 1, 1, Inches(0.2), Inches(2), Inches(1), Inches(0.5) +) +visible_table_shape.table.cell(0, 0).text = "visible table" +visible_chart_data = ChartData() +visible_chart_data.categories = ["visible"] +visible_chart_data.add_series("visible", (1,)) +visible_chart_shape = hidden_inventory_slide.shapes.add_chart( + XL_CHART_TYPE.COLUMN_CLUSTERED, + Inches(1.3), Inches(2), Inches(1.5), Inches(1), visible_chart_data, +) +visible_picture_shape = hidden_inventory_slide.shapes.add_picture( + "placeholder-picture.png", Inches(3), Inches(2), Inches(1), Inches(1) +) +hidden_inventory_slide.shapes._spTree.append( + smartart_frame(911, "Visible SmartArt", "rIdVisibleSmart") +) +hidden_inventory_prs.save("hidden-nontext-inventory.pptx") +hidden_inventory_reopened = open_validated_presentation("hidden-nontext-inventory.pptx") +hidden_inventory_slide = hidden_inventory_reopened.slides[0] +hidden_raw_names = [shape.name for shape in hidden_inventory_slide.shapes] +hidden_visible_names = [shape.name for shape in iter_shapes(hidden_inventory_slide.shapes)] +hidden_inventory = extract_slide_content(hidden_inventory_slide) +check("hidden non-text fixture retains direct and group-hidden objects (negative control)", + {"Hidden SmartArt", hidden_table_shape.name, hidden_chart_shape.name, + hidden_picture_shape.name, "demo group"} <= set(hidden_raw_names), + hidden_raw_names) +check("visibility walker prunes direct hidden leaves and every child of a hidden group", + "Hidden SmartArt" not in hidden_visible_names + and hidden_table_shape.name not in hidden_visible_names + and hidden_chart_shape.name not in hidden_visible_names + and hidden_picture_shape.name not in hidden_visible_names + and "secret grouped table" not in str(hidden_inventory), + (hidden_visible_names, hidden_inventory)) +check("all non-text inventories retain only visible siblings after hidden shapes", + len(hidden_inventory["tables"]) == 1 + and hidden_inventory["tables"][0][0][0]["text"] == "visible table" + and len(hidden_inventory["charts"]) == 1 + and hidden_inventory["charts"][0]["plots"][0]["series"][0]["values"] == [1.0] + and len(hidden_inventory["pictures"]) == 1 + and hidden_inventory["pictures"][0]["extension"] == "png" + and hidden_inventory["smartart"] == [{ + "name": "Visible SmartArt", "status": "unreadable", + "reason": "rIdVisibleSmart", + }], hidden_inventory) + +hidden_title_prs = Presentation() +hidden_title_slide = hidden_title_prs.slides.add_slide(hidden_title_prs.slide_layouts[5]) +hidden_title_slide.shapes.title.text = "confidential hidden title" +set_shape_hidden(hidden_title_slide.shapes.title) +hidden_title_prs.save("hidden-title.pptx") +hidden_title_slide = open_validated_presentation("hidden-title.pptx").slides[0] +hidden_title_shape = hidden_title_slide.shapes.title +guarded_title = ( + hidden_title_shape.text_frame.text + if hidden_title_shape is not None and not shape_is_hidden(hidden_title_shape) else "" +) +check("main slide summary does not expose a hidden title", guarded_title == "", guarded_title) + merged_prs = Presentation() merged_slide = merged_prs.slides.add_slide(merged_prs.slide_layouts[6]) merged_table = merged_slide.shapes.add_table( @@ -1992,6 +2203,11 @@ def name(self): cached_numeric_points(sparse_source, include_count=True) == {"point_count": 3, "points": [(0, 1.0)]}, cached_numeric_points(sparse_source, include_count=True)) +sparse_declared_budget = {"remaining": 3} +check("sparse numeric cache charges declared ptCount before materializing present points", + cached_numeric_points(sparse_source, point_budget=sparse_declared_budget) + == [(0, 1.0)] and sparse_declared_budget["remaining"] == 0, + sparse_declared_budget) check("category numeric cache can densify sparse positions as blanks", cached_numeric_points(sparse_source, fill_missing=True) == [(0, 1.0), (1, None), (2, None)], @@ -2007,6 +2223,53 @@ def name(self): ) check("oversized logical numeric counts fail before allocation", cached_numeric_points(huge_count_source, fill_missing=True) is None) + +# Sparse XY/bubble caches reserve their declared logical counts from one shared deck +# budget before iterating or materializing the few present nodes. +budgeted_xy_element = copy.deepcopy(xy_item._element) +for source_name in ("xVal", "yVal"): + numeric_cache = budgeted_xy_element.find(qn(f"c:{source_name}")) + numeric_cache = numeric_cache.find(qn("c:numRef") + "/" + qn("c:numCache")) + numeric_cache.find(qn("c:ptCount")).set("val", "60000") + for point in numeric_cache.findall(qn("c:pt"))[1:]: + numeric_cache.remove(point) +budgeted_xy_series = type("BudgetedXySeries", (), { + "_element": budgeted_xy_element, +})() +xy_declared_budget = {"remaining": MAX_CHART_POINTS} +xy_budgeted_content = series_content( + budgeted_xy_series, point_budget=xy_declared_budget +) +check("scatter X/Y caches charge declared sparse counts from the shared budget", + xy_budgeted_content.get("cache_status") == "unavailable" + and xy_declared_budget["remaining"] == 40000, + (xy_budgeted_content, xy_declared_budget)) + +bubble_plot = next( + plot for shape in Presentation("xy-bubble.pptx").slides[0].shapes + if shape.has_chart for plot in shape.chart.plots + if type(plot).__name__ == "BubblePlot" +) +bubble_item = list(bubble_plot.series)[0] +budgeted_bubble_element = copy.deepcopy(bubble_item._element) +for source_name in ("xVal", "yVal", "bubbleSize"): + numeric_cache = budgeted_bubble_element.find(qn(f"c:{source_name}")) + numeric_cache = numeric_cache.find(qn("c:numRef") + "/" + qn("c:numCache")) + numeric_cache.find(qn("c:ptCount")).set("val", "40000") + for point in numeric_cache.findall(qn("c:pt"))[1:]: + numeric_cache.remove(point) +budgeted_bubble_series = type("BudgetedBubbleSeries", (), { + "_element": budgeted_bubble_element, +})() +bubble_declared_budget = {"remaining": MAX_CHART_POINTS} +bubble_budgeted_content = series_content( + budgeted_bubble_series, point_budget=bubble_declared_budget +) +check("bubble X/Y/size caches share one declared-count budget", + bubble_budgeted_content.get("cache_status") == "unavailable" + and bubble_declared_budget["remaining"] == 20000, + (bubble_budgeted_content, bubble_declared_budget)) + actual_cache = x_source.xpath("./c:numRef/c:numCache")[0] x_source.xpath("./c:numRef")[0].remove(actual_cache) check("XY series with an unavailable cache is explicit", @@ -2681,6 +2944,13 @@ def unresolved_graphic_font_regions(shapes): check("ordinary table graphic frames are not mislabeled as chart or SmartArt", all(item["shape"] != triage_table_shape.name for item in unresolved_graphics), unresolved_graphics) +hidden_deck_unresolved_graphics = list(unresolved_graphic_font_regions( + iter_shapes(hidden_inventory_slide.shapes) +)) +check("font-region inventory excludes hidden charts and SmartArt while retaining visible siblings", + {item["shape"] for item in hidden_deck_unresolved_graphics} + == {visible_chart_shape.name, "Visible SmartArt"}, + hidden_deck_unresolved_graphics) try: if unresolved_graphics: raise LookupError(f"unresolved chart/SmartArt fonts: {unresolved_graphics}") From 0191149e1b75db54c20df9b44c2688c031e2cfb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=98=E6=89=AC?= Date: Tue, 18 Aug 2026 00:25:34 +0800 Subject: [PATCH 31/34] fix(document-skills): verify rendered PDF content --- .../skills/pdf/references/inspect.md | 188 ++++++-- .../document-skills/tests/pdf_fixture.py | 423 +++++++++++++++--- 2 files changed, 522 insertions(+), 89 deletions(-) diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md b/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md index b5d33a2..0146487 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md @@ -3,6 +3,7 @@ ```python import math import os +import re import fitz doc = fitz.open("input.pdf") @@ -11,19 +12,90 @@ if doc.needs_pass: if doc.authenticate(password) <= 0: raise RuntimeError("Encrypted PDF: set a valid PDF_PASSWORD before inspection") +DIRECT_CHARPROC_NAME = re.compile( + r"/(?:#[0-9A-Fa-f]{2}|[^#\s()<>\[\]{}/%])+" +) +DIRECT_CHARPROC_REFERENCE = re.compile(r"\s+([1-9]\d*)\s+\d+\s+R") + +def indirect_xref(value): + match = re.fullmatch(r"\s*([1-9]\d*)\s+\d+\s+R\s*", value or "") + return int(match.group(1)) if match else None + +def direct_charproc_xrefs(value): + """Parse only a conservative direct dictionary of name -> indirect-object entries.""" + value = (value or "").strip() + if not value.startswith("<<") or not value.endswith(">>"): + return "uninspectable", [] + body = value[2:-2] + position = 0 + references = [] + while position < len(body): + while position < len(body) and body[position].isspace(): + position += 1 + if position == len(body): + break + name_match = DIRECT_CHARPROC_NAME.match(body, position) + if name_match is None: + return "uninspectable", [] + reference_match = DIRECT_CHARPROC_REFERENCE.match(body, name_match.end()) + if reference_match is None: + return "malformed", [] + references.append(int(reference_match.group(1))) + position = reference_match.end() + return ("parsed", references) if references else ("malformed", []) + +def type3_charprocs_status(document, xref, font_type): + """Return verified, malformed, or uninspectable without reading glyph streams.""" + if font_type.replace(" ", "").casefold() != "type3": + return None + if xref <= 0: # A direct font dictionary has no xref for safe nested lookup. + return "uninspectable" + try: + charprocs_type, charprocs_value = document.xref_get_key(xref, "CharProcs") + except (RuntimeError, ValueError): + return "uninspectable" + if charprocs_type == "dict": + parse_status, glyph_xrefs = direct_charproc_xrefs(charprocs_value) + if parse_status != "parsed": + return parse_status + elif charprocs_type == "xref": + dictionary_xref = indirect_xref(charprocs_value) + if dictionary_xref is None: + return "malformed" + try: + dictionary_source = document.xref_object(dictionary_xref, compressed=True) + glyph_names = document.xref_get_keys(dictionary_xref) + except (RuntimeError, ValueError): + return "uninspectable" + if not dictionary_source.lstrip().startswith("<<") or not glyph_names: + return "malformed" + glyph_xrefs = [] + for glyph_name in glyph_names: + try: + glyph_type, glyph_value = document.xref_get_key(dictionary_xref, glyph_name) + except (RuntimeError, ValueError): + return "uninspectable" + glyph_xref = indirect_xref(glyph_value) if glyph_type == "xref" else None + if glyph_xref is None: + return "malformed" + glyph_xrefs.append(glyph_xref) + else: + return "malformed" + for glyph_xref in glyph_xrefs: + try: + if not document.xref_is_stream(glyph_xref): + return "malformed" + except (RuntimeError, ValueError): + return "uninspectable" + return "verified" + def font_inventory(document, page): - """Distinguish font files, self-contained Type3 glyphs, and referenced-only faces.""" + """Distinguish font files, verified Type3 glyph streams, and unknown cases.""" fonts = [] for entry in page.get_fonts(full=True): xref, extension, font_type, base_name, resource_name, encoding = entry[:6] is_type3 = font_type.replace(" ", "").casefold() == "type3" - type3_self_contained = False - if is_type3 and xref > 0: - try: - charprocs_type, _ = document.xref_get_key(xref, "CharProcs") - type3_self_contained = charprocs_type in {"dict", "xref"} - except (RuntimeError, ValueError): - type3_self_contained = False + charprocs_status = type3_charprocs_status(document, xref, font_type) embedded_bytes = 0 if xref > 0: try: @@ -31,6 +103,17 @@ def font_inventory(document, page): embedded_bytes = len(extracted[3] or b"") except (RuntimeError, ValueError): embedded_bytes = 0 + if is_type3: + embedded = ( + True if charprocs_status == "verified" else + (False if charprocs_status == "malformed" else None) + ) + self_contained = embedded + program_source = "type3-charprocs" + else: + embedded = embedded_bytes > 0 + self_contained = None + program_source = "font-file" if embedded_bytes else None fonts.append({ "xref": xref, "base_name": base_name, @@ -38,13 +121,11 @@ def font_inventory(document, page): "type": font_type, "encoding": encoding, "extension": extension, - "embedded": type3_self_contained or embedded_bytes > 0, + "embedded": embedded, "embedded_bytes": embedded_bytes, - "self_contained": type3_self_contained, - "program_source": ( - "type3-charprocs" if type3_self_contained else - ("font-file" if embedded_bytes else None) - ), + "self_contained": self_contained, + "charprocs_status": charprocs_status, + "program_source": program_source, }) return fonts @@ -79,6 +160,51 @@ def visible_clip(page, rectangle, *, already_rotated=False): clip = rotated & page.rect return None if clip.is_empty else clip +MAX_IMAGE_PLACEMENTS = 1_000 +MAX_IMAGE_SOURCE_PIXELS = 25_000_000 +MAX_TOTAL_IMAGE_SOURCE_PIXELS = 50_000_000 +MAX_IMAGE_RENDER_PIXELS = 4_000_000 +MAX_TOTAL_IMAGE_RENDER_PIXELS = 20_000_000 + +def viewable_images(page): + """Render bounded placement clips; unknown visibility keeps the page nonblank.""" + try: + placements = page.get_image_info() + except (RuntimeError, ValueError): + return [], [], True + if len(placements) > MAX_IMAGE_PLACEMENTS: + return placements, [], True + visible = [] + total_source_pixels = 0 + total_render_pixels = 0 + for placement in placements: + clip = visible_clip(page, placement.get("bbox")) + if clip is None: + continue + width, height = placement.get("width"), placement.get("height") + if (not isinstance(width, int) or isinstance(width, bool) or width <= 0 + or not isinstance(height, int) or isinstance(height, bool) or height <= 0): + return placements, visible, True + source_pixels = width * height + total_source_pixels += source_pixels + if (source_pixels > MAX_IMAGE_SOURCE_PIXELS + or total_source_pixels > MAX_TOTAL_IMAGE_SOURCE_PIXELS): + return placements, visible, True + render_pixels = math.ceil(clip.width) * math.ceil(clip.height) + total_render_pixels += render_pixels + if (render_pixels > MAX_IMAGE_RENDER_PIXELS + or total_render_pixels > MAX_TOTAL_IMAGE_RENDER_PIXELS): + return placements, visible, True + try: + pixmap = page.get_pixmap(clip=clip, alpha=True, annots=False) + except (RuntimeError, ValueError): + return placements, visible, True + if not pixmap.alpha: + return placements, visible, True + if any(pixmap.samples[pixmap.n - 1::pixmap.n]): + visible.append(placement) + return placements, visible, False + def rendered_interactives(page, items): rendered = [] visibility_unknown = False @@ -149,20 +275,21 @@ for page in doc: "crop_size": crop_size, "rotation": page.rotation, }) - visible_images = [ - info for info in page.get_image_info() - if visible_clip(page, info.get("bbox")) is not None - ] + image_placements, visible_images, image_visibility_unknown = viewable_images(page) drawings = page.get_drawings() widgets, annotations, links, interaction_visibility_unknown = viewable_interactives(page) is_blank = not ( page.get_text().strip() or visible_images or drawings - or widgets or annotations or links or interaction_visibility_unknown + or widgets or annotations or links + or image_visibility_unknown or interaction_visibility_unknown ) print(page.number + 1, "media_size:", media_size, "crop_size:", crop_size, "rotation:", page.rotation, "text_len:", len(page.get_text()), "resource_images:", len(page.get_images()), - "visible_image_placements:", len(visible_images), "drawings:", len(drawings), + "image_placements:", len(image_placements), + "visible_images:", len(visible_images), + "image_visibility_unknown:", image_visibility_unknown, + "drawings:", len(drawings), "widgets:", len(widgets), "annotations:", len(annotations), "links:", len(links), "interaction_visibility_unknown:", interaction_visibility_unknown, "blank:", is_blank) @@ -177,16 +304,21 @@ print("crop_size_consistent:", len({row["crop_size"] for row in page_geometry}) and viewable widgets, annotations, and links are all absent. Ignore interactive objects carrying invisible, hidden, or no-view flags, as well as empty, off-page, or unrendered appearances. `page.get_images()` lists every image XObject resource, including unused resources, and also - misses images embedded inline in the content stream. `page.get_image_info()` reports painted - inline and XObject placements without loading their bytes; retain only finite, nonempty placements - that intersect the rotated page via `visible_clip()`. Interactive form fields are widgets rather + misses images embedded inline in the content stream. `page.get_image_info()` reports invoked + inline and XObject placements without loading their bytes, but its boxes ignore graphics-state + clipping. Treat those boxes as diagnostics only. The bounded `viewable_images()` render checks + alpha for each page-intersecting placement; a count, source-pixel, render-pixel, or decoder limit + returns `image_visibility_unknown=True`, which must keep the page nonblank. Interactive form + fields are widgets rather than page text, so a three-content-stream predicate alone would misclassify a usable form page as blank. A blank page after generation usually means an overflowing flowable created it. - **Font inventory**: `page.get_fonts()` lists referenced fonts, including non-embedded base fonts. Use `doc.extract_font(xref)` as above for conventional font files. Type3 fonts are a - separate self-contained case: when the font dictionary has a `/CharProcs` dictionary, its glyph - programs are PDF content streams, so report `self_contained=True` and - `program_source="type3-charprocs"` even when no conventional font-file bytes extract. + separate case: report `charprocs_status="verified"` only for a nonempty `/CharProcs` dictionary + whose glyph entries point to actual streams. A direct font dictionary has no font xref and is + `uninspectable`, not falsely non-embedded. For Type3, `embedded` and `self_contained` are tri-state: + `True` for verified, `False` for malformed, and `None` when uninspectable. Its + `program_source="type3-charprocs"` remains explicit even when conventional bytes do not extract. Other referenced faces with no extractable program may be substituted on another machine. - **Page size consistency**: compare unrotated `(width, height)` pairs from `page.mediabox` and `page.cropbox`, and report `page.rotation` separately. Do not compare `page.rect`: it applies @@ -196,6 +328,6 @@ print("crop_size_consistent:", len({row["crop_size"] for row in page_geometry}) `pypdf.PdfReader` cross-check when provenance is unknown. Report findings as a table (page, media size, crop size, rotation, text chars, resource images, -visible image placements, drawings, widgets, annotations, links, interaction -visibility unknown, blank) - it +image placements, visible images, image visibility unknown, drawings, widgets, annotations, +links, interaction visibility unknown, blank) - it is what every downstream decision hangs off. diff --git a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py index e94e5df..4194dad 100644 --- a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py @@ -7,6 +7,7 @@ # python docx_fixture.py (deps: python-docx, pymupdf, soffice on PATH) import math import os +import re import sys import fitz @@ -14,7 +15,7 @@ import reportlab from pypdf.generic import ( ArrayObject, DecodedStreamObject, DictionaryObject, FloatObject, - NameObject, NumberObject, + NameObject, NullObject, NumberObject, ) from reportlab.lib.pagesizes import A4 from reportlab.pdfbase import pdfmetrics @@ -129,86 +130,202 @@ def verify_page_count(reader, expected_page_count): font_canvas.drawString(72, 750, "Embedded Vera") font_canvas.save() +DIRECT_CHARPROC_NAME = re.compile( + r"/(?:#[0-9A-Fa-f]{2}|[^#\s()<>\[\]{}/%])+" +) +DIRECT_CHARPROC_REFERENCE = re.compile(r"\s+([1-9]\d*)\s+\d+\s+R") + + +def indirect_xref(value): + match = re.fullmatch(r"\s*([1-9]\d*)\s+\d+\s+R\s*", value or "") + return int(match.group(1)) if match else None + + +def direct_charproc_xrefs(value): + value = (value or "").strip() + if not value.startswith("<<") or not value.endswith(">>"): + return "uninspectable", [] + body = value[2:-2] + position = 0 + references = [] + while position < len(body): + while position < len(body) and body[position].isspace(): + position += 1 + if position == len(body): + break + name_match = DIRECT_CHARPROC_NAME.match(body, position) + if name_match is None: + return "uninspectable", [] + reference_match = DIRECT_CHARPROC_REFERENCE.match(body, name_match.end()) + if reference_match is None: + return "malformed", [] + references.append(int(reference_match.group(1))) + position = reference_match.end() + return ("parsed", references) if references else ("malformed", []) + + +def type3_charprocs_status(document, xref, font_type): + if font_type.replace(" ", "").casefold() != "type3": + return None + if xref <= 0: + return "uninspectable" + try: + charprocs_type, charprocs_value = document.xref_get_key(xref, "CharProcs") + except (RuntimeError, ValueError): + return "uninspectable" + if charprocs_type == "dict": + parse_status, glyph_xrefs = direct_charproc_xrefs(charprocs_value) + if parse_status != "parsed": + return parse_status + elif charprocs_type == "xref": + dictionary_xref = indirect_xref(charprocs_value) + if dictionary_xref is None: + return "malformed" + try: + dictionary_source = document.xref_object(dictionary_xref, compressed=True) + glyph_names = document.xref_get_keys(dictionary_xref) + except (RuntimeError, ValueError): + return "uninspectable" + if not dictionary_source.lstrip().startswith("<<") or not glyph_names: + return "malformed" + glyph_xrefs = [] + for glyph_name in glyph_names: + try: + glyph_type, glyph_value = document.xref_get_key(dictionary_xref, glyph_name) + except (RuntimeError, ValueError): + return "uninspectable" + glyph_xref = indirect_xref(glyph_value) if glyph_type == "xref" else None + if glyph_xref is None: + return "malformed" + glyph_xrefs.append(glyph_xref) + else: + return "malformed" + for glyph_xref in glyph_xrefs: + try: + if not document.xref_is_stream(glyph_xref): + return "malformed" + except (RuntimeError, ValueError): + return "uninspectable" + return "verified" + + def font_inventory(document, page): fonts = [] for entry in page.get_fonts(full=True): xref, extension, font_type, base_name, resource_name, encoding = entry[:6] is_type3 = font_type.replace(" ", "").casefold() == "type3" - type3_self_contained = False - if is_type3 and xref > 0: - try: - charprocs_type, _ = document.xref_get_key(xref, "CharProcs") - type3_self_contained = charprocs_type in {"dict", "xref"} - except (RuntimeError, ValueError): - type3_self_contained = False + charprocs_status = type3_charprocs_status(document, xref, font_type) embedded_bytes = 0 if xref > 0: try: embedded_bytes = len((document.extract_font(xref)[3] or b"")) except (RuntimeError, ValueError): embedded_bytes = 0 + if is_type3: + embedded = ( + True if charprocs_status == "verified" else + (False if charprocs_status == "malformed" else None) + ) + self_contained = embedded + program_source = "type3-charprocs" + else: + embedded = embedded_bytes > 0 + self_contained = None + program_source = "font-file" if embedded_bytes else None fonts.append({ "base_name": base_name, "type": font_type, - "embedded": type3_self_contained or embedded_bytes > 0, + "embedded": embedded, "embedded_bytes": embedded_bytes, - "self_contained": type3_self_contained, - "program_source": ( - "type3-charprocs" if type3_self_contained else - ("font-file" if embedded_bytes else None) - ), + "self_contained": self_contained, + "charprocs_status": charprocs_status, + "program_source": program_source, }) return fonts font_doc = fitz.open("font-inventory.pdf") fonts = font_inventory(font_doc, font_doc[0]) check("font inventory labels the embedded TrueType face", - any("Vera" in item["base_name"] and item["embedded"] for item in fonts), fonts) + any("Vera" in item["base_name"] and item["embedded"] is True for item in fonts), fonts) check("font inventory labels referenced-only Helvetica as non-embedded", - any("Helvetica" in item["base_name"] and not item["embedded"] for item in fonts), fonts) + any("Helvetica" in item["base_name"] and item["embedded"] is False for item in fonts), fonts) # A Type3 font stores each glyph as PDF content in /CharProcs rather than as an # extractable conventional font file. -type3_writer = pypdf.PdfWriter() -type3_page = type3_writer.add_blank_page(width=612, height=792) -type3_glyph = DecodedStreamObject() -type3_glyph.set_data(b"500 0 0 0 500 700 d1 0 0 500 700 re f") -type3_glyph_ref = type3_writer._add_object(type3_glyph) -type3_charprocs_ref = type3_writer._add_object(DictionaryObject({ - NameObject("/A"): type3_glyph_ref, -})) -type3_encoding = DictionaryObject({ - NameObject("/Type"): NameObject("/Encoding"), - NameObject("/Differences"): ArrayObject([ - NumberObject(65), NameObject("/A"), - ]), -}) -type3_font = DictionaryObject({ - NameObject("/Type"): NameObject("/Font"), - NameObject("/Subtype"): NameObject("/Type3"), - NameObject("/Name"): NameObject("/FType3"), - NameObject("/FontBBox"): ArrayObject([ - NumberObject(0), NumberObject(0), NumberObject(500), NumberObject(700), - ]), - NameObject("/FontMatrix"): ArrayObject([ - FloatObject(0.001), NumberObject(0), NumberObject(0), - FloatObject(0.001), NumberObject(0), NumberObject(0), - ]), - NameObject("/CharProcs"): type3_charprocs_ref, - NameObject("/Encoding"): type3_encoding, - NameObject("/FirstChar"): NumberObject(65), - NameObject("/LastChar"): NumberObject(65), - NameObject("/Widths"): ArrayObject([NumberObject(500)]), - NameObject("/Resources"): DictionaryObject(), -}) -type3_font_ref = type3_writer._add_object(type3_font) -type3_page[NameObject("/Resources")] = DictionaryObject({ - NameObject("/Font"): DictionaryObject({NameObject("/FType3"): type3_font_ref}), -}) -type3_content = DecodedStreamObject() -type3_content.set_data(b"BT /FType3 72 Tf 72 700 Td (A) Tj ET") -type3_page[NameObject("/Contents")] = type3_writer._add_object(type3_content) -with open("type3-font.pdf", "wb") as output: - type3_writer.write(output) + + +def write_type3_pdf(path, *, charprocs_kind="indirect-dict", direct_font=False): + writer = pypdf.PdfWriter() + page = writer.add_blank_page(width=612, height=792) + glyph = DecodedStreamObject() + glyph.set_data(b"500 0 0 0 500 700 d1 0 0 500 700 re f") + glyph_ref = writer._add_object(glyph) + valid_charprocs = DictionaryObject({NameObject("/A"): glyph_ref}) + if charprocs_kind == "indirect-dict": + charprocs = writer._add_object(valid_charprocs) + elif charprocs_kind == "direct-dict": + charprocs = valid_charprocs + elif charprocs_kind == "direct-escaped-name": + charprocs = DictionaryObject({NameObject("/A#20B"): glyph_ref}) + elif charprocs_kind == "empty-direct": + charprocs = DictionaryObject() + elif charprocs_kind == "direct-number-entry": + charprocs = DictionaryObject({NameObject("/A"): NumberObject(42)}) + elif charprocs_kind == "indirect-array": + charprocs = writer._add_object(ArrayObject([glyph_ref])) + elif charprocs_kind == "indirect-number": + charprocs = writer._add_object(NumberObject(42)) + elif charprocs_kind == "indirect-null": + charprocs = writer._add_object(NullObject()) + elif charprocs_kind == "indirect-bad-glyph": + bad_glyph_ref = writer._add_object(NumberObject(42)) + charprocs = writer._add_object(DictionaryObject({ + NameObject("/A"): bad_glyph_ref, + })) + else: + raise ValueError(f"unsupported CharProcs fixture kind {charprocs_kind}") + encoding = DictionaryObject({ + NameObject("/Type"): NameObject("/Encoding"), + NameObject("/Differences"): ArrayObject([ + NumberObject(65), NameObject("/A"), + ]), + }) + font = DictionaryObject({ + NameObject("/Type"): NameObject("/Font"), + NameObject("/Subtype"): NameObject("/Type3"), + NameObject("/Name"): NameObject("/FType3"), + NameObject("/FontBBox"): ArrayObject([ + NumberObject(0), NumberObject(0), NumberObject(500), NumberObject(700), + ]), + NameObject("/FontMatrix"): ArrayObject([ + FloatObject(0.001), NumberObject(0), NumberObject(0), + FloatObject(0.001), NumberObject(0), NumberObject(0), + ]), + NameObject("/CharProcs"): charprocs, + NameObject("/Encoding"): encoding, + NameObject("/FirstChar"): NumberObject(65), + NameObject("/LastChar"): NumberObject(65), + NameObject("/Widths"): ArrayObject([NumberObject(500)]), + NameObject("/Resources"): DictionaryObject(), + }) + font_object = font if direct_font else writer._add_object(font) + page[NameObject("/Resources")] = DictionaryObject({ + NameObject("/Font"): DictionaryObject({NameObject("/FType3"): font_object}), + }) + content = DecodedStreamObject() + content.set_data(b"BT /FType3 72 Tf 72 700 Td (A) Tj ET") + page[NameObject("/Contents")] = writer._add_object(content) + with open(path, "wb") as output: + writer.write(output) + + +def type3_fixture_record(path): + document = fitz.open(path) + record = next(item for item in font_inventory(document, document[0]) + if item["type"].replace(" ", "").casefold() == "type3") + return document, record + + +write_type3_pdf("type3-font.pdf") type3_doc = fitz.open("type3-font.pdf") type3_entry = next(item for item in type3_doc[0].get_fonts(full=True) if item[2].replace(" ", "").casefold() == "type3") @@ -224,9 +341,60 @@ def font_inventory(document, page): check("font inventory classifies Type3 CharProcs as self-contained content", type3_record["embedded"] and type3_record["self_contained"] and type3_record["embedded_bytes"] == 0 + and type3_record["charprocs_status"] == "verified" and type3_record["program_source"] == "type3-charprocs", type3_record) +write_type3_pdf("type3-direct-charprocs.pdf", charprocs_kind="direct-dict") +direct_charprocs_doc, direct_charprocs_record = type3_fixture_record( + "type3-direct-charprocs.pdf" +) +check("direct nonempty Type3 CharProcs with stream glyphs verifies", + direct_charprocs_record["charprocs_status"] == "verified" + and direct_charprocs_record["embedded"] is True + and direct_charprocs_doc[0].get_text().strip() == "A", + direct_charprocs_record) + +write_type3_pdf( + "type3-escaped-charproc-name.pdf", charprocs_kind="direct-escaped-name", +) +_, escaped_charproc_record = type3_fixture_record("type3-escaped-charproc-name.pdf") +check("escaped PDF names in direct Type3 CharProcs remain verifiable", + escaped_charproc_record["charprocs_status"] == "verified" + and escaped_charproc_record["embedded"] is True, + escaped_charproc_record) + +write_type3_pdf( + "type3-direct-font.pdf", charprocs_kind="direct-dict", direct_font=True, +) +direct_font_doc, direct_font_record = type3_fixture_record("type3-direct-font.pdf") +direct_font_xref = next( + entry[0] for entry in direct_font_doc[0].get_fonts(full=True) + if entry[2].replace(" ", "").casefold() == "type3" +) +check("direct Type3 font dictionary is uninspectable rather than falsely non-embedded", + direct_font_xref == 0 + and direct_font_doc[0].get_text().strip() == "A" + and direct_font_record["charprocs_status"] == "uninspectable" + and direct_font_record["embedded"] is None + and direct_font_record["self_contained"] is None, + (direct_font_xref, direct_font_record)) + +malformed_type3_records = {} +for malformed_kind in ( + "empty-direct", "direct-number-entry", "indirect-array", + "indirect-number", "indirect-null", "indirect-bad-glyph", +): + malformed_path = f"type3-{malformed_kind}.pdf" + write_type3_pdf(malformed_path, charprocs_kind=malformed_kind) + _, malformed_type3_records[malformed_kind] = type3_fixture_record(malformed_path) +check("empty, wrong-type, and non-stream Type3 CharProcs fail closed as malformed", + all(record["charprocs_status"] == "malformed" + and record["embedded"] is False + and record["self_contained"] is False + for record in malformed_type3_records.values()), + malformed_type3_records) + # ---- SKILL.md postcheck: encrypted output is reopened with its password ------- encrypted_writer = pypdf.PdfWriter() encrypted_writer.append(r) @@ -333,6 +501,52 @@ def visible_clip(page, rectangle, *, already_rotated=False): return None if clip.is_empty else clip +MAX_IMAGE_PLACEMENTS = 1_000 +MAX_IMAGE_SOURCE_PIXELS = 25_000_000 +MAX_TOTAL_IMAGE_SOURCE_PIXELS = 50_000_000 +MAX_IMAGE_RENDER_PIXELS = 4_000_000 +MAX_TOTAL_IMAGE_RENDER_PIXELS = 20_000_000 + + +def viewable_images(page): + try: + placements = page.get_image_info() + except (RuntimeError, ValueError): + return [], [], True + if len(placements) > MAX_IMAGE_PLACEMENTS: + return placements, [], True + visible = [] + total_source_pixels = 0 + total_render_pixels = 0 + for placement in placements: + clip = visible_clip(page, placement.get("bbox")) + if clip is None: + continue + width, height = placement.get("width"), placement.get("height") + if (not isinstance(width, int) or isinstance(width, bool) or width <= 0 + or not isinstance(height, int) or isinstance(height, bool) or height <= 0): + return placements, visible, True + source_pixels = width * height + total_source_pixels += source_pixels + if (source_pixels > MAX_IMAGE_SOURCE_PIXELS + or total_source_pixels > MAX_TOTAL_IMAGE_SOURCE_PIXELS): + return placements, visible, True + render_pixels = math.ceil(clip.width) * math.ceil(clip.height) + total_render_pixels += render_pixels + if (render_pixels > MAX_IMAGE_RENDER_PIXELS + or total_render_pixels > MAX_TOTAL_IMAGE_RENDER_PIXELS): + return placements, visible, True + try: + pixmap = page.get_pixmap(clip=clip, alpha=True, annots=False) + except (RuntimeError, ValueError): + return placements, visible, True + if not pixmap.alpha: + return placements, visible, True + if any(pixmap.samples[pixmap.n - 1::pixmap.n]): + visible.append(placement) + return placements, visible, False + + def rendered_interactives(page, items): rendered = [] visibility_unknown = False @@ -933,14 +1147,12 @@ def stamp_placement(page, stamp_box): def inspected_page_is_blank(page): - visible_images = [ - info for info in page.get_image_info() - if visible_clip(page, info.get("bbox")) is not None - ] + _, visible_images, image_visibility_unknown = viewable_images(page) return not ( page.get_text().strip() or visible_images or page.get_drawings() or list(page.widgets() or ()) or list(page.annots() or ()) or page.get_links() + or image_visibility_unknown ) @@ -1003,6 +1215,95 @@ def inspected_page_is_blank(page): xobject_canvas.showPage() xobject_canvas.save() +def write_fully_clipped_image(path, *, inline): + clipped_canvas = canvas.Canvas(path, pagesize=(200, 200)) + clip_path = clipped_canvas.beginPath() + clip_path.rect(0, 0, 10, 10) + clipped_canvas.saveState() + clipped_canvas.clipPath(clip_path, stroke=0, fill=0) + draw = clipped_canvas.drawInlineImage if inline else clipped_canvas.drawImage + draw("inline-only-source.png", 60, 60, width=80, height=80) + clipped_canvas.restoreState() + clipped_canvas.showPage() + clipped_canvas.save() + + +clipped_image_results = {} +for clipped_kind, clipped_inline in (("xobject", False), ("inline", True)): + clipped_path = f"fully-clipped-{clipped_kind}.pdf" + write_fully_clipped_image(clipped_path, inline=clipped_inline) + clipped_doc = fitz.open(clipped_path) + clipped_page = clipped_doc[0] + placements, visible, visibility_unknown = viewable_images(clipped_page) + clipped_alpha = clipped_page.get_pixmap(alpha=True, annots=False) + clipped_image_results[clipped_kind] = { + "placements": len(placements), "visible": len(visible), + "unknown": visibility_unknown, + "painted_alpha": any(clipped_alpha.samples[clipped_alpha.n - 1::clipped_alpha.n]), + "blank": inspected_page_is_blank(clipped_page), + } +check("fully clipped XObject and inline placements remain diagnostic candidates", + all(result["placements"] == 1 for result in clipped_image_results.values()), + clipped_image_results) +check("alpha render excludes fully clipped XObject and inline images from blank evidence", + all(result == { + "placements": 1, "visible": 0, "unknown": False, + "painted_alpha": False, "blank": True, + } for result in clipped_image_results.values()), clipped_image_results) + +rotated_image_writer = pypdf.PdfWriter() +rotated_image_writer.append(pypdf.PdfReader("xobject-image.pdf")) +rotated_image_writer.pages[0].rotate(90) +with open("rotated-visible-image.pdf", "wb") as output: + rotated_image_writer.write(output) +rotated_image_doc = fitz.open("rotated-visible-image.pdf") +rotated_placements, rotated_visible, rotated_unknown = viewable_images(rotated_image_doc[0]) +check("visible image remains visible after page rotation", + len(rotated_placements) == len(rotated_visible) == 1 + and not rotated_unknown and not inspected_page_is_blank(rotated_image_doc[0]), + (rotated_placements, rotated_visible, rotated_unknown)) + +xobject_visibility_doc = fitz.open("xobject-image.pdf") +budget_results = {} +for budget_name in ( + "MAX_IMAGE_PLACEMENTS", "MAX_IMAGE_SOURCE_PIXELS", + "MAX_TOTAL_IMAGE_SOURCE_PIXELS", "MAX_IMAGE_RENDER_PIXELS", + "MAX_TOTAL_IMAGE_RENDER_PIXELS", +): + original_budget = globals()[budget_name] + globals()[budget_name] = 0 + try: + budget_placements, budget_visible, budget_unknown = viewable_images( + xobject_visibility_doc[0] + ) + budget_page_is_blank = inspected_page_is_blank(xobject_visibility_doc[0]) + finally: + globals()[budget_name] = original_budget + budget_results[budget_name] = ( + len(budget_placements), len(budget_visible), + budget_unknown, budget_page_is_blank, + ) +check("every image count/source/render budget fails closed as visibility unknown", + all(result == (1, 0, True, False) for result in budget_results.values()), + budget_results) + +class FailingImageRenderPage: + rotation_matrix = fitz.Matrix(1, 1) + rect = fitz.Rect(0, 0, 200, 200) + + def get_image_info(self): + return [{"bbox": (20, 20, 80, 80), "width": 12, "height": 12}] + + def get_pixmap(self, **kwargs): + raise RuntimeError("fixture image decoder failure") + + +failed_placements, failed_visible, failed_unknown = viewable_images( + FailingImageRenderPage() +) +check("image decoder failure is reported as visibility unknown", + len(failed_placements) == 1 and not failed_visible and failed_unknown) + # Editing can remove the only Do operation while leaving the image in /Resources. # The resource inventory is then nonempty, but no image is painted. unused_image_writer = pypdf.PdfWriter() From 27b3edcc0a2bb41102c89e3ac7f3a6d90f9616a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=98=E6=89=AC?= Date: Tue, 18 Aug 2026 00:35:59 +0800 Subject: [PATCH 32/34] fix(document-skills): preserve escaped Type3 glyph names --- .../skills/pdf/references/inspect.md | 19 ++---- .../document-skills/tests/pdf_fixture.py | 65 ++++++++++++++----- 2 files changed, 55 insertions(+), 29 deletions(-) diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md b/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md index 0146487..7e763a7 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md @@ -64,21 +64,16 @@ def type3_charprocs_status(document, xref, font_type): return "malformed" try: dictionary_source = document.xref_object(dictionary_xref, compressed=True) - glyph_names = document.xref_get_keys(dictionary_xref) + dictionary_is_stream = document.xref_is_stream(dictionary_xref) except (RuntimeError, ValueError): return "uninspectable" - if not dictionary_source.lstrip().startswith("<<") or not glyph_names: + dictionary_source = dictionary_source.strip() + if (dictionary_is_stream or not dictionary_source.startswith("<<") + or not dictionary_source.endswith(">>")): return "malformed" - glyph_xrefs = [] - for glyph_name in glyph_names: - try: - glyph_type, glyph_value = document.xref_get_key(dictionary_xref, glyph_name) - except (RuntimeError, ValueError): - return "uninspectable" - glyph_xref = indirect_xref(glyph_value) if glyph_type == "xref" else None - if glyph_xref is None: - return "malformed" - glyph_xrefs.append(glyph_xref) + parse_status, glyph_xrefs = direct_charproc_xrefs(dictionary_source) + if parse_status != "parsed": + return parse_status else: return "malformed" for glyph_xref in glyph_xrefs: diff --git a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py index 4194dad..b810119 100644 --- a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py @@ -183,21 +183,16 @@ def type3_charprocs_status(document, xref, font_type): return "malformed" try: dictionary_source = document.xref_object(dictionary_xref, compressed=True) - glyph_names = document.xref_get_keys(dictionary_xref) + dictionary_is_stream = document.xref_is_stream(dictionary_xref) except (RuntimeError, ValueError): return "uninspectable" - if not dictionary_source.lstrip().startswith("<<") or not glyph_names: + dictionary_source = dictionary_source.strip() + if (dictionary_is_stream or not dictionary_source.startswith("<<") + or not dictionary_source.endswith(">>")): return "malformed" - glyph_xrefs = [] - for glyph_name in glyph_names: - try: - glyph_type, glyph_value = document.xref_get_key(dictionary_xref, glyph_name) - except (RuntimeError, ValueError): - return "uninspectable" - glyph_xref = indirect_xref(glyph_value) if glyph_type == "xref" else None - if glyph_xref is None: - return "malformed" - glyph_xrefs.append(glyph_xref) + parse_status, glyph_xrefs = direct_charproc_xrefs(dictionary_source) + if parse_status != "parsed": + return parse_status else: return "malformed" for glyph_xref in glyph_xrefs: @@ -259,13 +254,19 @@ def write_type3_pdf(path, *, charprocs_kind="indirect-dict", direct_font=False): glyph = DecodedStreamObject() glyph.set_data(b"500 0 0 0 500 700 d1 0 0 500 700 re f") glyph_ref = writer._add_object(glyph) - valid_charprocs = DictionaryObject({NameObject("/A"): glyph_ref}) + glyph_name = NameObject( + "/A/B" if charprocs_kind in {"direct-escaped-name", "indirect-escaped-name"} + else "/A" + ) + valid_charprocs = DictionaryObject({glyph_name: glyph_ref}) if charprocs_kind == "indirect-dict": charprocs = writer._add_object(valid_charprocs) + elif charprocs_kind == "indirect-escaped-name": + charprocs = writer._add_object(valid_charprocs) elif charprocs_kind == "direct-dict": charprocs = valid_charprocs elif charprocs_kind == "direct-escaped-name": - charprocs = DictionaryObject({NameObject("/A#20B"): glyph_ref}) + charprocs = valid_charprocs elif charprocs_kind == "empty-direct": charprocs = DictionaryObject() elif charprocs_kind == "direct-number-entry": @@ -286,7 +287,7 @@ def write_type3_pdf(path, *, charprocs_kind="indirect-dict", direct_font=False): encoding = DictionaryObject({ NameObject("/Type"): NameObject("/Encoding"), NameObject("/Differences"): ArrayObject([ - NumberObject(65), NameObject("/A"), + NumberObject(65), glyph_name, ]), }) font = DictionaryObject({ @@ -358,12 +359,42 @@ def type3_fixture_record(path): write_type3_pdf( "type3-escaped-charproc-name.pdf", charprocs_kind="direct-escaped-name", ) -_, escaped_charproc_record = type3_fixture_record("type3-escaped-charproc-name.pdf") +escaped_charproc_doc, escaped_charproc_record = type3_fixture_record( + "type3-escaped-charproc-name.pdf" +) check("escaped PDF names in direct Type3 CharProcs remain verifiable", escaped_charproc_record["charprocs_status"] == "verified" - and escaped_charproc_record["embedded"] is True, + and escaped_charproc_record["embedded"] is True + and escaped_charproc_doc[0].get_text().strip() == "A", escaped_charproc_record) +write_type3_pdf( + "type3-indirect-escaped-charproc-name.pdf", + charprocs_kind="indirect-escaped-name", +) +indirect_escaped_doc, indirect_escaped_record = type3_fixture_record( + "type3-indirect-escaped-charproc-name.pdf" +) +indirect_escaped_xref = next( + entry[0] for entry in indirect_escaped_doc[0].get_fonts(full=True) + if entry[2].replace(" ", "").casefold() == "type3" +) +indirect_charprocs_type, indirect_charprocs_value = indirect_escaped_doc.xref_get_key( + indirect_escaped_xref, "CharProcs" +) +indirect_charprocs_xref = indirect_xref(indirect_charprocs_value) +indirect_charprocs_source = indirect_escaped_doc.xref_object( + indirect_charprocs_xref, compressed=True +) +check("escaped slash name survives in the raw indirect CharProcs dictionary", + indirect_charprocs_type == "xref" and "#2F" in indirect_charprocs_source.upper(), + indirect_charprocs_source) +check("escaped slash name in indirect Type3 CharProcs remains verified", + indirect_escaped_record["charprocs_status"] == "verified" + and indirect_escaped_record["embedded"] is True + and indirect_escaped_doc[0].get_text().strip() == "A", + indirect_escaped_record) + write_type3_pdf( "type3-direct-font.pdf", charprocs_kind="direct-dict", direct_font=True, ) From a5cd131415a647459590826a06b41e99e9805e46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=98=E6=89=AC?= Date: Tue, 18 Aug 2026 07:22:34 +0800 Subject: [PATCH 33/34] docs(document-skills): bound untrusted package loading Address the latest review round: add bounded XLSX/OPC package preflight with compression, member, materialization and range-token budgets; route all existing-package loads through validated snapshots; harden DOCX CJK/locator, PDF Type3 and PPTX merged-slot guidance; cover CSV formula neutralization, missing dimension scanning and internal hyperlink rewrites with real OPC fixtures. --- .../skills/docx/references/cjk.md | 3 +- .../skills/docx/references/read.md | 156 ++- .../skills/pdf/references/inspect.md | 91 +- .../skills/pptx/references/edit.md | 9 +- .../document-skills/skills/xlsx/SKILL.md | 20 +- .../skills/xlsx/references/csv.md | 130 +- .../skills/xlsx/references/edit.md | 86 +- .../skills/xlsx/references/formatting.md | 190 ++- .../skills/xlsx/references/package.md | 318 +++++ .../skills/xlsx/references/read.md | 236 ++-- .../document-skills/tests/docx_fixture.py | 301 +++- .../document-skills/tests/pdf_fixture.py | 187 ++- .../document-skills/tests/pptx_fixture.py | 67 + .../tests/requirements-fixtures.txt | 1 + .../document-skills/tests/xlsx_fixture.py | 1205 ++++++++++++++--- 15 files changed, 2433 insertions(+), 567 deletions(-) create mode 100644 plugins/Hylouis233/document-skills/skills/xlsx/references/package.md diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md b/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md index d239222..242466a 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/cjk.md @@ -209,7 +209,8 @@ section.left_margin, section.right_margin = Cm(2.8), Cm(2.6) or 0xF900 <= codepoint <= 0xFAFF # CJK compatibility ideographs or 0xFE30 <= codepoint <= 0xFE6F # CJK compatibility and small forms or 0xFF00 <= codepoint <= 0xFFEF # fullwidth and halfwidth forms - or 0x20000 <= codepoint <= 0x3134F # CJK unified ideograph extensions + or 0x20000 <= codepoint <= 0x3134F # CJK unified ideograph extensions B-G + or 0x31350 <= codepoint <= 0x33479 # Unicode 17 extensions H-J ) return "eastAsia" if uses_east_asian_slot else ( "ascii" if codepoint < 128 else "hAnsi" diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/read.md b/plugins/Hylouis233/document-skills/skills/docx/references/read.md index 5dfe8d5..bc76771 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/read.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/read.md @@ -20,9 +20,60 @@ from docx.table import Table from docx.text.paragraph import Paragraph from docx.text.run import Run +MC_NAMESPACE = "http://schemas.openxmlformats.org/markup-compatibility/2006" +MC_ALTERNATE_CONTENT = f"{{{MC_NAMESPACE}}}AlternateContent" +MC_CHOICE = f"{{{MC_NAMESPACE}}}Choice" +MC_FALLBACK = f"{{{MC_NAMESPACE}}}Fallback" +SUPPORTED_MC_NAMESPACE_URIS = { + MC_NAMESPACE, + qn("w:p").split("}", 1)[0][1:], + qn("r:id").split("}", 1)[0][1:], +} + +def alternate_content_branch(element): + """Select the first Choice whose required namespace URIs are understood.""" + fallback = None + for child in element.iterchildren(): + if child.tag == MC_CHOICE: + required_prefixes = (child.get("Requires") or "").split() + if required_prefixes and all( + child.nsmap.get(prefix) in SUPPORTED_MC_NAMESPACE_URIS + for prefix in required_prefixes + ): + return child + elif child.tag == MC_FALLBACK and fallback is None: + fallback = child + return fallback + +def unresolved_alternate_content(element): + return { + "kind": "AlternateContent", + "requires": [ + (child.get("Requires") or "").split() + for child in element.iterchildren() + if child.tag == MC_CHOICE + ], + "reason": "no supported Choice and no Fallback", + } + +def iter_effective_children(root): + """Yield one selected markup-compatibility branch, never Choice and Fallback.""" + for child in root.iterchildren(): + if child.tag != MC_ALTERNATE_CONTENT: + yield "element", child + continue + branch = alternate_content_branch(child) + if branch is None: + yield "unreadable", unresolved_alternate_content(child) + else: + yield from iter_effective_children(branch) + def iter_part_blocks(root, parent): """Yield each paragraph/table once, descending through block content controls.""" - for child in root.iterchildren(): + for child_kind, child in iter_effective_children(root): + if child_kind == "unreadable": + yield child_kind, child + continue if child.tag == qn("w:p"): yield "paragraph", Paragraph(child, parent) elif child.tag == qn("w:tbl"): @@ -41,16 +92,25 @@ def iter_part_blocks(root, parent): else: yield from iter_part_blocks(child, parent) -def iter_paragraph_runs(paragraph): - """Include runs wrapped by hyperlinks, fields, revisions, or inline content controls.""" +def iter_paragraph_items(paragraph): + """Include runs/unreadable records in their selected, rendered order.""" def walk(element): - for child in element.iterchildren(): + for child_kind, child in iter_effective_children(element): + if child_kind == "unreadable": + yield child_kind, child + continue if child.tag == qn("w:r"): - yield Run(child, paragraph) + yield "run", Run(child, paragraph) elif child.tag != qn("w:p"): # nested text-box paragraphs are yielded separately yield from walk(child) yield from walk(paragraph._p) +def iter_paragraph_runs(paragraph): + """Yield only runs; paragraph_text separately reports unresolved alternatives.""" + for kind, item in iter_paragraph_items(paragraph): + if kind == "run": + yield item + def legacy_symbol_record(symbol): """Report font-specific w:sym content without guessing at a Unicode mapping.""" font = symbol.get(qn("w:font")) @@ -59,14 +119,47 @@ def legacy_symbol_record(symbol): def run_text(run): pieces = [] - for child in run._r.xpath( - "w:br | w:cr | w:noBreakHyphen | w:ptab | w:t | w:tab | w:sym" - ): - pieces.append(legacy_symbol_record(child) if child.tag == qn("w:sym") else str(child)) + text_tags = { + qn("w:br"), qn("w:cr"), qn("w:noBreakHyphen"), qn("w:ptab"), + qn("w:t"), qn("w:tab"), qn("w:sym"), + } + for child_kind, child in iter_effective_children(run._r): + if child_kind == "unreadable": + pieces.append(f"[unreadable {child['kind']}: {child['reason']}]") + elif child.tag in text_tags: + pieces.append(legacy_symbol_record(child) if child.tag == qn("w:sym") else str(child)) return "".join(pieces) def paragraph_text(paragraph): - return "".join(run_text(run) for run in iter_paragraph_runs(paragraph)) + pieces = [] + for kind, item in iter_paragraph_items(paragraph): + pieces.append( + run_text(item) if kind == "run" + else f"[unreadable {item['kind']}: {item['reason']}]" + ) + return "".join(pieces) + +def cell_paragraph_text(paragraph): + """Render cell controls as visible markers while honoring one MC branch.""" + pieces = [] + for kind, item in iter_paragraph_items(paragraph): + if kind == "unreadable": + pieces.append(f"[unreadable {item['kind']}: {item['reason']}]") + continue + for child_kind, child in iter_effective_children(item._r): + if child_kind == "unreadable": + pieces.append(f"[unreadable {child['kind']}: {child['reason']}]") + elif child.tag == qn("w:t"): + pieces.append(child.text or "") + elif child.tag in (qn("w:tab"), qn("w:ptab")): + pieces.append("") + elif child.tag in (qn("w:br"), qn("w:cr")): + pieces.append("
") + elif child.tag == qn("w:noBreakHyphen"): + pieces.append("-") + elif child.tag == qn("w:sym"): + pieces.append(legacy_symbol_record(child)) + return "".join(pieces) def tc_text(tc, parent): """Cell text rebuilt per paragraph, keeping tabs and breaks visible. @@ -78,26 +171,27 @@ def tc_text(tc, parent): for kind, block in iter_part_blocks(tc, parent): if kind != "paragraph": # nested tables are represented recursively, not duplicated here continue - pieces = [] - for node in block._p.iter(): - if node.tag == qn("w:t"): - pieces.append(node.text or "") - elif node.tag == qn("w:tab"): - pieces.append("") - elif node.tag in (qn("w:br"), qn("w:cr")): - pieces.append("
") - elif node.tag == qn("w:noBreakHyphen"): - pieces.append("-") - elif node.tag == qn("w:sym"): - pieces.append(legacy_symbol_record(node)) - paragraphs.append("".join(pieces)) + paragraphs.append(cell_paragraph_text(block)) return " / ".join(paragraphs) +def iter_content_control_children(root, target_tag): + """Find physical rows/cells through sdtContent without entering nested tables.""" + for child_kind, child in iter_effective_children(root): + if child_kind == "unreadable": + raise ValueError( + f"unresolved AlternateContent while locating {target_tag}: {child}" + ) + if child.tag == target_tag: + yield child + elif child.tag == qn("w:sdt"): + for content in child.findall(qn("w:sdtContent")): + yield from iter_content_control_children(content, target_tag) + def table_content(table): rows = [] - for row in table.rows: + for row_element in iter_content_control_children(table._tbl, qn("w:tr")): rendered_cells = [] - row_properties = row._tr.find(qn("w:trPr")) + row_properties = row_element.find(qn("w:trPr")) grid_before_node = None if row_properties is None else row_properties.find(qn("w:gridBefore")) grid_after_node = None if row_properties is None else row_properties.find(qn("w:gridAfter")) grid_before = 0 if grid_before_node is None else int(grid_before_node.get(qn("w:val"), "0")) @@ -105,7 +199,7 @@ def table_content(table): column = grid_before # row.cells repeats a merge-origin proxy for every grid position it spans. # Walk physical w:tc elements and expose the merge structure instead. - for cell_element in row._tr.tc_lst: + for cell_element in iter_content_control_children(row_element, qn("w:tc")): cell_properties = cell_element.find(qn("w:tcPr")) grid_span = None if cell_properties is None else cell_properties.find(qn("w:gridSpan")) colspan = 1 if grid_span is None else int(grid_span.get(qn("w:val"), "1")) @@ -154,8 +248,14 @@ Notes: when a table is yielded, so table text is not also emitted as prose; `table_content()` handles nested tables recursively and emits `grid_before`, `grid_after`, `column`, `colspan`, and `vMerge` metadata for physical cells instead of duplicating merge-origin text through - `row.cells`. The row-level grid omissions are required for nonuniform tables whose cells do - not start in logical column zero or do not extend to the final grid column. + `row.cells`. It walks physical rows and cells through `w:sdtContent`; python-docx's public row + and cell collections omit those wrapped elements. The row-level grid omissions are required + for nonuniform tables whose cells do not start in logical column zero or do not extend to the + final grid column. + `mc:AlternateContent` is evaluated once: the first Choice whose required namespace URIs this + extractor understands wins, otherwise the Fallback is used. An alternative without either a + supported Choice or a Fallback is reported as unreadable; mutually exclusive branches must + never be concatenated. Imported `w:altChunk` HTML/RTF/document parts are not modeled by python-docx; the traversal reports their relationship target and content type as `unreadable` instead of silently presenting an incomplete extraction. Convert them with a trusted office renderer before diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md b/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md index 7e763a7..fa16703 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md @@ -160,6 +160,9 @@ MAX_IMAGE_SOURCE_PIXELS = 25_000_000 MAX_TOTAL_IMAGE_SOURCE_PIXELS = 50_000_000 MAX_IMAGE_RENDER_PIXELS = 4_000_000 MAX_TOTAL_IMAGE_RENDER_PIXELS = 20_000_000 +MAX_DRAWING_PATHS = 1_000 +MAX_DRAWING_RENDER_PIXELS = 4_000_000 +MAX_TOTAL_DRAWING_RENDER_PIXELS = 20_000_000 def viewable_images(page): """Render bounded placement clips; unknown visibility keeps the page nonblank.""" @@ -200,6 +203,71 @@ def viewable_images(page): visible.append(placement) return placements, visible, False +def drawing_bounds(drawing): + """Include stroke width around the path geometry returned by PyMuPDF.""" + try: + rectangle = fitz.Rect(drawing.get("rect")) + except (TypeError, ValueError): + raise ValueError("drawing has no finite bounding rectangle") from None + if not all(math.isfinite(value) for value in rectangle) or rectangle.is_infinite: + raise ValueError("drawing has no finite bounding rectangle") + rectangle.normalize() + path_type = drawing.get("type") + has_stroke = ( + "s" in path_type if isinstance(path_type, str) + else drawing.get("color") is not None + ) + if has_stroke: + width = drawing.get("width", 0) + if isinstance(width, bool): + raise ValueError("drawing has an invalid stroke width") + try: + width = float(width or 0) + except (TypeError, ValueError): + raise ValueError("drawing has an invalid stroke width") from None + if not math.isfinite(width) or width < 0: + raise ValueError("drawing has an invalid stroke width") + # PDF hairlines (width 0) still paint one device pixel at render time. + padding = max(width / 2, 0.5 if width == 0 else 0) + rectangle = fitz.Rect( + rectangle.x0 - padding, rectangle.y0 - padding, + rectangle.x1 + padding, rectangle.y1 + padding, + ) + return rectangle + +def viewable_drawings(page): + """Render bounded path clips; unknown visibility keeps the page nonblank.""" + try: + drawings = page.get_drawings() + except (RuntimeError, ValueError): + return [], [], True + if len(drawings) > MAX_DRAWING_PATHS: + return drawings, [], True + visible = [] + total_render_pixels = 0 + for drawing in drawings: + try: + bounds = drawing_bounds(drawing) + except ValueError: + return drawings, visible, True + clip = visible_clip(page, bounds) + if clip is None: + continue + render_pixels = math.ceil(clip.width) * math.ceil(clip.height) + total_render_pixels += render_pixels + if (render_pixels > MAX_DRAWING_RENDER_PIXELS + or total_render_pixels > MAX_TOTAL_DRAWING_RENDER_PIXELS): + return drawings, visible, True + try: + pixmap = page.get_pixmap(clip=clip, alpha=True, annots=False) + except (RuntimeError, ValueError): + return drawings, visible, True + if not pixmap.alpha: + return drawings, visible, True + if any(pixmap.samples[pixmap.n - 1::pixmap.n]): + visible.append(drawing) + return drawings, visible, False + def rendered_interactives(page, items): rendered = [] visibility_unknown = False @@ -271,12 +339,13 @@ for page in doc: "rotation": page.rotation, }) image_placements, visible_images, image_visibility_unknown = viewable_images(page) - drawings = page.get_drawings() + drawings, visible_drawings, drawing_visibility_unknown = viewable_drawings(page) widgets, annotations, links, interaction_visibility_unknown = viewable_interactives(page) is_blank = not ( - page.get_text().strip() or visible_images or drawings + page.get_text().strip() or visible_images or visible_drawings or widgets or annotations or links - or image_visibility_unknown or interaction_visibility_unknown + or image_visibility_unknown or drawing_visibility_unknown + or interaction_visibility_unknown ) print(page.number + 1, "media_size:", media_size, "crop_size:", crop_size, "rotation:", page.rotation, "text_len:", len(page.get_text()), @@ -284,7 +353,9 @@ for page in doc: "image_placements:", len(image_placements), "visible_images:", len(visible_images), "image_visibility_unknown:", image_visibility_unknown, - "drawings:", len(drawings), + "drawing_paths:", len(drawings), + "visible_drawings:", len(visible_drawings), + "drawing_visibility_unknown:", drawing_visibility_unknown, "widgets:", len(widgets), "annotations:", len(annotations), "links:", len(links), "interaction_visibility_unknown:", interaction_visibility_unknown, "blank:", is_blank) @@ -295,7 +366,7 @@ print("crop_size_consistent:", len({row["crop_size"] for row in page_geometry}) ## Checks worth automating -- **Blank page detection**: flag only when text, visible painted image placements, drawings, +- **Blank page detection**: flag only when text, visible painted image placements and vector paths, and viewable widgets, annotations, and links are all absent. Ignore interactive objects carrying invisible, hidden, or no-view flags, as well as empty, off-page, or unrendered appearances. `page.get_images()` lists every image XObject resource, including unused resources, and also @@ -304,7 +375,11 @@ print("crop_size_consistent:", len({row["crop_size"] for row in page_geometry}) clipping. Treat those boxes as diagnostics only. The bounded `viewable_images()` render checks alpha for each page-intersecting placement; a count, source-pixel, render-pixel, or decoder limit returns `image_visibility_unknown=True`, which must keep the page nonblank. Interactive form - fields are widgets rather + `page.get_drawings()` similarly reports paths that are off-page, fully clipped, hidden by page + state, or painted with zero opacity. The bounded `viewable_drawings()` checks each + page-intersecting path clip against the actual alpha render; path-count, render-pixel, or renderer + limits return `drawing_visibility_unknown=True`, which also keeps the page nonblank. Interactive + form fields are widgets rather than page text, so a three-content-stream predicate alone would misclassify a usable form page as blank. A blank page after generation usually means an overflowing flowable created it. - **Font inventory**: `page.get_fonts()` lists referenced fonts, including non-embedded base @@ -323,6 +398,6 @@ print("crop_size_consistent:", len({row["crop_size"] for row in page_geometry}) `pypdf.PdfReader` cross-check when provenance is unknown. Report findings as a table (page, media size, crop size, rotation, text chars, resource images, -image placements, visible images, image visibility unknown, drawings, widgets, annotations, -links, interaction visibility unknown, blank) - it +image placements, visible images, image visibility unknown, drawing paths, visible drawings, +drawing visibility unknown, widgets, annotations, links, interaction visibility unknown, blank) - it is what every downstream decision hangs off. diff --git a/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md b/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md index 9f6820d..b6e1ae2 100644 --- a/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md +++ b/plugins/Hylouis233/document-skills/skills/pptx/references/edit.md @@ -161,6 +161,8 @@ def iter_text_targets(path, shape): if shape.has_table: for row_index, row in enumerate(shape.table.rows): for column_index, cell in enumerate(row.cells): + if cell.is_spanned: # covered merge slots can retain stale, non-rendered text + continue yield f"{path}/table[{row_index},{column_index}]", cell.text_frame candidates = [] @@ -202,9 +204,10 @@ prs.save("input-edited.pptx") template, masters, notes, and animations. Edit in place, save to a new path. 2. Address shapes by slide index + shape name or matched text, and **require exactly one match** with an explicit exception (never a Python `assert`, which `python -O` removes). - The locator must search both shape text frames and every table cell, retaining a stable - `/table[row,column]` suffix. If copy repeats inside one table, set `target_location` as well as - the slide/shape selectors rather than choosing one. + The locator must search both shape text frames and every editable table cell, skipping grid + slots covered by a merge and retaining a stable `/table[row,column]` suffix. If copy repeats + inside one table, set `target_location` as well as the slide/shape selectors rather than + choosing one. 3. For formatted text, change `run.text` only when the target is wholly inside one run. Assigning `paragraph.text` or `text_frame.text` rebuilds runs and can discard run formatting and links. If the target spans runs, stop and make an explicitly reviewed run/XML edit. diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md b/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md index 68913a6..3fc76e8 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md @@ -12,12 +12,17 @@ contracts below. ## Step 0 - Check the toolchain ```bash -python -c "import openpyxl; print(openpyxl.__version__)" +python -c "import defusedxml, openpyxl; print(openpyxl.__version__, defusedxml.__version__)" ``` - `.csv`/`.tsv` -> standard library `csv` module is fine and often better (streaming). - `.xlsx`/`.xlsm`/`.xltx` -> openpyxl. +- Safe package preflight requires `defusedxml==0.7.1`; do not fall back to the standard XML + parser for untrusted OOXML parts. - Macro preservation: openpyxl keeps VBA in `.xlsm` only with `keep_vba=True` on load and save. +- Before **any** `openpyxl.load_workbook()` of an existing package, copy and use the bounded + same-handle loader in [references/package.md](references/package.md). `read_only=True` does + not bound shared strings, styles, or other package parts parsed during initialization. ## Step 1 - Classify the task @@ -28,6 +33,7 @@ python -c "import openpyxl; print(openpyxl.__version__)" | Build a new workbook (data + formulas + chart) | [references/create.md](references/create.md) | | CSV/TSV in or out, messy data cleanup | [references/csv.md](references/csv.md) | | Conditional formatting, structured tables, pivot-style aggregation | [references/formatting.md](references/formatting.md) | +| Safely open an existing OOXML package | [references/package.md](references/package.md) | ## Step 2 - Contracts that always apply @@ -46,14 +52,16 @@ python -c "import openpyxl; print(openpyxl.__version__)" `wb.calculation.calcMode`. You still cannot read results back without opening the file in a real spreadsheet app; verify formula strings and ranges structurally instead (see postcheck). -5. **Dimensions**: `ws.max_row`/`ws.max_column` reflect used range - trust them over guesses; - but scan for trailing blank-but-formatted rows when a file "looks" bigger than its data. +5. **Dimensions**: treat ``, `ws.max_row`, and `ws.max_column` as untrusted hints. + Discover logical cells with the sparse worksheet-XML scan in the read route; never expand an + unknown rectangular range merely to find its bounds. 6. Save to a new path first; overwrite only on explicit request. ## Step 3 - Postcheck (mandatory) -Save this as `postcheck.py`, then pass the output path followed by every sheet the task should -produce, for example `python postcheck.py output.xlsx Sales Summary`: +Save this as `postcheck.py`, copy `load_validated_workbook()` and its dependencies from +[references/package.md](references/package.md), then pass the output path followed by every +sheet the task should produce, for example `python postcheck.py output.xlsx Sales Summary`: ```python import openpyxl @@ -70,7 +78,7 @@ expected_number_formats = { expected_formulas = { # "Sales": {"D2": "=C2*1.08"}, } -wb = openpyxl.load_workbook(output_path) +wb = load_validated_workbook(output_path) def require(condition, message): if not condition: diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md index 3546957..d26f0d7 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/csv.md @@ -90,20 +90,28 @@ formula-injection protection**; do not present a raw export as safe to open in a if isinstance(value, str) and value.startswith("="): cell.data_type = "s" # openpyxl otherwise promotes it to an XLSX formula ``` -- XLSX -> CSV: use a separate `data_only=True` read so formulas export the cached values users - see, not formula strings. Pair it with a formula-preserving read and report missing caches. +- XLSX -> CSV: use a `data_only=True` read so formulas export the cached values users + see, not formula strings. Pair it with the sparse XML profile and report missing caches. `openpyxl` exposes both a missing cache and a present empty-string cache as `None`, so consult the worksheet XML. A nonempty `` is cached; an empty `` is a valid displayed blank only when the formula cell explicitly has string-result type `t="str"`. A missing `` or the bare empty `` that openpyxl writes for an uncalculated formula must fail closed. Cached values - can still be stale until a spreadsheet application recalculates the workbook: + can still be stale until a spreadsheet application recalculates the workbook. Copy the + bounded helpers from [package.md](package.md) and `worksheet_xml_profile()` from + [read.md](read.md) first: raw ZIP inspection and the value workbook must share one already + validated source identity. Refuse a logical CSV rectangle above the explicit cell budget; + never call an unbounded `iter_rows()` after `reset_dimensions()`. ```python import csv + import os import openpyxl import posixpath import zipfile from pathlib import Path + from tempfile import mkstemp + from openpyxl.utils import get_column_letter + from openpyxl.utils.cell import coordinate_to_tuple from xml.etree import ElementTree as ET FORMULA_OPERATORS = ("=", "+", "-", "@", "＝", "＋", "－", "＠") @@ -113,6 +121,9 @@ formula-injection protection**; do not present a raw export as safe to open in a CELL_TAG = f"{{{MAIN_NS}}}c" FORMULA_TAG = f"{{{MAIN_NS}}}f" VALUE_TAG = f"{{{MAIN_NS}}}v" + INLINE_STRING_TAG = f"{{{MAIN_NS}}}is" + MAX_EXPLICIT_CELLS = 1_000_000 + MAX_CSV_EXPORT_CELLS = 5_000_000 def begins_spreadsheet_formula(value): index = 0 @@ -145,73 +156,62 @@ formula-injection protection**; do not present a raw export as safe to open in a item.attrib["Target"] for item in relationships.iter(f"{{{PKG_REL_NS}}}Relationship") if item.attrib["Id"] == relationship_id ) - if target.startswith("/"): - return target.lstrip("/") - return posixpath.normpath(posixpath.join("xl", target)) - - def cached_formula_coordinates(archive, part): - """Find formula cells with a nonempty cache or an explicitly typed empty-string cache.""" - cached = set() - coordinate = None - cell_type = None - has_formula = value_seen = False - value_text = None - with archive.open(part) as source: - for event, element in ET.iterparse(source, events=("start", "end")): - if event == "start" and element.tag == CELL_TAG: - coordinate = element.attrib["r"] - cell_type = element.attrib.get("t") - has_formula = value_seen = False - value_text = None - elif event == "end" and coordinate is not None: - if element.tag == FORMULA_TAG: - has_formula = True - elif element.tag == VALUE_TAG: - value_seen = True - value_text = element.text - elif element.tag == CELL_TAG: - valid_cache = value_seen and ( - value_text not in (None, "") or cell_type == "str" - ) - if has_formula and valid_cache: - cached.add(coordinate) - coordinate = None - element.clear() - elif event == "end": - element.clear() - return cached + part = target.lstrip("/") if target.startswith("/") else posixpath.normpath( + posixpath.join("xl", target) + ) + require(part in archive.namelist(), f"worksheet part is missing: {part}") + return part input_path = "input.xlsx" sheet_name = "Data" export_mode = "safe" # use "raw" only for explicitly requested trusted machine interchange - with zipfile.ZipFile(input_path) as archive: - part = worksheet_part(archive, sheet_name) - cached_formula_cells = cached_formula_coordinates(archive, part) - formula_wb = openpyxl.load_workbook(input_path, read_only=True, data_only=False) - value_wb = openpyxl.load_workbook(input_path, read_only=True, data_only=True) - formula_ws, value_ws = formula_wb[sheet_name], value_wb[sheet_name] - # Producer-written metadata can look plausible while truncating real cells. - # Reset both paired streams before their first iter_rows() call. - formula_ws.reset_dimensions() - value_ws.reset_dimensions() - missing_caches = [] output_path = Path("output.csv") - temporary_path = output_path.with_suffix(output_path.suffix + ".tmp") - with temporary_path.open("w", newline="", encoding="utf-8") as output: - writer = csv.writer(output, delimiter=delimiter_for(output_path)) - for formula_row, value_row in zip(formula_ws.iter_rows(), value_ws.iter_rows()): - for formula_cell, value_cell in zip(formula_row, value_row): - if (formula_cell.data_type == "f" and value_cell.value is None - and formula_cell.coordinate not in cached_formula_cells): - missing_caches.append(formula_cell.coordinate) - writer.writerow([ - spreadsheet_csv_field(cell.value, mode=export_mode) for cell in value_row - ]) - formula_wb.close() - value_wb.close() - if missing_caches: - temporary_path.unlink(missing_ok=True) - raise RuntimeError(f"formula cells have no cached value: {missing_caches}") + temporary_path = None + try: + descriptor, temporary_name = mkstemp( + dir=output_path.parent, prefix=f".{output_path.name}.", suffix=".tmp" + ) + temporary_path = Path(temporary_name) + os.close(descriptor) + with validated_xlsx_source(input_path) as package_source: + value_wb = openpyxl.load_workbook(package_source, read_only=True, data_only=True) + try: + package_source.seek(0) + with zipfile.ZipFile(package_source) as archive: + part = worksheet_part(archive, sheet_name) + profile = worksheet_xml_profile(archive, part) + if profile["missing_formula_count"]: + raise RuntimeError( + "formula cells have no cached value: " + f"{profile['missing_formula_samples']}" + ) + value_ws = value_wb[sheet_name] + value_ws.reset_dimensions() + if profile["bounds"] is None: + rows = iter(()) + else: + min_row, min_column, max_row, max_column = profile["bounds"] + export_cells = (max_row - min_row + 1) * (max_column - min_column + 1) + require(export_cells <= MAX_CSV_EXPORT_CELLS, + f"CSV export rectangle is too large: {export_cells} cells") + rows = value_ws.iter_rows( + min_row=min_row, min_col=min_column, + max_row=max_row, max_col=max_column, + values_only=True, + ) + with temporary_path.open("w", newline="", encoding="utf-8") as output: + writer = csv.writer(output, delimiter=delimiter_for(output_path)) + for value_row in rows: + writer.writerow([ + spreadsheet_csv_field(value, mode=export_mode) for value in value_row + ]) + finally: + value_wb.close() + except Exception: + if temporary_path is not None: + temporary_path.unlink(missing_ok=True) + raise + require(temporary_path is not None, "CSV temporary output was not created") temporary_path.replace(output_path) ``` diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md index 4bb8a92..5951081 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md @@ -1,5 +1,10 @@ # Edit an existing workbook +Copy the bounded loaders from [package.md](package.md) into the script. The same preflight must +run before the main editable load and before any raw ZIP/XML round-trip inventory. Define +`load_with_round_trip_audit()` from the prerequisite block later on this page before executing +the edit block; the workbook it returns is the one that must be edited and saved. + ```python import openpyxl from datetime import date @@ -7,7 +12,14 @@ from openpyxl.formula import Tokenizer from openpyxl.styles import Font from openpyxl.utils.cell import coordinate_to_tuple, range_boundaries -wb = openpyxl.load_workbook("input.xlsx") # NOT data_only: that would drop all formulas +approved_feature_loss = False # set True only after showing the inventory to the user +wb, dropped_parts, stripped_extensions = load_with_round_trip_audit("input.xlsx") +if (dropped_parts or stripped_extensions) and not approved_feature_loss: + wb.close() + raise RuntimeError( + f"openpyxl would drop parts={dropped_parts!r}, extensions={stripped_extensions!r}; " + "report this exact inventory and obtain confirmation before editing" + ) ws = wb["Data"] def formula_text(value): @@ -38,16 +50,14 @@ def drawing_anchor_rows(drawing): rows.append(marker.row + 1) return tuple(rows) -def sparse_formula_cells(sheet): - """Walk instantiated formula cells only; worksheet bounds can span the full grid.""" +def sparse_cells(sheet): + """Walk instantiated cells only; worksheet bounds can span the full grid.""" if not hasattr(sheet, "_cells"): raise RuntimeError("structural edits require a normal writable Worksheet") - return ( - cell for cell in sorted( - sheet._cells.values(), key=lambda cell: (cell.row, cell.column) - ) - if cell.data_type == "f" - ) + return sorted(sheet._cells.values(), key=lambda cell: (cell.row, cell.column)) + +def sparse_formula_cells(sheet): + return (cell for cell in sparse_cells(sheet) if cell.data_type == "f") def structural_references(workbook): """Inventory formulas/ranges that insert_rows/delete_rows will not rewrite.""" @@ -59,6 +69,17 @@ def structural_references(workbook): for cell in sparse_formula_cells(sheet): refs.append(("cell formula", f"{owner}!{cell.coordinate}", formula_text(cell.value))) + for cell in sparse_cells(sheet): + hyperlink = cell.hyperlink + if hyperlink is None: + continue + location = getattr(hyperlink, "location", None) + target = getattr(hyperlink, "target", None) + refs.append(( + "cell hyperlink", + f"{owner}!{cell.coordinate}", + (getattr(hyperlink, "ref", None), location, target), + )) for table in sheet.tables.values(): refs.append(("table", owner + "!" + table.name, table.ref)) for merged_range in sheet.merged_cells.ranges: @@ -116,7 +137,7 @@ def formula_may_intersect_rows(owner_sheet, formula, shifted_sheet, start_row): tokens = Tokenizer(formula).items # These functions can manufacture references from strings or numeric # offsets that the RANGE-token audit below cannot see or rewrite safely. - unmodeled_reference_functions = {"indirect", "offset", "address"} + unmodeled_reference_functions = {"indirect", "offset", "address", "hyperlink"} if any( token.type == "FUNC" and token.subtype == "OPEN" and token.value.rstrip("(").rsplit(":", 1)[-1] @@ -271,25 +292,34 @@ wb.save("input-edited.xlsx") stripped.extend((name, uri, children) for _ in range(count)) return sorted(stripped, key=repr) - def round_trip_changes(path, **load_options): - before_names, before_extensions = archive_inventory(path) - wb = openpyxl.load_workbook(path, **load_options) # same options as the real edit - with TemporaryFile() as output: - wb.save(output) - output.seek(0) - after_names, after_extensions = archive_inventory(output) - wb.close() + def load_with_round_trip_audit_from_source(source, **load_options): + """Dry-run one workbook, then return a fresh edit copy from the same source.""" + require(not load_options.get("read_only"), + "round-trip audit requires a normal writable Workbook") + source.seek(0) + before_names, before_extensions = archive_inventory(source) + source.seek(0) + audit_workbook = openpyxl.load_workbook(source, **load_options) + try: + with TemporaryFile() as output: + audit_workbook.save(output) + output.seek(0) + after_names, after_extensions = archive_inventory(output) + finally: + audit_workbook.close() dropped = sorted(before_names - after_names) stripped_extensions = stripped_extension_records( before_extensions, after_extensions, before_names & after_names ) - return dropped, stripped_extensions + # Saving can consume image streams and other resources. Reload a fresh editable + # workbook from the same already validated identity instead of returning the dry-run copy. + source.seek(0) + editable_workbook = openpyxl.load_workbook(source, **load_options) + return editable_workbook, dropped, stripped_extensions - dropped, stripped = round_trip_changes("input.xlsx") - if dropped or stripped: - print("WARNING: saving with openpyxl will drop:", dropped) - print("WARNING: saving with openpyxl will strip (worksheet, URI, content):", stripped) - # report to the user and get confirmation before the first save + def load_with_round_trip_audit(path, **load_options): + with validated_xlsx_source(path) as source: + return load_with_round_trip_audit_from_source(source, **load_options) ``` (openpyxl re-serializes every sheet it touches, so byte-identity of sheets is not a @@ -302,7 +332,11 @@ wb.save("input-edited.xlsx") - `insert_rows`/`delete_rows` move cells but do **not** rewrite range references for you. Before structural edits, inventory cell formulas plus workbook defined names, tables, merged ranges, print areas/titles, autofilters, data validations, conditional formatting, and - chart-series formulas as above. + chart-series formulas and cell hyperlinks as above. Every hyperlink rewrite plan must update + the anchor `ref` when its cell moves; an internal link must also update a destination stored in + `location` or a `target` beginning with `#` when that destination moves. Treat formula-based + `HYPERLINK()` references as dynamic and require a manual rewrite plan because their destination + is a string token, not an ordinary range token. Refuse the insertion until every dependency that can intersect the shifted region has an explicit rewrite; after the edit, rerun both inventories and verify the expected references. - Styling: import `Font` and assign the style to each cell. A range such as `ws["A1:F1"]` @@ -312,7 +346,7 @@ wb.save("input-edited.xlsx") - Freeze panes and autofilter improve usability cheaply: `ws.freeze_panes = "A2"; ws.auto_filter.ref = ws.dimensions`. - Merged cells: avoid creating new merges; writing into a non-anchor merged cell raises. -- `.xlsm`: `load_workbook(path, keep_vba=True)` and save with the same suffix, or macros are +- `.xlsm`: `load_validated_workbook(path, keep_vba=True)` and save with the same suffix, or macros are stripped. - Do not delete sheets unless asked; hide instead (`ws.sheet_state = "hidden"`) when the goal is a cleaner tab bar. diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/formatting.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/formatting.md index 2b1cc79..5deeaab 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/formatting.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/formatting.md @@ -87,7 +87,10 @@ one you took: 1. **Formula sheet (live, recalculates)** - the default. A `SUMIFS`/`COUNTIFS`/`AVERAGEIFS` grid keyed on a unique-values column reproduces most pivot outputs and stays a formula per contract rule 1. Build every sheet reference from the real source sheet's name - - hard-coding `Data!` breaks on any workbook whose sheet is named differently: + hard-coding `Data!` breaks on any workbook whose sheet is named differently. Copy + `validated_xlsx_source()` from [package.md](package.md) and + `load_with_round_trip_audit_from_source()` plus its inventory dependencies from + [edit.md](edit.md) before running this route: ```python import openpyxl @@ -98,51 +101,51 @@ one you took: MAIN_NS = "http://schemas.openxmlformats.org/spreadsheetml/2006/main" DOC_REL_NS = "http://schemas.openxmlformats.org/officeDocument/2006/relationships" PKG_REL_NS = "http://schemas.openxmlformats.org/package/2006/relationships" + MAX_AGGREGATION_ROW_SPAN = 100_000 - def cached_formula_coordinates_for_sheet(path, sheet_name): + def cached_formula_coordinates_for_sheet(archive, sheet_name, wanted_coordinates): """Distinguish a typed cached blank from a formula with no cached result.""" - with zipfile.ZipFile(path) as archive: - workbook = ET.fromstring(archive.read("xl/workbook.xml")) - sheet = next( - item for item in workbook.iter(f"{{{MAIN_NS}}}sheet") - if item.attrib["name"] == sheet_name - ) - relationship_id = sheet.attrib[f"{{{DOC_REL_NS}}}id"] - relationships = ET.fromstring(archive.read("xl/_rels/workbook.xml.rels")) - target = next( - item.attrib["Target"] - for item in relationships.iter(f"{{{PKG_REL_NS}}}Relationship") - if item.attrib["Id"] == relationship_id - ) - part = target.lstrip("/") if target.startswith("/") else posixpath.normpath( - posixpath.join("xl", target) - ) - cached = set() - coordinate = cell_type = value_text = None - has_formula = value_seen = False - with archive.open(part) as source: - for event, element in ET.iterparse(source, events=("start", "end")): - if event == "start" and element.tag == f"{{{MAIN_NS}}}c": - coordinate = element.attrib["r"] - cell_type = element.attrib.get("t") - has_formula = value_seen = False - value_text = None - elif event == "end" and coordinate is not None: - if element.tag == f"{{{MAIN_NS}}}f": - has_formula = True - elif element.tag == f"{{{MAIN_NS}}}v": - value_seen = True - value_text = element.text - elif element.tag == f"{{{MAIN_NS}}}c": - valid_cache = value_seen and ( - value_text not in (None, "") or cell_type == "str" - ) - if has_formula and valid_cache: - cached.add(coordinate) - coordinate = None - element.clear() - elif event == "end": - element.clear() + workbook = ET.fromstring(archive.read("xl/workbook.xml")) + sheet = next( + item for item in workbook.iter(f"{{{MAIN_NS}}}sheet") + if item.attrib["name"] == sheet_name + ) + relationship_id = sheet.attrib[f"{{{DOC_REL_NS}}}id"] + relationships = ET.fromstring(archive.read("xl/_rels/workbook.xml.rels")) + target = next( + item.attrib["Target"] + for item in relationships.iter(f"{{{PKG_REL_NS}}}Relationship") + if item.attrib["Id"] == relationship_id + ) + part = target.lstrip("/") if target.startswith("/") else posixpath.normpath( + posixpath.join("xl", target) + ) + cached = set() + coordinate = cell_type = value_text = None + has_formula = value_seen = False + with archive.open(part) as source: + for event, element in ET.iterparse(source, events=("start", "end")): + if event == "start" and element.tag == f"{{{MAIN_NS}}}c": + coordinate = element.attrib["r"] + cell_type = element.attrib.get("t") + has_formula = value_seen = False + value_text = None + elif event == "end" and coordinate is not None: + if element.tag == f"{{{MAIN_NS}}}f": + has_formula = True + elif element.tag == f"{{{MAIN_NS}}}v": + value_seen = True + value_text = element.text + elif element.tag == f"{{{MAIN_NS}}}c": + valid_cache = value_seen and ( + value_text not in (None, "") or cell_type == "str" + ) + if has_formula and valid_cache and coordinate in wanted_coordinates: + cached.add(coordinate) + coordinate = None + element.clear() + elif event == "end": + element.clear() return cached def sheet_ref(sheet): @@ -151,31 +154,76 @@ one you took: escaped = sheet.title.replace("'", "''") return f"'{escaped}'!" - src = sheet_ref(ws) # e.g. "'Sales'!" or "'Raw Data'!" - source_path = "input.xlsx" # the same original path used to load formula-preserving `wb` - cached_region_formulas = cached_formula_coordinates_for_sheet(source_path, ws.title) - value_wb = openpyxl.load_workbook(source_path, read_only=True, data_only=True) - value_ws = value_wb[ws.title] - value_ws.reset_dimensions() - regions = [] - seen_region_keys = set() - missing_region_caches = [] - source_rows = ws.iter_rows(min_row=2, min_col=1, max_col=1) - value_rows = value_ws.iter_rows(min_row=2, min_col=1, max_col=1) - for source_row, value_row in zip(source_rows, value_rows): - source_cell, value_cell = source_row[0], value_row[0] - region = value_cell.value if source_cell.data_type == "f" else source_cell.value - if (source_cell.data_type == "f" and region is None - and source_cell.coordinate not in cached_region_formulas): - missing_region_caches.append(source_cell.coordinate) - continue - if region is None or region == "": # keep valid falsey values: 0 and False - continue - key = (type(region), region) # do not collapse False and numeric 0 - if key not in seen_region_keys: - seen_region_keys.add(key) - regions.append(region) # stable source order; no mixed-type sort - value_wb.close() + source_path = "input.xlsx" + source_sheet_name = "Data" # select this from the user's request / initial inventory + approved_feature_loss = False # set True only after showing the exact audit to the user + with validated_xlsx_source(source_path) as package_source: + wb, dropped_parts, stripped_extensions = load_with_round_trip_audit_from_source( + package_source, data_only=False + ) + if (dropped_parts or stripped_extensions) and not approved_feature_loss: + wb.close() + raise RuntimeError( + f"openpyxl would drop parts={dropped_parts!r}, " + f"extensions={stripped_extensions!r}; obtain confirmation before editing" + ) + ws = wb[source_sheet_name] + src = sheet_ref(ws) # e.g. "'Sales'!" or "'Raw Data'!" + require(hasattr(ws, "_cells"), + "aggregation requires a normal writable Worksheet") + source_cells = sorted( + (cell for cell in ws._cells.values() + if cell.row >= 2 and cell.column == 1 and cell.value is not None), + key=lambda cell: cell.row, + ) + if source_cells: + min_source_row, max_source_row = source_cells[0].row, source_cells[-1].row + row_span = max_source_row - min_source_row + 1 + require(row_span <= MAX_AGGREGATION_ROW_SPAN, + f"aggregation row span is too large: {row_span}") + wanted_formula_coordinates = { + cell.coordinate for cell in source_cells if cell.data_type == "f" + } + package_source.seek(0) + with zipfile.ZipFile(package_source) as archive: + cached_region_formulas = cached_formula_coordinates_for_sheet( + archive, ws.title, wanted_formula_coordinates + ) + package_source.seek(0) + value_wb = openpyxl.load_workbook( + package_source, read_only=True, data_only=True + ) + try: + value_ws = value_wb[ws.title] + value_ws.reset_dimensions() + regions = [] + seen_region_keys = set() + missing_region_caches = [] + if source_cells: + source_by_row = {cell.row: cell for cell in source_cells} + value_rows = value_ws.iter_rows( + min_row=min_source_row, max_row=max_source_row, + min_col=1, max_col=1, + ) + for row_index, value_row in enumerate(value_rows, start=min_source_row): + source_cell = source_by_row.get(row_index) + if source_cell is None: + continue + value_cell = value_row[0] + region = (value_cell.value if source_cell.data_type == "f" + else source_cell.value) + if (source_cell.data_type == "f" and region is None + and source_cell.coordinate not in cached_region_formulas): + missing_region_caches.append(source_cell.coordinate) + continue + if region is None or region == "": # keep valid falsey values: 0 and False + continue + key = (type(region), region) # keep False distinct from numeric 0 + if key not in seen_region_keys: + seen_region_keys.add(key) + regions.append(region) # stable order; no mixed-type sort + finally: + value_wb.close() if missing_region_caches: raise RuntimeError( f"aggregation keys have no cached value: {missing_region_caches}" @@ -198,8 +246,8 @@ one you took: write values, and **label the sheet** ("values as of generation, not recalculated"). 3. **User's Excel/template pivot** - when the workbook already has slicers or a pivot the - user maintains, edit around it and re-run the `round_trip_changes` check from - [edit.md](edit.md) before saving. + user maintains, edit around it and run the feature-loss audit from [edit.md](edit.md) before + saving. Use `load_with_round_trip_audit_from_source()` and edit only the workbook it returns. ## Postcheck additions diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/package.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/package.md new file mode 100644 index 0000000..372bb9a --- /dev/null +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/package.md @@ -0,0 +1,318 @@ +# Bounded XLSX package loading + +Every route that opens an existing `.xlsx`, `.xlsm`, or `.xltx` must preflight the package +before `openpyxl` or route-specific ZIP/XML inspection parses it. Copy these helpers into the +working script. `validated_xlsx_source()` supplies a bounded raw package handle; +`load_validated_workbook()` loads a normal workbook; and `open_validated_workbook()` keeps a +read-only source handle alive until its workbook is closed. Every helper validates and parses +the same open-file identity. + +```python +from contextlib import contextmanager +from pathlib import Path, PurePosixPath +from defusedxml import ElementTree as DefusedET +from openpyxl.utils.cell import range_boundaries +from tempfile import TemporaryFile +import openpyxl +import zipfile + +CONTENT_TYPES_NS = "http://schemas.openxmlformats.org/package/2006/content-types" +SPREADSHEETML_NAMESPACES = { + "http://schemas.openxmlformats.org/spreadsheetml/2006/main", + "http://purl.oclc.org/ooxml/spreadsheetml/main", +} +WORKSHEET_CELL_TAGS = { + f"{{{namespace}}}c" for namespace in SPREADSHEETML_NAMESPACES +} +WORKSHEET_RANGE_TAGS = { + f"{{{namespace}}}{local_name}" + for namespace in SPREADSHEETML_NAMESPACES + for local_name in ("mergeCell", "hyperlink") +} +WORKSHEET_SINGLE_CELL_REF_TAGS = { + f"{{{namespace}}}comment" for namespace in SPREADSHEETML_NAMESPACES +} +WORKSHEET_MULTI_RANGE_TAGS = { + f"{{{namespace}}}{local_name}" + for namespace in SPREADSHEETML_NAMESPACES + for local_name in ("conditionalFormatting", "dataValidation", "scenarios") +} +MAX_ARCHIVE_BYTES = 200 * 1024 * 1024 +MAX_MEMBERS = 10_000 +MAX_XML_PART = 20 * 1024 * 1024 +MAX_WORKSHEET_XML = 100 * 1024 * 1024 +MAX_ENTRY = 100 * 1024 * 1024 +MAX_TOTAL_UNCOMPRESSED = 500 * 1024 * 1024 +MAX_COMPRESSION_RATIO = 200 +MAX_WORKSHEET_MATERIALIZED_CELLS = 1_000_000 +MAX_WORKSHEET_RANGE_LIST_CHARS = 65_536 +MAX_WORKSHEET_RANGE_TOKENS = 100_000 + +def require(condition, message): + """Security gates must remain active under python -O.""" + if not condition: + raise ValueError(message) + +def canonical_member_name(name): + """Reject ambiguous, absolute, or parent-traversing OPC member names.""" + require(name and "\\" not in name and "\x00" not in name, + f"invalid archive member name: {name!r}") + is_directory = name.endswith("/") + path = name[:-1] if is_directory else name + require(path and not path.startswith("/"), f"absolute archive member path: {name}") + parts = path.split("/") + require(all(part not in ("", ".", "..") for part in parts), + f"non-canonical archive member path: {name}") + canonical = PurePosixPath(*parts).as_posix() + ("/" if is_directory else "") + require(canonical == name, f"non-canonical archive member path: {name}") + return canonical.casefold() + +def safe_xml_root(blob, member_name): + try: + return DefusedET.fromstring( + blob, forbid_dtd=True, forbid_entities=True, forbid_external=True + ) + except Exception as error: + raise ValueError(f"unsafe or malformed XML part: {member_name}") from error + +def worksheet_element_cell_cost(element, member_name): + """Bound cells that openpyxl creates while binding worksheet XML.""" + tag = element.tag + if tag in WORKSHEET_CELL_TAGS: + return 1 + if tag not in WORKSHEET_RANGE_TAGS | WORKSHEET_SINGLE_CELL_REF_TAGS: + return 0 + reference = element.get("ref") or "" + require(reference and len(reference) <= 64, + f"invalid materialized worksheet range in {member_name}: {reference!r}") + try: + min_column, min_row, max_column, max_row = range_boundaries(reference) + except (TypeError, ValueError) as error: + raise ValueError( + f"invalid materialized worksheet range in {member_name}: {reference!r}" + ) from error + require( + all(isinstance(value, int) for value in + (min_column, min_row, max_column, max_row)) + and 1 <= min_column <= max_column <= 16_384 + and 1 <= min_row <= max_row <= 1_048_576, + f"invalid materialized worksheet range in {member_name}: {reference!r}", + ) + if tag in WORKSHEET_SINGLE_CELL_REF_TAGS: + require( + min_column == max_column and min_row == max_row, + f"comment reference must identify one cell in {member_name}: {reference!r}", + ) + return (max_column - min_column + 1) * (max_row - min_row + 1) + +def worksheet_element_range_token_cost(element, member_name): + """Bound MultiCellRange objects constructed from worksheet sqref lists.""" + if element.tag not in WORKSHEET_MULTI_RANGE_TAGS: + return 0 + reference = element.get("sqref") or "" + require( + len(reference) <= MAX_WORKSHEET_RANGE_LIST_CHARS, + f"worksheet range list is too long in {member_name}", + ) + return len(reference.split()) + +def read_bounded_member(archive, info, limit, *, capture): + chunks = [] if capture else None + actual_size = 0 + with archive.open(info) as stream: + while chunk := stream.read(64 * 1024): + actual_size += len(chunk) + require(actual_size <= limit, f"part exceeded read limit: {info.filename}") + if chunks is not None: + chunks.append(chunk) + require(actual_size == info.file_size, f"size mismatch: {info.filename}") + return b"".join(chunks) if chunks is not None else actual_size + +def validate_xlsx_package(source): + """Bound every member before openpyxl sees this same seekable source.""" + source.seek(0, 2) + compressed_size = source.tell() + source.seek(0) + require(compressed_size <= MAX_ARCHIVE_BYTES, "compressed XLSX file size above limit") + try: + archive_context = zipfile.ZipFile(source) + except zipfile.BadZipFile as error: + raise ValueError("input is not a valid XLSX ZIP package") from error + with archive_context as archive: + infos = archive.infolist() + require(len(infos) <= MAX_MEMBERS, "archive member count above limit") + names = {info.filename for info in infos} + require(len(names) == len(infos), "duplicate archive member names are unsafe") + canonical_names = [canonical_member_name(info.filename) for info in infos] + require(len(set(canonical_names)) == len(canonical_names), + "archive member names collide case-insensitively") + require(all(not (info.flag_bits & 1) for info in infos), + "encrypted archive members are not supported") + require( + {"[Content_Types].xml", "_rels/.rels", "xl/workbook.xml"} <= names, + "required XLSX package parts are missing", + ) + require(sum(info.file_size for info in infos) <= MAX_TOTAL_UNCOMPRESSED, + "declared total uncompressed size above limit") + + content_types_info = next( + info for info in infos if info.filename == "[Content_Types].xml" + ) + require(content_types_info.file_size <= MAX_XML_PART, + "oversized XML part: [Content_Types].xml") + require( + content_types_info.file_size / max(content_types_info.compress_size, 1) + <= MAX_COMPRESSION_RATIO, + "suspicious compression ratio: [Content_Types].xml", + ) + content_types_blob = read_bounded_member( + archive, content_types_info, MAX_XML_PART, capture=True + ) + content_types_root = safe_xml_root(content_types_blob, "[Content_Types].xml") + require(content_types_root.tag == f"{{{CONTENT_TYPES_NS}}}Types", + "invalid content-types root") + default_types = {} + override_types = {} + for declaration in content_types_root: + if declaration.tag == f"{{{CONTENT_TYPES_NS}}}Default": + key = (declaration.get("Extension") or "").casefold() + target = default_types + elif declaration.tag == f"{{{CONTENT_TYPES_NS}}}Override": + part_name = declaration.get("PartName") or "" + require(part_name.startswith("/"), "invalid content-type part name") + key = canonical_member_name(part_name[1:]) + target = override_types + else: + continue + content_type = declaration.get("ContentType") or "" + require(key and content_type and key not in target, + "invalid or duplicate content-type declaration") + target[key] = content_type.partition(";")[0].strip().casefold() + + actual_total = len(content_types_blob) + worksheet_materialized_cells = 0 + worksheet_range_tokens = 0 + for info in infos: + require(info.file_size <= MAX_ENTRY, f"oversized part: {info.filename}") + require( + info.file_size / max(info.compress_size, 1) <= MAX_COMPRESSION_RATIO, + f"suspicious compression ratio: {info.filename}", + ) + extension = info.filename.rpartition(".")[2].casefold() + content_type = override_types.get( + canonical_member_name(info.filename), default_types.get(extension, "") + ) + is_xml = ( + info.filename.casefold().endswith((".xml", ".rels")) + or content_type in {"application/xml", "text/xml"} + or content_type.endswith("+xml") + ) + is_worksheet_xml = ( + is_xml + and info.filename.casefold().startswith("xl/worksheets/") + and content_type.endswith(".worksheet+xml") + ) + if is_xml: + xml_limit = MAX_WORKSHEET_XML if is_worksheet_xml else MAX_XML_PART + require(info.file_size <= xml_limit, + f"oversized XML part: {info.filename}") + if info.filename == "[Content_Types].xml": + blob = content_types_blob + else: + member_data = read_bounded_member( + archive, info, xml_limit if is_xml else MAX_ENTRY, + capture=is_xml and not is_worksheet_xml, + ) + actual_size = ( + len(member_data) if is_xml and not is_worksheet_xml else member_data + ) + actual_total += actual_size + require(actual_total <= MAX_TOTAL_UNCOMPRESSED, + "archive exceeded total read limit") + blob = member_data if is_xml and not is_worksheet_xml else None + if is_worksheet_xml: + try: + with archive.open(info) as stream: + for _, element in DefusedET.iterparse( + stream, events=("end",), forbid_dtd=True, + forbid_entities=True, forbid_external=True, + ): + worksheet_materialized_cells += worksheet_element_cell_cost( + element, info.filename + ) + worksheet_range_tokens += worksheet_element_range_token_cost( + element, info.filename + ) + require( + worksheet_materialized_cells + <= MAX_WORKSHEET_MATERIALIZED_CELLS, + "worksheet cell materialization budget exceeded", + ) + require( + worksheet_range_tokens <= MAX_WORKSHEET_RANGE_TOKENS, + "worksheet range-token budget exceeded", + ) + element.clear() + except Exception as error: + if isinstance(error, ValueError): + raise + raise ValueError( + f"unsafe or malformed XML part: {info.filename}" + ) from error + elif is_xml and info.filename != "[Content_Types].xml": + xml_root = safe_xml_root(blob, info.filename) + for element in xml_root.iter(): + worksheet_materialized_cells += worksheet_element_cell_cost( + element, info.filename + ) + worksheet_range_tokens += worksheet_element_range_token_cost( + element, info.filename + ) + require( + worksheet_materialized_cells + <= MAX_WORKSHEET_MATERIALIZED_CELLS, + "worksheet cell materialization budget exceeded", + ) + require( + worksheet_range_tokens <= MAX_WORKSHEET_RANGE_TOKENS, + "worksheet range-token budget exceeded", + ) + source.seek(0) + +@contextmanager +def validated_xlsx_source(path): + """Yield a private validated snapshot, isolated from later path mutation.""" + with Path(path).open("rb") as external_source, TemporaryFile() as source: + copied_size = 0 + while chunk := external_source.read(64 * 1024): + copied_size += len(chunk) + require(copied_size <= MAX_ARCHIVE_BYTES, + "compressed XLSX file size above limit") + source.write(chunk) + source.flush() + source.seek(0) + validate_xlsx_package(source) + source.seek(0) + yield source + +def load_validated_workbook(path, **options): + """Load a non-streaming workbook from the exact handle that passed preflight.""" + require(not options.get("read_only"), + "use open_validated_workbook for read_only=True") + with validated_xlsx_source(path) as source: + return openpyxl.load_workbook(source, **options) + +@contextmanager +def open_validated_workbook(path, **options): + """Keep the validated source alive until its openpyxl workbook is closed.""" + with validated_xlsx_source(path) as source: + workbook = openpyxl.load_workbook(source, **options) + try: + yield workbook + finally: + workbook.close() +``` + +Do not weaken these bounds in a route-specific copy. A task may choose smaller limits. If a +valid workbook exceeds a limit, report the limit and ask the user before raising it; never retry +an untrusted package without bounds. diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md index 17426e9..5e3a7aa 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/read.md @@ -1,10 +1,13 @@ # Read / profile a workbook +Copy the bounded loaders from [package.md](package.md) into the script before running this +route. The package preflight must finish before either openpyxl or the raw XML scan starts. + ```python -import openpyxl import posixpath import zipfile from openpyxl.utils import get_column_letter +from openpyxl.utils.cell import coordinate_to_tuple from xml.etree import ElementTree as ET MAIN_NS = "http://schemas.openxmlformats.org/spreadsheetml/2006/main" @@ -13,6 +16,9 @@ PKG_REL_NS = "http://schemas.openxmlformats.org/package/2006/relationships" CELL_TAG = f"{{{MAIN_NS}}}c" FORMULA_TAG = f"{{{MAIN_NS}}}f" VALUE_TAG = f"{{{MAIN_NS}}}v" +INLINE_STRING_TAG = f"{{{MAIN_NS}}}is" +MAX_EXPLICIT_CELLS = 1_000_000 +MAX_PROFILE_RECTANGLE_CELLS = 100_000 def worksheet_part(archive, sheet_name): workbook = ET.fromstring(archive.read("xl/workbook.xml")) @@ -26,127 +32,156 @@ def worksheet_part(archive, sheet_name): item.attrib["Target"] for item in relationships.iter(f"{{{PKG_REL_NS}}}Relationship") if item.attrib["Id"] == relationship_id ) - return target.lstrip("/") if target.startswith("/") else posixpath.normpath( + part = target.lstrip("/") if target.startswith("/") else posixpath.normpath( posixpath.join("xl", target) ) + require(part in archive.namelist(), f"worksheet part is missing: {part}") + return part -def cached_formula_coordinates(archive, part): - """Find nonempty caches plus explicitly typed empty-string caches.""" - cached = set() - coordinate = cell_type = value_text = None - has_formula = value_seen = False +def worksheet_xml_profile(archive, part): + """Discover physical logical cells without expanding their rectangular gaps.""" + min_row = min_column = max_row = max_column = None + explicit_cell_count = 0 + formula_count = missing_formula_count = 0 + missing_formula_samples = [] + coordinate = cell_type = value_text = formula_display = None + has_formula = value_seen = inline_string_seen = False with archive.open(part) as source: for event, element in ET.iterparse(source, events=("start", "end")): if event == "start" and element.tag == CELL_TAG: - coordinate = element.attrib["r"] + coordinate = element.attrib.get("r") + require(coordinate is not None, f"cell without a coordinate in {part}") cell_type = element.attrib.get("t") - has_formula = value_seen = False - value_text = None + has_formula = value_seen = inline_string_seen = False + value_text = formula_display = None elif event == "end" and coordinate is not None: if element.tag == FORMULA_TAG: has_formula = True + if element.text is not None: + formula_display = "=" + element.text + else: + details = ", ".join( + f"{key}={value!r}" for key, value in sorted(element.attrib.items()) + ) + formula_display = f"" elif element.tag == VALUE_TAG: value_seen = True value_text = element.text + elif element.tag == INLINE_STRING_TAG and cell_type == "inlineStr": + inline_string_seen = True elif element.tag == CELL_TAG: + explicit_cell_count += 1 + require(explicit_cell_count <= MAX_EXPLICIT_CELLS, + f"too many explicit worksheet cells in {part}") + row_index, column_index = coordinate_to_tuple(coordinate) + require(1 <= row_index <= 1_048_576 and 1 <= column_index <= 16_384, + f"cell coordinate outside XLSX limits: {coordinate}") + scalar_value = value_seen and ( + value_text not in (None, "") or cell_type == "str" + ) + populated = has_formula or scalar_value or inline_string_seen + if populated: + min_row = row_index if min_row is None else min(min_row, row_index) + min_column = (column_index if min_column is None + else min(min_column, column_index)) + max_row = row_index if max_row is None else max(max_row, row_index) + max_column = (column_index if max_column is None + else max(max_column, column_index)) valid_cache = value_seen and ( value_text not in (None, "") or cell_type == "str" ) - if has_formula and valid_cache: - cached.add(coordinate) + if has_formula: + formula_count += 1 + if not valid_cache: + missing_formula_count += 1 + if len(missing_formula_samples) < 10: + missing_formula_samples.append((coordinate, formula_display)) coordinate = None element.clear() elif event == "end": element.clear() - return cached - -input_path = "input.xlsx" -formula_wb = openpyxl.load_workbook(input_path, read_only=True, data_only=False) -value_wb = openpyxl.load_workbook(input_path, read_only=True, data_only=True) -with zipfile.ZipFile(input_path) as archive: - cached_formulas = { - sheet_name: cached_formula_coordinates(archive, worksheet_part(archive, sheet_name)) - for sheet_name in formula_wb.sheetnames + if max_row is None: + bounds, extent, first_populated_row = None, "A1:A1", None + else: + bounds = (min_row, min_column, max_row, max_column) + extent = ( + f"{get_column_letter(min_column)}{min_row}:" + f"{get_column_letter(max_column)}{max_row}" + ) + first_populated_row = min_row + return { + "part": part, + "bounds": bounds, + "extent": extent, + "first_populated_row": first_populated_row, + "formula_count": formula_count, + "missing_formula_count": missing_formula_count, + "missing_formula_samples": missing_formula_samples, + "explicit_cell_count": explicit_cell_count, } -print("sheets:", value_wb.sheetnames) -def formula_text(value): - if isinstance(value, str): - return value - if text := getattr(value, "text", None): # ArrayFormula - return text - # DataTableFormula has no .text; render stable, useful attributes, not an address repr. - fields = ("ref", "r1", "r2", "dt2D", "dtr", "ca", "del1", "del2") - details = ", ".join( - f"{name}={getattr(value, name)!r}" for name in fields if hasattr(value, name) +def bounded_sample_rows(worksheet, profile, *, max_cells=MAX_PROFILE_RECTANGLE_CELLS): + """Return a header plus at most five rows, rejecting expansion before iter_rows().""" + if profile["bounds"] is None: + return iter(()) + min_row, min_column, max_row, max_column = profile["bounds"] + sample_max_row = min(max_row, min_row + 5) + sample_cells = (sample_max_row - min_row + 1) * (max_column - min_column + 1) + require(sample_cells <= max_cells, + f"sample rectangle is too large: {worksheet.title} ({sample_cells} cells)") + return worksheet.iter_rows( + min_row=min_row, min_col=min_column, + max_row=sample_max_row, max_col=max_column, + values_only=True, ) - return f"{type(value).__name__}({details})" -def discover_dimension(worksheet): - """Scan an untrusted read-only stream without relying on its .""" - worksheet.reset_dimensions() - min_row = min_column = max_row = max_column = None - for row in worksheet.iter_rows(): - for cell in row: - # A styled-but-empty cell has coordinates in a read-only stream but is not data. - # Formula cells remain part of the logical range even when their cache is missing. - if getattr(cell, "value", None) is None and getattr(cell, "data_type", None) != "f": - continue - row_index = getattr(cell, "row", None) # EmptyCell has no coordinates - column_index = getattr(cell, "column", None) - if row_index is None or column_index is None: - continue - min_row = row_index if min_row is None else min(min_row, row_index) - min_column = column_index if min_column is None else min(min_column, column_index) - max_row = row_index if max_row is None else max(max_row, row_index) - max_column = column_index if max_column is None else max(max_column, column_index) - if max_row is None: - return "A1:A1", None - extent = ( - f"{get_column_letter(min_column)}{min_row}:" - f"{get_column_letter(max_column)}{max_row}" - ) - return extent, min_row +def worksheet_declared_dimension(worksheet): + """Return producer metadata without forcing an unsized read-only worksheet scan.""" + if worksheet.max_row is None or worksheet.max_column is None: + return None + return worksheet.calculate_dimension() -# Profile EVERY sheet by default; only narrow when the task names a specific sheet. -for sheet_name in value_wb.sheetnames: - formula_ws = formula_wb[sheet_name] - value_ws = value_wb[sheet_name] - # Read-only iteration is bounded by the sheet's metadata. A - # non-Excel producer can declare a plausible but truncated range (for - # example A1:B2 while data continues below it), so treat that metadata as - # untrusted: reset both streams and discover the real bounds before reading. - declared = formula_ws.calculate_dimension() - discovered, first_populated_row = discover_dimension(formula_ws) - # The value stream cannot distinguish a missing formula cache from a displayed blank, - # so it follows the logical bounds discovered from the formula-preserving stream. - value_ws.reset_dimensions() - if discovered != declared: - print(f"--- {sheet_name} --- declared {declared!r}; discovered real extent:") - print(f"--- {sheet_name} --- dims:", discovered) +input_path = "input.xlsx" +with validated_xlsx_source(input_path) as package_source: + value_wb = openpyxl.load_workbook(package_source, read_only=True, data_only=True) + try: + package_source.seek(0) + with zipfile.ZipFile(package_source) as archive: + profiles = { + sheet_name: worksheet_xml_profile( + archive, worksheet_part(archive, sheet_name) + ) + for sheet_name in value_wb.sheetnames + } + print("sheets:", value_wb.sheetnames) - if first_populated_row is None: - rows, header = iter(()), None - else: - rows = value_ws.iter_rows(min_row=first_populated_row, values_only=True) - header = next(rows, None) - print("header:", header) - for i, row in enumerate(rows): - if i >= 5: break - print(row) + # Profile EVERY sheet by default; only narrow when the task names a specific sheet. + for sheet_name in value_wb.sheetnames: + value_ws = value_wb[sheet_name] + declared = worksheet_declared_dimension(value_ws) + profile = profiles[sheet_name] + discovered = profile["extent"] + value_ws.reset_dimensions() + if discovered != declared: + print(f"--- {sheet_name} --- declared {declared!r}; discovered real extent:") + print(f"--- {sheet_name} --- dims:", discovered, + "explicit cells:", profile["explicit_cell_count"]) + + if profile["bounds"] is None: + rows, header = iter(()), None + else: + rows = bounded_sample_rows(value_ws, profile) + header = next(rows, None) + print("header:", header) + for row in rows: + print(row) - missing_cache_count = 0 - for formula_row, value_row in zip(formula_ws.iter_rows(), value_ws.iter_rows()): - for formula_cell, value_cell in zip(formula_row, value_row): - if (formula_cell.data_type == "f" and value_cell.value is None - and formula_cell.coordinate not in cached_formulas[sheet_name]): - missing_cache_count += 1 - if missing_cache_count <= 10: - print("formula without cached value:", formula_cell.coordinate, - formula_text(formula_cell.value)) - print("formulas without cached values:", missing_cache_count) -formula_wb.close() -value_wb.close() + for coordinate, formula in profile["missing_formula_samples"]: + print("formula without cached value:", coordinate, formula) + print("formulas:", profile["formula_count"], + "without cached values:", profile["missing_formula_count"]) + finally: + value_wb.close() ``` ## Rules @@ -157,10 +192,15 @@ value_wb.close() discovered first populated row; leading blank rows are not a header. - `read_only=True` streams large files; you lose random access (`ws["B2"]` works but is slow in read_only mode - iterate instead). +- A worksheet with no `` is unsized in read-only mode. Report its declared extent + as `None` and continue with the sparse XML profile; never call + `calculate_dimension(force=True)`, which scans the worksheet before the profile budgets apply. - `data_only=True` gives cached values. A file saved by a library (never opened in Excel) - may return `None` for formulas with no cache. Compare each cell with the corresponding cell - from a `data_only=False` workbook and detect formulas with `cell.data_type == "f"`; array and - data-table formulas may not be strings beginning with `=`, so a string-prefix test is incomplete. + may have no cache for a formula. Use the sparse XML profile's `` and `` inventory to + detect those cells without a second rectangular workbook traversal; array, shared, and + data-table formula records may not contain ordinary formula text. - Mixed-type columns: profile them (`set(type(v).__name__ for v in col)`) before converting; a column that is mostly numbers with a few text cells is a data-quality finding, not noise. -- Never load the full sheet into memory to "look at it" when `iter_rows` with a break would do. +- Never use an unbounded `iter_rows()` even with a later `break`: it can manufacture empty + cells before the caller regains control. Pass all four bounds from the sparse XML profile and + enforce a rectangle budget first. diff --git a/plugins/Hylouis233/document-skills/tests/docx_fixture.py b/plugins/Hylouis233/document-skills/tests/docx_fixture.py index 2a599a5..81a3657 100644 --- a/plugins/Hylouis233/document-skills/tests/docx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/docx_fixture.py @@ -156,8 +156,61 @@ def validate_docx_package(path): ) # ---- read.md includes block/inline content controls and controlled tables ------- -def iter_part_blocks(root, parent): +MC_NAMESPACE = "http://schemas.openxmlformats.org/markup-compatibility/2006" +MC_ALTERNATE_CONTENT = f"{{{MC_NAMESPACE}}}AlternateContent" +MC_CHOICE = f"{{{MC_NAMESPACE}}}Choice" +MC_FALLBACK = f"{{{MC_NAMESPACE}}}Fallback" +SUPPORTED_MC_NAMESPACE_URIS = { + MC_NAMESPACE, + qn("w:p").split("}", 1)[0][1:], + qn("r:id").split("}", 1)[0][1:], +} + + +def alternate_content_branch(element): + fallback = None + for child in element.iterchildren(): + if child.tag == MC_CHOICE: + required_prefixes = (child.get("Requires") or "").split() + if required_prefixes and all( + child.nsmap.get(prefix) in SUPPORTED_MC_NAMESPACE_URIS + for prefix in required_prefixes + ): + return child + elif child.tag == MC_FALLBACK and fallback is None: + fallback = child + return fallback + + +def unresolved_alternate_content(element): + return { + "kind": "AlternateContent", + "requires": [ + (child.get("Requires") or "").split() + for child in element.iterchildren() + if child.tag == MC_CHOICE + ], + "reason": "no supported Choice and no Fallback", + } + + +def iter_effective_children(root): for child in root.iterchildren(): + if child.tag != MC_ALTERNATE_CONTENT: + yield "element", child + continue + branch = alternate_content_branch(child) + if branch is None: + yield "unreadable", unresolved_alternate_content(child) + else: + yield from iter_effective_children(branch) + + +def iter_part_blocks(root, parent): + for child_kind, child in iter_effective_children(root): + if child_kind == "unreadable": + yield child_kind, child + continue if child.tag == qn("w:p"): yield "paragraph", Paragraph(child, parent) elif child.tag == qn("w:tbl"): @@ -177,16 +230,25 @@ def iter_part_blocks(root, parent): yield from iter_part_blocks(child, parent) -def iter_paragraph_runs(paragraph): +def iter_paragraph_items(paragraph): def walk(element): - for child in element.iterchildren(): + for child_kind, child in iter_effective_children(element): + if child_kind == "unreadable": + yield child_kind, child + continue if child.tag == qn("w:r"): - yield Run(child, paragraph) + yield "run", Run(child, paragraph) elif child.tag != qn("w:p"): yield from walk(child) yield from walk(paragraph._p) +def iter_paragraph_runs(paragraph): + for kind, item in iter_paragraph_items(paragraph): + if kind == "run": + yield item + + def legacy_symbol_record(symbol): font = symbol.get(qn("w:font")) character = symbol.get(qn("w:char")) @@ -195,38 +257,102 @@ def legacy_symbol_record(symbol): def run_text(run): pieces = [] - for child in run._r.xpath( - "w:br | w:cr | w:noBreakHyphen | w:ptab | w:t | w:tab | w:sym" - ): - pieces.append(legacy_symbol_record(child) if child.tag == qn("w:sym") else str(child)) + text_tags = { + qn("w:br"), qn("w:cr"), qn("w:noBreakHyphen"), qn("w:ptab"), + qn("w:t"), qn("w:tab"), qn("w:sym"), + } + for child_kind, child in iter_effective_children(run._r): + if child_kind == "unreadable": + pieces.append(f"[unreadable {child['kind']}: {child['reason']}]") + elif child.tag in text_tags: + pieces.append(legacy_symbol_record(child) if child.tag == qn("w:sym") else str(child)) return "".join(pieces) def paragraph_text(paragraph): - return "".join(run_text(run) for run in iter_paragraph_runs(paragraph)) + pieces = [] + for kind, item in iter_paragraph_items(paragraph): + pieces.append( + run_text(item) if kind == "run" + else f"[unreadable {item['kind']}: {item['reason']}]" + ) + return "".join(pieces) + + +def cell_paragraph_text(paragraph): + pieces = [] + for kind, item in iter_paragraph_items(paragraph): + if kind == "unreadable": + pieces.append(f"[unreadable {item['kind']}: {item['reason']}]") + continue + for child_kind, child in iter_effective_children(item._r): + if child_kind == "unreadable": + pieces.append(f"[unreadable {child['kind']}: {child['reason']}]") + elif child.tag == qn("w:t"): + pieces.append(child.text or "") + elif child.tag in (qn("w:tab"), qn("w:ptab")): + pieces.append("") + elif child.tag in (qn("w:br"), qn("w:cr")): + pieces.append("
") + elif child.tag == qn("w:noBreakHyphen"): + pieces.append("-") + elif child.tag == qn("w:sym"): + pieces.append(legacy_symbol_record(child)) + return "".join(pieces) + + +def tc_text(tc, parent): + paragraphs = [] + for kind, block in iter_part_blocks(tc, parent): + if kind == "paragraph": + paragraphs.append(cell_paragraph_text(block)) + return " / ".join(paragraphs) + + +def iter_content_control_children(root, target_tag): + for child_kind, child in iter_effective_children(root): + if child_kind == "unreadable": + raise ValueError( + f"unresolved AlternateContent while locating {target_tag}: {child}" + ) + if child.tag == target_tag: + yield child + elif child.tag == qn("w:sdt"): + for content in child.findall(qn("w:sdtContent")): + yield from iter_content_control_children(content, target_tag) def table_content(table): rows = [] - for row in table.rows: + for row_element in iter_content_control_children(table._tbl, qn("w:tr")): rendered_cells = [] - row_properties = row._tr.find(qn("w:trPr")) + row_properties = row_element.find(qn("w:trPr")) grid_before_node = None if row_properties is None else row_properties.find(qn("w:gridBefore")) grid_after_node = None if row_properties is None else row_properties.find(qn("w:gridAfter")) grid_before = 0 if grid_before_node is None else int(grid_before_node.get(qn("w:val"), "0")) grid_after = 0 if grid_after_node is None else int(grid_after_node.get(qn("w:val"), "0")) column = grid_before - for cell in row.cells: - items = [] - for kind, block in iter_part_blocks(cell._tc, cell): - if kind == "paragraph": - items.append(paragraph_text(block)) - elif kind == "table": - items.append(table_content(block)) - else: - items.append(block) - rendered_cells.append({"column": column, "items": items}) - column += 1 + for cell_element in iter_content_control_children(row_element, qn("w:tc")): + cell_properties = cell_element.find(qn("w:tcPr")) + grid_span = None if cell_properties is None else cell_properties.find(qn("w:gridSpan")) + colspan = 1 if grid_span is None else int(grid_span.get(qn("w:val"), "1")) + vertical = None if cell_properties is None else cell_properties.find(qn("w:vMerge")) + vertical_merge = None if vertical is None else vertical.get(qn("w:val"), "continue") + nested_tables = [] + unreadable = [] + for kind, block in iter_part_blocks(cell_element, table): + if kind == "table": + nested_tables.append(table_content(block)) + elif kind == "unreadable": + unreadable.append(block) + rendered_cells.append({ + "column": column, "colspan": colspan, + "vMerge": vertical_merge, + "text": tc_text(cell_element, table), + "tables": nested_tables, + "unreadable": unreadable, + }) + column += colspan rows.append({ "grid_before": grid_before, "cells": rendered_cells, "grid_after": grid_after, }) @@ -256,7 +382,6 @@ def wrap_in_sdt(element): row_properties.append(grid_after) nested_table = controlled_table.cell(0, 0).add_table(rows=1, cols=1) nested_table.cell(0, 0).text = "nested한" -wrap_in_sdt(controlled_table._tbl) inline_paragraph = sdt_doc.add_paragraph("before-") inline_run = inline_paragraph.add_run("inline한") inline_paragraph.add_run("-after") @@ -281,6 +406,11 @@ def wrap_in_sdt(element): controlled_table.cell(0, 0)._tc.insert( len(controlled_table.cell(0, 0)._tc) - 1, cell_alt_chunk ) +controlled_row_element = controlled_table.rows[0]._tr +controlled_cell_element = controlled_row_element.tc_lst[0] +wrap_in_sdt(controlled_cell_element) +wrap_in_sdt(controlled_row_element) +wrap_in_sdt(controlled_table._tbl) sdt_doc.save("content-control.docx") sdt_reopened = Document("content-control.docx") check("doc.paragraphs omits block content-control text (negative control)", @@ -305,9 +435,17 @@ def wrap_in_sdt(element): f"-legacy-{legacy_symbol_marker}-visible" in paragraph_text(legacy_paragraph), paragraph_text(legacy_paragraph)) walked_tables = [block for kind, block in walked_blocks if kind == "table"] +check("Table.rows omits a row wrapped by row-level sdtContent (negative control)", + len(walked_tables) == 1 and len(walked_tables[0].rows) == 0) +physical_rows = list(iter_content_control_children(walked_tables[0]._tbl, qn("w:tr"))) +check("tr.tc_lst omits a cell wrapped by cell-level sdtContent (negative control)", + len(physical_rows) == 1 and len(physical_rows[0].tc_lst) == 0) rendered_tables = [table_content(table) for table in walked_tables] check("content-control traversal emits a wrapped table", len(walked_tables) == 1 and "table한" in str(rendered_tables), rendered_tables) +check("table traversal emits the sdt-wrapped physical row and cell", + len(rendered_tables[0]) == 1 and len(rendered_tables[0][0]["cells"]) == 1, + rendered_tables) all_emitted = walked_text + [str(table) for table in rendered_tables] check("table text is emitted exactly once, not again as prose", sum(item.count("table한") for item in all_emitted) == 1, all_emitted) @@ -327,15 +465,112 @@ def wrap_in_sdt(element): } check("altChunk content is reported instead of silently omitted", unreadable_parts == [expected_alt_chunk], unreadable_parts) -cell_unreadable = [ - item for item in rendered_tables[0][0]["cells"][0]["items"] - if isinstance(item, dict) and item.get("kind") == "altChunk" -] +cell_unreadable = rendered_tables[0][0]["cells"][0]["unreadable"] check("altChunk content inside a table cell is also reported", cell_unreadable == [expected_alt_chunk], rendered_tables) check("altChunk payload is not misrepresented as extracted paragraph text", all("IMPORTED ALTCHUNK TEXT" not in text for text in walked_text), walked_text) + +# ---- read.md chooses exactly one mc:AlternateContent branch --------------------- +W14_NAMESPACE = "http://schemas.microsoft.com/office/word/2010/wordml" +W_NAMESPACE = qn("w:p").split("}", 1)[0][1:] + + +def wml_run(text): + run = OxmlElement("w:r") + text_element = OxmlElement("w:t") + text_element.text = text + run.append(text_element) + return run + + +def wml_paragraph(text): + paragraph = OxmlElement("w:p") + paragraph.append(wml_run(text)) + return paragraph + + +def wml_table(text): + source = Document() + table = source.add_table(rows=1, cols=1) + table.cell(0, 0).text = text + return copy.deepcopy(table._tbl) + + +def alternate_content(requires, choice_elements, fallback_elements=None): + element = etree.Element( + MC_ALTERNATE_CONTENT, + nsmap={"mc": MC_NAMESPACE, "w14": W14_NAMESPACE, "w": W_NAMESPACE}, + ) + choice = etree.SubElement(element, MC_CHOICE) + choice.set("Requires", requires) + for child in choice_elements: + choice.append(child) + if fallback_elements is not None: + fallback = etree.SubElement(element, MC_FALLBACK) + for child in fallback_elements: + fallback.append(child) + return element + + +mc_doc = Document() +mc_doc.element.body.insert( + len(mc_doc.element.body) - 1, + alternate_content( + "w14", + [wml_paragraph("UNSUPPORTED-CHOICE-BLOCK"), wml_table("UNSUPPORTED-CHOICE-TABLE")], + [wml_paragraph("FALLBACK-BLOCK"), wml_table("FALLBACK-TABLE")], + ), +) +mc_doc.element.body.insert( + len(mc_doc.element.body) - 1, + alternate_content( + "w", + [wml_paragraph("SUPPORTED-CHOICE-BLOCK")], + [wml_paragraph("UNSELECTED-FALLBACK-BLOCK")], + ), +) +mc_doc.element.body.insert( + len(mc_doc.element.body) - 1, + alternate_content("w14", [wml_paragraph("UNRESOLVED-HIDDEN-BLOCK")]), +) +mc_inline = mc_doc.add_paragraph("INLINE-BEFORE-") +mc_inline._p.append( + alternate_content( + "w14", [wml_run("UNSUPPORTED-CHOICE-INLINE")], [wml_run("FALLBACK-INLINE")] + ) +) +mc_inline.add_run("-INLINE-AFTER") +mc_doc.save("alternate-content.docx") +mc_reopened = Document("alternate-content.docx") +mc_blocks = list(iter_part_blocks(mc_reopened.element.body, mc_reopened)) +mc_paragraph_texts = [paragraph_text(block) for kind, block in mc_blocks if kind == "paragraph"] +mc_tables = [table_content(block) for kind, block in mc_blocks if kind == "table"] +mc_unreadable = [block for kind, block in mc_blocks if kind == "unreadable"] +mc_emitted = mc_paragraph_texts + [str(table) for table in mc_tables] +check("unsupported AlternateContent Choice selects only its Fallback paragraph and table", + any("FALLBACK-BLOCK" in text for text in mc_emitted) + and any("FALLBACK-TABLE" in text for text in mc_emitted) + and all("UNSUPPORTED-CHOICE" not in text for text in mc_emitted), + mc_emitted) +check("supported AlternateContent Choice wins over its Fallback", + any("SUPPORTED-CHOICE-BLOCK" in text for text in mc_emitted) + and all("UNSELECTED-FALLBACK-BLOCK" not in text for text in mc_emitted), + mc_emitted) +check("inline AlternateContent contributes exactly one selected run", + "INLINE-BEFORE-FALLBACK-INLINE-INLINE-AFTER" in mc_paragraph_texts + and all("UNSUPPORTED-CHOICE-INLINE" not in text for text in mc_paragraph_texts), + mc_paragraph_texts) +check("AlternateContent without a usable branch is explicitly unreadable", + mc_unreadable == [{ + "kind": "AlternateContent", + "requires": [["w14"]], + "reason": "no supported Choice and no Fallback", + }], mc_unreadable) +check("unresolved AlternateContent payload is not misrepresented as extracted text", + all("UNRESOLVED-HIDDEN-BLOCK" not in text for text in mc_emitted), mc_emitted) + # Per-run glyph validation must not let a different referenced font hide a missing glyph. fixture_cmaps = {"CJK Face": {ord("漢")}, "Latin Face": {ord("A")}} assigned_runs = [("CJK Face", "漢"), ("Latin Face", "漢")] @@ -387,6 +622,7 @@ def font_slot(character): or 0xFE30 <= codepoint <= 0xFE6F or 0xFF00 <= codepoint <= 0xFFEF or 0x20000 <= codepoint <= 0x3134F + or 0x31350 <= codepoint <= 0x33479 ) return "eastAsia" if uses_east_asian_slot else ("ascii" if codepoint < 128 else "hAnsi") @@ -396,6 +632,12 @@ def font_slot(character): check("Hangul Jamo use the east-Asian font slot", font_slot("ᄒ") == "eastAsia") check("Yi syllables use the east-Asian font slot", font_slot("ꀀ") == "eastAsia") check("CJK compatibility forms use the east-Asian font slot", font_slot("︰") == "eastAsia") +check("Unicode 17 Han Extension H start uses the east-Asian font slot", + font_slot(chr(0x31350)) == "eastAsia") +check("Unicode 17 Han Extension J end uses the east-Asian font slot", + font_slot(chr(0x33479)) == "eastAsia") +check("codepoint after the Unicode 17 Han ranges stays in hAnsi", + font_slot(chr(0x3347A)) == "hAnsi") check("ASCII text keeps the ascii font slot", font_slot("A") == "ascii") # Header/footer parts have no `.document`; effective styles close over the document. @@ -910,6 +1152,7 @@ def font_slot(character): or 0x3130 <= codepoint <= 0x318F or 0xA960 <= codepoint <= 0xA97F or 0xAC00 <= codepoint <= 0xD7FF or 0xF900 <= codepoint <= 0xFAFF or 0x20000 <= codepoint <= 0x3134F + or 0x31350 <= codepoint <= 0x33479 ) else ("ascii" if codepoint < 128 else "hAnsi") check("Hangul syllables use the East Asian slot", font_slot("한") == "eastAsia") @@ -918,6 +1161,10 @@ def font_slot(character): check("Hangul extended-A uses the East Asian slot", font_slot(chr(0xA960)) == "eastAsia") check("Hangul extended-B uses the East Asian slot", font_slot(chr(0xD7B0)) == "eastAsia") check("U+2E80 CJK radical uses the East Asian slot", font_slot(chr(0x2E80)) == "eastAsia") +check("Unicode 17 supplementary Han endpoints use the East Asian slot", + font_slot(chr(0x31350)) == font_slot(chr(0x33479)) == "eastAsia") +check("the codepoint after Unicode 17 Han remains hAnsi", + font_slot(chr(0x3347A)) == "hAnsi") check("Latin stays in the ascii slot", font_slot("A") == "ascii") check("non-CJK fullwidth-range-adjacent Latin-1 stays hAnsi", font_slot("é") == "hAnsi") diff --git a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py index b810119..94be938 100644 --- a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py @@ -537,6 +537,9 @@ def visible_clip(page, rectangle, *, already_rotated=False): MAX_TOTAL_IMAGE_SOURCE_PIXELS = 50_000_000 MAX_IMAGE_RENDER_PIXELS = 4_000_000 MAX_TOTAL_IMAGE_RENDER_PIXELS = 20_000_000 +MAX_DRAWING_PATHS = 1_000 +MAX_DRAWING_RENDER_PIXELS = 4_000_000 +MAX_TOTAL_DRAWING_RENDER_PIXELS = 20_000_000 def viewable_images(page): @@ -578,6 +581,73 @@ def viewable_images(page): return placements, visible, False +def drawing_bounds(drawing): + """Include stroke width around the path geometry returned by PyMuPDF.""" + try: + rectangle = fitz.Rect(drawing.get("rect")) + except (TypeError, ValueError): + raise ValueError("drawing has no finite bounding rectangle") from None + if not all(math.isfinite(value) for value in rectangle) or rectangle.is_infinite: + raise ValueError("drawing has no finite bounding rectangle") + rectangle.normalize() + path_type = drawing.get("type") + has_stroke = ( + "s" in path_type if isinstance(path_type, str) + else drawing.get("color") is not None + ) + if has_stroke: + width = drawing.get("width", 0) + if isinstance(width, bool): + raise ValueError("drawing has an invalid stroke width") + try: + width = float(width or 0) + except (TypeError, ValueError): + raise ValueError("drawing has an invalid stroke width") from None + if not math.isfinite(width) or width < 0: + raise ValueError("drawing has an invalid stroke width") + # PDF hairlines (width 0) still paint one device pixel at render time. + padding = max(width / 2, 0.5 if width == 0 else 0) + rectangle = fitz.Rect( + rectangle.x0 - padding, rectangle.y0 - padding, + rectangle.x1 + padding, rectangle.y1 + padding, + ) + return rectangle + + +def viewable_drawings(page): + """Render bounded path clips; unknown visibility keeps the page nonblank.""" + try: + drawings = page.get_drawings() + except (RuntimeError, ValueError): + return [], [], True + if len(drawings) > MAX_DRAWING_PATHS: + return drawings, [], True + visible = [] + total_render_pixels = 0 + for drawing in drawings: + try: + bounds = drawing_bounds(drawing) + except ValueError: + return drawings, visible, True + clip = visible_clip(page, bounds) + if clip is None: + continue + render_pixels = math.ceil(clip.width) * math.ceil(clip.height) + total_render_pixels += render_pixels + if (render_pixels > MAX_DRAWING_RENDER_PIXELS + or total_render_pixels > MAX_TOTAL_DRAWING_RENDER_PIXELS): + return drawings, visible, True + try: + pixmap = page.get_pixmap(clip=clip, alpha=True, annots=False) + except (RuntimeError, ValueError): + return drawings, visible, True + if not pixmap.alpha: + return drawings, visible, True + if any(pixmap.samples[pixmap.n - 1::pixmap.n]): + visible.append(drawing) + return drawings, visible, False + + def rendered_interactives(page, items): rendered = [] visibility_unknown = False @@ -662,10 +732,13 @@ class VisibleInteractiveProbe: widgets, annotations, links, interaction_visibility_unknown = viewable_interactives( widget_page ) +_, visible_widget_drawings, widget_drawing_visibility_unknown = viewable_drawings( + widget_page +) blank = ( not widget_page.get_text().strip() and not widget_page.get_images() - and not widget_page.get_drawings() + and not visible_widget_drawings and not widget_drawing_visibility_unknown and not widgets and not annotations and not links and not interaction_visibility_unknown ) check("widget-only form page exposes a widget", len(widgets) == 1, len(widgets)) @@ -1179,11 +1252,12 @@ def stamp_placement(page, stamp_box): def inspected_page_is_blank(page): _, visible_images, image_visibility_unknown = viewable_images(page) + _, visible_drawings, drawing_visibility_unknown = viewable_drawings(page) return not ( page.get_text().strip() or visible_images - or page.get_drawings() or list(page.widgets() or ()) + or visible_drawings or list(page.widgets() or ()) or list(page.annots() or ()) or page.get_links() - or image_visibility_unknown + or image_visibility_unknown or drawing_visibility_unknown ) @@ -1191,6 +1265,100 @@ def inspected_page_is_blank(page): check("form-only page exposes a widget", len(form_only_widgets) == 1) check("form-only page is not classified as blank", not form_only_is_blank) +# PyMuPDF inventories invoked paths even when page geometry, clipping, or opacity +# prevents them from contributing a rendered pixel. +drawing_visibility_canvas = canvas.Canvas( + "drawing-visibility.pdf", pagesize=(200, 200), pageCompression=0, +) +drawing_visibility_canvas.setLineWidth(2) +drawing_visibility_canvas.rect(40, 40, 80, 60, stroke=1, fill=0) +drawing_visibility_canvas.showPage() +drawing_visibility_canvas.setLineWidth(2) +drawing_visibility_canvas.rect(250, 40, 80, 60, stroke=1, fill=0) +drawing_visibility_canvas.showPage() +clipping_path = drawing_visibility_canvas.beginPath() +clipping_path.rect(0, 0, 10, 10) +drawing_visibility_canvas.saveState() +drawing_visibility_canvas.clipPath(clipping_path, stroke=0, fill=0) +drawing_visibility_canvas.setLineWidth(2) +drawing_visibility_canvas.rect(40, 40, 80, 60, stroke=1, fill=0) +drawing_visibility_canvas.restoreState() +drawing_visibility_canvas.showPage() +drawing_visibility_canvas.saveState() +drawing_visibility_canvas.setStrokeAlpha(0) +drawing_visibility_canvas.setLineWidth(2) +drawing_visibility_canvas.rect(40, 40, 80, 60, stroke=1, fill=0) +drawing_visibility_canvas.restoreState() +drawing_visibility_canvas.showPage() +no_paint_path = drawing_visibility_canvas.beginPath() +no_paint_path.rect(40, 40, 80, 60) +drawing_visibility_canvas.drawPath(no_paint_path, stroke=0, fill=0) +drawing_visibility_canvas.showPage() +drawing_visibility_canvas.save() + +drawing_visibility_doc = fitz.open("drawing-visibility.pdf") +drawing_visibility_results = [] +for drawing_page in drawing_visibility_doc: + drawing_paths, visible_drawing_paths, drawing_visibility_unknown = viewable_drawings( + drawing_page + ) + drawing_visibility_results.append(( + len(drawing_paths), len(visible_drawing_paths), drawing_visibility_unknown, + inspected_page_is_blank(drawing_page), + )) +check("drawing visibility uses rendered paint rather than raw path presence", + drawing_visibility_results == [ + (1, 1, False, False), # ordinary visible stroke + (1, 0, False, True), # fully off-page stroke + (1, 0, False, True), # stroke fully excluded by the active clip + (1, 0, False, True), # zero-opacity stroke still has a path record + (0, 0, False, True), # a path ended with PDF's no-paint operator + ], drawing_visibility_results) + + +class FailingDrawingRenderPage: + rect = fitz.Rect(0, 0, 200, 200) + rotation_matrix = fitz.Identity + + def get_drawings(self): + return [{"type": "f", "rect": fitz.Rect(20, 20, 80, 80)}] + + def get_pixmap(self, **kwargs): + raise RuntimeError("fixture drawing renderer failure") + + +failed_drawing_paths, failed_visible_drawings, failed_drawing_unknown = viewable_drawings( + FailingDrawingRenderPage() +) +failed_drawing_blank = not (failed_visible_drawings or failed_drawing_unknown) +check("drawing render failures keep blank-page classification fail closed", + len(failed_drawing_paths) == 1 and not failed_visible_drawings + and failed_drawing_unknown and not failed_drawing_blank, + (failed_drawing_paths, failed_visible_drawings, + failed_drawing_unknown, failed_drawing_blank)) + +drawing_budget_results = {} +for budget_name in ( + "MAX_DRAWING_PATHS", "MAX_DRAWING_RENDER_PIXELS", + "MAX_TOTAL_DRAWING_RENDER_PIXELS", +): + original_budget = globals()[budget_name] + globals()[budget_name] = 0 + try: + budget_paths, budget_visible, budget_unknown = viewable_drawings( + drawing_visibility_doc[0] + ) + budget_page_is_blank = inspected_page_is_blank(drawing_visibility_doc[0]) + finally: + globals()[budget_name] = original_budget + drawing_budget_results[budget_name] = ( + len(budget_paths), len(budget_visible), budget_unknown, budget_page_is_blank, + ) +check("every drawing count/render budget fails closed as visibility unknown", + all(result == (1, 0, True, False) + for result in drawing_budget_results.values()), + drawing_budget_results) + # Identical media/crop geometry remains consistent when one page has /Rotate 90. mixed_rotation_writer = PdfWriter() mixed_rotation_writer.append(PdfReader("form.pdf")) @@ -1735,19 +1903,6 @@ def stamp_line_directions(path, page_number): with open("overflow-graphics-rotated.pdf", "wb") as f: rotated_graphics_writer.write(f) - -def drawing_bounds(drawing): - rect = fitz.Rect(drawing["rect"]) - stroke_pad = ( - float(drawing.get("width") or 0) / 2 - if "s" in drawing.get("type", "") else 0 - ) - return fitz.Rect( - rect.x0 - stroke_pad, rect.y0 - stroke_pad, - rect.x1 + stroke_pad, rect.y1 + stroke_pad, - ) - - def overflow_pages(path, password=None): doc = fitz.open(path) if doc.needs_pass and doc.authenticate("") <= 0: diff --git a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py index 914b9e3..c352f0e 100644 --- a/plugins/Hylouis233/document-skills/tests/pptx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pptx_fixture.py @@ -2319,9 +2319,76 @@ def iter_text_targets(path, shape): if shape.has_table: for row_index, row in enumerate(shape.table.rows): for column_index, cell in enumerate(row.cells): + if cell.is_spanned: + continue yield f"{path}/table[{row_index},{column_index}]", cell.text_frame +merged_edit_old, merged_edit_new = "Merged edit target", "Edited merged target" +merged_edit_prs = Presentation() +merged_edit_slide = merged_edit_prs.slides.add_slide(merged_edit_prs.slide_layouts[6]) +merged_edit_table = merged_edit_slide.shapes.add_table( + 2, 2, Inches(1), Inches(1), Inches(4), Inches(2) +).table +merged_edit_table.cell(0, 0).text = merged_edit_old +merged_edit_table.cell(0, 0).merge(merged_edit_table.cell(1, 1)) +# Covered cells can retain stale text in a valid OPC package even though it is not rendered. +for merged_edit_coordinate in ((0, 1), (1, 0), (1, 1)): + merged_edit_table.cell(*merged_edit_coordinate).text = merged_edit_old +merged_edit_prs.save("merged-edit-source.pptx") + +merged_edit_reopened = Presentation("merged-edit-source.pptx") +merged_edit_shape = next( + shape for shape in merged_edit_reopened.slides[0].shapes if shape.has_table +) +merged_edit_candidates = [ + (location, text_frame) + for location, text_frame in iter_text_targets(merged_edit_shape.name, merged_edit_shape) + if merged_edit_old in text_frame.text +] +check( + "merged-table locator ignores stale text in covered merge slots", + len(merged_edit_candidates) == 1 + and merged_edit_candidates[0][0].endswith("/table[0,0]"), + [location for location, _ in merged_edit_candidates], +) +merged_edit_frame = merged_edit_candidates[0][1] +merged_edit_hits = [ + run + for paragraph in merged_edit_frame.paragraphs + for run in paragraph.runs + if merged_edit_old in run.text +] +require(len(merged_edit_hits) == 1, "expected one run in the merged-cell edit target") +merged_edit_hits[0].text = merged_edit_hits[0].text.replace( + merged_edit_old, merged_edit_new, 1 +) +merged_edit_reopened.save("merged-edit-result.pptx") + +merged_edit_result = Presentation("merged-edit-result.pptx") +merged_edit_result_table = next( + shape.table for shape in merged_edit_result.slides[0].shapes if shape.has_table +) +merged_edit_origin = merged_edit_result_table.cell(0, 0) +merged_edit_covered = [ + merged_edit_result_table.cell(*coordinate) + for coordinate in ((0, 1), (1, 0), (1, 1)) +] +check( + "merged-cell run edit persists without changing the merge topology", + merged_edit_origin.text == merged_edit_new + and merged_edit_origin.is_merge_origin + and not merged_edit_origin.is_spanned + and merged_edit_origin.span_width == 2 + and merged_edit_origin.span_height == 2 + and all(cell.is_spanned for cell in merged_edit_covered), + { + "origin": merged_edit_origin.text, + "covered": [(cell.text, cell.is_spanned) for cell in merged_edit_covered], + }, +) + + def iter_postcheck_text_frames(shapes, path=""): for shape in shapes: here = f"{path}/{shape.name}" if path else shape.name diff --git a/plugins/Hylouis233/document-skills/tests/requirements-fixtures.txt b/plugins/Hylouis233/document-skills/tests/requirements-fixtures.txt index fb549f2..39891dc 100644 --- a/plugins/Hylouis233/document-skills/tests/requirements-fixtures.txt +++ b/plugins/Hylouis233/document-skills/tests/requirements-fixtures.txt @@ -1,5 +1,6 @@ lxml==6.0.2 openpyxl==3.1.5 +defusedxml==0.7.1 Pillow==11.3.0 pymupdf==1.27.2.2 pypdf==6.9.2 diff --git a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py index bded508..cb6b30d 100644 --- a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py @@ -7,10 +7,11 @@ # python docx_fixture.py (deps: python-docx, pymupdf, soffice on PATH) import base64 import csv +import os import sys import zipfile from pathlib import Path -from tempfile import TemporaryFile +from tempfile import TemporaryFile, mkstemp import openpyxl @@ -23,6 +24,15 @@ def check(name, cond, extra=""): failures.append(name) +# Execute the canonical package.md snippet itself so fixtures cannot drift from the Skill. +package_reference = ( + Path(__file__).resolve().parents[1] / "skills" / "xlsx" / "references" / "package.md" +) +package_markdown = package_reference.read_text(encoding="utf-8") +package_code = package_markdown.split("```python", 1)[1].split("```", 1)[0] +exec(compile(package_code, str(package_reference), "exec"), globals()) + + # ---- csv.md snippet: sniffed dialect actually reaches the reader --------------- semicolon_csv = "region;units;note\nEU;120;first\nUS;80;second\n" with open("input.csv", "w", newline="", encoding="utf-8") as f: @@ -146,6 +156,284 @@ def delimiter_for(path): with zipfile.ZipFile("plain.xlsx") as zin: payload = {name: zin.read(name) for name in zin.namelist()} +# ---- package.md: preflight runs before openpyxl and bounds every package part ----- +try: + with open_validated_workbook("plain.xlsx") as validated_plain: + ordinary_package_loaded = validated_plain.sheetnames == ["Data", "Keep"] +except Exception as error: + ordinary_package_loaded = False + ordinary_package_error = error +check("bounded XLSX loader accepts an ordinary package", ordinary_package_loaded, + locals().get("ordinary_package_error")) + +compressed_payload = dict(payload) +compressed_payload["xl/styles.xml"] = ( + b'' + + b"A" * (1024 * 1024) + + b"" +) +with zipfile.ZipFile("compressed-xlsx-bomb.xlsx", "w", zipfile.ZIP_DEFLATED) as archive: + for name, data in compressed_payload.items(): + archive.writestr(name, data) +real_load_workbook = openpyxl.load_workbook +preflight_load_calls = [] +openpyxl.load_workbook = lambda *args, **kwargs: preflight_load_calls.append(args) or None +try: + with open_validated_workbook("compressed-xlsx-bomb.xlsx"): + compressed_bomb_rejected = False +except ValueError as error: + compressed_bomb_rejected = "suspicious compression ratio: xl/styles.xml" in str(error) +finally: + openpyxl.load_workbook = real_load_workbook +check("XLSX compression bomb is rejected before openpyxl runs", + compressed_bomb_rejected and preflight_load_calls == [], + (compressed_bomb_rejected, preflight_load_calls)) +check("XLSX preflight gates remain active under optimized Python", + __debug__ or compressed_bomb_rejected) + +original_member_limit = MAX_MEMBERS +try: + with zipfile.ZipFile("plain.xlsx") as archive: + MAX_MEMBERS = len(archive.infolist()) - 1 + with validated_xlsx_source("plain.xlsx"): + member_limit_rejected = False +except ValueError as error: + member_limit_rejected = str(error) == "archive member count above limit" +finally: + MAX_MEMBERS = original_member_limit +check("XLSX member-count limit rejects a real package before parsing", member_limit_rejected) + +metadata_names = [ + name for name in payload + if name.endswith((".xml", ".rels")) and not name.startswith("xl/worksheets/") +] +metadata_test_limit = max(len(payload[name]) for name in metadata_names) + 128 +oversized_metadata = dict(payload) +oversized_metadata["xl/styles.xml"] = ( + b'" +) +with zipfile.ZipFile("oversized-metadata.xlsx", "w", zipfile.ZIP_STORED) as archive: + for name, data in oversized_metadata.items(): + archive.writestr(name, data) +original_xml_limit = MAX_XML_PART +try: + MAX_XML_PART = metadata_test_limit + with validated_xlsx_source("oversized-metadata.xlsx"): + oversized_metadata_rejected = False +except ValueError as error: + oversized_metadata_rejected = str(error) == "oversized XML part: xl/styles.xml" +finally: + MAX_XML_PART = original_xml_limit +check("metadata XML part limit is enforced before openpyxl", oversized_metadata_rejected) + +large_worksheet_payload = dict(payload) +sheet_name = "xl/worksheets/sheet1.xml" +large_worksheet_payload[sheet_name] = large_worksheet_payload[sheet_name].replace( + b"", b"" +) +with zipfile.ZipFile("large-streamed-worksheet.xlsx", "w", zipfile.ZIP_STORED) as archive: + for name, data in large_worksheet_payload.items(): + archive.writestr(name, data) +original_xml_limit = MAX_XML_PART +original_worksheet_limit = MAX_WORKSHEET_XML +try: + MAX_XML_PART = metadata_test_limit + MAX_WORKSHEET_XML = len(large_worksheet_payload[sheet_name]) + 128 + with validated_xlsx_source("large-streamed-worksheet.xlsx"): + streamed_worksheet_passed = True +except Exception as error: + streamed_worksheet_passed = False + streamed_worksheet_error = error +finally: + MAX_XML_PART = original_xml_limit + MAX_WORKSHEET_XML = original_worksheet_limit +check("large worksheet XML uses the bounded streaming limit, not metadata limit", + streamed_worksheet_passed, locals().get("streamed_worksheet_error")) + +unsafe_xml_payload = dict(payload) +unsafe_xml_payload["xl/unsafe.xml"] = ( + '' + ']>&x;' +).encode("utf-16") +with zipfile.ZipFile("unsafe-utf16-xml.xlsx", "w", zipfile.ZIP_STORED) as archive: + for name, data in unsafe_xml_payload.items(): + archive.writestr(name, data) +try: + with validated_xlsx_source("unsafe-utf16-xml.xlsx"): + utf16_dtd_rejected = False +except ValueError as error: + utf16_dtd_rejected = "unsafe or malformed XML part: xl/unsafe.xml" in str(error) +check("defused XML parsing rejects UTF-16 DTD/entity parts", utf16_dtd_rejected) + +semantic_range_payloads = { + "merge-range-bomb.xlsx": ( + b'' + ), + "hyperlink-range-bomb.xlsx": ( + b'' + b'' + ), +} +for malicious_path, range_markup in semantic_range_payloads.items(): + malicious_payload = dict(payload) + malicious_payload[sheet_name] = malicious_payload[sheet_name].replace( + b"", range_markup + b"" + ) + with zipfile.ZipFile(malicious_path, "w", zipfile.ZIP_DEFLATED) as archive: + for name, data in malicious_payload.items(): + archive.writestr(name, data) + +from openpyxl.comments import Comment + +comment_wb = openpyxl.Workbook() +comment_wb.active["A1"].comment = Comment("bounded note", "fixture") +comment_wb.save("comment-base.xlsx") +comment_wb.close() +with zipfile.ZipFile("comment-base.xlsx") as archive: + comment_payload = {name: archive.read(name) for name in archive.namelist()} +comment_part = next( + name for name in comment_payload + if name.casefold().startswith("xl/comments/") and name.casefold().endswith(".xml") +) +comment_payload[comment_part] = comment_payload[comment_part].replace( + b'ref="A1"', b'ref="A1:XFD1048576"', 1 +) +with zipfile.ZipFile("comment-range-bomb.xlsx", "w", zipfile.ZIP_DEFLATED) as archive: + for name, data in comment_payload.items(): + archive.writestr(name, data) + +semantic_expected_errors = { + **{path: "worksheet cell materialization budget exceeded" + for path in semantic_range_payloads}, + "comment-range-bomb.xlsx": "comment reference must identify one cell", +} + +real_load_workbook = openpyxl.load_workbook +semantic_preflight_load_calls = [] +semantic_range_results = {} +openpyxl.load_workbook = ( + lambda *args, **kwargs: semantic_preflight_load_calls.append(args) or None +) +try: + for malicious_path, expected_error in semantic_expected_errors.items(): + try: + with open_validated_workbook(malicious_path): + semantic_range_results[malicious_path] = False + except ValueError as error: + semantic_range_results[malicious_path] = ( + expected_error in str(error) + ) + + original_materialized_cell_limit = MAX_WORKSHEET_MATERIALIZED_CELLS + try: + MAX_WORKSHEET_MATERIALIZED_CELLS = 1 + with open_validated_workbook("plain.xlsx"): + explicit_cell_limit_rejected = False + except ValueError as error: + explicit_cell_limit_rejected = ( + str(error) == "worksheet cell materialization budget exceeded" + ) + finally: + MAX_WORKSHEET_MATERIALIZED_CELLS = original_materialized_cell_limit +finally: + openpyxl.load_workbook = real_load_workbook + +check( + "merge, hyperlink, and comment ranges are bounded before openpyxl runs", + all(semantic_range_results.values()) and semantic_preflight_load_calls == [], + (semantic_range_results, semantic_preflight_load_calls), +) +check( + "explicit worksheet cells share the pre-load materialization budget", + explicit_cell_limit_rejected and semantic_preflight_load_calls == [], + (explicit_cell_limit_rejected, semantic_preflight_load_calls), +) +with open_validated_workbook("comment-base.xlsx") as validated_comment_wb: + valid_comment_loaded = ( + validated_comment_wb.active["A1"].comment is not None + and validated_comment_wb.active["A1"].comment.text == "bounded note" + ) +check("a valid single-cell comment survives bounded loading", valid_comment_loaded) + +range_list_payloads = { + "range-list-budget.xlsx": b'', + "range-list-whole-sheet.xlsx": ( + b'' + ), +} +for range_list_path, range_list_markup in range_list_payloads.items(): + range_list_payload = dict(payload) + range_list_payload[sheet_name] = range_list_payload[sheet_name].replace( + b"", range_list_markup + b"" + ) + with zipfile.ZipFile(range_list_path, "w", zipfile.ZIP_DEFLATED) as archive: + for name, data in range_list_payload.items(): + archive.writestr(name, data) + +original_range_token_limit = MAX_WORKSHEET_RANGE_TOKENS +original_range_list_char_limit = MAX_WORKSHEET_RANGE_LIST_CHARS +real_load_workbook = openpyxl.load_workbook +range_list_load_calls = [] +openpyxl.load_workbook = lambda *args, **kwargs: range_list_load_calls.append(args) or None +try: + MAX_WORKSHEET_RANGE_TOKENS = 2 + try: + with open_validated_workbook("range-list-budget.xlsx"): + range_token_budget_rejected = False + except ValueError as error: + range_token_budget_rejected = str(error) == "worksheet range-token budget exceeded" + MAX_WORKSHEET_RANGE_TOKENS = 1 + try: + with validated_xlsx_source("range-list-whole-sheet.xlsx"): + whole_sheet_range_accepted = True + except Exception as error: + whole_sheet_range_accepted = False + whole_sheet_range_error = error + MAX_WORKSHEET_RANGE_LIST_CHARS = 8 + try: + with validated_xlsx_source("range-list-whole-sheet.xlsx"): + long_range_list_rejected = False + except ValueError as error: + long_range_list_rejected = "worksheet range list is too long" in str(error) +finally: + MAX_WORKSHEET_RANGE_TOKENS = original_range_token_limit + MAX_WORKSHEET_RANGE_LIST_CHARS = original_range_list_char_limit + openpyxl.load_workbook = real_load_workbook + +check( + "worksheet range-list token growth is bounded before openpyxl runs", + range_token_budget_rejected and range_list_load_calls == [], + (range_token_budget_rejected, range_list_load_calls), +) +check( + "one whole-sheet sqref costs one range token rather than its cell area", + whole_sheet_range_accepted, + locals().get("whole_sheet_range_error"), +) +check("worksheet sqref character growth is bounded", long_range_list_rejected) + +Path("snapshot-source.xlsx").write_bytes(Path("plain.xlsx").read_bytes()) +replacement_wb = openpyxl.Workbook() +replacement_wb.active["A1"] = "UNVALIDATED" +replacement_wb.save("snapshot-replacement.xlsx") +replacement_wb.close() +with validated_xlsx_source("snapshot-source.xlsx") as snapshot_source: + Path("snapshot-source.xlsx").write_bytes( + Path("snapshot-replacement.xlsx").read_bytes() + ) + snapshot_wb = openpyxl.load_workbook(snapshot_source, read_only=True, data_only=True) + try: + snapshot_value = snapshot_wb.active["A1"].value + finally: + snapshot_wb.close() +check( + "validated source is a private snapshot isolated from later path changes", + snapshot_value == "Region", + snapshot_value, +) + # simulate an unsupported extension part (what a slicer/queries part looks like in the zip) payload["xl/slicers/slicer1.xml"] = b"" payload["xl/media/large.bin"] = b"x14:" * 32_768 # binary payload must never be marker-scanned @@ -172,7 +460,7 @@ def delimiter_for(path): payload["xl/worksheets/sheet2.xml"] = payload["xl/worksheets/sheet2.xml"].replace( b"", EXT_LIST_DROP + b"") -with zipfile.ZipFile("extended.xlsx", "w", zipfile.ZIP_DEFLATED) as zout: +with zipfile.ZipFile("extended.xlsx", "w", zipfile.ZIP_STORED) as zout: for name, data in payload.items(): zout.writestr(name, data) @@ -180,7 +468,7 @@ def delimiter_for(path): partial_payload["xl/worksheets/sheet1.xml"] = partial_payload[ "xl/worksheets/sheet1.xml" ].replace(EXT_LIST_BOTH, EXT_LIST_KEEP) -with zipfile.ZipFile("partial-extension.xlsx", "w", zipfile.ZIP_DEFLATED) as zout: +with zipfile.ZipFile("partial-extension.xlsx", "w", zipfile.ZIP_STORED) as zout: for name, data in partial_payload.items(): zout.writestr(name, data) @@ -246,28 +534,86 @@ def stripped_extension_records(before, after, common_names): return sorted(stripped, key=repr) -def round_trip_changes(path, **load_options): - before_names, before_extensions = archive_inventory(path) - wb = openpyxl.load_workbook(path, **load_options) # same options as the real edit - with TemporaryFile() as output: - wb.save(output) - output.seek(0) - after_names, after_extensions = archive_inventory(output) - wb.close() +def load_with_round_trip_audit_from_source(source, **load_options): + require(not load_options.get("read_only"), + "round-trip audit requires a normal writable Workbook") + source.seek(0) + before_names, before_extensions = archive_inventory(source) + source.seek(0) + audit_workbook = openpyxl.load_workbook(source, **load_options) + try: + with TemporaryFile() as output: + audit_workbook.save(output) + output.seek(0) + after_names, after_extensions = archive_inventory(output) + finally: + audit_workbook.close() dropped = sorted(before_names - after_names) stripped_extensions = stripped_extension_records( before_extensions, after_extensions, before_names & after_names ) - return dropped, stripped_extensions + source.seek(0) + editable_workbook = openpyxl.load_workbook(source, **load_options) + return editable_workbook, dropped, stripped_extensions + +def load_with_round_trip_audit(path, **load_options): + with validated_xlsx_source(path) as source: + return load_with_round_trip_audit_from_source(source, **load_options) -dropped, stripped = round_trip_changes("extended.xlsx") + +audited_extended, dropped, stripped = load_with_round_trip_audit("extended.xlsx") +audited_extended.close() check("injected slicer-like part is detected as dropped", "xl/slicers/slicer1.xml" in dropped, dropped) check("each stripped worksheet extension record is reported with worksheet and URI", sum(item[0] == "xl/worksheets/sheet1.xml" and item[1] == EXT_URI for item in stripped) == 2, stripped) -check("clean workbook reports nothing", round_trip_changes("plain.xlsx") == ([], [])) +audited_plain, clean_dropped, clean_stripped = load_with_round_trip_audit("plain.xlsx") +check("clean workbook reports nothing", (clean_dropped, clean_stripped) == ([], [])) +check("round-trip audit returns the actual editable workbook identity", + audited_plain["Data"]["A2"].value == "EU") +audited_plain.close() + +from openpyxl.drawing.image import Image as AuditImage +Path("audit-image.png").write_bytes(base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9ZlRYAAAAASUVORK5CYII=" +)) +image_input_wb = openpyxl.Workbook() +image_input_wb.active.title = "Data" +image_input_wb.active["A1"] = "before" +image_input_wb.active.add_image(AuditImage("audit-image.png"), "C3") +image_input_wb.save("image-roundtrip.xlsx") +image_editable, image_dropped, image_stripped = load_with_round_trip_audit( + "image-roundtrip.xlsx" +) +image_editable["Data"]["A1"] = "after" +image_editable.save("image-roundtrip-edited.xlsx") +image_editable.close() +image_reopened = openpyxl.load_workbook("image-roundtrip-edited.xlsx") +check("round-trip audit returns a fresh workbook whose image streams can be saved again", + image_dropped == [] and image_stripped == [] + and image_reopened["Data"]["A1"].value == "after" + and len(image_reopened["Data"]._images) == 1, + (image_dropped, image_stripped, len(image_reopened["Data"]._images))) +image_reopened.close() + +real_archive_inventory = archive_inventory +preflight_inventory_calls = [] +def tracking_archive_inventory(source): + preflight_inventory_calls.append(source) + return real_archive_inventory(source) +archive_inventory = tracking_archive_inventory +try: + load_with_round_trip_audit("compressed-xlsx-bomb.xlsx") + audit_bomb_rejected = False +except ValueError as error: + audit_bomb_rejected = "suspicious compression ratio: xl/styles.xml" in str(error) +finally: + archive_inventory = real_archive_inventory +check("round-trip audit rejects a package bomb before raw inventory", + audit_bomb_rejected and preflight_inventory_calls == [], + (audit_bomb_rejected, preflight_inventory_calls)) inventory_names, inventory_extensions = archive_inventory("extended.xlsx") check("binary archive parts are named but never marker-scanned", "xl/media/large.bin" in inventory_names @@ -362,9 +708,11 @@ def sheet_ref(sheet): from openpyxl.drawing.image import Image from openpyxl.formula import Tokenizer from openpyxl.formatting.rule import CellIsRule, ColorScaleRule, DataBarRule, FormulaRule +from openpyxl.utils import get_column_letter from openpyxl.utils.cell import coordinate_to_tuple, range_boundaries from openpyxl.workbook.defined_name import DefinedName from openpyxl.worksheet.datavalidation import DataValidation +from openpyxl.worksheet.hyperlink import Hyperlink from openpyxl.worksheet.table import Table @@ -425,15 +773,14 @@ def drawing_anchor_rows(drawing): return tuple(rows) -def sparse_formula_cells(sheet): +def sparse_cells(sheet): if not hasattr(sheet, "_cells"): raise RuntimeError("structural edits require a normal writable Worksheet") - return ( - cell for cell in sorted( - sheet._cells.values(), key=lambda cell: (cell.row, cell.column) - ) - if cell.data_type == "f" - ) + return sorted(sheet._cells.values(), key=lambda cell: (cell.row, cell.column)) + + +def sparse_formula_cells(sheet): + return (cell for cell in sparse_cells(sheet) if cell.data_type == "f") def structural_references(workbook): @@ -445,6 +792,17 @@ def structural_references(workbook): for cell in sparse_formula_cells(sheet): refs.append(("cell formula", f"{owner}!{cell.coordinate}", formula_text(cell.value))) + for cell in sparse_cells(sheet): + hyperlink = cell.hyperlink + if hyperlink is None: + continue + location = getattr(hyperlink, "location", None) + target = getattr(hyperlink, "target", None) + refs.append(( + "cell hyperlink", + f"{owner}!{cell.coordinate}", + (getattr(hyperlink, "ref", None), location, target), + )) for table in sheet.tables.values(): refs.append(("table", owner + "!" + table.name, table.ref)) for merged_range in sheet.merged_cells.ranges: @@ -478,6 +836,11 @@ def structural_references(workbook): return refs +def non_cell_references(workbook): + return [reference for reference in structural_references(workbook) + if reference[0] != "cell formula"] + + def cell_formula_references(workbook): refs = [] for sheet in workbook.worksheets: @@ -496,7 +859,7 @@ def formula_may_intersect_rows(owner_sheet, formula, shifted_sheet, start_row): if not isinstance(formula, str) or not formula.startswith("="): return True tokens = Tokenizer(formula).items - unmodeled_reference_functions = {"indirect", "offset", "address"} + unmodeled_reference_functions = {"indirect", "offset", "address", "hyperlink"} if any( token.type == "FUNC" and token.subtype == "OPEN" and token.value.rstrip("(").rsplit(":", 1)[-1] @@ -587,6 +950,59 @@ class LegacyWorkbook: formula_references, ) +hyperlink_wb = openpyxl.Workbook() +hyperlink_ws = hyperlink_wb.active +hyperlink_ws.title = "Data" +hyperlink_ws["A10"] = "destination" +hyperlink_ws["B2"] = "location link" +hyperlink_ws["B2"].hyperlink = Hyperlink(ref="B2", location="'Data'!A10") +hyperlink_ws["B3"] = "hash target link" +hyperlink_ws["B3"].hyperlink = Hyperlink(ref="B3", target="#'Data'!A10") +hyperlink_ws["B4"] = "external link" +hyperlink_ws["B4"].hyperlink = "https://example.com/" +hyperlink_references = non_cell_references(hyperlink_wb) +check( + "structural inventory records every hyperlink anchor and destination form", + ("cell hyperlink", "Data!B2", ("B2", "'Data'!A10", None)) + in hyperlink_references + and ("cell hyperlink", "Data!B3", ("B3", None, "#'Data'!A10")) + in hyperlink_references + and ("cell hyperlink", "Data!B4", ("B4", None, "https://example.com/")) + in hyperlink_references, + hyperlink_references, +) +check("cell hyperlinks block structural edits before insert_rows runs", + bool(hyperlink_references) and hyperlink_ws["A10"].value == "destination") + +# Negative controls prove why both destination and anchor metadata need a rewrite plan. +destination_stale_wb = openpyxl.Workbook() +destination_stale_ws = destination_stale_wb.active +destination_stale_ws.title = "Data" +destination_stale_ws["A10"] = "destination" +destination_stale_ws["B2"] = "jump" +destination_stale_ws["B2"].hyperlink = Hyperlink(ref="B2", location="'Data'!A10") +destination_stale_ws.insert_rows(5) +check("insert_rows leaves an internal hyperlink destination stale (negative control)", + destination_stale_ws["A11"].value == "destination" + and destination_stale_ws["B2"].hyperlink.location == "'Data'!A10") + +anchor_stale_wb = openpyxl.Workbook() +anchor_stale_ws = anchor_stale_wb.active +anchor_stale_ws.title = "Data" +anchor_stale_ws["B8"] = "moving link" +anchor_stale_ws["B8"].hyperlink = Hyperlink(ref="B8", target="https://example.com/") +anchor_stale_ws.insert_rows(5) +check("insert_rows leaves an external hyperlink anchor ref stale (negative control)", + anchor_stale_ws["B9"].value == "moving link" + and anchor_stale_ws["B9"].hyperlink.ref == "B8") +anchor_stale_wb.save("stale-hyperlink-anchor.xlsx") +anchor_stale_reopened = openpyxl.load_workbook("stale-hyperlink-anchor.xlsx") +check("stale hyperlink ref detaches from moved text after save/reopen (negative control)", + anchor_stale_reopened["Data"]["B8"].hyperlink is not None + and anchor_stale_reopened["Data"]["B9"].value == "moving link" + and anchor_stale_reopened["Data"]["B9"].hyperlink is None) +anchor_stale_reopened.close() + sparse_scan_wb = openpyxl.Workbook() sparse_scan_ws = sparse_scan_wb.active sparse_scan_ws.title = "Sparse" @@ -622,6 +1038,10 @@ class LegacyWorkbook: formula_may_intersect_rows("Data", '=SUM(INDIRECT("A5:A6"))', "Data", 5)) check("OFFSET numeric row references require a manual structural rewrite plan", formula_may_intersect_rows("Data", "=SUM(OFFSET(A1,4,0,2,1))", "Data", 5)) +check("HYPERLINK string destinations require a manual structural rewrite plan", + formula_may_intersect_rows( + "Data", '=HYPERLINK("#\'Data\'!A10","jump")', "Data", 5 + )) check("implicit-intersection INDIRECT references require a manual rewrite plan", formula_may_intersect_rows("Data", '=@INDIRECT("A5:A6")', "Data", 5)) check("OFFSET used by the range operator requires a manual rewrite plan", @@ -743,6 +1163,129 @@ class LegacyWorkbook: CELL_TAG = f"{{{MAIN_NS}}}c" FORMULA_TAG = f"{{{MAIN_NS}}}f" VALUE_TAG = f"{{{MAIN_NS}}}v" +INLINE_STRING_TAG = f"{{{MAIN_NS}}}is" +MAX_EXPLICIT_CELLS = 1_000_000 +MAX_PROFILE_RECTANGLE_CELLS = 100_000 + + +def worksheet_part(archive, sheet_name): + workbook = ET.fromstring(archive.read("xl/workbook.xml")) + sheet = next( + item for item in workbook.iter(f"{{{MAIN_NS}}}sheet") + if item.attrib["name"] == sheet_name + ) + relationship_id = sheet.attrib[f"{{{DOC_REL_NS}}}id"] + relationships = ET.fromstring(archive.read("xl/_rels/workbook.xml.rels")) + target = next( + item.attrib["Target"] for item in relationships.iter(f"{{{PKG_REL_NS}}}Relationship") + if item.attrib["Id"] == relationship_id + ) + part = target.lstrip("/") if target.startswith("/") else posixpath.normpath( + posixpath.join("xl", target) + ) + require(part in archive.namelist(), f"worksheet part is missing: {part}") + return part + + +def worksheet_xml_profile(archive, part): + min_row = min_column = max_row = max_column = None + explicit_cell_count = 0 + formula_count = missing_formula_count = 0 + missing_formula_samples = [] + coordinate = cell_type = value_text = formula_display = None + has_formula = value_seen = inline_string_seen = False + with archive.open(part) as source: + for event, element in ET.iterparse(source, events=("start", "end")): + if event == "start" and element.tag == CELL_TAG: + coordinate = element.attrib.get("r") + require(coordinate is not None, f"cell without a coordinate in {part}") + cell_type = element.attrib.get("t") + has_formula = value_seen = inline_string_seen = False + value_text = formula_display = None + elif event == "end" and coordinate is not None: + if element.tag == FORMULA_TAG: + has_formula = True + if element.text is not None: + formula_display = "=" + element.text + else: + details = ", ".join( + f"{key}={value!r}" for key, value in sorted(element.attrib.items()) + ) + formula_display = f"" + elif element.tag == VALUE_TAG: + value_seen = True + value_text = element.text + elif element.tag == INLINE_STRING_TAG and cell_type == "inlineStr": + inline_string_seen = True + elif element.tag == CELL_TAG: + explicit_cell_count += 1 + require(explicit_cell_count <= MAX_EXPLICIT_CELLS, + f"too many explicit worksheet cells in {part}") + row_index, column_index = coordinate_to_tuple(coordinate) + require(1 <= row_index <= 1_048_576 and 1 <= column_index <= 16_384, + f"cell coordinate outside XLSX limits: {coordinate}") + scalar_value = value_seen and ( + value_text not in (None, "") or cell_type == "str" + ) + populated = has_formula or scalar_value or inline_string_seen + if populated: + min_row = row_index if min_row is None else min(min_row, row_index) + min_column = (column_index if min_column is None + else min(min_column, column_index)) + max_row = row_index if max_row is None else max(max_row, row_index) + max_column = (column_index if max_column is None + else max(max_column, column_index)) + valid_cache = value_seen and ( + value_text not in (None, "") or cell_type == "str" + ) + if has_formula: + formula_count += 1 + if not valid_cache: + missing_formula_count += 1 + if len(missing_formula_samples) < 10: + missing_formula_samples.append((coordinate, formula_display)) + coordinate = None + element.clear() + elif event == "end": + element.clear() + if max_row is None: + bounds, extent, first_populated_row = None, "A1:A1", None + else: + bounds = (min_row, min_column, max_row, max_column) + extent = (f"{get_column_letter(min_column)}{min_row}:" + f"{get_column_letter(max_column)}{max_row}") + first_populated_row = min_row + return { + "part": part, + "bounds": bounds, + "extent": extent, + "first_populated_row": first_populated_row, + "formula_count": formula_count, + "missing_formula_count": missing_formula_count, + "missing_formula_samples": missing_formula_samples, + "explicit_cell_count": explicit_cell_count, + } + + +def bounded_sample_rows(worksheet, profile, *, max_cells=MAX_PROFILE_RECTANGLE_CELLS): + if profile["bounds"] is None: + return iter(()) + min_row, min_column, max_row, max_column = profile["bounds"] + sample_max_row = min(max_row, min_row + 5) + sample_cells = (sample_max_row - min_row + 1) * (max_column - min_column + 1) + require(sample_cells <= max_cells, + f"sample rectangle is too large: {worksheet.title} ({sample_cells} cells)") + return worksheet.iter_rows( + min_row=min_row, min_col=min_column, + max_row=sample_max_row, max_col=max_column, + values_only=True, + ) + + +def worksheet_declared_dimension(worksheet): + if worksheet.max_row is None or worksheet.max_column is None: + return None + return worksheet.calculate_dimension() wb_h = openpyxl.Workbook() wb_h.active.title = "First" @@ -774,43 +1317,45 @@ class LegacyWorkbook: with zipfile.ZipFile("multi.xlsx", "w", zipfile.ZIP_DEFLATED) as archive: for name, data in multi_members.items(): archive.writestr(name, data) -formula_wb = openpyxl.load_workbook("multi.xlsx", read_only=True, data_only=False) -value_wb = openpyxl.load_workbook("multi.xlsx", read_only=True, data_only=True) -profiled = list(value_wb.sheetnames) -uncached = {(sn, fc.coordinate, formula_text(fc.value)) - for sn in profiled - for frow, vrow in zip(formula_wb[sn].iter_rows(), value_wb[sn].iter_rows()) - for fc, vc in zip(frow, vrow) - if fc.data_type == "f" and vc.value is None} +with validated_xlsx_source("multi.xlsx") as multi_source: + formula_wb = openpyxl.load_workbook(multi_source, read_only=True, data_only=False) + multi_source.seek(0) + value_wb = openpyxl.load_workbook(multi_source, read_only=True, data_only=True) + try: + multi_source.seek(0) + with zipfile.ZipFile(multi_source) as archive: + profiled = list(value_wb.sheetnames) + multi_profiles = { + sheet_name: worksheet_xml_profile( + archive, worksheet_part(archive, sheet_name) + ) + for sheet_name in profiled + } + uncached = { + (sheet_name, coordinate, formula) + for sheet_name, profile in multi_profiles.items() + for coordinate, formula in profile["missing_formula_samples"] + } + paired_same_source = ( + next(formula_wb["First"].iter_rows(min_row=1, max_row=1))[0].value == "=1+1" + and next(value_wb["First"].iter_rows(min_row=1, max_row=1))[0].value is None + ) + finally: + formula_wb.close() + value_wb.close() check("multi-sheet profile iterates every sheet", profiled == ["First", "Second"], profiled) +check("formula/value read-only streams share one validated source identity", + paired_same_source) check("uncached formulas found on both sheets", {item[0] for item in uncached} == {"First", "Second"}, uncached) check("array-formula objects are detected by data_type", ("First", "A2", "=ROW(A2:A3)") in uncached, uncached) data_table_entry = next(item for item in uncached if item[:2] == ("Second", "B1")) check("data-table formulas have stable diagnostic text", - data_table_entry[2].startswith("DataTableFormula(ref='B1:B2', r1='C1'") - and "0x" not in data_table_entry[2], data_table_entry) -formula_wb.close() -value_wb.close() + data_table_entry[2].startswith("= {"A1", "A2"}, missing_caches) -check("XLSX-to-CSV does not leak formula strings into value output", - not any(value.startswith("=") for value in exported_values), exported_values) -check("XLSX-to-CSV safe mode neutralizes cached literal text for spreadsheet consumers", - {"'=literal", "'+literal", "'-literal", "'@literal"} <= set(exported_values), - exported_values) -formula_wb.close() -value_wb.close() +with validated_xlsx_source("multi.xlsx") as multi_csv_source: + multi_csv_value_wb = openpyxl.load_workbook( + multi_csv_source, read_only=True, data_only=True + ) + try: + multi_csv_source.seek(0) + with zipfile.ZipFile(multi_csv_source) as archive: + multi_csv_profile = worksheet_xml_profile( + archive, worksheet_part(archive, "First") + ) + missing_caches = [ + coordinate + for coordinate, _ in multi_csv_profile["missing_formula_samples"] + ] + finally: + multi_csv_value_wb.close() +check("XLSX-to-CSV reports formulas with no cached value before creating output", + set(missing_caches) >= {"A1", "A2"}, missing_caches) def write_formula_cache_fixture(path, cache_kind): @@ -914,32 +1449,63 @@ def write_formula_cache_fixture(path, cache_kind): archive.writestr(name, data) -def export_formula_values(source_path, output_path): +MAX_CSV_EXPORT_CELLS = 5_000_000 + + +def export_formula_values(source_path, output_path, *, max_cells=MAX_CSV_EXPORT_CELLS): sheet_name = "Data" - with zipfile.ZipFile(source_path) as archive: - part = worksheet_part(archive, sheet_name) - cached_cells = cached_formula_coordinates(archive, part) - formula_book = openpyxl.load_workbook(source_path, read_only=True, data_only=False) - value_book = openpyxl.load_workbook(source_path, read_only=True, data_only=True) - formula_sheet, value_sheet = formula_book[sheet_name], value_book[sheet_name] - formula_sheet.reset_dimensions() - value_sheet.reset_dimensions() - missing = [] destination = Path(output_path) - temporary = destination.with_suffix(destination.suffix + ".tmp") - with temporary.open("w", newline="", encoding="utf-8") as output: - writer = csv.writer(output) - for formula_row, value_row in zip(formula_sheet.iter_rows(), value_sheet.iter_rows()): - for formula_cell, value_cell in zip(formula_row, value_row): - if (formula_cell.data_type == "f" and value_cell.value is None - and formula_cell.coordinate not in cached_cells): - missing.append(formula_cell.coordinate) - writer.writerow([spreadsheet_csv_field(cell.value) for cell in value_row]) - formula_book.close() - value_book.close() - if missing: - temporary.unlink(missing_ok=True) - raise RuntimeError(f"formula cells have no cached value: {missing}") + temporary = None + try: + descriptor, temporary_name = mkstemp( + dir=destination.parent, prefix=f".{destination.name}.", suffix=".tmp" + ) + temporary = Path(temporary_name) + os.close(descriptor) + with validated_xlsx_source(source_path) as package_source: + value_book = openpyxl.load_workbook( + package_source, read_only=True, data_only=True + ) + try: + package_source.seek(0) + with zipfile.ZipFile(package_source) as archive: + profile = worksheet_xml_profile( + archive, worksheet_part(archive, sheet_name) + ) + if profile["missing_formula_count"]: + raise RuntimeError( + "formula cells have no cached value: " + f"{profile['missing_formula_samples']}" + ) + value_sheet = value_book[sheet_name] + value_sheet.reset_dimensions() + if profile["bounds"] is None: + rows = iter(()) + else: + min_row, min_column, max_row, max_column = profile["bounds"] + export_cells = ( + (max_row - min_row + 1) * (max_column - min_column + 1) + ) + require(export_cells <= max_cells, + f"CSV export rectangle is too large: {export_cells} cells") + rows = value_sheet.iter_rows( + min_row=min_row, min_col=min_column, + max_row=max_row, max_col=max_column, + values_only=True, + ) + with temporary.open("w", newline="", encoding="utf-8") as output: + writer = csv.writer(output) + for value_row in rows: + writer.writerow([ + spreadsheet_csv_field(value) for value in value_row + ]) + finally: + value_book.close() + except Exception: + if temporary is not None: + temporary.unlink(missing_ok=True) + raise + require(temporary is not None, "CSV temporary output was not created") temporary.replace(destination) @@ -964,23 +1530,12 @@ def export_formula_values(source_path, output_path): def profile_missing_formula_caches(source_path): - with zipfile.ZipFile(source_path) as archive: - cached_cells = cached_formula_coordinates(archive, worksheet_part(archive, "Data")) - formula_book = openpyxl.load_workbook(source_path, read_only=True, data_only=False) - value_book = openpyxl.load_workbook(source_path, read_only=True, data_only=True) - formula_sheet, value_sheet = formula_book["Data"], value_book["Data"] - formula_sheet.reset_dimensions() - value_sheet.reset_dimensions() - missing = [ - formula_cell.coordinate - for formula_row, value_row in zip(formula_sheet.iter_rows(), value_sheet.iter_rows()) - for formula_cell, value_cell in zip(formula_row, value_row) - if (formula_cell.data_type == "f" and value_cell.value is None - and formula_cell.coordinate not in cached_cells) - ] - formula_book.close() - value_book.close() - return missing + with validated_xlsx_source(source_path) as package_source: + with zipfile.ZipFile(package_source) as archive: + profile = worksheet_xml_profile( + archive, worksheet_part(archive, "Data") + ) + return [coordinate for coordinate, _ in profile["missing_formula_samples"]] check("workbook profiling accepts nonempty and typed blank formula caches", @@ -1027,30 +1582,70 @@ def write_region_key_fixture(path, cache_kind): archive.writestr(name, data) -def aggregation_regions(source_path): - formula_book = openpyxl.load_workbook(source_path, data_only=False) - source_sheet = formula_book["Data"] - with zipfile.ZipFile(source_path) as archive: - cached_cells = cached_formula_coordinates(archive, worksheet_part(archive, "Data")) - value_book = openpyxl.load_workbook(source_path, read_only=True, data_only=True) - value_sheet = value_book["Data"] - value_sheet.reset_dimensions() - regions = [] - missing = [] - for source_row, value_row in zip( - source_sheet.iter_rows(min_row=2, min_col=1, max_col=1), - value_sheet.iter_rows(min_row=2, min_col=1, max_col=1), - ): - source_cell, value_cell = source_row[0], value_row[0] - region = value_cell.value if source_cell.data_type == "f" else source_cell.value - if (source_cell.data_type == "f" and region is None - and source_cell.coordinate not in cached_cells): - missing.append(source_cell.coordinate) - continue - if region is not None and region != "": - regions.append(region) - value_book.close() - formula_book.close() +def aggregation_regions(source_path, *, max_row_span=100_000, + approved_feature_loss=False): + with validated_xlsx_source(source_path) as package_source: + formula_book, dropped_parts, stripped_extensions = ( + load_with_round_trip_audit_from_source(package_source, data_only=False) + ) + if (dropped_parts or stripped_extensions) and not approved_feature_loss: + formula_book.close() + raise RuntimeError( + f"openpyxl would drop parts={dropped_parts!r}, " + f"extensions={stripped_extensions!r}" + ) + value_book = None + try: + source_sheet = formula_book["Data"] + source_cells = sorted( + (cell for cell in source_sheet._cells.values() + if cell.row >= 2 and cell.column == 1 and cell.value is not None), + key=lambda cell: cell.row, + ) + if source_cells: + min_source_row, max_source_row = source_cells[0].row, source_cells[-1].row + row_span = max_source_row - min_source_row + 1 + require(row_span <= max_row_span, + f"aggregation row span is too large: {row_span}") + wanted_formula_coordinates = { + cell.coordinate for cell in source_cells if cell.data_type == "f" + } + package_source.seek(0) + with zipfile.ZipFile(package_source) as archive: + cached_cells = cached_formula_coordinates( + archive, worksheet_part(archive, "Data"), wanted_formula_coordinates + ) + package_source.seek(0) + value_book = openpyxl.load_workbook( + package_source, read_only=True, data_only=True + ) + value_sheet = value_book["Data"] + value_sheet.reset_dimensions() + regions = [] + missing = [] + if source_cells: + source_by_row = {cell.row: cell for cell in source_cells} + value_rows = value_sheet.iter_rows( + min_row=min_source_row, max_row=max_source_row, + min_col=1, max_col=1, + ) + for row_index, value_row in enumerate(value_rows, start=min_source_row): + source_cell = source_by_row.get(row_index) + if source_cell is None: + continue + value_cell = value_row[0] + region = (value_cell.value if source_cell.data_type == "f" + else source_cell.value) + if (source_cell.data_type == "f" and region is None + and source_cell.coordinate not in cached_cells): + missing.append(source_cell.coordinate) + continue + if region is not None and region != "": + regions.append(region) + finally: + if value_book is not None: + value_book.close() + formula_book.close() if missing: raise RuntimeError(f"aggregation keys have no cached value: {missing}") return regions @@ -1075,6 +1670,20 @@ def aggregation_regions(source_path): check("aggregation fails closed when a formula key has no cached value", missing_region_cache_rejected) +feature_loss_payload = { + name: data for name, data in payload.items() if name != "xl/media/large.bin" +} +with zipfile.ZipFile("aggregation-feature-loss.xlsx", "w", zipfile.ZIP_STORED) as archive: + for name, data in feature_loss_payload.items(): + archive.writestr(name, data) +try: + aggregation_regions("aggregation-feature-loss.xlsx") + aggregation_feature_loss_rejected = False +except RuntimeError as error: + aggregation_feature_loss_rejected = "openpyxl would drop parts=" in str(error) +check("aggregation feature-loss audit gates the exact workbook later edited", + aggregation_feature_loss_rejected) + export_formula_values("cached-value.xlsx", "cached-value.csv") with open("cached-value.csv", newline="", encoding="utf-8") as exported: cached_value_rows = list(csv.reader(exported)) @@ -1086,6 +1695,7 @@ def aggregation_regions(source_path): check("cached empty-string formula result exports as a displayed blank", cached_empty_rows == [[""]], cached_empty_rows) Path("missing-cache.csv").write_text("sentinel\n", encoding="utf-8") +Path("missing-cache.csv.tmp").write_text("unrelated temporary\n", encoding="utf-8") try: export_formula_values("missing-cache.xlsx", "missing-cache.csv") missing_cache_rejected = False @@ -1094,8 +1704,11 @@ def aggregation_regions(source_path): check("formula with no XML cache element is rejected", missing_cache_rejected) check("failed cache audit preserves the prior destination and removes its temporary file", Path("missing-cache.csv").read_text(encoding="utf-8") == "sentinel\n" - and not Path("missing-cache.csv.tmp").exists()) + and Path("missing-cache.csv.tmp").read_text(encoding="utf-8") + == "unrelated temporary\n" + and list(Path(".").glob(".missing-cache.csv.*.tmp")) == []) Path("bare-empty-cache.csv").write_text("sentinel\n", encoding="utf-8") +Path("bare-empty-cache.csv.tmp").write_text("unrelated temporary\n", encoding="utf-8") try: export_formula_values("bare-empty-cache.xlsx", "bare-empty-cache.csv") bare_empty_rejected = False @@ -1105,7 +1718,79 @@ def aggregation_regions(source_path): bare_empty_rejected) check("bare-empty rejection preserves destination and removes temporary output", Path("bare-empty-cache.csv").read_text(encoding="utf-8") == "sentinel\n" - and not Path("bare-empty-cache.csv.tmp").exists()) + and Path("bare-empty-cache.csv.tmp").read_text(encoding="utf-8") + == "unrelated temporary\n" + and list(Path(".").glob(".bare-empty-cache.csv.*.tmp")) == []) + +csv_sparse_wb = openpyxl.Workbook() +csv_sparse_ws = csv_sparse_wb.active +csv_sparse_ws.title = "Data" +csv_sparse_ws["A1"], csv_sparse_ws["A2"] = "Region", "EU" +csv_sparse_ws["XFD1048576"].number_format = "0.00" +csv_sparse_wb.save("csv-style-extreme.xlsx") +export_formula_values("csv-style-extreme.xlsx", "csv-style-extreme.csv") +with open("csv-style-extreme.csv", newline="", encoding="utf-8") as exported: + csv_sparse_rows = list(csv.reader(exported)) +check("CSV export ignores a style-only extreme cell without rectangular expansion", + csv_sparse_rows == [["Region"], ["EU"]], csv_sparse_rows) + +csv_far_wb = openpyxl.Workbook() +csv_far_ws = csv_far_wb.active +csv_far_ws.title = "Data" +csv_far_ws["A1"], csv_far_ws["XFD1048576"] = "near", "far" +csv_far_wb.save("csv-far-values.xlsx") +Path("csv-far-values.csv").write_text("sentinel\n", encoding="utf-8") +Path("csv-far-values.csv.tmp").write_text("unrelated temporary\n", encoding="utf-8") +try: + export_formula_values("csv-far-values.xlsx", "csv-far-values.csv", max_cells=1_000) + csv_rectangle_rejected = False +except ValueError as error: + csv_rectangle_rejected = "CSV export rectangle is too large" in str(error) +check("CSV export rejects a far-apart logical rectangle before iteration", + csv_rectangle_rejected + and Path("csv-far-values.csv").read_text(encoding="utf-8") == "sentinel\n" + and Path("csv-far-values.csv.tmp").read_text(encoding="utf-8") + == "unrelated temporary\n" + and list(Path(".").glob(".csv-far-values.csv.*.tmp")) == []) + +aggregation_sparse_wb = openpyxl.Workbook() +aggregation_sparse_ws = aggregation_sparse_wb.active +aggregation_sparse_ws.title = "Data" +aggregation_sparse_ws["A1"], aggregation_sparse_ws["A2"] = "Region", "EU" +aggregation_sparse_ws["XFD1048576"].number_format = "0.00" +aggregation_sparse_wb.save("aggregation-style-extreme.xlsx") +check("aggregation ignores an extreme style-only cell without iterating a million rows", + aggregation_regions("aggregation-style-extreme.xlsx") == ["EU"]) + +aggregation_far_wb = openpyxl.Workbook() +aggregation_far_ws = aggregation_far_wb.active +aggregation_far_ws.title = "Data" +aggregation_far_ws["A1"], aggregation_far_ws["A2"] = "Region", "near" +aggregation_far_ws["A1048576"] = "far" +aggregation_far_wb.save("aggregation-far-values.xlsx") +try: + aggregation_regions("aggregation-far-values.xlsx", max_row_span=1_000) + aggregation_span_rejected = False +except ValueError as error: + aggregation_span_rejected = "aggregation row span is too large" in str(error) +check("aggregation rejects far-apart keys before iter_rows", aggregation_span_rejected) + +real_worksheet_xml_profile = worksheet_xml_profile +bomb_profile_calls = [] +def tracking_worksheet_xml_profile(*args, **kwargs): + bomb_profile_calls.append(args) + return real_worksheet_xml_profile(*args, **kwargs) +worksheet_xml_profile = tracking_worksheet_xml_profile +try: + export_formula_values("compressed-xlsx-bomb.xlsx", "bomb-export.csv") + csv_bomb_rejected = False +except ValueError as error: + csv_bomb_rejected = "suspicious compression ratio: xl/styles.xml" in str(error) +finally: + worksheet_xml_profile = real_worksheet_xml_profile +check("CSV route rejects a package bomb before raw worksheet inventory", + csv_bomb_rejected and bomb_profile_calls == [], + (csv_bomb_rejected, bomb_profile_calls)) # SKILL.md contract: fullCalcOnLoad makes viewers recalculate even in manual calc mode. calc_wb = openpyxl.Workbook() @@ -1284,55 +1969,100 @@ def add_demo_formatting(sheet): for name, data in members.items(): archive.writestr(name, data) -from openpyxl.utils import get_column_letter - +with validated_xlsx_source("dimension.xlsx") as dimension_source: + dim_value = openpyxl.load_workbook( + dimension_source, read_only=True, data_only=True + ) + try: + dim_ws_ro = dim_value.active + check("plausible but truncated dimension limits streaming (negative control)", + dim_ws_ro.calculate_dimension() == "A1:B2" and dim_ws_ro.max_row == 2, + (dim_ws_ro.calculate_dimension(), dim_ws_ro.max_row)) + original_dimension_iter_rows = dim_ws_ro.iter_rows + dim_ws_ro.iter_rows = lambda *args, **kwargs: (_ for _ in ()).throw( + RuntimeError("XML discovery must not expand a worksheet rectangle") + ) + dimension_source.seek(0) + with zipfile.ZipFile(dimension_source) as archive: + dimension_profile = worksheet_xml_profile( + archive, worksheet_part(archive, dim_ws_ro.title) + ) + dim_ws_ro.iter_rows = original_dimension_iter_rows + dim_ws_ro.reset_dimensions() + dimension_rows = list(bounded_sample_rows(dim_ws_ro, dimension_profile)) + finally: + dim_value.close() +check("sparse XML profile ignores truncated dimension and retains uncached formulas", + dimension_profile["extent"] == "A1:C4" + and dimension_profile["first_populated_row"] == 1 + and dimension_profile["formula_count"] == 1 + and dimension_profile["missing_formula_samples"][0][0] == "C4", + dimension_profile) +check("bounded sampling uses the XML extent after resetting producer metadata", + len(dimension_rows) == 4 + and dimension_rows[2][:2] == (3, 4), + dimension_rows) + +missing_dimension_wb = openpyxl.Workbook() +missing_dimension_ws = missing_dimension_wb.active +missing_dimension_ws.append(["Region", "Units"]) +missing_dimension_ws.append(["EU", 120]) +missing_dimension_ws.append(["US", 80]) +missing_dimension_wb.save("missing-dimension.xlsx") +with zipfile.ZipFile("missing-dimension.xlsx") as archive: + missing_dimension_members = { + name: archive.read(name) for name in archive.namelist() + } +missing_dimension_xml = missing_dimension_members["xl/worksheets/sheet1.xml"] +missing_dimension_tag = b'' +check("missing-dimension fixture starts with the expected producer metadata", + missing_dimension_tag in missing_dimension_xml) +missing_dimension_members["xl/worksheets/sheet1.xml"] = ( + missing_dimension_xml.replace(missing_dimension_tag, b"") +) +with zipfile.ZipFile("missing-dimension.xlsx", "w") as archive: + for name, data in missing_dimension_members.items(): + archive.writestr(name, data) -def discover_dimension(worksheet): - worksheet.reset_dimensions() - min_row = min_column = max_row = max_column = None - for row in worksheet.iter_rows(): - for cell in row: - if getattr(cell, "value", None) is None and getattr(cell, "data_type", None) != "f": - continue - row_index = getattr(cell, "row", None) - column_index = getattr(cell, "column", None) - if row_index is None or column_index is None: - continue - min_row = row_index if min_row is None else min(min_row, row_index) - min_column = column_index if min_column is None else min(min_column, column_index) - max_row = row_index if max_row is None else max(max_row, row_index) - max_column = column_index if max_column is None else max(max_column, column_index) - if max_row is None: - return "A1:A1", None - extent = (f"{get_column_letter(min_column)}{min_row}:" - f"{get_column_letter(max_column)}{max_row}") - return extent, min_row - - -dim_value = openpyxl.load_workbook("dimension.xlsx", read_only=True, data_only=True) -dim_formula = openpyxl.load_workbook("dimension.xlsx", read_only=True, data_only=False) -dim_ws_ro = dim_value.active -check("plausible but truncated dimension limits streaming (negative control)", - dim_ws_ro.calculate_dimension() == "A1:B2" and dim_ws_ro.max_row == 2, - (dim_ws_ro.calculate_dimension(), dim_ws_ro.max_row)) -streamed_before_reset = [row for row in dim_ws_ro.iter_rows(values_only=True)] -discovered_value_dimension, discovered_value_first_row = discover_dimension(dim_ws_ro) -discovered_formula_dimension, discovered_formula_first_row = discover_dimension(dim_formula.active) -streamed_after_reset = [row for row in dim_ws_ro.iter_rows(values_only=True)] -dim_value.close() -dim_formula.close() -check("formula-preserving dimension scan retains uncached formulas in the logical range", - discovered_formula_dimension == "A1:C4" - and discovered_formula_first_row == 1 - and discovered_value_dimension == "A1:B3" - and discovered_value_first_row == 1, - ((discovered_value_dimension, discovered_value_first_row), - (discovered_formula_dimension, discovered_formula_first_row))) -check("reset_dimensions restores the real extent", - len(streamed_before_reset) == 2 - and len(streamed_after_reset) == 4 - and streamed_after_reset[2][:2] == (3, 4), - (len(streamed_before_reset), streamed_after_reset[-1])) +with validated_xlsx_source("missing-dimension.xlsx") as missing_dimension_source: + missing_dimension_value_wb = openpyxl.load_workbook( + missing_dimension_source, read_only=True, data_only=True + ) + try: + missing_dimension_value_ws = missing_dimension_value_wb.active + forced_dimension_scans = [] + original_forced_dimension_scan = missing_dimension_value_ws._calculate_dimension + + def reject_forced_dimension_scan(): + forced_dimension_scans.append(True) + raise RuntimeError("read route must not force a worksheet dimension scan") + + missing_dimension_value_ws._calculate_dimension = reject_forced_dimension_scan + missing_dimension_source.seek(0) + with zipfile.ZipFile(missing_dimension_source) as archive: + missing_dimension_profile = worksheet_xml_profile( + archive, worksheet_part(archive, missing_dimension_value_ws.title) + ) + missing_dimension_declared = worksheet_declared_dimension( + missing_dimension_value_ws + ) + missing_dimension_value_ws.reset_dimensions() + missing_dimension_rows = list(bounded_sample_rows( + missing_dimension_value_ws, missing_dimension_profile + )) + missing_dimension_value_ws._calculate_dimension = original_forced_dimension_scan + finally: + missing_dimension_value_wb.close() +check("missing dimension remains unsized before raw XML routing", + missing_dimension_declared is None + and missing_dimension_profile["extent"] == "A1:B3", + (missing_dimension_declared, missing_dimension_profile)) +check("missing dimension route samples from XML bounds without a forced scan", + forced_dimension_scans == [] + and missing_dimension_rows == [ + ("Region", "Units"), ("EU", 120), ("US", 80) + ], + (forced_dimension_scans, missing_dimension_rows)) csv_formula_wb = openpyxl.load_workbook("dimension.xlsx", read_only=True, data_only=False) csv_value_wb = openpyxl.load_workbook("dimension.xlsx", read_only=True, data_only=True) @@ -1359,36 +2089,75 @@ def discover_dimension(worksheet): offset_ws["A8"], offset_ws["B8"] = "EU", 120 offset_ws["A2"].number_format = "0.00" # styled but empty: not part of the data range offset_ws["C8"] = "=SUM(B8)" # uncached formula: remains part of the range +offset_ws["XFD1048576"].number_format = "0.00" # extreme style-only physical cell offset_wb.save("leading-blank-rows.xlsx") -offset_formula_wb = openpyxl.load_workbook( - "leading-blank-rows.xlsx", read_only=True, data_only=False, -) -offset_value_wb = openpyxl.load_workbook( - "leading-blank-rows.xlsx", read_only=True, data_only=True, -) -offset_formula_extent, offset_formula_first = discover_dimension(offset_formula_wb.active) -offset_value_wb.active.reset_dimensions() -offset_rows = offset_value_wb.active.iter_rows( - min_row=offset_formula_first, values_only=True, -) -offset_header = next(offset_rows, None) -offset_sample = next(offset_rows, None) +with validated_xlsx_source("leading-blank-rows.xlsx") as offset_source: + offset_value_wb = openpyxl.load_workbook( + offset_source, read_only=True, data_only=True, + ) + try: + offset_value_ws = offset_value_wb.active + original_offset_iter_rows = offset_value_ws.iter_rows + offset_value_ws.iter_rows = lambda *args, **kwargs: (_ for _ in ()).throw( + RuntimeError("XML profile must not call iter_rows") + ) + offset_source.seek(0) + with zipfile.ZipFile(offset_source) as archive: + offset_profile = worksheet_xml_profile( + archive, worksheet_part(archive, offset_value_ws.title) + ) + offset_value_ws.iter_rows = original_offset_iter_rows + offset_value_ws.reset_dimensions() + offset_rows = bounded_sample_rows(offset_value_ws, offset_profile) + offset_header = next(offset_rows, None) + offset_sample = next(offset_rows, None) + finally: + offset_value_wb.close() check("styled empty cells do not move the formula-preserving logical range", - (offset_formula_extent, offset_formula_first) == ("A7:C8", 7), - (offset_formula_extent, offset_formula_first)) + (offset_profile["extent"], offset_profile["first_populated_row"]) + == ("A7:C8", 7), offset_profile) check("header sampling skips six leading blank rows", offset_header[:2] == ("Region", "Units") and offset_sample[:2] == ("EU", 120), (offset_header, offset_sample)) -offset_formula_wb.close() -offset_value_wb.close() + +far_wb = openpyxl.Workbook() +far_ws = far_wb.active +far_ws["A1"] = "near" +far_ws["XFD1048576"] = "far" +far_wb.save("far-logical-cells.xlsx") +with validated_xlsx_source("far-logical-cells.xlsx") as far_source: + far_value_wb = openpyxl.load_workbook(far_source, read_only=True, data_only=True) + try: + far_source.seek(0) + with zipfile.ZipFile(far_source) as archive: + far_profile = worksheet_xml_profile( + archive, worksheet_part(archive, far_value_wb.active.title) + ) + far_iter_calls = [] + far_value_wb.active.iter_rows = ( + lambda *args, **kwargs: far_iter_calls.append((args, kwargs)) or iter(()) + ) + try: + bounded_sample_rows(far_value_wb.active, far_profile, max_cells=1_000) + far_budget_rejected = False + except ValueError as error: + far_budget_rejected = "sample rectangle is too large" in str(error) + finally: + far_value_wb.close() +check("real far-apart values hit the rectangle budget before iter_rows", + far_budget_rejected and far_iter_calls == [], (far_profile, far_iter_calls)) empty_dimension_wb = openpyxl.Workbook() empty_dimension_wb.save("empty-dimension.xlsx") -empty_dimension_ro = openpyxl.load_workbook("empty-dimension.xlsx", read_only=True) +with validated_xlsx_source("empty-dimension.xlsx") as empty_source: + with zipfile.ZipFile(empty_source) as archive: + empty_profile = worksheet_xml_profile( + archive, worksheet_part(archive, "Sheet") + ) check("dimension scan handles an actually empty worksheet", - discover_dimension(empty_dimension_ro.active) == ("A1:A1", None)) -empty_dimension_ro.close() + empty_profile["extent"] == "A1:A1" + and empty_profile["first_populated_row"] is None) print("\n" + ("ALL XLSX FIXTURES PASSED" if not failures else f"{len(failures)} FAILURES: {failures}")) From ece7467e31fa26b9ee1f8ccc6ba36495c2cd157e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=98=E6=89=AC?= Date: Tue, 18 Aug 2026 13:29:25 +0800 Subject: [PATCH 34/34] docs(document-skills): verify rendered package content --- .../skills/docx/references/read.md | 134 ++++++++++++++-- .../skills/docx/references/review.md | 44 +++++- .../skills/pdf/references/inspect.md | 51 +++++- .../document-skills/skills/xlsx/SKILL.md | 15 ++ .../skills/xlsx/references/edit.md | 3 + .../document-skills/tests/docx_fixture.py | 147 +++++++++++++++++- .../document-skills/tests/pdf_fixture.py | 117 +++++++++++++- .../document-skills/tests/xlsx_fixture.py | 63 ++++++++ 8 files changed, 552 insertions(+), 22 deletions(-) diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/read.md b/plugins/Hylouis233/document-skills/skills/docx/references/read.md index bc76771..06611a2 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/read.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/read.md @@ -14,11 +14,122 @@ tables). Prefer this when the goal is content, not coordinates. ## Structured access (python-docx) ```python +from contextlib import contextmanager +import zipfile +from pathlib import Path +from tempfile import TemporaryFile + from docx import Document from docx.oxml.ns import qn from docx.table import Table from docx.text.paragraph import Paragraph from docx.text.run import Run +from lxml import etree + +MAX_ARCHIVE_BYTES = 200 * 1024 * 1024 +MAX_MEMBERS = 10_000 +MAX_XML_PART = 20 * 1024 * 1024 +MAX_ENTRY = 100 * 1024 * 1024 +MAX_TOTAL_UNCOMPRESSED = 500 * 1024 * 1024 +MAX_COMPRESSION_RATIO = 200 +CONTENT_TYPES_NAMESPACE = "http://schemas.openxmlformats.org/package/2006/content-types" + +def require(condition, message): + if not condition: + raise ValueError(message) + +SAFE_XML_PARSER = etree.XMLParser( + load_dtd=False, resolve_entities=False, no_network=True, + huge_tree=False, recover=False, +) + +def xml_content_type(value): + media_type = (value or "").split(";", 1)[0].strip().casefold() + return media_type in {"application/xml", "text/xml"} or media_type.endswith("+xml") + +def declared_xml_parts(archive, infos): + """Classify XML by OPC declarations, not filename spelling alone.""" + by_name = {info.filename: info for info in infos} + content_types_info = by_name.get("[Content_Types].xml") + require(content_types_info is not None, "missing [Content_Types].xml") + require(content_types_info.file_size <= MAX_XML_PART, + "oversized XML part: [Content_Types].xml") + with archive.open(content_types_info) as stream: + content_types_blob = stream.read(MAX_XML_PART + 1) + require(len(content_types_blob) <= MAX_XML_PART, + "oversized XML part: [Content_Types].xml") + root = etree.fromstring(content_types_blob, parser=SAFE_XML_PARSER) + require(root.tag == f"{{{CONTENT_TYPES_NAMESPACE}}}Types", + "invalid [Content_Types].xml root") + defaults = {} + overrides = {} + for child in root: + if child.tag == f"{{{CONTENT_TYPES_NAMESPACE}}}Default": + extension = (child.get("Extension") or "").casefold() + require(extension and extension not in defaults, + "invalid duplicate content-type default") + defaults[extension] = child.get("ContentType") or "" + elif child.tag == f"{{{CONTENT_TYPES_NAMESPACE}}}Override": + part_name = child.get("PartName") or "" + require(part_name.startswith("/") and part_name[1:] not in overrides, + "invalid duplicate content-type override") + overrides[part_name[1:]] = child.get("ContentType") or "" + xml_names = {"[Content_Types].xml"} + for info in infos: + suffix = info.filename.rsplit(".", 1)[1].casefold() if "." in info.filename else "" + content_type = overrides.get(info.filename, defaults.get(suffix, "")) + if (info.filename.casefold().endswith((".xml", ".rels")) + or xml_content_type(content_type)): + xml_names.add(info.filename) + return xml_names + +def validate_docx_archive(archive): + infos = archive.infolist() + require(len(infos) <= MAX_MEMBERS, "archive member count above limit") + names = {info.filename for info in infos} + require(len(names) == len(infos), "duplicate archive member names are unsafe") + require("[Content_Types].xml" in names and "word/document.xml" in names, + "missing required OPC members") + require(sum(info.file_size for info in infos) <= MAX_TOTAL_UNCOMPRESSED, + "declared total uncompressed size above limit") + xml_names = declared_xml_parts(archive, infos) + actual_total = 0 + for info in infos: + require(info.file_size <= MAX_ENTRY, f"oversized part: {info.filename}") + require(info.file_size / max(info.compress_size, 1) <= MAX_COMPRESSION_RATIO, + f"suspicious compression ratio: {info.filename}") + is_xml = info.filename in xml_names + if is_xml: + require(info.file_size <= MAX_XML_PART, f"oversized XML part: {info.filename}") + chunks = [] + actual_size = 0 + with archive.open(info) as stream: + while chunk := stream.read(64 * 1024): + actual_size += len(chunk) + actual_total += len(chunk) + require(actual_size <= MAX_ENTRY, f"part exceeded read limit: {info.filename}") + require(actual_total <= MAX_TOTAL_UNCOMPRESSED, + "archive exceeded total read limit") + if is_xml: + chunks.append(chunk) + require(actual_size == info.file_size, f"size mismatch: {info.filename}") + if is_xml: + etree.fromstring(b"".join(chunks), parser=SAFE_XML_PARSER) + +@contextmanager +def validated_docx_source(path): + """Yield one private, bounded snapshot for both validation and python-docx.""" + with Path(path).open("rb") as external_source, TemporaryFile() as source: + copied = 0 + while chunk := external_source.read(64 * 1024): + copied += len(chunk) + require(copied <= MAX_ARCHIVE_BYTES, "compressed DOCX file size above limit") + source.write(chunk) + source.seek(0) + with zipfile.ZipFile(source) as archive: + validate_docx_archive(archive) + source.seek(0) + yield source MC_NAMESPACE = "http://schemas.openxmlformats.org/markup-compatibility/2006" MC_ALTERNATE_CONTENT = f"{{{MC_NAMESPACE}}}AlternateContent" @@ -227,17 +338,18 @@ def table_content(table): }) return rows -doc = Document("input.docx") -content_controls = list(doc.element.body.iter(qn("w:sdt"))) -blocks = list(iter_part_blocks(doc.element.body, doc)) -print("content controls:", len(content_controls), "top-level blocks:", len(blocks)) -for kind, block in blocks: - if kind == "paragraph": - print(block.style.name, "|", paragraph_text(block)) - elif kind == "table": - print("table |", table_content(block)) - else: - print("unreadable |", block) +with validated_docx_source("input.docx") as source: + doc = Document(source) + content_controls = list(doc.element.body.iter(qn("w:sdt"))) + blocks = list(iter_part_blocks(doc.element.body, doc)) + print("content controls:", len(content_controls), "top-level blocks:", len(blocks)) + for kind, block in blocks: + if kind == "paragraph": + print(block.style.name, "|", paragraph_text(block)) + elif kind == "table": + print("table |", table_content(block)) + else: + print("unreadable |", block) ``` Notes: diff --git a/plugins/Hylouis233/document-skills/skills/docx/references/review.md b/plugins/Hylouis233/document-skills/skills/docx/references/review.md index 24fe912..87b5201 100644 --- a/plugins/Hylouis233/document-skills/skills/docx/references/review.md +++ b/plugins/Hylouis233/document-skills/skills/docx/references/review.md @@ -24,6 +24,7 @@ MAX_XML_PART = 20 * 1024 * 1024 MAX_ENTRY = 100 * 1024 * 1024 MAX_TOTAL_UNCOMPRESSED = 500 * 1024 * 1024 MAX_COMPRESSION_RATIO = 200 +CONTENT_TYPES_NAMESPACE = "http://schemas.openxmlformats.org/package/2006/content-types" # Security limits must survive `python -O` (which strips assert statements), # so every check raises explicitly instead of asserting. @@ -38,6 +39,46 @@ safe_xml_parser = etree.XMLParser( huge_tree=False, recover=False, ) + +def xml_content_type(value): + media_type = (value or "").split(";", 1)[0].strip().casefold() + return media_type in {"application/xml", "text/xml"} or media_type.endswith("+xml") + +def declared_xml_parts(archive, infos): + """Classify XML from OPC declarations plus conventional suffixes.""" + by_name = {info.filename: info for info in infos} + content_types_info = by_name.get("[Content_Types].xml") + require(content_types_info is not None, "missing [Content_Types].xml") + require(content_types_info.file_size <= MAX_XML_PART, + "oversized XML part: [Content_Types].xml") + with archive.open(content_types_info) as stream: + content_types_blob = stream.read(MAX_XML_PART + 1) + require(len(content_types_blob) <= MAX_XML_PART, + "oversized XML part: [Content_Types].xml") + root = etree.fromstring(content_types_blob, parser=safe_xml_parser) + require(root.tag == f"{{{CONTENT_TYPES_NAMESPACE}}}Types", + "invalid [Content_Types].xml root") + defaults = {} + overrides = {} + for child in root: + if child.tag == f"{{{CONTENT_TYPES_NAMESPACE}}}Default": + extension = (child.get("Extension") or "").casefold() + require(extension and extension not in defaults, + "invalid duplicate content-type default") + defaults[extension] = child.get("ContentType") or "" + elif child.tag == f"{{{CONTENT_TYPES_NAMESPACE}}}Override": + part_name = child.get("PartName") or "" + require(part_name.startswith("/") and part_name[1:] not in overrides, + "invalid duplicate content-type override") + overrides[part_name[1:]] = child.get("ContentType") or "" + xml_names = {"[Content_Types].xml"} + for info in infos: + suffix = info.filename.rsplit(".", 1)[1].casefold() if "." in info.filename else "" + content_type = overrides.get(info.filename, defaults.get(suffix, "")) + if (info.filename.casefold().endswith((".xml", ".rels")) + or xml_content_type(content_type)): + xml_names.add(info.filename) + return xml_names # Check the package itself before ZipFile materializes its central directory. require(Path(path).stat().st_size <= MAX_ARCHIVE_BYTES, "compressed DOCX file size above limit") @@ -51,12 +92,13 @@ with zipfile.ZipFile(path) as z: "missing required OPC members") require(sum(info.file_size for info in infos) <= MAX_TOTAL_UNCOMPRESSED, "declared total uncompressed size above limit") + xml_names = declared_xml_parts(z, infos) actual_total = 0 for info in infos: require(info.file_size <= MAX_ENTRY, f"oversized part: {info.filename}") ratio = info.file_size / max(info.compress_size, 1) require(ratio <= MAX_COMPRESSION_RATIO, f"suspicious compression ratio: {info.filename}") - is_xml = info.filename.endswith((".xml", ".rels")) + is_xml = info.filename in xml_names if is_xml: require(info.file_size <= MAX_XML_PART, f"oversized XML part: {info.filename}") chunks = [] diff --git a/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md b/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md index fa16703..ec68e5f 100644 --- a/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md +++ b/plugins/Hylouis233/document-skills/skills/pdf/references/inspect.md @@ -163,6 +163,48 @@ MAX_TOTAL_IMAGE_RENDER_PIXELS = 20_000_000 MAX_DRAWING_PATHS = 1_000 MAX_DRAWING_RENDER_PIXELS = 4_000_000 MAX_TOTAL_DRAWING_RENDER_PIXELS = 20_000_000 +MAX_TEXT_SPANS = 10_000 +MAX_TEXT_RENDER_PIXELS = 4_000_000 +MAX_TOTAL_TEXT_RENDER_PIXELS = 20_000_000 + +def viewable_text(page): + """Use text render mode, opacity, clipping, and bounded alpha rendering.""" + try: + spans = page.get_texttrace() + except (RuntimeError, ValueError): + return [], [], True + if len(spans) > MAX_TEXT_SPANS: + return spans, [], True + visible = [] + total_render_pixels = 0 + for span in spans: + try: + text = "".join(chr(character[0]) for character in span.get("chars", ())) + render_type = int(span.get("type")) + opacity = float(span.get("opacity")) + except (TypeError, ValueError, OverflowError): + return spans, visible, True + if not text.strip() or render_type > 1 or opacity <= 0: + continue + if render_type not in (0, 1) or not math.isfinite(opacity): + return spans, visible, True + clip = visible_clip(page, span.get("bbox")) + if clip is None: + continue + render_pixels = math.ceil(clip.width) * math.ceil(clip.height) + total_render_pixels += render_pixels + if (render_pixels > MAX_TEXT_RENDER_PIXELS + or total_render_pixels > MAX_TOTAL_TEXT_RENDER_PIXELS): + return spans, visible, True + try: + pixmap = page.get_pixmap(clip=clip, alpha=True, annots=False) + except (RuntimeError, ValueError): + return spans, visible, True + if not pixmap.alpha: + return spans, visible, True + if any(pixmap.samples[pixmap.n - 1::pixmap.n]): + visible.append(span) + return spans, visible, False def viewable_images(page): """Render bounded placement clips; unknown visibility keeps the page nonblank.""" @@ -338,17 +380,20 @@ for page in doc: "crop_size": crop_size, "rotation": page.rotation, }) + text_spans, visible_text, text_visibility_unknown = viewable_text(page) image_placements, visible_images, image_visibility_unknown = viewable_images(page) drawings, visible_drawings, drawing_visibility_unknown = viewable_drawings(page) widgets, annotations, links, interaction_visibility_unknown = viewable_interactives(page) is_blank = not ( - page.get_text().strip() or visible_images or visible_drawings + visible_text or visible_images or visible_drawings or widgets or annotations or links - or image_visibility_unknown or drawing_visibility_unknown + or text_visibility_unknown or image_visibility_unknown or drawing_visibility_unknown or interaction_visibility_unknown ) print(page.number + 1, "media_size:", media_size, "crop_size:", crop_size, "rotation:", page.rotation, "text_len:", len(page.get_text()), + "text_spans:", len(text_spans), "visible_text_spans:", len(visible_text), + "text_visibility_unknown:", text_visibility_unknown, "resource_images:", len(page.get_images()), "image_placements:", len(image_placements), "visible_images:", len(visible_images), @@ -366,7 +411,7 @@ print("crop_size_consistent:", len({row["crop_size"] for row in page_geometry}) ## Checks worth automating -- **Blank page detection**: flag only when text, visible painted image placements and vector paths, +- **Blank page detection**: flag only when rendered text, visible painted image placements and vector paths, and viewable widgets, annotations, and links are all absent. Ignore interactive objects carrying invisible, hidden, or no-view flags, as well as empty, off-page, or unrendered appearances. `page.get_images()` lists every image XObject resource, including unused resources, and also diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md b/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md index 3fc76e8..a15a40f 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/SKILL.md @@ -78,6 +78,11 @@ expected_number_formats = { expected_formulas = { # "Sales": {"D2": "=C2*1.08"}, } +# Populate every expected sheet with the exact used range required by the task. +expected_dimensions = { + # "Sales": "A1:E20", + # "Summary": "A1:C8", +} wb = load_validated_workbook(output_path) def require(condition, message): @@ -98,6 +103,10 @@ def formula_text(value): print("sheets:", wb.sheetnames) missing = set(expected_sheets) - set(wb.sheetnames) require(not missing, f"missing expected sheets: {sorted(missing)}") +require( + set(expected_dimensions) == set(expected_sheets), + "expected_dimensions must declare the exact used range for every expected sheet", +) calc = wb.calculation print("calcMode:", getattr(calc, "calcMode", None), "fullCalcOnLoad:", getattr(calc, "fullCalcOnLoad", None)) @@ -109,6 +118,12 @@ if any(expected_formulas.values()): ) for ws in wb.worksheets: print(f"{ws.title} dims:", ws.dimensions) + if ws.title in expected_dimensions: + require( + ws.dimensions == expected_dimensions[ws.title], + f"{ws.title}: expected used range {expected_dimensions[ws.title]!r}, " + f"got {ws.dimensions!r}", + ) # `expected_formulas` is the task contract, so verify those coordinates directly. # Never call unbounded iter_rows(): one styled extreme cell can make the rectangle huge. actual_formulas = {} diff --git a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md index 5951081..cf36e83 100644 --- a/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md +++ b/plugins/Hylouis233/document-skills/skills/xlsx/references/edit.md @@ -296,6 +296,9 @@ wb.save("input-edited.xlsx") """Dry-run one workbook, then return a fresh edit copy from the same source.""" require(not load_options.get("read_only"), "round-trip audit requires a normal writable Workbook") + require(load_options.get("rich_text", True) is True, + "round-trip audit must preserve rich-text cell runs") + load_options["rich_text"] = True source.seek(0) before_names, before_extensions = archive_inventory(source) source.seek(0) diff --git a/plugins/Hylouis233/document-skills/tests/docx_fixture.py b/plugins/Hylouis233/document-skills/tests/docx_fixture.py index 81a3657..8124ba5 100644 --- a/plugins/Hylouis233/document-skills/tests/docx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/docx_fixture.py @@ -11,8 +11,9 @@ import sys import unicodedata import zipfile +from contextlib import contextmanager from pathlib import Path, PurePosixPath -from tempfile import TemporaryDirectory +from tempfile import TemporaryDirectory, TemporaryFile import fitz from docx import Document @@ -58,6 +59,7 @@ def check(name, cond, extra=""): MAX_ENTRY = 100 * 1024 * 1024 MAX_TOTAL_UNCOMPRESSED = 500 * 1024 * 1024 MAX_COMPRESSION_RATIO = 200 +CONTENT_TYPES_NAMESPACE = "http://schemas.openxmlformats.org/package/2006/content-types" MAX_MEMBER_COMPONENT_BYTES = 255 MAX_MEMBER_COMPONENT_UTF16_UNITS = 255 MAX_MEMBER_PATH_BYTES = 1024 @@ -70,6 +72,47 @@ def require(condition, message): raise ValueError(message) +def xml_content_type(value): + media_type = (value or "").split(";", 1)[0].strip().casefold() + return media_type in {"application/xml", "text/xml"} or media_type.endswith("+xml") + + +def declared_xml_parts(archive, infos): + by_name = {info.filename: info for info in infos} + content_types_info = by_name.get("[Content_Types].xml") + require(content_types_info is not None, "missing [Content_Types].xml") + require(content_types_info.file_size <= MAX_XML_PART, + "oversized XML part: [Content_Types].xml") + with archive.open(content_types_info) as stream: + content_types_blob = stream.read(MAX_XML_PART + 1) + require(len(content_types_blob) <= MAX_XML_PART, + "oversized XML part: [Content_Types].xml") + root = etree.fromstring(content_types_blob, parser=safe_xml_parser) + require(root.tag == f"{{{CONTENT_TYPES_NAMESPACE}}}Types", + "invalid [Content_Types].xml root") + defaults = {} + overrides = {} + for child in root: + if child.tag == f"{{{CONTENT_TYPES_NAMESPACE}}}Default": + extension = (child.get("Extension") or "").casefold() + require(extension and extension not in defaults, + "invalid duplicate content-type default") + defaults[extension] = child.get("ContentType") or "" + elif child.tag == f"{{{CONTENT_TYPES_NAMESPACE}}}Override": + part_name = child.get("PartName") or "" + require(part_name.startswith("/") and part_name[1:] not in overrides, + "invalid duplicate content-type override") + overrides[part_name[1:]] = child.get("ContentType") or "" + xml_names = {"[Content_Types].xml"} + for info in infos: + suffix = info.filename.rsplit(".", 1)[1].casefold() if "." in info.filename else "" + content_type = overrides.get(info.filename, defaults.get(suffix, "")) + if (info.filename.casefold().endswith((".xml", ".rels")) + or xml_content_type(content_type)): + xml_names.add(info.filename) + return xml_names + + def validate_docx_package(path): require( Path(path).stat().st_size <= MAX_ARCHIVE_BYTES, @@ -88,6 +131,7 @@ def validate_docx_package(path): sum(info.file_size for info in infos) <= MAX_TOTAL_UNCOMPRESSED, "declared archive size exceeds the review limit", ) + xml_names = declared_xml_parts(archive, infos) actual_total = 0 for info in infos: require(info.file_size <= MAX_ENTRY, f"oversized part: {info.filename}") @@ -95,7 +139,7 @@ def validate_docx_package(path): info.file_size / max(info.compress_size, 1) <= MAX_COMPRESSION_RATIO, f"suspicious compression ratio: {info.filename}", ) - is_xml = info.filename.endswith((".xml", ".rels")) + is_xml = info.filename in xml_names if is_xml: require(info.file_size <= MAX_XML_PART, f"oversized XML part: {info.filename}") chunks = [] @@ -116,6 +160,58 @@ def validate_docx_package(path): etree.fromstring(b"".join(chunks), parser=safe_xml_parser) +@contextmanager +def validated_docx_source(path): + with Path(path).open("rb") as external_source, TemporaryFile() as source: + copied = 0 + while chunk := external_source.read(64 * 1024): + copied += len(chunk) + require(copied <= MAX_ARCHIVE_BYTES, "compressed DOCX file size above limit") + source.write(chunk) + source.seek(0) + with zipfile.ZipFile(source) as archive: + infos = archive.infolist() + require(len(infos) <= MAX_MEMBERS, "archive member count above limit") + names = {info.filename for info in infos} + require(len(names) == len(infos), "duplicate archive member names are unsafe") + require("[Content_Types].xml" in names and "word/document.xml" in names, + "missing required OPC members") + require(sum(info.file_size for info in infos) <= MAX_TOTAL_UNCOMPRESSED, + "declared total uncompressed size above limit") + xml_names = declared_xml_parts(archive, infos) + actual_total = 0 + for info in infos: + require(info.file_size <= MAX_ENTRY, f"oversized part: {info.filename}") + require(info.file_size / max(info.compress_size, 1) <= MAX_COMPRESSION_RATIO, + f"suspicious compression ratio: {info.filename}") + is_xml = info.filename in xml_names + if is_xml: + require(info.file_size <= MAX_XML_PART, + f"oversized XML part: {info.filename}") + chunks = [] + actual_size = 0 + with archive.open(info) as stream: + while chunk := stream.read(64 * 1024): + actual_size += len(chunk) + actual_total += len(chunk) + require(actual_size <= MAX_ENTRY, + f"part exceeded read limit: {info.filename}") + require(actual_total <= MAX_TOTAL_UNCOMPRESSED, + "archive exceeded total read limit") + if is_xml: + chunks.append(chunk) + require(actual_size == info.file_size, f"size mismatch: {info.filename}") + if is_xml: + etree.fromstring(b"".join(chunks), parser=safe_xml_parser) + source.seek(0) + yield source + + +def load_validated_docx(path, loader=Document): + with validated_docx_source(path) as source: + return loader(source) + + health_doc = Document() health_doc.add_paragraph("bounded health check") health_doc.save("healthy.docx") @@ -139,6 +235,53 @@ def validate_docx_package(path): "archive safety checks remain active under optimized Python", __debug__ or archive_bomb_rejected, ) +loader_calls = [] +try: + load_validated_docx( + "compressed-bomb.docx", + loader=lambda source: loader_calls.append(source), + ) + read_bomb_rejected = False +except ValueError: + read_bomb_rejected = True +check("structured DOCX read rejects a bomb before python-docx is called", + read_bomb_rejected and loader_calls == [], loader_calls) +loaded_healthy = load_validated_docx("healthy.docx") +check("structured DOCX read validates and loads one private snapshot", + loaded_healthy.paragraphs[0].text == "bounded health check") + + +def docx_with_extra_part(source_path, output_path, name, payload, content_type=None): + with zipfile.ZipFile(source_path) as source: + members = {info.filename: source.read(info) for info in source.infolist()} + if content_type is not None: + root = etree.fromstring(members["[Content_Types].xml"], parser=safe_xml_parser) + override = etree.SubElement(root, f"{{{CONTENT_TYPES_NAMESPACE}}}Override") + override.set("PartName", "/" + name) + override.set("ContentType", content_type) + members["[Content_Types].xml"] = etree.tostring(root, xml_declaration=True, + encoding="UTF-8") + members[name] = payload + with zipfile.ZipFile(output_path, "w", zipfile.ZIP_DEFLATED) as output: + for member_name, member_data in members.items(): + output.writestr(member_name, member_data) + + +docx_with_extra_part("healthy.docx", "uppercase-xml.docx", "custom/BROKEN.XML", b"") diff --git a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py index 94be938..0581a20 100644 --- a/plugins/Hylouis233/document-skills/tests/pdf_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/pdf_fixture.py @@ -540,6 +540,48 @@ def visible_clip(page, rectangle, *, already_rotated=False): MAX_DRAWING_PATHS = 1_000 MAX_DRAWING_RENDER_PIXELS = 4_000_000 MAX_TOTAL_DRAWING_RENDER_PIXELS = 20_000_000 +MAX_TEXT_SPANS = 10_000 +MAX_TEXT_RENDER_PIXELS = 4_000_000 +MAX_TOTAL_TEXT_RENDER_PIXELS = 20_000_000 + + +def viewable_text(page): + try: + spans = page.get_texttrace() + except (RuntimeError, ValueError): + return [], [], True + if len(spans) > MAX_TEXT_SPANS: + return spans, [], True + visible = [] + total_render_pixels = 0 + for span in spans: + try: + text = "".join(chr(character[0]) for character in span.get("chars", ())) + render_type = int(span.get("type")) + opacity = float(span.get("opacity")) + except (TypeError, ValueError, OverflowError): + return spans, visible, True + if not text.strip() or render_type > 1 or opacity <= 0: + continue + if render_type not in (0, 1) or not math.isfinite(opacity): + return spans, visible, True + clip = visible_clip(page, span.get("bbox")) + if clip is None: + continue + render_pixels = math.ceil(clip.width) * math.ceil(clip.height) + total_render_pixels += render_pixels + if (render_pixels > MAX_TEXT_RENDER_PIXELS + or total_render_pixels > MAX_TOTAL_TEXT_RENDER_PIXELS): + return spans, visible, True + try: + pixmap = page.get_pixmap(clip=clip, alpha=True, annots=False) + except (RuntimeError, ValueError): + return spans, visible, True + if not pixmap.alpha: + return spans, visible, True + if any(pixmap.samples[pixmap.n - 1::pixmap.n]): + visible.append(span) + return spans, visible, False def viewable_images(page): @@ -736,7 +778,7 @@ class VisibleInteractiveProbe: widget_page ) blank = ( - not widget_page.get_text().strip() + not viewable_text(widget_page)[1] and not viewable_text(widget_page)[2] and not widget_page.get_images() and not visible_widget_drawings and not widget_drawing_visibility_unknown and not widgets and not annotations and not links and not interaction_visibility_unknown @@ -1251,13 +1293,15 @@ def stamp_placement(page, stamp_box): def inspected_page_is_blank(page): + _, visible_text, text_visibility_unknown = viewable_text(page) _, visible_images, image_visibility_unknown = viewable_images(page) _, visible_drawings, drawing_visibility_unknown = viewable_drawings(page) + widgets, annotations, links, interaction_visibility_unknown = viewable_interactives(page) return not ( - page.get_text().strip() or visible_images - or visible_drawings or list(page.widgets() or ()) - or list(page.annots() or ()) or page.get_links() - or image_visibility_unknown or drawing_visibility_unknown + visible_text or visible_images or visible_drawings + or widgets or annotations or links + or text_visibility_unknown or image_visibility_unknown + or drawing_visibility_unknown or interaction_visibility_unknown ) @@ -1265,6 +1309,69 @@ def inspected_page_is_blank(page): check("form-only page exposes a widget", len(form_only_widgets) == 1) check("form-only page is not classified as blank", not form_only_is_blank) +# Extractable text can still be visually absent because of Tr=3, alpha=0, +# clipping, or off-page geometry. +text_visibility_canvas = canvas.Canvas( + "text-visibility.pdf", pagesize=(200, 200), pageCompression=0, +) +text_visibility_canvas.drawString(40, 100, "visible") +text_visibility_canvas.showPage() +invisible_text = text_visibility_canvas.beginText(40, 100) +invisible_text.setTextRenderMode(3) +invisible_text.textOut("render-mode-hidden") +text_visibility_canvas.drawText(invisible_text) +text_visibility_canvas.showPage() +text_visibility_canvas.saveState() +text_visibility_canvas.setFillAlpha(0) +text_visibility_canvas.drawString(40, 100, "zero-opacity") +text_visibility_canvas.restoreState() +text_visibility_canvas.showPage() +text_clip = text_visibility_canvas.beginPath() +text_clip.rect(0, 0, 10, 10) +text_visibility_canvas.saveState() +text_visibility_canvas.clipPath(text_clip, stroke=0, fill=0) +text_visibility_canvas.drawString(40, 100, "clipped") +text_visibility_canvas.restoreState() +text_visibility_canvas.showPage() +text_visibility_canvas.drawString(250, 100, "off-page") +text_visibility_canvas.showPage() +text_visibility_canvas.save() + +text_visibility_doc = fitz.open("text-visibility.pdf") +text_visibility_results = [] +for text_page in text_visibility_doc: + text_spans, visible_text_spans, text_visibility_unknown = viewable_text(text_page) + text_visibility_results.append(( + len(text_spans), len(visible_text_spans), text_visibility_unknown, + inspected_page_is_blank(text_page), + )) +check("blank-page text detection uses rendered visibility", + text_visibility_results == [ + (1, 1, False, False), + (1, 0, False, True), + (1, 0, False, True), + (1, 0, False, True), + (1, 0, False, True), + ], text_visibility_results) + +text_budget_results = {} +for budget_name in ( + "MAX_TEXT_SPANS", "MAX_TEXT_RENDER_PIXELS", "MAX_TOTAL_TEXT_RENDER_PIXELS", +): + original_budget = globals()[budget_name] + globals()[budget_name] = 0 + try: + spans, visible_spans, visibility_unknown = viewable_text(text_visibility_doc[0]) + page_is_blank = inspected_page_is_blank(text_visibility_doc[0]) + finally: + globals()[budget_name] = original_budget + text_budget_results[budget_name] = ( + len(spans), len(visible_spans), visibility_unknown, page_is_blank, + ) +check("every text count/render budget fails closed as visibility unknown", + all(result == (1, 0, True, False) for result in text_budget_results.values()), + text_budget_results) + # PyMuPDF inventories invoked paths even when page geometry, clipping, or opacity # prevents them from contributing a rendered pixel. drawing_visibility_canvas = canvas.Canvas( diff --git a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py index cb6b30d..221a3e6 100644 --- a/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py +++ b/plugins/Hylouis233/document-skills/tests/xlsx_fixture.py @@ -14,6 +14,8 @@ from tempfile import TemporaryFile, mkstemp import openpyxl +from openpyxl.cell.rich_text import CellRichText, TextBlock +from openpyxl.cell.text import InlineFont failures = [] @@ -537,6 +539,9 @@ def stripped_extension_records(before, after, common_names): def load_with_round_trip_audit_from_source(source, **load_options): require(not load_options.get("read_only"), "round-trip audit requires a normal writable Workbook") + require(load_options.get("rich_text", True) is True, + "round-trip audit must preserve rich-text cell runs") + load_options["rich_text"] = True source.seek(0) before_names, before_extensions = archive_inventory(source) source.seek(0) @@ -575,6 +580,37 @@ def load_with_round_trip_audit(path, **load_options): audited_plain["Data"]["A2"].value == "EU") audited_plain.close() +rich_input = openpyxl.Workbook() +rich_input.active.title = "Rich" +rich_input.active["A1"] = CellRichText( + TextBlock(InlineFont(b=True), "Bold"), " and plain" +) +rich_input.save("rich-roundtrip.xlsx") +rich_editable, rich_dropped, rich_stripped = load_with_round_trip_audit( + "rich-roundtrip.xlsx" +) +rich_editable["Rich"]["B1"] = "edited" +rich_editable.save("rich-roundtrip-edited.xlsx") +rich_editable.close() +rich_reopened = openpyxl.load_workbook("rich-roundtrip-edited.xlsx", rich_text=True) +rich_value = rich_reopened["Rich"]["A1"].value +check( + "round-trip audit preserves rich-text runs while editing another cell", + rich_dropped == [] and rich_stripped == [] + and isinstance(rich_value, CellRichText) + and isinstance(rich_value[0], TextBlock) and rich_value[0].font.b is True + and str(rich_value) == "Bold and plain" + and rich_reopened["Rich"]["B1"].value == "edited", + (rich_dropped, rich_stripped, type(rich_value), rich_value), +) +rich_reopened.close() +try: + load_with_round_trip_audit("plain.xlsx", rich_text=False) + rich_text_opt_out_rejected = False +except ValueError as error: + rich_text_opt_out_rejected = "must preserve rich-text" in str(error) +check("round-trip audit rejects rich-text flattening opt-outs", rich_text_opt_out_rejected) + from openpyxl.drawing.image import Image as AuditImage Path("audit-image.png").write_bytes(base64.b64decode( "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9Y9ZlRYAAAAASUVORK5CYII=" @@ -1854,6 +1890,33 @@ def expected_formula_inventory(sheet, expected): formula_bound_ws.iter_rows = original_iter_rows check("formula postcheck uses bounded public coordinate lookups", bounded_formulas == {"D2": "=1+1"}, bounded_formulas) + +dimension_contract_wb = openpyxl.Workbook() +dimension_contract_ws = dimension_contract_wb.active +dimension_contract_ws.title = "Expected" +dimension_contract_ws["A1"] = "header" +dimension_contract_ws["C3"] = "tail" +expected_dimension_contract = {"Expected": "A1:C3"} +try: + require( + dimension_contract_ws.dimensions == expected_dimension_contract["Expected"], + "unexpected used range", + ) + exact_dimension_passed = True +except ValueError: + exact_dimension_passed = False +dimension_contract_ws["D4"] = "unintended" +try: + require( + dimension_contract_ws.dimensions == expected_dimension_contract["Expected"], + "unexpected used range", + ) + stale_dimension_rejected = False +except ValueError: + stale_dimension_rejected = True +check("postcheck accepts the exact task-declared worksheet range", exact_dimension_passed) +check("postcheck rejects unintended cells outside the task-declared range", + stale_dimension_rejected, dimension_contract_ws.dimensions) formula_bound_reopened.close()