diff --git a/packages/document-schema.js/README.md b/packages/document-schema.js/README.md index 1627cbbd4..efcd9bfd9 100644 --- a/packages/document-schema.js/README.md +++ b/packages/document-schema.js/README.md @@ -108,7 +108,7 @@ const laidOut = DocumentTreeSchema.parse({ - **Groups** are `{ node, children }` where `node` embeds either an anchor paragraph (heading groups and list-item groups carry the full `ContentParagraph` — runs, formatting, frames — never a projected text label) or a container descriptor: `{ kind: 'section', pageSize, margins }`, `{ kind: 'slide', size, notes }`, `{ kind: 'sheet', name, cells, columns, rows, printSettings }`, `{ kind: 'drawPage', size }`, each tagged with a `kind` the flat container type does not carry, or a shape group's untagged frame descriptor, or — since 4.1.0 — a **construct descriptor** (see [Fidelity constructs](#fidelity-constructs)). - **Bare leaves** carry their own `kind` and never `children`. Discrimination is structural on `node`+`children`, not on the presence of a `kind`. Every `ContentBlock` kind is a legal leaf except the two construct boundary markers, which are the flat form's encoding of something the tree carries as a group (see [Constructs in the flat form](#constructs-in-the-flat-form)). -- **Section groups are mandatory** — one per `ContentSection` — because a section carries pre-layout page geometry (`pageSize`/`margins`, plus the optional `breakType` naming how the section begins — nextPage/continuous/evenPage/oddPage, absent meaning the producer's own default) that a rendered `pages` array cannot hold. +- **Section groups are mandatory** — one per `ContentSection` — because a section carries pre-layout page geometry (`pageSize`/`margins`, plus the optional `breakType` naming how the section begins — nextPage/continuous/evenPage/oddPage, absent meaning the producer's own default) and its optional page furniture (`headers`/`footers`, per-slot block flows in WordprocessingML's own default/even/first reference vocabulary — ExaDev/documents.js#1128) that a rendered `pages` array cannot hold. - **Grouping never crosses container boundaries**: a shape is its own group with its inner blocks grouped inside it (never a slide's paragraphs flattened across its shapes — that is a TOC projection, not a decomposition); a sheet's grid rides on the sheet node with images and embedded documents as children; embedded documents stay intact as one leaf. - **Style refs ride on group wrappers only** — a group may carry `style: string` naming a `styles` table entry; `ContentDocument` nodes carry no ref field, so the flat codec-exchange form is always fully materialised. diff --git a/packages/document-schema.js/src/bijection.test.ts b/packages/document-schema.js/src/bijection.test.ts index b13768632..9f870e6dd 100644 --- a/packages/document-schema.js/src/bijection.test.ts +++ b/packages/document-schema.js/src/bijection.test.ts @@ -437,6 +437,28 @@ function corpus(): readonly CorpusEntry[] { sections: [{ ...SECTION_GEOMETRY, blocks: [paragraph("body")] }], }, }, + { + name: "wordprocessing section carrying page furniture in every slot (ExaDev/documents.js#1128)", + content: { + kind: "wordprocessing", + metadata: {}, + sections: [ + { + ...SECTION_GEOMETRY, + headers: { + default: [paragraph("header default")], + even: [paragraph("header even")], + first: [paragraph("header first")], + }, + footers: { + default: [paragraph("footer default")], + even: [paragraph("footer even")], + }, + blocks: [paragraph("body")], + }, + ], + }, + }, { name: "presentation with several shapes, list nesting inside each, and a heading-styled leaf", content: { diff --git a/packages/document-schema.js/src/content-json-schema-defs.test.ts b/packages/document-schema.js/src/content-json-schema-defs.test.ts index 35a79cde0..c164add59 100644 --- a/packages/document-schema.js/src/content-json-schema-defs.test.ts +++ b/packages/document-schema.js/src/content-json-schema-defs.test.ts @@ -54,6 +54,7 @@ import { ContentSubpathSchema, ContentVectorSchema, ContentCellValueSchema, + ContentPageFurnitureSchema, } from "./content"; import { CONTENT_DEFS } from "./content-json-schema-defs"; import { @@ -123,6 +124,7 @@ import { SourceResidueSchema } from "./source"; // Comparison strategy: a bare `z.toJSONSchema(SomeSchema)` call, run in isolation, would INLINE every nested schema it encounters (ColorSchema inside ContentRunSchema, AlignmentSchema inside ContentParagraphSchema, etc.) rather than emit the `{ $ref: '#/$defs/X' }` pointers CONTENT_DEFS itself uses -- because those nested schemas aren't registered anywhere. To reproduce the exact cross-reference shape CONTENT_DEFS hand-authors, this test registers the identical set of real schemas under the identical id strings CONTENT_DEFS uses as its own $defs keys, with a `uri` callback matching the `#/$defs/` convention CONTENT_DEFS was written against -- confirmed empirically (see this file's own construction) to make Zod's registry-based multi-schema generation emit exactly that $ref shape for every registered schema referenced from within another. Each per-schema result still carries its own top-level `$schema`/`$id` (since z.toJSONSchema(registry, ...) treats every registered schema as its own standalone root), which CONTENT_DEFS's own nested fragments never have -- those two keys are stripped before comparison, since they're an artefact of testing each fragment as a registry root rather than a real structural difference. const REGISTERED_SCHEMAS = { + ContentPageFurniture: ContentPageFurnitureSchema, Color: ColorSchema, Box: BoxSchema, LayoutFrame: LayoutFrameSchema, diff --git a/packages/document-schema.js/src/content-json-schema-defs.ts b/packages/document-schema.js/src/content-json-schema-defs.ts index 2e45de6e0..f1862a2b6 100644 --- a/packages/document-schema.js/src/content-json-schema-defs.ts +++ b/packages/document-schema.js/src/content-json-schema-defs.ts @@ -603,12 +603,24 @@ export const CONTENT_DEFS: Record = { type: "string", enum: ["nextPage", "continuous", "evenPage", "oddPage"], }, + headers: { $ref: "#/$defs/ContentPageFurniture" }, + footers: { $ref: "#/$defs/ContentPageFurniture" }, source: { $ref: "#/$defs/SourceResidue" }, kind: { type: "string", const: "section" }, }, required: ["pageSize", "margins", "kind"], additionalProperties: false, }, + // The per-slot page-furniture block flows a ContentSection's headers/footers fields carry (src/content.ts's ContentPageFurnitureSchema). Hand-authored here for the same recursive reason as every other block-array shape: the slots hold ContentBlock, which is the hand-written structural guard in Zod and needs its JSON spelling stated alongside. + ContentPageFurniture: { + type: "object", + properties: { + default: { type: "array", items: { $ref: "#/$defs/ContentBlock" } }, + even: { type: "array", items: { $ref: "#/$defs/ContentBlock" } }, + first: { type: "array", items: { $ref: "#/$defs/ContentBlock" } }, + }, + additionalProperties: false, + }, SlideDescriptor: { type: "object", properties: { diff --git a/packages/document-schema.js/src/content.ts b/packages/document-schema.js/src/content.ts index c2d389e37..ae26d196c 100644 --- a/packages/document-schema.js/src/content.ts +++ b/packages/document-schema.js/src/content.ts @@ -661,6 +661,14 @@ export const ContentBlockSchema: z.ZodType = ContentConstructEndSchema, ]); +// One furniture kind's per-slot block flows -- see ContentSectionSchema's own headers/footers comment for the slot vocabulary's format evidence. A slot is absent when the section states no furniture for it: an absent default slot with a present even slot is the even/odd-headers shape, not a gap (ExaDev/documents.js#1128). +export const ContentPageFurnitureSchema = z.object({ + default: z.lazy(() => z.array(ContentBlockSchema)).optional(), + even: z.lazy(() => z.array(ContentBlockSchema)).optional(), + first: z.lazy(() => z.array(ContentBlockSchema)).optional(), +}); +export type ContentPageFurniture = z.infer; + // A docx section: a run of pages sharing one page size/margins (a w:sectPr boundary starts a new one). export const ContentSectionSchema = z.object({ pageSize: PageSizeSchema, @@ -670,6 +678,9 @@ export const ContentSectionSchema = z.object({ breakType: z .enum(["nextPage", "continuous", "evenPage", "oddPage"]) .optional(), + // The page furniture this section repeats on its rendered pages -- the block flow a header or footer paints -- in the three-slot vocabulary WordprocessingML itself defines (w:headerReference/w:footerReference's own @w:type values default/even/first, with evenAndOddHeaders gating the even slot). Every page-furniture-carrying format narrows onto it: an ODF master page's style:header/style:header-left pair is default/even, its style:header-first the first slot; a WordPerfect D6 header's own occurrence bits (occurs on odd pages / occurs on even pages) state odd-only -> default, even-only -> even, both -> default. Slot-less decorations the vocabulary cannot state -- a watermark is neither header nor footer and owns no parity -- stay outside it. Section-scoped rather than document-level for the reason breakType is: page furniture belongs to the section that renders it, and a document with two sections may give each its own header. + headers: ContentPageFurnitureSchema.optional(), + footers: ContentPageFurnitureSchema.optional(), source: SourceResidueSchema.optional(), // quarantined residue -- opaque text this format carries and no other format interprets (src/source.ts); rides the tree's section descriptor automatically (omit+extend, src/package-node.ts) }); export type ContentSection = z.infer; diff --git a/packages/wpd-codec/README.md b/packages/wpd-codec/README.md index e8fda0dd0..55732ae53 100644 --- a/packages/wpd-codec/README.md +++ b/packages/wpd-codec/README.md @@ -174,8 +174,13 @@ This is a bet against libwpd's _code_, not against citing its _data_: the charac Everything below is recognised by the tokeniser and skipped by the fold, so a document containing it still reads — losing that construct's own structure, never the surrounding text. Each is reported through the diagnostic sink rather than passed over in silence. -- **Headers, footers, footnotes, and endnotes** (the 0xD6 and 0xD7 groups). Reported through `wpd/header-footer-dropped` and `wpd/note-dropped`. The text is genuinely recoverable — each function names a General WP Text packet (type 0x08) holding its own function-code stream, which this package's tokeniser and fold would read (the same packet type boxes now resolve their own text content through) — but the flat `ContentDocument` has no page-furniture position for a header or footer and no note position for a footnote body. That body's real home is `document-schema.js`'s tree-only `definitions` table, which a codec producing the flat form cannot reach; it is the same gap `rtf-codec` documents for its own equivalent constructs, and it closes at the schema boundary rather than here — the one gap in this list that stays blocked pending a DocumentTree-producing wpd-codec, distinct from every other entry, which is a parsing-effort gap rather than a schema-shape one. - **A box whose content is a presentation, video, macro, sound, or external payload**, and a box relying on its own template's inherited geometry rather than an explicit function-level position/size override. Reported through `wpd/box-content-unresolved` and `wpd/box-frame-unresolved` respectively. An image box IS lifted when its content packet carries a whole PNG or JPEG payload: the packet's raw bytes are scanned by signature and structural walk (`src/stream/image.ts` — the chunk chain to IEND for PNG, the marker segments to EOI for JPEG), never by guessing at a container header, and the span lifts as a `ContentImageBlock` sized by the box's own frame, with an absolute-from-page-edge position carried as the image's `floatPosition`. What stays unresolved: a WPG vector graphic (a distinct binary graphics format `ContentImageBlock`'s `png`/`jpeg`/`svg`/`gif` set cannot hold as recovered, and which this package does not decode — a project on the scale of this package's own WordPerfect reader, not a wiring job) and a native OLE object (see the next bullet). A box with no function-level content override at all — relying entirely on its template's own rendering defaults — is reported through `wpd/box-dropped`, unchanged from before. +- **Watermarks** (the 0xD6 group's two watermark subfunctions). Reported through `wpd/header-footer-dropped`: a watermark is neither a header nor a footer and owns no parity, so the shared page-furniture vocabulary has no slot for one. +- **A second header or footer claiming a slot a first already filled.** WordPerfect's own A/B two-slot-per-kind mechanism is a shape the shared one-flow-per-slot vocabulary does not carry; the first function to claim a slot is the one lifted, and the collision is reported through `wpd/header-footer-dropped`. + +Headers and footers themselves are LIFTED (ExaDev/documents.js#1128): a D6 function's occurrence bits narrow onto the shared furniture vocabulary's slots (`ContentSection.headers`/`footers`, `default`/`even` — odd-only and both-parities are the default slot, even-only the even slot), its body folded from the General WP Text packet its first prefix ID names. Footnotes and endnotes are anchored in the flat form (a footnote/endnote anchor construct around the reference site, `definition` naming `note-1`, `note-2`, ... in document order) and their bodies are carried by `readWpd` as definitions-table entries in the tree form — the flat `readWpdContent` reports each still-borne body through `wpd/note-dropped`, since the flat `ContentDocument` genuinely has no home for one (`rtf-codec` documents the same split for its own equivalent constructs). + +- **A box whose content is an image, presentation, video, macro, sound, or external payload**, and a box relying on its own template's inherited geometry rather than an explicit function-level position/size override. Reported through `wpd/box-content-unresolved` and `wpd/box-frame-unresolved` respectively. An image box's own content resolves to a Graphics Filename prefix packet (type 0x40) whose children carry either WPG vector graphics (a distinct binary graphics format `ContentImageBlock`'s `png`/`jpeg`/`svg`/`gif` set cannot hold as recovered, and which this package does not decode — a project on the scale of this package's own WordPerfect reader, not a wiring job) or a native OLE object (see the next bullet). A box with no function-level content override at all — relying entirely on its template's own rendering defaults — is reported through `wpd/box-dropped`, unchanged from before. - **Embedded OLE objects**, stored under the compound file's `PerfectOffice_OBJECTS` storage and named by an image box's Graphics Filename packet's own `0x70`/`0x71` (OLE Object Descriptor / OLE Object Data) children. `archive-codec`'s compound-file reader already reaches that storage, which is how `ooxml.js` recovers a ZIP-payload embedded object — but a WordPerfect OLE object's payload is a native OLE server's own stream rather than a nested document package (`ooxml.js`'s own equivalent case, a classic OLE1 `.bin` payload with no `Package` stream, stays opaque by the identical scope boundary), so recovering one generically is a project in its own right, not a wiring job. - **The counter groups** (0xD8, 0xD9, 0xDB, 0xDC): setting, numbering-method, increment and decrement carry no text and change no structure this reader models, so only the Display Number group's own paragraph-number pair is read. - **Every merge subfunction other than FIELD** (ASSIGN, CALL, IF, FOR, CASE, and the rest of WordPerfect's own merge scripting language) and **cross-references** (0xD5). A cross-reference's displayed text survives as ordinary text; its target binding does not. Reported through `wpd/merge-code-dropped` and `wpd/cross-reference-flattened`. Unlike FIELD, these can legitimately wrap whole paragraphs of body text as control flow, which the run-scoped field construct's own one-paragraph extent cannot express regardless — a schema gap for a scripting language's control flow, not a parsing gap. diff --git a/packages/wpd-codec/src/diagnostics.ts b/packages/wpd-codec/src/diagnostics.ts index 6fe6a378d..5cc5de625 100644 --- a/packages/wpd-codec/src/diagnostics.ts +++ b/packages/wpd-codec/src/diagnostics.ts @@ -28,9 +28,11 @@ export const WpdDiagnosticCodes = { OutlineNumberRegenerated: "wpd/outline-number-regenerated", // The document contains a box: a figure, text box, equation, or graphic. Its contents are not read; see the README's Remaining scope. BoxDropped: "wpd/box-dropped", - // The document contains a footnote or endnote. Its reference site is where this fires; the note's own text lives in a prefix packet the flat content model has nowhere to put. + // The document contains a footnote or endnote whose body the flat ContentDocument has no home for. Its reference anchor IS emitted (a footnote/endnote anchor construct around the reference site); the body is lifted into the tree form's definitions table by readWpd, so this fires only on the flat readWpdContent. NoteDropped: "wpd/note-dropped", - // The document declares a header, footer, or watermark. The flat content model has no page-furniture position for one. + // A note's On/Off reference pair straddled a paragraph boundary, which the run-scoped anchor cannot express. + NoteSpansParagraphs: "wpd/note-spans-paragraphs", + // The document declares a watermark, or a second header/footer function claims a slot a first already filled (WordPerfect's own A/B two-slot-per-kind mechanism, a shape the shared one-flow-per-slot vocabulary does not carry). A plain header or footer with a resolvable body is NOT dropped -- it lands in ContentSection.headers/footers. HeaderFooterDropped: "wpd/header-footer-dropped", // The document contains a cross-reference. Its displayed text survives as ordinary text; the reference's own target binding does not. CrossReferenceFlattened: "wpd/cross-reference-flattened", diff --git a/packages/wpd-codec/src/read-structure.test.ts b/packages/wpd-codec/src/read-structure.test.ts index c8cbc5e8d..47f82bc17 100644 --- a/packages/wpd-codec/src/read-structure.test.ts +++ b/packages/wpd-codec/src/read-structure.test.ts @@ -608,26 +608,29 @@ describe("document metadata", () => { }); describe("constructs this reader does not lift", () => { - // Each of these is recognised by the tokeniser and skipped by the fold, so a document containing it still reads -- and says what it lost rather than passing over it in silence. + // Each of these is recognised by the tokeniser and skipped by the fold, so a document containing it still reads -- and says what it lost rather than passing over it in silence. Group 0xD6 no longer appears here: a header or footer function is LIFTED into ContentSection.headers/footers (see the page-furniture describe below), and the watermark subfunction -- the one D6 shape the vocabulary has no slot for -- needs its own subgroup, which the empty-occurrence default of these bare fixtures cannot state. it.each([ - [0xdf, WpdDiagnosticCodes.BoxDropped], - [0xd7, WpdDiagnosticCodes.NoteDropped], - [0xd6, WpdDiagnosticCodes.HeaderFooterDropped], - [0xd5, WpdDiagnosticCodes.CrossReferenceFlattened], - [0xde, WpdDiagnosticCodes.MergeCodeDropped], - ])("reports group %i through the diagnostic sink", (group, code) => { - const { document, diagnostics } = readWithDiagnostics([ - ...text("before"), - ...variableFunction({ group, subgroup: 0x00 }), - ...text("after"), - ]); - expect( - paragraphsOf(document)[0] - ?.runs.map((run) => run.text) - .join(""), - ).toBe("beforeafter"); - expect( - diagnostics.filter((diagnostic) => diagnostic.code === code), - ).toHaveLength(1); - }); + [0xdf, WpdDiagnosticCodes.BoxDropped, 0x00], + [0xd7, WpdDiagnosticCodes.NoteDropped, 0x00], + [0xd6, WpdDiagnosticCodes.HeaderFooterDropped, 0x04], + [0xd5, WpdDiagnosticCodes.CrossReferenceFlattened, 0x00], + [0xde, WpdDiagnosticCodes.MergeCodeDropped, 0x00], + ])( + "reports group %i through the diagnostic sink", + (group, code, subgroup) => { + const { document, diagnostics } = readWithDiagnostics([ + ...text("before"), + ...variableFunction({ group, subgroup }), + ...text("after"), + ]); + expect( + paragraphsOf(document)[0] + ?.runs.map((run) => run.text) + .join(""), + ).toBe("beforeafter"); + expect( + diagnostics.filter((diagnostic) => diagnostic.code === code), + ).toHaveLength(1); + }, + ); }); diff --git a/packages/wpd-codec/src/read.test.ts b/packages/wpd-codec/src/read.test.ts index d773947cc..8dc8599a0 100644 --- a/packages/wpd-codec/src/read.test.ts +++ b/packages/wpd-codec/src/read.test.ts @@ -770,3 +770,176 @@ describe("boxes", () => { ).toHaveLength(1); }); }); + +describe("page furniture and notes (D6/D7, #1128)", () => { + function generalWpTextPacket(documentArea: readonly number[]) { + const header = [ + 1, + 0, + 6, + 0, + documentArea.length & 0xff, + (documentArea.length >>> 8) & 0xff, + ]; + return { + packetType: 0x08, + bytes: new Uint8Array([...header, ...documentArea]), + }; + } + + function headerFunction(subgroup: number, occurrence: number): number[] { + return variableFunction({ + group: 0xd6, + subgroup, + prefixIds: [1], + nonDeletable: [occurrence, 0], + }); + } + + it("lifts a header occurring on odd pages into the section's default header slot", () => { + const document = readDocumentArea( + [...text("body"), ...headerFunction(0x00, 0x01)], + [generalWpTextPacket(text("Confidential draft"))], + ); + if (document.kind !== "wordprocessing") + throw new Error("expected wordprocessing"); + const section = document.sections[0]; + if (section === undefined) throw new Error("expected a section"); + expect( + section.headers?.default?.map((b) => + b.kind === "paragraph" ? b.runs.map((run) => run.text).join("") : "", + ), + ).toEqual(["Confidential draft"]); + expect(section.headers?.even).toBeUndefined(); + expect(section.footers).toBeUndefined(); + }); + + it("lifts an even-only footer into the even slot", () => { + const document = readDocumentArea( + [...text("body"), ...headerFunction(0x02, 0x02)], + [generalWpTextPacket(text("Page footer"))], + ); + if (document.kind !== "wordprocessing") + throw new Error("expected wordprocessing"); + const section = document.sections[0]; + if (section === undefined) throw new Error("expected a section"); + expect( + section.footers?.even?.map((b) => + b.kind === "paragraph" ? b.runs.map((run) => run.text).join("") : "", + ), + ).toEqual(["Page footer"]); + expect(section.footers?.default).toBeUndefined(); + }); + + it("reports a watermark, which the furniture vocabulary has no slot for", () => { + const diagnostics: WpdDiagnostic[] = []; + readWpdContent( + buildWpdFile( + [ + ...text("body"), + ...variableFunction({ + group: 0xd6, + subgroup: 0x04, + prefixIds: [1], + nonDeletable: [0x03, 0], + }), + ], + [generalWpTextPacket(text("DRAFT"))], + ), + { sink: (d) => diagnostics.push(d) }, + ); + expect( + diagnostics.filter((d) => d.code === "wpd/header-footer-dropped"), + ).toHaveLength(1); + }); + + it("keeps the first header when a second claims the same slot", () => { + const diagnostics: WpdDiagnostic[] = []; + const document = readWpdContent( + buildWpdFile( + [ + ...text("body"), + ...headerFunction(0x00, 0x01), + ...headerFunction(0x01, 0x01), + ], + [generalWpTextPacket(text("First header"))], + ), + { sink: (d) => diagnostics.push(d) }, + ); + if (document.kind !== "wordprocessing") + throw new Error("expected wordprocessing"); + const section = document.sections[0]; + if (section === undefined) throw new Error("expected a section"); + expect( + section.headers?.default?.map((b) => + b.kind === "paragraph" ? b.runs.map((run) => run.text).join("") : "", + ), + ).toEqual(["First header"]); + expect( + diagnostics.some((d) => d.code === "wpd/header-footer-dropped"), + ).toBe(true); + }); + + it("anchors a footnote reference in the flat form and carries its body in the tree's definitions table", () => { + const noteBody = generalWpTextPacket(text("The fine print")); + const documentArea = [ + ...text("See this"), + ...variableFunction({ group: 0xd7, subgroup: 0x00, prefixIds: [1] }), + ...text("1"), + ...variableFunction({ group: 0xd7, subgroup: 0x01 }), + ...text(" point"), + ]; + const diagnostics: WpdDiagnostic[] = []; + const flat = readWpdContent(buildWpdFile(documentArea, [noteBody]), { + sink: (d) => diagnostics.push(d), + }); + if (flat.kind !== "wordprocessing") + throw new Error("expected wordprocessing"); + const paragraph = flat.sections[0]?.blocks.find( + (b): b is Extract => + b.kind === "paragraph", + ); + expect(paragraph?.constructs?.[0]?.descriptor.kind).toBe("anchor"); + const anchor = paragraph?.constructs?.[0]?.descriptor; + if (anchor?.kind === "anchor") { + expect(anchor.anchorType).toBe("footnote"); + expect(anchor.name).toBe("1"); + expect(anchor.definition).toBe("note-1"); + } else { + throw new Error("expected an anchor descriptor"); + } + // The flat form reports the body it cannot carry. + expect( + diagnostics.filter((d) => d.code === "wpd/note-dropped"), + ).toHaveLength(1); + + const tree = readWpd(buildWpdFile(documentArea, [noteBody])); + // The definitions table is deliberately tenant-loose (document-schema.js's own design), so the whole entry is asserted in one toEqual rather than through typed field access. + expect(tree.definitions?.["note-1"]).toEqual({ + kind: "footnote", + marker: "1", + blocks: [ + { + kind: "paragraph", + runs: [{ text: "The fine print" }], + }, + ], + }); + }); + + it("carries an endnote pair as the endnote tenant", () => { + const tree = readWpd( + buildWpdFile( + [ + ...text("Note"), + ...variableFunction({ group: 0xd7, subgroup: 0x02, prefixIds: [1] }), + ...text("2"), + ...variableFunction({ group: 0xd7, subgroup: 0x03 }), + ], + [generalWpTextPacket(text("The endnote body"))], + ), + ); + const definition = tree.definitions?.["note-1"]; + expect(definition?.kind).toBe("endnote"); + }); +}); diff --git a/packages/wpd-codec/src/read.ts b/packages/wpd-codec/src/read.ts index 0739bec5c..dd8a49f72 100644 --- a/packages/wpd-codec/src/read.ts +++ b/packages/wpd-codec/src/read.ts @@ -5,6 +5,7 @@ import type { ContentCellFill, ContentDocument, ContentEmbeddedObjectBlock, + ContentPageFurniture, ContentParagraph, ContentRun, ContentTableCell, @@ -16,6 +17,7 @@ import type { import { assembleTree } from "document-schema.js"; import { bytesToBase64 } from "./bytes/base64"; import { uint16At } from "./bytes/view"; +import { readFurnitureClaim } from "./stream/furniture"; import { openWpdDocument, type WpdDocumentContainer, @@ -239,6 +241,23 @@ interface ReaderState { pendingConstructs: RunConstructExtent[]; // The run index (into `runs`) a FIELD On merge code opened, or undefined when no FIELD scope is currently open. Reset to undefined -- abandoning the in-progress field, rather than reused across paragraphs -- whenever a paragraph flushes with one still open: `runs` is spliced empty by flushParagraph, so an index into the paragraph that just closed means nothing in the one that follows. openMergeFieldStartRun: number | undefined; + // The page furniture a D6 function has filled so far, per kind, keyed by the shared vocabulary's slots. A second function claiming a slot a first already filled is reported rather than overwritten -- WordPerfect's own A/B two-slot-per-kind mechanism is a shape the one-flow-per-slot vocabulary does not carry. + readonly headers: ContentPageFurniture; + readonly footers: ContentPageFurniture; + readonly furnitureFilled: Set; + // The note reference a D7 On function opened, or undefined when none is currently open. Abandoned at a paragraph boundary exactly like a merge FIELD, for the identical run-index reason. + openNote: + | { anchorType: "footnote" | "endnote"; startRun: number; prefixId: number } + | undefined; + // Every note whose body a D7 On/Off pair named, in document order, bodies folded from each function's own General WP Text packet. The flat ContentDocument has no home for a note body (its real home is the tree's definitions table), so readWpdContent reports these and readWpd carries them. + readonly notes: WpdNoteDefinition[]; +} + +// One lifted note, shaped exactly as the definitions-table note tenant the tree form carries ({ kind, marker, blocks } -- the tenant vocabulary markdown-codec's own footnote definitions established): the reference marker is the text the D7 On/Off pair encloses, the body the packet its prefix ID names. +export interface WpdNoteDefinition { + readonly anchorType: "footnote" | "endnote"; + readonly marker: string; + readonly blocks: readonly ContentBlock[]; } // The direct-formatting state a style packet's own "beginning style text" block can change, snapshotted before applying that block so the style's own scope closer can restore exactly what it overrode -- the same fields a Font Face Change, Font Size Change, character-colour function, or Attribute On/Off can change directly in the main stream, because a style's begin block is folded through the identical applyToken dispatch those use. @@ -312,6 +331,16 @@ function targetBlocks(state: ReaderState): ContentBlock[] { // Closes the current paragraph. Called for every hard return, so a document with two consecutive hard returns genuinely produces an empty paragraph between them -- that blank line is content the author typed, not an artefact. function flushParagraph(state: ReaderState, sink: WpdDiagnosticSink): void { flushRun(state); + // A note's D7 On/Off pair encloses a reference site -- a run-scoped extent, the identical constraint a merge FIELD has: an On with no Off before this paragraph closed means `runs` is about to be emptied and the start index means nothing in the next paragraph. Abandoned rather than carried, with the diagnostic saying so. + if (state.openNote !== undefined) { + state.openNote = undefined; + reportOnce( + state, + sink, + WpdDiagnosticCodes.NoteSpansParagraphs, + "A footnote or endnote's own On/Off pair straddled a paragraph boundary, which the run-scoped note anchor cannot express; its reference text became ordinary paragraph text with no note anchor.", + ); + } if (state.openMergeFieldStartRun !== undefined) { // A FIELD On with no matching FIELD Off before this paragraph closed: `runs` is about to be spliced empty, so the run index this field opened at means nothing in the paragraph that follows. Abandoned rather than carried forward -- the run-level extent mechanism cannot express a construct spanning two paragraphs, so no construct is emitted for this occurrence, and the diagnostic says so rather than the field silently vanishing with no trace. state.openMergeFieldStartRun = undefined; @@ -946,6 +975,171 @@ function applyCharacterGroup( } // FIELD On opens a run-scoped extent at the run boundary it sits at; FIELD Off closes it and tags the runs in between as a FieldDescriptor construct, `instruction` being exactly the field-code text that flowed through as ordinary characters between the two -- so a merge field's own displayed spelling is both kept as real run content (a template genuinely shows its own field codes, not a merged result) and tagged as a placeholder rather than typed prose. Every other merge subfunction still reports through the diagnostic sink, unchanged. +// Lifts a D6 header/footer function's body into the section's page furniture: the claim (which kind, which slot) comes from stream/furniture.ts's own subgroup + occurrence-byte reading, the body from the General WP Text packet the function's first prefix ID names -- folded through the identical tokeniser and fold the main document area uses, exactly as a box's own text content is. A watermark, a function whose claim narrows onto nothing, an unresolvable packet, or a second function claiming a slot a first already filled stays reported rather than guessed at. +function applyHeaderFooterGroup( + state: ReaderState, + token: Extract, + container: WpdDocumentContainer, + sink: WpdDiagnosticSink, +): void { + const claim = readFurnitureClaim(token.subgroup, token.nonDeletable); + if (claim === "none") { + return; + } + if (claim === "watermark") { + reportOnce( + state, + sink, + WpdDiagnosticCodes.HeaderFooterDropped, + "This document declares a watermark, which is neither a header nor a footer and owns no parity -- the shared page-furniture vocabulary has no slot for one.", + ); + return; + } + const slotKey = `${claim.kind}:${claim.slot}`; + if (state.furnitureFilled.has(slotKey)) { + reportOnce( + state, + sink, + WpdDiagnosticCodes.HeaderFooterDropped, + `This document declares a second ${claim.kind} for the ${claim.slot} slot -- WordPerfect's own A/B two-slot-per-kind mechanism, which the shared one-flow-per-slot page-furniture vocabulary does not carry; the first ${claim.kind} to claim the slot is the one lifted.`, + ); + return; + } + const blocks = furnitureBodyBlocks(state, token, container, sink); + if (blocks === undefined) { + return; + } + state.furnitureFilled.add(slotKey); + const furniture = claim.kind === "header" ? state.headers : state.footers; + furniture[claim.slot] = blocks; +} + +// A D6 function's own body: the General WP Text packet its first prefix ID names, folded to blocks through the identical machinery the main stream uses. Reports and answers undefined when the packet cannot be resolved or read -- the honest-or-nothing contract every other body resolution here holds. +function furnitureBodyBlocks( + state: ReaderState, + token: Extract, + container: WpdDocumentContainer, + sink: WpdDiagnosticSink, +): ContentBlock[] | undefined { + const prefixId = token.prefixIds[0]; + const packet = + prefixId === undefined + ? undefined + : packetByPrefixId(container.packets, prefixId); + if (packet?.packetType !== PACKET_TYPE_GENERAL_WP_TEXT) { + reportOnce( + state, + sink, + WpdDiagnosticCodes.HeaderFooterDropped, + "This document declares a header or footer whose body packet this reader could not resolve; it was not lifted.", + ); + return undefined; + } + const textBlocks = readGeneralWpTextBlocks(packet.bytes); + if (textBlocks === undefined) { + reportOnce( + state, + sink, + WpdDiagnosticCodes.HeaderFooterDropped, + "This document declares a header or footer whose body packet this reader could not read; it was not lifted.", + ); + return undefined; + } + const nested = tokeniseDocumentArea(textBlocks, 0, textBlocks.length); + return foldTokens(nested, container, sink).blocks; +} + +// The note subfunctions, per WPFF "D7 Footnote/Endnote Functions": even-numbered codes are the On functions (0 Footnote On, 2 Endnote On, each naming its body packet through its first prefix ID), odd-numbered the Off (1 Footnote Off, 3 Endnote Off). Each On's body is encased between its own On and Off. +// https://github.com/OneWingedShark/WordPerfect/blob/master/doc/SDK_Help/FileFormats/WPFF_D7-FootnoteEndNote.htm +const FOOTNOTE_ON = 0x00; +const FOOTNOTE_OFF = 0x01; +const ENDNOTE_ON = 0x02; +const ENDNOTE_OFF = 0x03; + +// Lifts a D7 note pair as the shared model's note-anchor construct plus a definitions-ready body: the On opens a run-scoped anchor extent at the reference site (the encased text is the reference marker -- typically the note's own rendered number), the Off closes it and resolves the body from the packet the On's first prefix ID named. The anchor descriptor's definition key is the position it will hold in the tree form's definitions table (note-1, note-2, ... in document order), which readWpd splices in; the flat readWpdContent emits the anchor and reports the body, whose real home is exactly that table. +function applyNoteGroup( + state: ReaderState, + token: Extract, + container: WpdDocumentContainer, + sink: WpdDiagnosticSink, +): void { + if (token.subgroup === FOOTNOTE_ON || token.subgroup === ENDNOTE_ON) { + flushRun(state); + const prefixId = token.prefixIds[0]; + if (prefixId === undefined) { + reportOnce( + state, + sink, + WpdDiagnosticCodes.NoteDropped, + "This document contains a footnote or endnote whose body packet this reader could not resolve; only its reference text survived.", + ); + return; + } + state.openNote = { + anchorType: token.subgroup === FOOTNOTE_ON ? "footnote" : "endnote", + startRun: state.runs.length, + prefixId, + }; + return; + } + const closing = + token.subgroup === FOOTNOTE_OFF + ? "footnote" + : token.subgroup === ENDNOTE_OFF + ? "endnote" + : undefined; + if (closing === undefined) { + return; + } + const open = state.openNote; + state.openNote = undefined; + if (open?.anchorType !== closing) { + // An Off with no matching On -- a stream whose note pairs do not pair, or an On this reader already abandoned at a paragraph boundary (flushParagraph). Nothing to anchor. + return; + } + flushRun(state); + const endRun = state.runs.length; + const marker = + state.runs + .slice(open.startRun, endRun) + .map((run) => run.text) + .join("") || String(state.notes.length + 1); + const definition = `note-${state.notes.length + 1}`; + state.pendingConstructs.push({ + descriptor: { + kind: "anchor", + anchorType: open.anchorType, + name: marker, + definition, + }, + startRun: open.startRun, + endRun, + }); + const packet = packetByPrefixId(container.packets, open.prefixId); + const textBlocks = + packet?.packetType === PACKET_TYPE_GENERAL_WP_TEXT + ? readGeneralWpTextBlocks(packet.bytes) + : undefined; + const nested = + textBlocks === undefined + ? undefined + : tokeniseDocumentArea(textBlocks, 0, textBlocks.length); + const blocks = + nested === undefined + ? undefined + : foldTokens(nested, container, sink).blocks; + if (blocks === undefined) { + reportOnce( + state, + sink, + WpdDiagnosticCodes.NoteDropped, + "This document contains a footnote or endnote whose body packet this reader could not read; its reference anchor survives and its body does not.", + ); + return; + } + state.notes.push({ anchorType: open.anchorType, marker, blocks }); +} + function applyMergeGroup( state: ReaderState, token: Extract, @@ -1211,20 +1405,10 @@ function applyVariableFunction( ); return; case HEADER_FOOTER_GROUP: - reportOnce( - state, - sink, - WpdDiagnosticCodes.HeaderFooterDropped, - "This document declares a header, footer, or watermark, which the flat content model has no page-furniture position for.", - ); + applyHeaderFooterGroup(state, token, container, sink); return; case FOOTNOTE_ENDNOTE_GROUP: - reportOnce( - state, - sink, - WpdDiagnosticCodes.NoteDropped, - "This document contains a footnote or endnote; its text lives in a prefix packet the flat content model has nowhere to put.", - ); + applyNoteGroup(state, token, container, sink); return; case MERGE_GROUP: applyMergeGroup(state, token, sink); @@ -1289,6 +1473,9 @@ function applyFixedFunction( interface FoldResult { readonly blocks: ContentBlock[]; readonly page: PageState; + readonly headers: ContentPageFurniture; + readonly footers: ContentPageFurniture; + readonly notes: readonly WpdNoteDefinition[]; } // One token's own effect on the reader state, shared by the main document-area walk and any sub-stream folded through the identical function-code vocabulary -- currently a style packet's own "beginning style text" block (applyStylePacketBegin above), which carries the same font/attribute/colour-change functions the main stream does and means them identically. @@ -1359,6 +1546,11 @@ function foldTokens( reported: new Set(), pendingConstructs: [], openMergeFieldStartRun: undefined, + headers: {}, + footers: {}, + furnitureFilled: new Set(), + openNote: undefined, + notes: [], }; for (const token of tokens) { @@ -1375,7 +1567,13 @@ function foldTokens( closeRow(state.table); closeTable(state); } - return { blocks: state.blocks, page: state.page }; + return { + blocks: state.blocks, + page: state.page, + headers: state.headers, + footers: state.footers, + notes: state.notes, + }; } // The document's own metadata, from the Extended Document Summary prefix packet. A document that carries no summary packet gets an empty envelope -- the honest answer, rather than fields invented from the file's structure. @@ -1399,7 +1597,18 @@ export function readWpdContent( container.documentAreaOffset, container.documentAreaEnd, ); - const { blocks, page } = foldTokens(tokens, container, sink); + const { blocks, page, headers, footers, notes } = foldTokens( + tokens, + container, + sink, + ); + // The note bodies' real home is the tree form's definitions table, which the flat form cannot reach -- readWpd carries them. Each still-borne body says so here rather than passing in silence; the anchor itself is in the blocks. + for (const note of notes) { + sink({ + code: WpdDiagnosticCodes.NoteDropped, + message: `This document contains a ${note.anchorType} whose body the flat ContentDocument has no home for; its reference anchor survives and readWpd lifts the body into the tree form's definitions table.`, + }); + } return { kind: "wordprocessing", metadata: readMetadata(container), @@ -1415,16 +1624,65 @@ export function readWpdContent( bottomPt: page.bottomPt ?? DEFAULT_MARGIN_PT, leftPt: page.leftPt ?? DEFAULT_MARGIN_PT, }, + ...(Object.keys(headers).length > 0 ? { headers } : {}), + ...(Object.keys(footers).length > 0 ? { footers } : {}), blocks, }, ], }; } -// The same read, one level up: the tree-form DocumentTree every other codec in the family also offers, assembled from the flat document by document-schema.js's own transform. +// The same read, one level up: the tree-form DocumentTree every other codec in the family also offers, assembled from the flat document by document-schema.js's own transform -- plus the one fact only the tree can hold: every note body the flat form cannot carry rides as a definitions-table entry ({ kind: 'footnote' | 'endnote', marker, blocks } -- the tenant vocabulary markdown-codec's own footnote definitions established), keyed by the definition id each anchor descriptor in the flat content already names (note-1, note-2, ... in document order). The identical splice markdown-codec's own tree reader performs for its link table. export function readWpd( bytes: Uint8Array, options: ReadWpdOptions = {}, ): DocumentTree { - return assembleTree(readWpdContent(bytes, options)); + // The flat read's per-note NoteDropped diagnostics would be lies at this level -- the tree DOES carry the bodies -- so the internal read runs with a silent sink and the bodies are collected straight from the fold, exactly the shape readWpdContent discards. + const container = openWpdDocument(bytes, { password: options.password }); + const tokens = tokeniseDocumentArea( + container.bytes, + container.documentAreaOffset, + container.documentAreaEnd, + ); + const sink = options.sink ?? NOOP_WPD_DIAGNOSTIC_SINK; + const { notes, ...flatRest } = foldTokens(tokens, container, sink); + const document: ContentDocument = { + kind: "wordprocessing", + metadata: readMetadata(container), + sections: [ + { + pageSize: { + widthPt: flatRest.page.widthPt ?? DEFAULT_PAGE_WIDTH_PT, + heightPt: flatRest.page.heightPt ?? DEFAULT_PAGE_HEIGHT_PT, + }, + margins: { + topPt: flatRest.page.topPt ?? DEFAULT_MARGIN_PT, + rightPt: flatRest.page.rightPt ?? DEFAULT_MARGIN_PT, + bottomPt: flatRest.page.bottomPt ?? DEFAULT_MARGIN_PT, + leftPt: flatRest.page.leftPt ?? DEFAULT_MARGIN_PT, + }, + ...(Object.keys(flatRest.headers).length > 0 + ? { headers: flatRest.headers } + : {}), + ...(Object.keys(flatRest.footers).length > 0 + ? { footers: flatRest.footers } + : {}), + blocks: flatRest.blocks, + }, + ], + }; + const assembled = assembleTree(document); + if (notes.length === 0) { + return assembled; + } + const definitions = Object.fromEntries( + notes.map((note, index) => [ + `note-${index + 1}`, + { kind: note.anchorType, marker: note.marker, blocks: note.blocks }, + ]), + ); + return { + ...assembled, + definitions: { ...assembled.definitions, ...definitions }, + }; } diff --git a/packages/wpd-codec/src/stream/furniture.ts b/packages/wpd-codec/src/stream/furniture.ts new file mode 100644 index 000000000..a6084967c --- /dev/null +++ b/packages/wpd-codec/src/stream/furniture.ts @@ -0,0 +1,51 @@ +// -- Page furniture, per WPFF "D6 Header/Footer Functions" -- +// +// The D6 group states a document's headers, footers, and watermarks. Each function names its body as the prefix ID of a General WP Text packet (type 0x08) -- the identical packet type a box's own text content rides -- and its non-deletable data is exactly two bytes, of which the first is the occurrence byte: bit 0 "does occur on odd pages", bit 1 "does occur on even pages" (bit 2 states a vertical-text watermark and bits 3/4 watermark display space, none of which the shared furniture model asks about). +// +// https://github.com/OneWingedShark/WordPerfect/blob/master/doc/SDK_Help/FileFormats/WPFF_D6-HeaderFooter.htm + +export const HEADER_FOOTER_GROUP = 0xd6; + +// The SDK's own subfunction table: "0 Header A, 1 Header B, 2 Footer A, 3 Footer B, 4 Watermark A, 5 Watermark B". The A/B pair is WordPerfect's own two-slot-per-kind mechanism -- a page shows Header A until a Header B supersedes it -- which the shared furniture vocabulary does not carry: a slot holds one flow, and a second function claiming a slot a first already fills is reported rather than silently overwritten. +export const HEADER_A = 0x00; +export const HEADER_B = 0x01; +export const FOOTER_A = 0x02; +export const FOOTER_B = 0x03; +export const WATERMARK_A = 0x04; +export const WATERMARK_B = 0x05; + +// The occurrence byte's own two bits the furniture model asks about. +const OCCURS_ON_ODD = 1 << 0; +const OCCURS_ON_EVEN = 1 << 1; + +// What one D6 function claims: which furniture kind it is, and which of the shared vocabulary's three slots (default/even/first, WordprocessingML's own headerReference/@w:type values) its occurrence bits narrow onto. Odd-only is the default slot (the ordinary single-header document states exactly that); even-only is the even slot; both parities is the default slot too, since a flow occurring on every page IS the default. A function claiming neither parity is suppressed in its own file and claims nothing here. undefined answers watermark -- the one kind the vocabulary has no slot for (a watermark is neither header nor footer and owns no parity). +export interface WpdFurnitureClaim { + readonly kind: "header" | "footer"; + readonly slot: "default" | "even"; +} + +export function readFurnitureClaim( + subgroup: number, + nonDeletable: Uint8Array, +): WpdFurnitureClaim | "watermark" | "none" { + if (subgroup === WATERMARK_A || subgroup === WATERMARK_B) { + return "watermark"; + } + const kind = + subgroup === HEADER_A || subgroup === HEADER_B + ? ("header" as const) + : subgroup === FOOTER_A || subgroup === FOOTER_B + ? ("footer" as const) + : undefined; + if (kind === undefined) { + return "none"; + } + const occurrence = nonDeletable[0] ?? 0; + const odd = (occurrence & OCCURS_ON_ODD) !== 0; + const even = (occurrence & OCCURS_ON_EVEN) !== 0; + if (!odd && !even) { + return "none"; + } + // Even-only is the even slot; odd-only and both-parities are the default slot, per the narrowing above. + return { kind, slot: even && !odd ? "even" : "default" }; +}