Skip to content

Commit 6f31cea

Browse files
committed
Merge branch 'fix/parsers-office-structure' into eval/kb-parser-quality
# Conflicts: # apps/sim/lib/file-parsers/sniff.ts
2 parents df65598 + 2f0f498 commit 6f31cea

14 files changed

Lines changed: 594 additions & 139 deletions

‎apps/sim/lib/file-parsers/docx-parser.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import {
1212
isHtmlComplexityError,
1313
} from '@/lib/file-parsers/html-parser'
1414
import { parseOfficeText } from '@/lib/file-parsers/officeparser-module'
15+
import { isEncryptedOoxmlContainer } from '@/lib/file-parsers/ooxml-encryption'
1516
import type { FileParseOptions, FileParseResult, FileParser } from '@/lib/file-parsers/types'
1617
import { sanitizeTextForUTF8 } from '@/lib/file-parsers/utils'
1718
import { assertOoxmlArchiveWithinLimits } from '@/lib/file-parsers/zip-guard'
@@ -106,6 +107,14 @@ export class DocxParser implements FileParser {
106107
extractionErrors.push(officeError)
107108
}
108109

110+
if (isEncryptedOoxmlContainer(buffer)) {
111+
throw new FileParserError(
112+
'encrypted_file',
113+
'This document is encrypted or password-protected',
114+
extractionErrors.length > 0 ? new AggregateError(extractionErrors) : undefined
115+
)
116+
}
117+
109118
const isZipFile = buffer.length >= 2 && buffer[0] === 0x50 && buffer[1] === 0x4b
110119
if (!isZipFile) {
111120
const textContent = buffer.toString('utf8').trim()

‎apps/sim/lib/file-parsers/html-parser.test.ts‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,46 @@ describe('HtmlParser', () => {
141141
expect(result.content).not.toContain('fourth nested')
142142
})
143143

144+
it('renders a nested table inside its cell exactly once', async () => {
145+
const buffer = Buffer.from(
146+
`<body><table><tbody><tr><td>Outer A</td><td><p>Intro</p>` +
147+
`<table><tr><td>Inner 1</td><td>Inner 2</td></tr><tr><td>Inner 3</td></tr></table>` +
148+
`</td></tr><tr><th>Outer B</th><td>Plain</td></tr></tbody></table></body>`
149+
)
150+
151+
const result = await parser.parseBuffer(buffer)
152+
153+
expect(result.content).toContain('| Outer A | Intro Inner 1 / Inner 2 / Inner 3 |')
154+
expect(result.content).toContain('| Outer B | Plain |')
155+
for (const cell of ['Outer A', 'Inner 1', 'Inner 2', 'Inner 3', 'Outer B', 'Plain']) {
156+
expect(result.content.split(cell)).toHaveLength(2)
157+
}
158+
expect(result.content.match(/\[Table\]/g)).toHaveLength(1)
159+
expect(result.metadata?.tableCount).toBe(2)
160+
})
161+
162+
it('separates block elements inside a list item', async () => {
163+
const buffer = Buffer.from(
164+
`<body><ul><li><div><p>Versions</p><p>Release Information</p></div></li></ul></body>`
165+
)
166+
167+
const result = await parser.parseBuffer(buffer)
168+
169+
expect(result.content).toContain('• Versions Release Information')
170+
})
171+
172+
it('drops endnote return links but keeps the endnote text', async () => {
173+
const buffer = Buffer.from(
174+
`<body><p>Body<sup><a href="#endnote-1" id="endnote-ref-1">[1]</a></sup></p>` +
175+
`<ol><li id="endnote-1"><p>End text <a href="#endnote-ref-1">↑</a></p></li></ol></body>`
176+
)
177+
178+
const result = await parser.parseBuffer(buffer)
179+
180+
expect(result.content).toContain('1. End text')
181+
expect(result.content).not.toContain('↑')
182+
})
183+
144184
it('drops footnote return links but keeps the footnote text', async () => {
145185
const buffer = Buffer.from(
146186
`<body><p>Body<sup><a href="#footnote-1" id="footnote-ref-1">[1]</a></sup></p>` +

‎apps/sim/lib/file-parsers/html-parser.ts‎

Lines changed: 73 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -108,8 +108,8 @@ const NON_CONTENT_SELECTOR = 'script, style, noscript, meta, link, iframe, objec
108108
/** Block elements inside a table cell; `.text()` would otherwise glue their words together. */
109109
const CELL_BLOCK_SELECTOR = 'p, div, li, br, h1, h2, h3, h4, h5, h6, tr'
110110

111-
/** mammoth renders a footnote's return link as `<a href="#footnote-ref-N">↑</a>`. */
112-
const FOOTNOTE_BACKLINK_SELECTOR = 'a[href^="#footnote-ref"]'
111+
/** mammoth renders a footnote's or endnote's return link as `<a href="#footnote-ref-N">↑</a>`. */
112+
const FOOTNOTE_BACKLINK_SELECTOR = 'a[href^="#footnote-ref"], a[href^="#endnote-ref"]'
113113

114114
/**
115115
* Strips the non-content markup and HTML comments from a loaded document so the
@@ -318,7 +318,9 @@ function processListItem(
318318
return
319319
}
320320
}
321-
const text = $(child).text().replace(/\s+/g, ' ').trim()
321+
const $child = $(child)
322+
$child.find(CELL_BLOCK_SELECTOR).after(' ')
323+
const text = $child.text().replace(/\s+/g, ' ').trim()
322324
if (text) ownText.push(text)
323325
})
324326

@@ -335,8 +337,69 @@ function processListItem(
335337
}
336338
}
337339

340+
/** A table's own rows: direct `<tr>` children and those under its section elements. */
341+
function directRows(
342+
$: cheerio.CheerioAPI,
343+
table: cheerio.Cheerio<AnyNode>
344+
): cheerio.Cheerio<AnyNode> {
345+
return table.children('thead, tbody, tfoot').children('tr').add(table.children('tr'))
346+
}
347+
348+
/** Nested tables whose nearest enclosing table is the cell's own. */
349+
function topLevelNestedTables(
350+
$: cheerio.CheerioAPI,
351+
cell: cheerio.Cheerio<AnyNode>
352+
): cheerio.Cheerio<AnyNode> {
353+
const own = cell.closest('table').get(0)
354+
return cell.find('table').filter((_, nested) => $(nested).parents('table').get(0) === own)
355+
}
356+
357+
/**
358+
* The cells of a table in reading order, each rendered with {@link cellText},
359+
* for a table nested inside another table's cell.
360+
*/
361+
function flattenedTableCells($: cheerio.CheerioAPI, table: cheerio.Cheerio<AnyNode>): string[] {
362+
const cells: string[] = []
363+
directRows($, table).each((_, row) => {
364+
$(row)
365+
.children('td, th')
366+
.each((_, cell) => {
367+
const text = cellText($, $(cell))
368+
if (text) cells.push(text)
369+
})
370+
})
371+
return cells
372+
}
373+
374+
/**
375+
* One cell's text on a single line. Block elements inside the cell get a space
376+
* so adjacent paragraphs do not glue together — this mutates the live DOM, and
377+
* runs before `extractHeadings`/`extractLinks`, so heading or link text inside a
378+
* cell gains those spaces too. A nested table is rendered on a clone of the cell
379+
* as its cells joined with ` / `, without `[Table]` markers or pipes, so the
380+
* outer row stays one line and the inner text appears exactly once.
381+
*/
382+
function cellText($: cheerio.CheerioAPI, cell: cheerio.Cheerio<AnyNode>): string {
383+
const nested = topLevelNestedTables($, cell)
384+
if (nested.length === 0) {
385+
cell.find(CELL_BLOCK_SELECTOR).after(' ')
386+
return cell.text().replace(/\s+/g, ' ').trim()
387+
}
388+
389+
const clone = cell.clone()
390+
topLevelNestedTables($, clone).each((_, table) => {
391+
const $table = $(table)
392+
$table.replaceWith(` ${flattenedTableCells($, $table).join(' / ')} `)
393+
})
394+
clone.find(CELL_BLOCK_SELECTOR).after(' ')
395+
return clone.text().replace(/\s+/g, ' ').trim()
396+
}
397+
338398
/**
339-
* Process table elements to extract structured data
399+
* Renders a table as `[Table]`, one `| a | b |` line per row, `[/Table]`. Only
400+
* the table's own rows and each row's own cells are visited, so a nested table
401+
* contributes to its containing cell (see {@link cellText}) and is never
402+
* emitted a second time as rows of its own.
340403
*/
341404
function processTable(
342405
$: cheerio.CheerioAPI,
@@ -345,16 +408,14 @@ function processTable(
345408
): void {
346409
contentParts.push('\n[Table]')
347410

348-
table.find('tr').each((_, row) => {
349-
const $row = $(row)
411+
directRows($, table).each((_, row) => {
350412
const cells: string[] = []
351413

352-
$row.find('td, th').each((_, cell) => {
353-
const $cell = $(cell)
354-
$cell.find(CELL_BLOCK_SELECTOR).after(' ')
355-
const cellText = $cell.text().replace(/\s+/g, ' ').trim()
356-
cells.push(cellText || '')
357-
})
414+
$(row)
415+
.children('td, th')
416+
.each((_, cell) => {
417+
cells.push(cellText($, $(cell)))
418+
})
358419

359420
if (cells.length > 0) {
360421
contentParts.push(`| ${cells.join(' | ')} |`)
@@ -397,7 +458,6 @@ export class HtmlParser implements FileParser {
397458
const htmlContent = decoded.text
398459
const $ = cheerio.load(htmlContent)
399460

400-
// Extract meta information before removing tags
401461
const title = $('title').text().trim()
402462
const metaDescription = $('meta[name="description"]').attr('content') || ''
403463

‎apps/sim/lib/file-parsers/odf-text.test.ts‎

Lines changed: 58 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
/**
22
* @vitest-environment node
33
*/
4-
import JSZip from 'jszip'
5-
import { describe, expect, it } from 'vitest'
4+
import JSZip, { type JSZipObject } from 'jszip'
5+
import { afterEach, describe, expect, it, vi } from 'vitest'
6+
import type { FileParserError } from '@/lib/file-parsers/errors'
67
import { extractOpenDocumentText } from '@/lib/file-parsers/odf-text'
8+
import { MAX_OFFICE_XML_PART_BYTES } from '@/lib/file-parsers/office-text'
79

810
const NS =
911
'xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0" xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0" xmlns:table="urn:oasis:names:tc:opendocument:xmlns:table:1.0" xmlns:draw="urn:oasis:names:tc:opendocument:xmlns:drawing:1.0" xmlns:presentation="urn:oasis:names:tc:opendocument:xmlns:presentation:1.0" xmlns:dc="http://purl.org/dc/elements/1.1/"'
@@ -22,6 +24,10 @@ async function buildOdf(bodyXml: string, extraParts: Record<string, string> = {}
2224
const text = (body: string) => buildOdf(`<office:text>${body}</office:text>`)
2325

2426
describe('extractOpenDocumentText', () => {
27+
afterEach(() => {
28+
vi.restoreAllMocks()
29+
})
30+
2531
it('drops annotations so the surrounding sentence stays intact', async () => {
2632
const buffer = await text(
2733
`<text:p>Aaa <office:annotation><dc:creator>M</dc:creator><text:sender-initials>M</text:sender-initials><text:p>First comment.</text:p></office:annotation>comment ccc.<office:annotation-end office:name="c1"/></text:p>`
@@ -85,6 +91,56 @@ describe('extractOpenDocumentText', () => {
8591
)
8692
})
8793

94+
it('flattens a nested table into its cell without markers', async () => {
95+
const cell = (inner: string) => `<table:table-cell>${inner}</table:table-cell>`
96+
const p = (t: string) => `<text:p>${t}</text:p>`
97+
const inner = `<table:table><table:table-row>${cell(p('In 1'))}${cell(p('In 2'))}</table:table-row><table:table-row>${cell(p('In 3'))}</table:table-row></table:table>`
98+
const buffer = await text(
99+
`<table:table><table:table-row>${cell(p('Out A'))}${cell(p('Intro') + inner)}</table:table-row></table:table>`
100+
)
101+
102+
const result = await extractOpenDocumentText(buffer)
103+
104+
expect(result).toBe('[Table]\n| Out A | Intro In 1 / In 2 / In 3 |\n[/Table]')
105+
})
106+
107+
it('rejects an archive without content.xml as invalid_format', async () => {
108+
const zip = new JSZip()
109+
zip.file('mimetype', 'application/vnd.oasis.opendocument.text', { compression: 'STORE' })
110+
zip.file('junk.txt', 'not a document')
111+
const buffer = (await zip.generateAsync({ type: 'nodebuffer' })) as Buffer
112+
113+
await expect(extractOpenDocumentText(buffer)).rejects.toMatchObject<FileParserError>({
114+
code: 'invalid_format',
115+
})
116+
})
117+
118+
it('returns an empty string for a present but textless body', async () => {
119+
expect(await extractOpenDocumentText(await text('<text:p/>'))).toBe('')
120+
})
121+
122+
it('emits an image frame as its alternative text', async () => {
123+
const buffer = await text(
124+
`<text:p><draw:frame draw:name="Image1"><draw:image xlink:href="Pictures/a.png" xmlns:xlink="http://www.w3.org/1999/xlink"/><svg:title xmlns:svg="urn:oasis:names:tc:opendocument:xmlns:svg-compatible:1.0">Org chart</svg:title></draw:frame></text:p>`
125+
)
126+
127+
expect(await extractOpenDocumentText(buffer)).toBe('[Image: Org chart]')
128+
})
129+
130+
it('rejects a content part above the per-part size cap before parsing it', async () => {
131+
const buffer = await text('<text:p>Small</text:p>')
132+
const zip = await JSZip.loadAsync(buffer)
133+
const entry = zip.file('content.xml') as JSZipObject & {
134+
_data: { uncompressedSize: number }
135+
}
136+
entry._data.uncompressedSize = MAX_OFFICE_XML_PART_BYTES + 1
137+
vi.spyOn(JSZip, 'loadAsync').mockResolvedValueOnce(zip)
138+
139+
await expect(extractOpenDocumentText(buffer)).rejects.toMatchObject<FileParserError>({
140+
code: 'complexity_limit',
141+
})
142+
})
143+
88144
it('includes embedded object content parts after the main document', async () => {
89145
const buffer = await buildOdf(`<office:text><text:p>Main</text:p></office:text>`, {
90146
'Object 1/content.xml': `<?xml version="1.0"?><office:document-content ${NS}><office:body><office:text><text:p>Embedded</text:p></office:text></office:body></office:document-content>`,

0 commit comments

Comments
 (0)