Skip to content

Commit feb7617

Browse files
waleedlatif1claude
andcommitted
fix(parsers): keep table headers, bound PDF assembly, disable heading markers
Furniture suppression treated any band text repeating across pages as a running header, which deleted multi-page table headers (IRS tax tables, NIST recommendation tables, EIC tables, DFAST captions). A band group is now exempt when it runs into the body at line pitch, when its key also occurs in body positions, or when dropping it would orphan a hyphenated word; folio candidates get the same flow test so edge table cells survive, and roman numerals must parse and fit the page count. joinLines accumulated the page in one string and ran anchored regexes over it per line, which was quadratic (40k lines: 72 s, now 7 ms); the hyphen and compound scans are bounded to the line tail, assembly yields to the event loop and honours the abort signal, and the line count and word set are capped. Geometry separators no longer count against the character budget, and preview output that overflows after decoration sets the truncated flag. Heading markers are off by default: on documents dominated by table or footnote text the estimated body height turned prose into headings that the chunker then split per line. The estimator now weighs prose-like lines only and skips runs of same-height lines for when it is enabled. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
1 parent e0b71c4 commit feb7617

7 files changed

Lines changed: 710 additions & 123 deletions

File tree

‎apps/sim/lib/file-parsers/pdf-furniture.test.ts‎

Lines changed: 122 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -12,19 +12,22 @@ import {
1212
import type { PdfLine } from '@/lib/file-parsers/pdf-lines'
1313

1414
const PAGE_HEIGHT = 792
15+
const PITCH = 12
1516

16-
function line(text: string, y: number): PdfLine {
17-
return { text, y, height: 11 }
17+
function line(text: string, y: number, height = 11): PdfLine {
18+
return { text, y, height }
1819
}
1920

20-
/** A page with a top header, a bottom footer, and two body lines. */
21+
/** Body lines at a regular pitch running down from `top`. */
22+
function body(count: number, top: number, label = 'Body'): PdfLine[] {
23+
return Array.from({ length: count }, (_, i) => line(`${label} line ${i + 1}`, top - i * PITCH))
24+
}
25+
26+
/** A page with a top header, a bottom footer, and a body separated from both by a margin gap. */
2127
function page(index: number, options: { header?: string; footer?: string } = {}): PdfPageLines {
2228
const lines: PdfLine[] = []
2329
if (options.header) lines.push(line(options.header, 729))
24-
lines.push(
25-
line(`Body line one of page ${index}`, 600),
26-
line(`Body line two of page ${index}`, 585)
27-
)
30+
lines.push(...body(40, 660, `Page ${index}`))
2831
if (options.footer) lines.push(line(options.footer, 55))
2932
return { lines, pageHeight: PAGE_HEIGHT }
3033
}
@@ -33,25 +36,6 @@ function texts(pages: PdfLine[][]): string[][] {
3336
return pages.map((lines) => lines.map((entry) => entry.text))
3437
}
3538

36-
describe('page numbers outside the band', () => {
37-
it('drops a folio that is the last line of a page even inside a wide margin', () => {
38-
const page = (n: number) => ({
39-
pageHeight: 842,
40-
lines: [
41-
{ text: 'Body text of the page.', y: 700, height: 10 },
42-
{ text: String(n), y: 189, height: 10 },
43-
],
44-
})
45-
const result = suppressFurniture([page(1), page(2), page(3)])
46-
47-
expect(result.map((lines) => lines.map((line) => line.text))).toEqual([
48-
['Body text of the page.'],
49-
['Body text of the page.'],
50-
['Body text of the page.'],
51-
])
52-
})
53-
})
54-
5539
describe('suppressFurniture', () => {
5640
it('drops a header repeated on enough pages but keeps its first occurrence', () => {
5741
const pages = [1, 2, 3, 4].map((i) => page(i, { header: 'ACME Corp — Internal Use Only' }))
@@ -61,7 +45,7 @@ describe('suppressFurniture', () => {
6145
expect(result[0]).toContain('ACME Corp — Internal Use Only')
6246
for (const remaining of result.slice(1)) {
6347
expect(remaining).not.toContain('ACME Corp — Internal Use Only')
64-
expect(remaining).toHaveLength(2)
48+
expect(remaining).toHaveLength(40)
6549
}
6650
})
6751

@@ -114,12 +98,30 @@ describe('suppressFurniture', () => {
11498

11599
it('drops page numbers in the bands regardless of repetition', () => {
116100
const pages: PdfPageLines[] = [
117-
{ lines: [line('Page 1 of 3', 55), line('Body', 600)], pageHeight: PAGE_HEIGHT },
118-
{ lines: [line('2', 55), line('Body', 600), line('iv', 729)], pageHeight: PAGE_HEIGHT },
119-
{ lines: [line('- 3 -', 55), line('Body', 600), line('925', 60)], pageHeight: PAGE_HEIGHT },
101+
{ lines: [line('Page 1 of 3', 55), ...body(3, 600)], pageHeight: PAGE_HEIGHT },
102+
{ lines: [line('2', 55), ...body(3, 600), line('ii', 729)], pageHeight: PAGE_HEIGHT },
103+
{ lines: [line('- 3 -', 55), ...body(3, 600), line('925', 60)], pageHeight: PAGE_HEIGHT },
120104
]
121105

122-
expect(texts(suppressFurniture(pages))).toEqual([['Body'], ['Body'], ['Body', '925']])
106+
const result = texts(suppressFurniture(pages))
107+
108+
expect(result[0]).toEqual(['Body line 1', 'Body line 2', 'Body line 3'])
109+
expect(result[1]).toEqual(['Body line 1', 'Body line 2', 'Body line 3'])
110+
expect(result[2]).toEqual(['Body line 1', 'Body line 2', 'Body line 3', '925'])
111+
})
112+
113+
it('drops a folio outside the band only when a margin gap separates it from the text', () => {
114+
const pages = [1, 2].map((i) => ({
115+
lines: [...body(20, 700), line(`${i}`, 100)],
116+
pageHeight: PAGE_HEIGHT,
117+
}))
118+
const tableCells = [1, 2].map(() => ({
119+
lines: [...body(50, 700), line('1', 700 - 50 * PITCH)],
120+
pageHeight: PAGE_HEIGHT,
121+
}))
122+
123+
for (const kept of texts(suppressFurniture(pages))) expect(kept).toHaveLength(20)
124+
for (const kept of texts(suppressFurniture(tableCells))) expect(kept).toContain('1')
123125
})
124126

125127
it('ignores band text longer than the furniture cap, such as a repeated table header', () => {
@@ -135,7 +137,7 @@ describe('suppressFurniture', () => {
135137
it('merges same-baseline fragments into one key before matching', () => {
136138
const pages = [1, 2, 3, 4].map((i) => ({
137139
lines: [
138-
line('Body', 600),
140+
...body(3, 600),
139141
line(`${i}`, 31.3),
140142
line('Chapter 1', 31.3),
141143
line('Publication 17 (2025)', 32.5),
@@ -145,8 +147,83 @@ describe('suppressFurniture', () => {
145147

146148
const result = texts(suppressFurniture(pages))
147149

148-
expect(result[0]).toEqual(['Body', '1', 'Chapter 1', 'Publication 17 (2025)'])
149-
expect(result[3]).toEqual(['Body'])
150+
expect(result[0].slice(3)).toEqual(['1', 'Chapter 1', 'Publication 17 (2025)'])
151+
expect(result[3]).toHaveLength(3)
152+
})
153+
154+
it('keeps a table header that repeats at the top of every page while dropping the running header above it', () => {
155+
const pages = Array.from({ length: 5 }, () => {
156+
const rows = Array.from({ length: 50 }, (_, i) =>
157+
line(`${1000 + i} ${2000 + i} ${3000 + i}`, 740 - (i + 1) * 8, 7.5)
158+
)
159+
return {
160+
lines: [
161+
line('2025 Tax Table — Continued', 772, 10),
162+
line('Single Married filing jointly Head of household', 740, 7.5),
163+
...rows,
164+
line('Need more information? Visit IRS.gov.', 31, 10),
165+
],
166+
pageHeight: PAGE_HEIGHT,
167+
}
168+
})
169+
170+
const result = texts(suppressFurniture(pages))
171+
172+
for (const kept of result) {
173+
expect(kept).toContain('Single Married filing jointly Head of household')
174+
}
175+
expect(result.flat().filter((text) => text === '2025 Tax Table — Continued')).toHaveLength(1)
176+
expect(
177+
result.flat().filter((text) => text === 'Need more information? Visit IRS.gov.')
178+
).toHaveLength(1)
179+
})
180+
181+
it('keeps a footnote that sits directly under the body while dropping the footer below it', () => {
182+
const pages = Array.from({ length: 4 }, () => ({
183+
lines: [
184+
...body(52, 700),
185+
line(
186+
'* This column must also be used by a qualifying surviving spouse.',
187+
700 - 52 * PITCH,
188+
8
189+
),
190+
line('Visit IRS.gov.', 30, 10),
191+
],
192+
pageHeight: PAGE_HEIGHT,
193+
}))
194+
195+
const result = texts(suppressFurniture(pages))
196+
197+
for (const kept of result) {
198+
expect(kept).toContain('* This column must also be used by a qualifying surviving spouse.')
199+
}
200+
expect(result.flat().filter((text) => text === 'Visit IRS.gov.')).toHaveLength(1)
201+
})
202+
203+
it('keeps a band line whose text also occurs in body positions', () => {
204+
const pages = [1, 2, 3, 4].map((i) => ({
205+
lines: [
206+
line('Quarter Revenue Margin', 760),
207+
...body(40, 660),
208+
...(i === 1 ? [line('Quarter Revenue Margin', 400)] : []),
209+
],
210+
pageHeight: PAGE_HEIGHT,
211+
}))
212+
213+
for (const kept of texts(suppressFurniture(pages))) {
214+
expect(kept).toContain('Quarter Revenue Margin')
215+
}
216+
})
217+
218+
it('never drops a band line that would leave a hyphenated word orphaned', () => {
219+
const pages = [1, 2, 3, 4].map(() => ({
220+
lines: [line('Married filing sepa-', 760), ...body(40, 660)],
221+
pageHeight: PAGE_HEIGHT,
222+
}))
223+
224+
for (const kept of texts(suppressFurniture(pages))) {
225+
expect(kept).toContain('Married filing sepa-')
226+
}
150227
})
151228

152229
it('leaves body text alone even when it repeats', () => {
@@ -189,7 +266,8 @@ describe('isPageNumber', () => {
189266
expect(isPageNumber('page 3 of 10', 10)).toBe(true)
190267
expect(isPageNumber('3 / 10', 10)).toBe(true)
191268
expect(isPageNumber('7', 10)).toBe(true)
192-
expect(isPageNumber('xiv', 10)).toBe(true)
269+
expect(isPageNumber('xiv', 20)).toBe(true)
270+
expect(isPageNumber('CD', 500)).toBe(true)
193271
expect(isPageNumber('— 12 —', 20)).toBe(true)
194272
})
195273

@@ -199,6 +277,15 @@ describe('isPageNumber', () => {
199277
expect(isPageNumber('Chapter 1', 10)).toBe(false)
200278
expect(isPageNumber('civilian', 10)).toBe(false)
201279
})
280+
281+
it('rejects words that merely look like roman numerals', () => {
282+
for (const word of ['mix', 'mild', 'civil', 'vivid', 'mimic', 'dim']) {
283+
expect(isPageNumber(word, 1000)).toBe(false)
284+
}
285+
expect(isPageNumber('CD', 10)).toBe(false)
286+
expect(isPageNumber('xiv', 10)).toBe(false)
287+
expect(isPageNumber('ii', 1)).toBe(false)
288+
})
202289
})
203290

204291
describe('furnitureThreshold', () => {

0 commit comments

Comments
 (0)