Merge branch 'fix/parsers-input-hygiene' into eval/kb-parser-quality

Author: waleedlatif1

apps/docs/openapi-v2-files-audit.json

@@ -745,7 +745,7 @@
       "get": {
         "operationId": "readFileText",
         "summary": "Read File Text",
-        "description": "Extract text without changing the file. Use Unzip File to unpack archives or Download File for original bytes. Unsupported types return `400`, compiling documents return `409`, and oversized files return `413`. `degraded: true` indicates incomplete or synthesized text, including some legacy `.doc` and `.ppt` results; `truncated: true` indicates a parser limit.\n\nOAuth scope: `api:read`.",
+        "description": "Extract text without changing the file. Use Unzip File to unpack archives or Download File for original bytes. Unsupported types return `400`, compiling documents return `409`, and oversized files return `413`. `degraded: true` indicates incomplete or synthesized text, such as the legacy `.pptx` fallback; `truncated: true` indicates a parser limit.\n\nOAuth scope: `api:read`.",
         "x-sim-operation": "files.read_content",
         "x-oauth-scope": "api:read",
         "tags": ["Files"],

apps/sim/connectors/bitbucket/bitbucket.test.ts

@@ -774,7 +774,7 @@ describe('bitbucket getDocument', () => {
     expect(doc?.skippedReason).toMatch(/Binary/)
   })
 
-  it('surfaces non-UTF-8 source as skipped instead of indexing replacement characters', async () => {
+  it('decodes non-UTF-8 source as Windows-1252 instead of skipping or indexing replacement characters', async () => {
     mockApi([
       [
         /\/src\/[a-f0-9]+\/latin1\.txt$/,
@@ -784,8 +784,9 @@ describe('bitbucket getDocument', () => {
 
     const doc = await bitbucketConnector.getDocument(ACCESS_TOKEN, CONFIG, 'file:latin1.txt', {})
 
-    expect(doc?.skippedReason).toMatch(/Non-UTF-8/)
-    expect(doc?.content).toBe('')
+    expect(doc?.skippedReason).toBeUndefined()
+    expect(doc?.content).toContain('café')
+    expect(doc?.content).not.toContain('\uFFFD')
   })
 
   it('returns null for a file the ref no longer carries', async () => {

apps/sim/connectors/bitbucket/bitbucket.ts

@@ -1,5 +1,6 @@
 import { createLogger } from '@sim/logger'
 import { getErrorMessage, toError } from '@sim/utils/errors'
+import { decodeTextBuffer } from '@/lib/file-parsers/utils'
 import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
 import { bitbucketConnectorMeta } from '@/connectors/bitbucket/meta'
 import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
@@ -81,7 +82,6 @@ const BINARY_SNIFF_BYTES = 8000
  */
 const MAX_TREE_DEPTH = 5
 const BINARY_SKIP_REASON = 'Binary file was not indexed'
-const NON_UTF8_SKIP_REASON = 'Non-UTF-8 file was not indexed'
 /**
  * Bitbucket answers a raw read of an LFS-managed file with a 301 to Atlassian's
  * media services platform. The connector deliberately surfaces the file as
@@ -1240,13 +1240,7 @@ export const bitbucketConnector: ConnectorConfig = {
         return markSkipped(stub, BINARY_SKIP_REASON)
       }
 
-      let text: string
-      try {
-        text = new TextDecoder('utf-8', { fatal: true }).decode(buffer)
-      } catch {
-        logger.info('Skipping non-UTF-8 Bitbucket file', { path })
-        return markSkipped(stub, NON_UTF8_SKIP_REASON)
-      }
+      const text = decodeTextBuffer(buffer).text
 
       const body = composeBody(stub.title, text)
       if (!body.trim()) return null

apps/sim/connectors/box/box.ts

@@ -133,7 +133,6 @@ const REPRESENTATION_EXTENSIONS = new Set([
   'odt',
   'otp',
   'pdf',
-  'ppt',
   'pptx',
   'rtf',
   'vi',

apps/sim/connectors/databricks/databricks.ts

@@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger'
 import { getErrorMessage, toError } from '@sim/utils/errors'
 import { truncate } from '@sim/utils/string'
 import { validateDatabricksWorkspaceHost } from '@/lib/core/security/input-validation'
+import { decodeTextBuffer } from '@/lib/file-parsers/utils'
 import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
 import {
   DATABRICKS_CONTENT_TYPES,
@@ -586,7 +587,7 @@ async function exportNotebook(
     return { skippedReason: sizeLimitSkipReason(CONNECTOR_MAX_FILE_BYTES) }
   }
 
-  return { content: decoded.toString('utf8') }
+  return { content: decodeTextBuffer(decoded).text }
 }
 
 /**

apps/sim/connectors/google-drive/google-drive.ts

@@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger'
 import { getErrorMessage, toError } from '@sim/utils/errors'
 import { isPlainRecord } from '@sim/utils/object'
 import { mapWithConcurrency } from '@/lib/core/utils/concurrency'
+import { decodeTextBuffer } from '@/lib/file-parsers/utils'
 import {
   type DrivePermission,
   driveFileAcl,
@@ -194,10 +195,10 @@ async function fetchFilePayload(accessToken: string, file: DriveFile): Promise<F
         },
       }
     }
-    return { content: bytes.toString('utf8'), mimeType: 'text/plain' }
+    return { content: decodeTextBuffer(bytes).text, mimeType: 'text/plain' }
   }
   if (file.mimeType === 'text/html') {
-    const html = (await downloadFile(accessToken, file.id)).toString('utf8')
+    const html = decodeTextBuffer(await downloadFile(accessToken, file.id)).text
     return { content: htmlToPlainText(html), mimeType: 'text/plain' }
   }
   const raw = rawFileType(file)
@@ -210,7 +211,7 @@ async function fetchFilePayload(accessToken: string, file: DriveFile): Promise<F
   }
   if (isSupportedTextFile(file.mimeType)) {
     return {
-      content: (await downloadFile(accessToken, file.id)).toString('utf8'),
+      content: decodeTextBuffer(await downloadFile(accessToken, file.id)).text,
       mimeType: 'text/plain',
     }
   }

apps/sim/connectors/s3/s3.ts

@@ -4,6 +4,7 @@ import { isLoopbackHostname } from '@sim/security/ssrf'
 import { getErrorMessage, toError } from '@sim/utils/errors'
 import { truncate } from '@sim/utils/string'
 import { isHosted } from '@/lib/core/config/env-flags'
+import { decodeTextBuffer } from '@/lib/file-parsers/utils'
 import { secureFetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server'
 import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils'
 import { s3ConnectorMeta } from '@/connectors/s3/meta'
@@ -658,7 +659,7 @@ export const s3Connector: ConnectorConfig = {
           sizeLimitSkipReason(MAX_FILE_SIZE)
         )
       }
-      const raw = body.toString('utf-8')
+      const raw = decodeTextBuffer(body).text
       const content = MARKUP_EXTENSIONS.has(getExtension(key)) ? htmlToPlainText(raw) : raw
       if (!content.trim()) return null
 

apps/sim/connectors/sftp/sftp.ts

@@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger'
 import { getErrorMessage, toError } from '@sim/utils/errors'
 import { type Attributes, type Client, type SFTPWrapper, utils as ssh2Utils } from 'ssh2'
 import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
+import { decodeTextBuffer } from '@/lib/file-parsers/utils'
 import {
   createSftpConnection,
   getFileType,
@@ -779,7 +780,7 @@ export const sftpConnector: ConnectorConfig = {
         return markSkipped(stub, 'File appears to be binary and was not indexed')
       }
 
-      const raw = buffer.toString('utf-8')
+      const raw = decodeTextBuffer(buffer).text
       const content = HTML_EXTENSIONS.has(getExtension(remotePath)) ? htmlToPlainText(raw) : raw
       if (!content.trim()) return null
 

apps/sim/lib/api/contracts/v2/openapi/files-audit.ts

@@ -393,7 +393,7 @@ const declaredRoutes = [
       operationId: 'readFileText',
       summary: 'Read File Text',
       description:
-        'Extract text without changing the file. Use Unzip File to unpack archives or Download File for original bytes. Unsupported types return `400`, compiling documents return `409`, and oversized files return `413`. `degraded: true` indicates incomplete or synthesized text, including some legacy `.doc` and `.ppt` results; `truncated: true` indicates a parser limit.',
+        'Extract text without changing the file. Use Unzip File to unpack archives or Download File for original bytes. Unsupported types return `400`, compiling documents return `409`, and oversized files return `413`. `degraded: true` indicates incomplete or synthesized text, such as the legacy `.pptx` fallback; `truncated: true` indicates a parser limit.',
       errors: [...RESOURCE_CONFLICT_ERRORS, 'PayloadTooLarge'],
       success: { description: 'The extracted text and its extraction-quality flags.' },
     }),

apps/sim/lib/file-parsers/doc-parser.test.ts

@@ -37,6 +37,7 @@ interface WordSections {
   footers?: string
   footnotes?: string
   endnotes?: string
+  textboxes?: string
 }
 
 /** The accessor surface of word-extractor's `Document`, with empty sections by default. */
@@ -48,7 +49,7 @@ function wordDocument(sections: WordSections) {
     getFootnotes: () => sections.footnotes ?? '',
     getEndnotes: () => sections.endnotes ?? '',
     getAnnotations: () => '',
-    getTextboxes: () => '',
+    getTextboxes: () => sections.textboxes ?? '',
   }
 }
 
@@ -169,6 +170,7 @@ describe('DocParser.parseBuffer', () => {
         footers: 'Page footer',
         footnotes: '',
         endnotes: 'An endnote',
+        textboxes: 'Pull quote in a text box',
       })
     )
 
@@ -177,7 +179,7 @@ describe('DocParser.parseBuffer', () => {
     expect(mockWordExtract).toHaveBeenCalledOnce()
     expect(mockParseOfficeText).not.toHaveBeenCalled()
     expect(result.content).toBe(
-      'Body paragraph “quoted”\n\nRunning header\n\nPage footer\n\nAn endnote'
+      'Body paragraph “quoted”\n\nRunning header\n\nPage footer\n\nAn endnote\n\nPull quote in a text box'
     )
     expect(result.metadata).toMatchObject({
       extractionMethod: 'word-extractor',
@@ -200,18 +202,26 @@ describe('DocParser.parseBuffer', () => {
     expect((error as Error).message).toMatch(/Word 6\/95/)
   })
 
-  it('maps any other word-extractor failure to invalid_format with the cause retained', async () => {
-    const libraryError = new Error('Invalid Short Sector Allocation Table')
-    mockWordExtract.mockRejectedValue(libraryError)
-
-    const error = await new DocParser()
-      .parseBuffer(buildLegacyOleDoc())
-      .catch((caught: unknown) => caught)
-
-    expect(error).toBeInstanceOf(FileParserError)
-    expect(error).toMatchObject({ code: 'invalid_format' })
-    expect((error as FileParserError).cause).toBe(libraryError)
-  })
+  it.each([
+    new Error('Invalid Short Sector Allocation Table'),
+    new RangeError('Attempt to access memory outside buffer bounds'),
+  ])(
+    'maps any other word-extractor failure to a stable invalid_format message with the cause retained: %s',
+    async (libraryError) => {
+      mockWordExtract.mockRejectedValue(libraryError)
+
+      const error = await new DocParser()
+        .parseBuffer(buildLegacyOleDoc())
+        .catch((caught: unknown) => caught)
+
+      expect(error).toBeInstanceOf(FileParserError)
+      expect(error).toMatchObject({
+        code: 'invalid_format',
+        message: 'This .doc file could not be read',
+      })
+      expect((error as FileParserError).cause).toBe(libraryError)
+    }
+  )
 
   it('reports a legacy .doc with no text as no_extractable_text', async () => {
     mockWordExtract.mockResolvedValue(wordDocument({ body: '   \n' }))