|
| 1 | +import remarkGfm from 'remark-gfm' |
| 2 | +import remarkParse from 'remark-parse' |
| 3 | +import { unified } from 'unified' |
| 4 | +import { describe, expect, it } from 'vitest' |
| 5 | +import { remarkPlainText } from '@/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/remark-plain-text' |
| 6 | + |
| 7 | +const reference = unified().use(remarkParse).use(remarkGfm) |
| 8 | +const optimized = unified().use(remarkParse).use(remarkGfm).use(remarkPlainText) |
| 9 | + |
| 10 | +describe('plain-paragraph parsing', () => { |
| 11 | + it('matches remark positions, whitespace, and Unicode text', () => { |
| 12 | + for (const text of [ |
| 13 | + 'Hello world! ', |
| 14 | + 'Hello ', |
| 15 | + 'A café, 日本語, العربية, 😀 and nonbreaking\u00a0spaces.\u00a0', |
| 16 | + 'A sentence with (parentheses), quotes, 1.23 and 20% off.', |
| 17 | + '12)word', |
| 18 | + 'Plain\u000btext\u000ctext', |
| 19 | + ]) { |
| 20 | + for (let end = 0; end <= text.length; end++) { |
| 21 | + const prefix = text.slice(0, end) |
| 22 | + expect(optimized.parse(prefix), JSON.stringify(prefix)).toEqual(reference.parse(prefix)) |
| 23 | + } |
| 24 | + } |
| 25 | + }) |
| 26 | + |
| 27 | + it('preserves Markdown interpretation when a plain prefix gains syntax', () => { |
| 28 | + const cases = [ |
| 29 | + 'Text **bold**, _italic_, ~~deleted~~ and `code`.', |
| 30 | + 'Text [link](https://example.com), .', |
| 31 | + 'Text www.example.com, https://example.com and user@example.com.', |
| 32 | + 'Text WWW.EXAMPLE.COM and foo@bar.test.', |
| 33 | + 'Text & and A and <b>raw</b>.', |
| 34 | + 'Text \\*escaped\\* and \\\nline break.', |
| 35 | + 'Text \nline break.\r\n\r\nNext paragraph.', |
| 36 | + 'Text\u0000replacement\tand tab', |
| 37 | + '1. Ordered list\n2. Next item', |
| 38 | + '1) Ordered list', |
| 39 | + ' Indented text\n\n code', |
| 40 | + '# Heading\n\n> Quote\n\n---\n\n- [x] task', |
| 41 | + 'Heading\n=======\n\n| Table |\n| --- |\n| Cell |', |
| 42 | + '[link][ref]\n\n[ref]: https://example.com', |
| 43 | + '```typescript\nconst text = `multiline\nvalue`\n```', |
| 44 | + '\ufeffText', |
| 45 | + ] |
| 46 | + for (const text of cases) { |
| 47 | + for (let end = 0; end <= text.length; end++) { |
| 48 | + const prefix = text.slice(0, end) |
| 49 | + expect(optimized.parse(prefix), JSON.stringify(prefix)).toEqual(reference.parse(prefix)) |
| 50 | + } |
| 51 | + } |
| 52 | + }) |
| 53 | + |
| 54 | + it('matches the reference on growing paragraphs and every ASCII character', () => { |
| 55 | + const cases = [ |
| 56 | + 'A plain paragraph describing the workflow. '.repeat(50), |
| 57 | + '## Heading\n\nA paragraph.\n\n- A list item\n\n| Column |\n| --- |\n| Cell |\n\n'.repeat(30), |
| 58 | + `\`\`\`typescript\n${'const value = items.map((item) => item.value)\n'.repeat(40)}\`\`\``, |
| 59 | + 'A citation. <source>{"url":"https://example.com"}</source>\n\n'.repeat(30), |
| 60 | + ] |
| 61 | + for (const text of cases) { |
| 62 | + for (let end = 7; end <= text.length; end += 7) { |
| 63 | + expect(optimized.parse(text.slice(0, end))).toEqual(reference.parse(text.slice(0, end))) |
| 64 | + } |
| 65 | + } |
| 66 | + for (let char = 0; char < 128; char++) { |
| 67 | + const text = `Before ${String.fromCharCode(char)} after ` |
| 68 | + expect(optimized.parse(text), JSON.stringify(text)).toEqual(reference.parse(text)) |
| 69 | + } |
| 70 | + }) |
| 71 | +}) |
0 commit comments