diff --git a/src/lib/embedder.ts b/src/lib/embedder.ts index b6a8011..3495567 100644 --- a/src/lib/embedder.ts +++ b/src/lib/embedder.ts @@ -1,4 +1,5 @@ import { pipeline, type FeatureExtractionPipeline } from '@huggingface/transformers'; +import { firstBodyParagraph } from './text.js'; const MODEL = 'Xenova/all-MiniLM-L6-v2'; @@ -32,7 +33,7 @@ export class Embedder { tags: string[], content: string, ): string { - const firstParagraph = content.split(/\n\n+/)[0] ?? ''; + const firstParagraph = firstBodyParagraph(content).trim(); const parts = [title]; if (tags.length > 0) { parts.push(tags.join(', ')); diff --git a/src/lib/store.ts b/src/lib/store.ts index 4771b58..2708208 100644 --- a/src/lib/store.ts +++ b/src/lib/store.ts @@ -1,6 +1,7 @@ import Database from 'better-sqlite3'; import * as sqliteVec from 'sqlite-vec'; import type { ParsedNode, ParsedEdge, SearchResult } from './types.js'; +import { firstBodyParagraph } from './text.js'; export class Store { db: Database.Database; @@ -281,7 +282,7 @@ export class Store { } function firstParagraph(content: string, maxLen: number): string { - const para = content.split(/\n\n+/).find(p => p.trim().length > 0 && !p.startsWith('#')); + const para = firstBodyParagraph(content); if (!para) return ''; const trimmed = para.trim(); return trimmed.length > maxLen ? trimmed.slice(0, maxLen) + '...' : trimmed; diff --git a/src/lib/text.ts b/src/lib/text.ts new file mode 100644 index 0000000..2564808 --- /dev/null +++ b/src/lib/text.ts @@ -0,0 +1,14 @@ +/** + * Returns the first non-empty, non-heading paragraph in `content`. + * + * `gray-matter` leaves a leading newline after stripping the YAML + * frontmatter, so the first paragraph from a naive `split('\n\n')` is often + * `'\n# Title'` — which only looks like a heading once trimmed. Trimming + * before the `#` check ensures the title line is correctly skipped in + * favor of the actual body text. + */ +export function firstBodyParagraph(content: string): string { + return ( + content.split(/\n\n+/).find(p => p.trim().length > 0 && !p.trim().startsWith('#')) ?? '' + ); +} diff --git a/test/embedder.test.ts b/test/embedder.test.ts index a20a9f1..dc1f036 100644 --- a/test/embedder.test.ts +++ b/test/embedder.test.ts @@ -39,6 +39,18 @@ describe('Embedder', () => { expect(text).toContain('theoretical framework'); expect(text).not.toContain('More details here'); }); + + it('includes the body paragraph, not the title heading, when content has a gray-matter-style leading newline', () => { + // gray-matter leaves a leading newline after stripping YAML frontmatter, + // so the raw content starts with '\n# Widget Theory'. + const text = Embedder.buildEmbeddingText( + 'Widget Theory', + [], + '\n# Widget Theory\n\nA theoretical framework for understanding component interactions.\n\n## Section', + ); + expect(text).toContain('A theoretical framework for understanding component interactions'); + expect(text).not.toContain('# Widget Theory'); + }); }); function cosineSimilarity(a: Float32Array, b: Float32Array): number { diff --git a/test/store.test.ts b/test/store.test.ts index 1f8ac92..7248f05 100644 --- a/test/store.test.ts +++ b/test/store.test.ts @@ -110,6 +110,23 @@ describe('Store', () => { expect(results[0].excerpt).toContain('framework'); }); + it('vector search excerpt skips the title heading when content has a gray-matter-style leading newline', () => { + store.upsertNode({ + id: 'test.md', + title: 'Widget Theory', + // gray-matter leaves a leading newline after stripping frontmatter, + // so a naive split on '\n\n' yields '\n# Widget Theory' as paragraph 0. + content: '\n# Widget Theory\n\nA framework for understanding component interactions.\n\n## Section', + frontmatter: {}, + }); + const embedding = new Float32Array(384).fill(0.1); + store.upsertEmbedding('test.md', embedding); + const results = store.searchVector(embedding, 5); + expect(results.length).toBeGreaterThan(0); + expect(results[0].excerpt).toContain('A framework for understanding component interactions'); + expect(results[0].excerpt).not.toContain('# Widget Theory'); + }); + it('counts edges for a node', () => { store.upsertNode({ id: 'a.md', title: 'A', content: '', frontmatter: {} }); store.upsertNode({ id: 'b.md', title: 'B', content: '', frontmatter: {} }); diff --git a/test/text.test.ts b/test/text.test.ts new file mode 100644 index 0000000..e1a617e --- /dev/null +++ b/test/text.test.ts @@ -0,0 +1,21 @@ +import { describe, it, expect } from 'vitest'; +import { firstBodyParagraph } from '../src/lib/text.js'; + +describe('firstBodyParagraph', () => { + it('returns the first non-empty, non-heading paragraph', () => { + const content = '# Title\n\nFirst body paragraph.\n\nSecond paragraph.'; + expect(firstBodyParagraph(content)).toBe('First body paragraph.'); + }); + + it('skips a leading heading even when preceded by a stray newline', () => { + // gray-matter leaves a leading newline after stripping YAML frontmatter, + // so the first split segment is '\n# Title', not '# Title'. + const content = '\n# Title\n\nFirst body paragraph.\n\n## Section'; + expect(firstBodyParagraph(content)).toBe('First body paragraph.'); + }); + + it('returns empty string when content has no body paragraph', () => { + expect(firstBodyParagraph('\n# Title Only')).toBe(''); + expect(firstBodyParagraph('')).toBe(''); + }); +});