diff --git a/.env.example b/.env.example index 2d476a34..92a20f31 100644 --- a/.env.example +++ b/.env.example @@ -28,6 +28,7 @@ AI_RESPONSE_MODEL= # Override AI response model (default: claude-sonn AI_CONFIDENCE_MODEL= # Override confidence scoring model AI_CLASSIFIER_MODEL= # Override ticket classifier model AI_SENTIMENT_MODEL= # Override sentiment analysis model +AI_QUERY_DISTILLER_MODEL= # Override the docs-search query distiller model # ─── Shadow Mode ───────────────────────────────────────────────────────────── # Set to 'true' to run the full AI pipeline but LOG responses instead of posting @@ -49,7 +50,7 @@ DISCORD_BOT_TOKEN="" # Bot token used by the discord-bot app DISCORD_CLIENT_ID="" DISCORD_CLIENT_SECRET="" # OAuth client secret (dashboard Discord login) DISCORD_MCP_TOKEN="" # Shared secret guarding the discord-mcp server -MONITORED_CHANNEL_IDS="" +MONITORED_CHANNEL_IDS="" # Forum/channel IDs the bot answers in. REQUIRED — empty means no channels DISCORD_DIGEST_CHANNEL_ID= # Channel for onboarding digest posts # Guild (server) IDs per community — replaces the old single GUILD_ID DISCORD_GUILD_COPILOTKIT= # CopilotKit Discord server ID diff --git a/apps/discord-bot/src/__tests__/message-create.test.ts b/apps/discord-bot/src/__tests__/message-create.test.ts index 25b4ba77..53033e5d 100644 --- a/apps/discord-bot/src/__tests__/message-create.test.ts +++ b/apps/discord-bot/src/__tests__/message-create.test.ts @@ -35,6 +35,7 @@ vi.mock('discord.js', async (importOriginal) => { import { handleMessageCreate } from '../events/message-create.js'; import { prisma } from '@copilotkit/outpost/db'; import { createJob } from '@copilotkit/outpost/queue'; +import { isShadowMode, handleShadowMessage } from '../lib/shadow-mode.js'; const TICKET = { id: 'ticket-1', @@ -68,6 +69,7 @@ function makeMessage(overrides: Record = {}) { describe('handleMessageCreate', () => { beforeEach(() => { + vi.mocked(isShadowMode).mockReturnValue(false); // findTicketByThreadId returns the existing ticket vi.mocked(prisma.ticket.findFirst).mockResolvedValue(TICKET as ReturnType extends Promise ? T : never); vi.mocked(prisma.message.create).mockResolvedValue({ @@ -149,6 +151,53 @@ describe('handleMessageCreate', () => { }); }); + // Regression: Discord dispatches BOTH ThreadCreate and MessageCreate for a + // new forum post. handleThreadCreate already ingests the starter message, + // so handling it again here enqueued a SECOND AI_RESPONSE job for the same + // ticket — the same question retrieved and answered twice, ~0.2s apart. + // A thread's starter message shares the thread's own ID. + it('ignores the thread starter message already ingested by ThreadCreate', async () => { + const starter = makeMessage({ id: 'thread-123' }); + + await handleMessageCreate(starter); + + expect(prisma.ticket.findFirst).not.toHaveBeenCalled(); + expect(prisma.message.create).not.toHaveBeenCalled(); + expect(createJob).not.toHaveBeenCalled(); + }); + + it('ignores the thread starter message in shadow mode too', async () => { + vi.mocked(isShadowMode).mockReturnValue(true); + const starter = makeMessage({ id: 'thread-123' }); + + await handleMessageCreate(starter); + + expect(handleShadowMessage).not.toHaveBeenCalled(); + expect(createJob).not.toHaveBeenCalled(); + }); + + // The gate above must not swallow real replies. Asserted on the message + // record rather than on an enqueue: since #172/#191, `InboundHandler` never + // enqueues AI_RESPONSE for a reply on ANY platform — Outpost answers once per + // ticket, on the opening message, and a human owns the thread after that. This + // test predates that rule and asserted the enqueue, which is why it survived + // the textual merge and then failed. What it is actually here to prove is that + // `message.id === threadId` distinguishes the starter message from a reply, + // and the message record is what shows that. + it('still processes genuine replies in the same thread', async () => { + const reply = makeMessage({ id: 'msg-777' }); + + await handleMessageCreate(reply); + + expect(prisma.message.create).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ ticketId: 'ticket-1', type: 'USER' }), + }), + ); + // And the one-answer rule still holds: a reply enqueues nothing. + expect(createJob).not.toHaveBeenCalled(); + }); + it('uses DiscordAdapter.parseInboundEvent to normalize message events', async () => { const message = makeMessage(); await handleMessageCreate(message); diff --git a/apps/discord-bot/src/__tests__/thread-create.test.ts b/apps/discord-bot/src/__tests__/thread-create.test.ts index bbdd7f27..677af211 100644 --- a/apps/discord-bot/src/__tests__/thread-create.test.ts +++ b/apps/discord-bot/src/__tests__/thread-create.test.ts @@ -11,15 +11,18 @@ vi.mock('../lib/shadow-mode.js', () => ({ handleShadowThreadCreate: vi.fn().mockResolvedValue('shadow-ticket-id'), })); -vi.mock('../config.js', () => ({ - config: { +// Mutable so the fail-closed case can empty MONITORED_CHANNEL_IDS. +const { testConfig } = vi.hoisted(() => ({ + testConfig: { discordToken: 'test-token', clientId: 'test-client-id', guildId: 'test-guild-id', - monitoredChannelIds: ['forum-channel-1'], + monitoredChannelIds: ['forum-channel-1'] as string[], }, })); +vi.mock('../config.js', () => ({ config: testConfig })); + // Mock discord.js REST to prevent real HTTP calls vi.mock('discord.js', async (importOriginal) => { const actual = await importOriginal() as Record; @@ -36,6 +39,7 @@ vi.mock('discord.js', async (importOriginal) => { import { handleThreadCreate } from '../events/thread-create.js'; import { prisma } from '@copilotkit/outpost/db'; import { createJob } from '@copilotkit/outpost/queue'; +import { isShadowMode, handleShadowThreadCreate } from '../lib/shadow-mode.js'; import { PlatformDiscordAdapter } from '@copilotkit/outpost/shared/platforms'; function makeThread(overrides: Record = {}) { @@ -57,6 +61,9 @@ function makeThread(overrides: Record = {}) { describe('handleThreadCreate', () => { beforeEach(() => { + testConfig.monitoredChannelIds = ['forum-channel-1']; + vi.mocked(isShadowMode).mockReturnValue(false); + vi.mocked(prisma.ticket.create).mockResolvedValue({ id: 'ticket-internal-id', displayId: 'TKT-AB12CD34', @@ -204,6 +211,78 @@ describe('handleThreadCreate', () => { consoleSpy.mockRestore(); }); + // An unset MONITORED_CHANNEL_IDS used to mean "monitor every channel", so a + // missing env var silently opted the whole guild into a retrieval + + // generation cycle per thread. It now fails closed. + it('ignores every thread when MONITORED_CHANNEL_IDS is empty', async () => { + testConfig.monitoredChannelIds = []; + + await handleThreadCreate(makeThread(), true); + + expect(prisma.ticket.create).not.toHaveBeenCalled(); + expect(createJob).not.toHaveBeenCalled(); + }); + + // Announcements and release notes are threads too — they should not spend a + // full retrieval + generation cycle. + it('ignores a thread that does not read as a support request', async () => { + const thread = makeThread({ + name: 'v1.10.0 released', + fetchStarterMessage: vi.fn().mockResolvedValue({ + content: 'v1.10.0 is out. Release notes are in the changelog.', + author: { tag: 'Maintainer#0001', id: 'user-1', username: 'Maintainer' }, + }), + }); + + await handleThreadCreate(thread, true); + + expect(prisma.ticket.create).not.toHaveBeenCalled(); + expect(createJob).not.toHaveBeenCalled(); + }); + + it('answers an announcement-shaped thread that @-mentions the bot', async () => { + const thread = makeThread({ + name: 'v1.10.0 released', + fetchStarterMessage: vi.fn().mockResolvedValue({ + content: '<@test-client-id> v1.10.0 is out. Notes in the changelog.', + author: { tag: 'Maintainer#0001', id: 'user-1', username: 'Maintainer' }, + }), + }); + + await handleThreadCreate(thread, true); + + expect(prisma.ticket.create).toHaveBeenCalled(); + }); + + it('answers a thread whose question is only in the title', async () => { + const thread = makeThread({ + name: 'How do I render generative UI?', + fetchStarterMessage: vi.fn().mockResolvedValue({ + content: 'Details below.', + author: { tag: 'TestUser#1234', id: 'user-456', username: 'TestUser' }, + }), + }); + + await handleThreadCreate(thread, true); + + expect(prisma.ticket.create).toHaveBeenCalled(); + }); + + it('applies the support-request gate in shadow mode too', async () => { + vi.mocked(isShadowMode).mockReturnValue(true); + const thread = makeThread({ + name: 'v1.10.0 released', + fetchStarterMessage: vi.fn().mockResolvedValue({ + content: 'v1.10.0 is out. Release notes are in the changelog.', + author: { tag: 'Maintainer#0001', id: 'user-1', username: 'Maintainer' }, + }), + }); + + await handleThreadCreate(thread, true); + + expect(handleShadowThreadCreate).not.toHaveBeenCalled(); + }); + it('uses DiscordAdapter.parseInboundEvent to normalize the thread event', async () => { const thread = makeThread(); await handleThreadCreate(thread, true); diff --git a/apps/discord-bot/src/events/message-create.ts b/apps/discord-bot/src/events/message-create.ts index d4fde17b..b5d9c777 100644 --- a/apps/discord-bot/src/events/message-create.ts +++ b/apps/discord-bot/src/events/message-create.ts @@ -37,6 +37,14 @@ export async function handleMessageCreate(message: Message): Promise { const threadId = message.channel.id; + // Discord dispatches BOTH ThreadCreate and MessageCreate for a new forum + // post, and handleThreadCreate has already ingested this exact message as + // the ticket's first message. Processing it again enqueues a second + // AI_RESPONSE job for the same ticket, so the same question is retrieved + // and answered twice. A thread's starter message shares the thread's ID — + // that identity is what makes this detectable. + if (message.id === threadId) return; + try { // Look up the ticket associated with this thread (for shadow mode check) const ticket = await findTicketByThreadId(threadId); diff --git a/apps/discord-bot/src/events/thread-create.ts b/apps/discord-bot/src/events/thread-create.ts index 3eccb568..e7b79aab 100644 --- a/apps/discord-bot/src/events/thread-create.ts +++ b/apps/discord-bot/src/events/thread-create.ts @@ -2,7 +2,7 @@ import { ChannelType, type ThreadChannel } from 'discord.js'; import { prisma } from '@copilotkit/outpost/db'; import { createJob } from '@copilotkit/outpost/queue'; import { PlatformDiscordAdapter, InboundHandler } from '@copilotkit/outpost/shared/platforms'; -import { generateTicketId } from '@copilotkit/outpost/shared'; +import { generateTicketId, isSupportRequest } from '@copilotkit/outpost/shared'; import type { CreateJobFn } from '@copilotkit/outpost/shared'; import { config } from '../config.js'; import { isShadowMode, handleShadowThreadCreate } from '../lib/shadow-mode.js'; @@ -25,6 +25,26 @@ const createJobFn: CreateJobFn = async ( ); }; +/** Log the unconfigured-channel warning once, not once per thread. */ +let warnedUnconfiguredChannels = false; +function warnUnconfiguredChannels(): void { + if (warnedUnconfiguredChannels) return; + warnedUnconfiguredChannels = true; + console.warn( + '[Discord Bot] MONITORED_CHANNEL_IDS is empty — ignoring all threads. ' + + 'Set it to the forum channel IDs Outpost should answer in.', + ); +} + +/** + * A forum post's title often carries the question while the body carries the + * repro, so both are considered. The bot's own application ID doubles as its + * user ID, so an @-mention of the bot always qualifies. + */ +function shouldAnswer(threadName: string, content: string): boolean { + return isSupportRequest(`${threadName}\n${content}`, { botUserId: config.clientId }); +} + export async function handleThreadCreate(thread: ThreadChannel, newlyCreated: boolean): Promise { if (!newlyCreated) return; @@ -32,13 +52,15 @@ export async function handleThreadCreate(thread: ThreadChannel, newlyCreated: bo const parentId = thread.parentId; if (!parentId) return; - // If monitoredChannelIds is configured, only track those channels. - // If empty, monitor all channels (useful for development). - const isMonitored = - config.monitoredChannelIds.length === 0 || - config.monitoredChannelIds.includes(parentId); + // Fail CLOSED on an unset MONITORED_CHANNEL_IDS. Treating "empty" as + // "every channel" meant a missing env var silently opted the whole guild + // into a retrieval + generation cycle per thread. + if (config.monitoredChannelIds.length === 0) { + warnUnconfiguredChannels(); + return; + } - if (!isMonitored) return; + if (!config.monitoredChannelIds.includes(parentId)) return; // Only handle public/private threads (includes forum posts) if ( @@ -52,6 +74,12 @@ export async function handleThreadCreate(thread: ThreadChannel, newlyCreated: bo if (isShadowMode()) { const starterMessage = await thread.fetchStarterMessage(); const content = starterMessage?.content ?? ''; + if (!shouldAnswer(thread.name, content)) { + console.log( + `[Discord Bot] Thread ${thread.id} does not read as a support request, skipping`, + ); + return; + } const authorTag = starterMessage?.author.tag ?? 'Unknown'; const authorId = starterMessage?.author.id ?? ''; const displayId = generateTicketId(); @@ -71,6 +99,15 @@ export async function handleThreadCreate(thread: ThreadChannel, newlyCreated: bo // Fetch the starter message (first message in the thread) const starterMessage = await thread.fetchStarterMessage(); + // Announcements and release notes are threads too — only spend a full + // retrieval + generation cycle on something that reads like a question. + if (!shouldAnswer(thread.name, starterMessage?.content ?? '')) { + console.log( + `[Discord Bot] Thread ${thread.id} does not read as a support request, skipping`, + ); + return; + } + // Parse the raw event through the platform adapter const inboundMessage = adapter.parseInboundEvent({ thread, diff --git a/packages/outpost/ai/src/config.ts b/packages/outpost/ai/src/config.ts index 01dee226..bb3e9726 100644 --- a/packages/outpost/ai/src/config.ts +++ b/packages/outpost/ai/src/config.ts @@ -35,6 +35,15 @@ export const config = { /** Maximum tokens for classification */ maxClassifierTokens: 512, + /** Model used to distill a raw message body into a docs-search query (cheap, fast) */ + queryDistillerModel: process.env.AI_QUERY_DISTILLER_MODEL ?? 'claude-haiku-4-5-20251001', + + /** Maximum tokens for query distillation — the output is one short query */ + maxQueryDistillerTokens: 128, + + /** Temperature for query distillation (lower = more deterministic) */ + queryDistillerTemperature: 0, + /** Model used for sentiment analysis (cheap, fast) */ sentimentModel: process.env.AI_SENTIMENT_MODEL ?? 'claude-haiku-4-5-20251001', diff --git a/packages/outpost/ai/src/index.ts b/packages/outpost/ai/src/index.ts index 75857e14..9bdfdd99 100644 --- a/packages/outpost/ai/src/index.ts +++ b/packages/outpost/ai/src/index.ts @@ -16,6 +16,8 @@ export { AI_DISCLAIMER_REVIEWED, ResponseFormatter, } from './formatter.js'; +export { SearchQueryBuilder, heuristicSearchQuery } from './query.js'; +export type { SearchQuery } from './query.js'; export { AIPipeline, SUPPRESSED_RESPONSE_TEXT } from './pipeline.js'; export { analyzeSentiment } from './sentiment.js'; export { scoreEngagement } from './engagement.js'; diff --git a/packages/outpost/ai/src/pipeline.test.ts b/packages/outpost/ai/src/pipeline.test.ts index 051bfa0f..f43f6a20 100644 --- a/packages/outpost/ai/src/pipeline.test.ts +++ b/packages/outpost/ai/src/pipeline.test.ts @@ -32,9 +32,13 @@ const mockHeuristicScore = vi.fn(); const mockClassify = vi.fn(); const mockHeuristicClassify = vi.fn(); const mockFormat = vi.fn(); +const mockBuildQuery = vi.fn(); function createPipeline() { return new AIPipeline({ + queryBuilder: { + build: mockBuildQuery, + } as never, pathfinder: { searchDocs: mockSearchDocs, exploreDocs: vi.fn(), @@ -94,7 +98,14 @@ describe('AIPipeline', () => { vi.resetAllMocks(); pipeline = createPipeline(); - // Set up defaults + // Set up defaults. The query builder passes the question through + // untouched unless a test overrides it. + mockBuildQuery.mockImplementation(async (question: string) => ({ + query: question, + sanitized: question, + degraded: false, + tokenUsage: { inputTokens: 0, outputTokens: 0 }, + })); mockSearchDocs.mockResolvedValue(sampleSearchResults); mockGenerate.mockResolvedValue(sampleGeneratedResponse); mockScore.mockResolvedValue(sampleConfidence); @@ -119,6 +130,64 @@ describe('AIPipeline', () => { expect(result.latencyMs).toBeGreaterThanOrEqual(0); }); + // Regression: the raw inbound body used to be forwarded verbatim as the + // docs-search query, so Discord mentions, custom emoji, pasted channel + // sidebars, and issue-template boilerplate all reached the embedder. + it('searches with the distilled query, not the raw body', async () => { + const rawBody = '<@!123> hey <#456> — how do I render generative UI?'; + mockBuildQuery.mockResolvedValue({ + query: 'render generative UI', + sanitized: 'hey — how do I render generative UI?', + degraded: false, + tokenUsage: { inputTokens: 40, outputTokens: 8 }, + }); + + await pipeline.generateSupportResponse(rawBody, { source: 'discord' }); + + expect(mockBuildQuery).toHaveBeenCalledWith(rawBody); + expect(mockSearchDocs).toHaveBeenCalledWith({ query: 'render generative UI' }); + }); + + it('generates from the sanitized body, not the distilled query', async () => { + mockBuildQuery.mockResolvedValue({ + query: 'render generative UI', + sanitized: 'hey — how do I render generative UI?', + degraded: false, + tokenUsage: { inputTokens: 40, outputTokens: 8 }, + }); + + await pipeline.generateSupportResponse('<@!123> hey — how do I render generative UI?', { + source: 'discord', + }); + + expect(mockGenerate).toHaveBeenCalledWith( + expect.objectContaining({ question: 'hey — how do I render generative UI?' }), + expect.any(Array), + undefined, + ); + expect(mockScore).toHaveBeenCalledWith( + 'hey — how do I render generative UI?', + expect.any(String), + expect.any(Array), + ); + }); + + it('counts the distiller tokens in the aggregate usage', async () => { + mockBuildQuery.mockResolvedValue({ + query: 'render generative UI', + sanitized: 'how do I render generative UI?', + degraded: false, + tokenUsage: { inputTokens: 40, outputTokens: 8 }, + }); + + const result = await pipeline.generateSupportResponse('question', { + source: 'discord', + }); + + expect(result.tokenUsage.inputTokens).toBe(740); // 40 + 500 + 200 + expect(result.tokenUsage.outputTokens).toBe(138); // 8 + 100 + 30 + }); + it('should pass the source channel into the generator context', async () => { await pipeline.generateSupportResponse('test question', { source: 'discord' }); @@ -845,6 +914,12 @@ describe('AIPipeline confidence calibration', () => { beforeEach(() => { vi.resetAllMocks(); pipeline = createPipeline(); + mockBuildQuery.mockImplementation(async (question: string) => ({ + query: question, + sanitized: question, + degraded: false, + tokenUsage: { inputTokens: 0, outputTokens: 0 }, + })); mockSearchDocs.mockResolvedValue(sampleSearchResults); mockGenerate.mockResolvedValue(sampleGeneratedResponse); // generator score 0.85 mockScore.mockResolvedValue({ diff --git a/packages/outpost/ai/src/pipeline.ts b/packages/outpost/ai/src/pipeline.ts index 3c490f31..8d2a0d7b 100644 --- a/packages/outpost/ai/src/pipeline.ts +++ b/packages/outpost/ai/src/pipeline.ts @@ -19,6 +19,7 @@ import { AI_DISCLAIMER_REVIEWED, ResponseFormatter, } from './formatter.js'; +import { SearchQueryBuilder } from './query.js'; import { validateConfig } from './config.js'; /** @@ -68,6 +69,7 @@ export class AIPipeline { private confidenceScorer: ConfidenceScorer; private classifier: TicketClassifier; private formatter: ResponseFormatter; + private queryBuilder: SearchQueryBuilder; constructor(options?: { pathfinder?: PathfinderClient; @@ -75,6 +77,7 @@ export class AIPipeline { confidenceScorer?: ConfidenceScorer; classifier?: TicketClassifier; formatter?: ResponseFormatter; + queryBuilder?: SearchQueryBuilder; }) { validateConfig(); this.pathfinder = options?.pathfinder ?? new PathfinderClient(); @@ -82,6 +85,7 @@ export class AIPipeline { this.confidenceScorer = options?.confidenceScorer ?? new ConfidenceScorer(); this.classifier = options?.classifier ?? new TicketClassifier(); this.formatter = options?.formatter ?? new ResponseFormatter(); + this.queryBuilder = options?.queryBuilder ?? new SearchQueryBuilder(); } /** @@ -97,11 +101,18 @@ export class AIPipeline { const startTime = Date.now(); const totalTokenUsage: TokenUsage = { inputTokens: 0, outputTokens: 0 }; + // Step 0: Strip platform markup and distill a focused search query. + // The raw body carries mentions, custom emoji, pasted channel sidebars, + // and issue-template boilerplate — none of which belongs in an embedding. + const searchQuery = await this.queryBuilder.build(question); + totalTokenUsage.inputTokens += searchQuery.tokenUsage.inputTokens; + totalTokenUsage.outputTokens += searchQuery.tokenUsage.outputTokens; + // Step 1: Query Pathfinder for relevant content let searchResults: SearchResult[]; try { searchResults = await this.pathfinder.searchDocs({ - query: question, + query: searchQuery.query, }); } catch (error) { console.error( @@ -110,9 +121,12 @@ export class AIPipeline { searchResults = []; } - // Step 2: Generate response + // Step 2: Generate response. + // Generation gets the SANITIZED body, not the distilled query — the + // distillation is lossy on purpose and only good enough for retrieval, + // while the answer needs the reporter's full context and code. const pipelineContext: PipelineContext = { - question, + question: searchQuery.sanitized, source: options.source, }; @@ -126,7 +140,7 @@ export class AIPipeline { // (sequential, not parallel — the scorer needs the real text to // produce a meaningful signal, not a retrieval-quality proxy). const confidenceAssessment = await this.confidenceScorer - .score(question, generatedResponse.text, searchResults) + .score(searchQuery.sanitized, generatedResponse.text, searchResults) .catch((error) => { console.error( `[Pipeline] Confidence scoring failed: ${error instanceof Error ? error.message : String(error)}`, @@ -325,11 +339,14 @@ export class AIPipeline { question: string, options: PipelineOptions, ): AsyncIterable { + // Sanitize + distill first, same as the non-streaming path. + const searchQuery = await this.queryBuilder.build(question); + // Fetch search results first let searchResults: SearchResult[]; try { searchResults = await this.pathfinder.searchDocs({ - query: question, + query: searchQuery.query, }); } catch (error) { console.error( @@ -340,7 +357,7 @@ export class AIPipeline { } const pipelineContext: PipelineContext = { - question, + question: searchQuery.sanitized, source: options.source, }; diff --git a/packages/outpost/ai/src/query.test.ts b/packages/outpost/ai/src/query.test.ts new file mode 100644 index 00000000..3c4e46c3 --- /dev/null +++ b/packages/outpost/ai/src/query.test.ts @@ -0,0 +1,203 @@ +import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest'; +import { LLMock } from '@copilotkit/aimock'; +import { SearchQueryBuilder, heuristicSearchQuery } from './query.js'; + +// ─── aimock setup ─────────────────────────────────────────────────────────── + +let mock: LLMock; +let originalBaseUrl: string | undefined; + +beforeAll(async () => { + mock = new LLMock({ port: 0 }); + await mock.start(); + originalBaseUrl = process.env.ANTHROPIC_BASE_URL; + process.env.ANTHROPIC_BASE_URL = mock.url; +}); + +afterAll(async () => { + if (originalBaseUrl === undefined) { + delete process.env.ANTHROPIC_BASE_URL; + } else { + process.env.ANTHROPIC_BASE_URL = originalBaseUrl; + } + await mock.stop(); +}); + +beforeEach(() => { + mock.reset(); +}); + +// ─── Fixtures ─────────────────────────────────────────────────────────────── + +/** + * A forum post long enough to trip distillation, carrying Discord markup, a + * pasted channel sidebar, and issue-template boilerplate around one question. + */ +const NOISY_POST = [ + '<:copilotkit:1187213988392189962> hey <@!284920412034990081> :wave:', + '', + 'I posted this in <#1205139168783503400> already but reposting here since', + '<#1379082175625953370> said this was the right place to ask about it.', + '', + 'Channels', + '# ┃welcome', + '# ┃announcements', + '# ┃support', + '', + '## Pre-flight Checklist', + '- [x] I have searched existing issues', + '- [ ] I am willing to submit a PR', + '', + '### ♻️ Reproduction Steps', + '1. npx create-next-app', + '2. install @copilotkit/react-core and wire up the provider', + '', + 'My actual question: how do I render a custom React component from a tool call', + 'with useCopilotAction? Generative UI never renders, the tool returns text.', + '', + 'Docs I already read: https://docs.copilotkit.ai/generative-ui ', +].join('\n'); + +// ─── Tests ────────────────────────────────────────────────────────────────── + +describe('heuristicSearchQuery', () => { + it('returns an empty string when there is no prose left', () => { + expect(heuristicSearchQuery('')).toBe(''); + expect(heuristicSearchQuery('```\nconst x = 1;\n```')).toBe(''); + }); + + it('prefers the sentences that carry the question mark', () => { + const query = heuristicSearchQuery( + 'Thanks for the release! I upgraded this morning.\nHow do I register a tool?', + ); + expect(query).toBe('How do I register a tool?'); + }); + + it('drops code fences, stack frames, and bare URLs', () => { + const query = heuristicSearchQuery( + [ + 'Why does the runtime throw?', + '```ts', + 'const runtime = new CopilotRuntime();', + '```', + ' at handler (/app/src/index.ts:12:5)', + 'https://docs.copilotkit.ai/runtime', + ].join('\n'), + ); + expect(query).toBe('Why does the runtime throw?'); + }); + + it('falls back to the opening prose when nothing is phrased as a question', () => { + const query = heuristicSearchQuery('The provider crashes on mount with a null ref.'); + expect(query).toBe('The provider crashes on mount with a null ref.'); + }); + + it('caps the query length on a word boundary', () => { + const query = heuristicSearchQuery('word '.repeat(200)); + expect(query.length).toBeLessThanOrEqual(300); + expect(query.endsWith('word')).toBe(true); + }); +}); + +describe('SearchQueryBuilder', () => { + let builder: SearchQueryBuilder; + + beforeEach(() => { + builder = new SearchQueryBuilder({ apiKey: 'test-key' }); + }); + + it('returns an empty query for an empty body without calling the model', async () => { + const result = await builder.build(' '); + + expect(result.query).toBe(''); + expect(result.degraded).toBe(false); + expect(mock.getRequests()).toHaveLength(0); + }); + + it('passes a short, already-focused message straight through', async () => { + const result = await builder.build('How do I register a tool with useCopilotAction?'); + + expect(result.query).toBe('How do I register a tool with useCopilotAction?'); + expect(result.degraded).toBe(false); + expect(mock.getRequests()).toHaveLength(0); + }); + + it('sanitizes platform markup even on the short-circuit path', async () => { + const result = await builder.build('<@!123> does <#456> support SSR? <:ck:789>'); + + expect(result.query).toBe('does support SSR?'); + expect(result.sanitized).toBe('does support SSR?'); + }); + + it('distills a long noisy body into a focused query', async () => { + mock.onMessage(/./, { + content: 'render a custom React component from a useCopilotAction tool call', + usage: { input_tokens: 210, output_tokens: 18 }, + }); + + const result = await builder.build(NOISY_POST); + + expect(result.query).toBe( + 'render a custom React component from a useCopilotAction tool call', + ); + expect(result.degraded).toBe(false); + expect(result.tokenUsage).toEqual({ inputTokens: 210, outputTokens: 18 }); + }); + + it('sends the sanitized body — not the raw one — to the distiller', async () => { + mock.onMessage(/./, { + content: 'generative ui with useCopilotAction', + usage: { input_tokens: 210, output_tokens: 12 }, + }); + + await builder.build(NOISY_POST); + + const sent = JSON.stringify(mock.getLastRequest()); + expect(sent).not.toContain('1205139168783503400'); + expect(sent).not.toContain('Pre-flight Checklist'); + expect(sent).not.toContain('I have searched existing issues'); + }); + + it('never returns the raw body as the query', async () => { + mock.onMessage(/./, { + content: 'generative ui with useCopilotAction', + usage: { input_tokens: 210, output_tokens: 12 }, + }); + + const result = await builder.build(NOISY_POST); + + expect(result.query.length).toBeLessThan(NOISY_POST.length / 4); + expect(result.query).not.toContain('<#'); + expect(result.query).not.toContain('┃'); + }); + + it('falls back to the heuristic when the distiller fails', async () => { + mock.onMessage(/./, { error: { message: 'upstream unavailable' }, status: 503 }); + + const result = await builder.build(NOISY_POST); + + expect(result.degraded).toBe(true); + expect(result.query).toContain('how do I render a custom React component'); + expect(result.tokenUsage).toEqual({ inputTokens: 0, outputTokens: 0 }); + }); + + it('falls back to the heuristic when the distiller returns nothing', async () => { + mock.onMessage(/./, { content: ' ', usage: { input_tokens: 10, output_tokens: 0 } }); + + const result = await builder.build(NOISY_POST); + + expect(result.degraded).toBe(true); + expect(result.query).not.toBe(''); + }); + + it('truncates an over-long distilled query', async () => { + mock.onMessage(/./, { + content: 'query '.repeat(200), + usage: { input_tokens: 210, output_tokens: 400 }, + }); + + const result = await builder.build(NOISY_POST); + + expect(result.query.length).toBeLessThanOrEqual(300); + }); +}); diff --git a/packages/outpost/ai/src/query.ts b/packages/outpost/ai/src/query.ts new file mode 100644 index 00000000..701a7729 --- /dev/null +++ b/packages/outpost/ai/src/query.ts @@ -0,0 +1,169 @@ +import Anthropic from '@anthropic-ai/sdk'; +import { sanitizePlatformMarkup } from '@copilotkit/outpost/shared'; +import type { TokenUsage } from './types.js'; +import { config } from './config.js'; + +const DISTILL_SYSTEM_PROMPT = `You turn a raw customer support message into a documentation search query for CopilotKit, an open-source AI framework. + +Respond with ONLY the search query — no quotes, no preamble, no explanation. + +Rules: +- Capture what the person is actually trying to do or what is failing. +- Keep the specific API names, package names, error types, and framework names they mentioned. +- Drop greetings, pleasantries, issue-template boilerplate, environment dumps, stack traces, and anything pasted by accident. +- Write it as a short natural-language query, not keywords separated by commas. +- Maximum 30 words. If the message asks several things, cover the primary one.`; + +/** + * A message this short and this plain is already a usable query — distilling it + * would cost a round trip to say the same thing back. + */ +const DISTILL_SKIP_CHARS = 200; + +/** Hard cap on the query handed to Pathfinder, whatever produced it. */ +const MAX_QUERY_CHARS = 300; + +/** Result of turning a raw inbound body into a docs-search query. */ +export interface SearchQuery { + /** The focused query to send to Pathfinder. */ + query: string; + /** The full message with platform markup and boilerplate removed. */ + sanitized: string; + /** True when the LLM distiller was unavailable and the heuristic ran instead. */ + degraded: boolean; + tokenUsage: TokenUsage; +} + +/** Fenced and inline code — signal for the answer, noise for the embedder. */ +const CODE_FENCE = /```[\s\S]*?```/g; +const INLINE_CODE = /`[^`\n]+`/g; +const BARE_URL = /https?:\/\/\S+/g; +const STACK_FRAME = /^\s*at\s+\S+.*$/gm; + +/** + * Build a docs-search query from a sanitized body without calling an LLM. + * + * Prefers the sentences that carry a question mark — in a long forum post the + * question is almost always the part with the `?` — and otherwise falls back to + * the opening prose. + */ +export function heuristicSearchQuery(sanitized: string): string { + const prose = sanitized + .replace(CODE_FENCE, ' ') + .replace(STACK_FRAME, ' ') + .replace(INLINE_CODE, ' ') + .replace(BARE_URL, ' ') + .replace(/[^\S\n]+/g, ' ') + .trim(); + + if (!prose) return ''; + + // Split on blank lines first: a paragraph break ends a thought, but a bare + // newline usually does not — hard-wrapped issue bodies routinely break a + // single question across two lines. + const sentences = prose + .split(/\n\s*\n/) + .flatMap((paragraph) => { + const joined = paragraph.replace(/\s+/g, ' ').trim(); + return joined.match(/[^.!?]+[.!?]+|[^.!?]+$/g) ?? []; + }) + .map((sentence) => sentence.trim()) + .filter(Boolean); + + if (sentences.length === 0) return truncateQuery(prose.replace(/\s+/g, ' ')); + + const questions = sentences.filter((sentence) => sentence.includes('?')); + const picked = questions.length > 0 ? questions : sentences; + + return truncateQuery(picked.join(' ').replace(/\s+/g, ' ').trim()); +} + +/** Trim to MAX_QUERY_CHARS on a word boundary. */ +function truncateQuery(query: string): string { + if (query.length <= MAX_QUERY_CHARS) return query; + const clipped = query.slice(0, MAX_QUERY_CHARS); + const lastSpace = clipped.lastIndexOf(' '); + return (lastSpace > MAX_QUERY_CHARS / 2 ? clipped.slice(0, lastSpace) : clipped).trim(); +} + +/** + * Turns a raw inbound message body into a focused documentation-search query. + * + * Two stages, mirroring TicketClassifier's shape: sanitize the platform markup + * deterministically, then distill the remaining prose with Claude Haiku, with a + * heuristic fallback so retrieval never fails just because the LLM is down. + * + * Without this the entire body — up to 8000 characters of mentions, sidebar + * pastes, and checklist boilerplate — was embedded as the search query. + */ +export class SearchQueryBuilder { + private client: Anthropic; + private model: string; + + constructor(options?: { apiKey?: string; model?: string }) { + this.client = new Anthropic({ + apiKey: options?.apiKey ?? config.anthropicApiKey, + }); + this.model = options?.model ?? config.queryDistillerModel; + } + + /** + * Sanitize, then distill. Never throws — a failed distillation degrades to + * the heuristic rather than dropping the search. + */ + async build(rawQuestion: string): Promise { + const sanitized = sanitizePlatformMarkup(rawQuestion); + const noTokens: TokenUsage = { inputTokens: 0, outputTokens: 0 }; + + if (!sanitized) { + return { query: '', sanitized, degraded: false, tokenUsage: noTokens }; + } + + // Short, already-focused messages are their own best query. + if (sanitized.length <= DISTILL_SKIP_CHARS) { + return { + query: truncateQuery(sanitized.replace(/\s+/g, ' ')), + sanitized, + degraded: false, + tokenUsage: noTokens, + }; + } + + try { + const message = await this.client.messages.create({ + model: this.model, + max_tokens: config.maxQueryDistillerTokens, + temperature: config.queryDistillerTemperature, + system: DISTILL_SYSTEM_PROMPT, + messages: [{ role: 'user', content: sanitized.slice(0, 4000) }], + }); + + const text = message.content[0]?.type === 'text' ? message.content[0].text : ''; + const distilled = truncateQuery(text.replace(/\s+/g, ' ').trim()); + + if (!distilled) { + throw new Error('distiller returned an empty query'); + } + + return { + query: distilled, + sanitized, + degraded: false, + tokenUsage: { + inputTokens: message.usage.input_tokens, + outputTokens: message.usage.output_tokens, + }, + }; + } catch (error) { + console.error( + `[SearchQuery] Distillation failed, falling back to heuristic: ${error instanceof Error ? error.message : String(error)}`, + ); + return { + query: heuristicSearchQuery(sanitized), + sanitized, + degraded: true, + tokenUsage: noTokens, + }; + } + } +} diff --git a/packages/outpost/shared/src/__tests__/text.test.ts b/packages/outpost/shared/src/__tests__/text.test.ts new file mode 100644 index 00000000..eb11e31f --- /dev/null +++ b/packages/outpost/shared/src/__tests__/text.test.ts @@ -0,0 +1,124 @@ +import { describe, it, expect } from 'vitest'; +import { sanitizePlatformMarkup, isSupportRequest } from '../text.js'; + +/** + * A forum post that carries every noise source we have seen reach the docs + * embedder verbatim: Discord markup, a pasted channel sidebar, and GitHub + * issue-template boilerplate. + */ +const RAW_DISCORD_POST = [ + '<:copilotkit:1187213988392189962> hey <@!284920412034990081> :wave:', + '', + 'I posted this in <#1205139168783503400> already but reposting here since <#1379082175625953370> said this is the right place.', + '', + 'Channels', + '# ┃welcome', + '# ┃announcements', + '# ┃support', + 'Voice Channels', + '🔊 Lounge', + '', + '## Pre-flight Checklist', + '- [x] I have searched existing issues', + '- [ ] I am willing to submit a PR', + '', + '### ♻️ Reproduction Steps', + '1. npx create-next-app', + '', + 'How do I render a custom React component from a tool call? ', +].join('\n'); + +describe('sanitizePlatformMarkup', () => { + it('returns an empty string for empty input', () => { + expect(sanitizePlatformMarkup('')).toBe(''); + }); + + it('strips Discord channel, user, and role mentions', () => { + const out = sanitizePlatformMarkup('see <#123> and ask <@!456> or <@&789>'); + expect(out).not.toMatch(/<[#@]/); + expect(out).toContain('see'); + expect(out).toContain('and ask'); + }); + + it('strips Discord custom emoji, shortcodes, and timestamps', () => { + const out = sanitizePlatformMarkup( + '<:ck:1187213988392189962> shipped :tada: at ', + ); + expect(out).toBe('shipped at'); + }); + + it('keeps Slack link and channel labels while dropping the markup', () => { + expect(sanitizePlatformMarkup('read ')).toBe( + 'read the docs', + ); + expect(sanitizePlatformMarkup('ask in <#C123ABC|support>')).toBe('ask in support'); + expect(sanitizePlatformMarkup('cc <@U123ABC> ')).toBe('cc'); + }); + + it('drops issue-template checklists, HTML comments, and boilerplate headings', () => { + const out = sanitizePlatformMarkup( + [ + '', + '## Pre-flight Checklist', + '- [x] I have searched existing issues', + '### Reproduction Steps', + 'run the dev server', + ].join('\n'), + ); + expect(out).not.toContain('Pre-flight'); + expect(out).not.toContain('searched existing issues'); + expect(out).not.toContain('please fill this in'); + // Headings that carry signal survive, without their markers. + expect(out).toContain('Reproduction Steps'); + expect(out).toContain('run the dev server'); + }); + + it('drops pasted channel-sidebar rows', () => { + const out = sanitizePlatformMarkup(RAW_DISCORD_POST); + expect(out).not.toContain('┃welcome'); + expect(out).not.toContain('┃announcements'); + expect(out).not.toContain('🔊 Lounge'); + }); + + it('leaves the reporter question intact and shrinks the body substantially', () => { + const out = sanitizePlatformMarkup(RAW_DISCORD_POST); + expect(out).toContain('How do I render a custom React component from a tool call?'); + expect(out.length).toBeLessThan(RAW_DISCORD_POST.length / 2); + }); + + it('leaves fenced code untouched', () => { + const withCode = ['here:', '```ts', 'const x = a <@ b; // not a mention', '```'].join('\n'); + expect(sanitizePlatformMarkup(withCode)).toContain('const x = a <@ b; // not a mention'); + }); +}); + +describe('isSupportRequest', () => { + it('accepts anything with a question mark', () => { + expect(isSupportRequest('does this work with Next.js?')).toBe(true); + }); + + it('accepts help phrasing without a question mark', () => { + expect(isSupportRequest('I am stuck wiring up useCopilotAction')).toBe(true); + expect(isSupportRequest('the runtime is not working after the upgrade')).toBe(true); + }); + + it('accepts error signatures', () => { + expect(isSupportRequest('TypeError: Cannot read properties of undefined')).toBe(true); + expect(isSupportRequest(' at handler (/app/src/index.ts:12:5)')).toBe(true); + }); + + it('rejects pure announcements', () => { + expect(isSupportRequest('v1.10.0 is out. Release notes in the changelog.')).toBe(false); + expect(isSupportRequest('Office hours start in 10 minutes. See you there!')).toBe(false); + }); + + it('rejects an empty or markup-only body', () => { + expect(isSupportRequest('')).toBe(false); + expect(isSupportRequest('<:tada:123> <@!456>')).toBe(false); + }); + + it('accepts a direct @-mention of the bot even without help phrasing', () => { + expect(isSupportRequest('<@!999> take a look', { botUserId: '999' })).toBe(true); + expect(isSupportRequest('<@!111> take a look', { botUserId: '999' })).toBe(false); + }); +}); diff --git a/packages/outpost/shared/src/index.ts b/packages/outpost/shared/src/index.ts index 7f2394f6..5f42cca4 100644 --- a/packages/outpost/shared/src/index.ts +++ b/packages/outpost/shared/src/index.ts @@ -1,6 +1,7 @@ export * from './types.js'; export * from './constants.js'; export * from './utils.js'; +export * from './text.js'; export * from './dispatch/index.js'; export * from './sla/index.js'; export * from './onboarding/index.js'; diff --git a/packages/outpost/shared/src/text.ts b/packages/outpost/shared/src/text.ts new file mode 100644 index 00000000..ffea08ec --- /dev/null +++ b/packages/outpost/shared/src/text.ts @@ -0,0 +1,188 @@ +/** + * Inbound-text hygiene — shared by every platform. + * + * Raw message bodies arrive full of platform markup (Discord mention/emoji + * tokens, Slack link syntax) and issue-template boilerplate (pre-flight + * checklists, HTML comments, pasted channel sidebars). None of it is signal: + * forwarded verbatim it dilutes docs-search embeddings and pads generation + * prompts. The only sanitizer that existed before this ran on the OUTBOUND + * reply (see ai/formatter.ts), so nothing cleaned the inbound side. + * + * Everything here is pure and dependency-free so bots, the queue, and the AI + * package can all share it. + */ + +/** Discord custom emoji: `<:name:id>` / `` (animated). */ +const DISCORD_CUSTOM_EMOJI = //gi; + +/** Discord channel mention: `<#123>`. */ +const DISCORD_CHANNEL_MENTION = /<#\d+>/g; + +/** Discord user/role mention: `<@123>`, `<@!123>`, `<@&123>`. */ +const DISCORD_USER_MENTION = /<@[!&]?\d+>/g; + +/** Discord relative timestamp: ``. */ +const DISCORD_TIMESTAMP = //g; + +/** Slack channel mention with label: `<#C123|general>` -> `general`. */ +const SLACK_CHANNEL_MENTION = /<#[CG][A-Z0-9]+\|([^>]*)>/g; + +/** Slack user or user-group mention: `<@U123>`, `<@W123|name>`. */ +const SLACK_USER_MENTION = /<@[UWG][A-Z0-9]+(?:\|[^>]*)?>/g; + +/** Slack broadcast: ``, ``, ``. */ +const SLACK_BROADCAST = /]*)?>/gi; + +/** Slack link with label: `` -> `label`. */ +const SLACK_LABELLED_LINK = /<(https?:\/\/[^|>\s]+)\|([^>]*)>/g; + +/** Slack bare link: `` -> `https://x`. */ +const SLACK_BARE_LINK = /<(https?:\/\/[^>\s]+)>/g; + +/** Emoji shortcode standing alone as a token: ` :wave: `. */ +const EMOJI_SHORTCODE = /(^|\s):[a-z0-9_+-]{2,32}:(?=\s|$)/gi; + +/** HTML comment — issue templates hide their instructions in these. */ +const HTML_COMMENT = //g; + +/** Markdown task-list line: `- [x] I have searched existing issues`. */ +const TASK_LIST_LINE = /^\s*[-*]\s*\[[ xX]\]\s.*$/; + +/** + * Zero-width joiners and variation selectors. Dropped up front so the + * decoration patterns below can reason about single code points. + */ +const ZERO_WIDTH = /\u200D|[\uFE00-\uFE0F]/g; + +/** Leading markdown heading markers plus any decorative emoji: `### Steps`. */ +const HEADING_PREFIX = /^\s{0,3}#{1,6}[\s\p{Extended_Pictographic}]*/u; + +/** + * A line made up only of decoration — symbols, punctuation, and the box-drawing + * glyphs Discord uses in its channel sidebar, with no words. The ranges are + * U+2000-206F general punctuation and U+2190-2BFF arrows through miscellaneous + * symbols, which covers box drawing at U+2500-257F. + */ +const DECORATION_ONLY_LINE = + /^[\s#*>|\-_=~.,:;!?()[\]{}\u2000-\u206F\u2190-\u2BFF\p{Extended_Pictographic}]*$/u; + +/** + * A pasted Discord channel-sidebar row. The leading vertical box-drawing glyph + * (U+2502-254B) and the speaker emoji (U+1F508-1F50A, U+1F4E2) are the sidebar's + * own decoration — no prose line starts with either. + */ +const SIDEBAR_ROW = /^[\s#*>-]*(?:[\u2502-\u254B]|[\u{1F4E2}\u{1F508}-\u{1F50A}])\s*\S/u; + +/** + * Issue-template section headings that never carry the reporter's question. + * Matched case-insensitively against the heading text after markers and emoji + * are stripped. Deliberately short — headings like "Reproduction Steps" or + * "Describe the bug" DO carry signal and must survive. + */ +const BOILERPLATE_HEADINGS = new Set([ + 'pre-flight checklist', + 'preflight checklist', + 'checklist', + 'code of conduct', + 'terms', + 'prerequisites', +]); + +/** + * Strip platform markup and issue-template boilerplate from an inbound message + * body, leaving the reporter's own prose and code intact. + * + * Conservative by design: it removes tokens that are unambiguously markup, not + * anything that might be content. Semantic narrowing (picking the actual + * question out of a long post) is a separate concern — see the AI package's + * SearchQueryBuilder. + */ +export function sanitizePlatformMarkup(text: string): string { + if (!text) return ''; + + const stripped = text + .replace(ZERO_WIDTH, '') + .replace(HTML_COMMENT, ' ') + .replace(SLACK_LABELLED_LINK, '$2') + .replace(SLACK_BARE_LINK, '$1') + .replace(SLACK_CHANNEL_MENTION, '$1') + .replace(SLACK_USER_MENTION, ' ') + .replace(SLACK_BROADCAST, ' ') + .replace(DISCORD_CUSTOM_EMOJI, ' ') + .replace(DISCORD_CHANNEL_MENTION, ' ') + .replace(DISCORD_USER_MENTION, ' ') + .replace(DISCORD_TIMESTAMP, ' ') + .replace(EMOJI_SHORTCODE, '$1'); + + const kept: string[] = []; + let insideCodeFence = false; + + for (const line of stripped.split('\n')) { + // Code fences are content — pass every line through untouched while open. + if (/^\s*```/.test(line)) { + insideCodeFence = !insideCodeFence; + kept.push(line); + continue; + } + if (insideCodeFence) { + kept.push(line); + continue; + } + + if (TASK_LIST_LINE.test(line)) continue; + if (SIDEBAR_ROW.test(line)) continue; + + const headingPrefix = line.match(HEADING_PREFIX)?.[0]; + if (headingPrefix !== undefined && headingPrefix.trimStart().startsWith('#')) { + const headingText = line.slice(headingPrefix.length).trim(); + if (BOILERPLATE_HEADINGS.has(headingText.toLowerCase())) continue; + if (headingText) kept.push(headingText); + continue; + } + + if (line.trim() && DECORATION_ONLY_LINE.test(line)) continue; + + kept.push(line); + } + + // Collapse the runs of blank lines and spaces the removals left behind. + return kept + .join('\n') + .replace(/[^\S\n]+/g, ' ') + .replace(/ +\n/g, '\n') + .replace(/\n{3,}/g, '\n\n') + .trim(); +} + +/** Phrasings that mark a message as asking for help rather than announcing. */ +const HELP_PATTERNS: RegExp[] = [ + /\b(how (do|can|to|would|should)|what('s| is| are)|is there|is it possible|any (idea|one|body)|does any)/i, + /\b(help|stuck|confused|struggling|trying to|unable to|can'?t|cannot|won'?t|doesn'?t|didn'?t)\b/i, + /\b(not working|no luck|fails?|failing|failed|broken|crash(es|ing)?|hangs?)\b/i, + /\b(error|exception|traceback|stack ?trace)\b/i, + /\b(expected .{0,40}(but|instead)|instead of|but it|however it)\b/i, + /(TypeError|ReferenceError|SyntaxError|RangeError|ENOENT|ECONNREFUSED)/, + /^\s*at\s+\S+\s*\(.*:\d+:\d+\)/m, +]; + +/** + * Decide whether an inbound message is a support request worth spending a full + * retrieval + generation cycle on. + * + * Deliberately permissive — a missed announcement costs nothing, a missed + * support request costs a customer. Anything with a question mark, a help + * phrasing, an error signature, or a direct @-mention of the bot qualifies. + */ +export function isSupportRequest(content: string, options?: { botUserId?: string }): boolean { + // The bot-mention check runs on the RAW text: sanitizing strips mentions. + if (options?.botUserId && new RegExp(`<@[!&]?${options.botUserId}>`).test(content)) { + return true; + } + + const cleaned = sanitizePlatformMarkup(content); + if (!cleaned) return false; + + if (cleaned.includes('?')) return true; + + return HELP_PATTERNS.some((pattern) => pattern.test(cleaned)); +}