From c91a170219693a90fccae351d7223536d0fb2f6b Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Sat, 12 Sep 2026 11:07:02 -0700 Subject: [PATCH 1/3] Stop the GitHub triage relay amplifying issue spam Two throwaway accounts filed 23 SEO backlink issues on CopilotKit/CopilotKit on 2026-09-10/11. Outpost relayed each ~3.3 KB body verbatim into search-docs and search-code on mcp.copilotkit.ai within ~4s of creation and answered them publicly, producing 46 query_log rows that then ranked in Top Queries and fed the weekly Notion report and the monthly gap-analysis LLM prompt. Three levers: - A link-spam gate in the issues.opened webhook, BEFORE a ticket exists, so no AI job is enqueued and nothing is relayed or posted. Measured on the full issue history of CopilotKit/CopilotKit: 23/23 spam, 0/1264 legitimate. - A hard cap on the MCP search query. Content-independent, so it bounds the next campaign too; the generator still sees the full body. - X-Pathfinder-Source on the MCP initialize request, so this relay's traffic is attributable and excludable downstream. --- .../src/__tests__/spam-filter.test.ts | 145 ++++++++++++++++++ apps/github-app/src/lib/spam-filter.ts | 129 ++++++++++++++++ apps/github-app/src/webhooks/issues-opened.ts | 22 +++ packages/outpost/ai/src/config.ts | 24 +++ packages/outpost/ai/src/pathfinder.test.ts | 127 ++++++++++++++- packages/outpost/ai/src/pathfinder.ts | 55 +++++-- 6 files changed, 490 insertions(+), 12 deletions(-) create mode 100644 apps/github-app/src/__tests__/spam-filter.test.ts create mode 100644 apps/github-app/src/lib/spam-filter.ts diff --git a/apps/github-app/src/__tests__/spam-filter.test.ts b/apps/github-app/src/__tests__/spam-filter.test.ts new file mode 100644 index 00000000..044902d5 --- /dev/null +++ b/apps/github-app/src/__tests__/spam-filter.test.ts @@ -0,0 +1,145 @@ +import { describe, it, expect } from 'vitest'; +import { + isLikelySpamIssue, + hasFencedCode, + maxLinksToOneThirdPartyHost, +} from '../lib/spam-filter.js'; + +/** + * Shaped after the 2026-09-10 campaign: ~3.3 KB of marketing prose, no code, + * the same third-party host linked over and over. + */ +function spamBody(host = '1rank.app', links = 8): string { + const para = + 'Search visibility is the difference between a business that gets found and one ' + + 'that does not. Modern buyers start with a query, and the page they click is the ' + + 'page that answers it fastest. '; + const linked = Array.from( + { length: links }, + (_, i) => `Read more at [our guide ${i}](https://${host}/guide-${i}). `, + ).join(''); + let body = linked; + while (body.length < 3300) body += para; + return body; +} + +/** A long, link-heavy, genuinely useful bug report — the class that must survive. */ +function bugReportBody(): string { + return [ + '### Reproduction', + '', + 'Repro repo: https://github.com/someone/repro', + 'Related: https://github.com/CopilotKit/CopilotKit/issues/1', + 'Docs I followed: https://docs.copilotkit.ai/quickstart', + 'Upstream bug: https://github.com/langchain-ai/langgraph/issues/9', + 'Stackblitz: https://stackblitz.com/edit/a', + 'Sandbox: https://codesandbox.io/s/b', + '', + '```ts', + 'const { visibleMessages } = useCopilotChat();', + 'console.log(visibleMessages);', + '```', + '', + 'x'.repeat(3000), + ].join('\n'); +} + +describe('maxLinksToOneThirdPartyHost', () => { + it('counts repeats of the same third-party host', () => { + expect( + maxLinksToOneThirdPartyHost('a https://1rank.app/x b https://1rank.app/y'), + ).toBe(2); + }); + + it('does not count our own hosts or GitHub, at any subdomain', () => { + const body = [ + 'https://github.com/a', + 'https://github.com/b', + 'https://raw.githubusercontent.com/c', + 'https://docs.copilotkit.ai/d', + 'https://copilotkit.ai/e', + 'http://localhost:3000/f', + ].join(' '); + expect(maxLinksToOneThirdPartyHost(body)).toBe(0); + }); + + it('normalises www. and a port so one host is not counted as three', () => { + expect( + maxLinksToOneThirdPartyHost( + 'https://www.1rank.app/a https://1rank.app/b https://1rank.app:443/c', + ), + ).toBe(3); + }); + + it('returns the MAX for one host, not the total across hosts', () => { + // Six links, but spread three ways — nothing is being advertised. + const body = 'https://a.io/1 https://a.io/2 https://b.io/1 https://b.io/2 https://c.io/1 https://c.io/2'; + expect(maxLinksToOneThirdPartyHost(body)).toBe(2); + }); +}); + +describe('hasFencedCode', () => { + it('is true for an opened-and-closed block', () => { + expect(hasFencedCode('text\n```ts\ncode\n```\n')).toBe(true); + }); + + it('is false for a single stray fence', () => { + expect(hasFencedCode('text ``` more text')).toBe(false); + }); +}); + +describe('isLikelySpamIssue', () => { + it('flags the campaign shape: untrusted, long, no code, one host repeated', () => { + expect(isLikelySpamIssue({ body: spamBody(), authorAssociation: 'NONE' })).toBe(true); + }); + + // Every AND term gets its own negative case: a gate that silently eats real + // reports is worse than the spam it was built for, because nothing on the + // issue records that a decision was made. + it('never flags a trusted author, whatever they write', () => { + for (const assoc of ['OWNER', 'MEMBER', 'COLLABORATOR', 'CONTRIBUTOR']) { + expect(isLikelySpamIssue({ body: spamBody(), authorAssociation: assoc })).toBe(false); + } + }); + + it('matches author_association case-insensitively', () => { + expect(isLikelySpamIssue({ body: spamBody(), authorAssociation: 'member' })).toBe(false); + }); + + it('does not flag a long, link-heavy bug report that carries a repro', () => { + expect(isLikelySpamIssue({ body: bugReportBody(), authorAssociation: 'NONE' })).toBe( + false, + ); + }); + + it('does not flag a short body even if it is all links', () => { + expect( + isLikelySpamIssue({ body: spamBody().slice(0, 1200), authorAssociation: 'NONE' }), + ).toBe(false); + }); + + it('does not flag a long body whose links are spread across hosts', () => { + const body = + Array.from({ length: 12 }, (_, i) => `https://host${i}.example/x `).join('') + + 'y'.repeat(3000); + expect(isLikelySpamIssue({ body, authorAssociation: 'NONE' })).toBe(false); + }); + + it('does not flag a long body that only links GitHub and our docs', () => { + const body = + Array.from({ length: 12 }, (_, i) => `https://github.com/CopilotKit/x/issues/${i} `) + .join('') + 'y'.repeat(3000); + expect(isLikelySpamIssue({ body, authorAssociation: 'NONE' })).toBe(false); + }); + + it('handles an empty or missing body without throwing', () => { + expect(isLikelySpamIssue({ body: '', authorAssociation: 'NONE' })).toBe(false); + expect(isLikelySpamIssue({ body: null, authorAssociation: 'NONE' })).toBe(false); + expect(isLikelySpamIssue({ body: undefined, authorAssociation: undefined })).toBe(false); + }); + + it('treats a missing author_association as untrusted, not as trusted', () => { + // Fail closed on the spam side: an absent field must not become a bypass. + expect(isLikelySpamIssue({ body: spamBody(), authorAssociation: null })).toBe(true); + }); +}); diff --git a/apps/github-app/src/lib/spam-filter.ts b/apps/github-app/src/lib/spam-filter.ts new file mode 100644 index 00000000..d1952715 --- /dev/null +++ b/apps/github-app/src/lib/spam-filter.ts @@ -0,0 +1,129 @@ +/** + * Link-spam gate for inbound GitHub issues. + * + * Why this exists. On 2026-09-10/11 two throwaway accounts + * (`sarahnicholas1327-lgtm`, `kaylaford203-beep`) filed 23 SEO backlink issues on + * CopilotKit/CopilotKit — ~3.3 KB of marketing prose each, every one carrying a + * dozen-odd links to `1rank.app` / `zagfro.com`. Outpost relayed each body + * VERBATIM into `search-docs` + `search-code` on mcp.copilotkit.ai within ~4 + * seconds of creation, then answered the issue publicly. That is amplification: + * the spammer wrote once and got a machine-generated reply and 46 rows of + * retrieval traffic for free, and the blobs went on to pollute Pathfinder's Top + * Queries panel, the weekly Notion report, and the monthly gap-analysis LLM + * prompt. + * + * The gate is deliberately narrow. Its job is to drop a body that is a link + * advertisement, NOT to score quality — a bad bug report still deserves an + * answer, and a false positive here is silent (no ticket, no reply, no trace on + * the issue). All four conditions must hold. + * + * Measured on the FULL issue history of CopilotKit/CopilotKit (1,264 non-spam + * issues by 1,264-issue authorship, plus all 23 known spam issues): + * + * | rule | spam | legit | + * |-------------------------------------------------------------|-------|----------| + * | body >= 3000 chars AND >= 6 urls | 23/23 | 15/1264 | + * | + untrusted author | 23/23 | 14/1264 | + * | THIS RULE (untrusted, long, no code fence, one host >= 5x) | 23/23 | 0/1264 | + * + * A length-and-url rule alone is NOT sufficient: it eats real bug reports + * (#2667 is 40 KB with 16 urls, #3510 is 13 KB with 26). The two terms that buy + * the separation are the ones that describe an advertisement rather than a + * report: it contains no fenced code block, and its links point over and over at + * the SAME third-party host. A stack trace or a repro has code in it; a backlink + * campaign does not, because the links are the payload. + * + * Honest scope: this is fitted on one campaign and one repo's history. It is a + * discriminator, not a general spam classifier, and the next campaign may look + * different. The truncation and `X-Pathfinder-Source` changes in + * `packages/outpost/ai/src/pathfinder.ts` are the content-independent half of + * the defence and do not depend on this rule firing. + */ + +/** + * `author_association` values that mean the author has a real relationship with + * the repo. A spam account is `NONE` by construction — it has never had a PR or + * a commit merged. Requiring untrusted authorship means a maintainer or a prior + * contributor can never be silenced by this gate, whatever they write. + * + * `CONTRIBUTOR` is included as trusted: it means GitHub has already seen a + * merged commit from this account in this repo. + */ +const TRUSTED_ASSOCIATIONS: ReadonlySet = new Set([ + 'OWNER', + 'MEMBER', + 'COLLABORATOR', + 'CONTRIBUTOR', +]); + +/** + * Hosts that do not count towards "links at one third-party host". Our own docs + * and GitHub itself are what a legitimate issue links to repeatedly — a reporter + * citing eight `github.com/...` permalinks is doing exactly the right thing. + * Matched on the host or any subdomain of it. + */ +const FIRST_PARTY_HOSTS: readonly string[] = [ + 'github.com', + 'githubusercontent.com', + 'copilotkit.ai', + 'ag-ui.com', + 'localhost', +]; + +/** Minimum body length before the gate will consider anything. */ +const MIN_BODY_CHARS = 1500; + +/** Minimum links to ONE third-party host before the body reads as an ad. */ +const MIN_LINKS_TO_ONE_HOST = 5; + +function isFirstParty(host: string): boolean { + return FIRST_PARTY_HOSTS.some((h) => host === h || host.endsWith(`.${h}`)); +} + +/** + * The largest number of links in `body` pointing at a single third-party host. + * + * Counts raw URL occurrences rather than markdown links: the spam bodies mix + * `[anchor](url)` with bare urls and with the same domain written as plain text + * inside a sentence, and only the total is stable across those spellings. + */ +export function maxLinksToOneThirdPartyHost(body: string): number { + const counts = new Map(); + for (const match of body.matchAll(/https?:\/\/([^/\s)>\]"'`]+)/gi)) { + const host = match[1].toLowerCase().replace(/^www\./, '').replace(/:\d+$/, ''); + if (isFirstParty(host)) continue; + counts.set(host, (counts.get(host) ?? 0) + 1); + } + let max = 0; + for (const n of counts.values()) max = Math.max(max, n); + return max; +} + +/** Whether `body` contains at least one fenced code block. */ +export function hasFencedCode(body: string): boolean { + // Two fence markers, i.e. an opened AND closed block. A single stray "```" + // is not evidence of a repro. + return (body.match(/```/g)?.length ?? 0) >= 2; +} + +export interface SpamCandidate { + /** The issue body as filed. */ + body: string | null | undefined; + /** The webhook payload's `issue.author_association`. */ + authorAssociation: string | null | undefined; +} + +/** + * Whether this issue is a link advertisement that must NOT be relayed. + * + * Returns false for anything it is not sure about: an empty body, a trusted + * author, a short body, a body with a repro in it, or a body whose links are + * spread across hosts. + */ +export function isLikelySpamIssue({ body, authorAssociation }: SpamCandidate): boolean { + if (!body) return false; + if (TRUSTED_ASSOCIATIONS.has((authorAssociation ?? '').toUpperCase())) return false; + if (body.length < MIN_BODY_CHARS) return false; + if (hasFencedCode(body)) return false; + return maxLinksToOneThirdPartyHost(body) >= MIN_LINKS_TO_ONE_HOST; +} diff --git a/apps/github-app/src/webhooks/issues-opened.ts b/apps/github-app/src/webhooks/issues-opened.ts index d0c1d85a..be7182ea 100644 --- a/apps/github-app/src/webhooks/issues-opened.ts +++ b/apps/github-app/src/webhooks/issues-opened.ts @@ -5,6 +5,7 @@ import { InboundHandler, GitHubPlatformAdapter } from '@copilotkit/outpost/share import type { InboundPrismaLike, CreateJobFn } from '@copilotkit/outpost/shared'; import { getOctokit } from '../lib/github-client.js'; import { isRepoAllowed } from '../lib/repo-allowlist.js'; +import { isLikelySpamIssue } from '../lib/spam-filter.js'; import { config } from '../config.js'; export async function handleIssueOpened( @@ -24,6 +25,27 @@ export async function handleIssueOpened( return; } + // Drop link-spam BEFORE a ticket exists. This is the gate, not a + // post-filter: creating the ticket is what enqueues the AI job, and that job + // is what forwards the body verbatim into `search-docs`/`search-code` on + // mcp.copilotkit.ai and then posts a public reply. Returning here means the + // spammer gets nothing — no answer to point at, no retrieval traffic, no row + // in the analytics that feed the weekly report and the gap-analysis prompt. + // See lib/spam-filter.ts for the rule and the measurements behind it. + if ( + isLikelySpamIssue({ + body: issue.body, + authorAssociation: issue.author_association, + }) + ) { + console.log( + `[GitHub App] Ignoring link-spam issue ${repository.full_name}#${issue.number} ` + + `by ${sender.login} (${issue.body?.length ?? 0} chars, untrusted author, ` + + `no code block, links concentrated at one third-party host)`, + ); + return; + } + try { const adapter = new GitHubPlatformAdapter({ octokit: getOctokit() }); const message = adapter.parseInboundEvent({ diff --git a/packages/outpost/ai/src/config.ts b/packages/outpost/ai/src/config.ts index 01dee226..3f80d44a 100644 --- a/packages/outpost/ai/src/config.ts +++ b/packages/outpost/ai/src/config.ts @@ -73,6 +73,30 @@ export const config = { sessionTtlMs: 30 * 60 * 1000, /** Reconnect grace period before TTL expiry (5 minutes) */ refreshBeforeExpiryMs: 5 * 60 * 1000, + /** + * Hard cap on the characters sent as an MCP search `query`. + * + * A retrieval query is an embedding input, not a transcript: the issue + * body still reaches the generator in full, only the SEARCH string is + * capped. Measured on 7 days of Pathfinder's `query_log`, the longest + * query from any client that is NOT a relay is 194 characters, while the + * SEO-spam bodies this relay forwarded verbatim ran 3,304-3,631 — and + * scored a feeble 0.33-0.43 cosine for it, so the long tail was buying + * nothing. 1000 leaves ~5x headroom over every observed human query. + */ + maxQueryChars: parseInt(process.env.PATHFINDER_MAX_QUERY_CHARS ?? '1000', 10), + /** + * Value sent as `X-Pathfinder-Source` on the MCP `initialize` request. + * + * Pathfinder captures this header ONCE, at session initialisation, and + * closes over it for the session's lifetime — so it must ride on + * `initialize`, never on an individual `tools/call`. It exists so this + * relay's traffic is attributable and, more to the point, EXCLUDABLE: + * every query from here is machine traffic derived from someone else's + * text, and it should not rank in Top Queries or seed a gap-analysis + * prompt just because it was loud. + */ + sourceTag: process.env.PATHFINDER_SOURCE ?? 'outpost', }, } as const; diff --git a/packages/outpost/ai/src/pathfinder.test.ts b/packages/outpost/ai/src/pathfinder.test.ts index 4731aecc..e0f94911 100644 --- a/packages/outpost/ai/src/pathfinder.test.ts +++ b/packages/outpost/ai/src/pathfinder.test.ts @@ -1,5 +1,6 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; -import { PathfinderClient } from './pathfinder.js'; +import { PathfinderClient, capQuery } from './pathfinder.js'; +import { config } from './config.js'; // Mock global fetch const mockFetch = vi.fn(); @@ -481,3 +482,127 @@ describe('PathfinderClient', () => { }); }); }); + +/** A minimal well-formed docs reply, shared by the relay-hygiene tests. */ +const SNIPPETS = [ + 'SNIPPET 1', + 'TITLE: Actions', + 'SOURCE: https://docs.copilotkit.ai/actions', + 'CONTENT:', + 'Use useCopilotAction to register a frontend action.', +].join('\n'); + +// The SEO-spam wave of 2026-09-10/11 got here because the relay forwards a +// GitHub issue body VERBATIM as the retrieval query: 23 spam issues became 46 +// `query_log` rows of 3.3 KB marketing copy, which then ranked in Top Queries +// and seeded the gap-analysis LLM prompt. These two guards are the +// content-independent half of the fix — they bound the NEXT campaign too — so +// they are asserted at the wire, on the JSON that actually leaves the process. +describe('PathfinderClient — relay hygiene', () => { + let client: PathfinderClient; + + beforeEach(() => { + client = new PathfinderClient(BASE); + mockFetch.mockReset(); + }); + + afterEach(() => { + client.disconnect(); + }); + + it('identifies itself with X-Pathfinder-Source on initialize', async () => { + mockConnect('sess-src'); + + await client.connect(); + + const initHeaders = mockFetch.mock.calls[0][1].headers as Record; + expect(initHeaders['X-Pathfinder-Source']).toBe(config.pathfinder.sourceTag); + }); + + // Pathfinder captures the header ONCE, when the session is minted, and closes + // over it for the session's lifetime. A tool call that carried it would be + // ignored, so asserting it on `initialize` specifically is the point. + it('sends the source header on the session-minting request, not per tool call', async () => { + mockConnect('sess-src'); + mockFetch.mockResolvedValueOnce( + mkResp({ body: jsonRpc({ content: [{ type: 'text', text: SNIPPETS }] }) }), + ); + + await client.searchDocs({ query: 'actions' }); + + const toolHeaders = mockFetch.mock.calls[2][1].headers as Record; + expect(toolHeaders['Mcp-Session-Id']).toBe('sess-src'); + expect(toolHeaders['X-Pathfinder-Source']).toBeUndefined(); + }); + + it.each(['search-docs', 'search-code'] as const)( + 'caps an oversized %s query before it reaches the wire', + async (tool) => { + mockConnect(); + mockFetch.mockResolvedValueOnce( + mkResp({ body: jsonRpc({ content: [{ type: 'text', text: SNIPPETS }] }) }), + ); + + // Shaped like the real thing: long, and with no whitespace anywhere + // near the cut so the word-boundary pull-back cannot mask the cap. + const blob = 'spam '.repeat(400) + 'x'.repeat(500); + expect(blob.length).toBeGreaterThan(2000); + + if (tool === 'search-docs') { + await client.searchDocs({ query: blob }); + } else { + await client.searchCode({ query: blob }); + } + + const sent = JSON.parse(mockFetch.mock.calls[2][1].body); + expect(sent.params.name).toBe(tool); + expect(sent.params.arguments.query.length).toBeLessThanOrEqual( + config.pathfinder.maxQueryChars, + ); + // The HEAD is kept — that is where a real question lives. + expect(blob.startsWith(sent.params.arguments.query)).toBe(true); + }, + ); + + it('leaves a normal-length query untouched', async () => { + mockConnect(); + mockFetch.mockResolvedValueOnce( + mkResp({ body: jsonRpc({ content: [{ type: 'text', text: SNIPPETS }] }) }), + ); + + await client.searchDocs({ query: 'how do I self-host the runtime?' }); + + const sent = JSON.parse(mockFetch.mock.calls[2][1].body); + expect(sent.params.arguments.query).toBe('how do I self-host the runtime?'); + }); +}); + +describe('capQuery', () => { + it('returns the input unchanged when it already fits', () => { + expect(capQuery('short', 100)).toBe('short'); + }); + + it('cuts at a word boundary when one is near the cut', () => { + // Boundary at 16 of 18 — inside the trailing 15% the pull-back looks in. + const out = capQuery('alpha beta gamma delta epsilon', 18); + expect(out.length).toBeLessThanOrEqual(18); + expect(out).toBe('alpha beta gamma'); + }); + + // Deliberate: the pull-back only looks at the last 15%, so it can never + // discard a meaningful share of the query to chase a boundary. Losing a few + // characters of one token beats losing a sentence of context. + it('accepts a mid-token cut rather than reaching far back for a boundary', () => { + expect(capQuery('alpha beta gamma delta epsilon', 20)).toBe('alpha beta gamma del'); + }); + + // A single unbroken token has no boundary to fall back to; a hard cut is + // still better than shipping the whole blob. + it('hard-cuts when there is no late whitespace to fall back to', () => { + expect(capQuery('x'.repeat(50), 10)).toBe('x'.repeat(10)); + }); + + it('treats a non-positive cap as "no cap" rather than emptying the query', () => { + expect(capQuery('anything', 0)).toBe('anything'); + }); +}); diff --git a/packages/outpost/ai/src/pathfinder.ts b/packages/outpost/ai/src/pathfinder.ts index 29400051..63f0c6bb 100644 --- a/packages/outpost/ai/src/pathfinder.ts +++ b/packages/outpost/ai/src/pathfinder.ts @@ -35,6 +35,31 @@ function blobUrl(repository: string | undefined, path: string | undefined): stri return `${repo}/blob/main/${path.replace(/^\/+/, '')}`; } +/** + * Cap what goes out as an MCP search `query`. + * + * The relay forwards a whole GitHub issue body as the retrieval string. That is + * wrong twice over. It is bad retrieval — a 3.3 KB marketing blob scored 0.33 + * cosine against our docs, worse than the one-line questions it sits beside — + * and it is an amplification channel: whatever an anonymous stranger types + * arrives verbatim in Pathfinder's `query_log`, its Top Queries panel, the + * weekly Notion report, and the monthly gap-analysis LLM prompt. Capping it is + * content-independent: it bounds the NEXT campaign too, whatever it advertises. + * + * The head is kept rather than the tail because the opening sentences are where + * the question lives — a bug report leads with the symptom and trails into + * environment dumps. The cut is pulled back to the last whitespace in the final + * 15% so a query does not end mid-token, which is noise to an embedding. + * + * Exported for the test that proves the cap actually reaches the wire. + */ +export function capQuery(query: string, maxChars: number): string { + if (maxChars <= 0 || query.length <= maxChars) return query; + const head = query.slice(0, maxChars); + const lastSpace = head.search(/\s\S*$/); + return (lastSpace > maxChars * 0.85 ? head.slice(0, lastSpace) : head).trimEnd(); +} + /** * The Pathfinder search tools, verified against `tools/list` on * https://mcp.copilotkit.ai/mcp. All four take the same arguments @@ -96,16 +121,22 @@ export class PathfinderClient { // would fall back forever. this.reset(); - const { body, sessionId } = await this.post({ - jsonrpc: '2.0', - id: this.nextId++, - method: 'initialize', - params: { - protocolVersion: '2024-11-05', - capabilities: {}, - clientInfo: { name: 'outpost', version: '1.0.0' }, + const { body, sessionId } = await this.post( + { + jsonrpc: '2.0', + id: this.nextId++, + method: 'initialize', + params: { + protocolVersion: '2024-11-05', + capabilities: {}, + clientInfo: { name: 'outpost', version: '1.0.0' }, + }, }, - }); + // Identify the relay. Pathfinder reads `X-Pathfinder-Source` only on + // the request that mints the session and closes over it for every + // later tool call, so `initialize` is the one place it can be set. + { 'X-Pathfinder-Source': config.pathfinder.sourceTag }, + ); const parsed = this.parseJsonRpc(body); if (parsed.error) { @@ -145,6 +176,7 @@ export class PathfinderClient { */ private async post( message: Record, + extraHeaders?: Record, ): Promise<{ body: string; sessionId: string | null }> { const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), config.pathfinder.requestTimeoutMs); @@ -157,6 +189,7 @@ export class PathfinderClient { if (this.sessionId) { headers['Mcp-Session-Id'] = this.sessionId; } + Object.assign(headers, extraHeaders); const response = await fetch(this.endpoint, { method: 'POST', @@ -376,7 +409,7 @@ export class PathfinderClient { private async search(tool: SearchTool, query: PathfinderQuery): Promise { try { const result = await this.callTool(tool, { - query: query.query, + query: capQuery(query.query, config.pathfinder.maxQueryChars), limit: query.limit ?? config.pathfinder.defaultLimit, min_score: query.minScore ?? config.pathfinder.defaultMinScore, }); @@ -419,7 +452,7 @@ export class PathfinderClient { async searchDocs(query: PathfinderQuery): Promise { try { const result = await this.callTool('search-docs', { - query: query.query, + query: capQuery(query.query, config.pathfinder.maxQueryChars), limit: query.limit ?? config.pathfinder.defaultLimit, min_score: query.minScore ?? config.pathfinder.defaultMinScore, }); From b1077c753549cf2658ea6a6b0838575131cfca91 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Sat, 12 Sep 2026 11:07:43 -0700 Subject: [PATCH 2/3] Add webhook-level tests for the spam gate Mutation-checked: neutering isLikelySpamIssue fails 3 of these, neutering capQuery and the source header fails 6 in pathfinder.test.ts. --- .../src/__tests__/issues-opened.test.ts | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/apps/github-app/src/__tests__/issues-opened.test.ts b/apps/github-app/src/__tests__/issues-opened.test.ts index 84e76d31..619207d3 100644 --- a/apps/github-app/src/__tests__/issues-opened.test.ts +++ b/apps/github-app/src/__tests__/issues-opened.test.ts @@ -171,4 +171,51 @@ describe('handleIssueOpened', () => { expect(prisma.ticketExternalLink.create).not.toHaveBeenCalled(); expect(mockPostSystemMessage).not.toHaveBeenCalled(); }); + + // The gate has to sit HERE, before the ticket exists. Creating the ticket is + // what enqueues the AI job, and that job is what forwards the body verbatim + // into search-docs/search-code on mcp.copilotkit.ai and posts a public + // reply. A filter further down would still have paid for the relay. + it('ignores a link-spam issue without creating a ticket or relaying anything', async () => { + const event = makeEvent({ + issue: { + body: + Array.from( + { length: 8 }, + (_, i) => `Read [our SEO guide ${i}](https://1rank.app/g-${i}). `, + ).join('') + 'Search visibility wins customers. '.repeat(100), + author_association: 'NONE', + }, + }); + + await handleIssueOpened(event); + + expect(mockParseInboundEvent).not.toHaveBeenCalled(); + expect(mockHandle).not.toHaveBeenCalled(); + expect(prisma.ticketExternalLink.create).not.toHaveBeenCalled(); + expect(mockPostResponse).not.toHaveBeenCalled(); + }); + + // The other half of the same guarantee: a long, link-carrying bug report + // from a first-time reporter still gets answered. + it('still relays a long bug report from an untrusted author', async () => { + const event = makeEvent({ + issue: { + body: [ + 'Repro: https://github.com/someone/repro', + 'Docs: https://docs.copilotkit.ai/quickstart', + '```ts', + 'useCopilotAction({ name: "x" });', + '```', + 'Stack trace follows. '.repeat(200), + ].join('\n'), + author_association: 'NONE', + }, + }); + + await handleIssueOpened(event); + + expect(mockHandle).toHaveBeenCalled(); + expect(prisma.ticketExternalLink.create).toHaveBeenCalled(); + }); }); From 2877f820fa13fed6ba7b38dc0ec2592df984c71f Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Sat, 12 Sep 2026 11:09:07 -0700 Subject: [PATCH 3/3] Apply prettier to the files this change touches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI format-checks every changed file, and these were already non-conforming on main — so the pre-existing violations have to go with them. --- .../src/__tests__/issues-opened.test.ts | 10 ++++++---- .../src/__tests__/spam-filter.test.ts | 17 ++++++++--------- apps/github-app/src/lib/spam-filter.ts | 5 ++++- apps/github-app/src/webhooks/issues-opened.ts | 6 ++---- packages/outpost/ai/src/config.ts | 2 +- packages/outpost/ai/src/pathfinder.test.ts | 11 +++++++---- packages/outpost/ai/src/pathfinder.ts | 5 ++++- 7 files changed, 32 insertions(+), 24 deletions(-) diff --git a/apps/github-app/src/__tests__/issues-opened.test.ts b/apps/github-app/src/__tests__/issues-opened.test.ts index 619207d3..92d44268 100644 --- a/apps/github-app/src/__tests__/issues-opened.test.ts +++ b/apps/github-app/src/__tests__/issues-opened.test.ts @@ -80,17 +80,17 @@ function makeEvent(overrides: Record = {}): EmitterWebhookEvent title: 'Bug: CopilotKit crashes on init', body: 'When I call useCopilotKit() in my Next.js app, it crashes.', html_url: 'https://github.com/CopilotKit/CopilotKit/issues/42', - ...(overrides.issue as Record ?? {}), + ...((overrides.issue as Record) ?? {}), }, repository: { full_name: 'CopilotKit/CopilotKit', - ...(overrides.repository as Record ?? {}), + ...((overrides.repository as Record) ?? {}), }, sender: { login: 'user123', id: 999, type: 'User', - ...(overrides.sender as Record ?? {}), + ...((overrides.sender as Record) ?? {}), }, ...overrides, }, @@ -104,7 +104,9 @@ describe('handleIssueOpened', () => { ticketId: 'ticket-internal-id', plugin: 'github', externalId: 'CopilotKit/CopilotKit#42', - } as ReturnType extends Promise ? T : never); + } as ReturnType extends Promise + ? T + : never); }); it('uses GitHubPlatformAdapter to parse the event', async () => { diff --git a/apps/github-app/src/__tests__/spam-filter.test.ts b/apps/github-app/src/__tests__/spam-filter.test.ts index 044902d5..04902eee 100644 --- a/apps/github-app/src/__tests__/spam-filter.test.ts +++ b/apps/github-app/src/__tests__/spam-filter.test.ts @@ -46,9 +46,7 @@ function bugReportBody(): string { describe('maxLinksToOneThirdPartyHost', () => { it('counts repeats of the same third-party host', () => { - expect( - maxLinksToOneThirdPartyHost('a https://1rank.app/x b https://1rank.app/y'), - ).toBe(2); + expect(maxLinksToOneThirdPartyHost('a https://1rank.app/x b https://1rank.app/y')).toBe(2); }); it('does not count our own hosts or GitHub, at any subdomain', () => { @@ -73,7 +71,8 @@ describe('maxLinksToOneThirdPartyHost', () => { it('returns the MAX for one host, not the total across hosts', () => { // Six links, but spread three ways — nothing is being advertised. - const body = 'https://a.io/1 https://a.io/2 https://b.io/1 https://b.io/2 https://c.io/1 https://c.io/2'; + const body = + 'https://a.io/1 https://a.io/2 https://b.io/1 https://b.io/2 https://c.io/1 https://c.io/2'; expect(maxLinksToOneThirdPartyHost(body)).toBe(2); }); }); @@ -107,9 +106,7 @@ describe('isLikelySpamIssue', () => { }); it('does not flag a long, link-heavy bug report that carries a repro', () => { - expect(isLikelySpamIssue({ body: bugReportBody(), authorAssociation: 'NONE' })).toBe( - false, - ); + expect(isLikelySpamIssue({ body: bugReportBody(), authorAssociation: 'NONE' })).toBe(false); }); it('does not flag a short body even if it is all links', () => { @@ -127,8 +124,10 @@ describe('isLikelySpamIssue', () => { it('does not flag a long body that only links GitHub and our docs', () => { const body = - Array.from({ length: 12 }, (_, i) => `https://github.com/CopilotKit/x/issues/${i} `) - .join('') + 'y'.repeat(3000); + Array.from( + { length: 12 }, + (_, i) => `https://github.com/CopilotKit/x/issues/${i} `, + ).join('') + 'y'.repeat(3000); expect(isLikelySpamIssue({ body, authorAssociation: 'NONE' })).toBe(false); }); diff --git a/apps/github-app/src/lib/spam-filter.ts b/apps/github-app/src/lib/spam-filter.ts index d1952715..a5152cde 100644 --- a/apps/github-app/src/lib/spam-filter.ts +++ b/apps/github-app/src/lib/spam-filter.ts @@ -90,7 +90,10 @@ function isFirstParty(host: string): boolean { export function maxLinksToOneThirdPartyHost(body: string): number { const counts = new Map(); for (const match of body.matchAll(/https?:\/\/([^/\s)>\]"'`]+)/gi)) { - const host = match[1].toLowerCase().replace(/^www\./, '').replace(/:\d+$/, ''); + const host = match[1] + .toLowerCase() + .replace(/^www\./, '') + .replace(/:\d+$/, ''); if (isFirstParty(host)) continue; counts.set(host, (counts.get(host) ?? 0) + 1); } diff --git a/apps/github-app/src/webhooks/issues-opened.ts b/apps/github-app/src/webhooks/issues-opened.ts index be7182ea..704e0eae 100644 --- a/apps/github-app/src/webhooks/issues-opened.ts +++ b/apps/github-app/src/webhooks/issues-opened.ts @@ -15,13 +15,11 @@ export async function handleIssueOpened( console.log( `[GitHub App] Issue opened: ${repository.full_name}#${issue.number} ` + - `"${issue.title}" by ${sender.login}`, + `"${issue.title}" by ${sender.login}`, ); if (!isRepoAllowed(repository.full_name, config.allowedRepos)) { - console.log( - `[GitHub App] Ignoring issue on non-allowlisted repo ${repository.full_name}`, - ); + console.log(`[GitHub App] Ignoring issue on non-allowlisted repo ${repository.full_name}`); return; } diff --git a/packages/outpost/ai/src/config.ts b/packages/outpost/ai/src/config.ts index 3f80d44a..1f0b23e8 100644 --- a/packages/outpost/ai/src/config.ts +++ b/packages/outpost/ai/src/config.ts @@ -110,7 +110,7 @@ export function validateConfig(): void { if (!config.anthropicApiKey) { throw new Error( '[AI Config] ANTHROPIC_API_KEY is required but not set. ' + - 'Set the ANTHROPIC_API_KEY environment variable before starting the pipeline.', + 'Set the ANTHROPIC_API_KEY environment variable before starting the pipeline.', ); } } diff --git a/packages/outpost/ai/src/pathfinder.test.ts b/packages/outpost/ai/src/pathfinder.test.ts index e0f94911..25594797 100644 --- a/packages/outpost/ai/src/pathfinder.test.ts +++ b/packages/outpost/ai/src/pathfinder.test.ts @@ -339,8 +339,7 @@ describe('PathfinderClient', () => { content: [ { type: 'text', - text: - 'SNIPPET 1\nTITLE: CopilotKit Actions\nSOURCE: https://docs.copilotkit.ai/actions\nCONTENT:\nuseCopilotAction lets you define actions.\n\n---\n\nSNIPPET 2\nTITLE: Getting Started\nSOURCE: https://docs.copilotkit.ai/quickstart\nCONTENT:\nInstall CopilotKit with npm install.', + text: 'SNIPPET 1\nTITLE: CopilotKit Actions\nSOURCE: https://docs.copilotkit.ai/actions\nCONTENT:\nuseCopilotAction lets you define actions.\n\n---\n\nSNIPPET 2\nTITLE: Getting Started\nSOURCE: https://docs.copilotkit.ai/quickstart\nCONTENT:\nInstall CopilotKit with npm install.', }, ], }, @@ -453,8 +452,12 @@ describe('PathfinderClient', () => { it('returns empty when both MCP and fallback fail', async () => { mockConnect(); - mockFetch.mockResolvedValueOnce(mkResp({ ok: false, status: 500, statusText: 'Error' })); - mockFetch.mockResolvedValueOnce(mkResp({ ok: false, status: 500, statusText: 'Error' })); + mockFetch.mockResolvedValueOnce( + mkResp({ ok: false, status: 500, statusText: 'Error' }), + ); + mockFetch.mockResolvedValueOnce( + mkResp({ ok: false, status: 500, statusText: 'Error' }), + ); const results = await client.searchDocs({ query: 'anything' }); expect(results).toEqual([]); diff --git a/packages/outpost/ai/src/pathfinder.ts b/packages/outpost/ai/src/pathfinder.ts index 63f0c6bb..17a747a1 100644 --- a/packages/outpost/ai/src/pathfinder.ts +++ b/packages/outpost/ai/src/pathfinder.ts @@ -547,7 +547,10 @@ export class PathfinderClient { const score = matchCount / queryTerms.length; // Extract title from first line - const firstLine = section.split('\n')[0].replace(/^#+\s*/, '').trim(); + const firstLine = section + .split('\n')[0] + .replace(/^#+\s*/, '') + .trim(); return { title: firstLine || 'Documentation',