Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -70,3 +70,9 @@ yarn.lock

# Local AI review pipeline session state (create-review / review-changes)
.chalk/

# Compose auto-loads this exact filename and it is per-developer by convention.
# One was committed by accident: `ports: !override` REPLACES the base list, so
# postgres stopped publishing 5432 and the README's setup path (db:push against
# localhost:5432) failed on a fresh clone.
docker-compose.override.yml
14 changes: 6 additions & 8 deletions apps/discord-bot/src/lib/shadow-mode.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,10 @@
*
* This enables a parallel-run validation period before full cutover.
*/
export function isShadowMode(): boolean {
return process.env.SHADOW_MODE === 'true';
}
// Re-exported rather than reimplemented. Three copies of this predicate existed
// and all three compared `=== 'true'`; a shared one is the only version of this
// that stays fixed.
export { isShadowMode } from '@copilotkit/outpost/shared';

export interface ShadowResponse {
ticketId: string;
Expand Down Expand Up @@ -48,7 +49,7 @@

console.log(
`[Shadow Mode] Logged response for ticket ${response.ticketId} ` +
`(${response.responseTimeMs}ms)`,
`(${response.responseTimeMs}ms)`,
);
}

Expand Down Expand Up @@ -115,10 +116,7 @@

return ticket.id;
} catch (error) {
console.error(
`[Shadow Mode] Failed to create ticket for thread ${thread.id}:`,
error,
);
console.error(`[Shadow Mode] Failed to create ticket for thread ${thread.id}:`, error);
return null;
}
}
Expand All @@ -129,7 +127,7 @@
export async function handleShadowMessage(
message: Message,
ticketId: string,
threadId: string,

Check warning on line 130 in apps/discord-bot/src/lib/shadow-mode.ts

View workflow job for this annotation

GitHub Actions / Lint, Typecheck & Test

'threadId' is defined but never used. Allowed unused args must match /^_/u
): Promise<void> {
try {
await prisma.message.create({
Expand Down
23 changes: 23 additions & 0 deletions apps/worker/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,31 @@ import {
handleGithubReactionPoll,
handlePendingResponseSweep,
} from '@copilotkit/outpost/queue';
import { isShadowMode } from '@copilotkit/outpost/shared';
import { buildSyncEngine } from './build-sync-engine.js';

// ─── Announce the resolved posting mode ───────────────────────────────────

// One line, at boot, next to the other fail-fast-at-boot decision below.
//
// Without it an operator has no way to answer "which mode am I in?" except to
// wait for a job and infer it from which line got printed. That is a bad way to
// learn the answer for the one flag standing between a parallel-run window and
// machine-generated text arriving in a stranger's support thread.
//
// Deliberately a log and not a throw. Throwing when SHADOW_MODE is absent in
// production is a real behaviour change — staging is documented as
// `SHADOW_MODE=true`, so a variable that fails to carry to a new replica is a
// silent fail-open that this log makes visible but does not prevent. That
// stronger version belongs with the startup-assertion work, not here.
console.log(
isShadowMode()
? '[Worker] SHADOW MODE ON — responses are generated and recorded, never posted.'
: `[Worker] SHADOW MODE OFF — responses WILL post to real community surfaces (SHADOW_MODE=${
process.env.SHADOW_MODE ?? 'unset'
}).`,
);

// ─── Build SyncEngine for TRACKER_SYNC handler ────────────────────────────

// BOOT SEMANTICS — deliberate change. This is a top-level await that performs
Expand Down
7 changes: 5 additions & 2 deletions docs/deployment.md
Original file line number Diff line number Diff line change
Expand Up @@ -209,8 +209,11 @@ worker's gate is the one that covers **every** platform (GitHub, Slack, Teams) p
digest job, because that is where the adapter call lives — so a staging environment must
have it set on `outpost-worker`, not only on a bot.

When adding any new outbound post path, check `SHADOW_MODE` before posting — otherwise
staging will deliver to real users regardless of the flag.
When adding any new outbound post path, gate it on `isShadowMode()` from
`@copilotkit/outpost/shared` — not on `process.env.SHADOW_MODE` directly. Reading the
variable is what produced three separate copies of `=== 'true'`, all three of which
treated `SHADOW_MODE=TRUE` as "not shadow mode" and posted for real. The helper is the
only version of this that stays fixed.

### Promotion workflow

Expand Down
3 changes: 3 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@
"format": "prettier --write \"**/*.{ts,tsx,js,jsx,json,md}\"",
"format:check": "prettier --check \"**/*.{ts,tsx,js,jsx,json,md}\""
},
"dependencies": {
"@copilotkit/outpost": "workspace:*"
},
"devDependencies": {
"@eslint/js": "^9.39.4",
"@linear/sdk": "^81.0.0",
Expand Down
49 changes: 38 additions & 11 deletions packages/outpost/queue/src/__tests__/ai-response.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -72,17 +72,17 @@ const mockGetAdapter = vi.fn().mockReturnValue({
fetchUserInfo: vi.fn(),
});

vi.mock('@copilotkit/outpost/shared', () => ({
AI_CONFIDENCE: {
AUTO_RESPOND: 0.9,
HIGH_THRESHOLD: 0.8,
SUGGEST: 0.7,
MEDIUM_THRESHOLD: 0.5,
ESCALATE: 0.4,
},
MAX_JOB_ATTEMPTS: 5,
BACKOFF_BASE_MS: 1000,
BACKOFF_MAX_MS: 300_000,
// Partial mock, so `isShadowMode` below is the shipped function rather than a
// re-implementation of it. The four constants that used to be stubbed here were
// checked against `shared/src/constants.ts` and are identical, so the spread
// supplies them.
//
// `calculateBackoff` stays overridden on purpose: the real one
// (`shared/src/utils.ts:40`) adds `Math.random() * BACKOFF_BASE_MS` of jitter,
// so taking it from the spread would make any assertion on a retry delay
// nondeterministic. This override is the deterministic half of it.
vi.mock('@copilotkit/outpost/shared', async (importOriginal) => ({
...(await importOriginal<typeof import('@copilotkit/outpost/shared')>()),
calculateBackoff: (attempt: number) => 1000 * Math.pow(2, attempt),
}));

Expand Down Expand Up @@ -738,6 +738,33 @@ describe('handleAiResponse', () => {
}
});

// The fence, and the one that matters most: this is the handler that reaches
// real Discord and GitHub surfaces. Every other shadow test in this file
// uses `'true'`, the one spelling that read the same before and after the
// fail-closed change — so reverting `isShadowMode` to `=== 'true'` left all
// 145 tests in the two queue suites green. Each of these used to post.
it.each(['1', 'TRUE', 'yes', 'on', ' true ', 'YES'])(
'skips post-back when SHADOW_MODE=%j',
async (value) => {
const originalShadow = process.env.SHADOW_MODE;
try {
process.env.SHADOW_MODE = value;
mockPrismaTicket.findUnique.mockResolvedValue(sampleTicket);
mockHasAdapter.mockReturnValue(true);

const result = await handleAiResponse(
{ ticketId: 'tkt-1', source: 'discord' },
makeContext(),
);

expect(result.success).toBe(true);
expect(mockPostResponse).not.toHaveBeenCalled();
} finally {
restoreShadowMode(originalShadow);
}
},
);

it('succeeds even if post-back fails (non-fatal)', async () => {
mockPrismaTicket.findUnique.mockResolvedValue(sampleTicket);
mockHasAdapter.mockReturnValue(true);
Expand Down
70 changes: 52 additions & 18 deletions packages/outpost/queue/src/__tests__/onboarding-digest.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,14 @@ vi.mock('@copilotkit/outpost/db', () => ({
},
}));

vi.mock('@copilotkit/outpost/shared', () => ({
// Partial mock: everything real except the one function this file needs to
// pin. The previous version listed its exports explicitly, which meant
// re-implementing `isShadowMode` — so a diff whose whole point was deleting
// three copies of the comparison added a fourth, and it had already drifted
// (no EXPLICITLY_ON, no warn). Spreading the real module means the SHADOW_MODE
// tests below exercise the shipped function instead of a lookalike.
vi.mock('@copilotkit/outpost/shared', async (importOriginal) => ({
...(await importOriginal<typeof import('@copilotkit/outpost/shared')>()),
computeFunnelMetrics: vi.fn().mockReturnValue({
stageCounts: { JOINED: 3, CONTACTED: 2, RESPONDED: 1, MEETING_BOOKED: 0 },
conversionRates: {
Expand Down Expand Up @@ -84,7 +91,7 @@ describe('handleOnboardingDigest', () => {

it('queries members for the given date range', async () => {
mockOnboardingMember.findMany
.mockResolvedValueOnce([makeMemberRow()]) // date-filtered query
.mockResolvedValueOnce([makeMemberRow()]) // date-filtered query
.mockResolvedValueOnce([makeMemberRow()]); // all-members query for metrics

const ctx = makeContext();
Expand All @@ -102,8 +109,8 @@ describe('handleOnboardingDigest', () => {

it('handles zero new members gracefully', async () => {
mockOnboardingMember.findMany
.mockResolvedValueOnce([]) // no members for the day
.mockResolvedValueOnce([]); // no members overall
.mockResolvedValueOnce([]) // no members for the day
.mockResolvedValueOnce([]); // no members overall

const ctx = makeContext();
const result = await handleOnboardingDigest({ date: '2026-04-15' }, ctx);
Expand All @@ -118,9 +125,7 @@ describe('handleOnboardingDigest', () => {
const ctx = makeContext();
await handleOnboardingDigest({ date: '2026-04-15' }, ctx);

const progressCalls = ctx.reportProgress.mock.calls.map(
(c: number[]) => c[0],
);
const progressCalls = ctx.reportProgress.mock.calls.map((c: number[]) => c[0]);
expect(progressCalls).toEqual([10, 50, 70, 90, 100]);
});

Expand All @@ -142,9 +147,7 @@ describe('handleOnboardingDigest', () => {
makeMemberRow({ id: 'om-3', username: 'charlie#9012' }),
];

mockOnboardingMember.findMany
.mockResolvedValueOnce(members)
.mockResolvedValueOnce(members);
mockOnboardingMember.findMany.mockResolvedValueOnce(members).mockResolvedValueOnce(members);

const ctx = makeContext();
const result = await handleOnboardingDigest({ date: '2026-04-15' }, ctx);
Expand Down Expand Up @@ -221,14 +224,46 @@ describe('handleOnboardingDigest', () => {

expect(result.success).toBe(true);
expect(mockFetch).not.toHaveBeenCalled();
expect(consoleSpy).toHaveBeenCalledWith(
expect.stringContaining('Shadow mode'),
);
expect(consoleSpy).toHaveBeenCalledWith(expect.stringContaining('Shadow mode'));

consoleSpy.mockRestore();
vi.unstubAllGlobals();
});

// The fence. Every other shadow test here uses `'true'`, which is the one
// spelling that behaves identically before and after the fail-closed change
// — so reverting `isShadowMode` to `=== 'true'` left this whole file green.
// These are the spellings that used to post for real.
it.each(['1', 'TRUE', 'yes', 'on', ' true ', 'YES'])(
'does not post to Discord when SHADOW_MODE=%j',
async (value) => {
process.env.DISCORD_TOKEN = 'test-bot-token';
process.env.DISCORD_DIGEST_CHANNEL_ID = '1234567890';
process.env.SHADOW_MODE = value;

mockOnboardingMember.findMany
.mockResolvedValueOnce([makeMemberRow()])
.mockResolvedValueOnce([makeMemberRow()]);

const mockFetch = vi.fn().mockResolvedValue({
ok: true,
json: async () => ({ id: 'msg-1' }),
});
vi.stubGlobal('fetch', mockFetch);
const consoleSpy = vi.spyOn(console, 'log').mockImplementation(() => {});

const ctx = makeContext();
const result = await handleOnboardingDigest({ date: '2026-04-15' }, ctx);

expect(result.success).toBe(true);
expect(mockFetch).not.toHaveBeenCalled();
expect(consoleSpy).toHaveBeenCalledWith(expect.stringContaining('Shadow mode'));

consoleSpy.mockRestore();
vi.unstubAllGlobals();
},
);

it('posts to Discord when SHADOW_MODE is explicitly false', async () => {
process.env.DISCORD_TOKEN = 'test-bot-token';
process.env.DISCORD_DIGEST_CHANNEL_ID = '1234567890';
Expand Down Expand Up @@ -257,9 +292,7 @@ describe('handleOnboardingDigest', () => {
process.env.DISCORD_TOKEN = 'test-bot-token';
process.env.DISCORD_DIGEST_CHANNEL_ID = '1234567890';

mockOnboardingMember.findMany
.mockResolvedValueOnce([])
.mockResolvedValueOnce([]);
mockOnboardingMember.findMany.mockResolvedValueOnce([]).mockResolvedValueOnce([]);

const mockFetch = vi.fn().mockResolvedValue({
ok: false,
Expand All @@ -269,8 +302,9 @@ describe('handleOnboardingDigest', () => {
vi.stubGlobal('fetch', mockFetch);

const ctx = makeContext();
await expect(handleOnboardingDigest({ date: '2026-04-15' }, ctx))
.rejects.toThrow('Discord API error 403');
await expect(handleOnboardingDigest({ date: '2026-04-15' }, ctx)).rejects.toThrow(
'Discord API error 403',
);

vi.unstubAllGlobals();
});
Expand Down
4 changes: 2 additions & 2 deletions packages/outpost/queue/src/handlers/ai-response.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@

import { prisma } from '@copilotkit/outpost/db';
import { AIPipeline } from '@copilotkit/outpost/ai';
import { AI_CONFIDENCE, MAX_JOB_ATTEMPTS } from '@copilotkit/outpost/shared';
import { AI_CONFIDENCE, MAX_JOB_ATTEMPTS, isShadowMode } from '@copilotkit/outpost/shared';
import type { PlatformTarget, TicketSource } from '@copilotkit/outpost/shared';
import { hasAdapter, getAdapter } from '@copilotkit/outpost/shared/platforms';
import { getFeedbackCalibration } from '../feedback-calibration.js';
Expand Down Expand Up @@ -821,7 +821,7 @@ export async function handleAiResponse(
}

let responseDelivered = false;
if (process.env.SHADOW_MODE === 'true') {
if (isShadowMode()) {
try {
await prisma.message.create({
data: {
Expand Down
20 changes: 14 additions & 6 deletions packages/outpost/queue/src/handlers/onboarding-digest.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
*/

import { prisma } from '@copilotkit/outpost/db';
import { computeFunnelMetrics } from '@copilotkit/outpost/shared';
import { computeFunnelMetrics, isShadowMode } from '@copilotkit/outpost/shared';
import type { OnboardingMember } from '@copilotkit/outpost/shared';
import type { OnboardingDigestPayload, JobResult, JobHandlerContext } from '../types.js';

Expand Down Expand Up @@ -106,25 +106,33 @@ export async function handleOnboardingDigest(
digestLines.push('');
digestLines.push('Funnel Summary (all time):');
digestLines.push(` Joined: ${metrics.stageCounts.JOINED}`);
digestLines.push(` Contacted: ${metrics.stageCounts.CONTACTED} (${metrics.conversionRates.joinedToContacted}%)`);
digestLines.push(` Responded: ${metrics.stageCounts.RESPONDED} (${metrics.conversionRates.contactedToResponded}%)`);
digestLines.push(` Meeting Booked: ${metrics.stageCounts.MEETING_BOOKED} (${metrics.conversionRates.respondedToMeetingBooked}%)`);
digestLines.push(
` Contacted: ${metrics.stageCounts.CONTACTED} (${metrics.conversionRates.joinedToContacted}%)`,
);
digestLines.push(
` Responded: ${metrics.stageCounts.RESPONDED} (${metrics.conversionRates.contactedToResponded}%)`,
);
digestLines.push(
` Meeting Booked: ${metrics.stageCounts.MEETING_BOOKED} (${metrics.conversionRates.respondedToMeetingBooked}%)`,
);

const digest = digestLines.join('\n');

await context.reportProgress(90);

// Deliver the digest
const channelId = process.env.DISCORD_DIGEST_CHANNEL_ID;
if (process.env.SHADOW_MODE === 'true') {
if (isShadowMode()) {
// Shadow mode (staging): log the digest instead of posting it, so a
// staging worker never delivers to a real Discord channel.
console.log(`[Onboarding Digest] Shadow mode — skipping Discord post:\n${digest}`);
} else if (channelId) {
await postToDiscord(channelId, digest);
} else {
// Development fallback when DISCORD_DIGEST_CHANNEL_ID is not set
console.log(`[Onboarding Digest] DISCORD_DIGEST_CHANNEL_ID not set, logging to console:\n${digest}`);
console.log(
`[Onboarding Digest] DISCORD_DIGEST_CHANNEL_ID not set, logging to console:\n${digest}`,
);
}

await context.reportProgress(100);
Expand Down
Loading
Loading