fix(webapp): dashboard agent maintenance moves into the agent project (#4599)

## What & why

The dashboard agent's upkeep — retention deletes and the investigation
sweep — ran as cron jobs on the webapp's common worker, even though it
only touches the agent's own datastore. This moves that upkeep into the
agent's Trigger project as scheduled tasks (TRI-13182).

## What's inside

**Retention** — `internal-packages/dashboard-agent/src/maintenance.ts`,
a daily task (03:00 UTC). Deletes turn evals older than 30 days,
hard-deletes chats soft-deleted more than 30 days ago, and purges
terminal watches and submission rows older than 7 days. It used to run
every 5 minutes; nothing needs a hard delete that fast, so it is daily
now, draining in bounded batches and warning if it hits the cap. It
retries (3 attempts) because the next run is a day away. It connects
with `DASHBOARD_AGENT_DATABASE_URL`, falling back to `DATABASE_URL` like
every other task in the package (the deletes are confined to the agent's
own Postgres schema), and skips when neither is set.

**Investigation sweep** — `src/investigation-sweep.ts`, every 5 minutes,
same as before: settles investigation cards stuck `in_progress`
(30-minute window, attempt cap, force-abandon note). It keeps the fast
cadence because it fixes live state the UI is showing.

**What stays in the webapp.** The watch finalize/deliver sweep and batch
rearm: they cover a dead agent-side tick chain — a backstop can't live
inside the thing it backstops — and they need the main database and the
alerts worker. The org-deletion chat purge also stays: deletion must not
depend on the agent project being deployed. The removed cron job keeps a
cron-less tombstone entry so already-queued items drain cleanly; remove
it in a follow-up.

**Test plumbing** — the drizzle migration replayer that webapp tests
hand-rolled is now exported once from
`@internal/dashboard-agent-db/testing`; the moved tests live in the
agent package as `src/*.test.ts` against real Postgres.

## Testing

Agent package: retention passes (backlog drain, batch cap, no-op guard,
chat-delete cascade) and the sweep, on testcontainers Postgres. Webapp:
the watch/chat suites, plus a test that a settlement card stops the
dashboard spinner. Full typecheck on both.

Author: kathiekiwi

.server-changes/dashboard-agent-maintenance-move.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: improvement
+---
+
+Routine cleanup of old dashboard agent data now runs on its own schedule.

apps/webapp/app/services/dashboardAgentChatRetention.server.ts

@@ -1,71 +1,14 @@
 /**
- * Retention for soft-deleted chats. A deleted chat is kept for a grace window and then
- * hard-deleted with all its child rows; one bounded statement per run, oldest first.
- * Also the eventual purge behind organization deletion, which soft-deletes the org's
- * chats so this same sweep removes them.
+ * The purge behind organization deletion: it soft-deletes the org's chats, and retention
+ * hard-deletes them once the window passes.
  */
 
-import {
-  hardDeleteChatsSoftDeletedBefore,
-  softDeleteChatsForOrganization,
-} from "@internal/dashboard-agent-db";
+import { softDeleteChatsForOrganization } from "@internal/dashboard-agent-db";
 import { dashboardAgentDb } from "~/services/dashboardAgentDb.server";
-import { logger } from "~/services/logger.server";
 
 /**
- * How long a soft-deleted chat is kept before it and its children are hard-deleted.
- * Long enough that an accidental delete can still be investigated; org deletion soft-
- * deletes the org's chats, so those are removed the same way once the window passes.
- */
-export const CHAT_SOFT_DELETE_RETENTION_MS = 30 * 24 * 60 * 60 * 1000;
-
-/** Per-run cap. Retention is one bounded statement, not a row-at-a-time loop. */
-const RETENTION_BATCH_LIMIT = 500;
-
-export type ChatRetentionResult = {
-  /** Soft-deleted chats past the retention window dropped this run. */
-  purged: number;
-  failed: number;
-};
-
-export type ChatRetentionDeps = {
-  now?: () => Date;
-  limit?: number;
-  /** Hard-delete chats soft-deleted before `before`. Returns how many went. */
-  purge?: (params: { before: Date; limit: number }) => Promise<number>;
-};
-
-export async function sweepDashboardAgentSoftDeletedChats(
-  deps: ChatRetentionDeps = {}
-): Promise<ChatRetentionResult> {
-  const now = deps.now?.() ?? new Date();
-  const limit = deps.limit ?? RETENTION_BATCH_LIMIT;
-  const purge =
-    deps.purge ?? ((params) => hardDeleteChatsSoftDeletedBefore(dashboardAgentDb, params));
-
-  const result: ChatRetentionResult = { purged: 0, failed: 0 };
-
-  try {
-    result.purged = await purge({
-      before: new Date(now.getTime() - CHAT_SOFT_DELETE_RETENTION_MS),
-      limit,
-    });
-  } catch (error) {
-    result.failed++;
-    logger.error("Dashboard agent chat retention failed", { error });
-  }
-
-  if (result.failed > 0) {
-    throw new Error("The dashboard agent chat retention pass failed");
-  }
-
-  return result;
-}
-
-/**
- * Soft-delete every chat belonging to a deleted organization. The retention sweep above
- * hard-deletes them once the window passes, so the org-deletion request never runs a
- * cross-database hard delete.
+ * Soft-delete every chat belonging to a deleted organization. Retention hard-deletes them
+ * once the window passes, so the org-deletion request never runs a cross-database hard delete.
  */
 export async function purgeDashboardAgentChatsForOrganization(params: {
   organizationId: string;

apps/webapp/app/services/dashboardAgentEvalRetention.server.ts

@@ -6,8 +6,6 @@
 import {
   cancelWatch,
   claimWatchAlertDispatch,
-  deleteTerminalWatchesOlderThan,
-  deleteWatchSubmissionsOlderThan,
   listExpiredActiveWatches,
   listWatchBatchGroupsToArm,
   listWatchesAwaitingDelivery,
@@ -57,12 +55,6 @@ export const WATCH_DELIVERY_GRACE_MS = 5 * 60 * 1000;
 /** Per-run cap for each half of the sweep. Oldest first, so the rest land next run. */
 const SWEEP_BATCH_LIMIT = 100;
 
-/** How long a terminal watch is kept. Its outcome also lives in the chat transcript. */
-export const WATCH_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
-
-/** Higher than the other caps: retention is one statement, not a row-at-a-time loop. */
-const RETENTION_BATCH_LIMIT = 500;
-
 /**
  * How many rows one sweep handles at once. An incident expires a whole group together, and a
  * bound is what stops one slow tenant spending the entire visibility window.
@@ -91,10 +83,6 @@ export type WatchSweepResult = {
   redelivered: number;
   /** Decided but not handed over, with no agent project. They stay owed. */
   deliveryDeferred: number;
-  /** Long-terminal rows dropped by retention. */
-  purged: number;
-  /** Ledger rows dropped by retention. */
-  purgedSubmissions: number;
   failed: number;
 };
 
@@ -113,10 +101,6 @@ export type WatchSweepDeps = {
   deliver?: (watch: Watch) => Promise<void>;
   /** Gates the delivery half only. Finalization never depends on it. */
   configured?: () => boolean;
-  /** Drop terminal rows older than `before`. Returns how many went. */
-  purgeTerminal?: (params: { before: Date; limit: number }) => Promise<number>;
-  /** Drop submission-ledger rows older than `before`. */
-  purgeSubmissions?: (params: { before: Date; limit: number }) => Promise<number>;
   /** How many rows are handled at once. */
   concurrency?: number;
 };
@@ -332,11 +316,6 @@ export async function sweepDashboardAgentWatches(
   const listAwaitingDelivery =
     deps.listAwaitingDelivery ??
     ((params) => listWatchesAwaitingDelivery(dashboardAgentDb, params));
-  const purgeTerminal =
-    deps.purgeTerminal ?? ((params) => deleteTerminalWatchesOlderThan(dashboardAgentDb, params));
-  const purgeSubmissions =
-    deps.purgeSubmissions ??
-    ((params) => deleteWatchSubmissionsOlderThan(dashboardAgentDb, params));
 
   const result: WatchSweepResult = {
     overdue: 0,
@@ -347,8 +326,6 @@ export async function sweepDashboardAgentWatches(
     undelivered: 0,
     redelivered: 0,
     deliveryDeferred: 0,
-    purged: 0,
-    purgedSubmissions: 0,
     failed: 0,
   };
 
@@ -435,18 +412,6 @@ export async function sweepDashboardAgentWatches(
     }
   }
 
-  // Retention runs last, over rows both halves are finished with. Its own try/catch so a
-  // lost retention pass can't mask the other failures.
-  try {
-    const before = new Date(now.getTime() - WATCH_RETENTION_MS);
-    result.purged = await purgeTerminal({ before, limit: RETENTION_BATCH_LIMIT });
-    // The ledger's rows age out on the same window: past it no client is still retrying.
-    result.purgedSubmissions = await purgeSubmissions({ before, limit: RETENTION_BATCH_LIMIT });
-  } catch (error) {
-    result.failed++;
-    logger.error("Dashboard agent watch sweep: failed to purge terminal watches", { error });
-  }
-
   if (result.failed > 0) {
     throw new Error(`The dashboard agent watch sweep failed on ${result.failed} watches`);
   }

apps/webapp/app/services/dashboardAgentWatchSweep.server.ts

@@ -11,12 +11,7 @@ import {
   runAttioUserSync,
   runAttioWorkspaceSync,
 } from "~/services/attio.server";
-import {
-  purgeDashboardAgentChatsForOrganization,
-  sweepDashboardAgentSoftDeletedChats,
-} from "~/services/dashboardAgentChatRetention.server";
-import { sweepDashboardAgentTurnEvals } from "~/services/dashboardAgentEvalRetention.server";
-import { sweepDashboardAgentInvestigations } from "~/services/dashboardAgentInvestigationSweep.server";
+import { purgeDashboardAgentChatsForOrganization } from "~/services/dashboardAgentChatRetention.server";
 import {
   rearmDashboardAgentWatchBatches,
   sweepDashboardAgentWatches,
@@ -47,7 +42,7 @@ function initializeWorker() {
 
   logger.debug(`👨‍🏭 Initializing common worker at host ${env.COMMON_WORKER_REDIS_HOST}`);
 
-  // Only schedule the agent maintenance cron where the agent is actually set up. Otherwise
+  // Only schedule the agent watch cron where the agent is actually set up. Otherwise
   // its sweeps hit a missing schema and drip a dead-letter entry every run.
   const dashboardAgentConfigured =
     env.DASHBOARD_AGENT_ENABLED === "1" || Boolean(env.DASHBOARD_AGENT_DATABASE_URL);
@@ -161,16 +156,15 @@ function initializeWorker() {
           maxAttempts: 5,
         },
       },
-      // Stuck investigation cards and turn-eval retention.
+      // @deprecated, moved to the dashboard agent project; remove once the queue drains.
       "dashboardAgent.maintenance": {
         schema: CronSchema,
-        visibilityTimeoutMs: 60_000 * 5,
-        ...(dashboardAgentConfigured ? { cron: "*/5 * * * *", jitterInMs: 30_000 } : {}),
+        visibilityTimeoutMs: 60_000,
         retry: {
           maxAttempts: 1,
         },
       },
-      // The watch backstops: expiry, wake redelivery, retention and dead batch chains.
+      // The watch backstops: expiry, wake redelivery and dead batch chains.
       "dashboardAgent.watchMaintenance": {
         schema: CronSchema,
         visibilityTimeoutMs: 60_000 * 5,
@@ -179,7 +173,7 @@ function initializeWorker() {
           maxAttempts: 1,
         },
       },
-      // Soft-deletes a deleted organization's chats; the maintenance sweep purges them.
+      // Soft-deletes a deleted organization's chats; retention hard-deletes them later.
       "dashboardAgent.purgeOrganization": {
         schema: z.object({
           organizationId: z.string(),
@@ -247,48 +241,15 @@ function initializeWorker() {
         const service = new BulkActionService();
         await service.process(payload.bulkActionId);
       },
-      "dashboardAgent.maintenance": async () => {
-        // Each backstop runs independently; the first failure is rethrown at the end.
-        let failure: unknown;
-
-        try {
-          const investigations = await sweepDashboardAgentInvestigations();
-          if (investigations.stale > 0) {
-            logger.debug("Dashboard agent investigation sweep", investigations);
-          }
-        } catch (error) {
-          failure ??= error;
-        }
-
-        // Retention on the judged-turn rows. Independent of the agent being configured.
-        try {
-          const evals = await sweepDashboardAgentTurnEvals();
-          if (evals.purged > 0) {
-            logger.debug("Dashboard agent turn-eval retention", evals);
-          }
-        } catch (error) {
-          failure ??= error;
-        }
-
-        // Hard-delete chats soft-deleted past the retention window, with their children.
-        try {
-          const chats = await sweepDashboardAgentSoftDeletedChats();
-          if (chats.purged > 0) {
-            logger.debug("Dashboard agent chat retention", chats);
-          }
-        } catch (error) {
-          failure ??= error;
-        }
-
-        if (failure) throw failure;
-      },
+      // @deprecated, moved to the dashboard agent project; remove once the queue drains.
+      "dashboardAgent.maintenance": async () => {},
       "dashboardAgent.watchMaintenance": async () => {
         // Each backstop runs independently; the first failure is rethrown at the end.
         let failure: unknown;
 
         try {
           const watches = await sweepDashboardAgentWatches();
-          if (watches.overdue > 0 || watches.undelivered > 0 || watches.purged > 0) {
+          if (watches.overdue > 0 || watches.undelivered > 0) {
             logger.debug("Dashboard agent watch sweep", watches);
           }
         } catch (error) {