feat(worker): add Sentry error tracking and source maps for debuggable builds

.env.example

@@ -107,3 +107,18 @@ NODE_OPTIONS="--max_old_space_size=4096"
 # ─────────────────────────────────────────────
 # Log verbosity: error | warn | info | debug
 LOG_LEVEL=debug
+
+# ─────────────────────────────────────────────
+# Error tracking (Sentry)
+# ─────────────────────────────────────────────
+# Leave SENTRY_DSN empty to disable Sentry entirely (no-op).
+# Get a DSN from sentry.io → Project Settings → Client Keys (DSN).
+SENTRY_DSN=
+# Logical environment shown in Sentry (defaults to NODE_ENV).
+SENTRY_ENVIRONMENT=
+# Release/commit identifier so events are grouped by deploy. Set to the git SHA in CI.
+SENTRY_RELEASE=
+# Performance tracing sample rate, 0–1. Default 0 (errors only) to protect quota.
+SENTRY_TRACES_SAMPLE_RATE=0
+# Identifies which worker process emitted an event (e.g. "worker-chat").
+WORKER_NAME=

apps/worker/docker/rootfs/usr/local/bin/docker-entrypoint.sh

@@ -4,6 +4,14 @@ set -eu
 
 WORKER_DIST_DIR="/app/apps/worker/dist"
 NODE_BIN="/usr/local/bin/node"
+INSTRUMENT_SCRIPT="${WORKER_DIST_DIR}/instrument.mjs"
+# Debug + observability flags applied to every worker process:
+#   --enable-source-maps : remap stack traces from bundled .mjs back to .ts
+#   --import instrument   : initialize Sentry before any worker module loads
+# Passed directly (not via NODE_OPTIONS) so a runtime NODE_OPTIONS
+# (e.g. --max_old_space_size) cannot clobber them. Unquoted on use so it
+# word-splits into separate flags.
+NODE_FLAGS="--enable-source-maps --import ${INSTRUMENT_SCRIPT}"
 # ALL_WORKERS="chat integration ai-agent default webhook trigger analytics schedule sequence-scheduler sequence-producer sequence-consumer"
 ALL_WORKERS="chat integration ai-agent default webhook trigger analytics schedule sequence-scheduler"
 
@@ -76,7 +84,9 @@ run_all_workers() {
 
   trap 'handle_shutdown' INT TERM
 
-  # Validate all worker entrypoints first so we never start partially.
+  # Validate the Sentry instrument and all worker entrypoints first so we
+  # never start partially.
+  require_script "$INSTRUMENT_SCRIPT"
   for worker in $ALL_WORKERS; do
     script="$(resolve_worker_script "$worker")"
     require_script "$script"
@@ -85,7 +95,8 @@ run_all_workers() {
   for worker in $ALL_WORKERS; do
     script="$(resolve_worker_script "$worker")"
     echo "Starting worker: $worker ($script)"
-    "$NODE_BIN" "$script" &
+    # shellcheck disable=SC2086 # NODE_FLAGS must word-split into separate flags
+    "$NODE_BIN" $NODE_FLAGS "$script" &
     pids="$pids $!"
   done
 
@@ -128,8 +139,10 @@ case "${1:-}" in
           print_usage
           exit 3
         }
+        require_script "$INSTRUMENT_SCRIPT"
         require_script "$script"
-        exec "$NODE_BIN" "$script"
+        # shellcheck disable=SC2086 # NODE_FLAGS must word-split into separate flags
+        exec "$NODE_BIN" $NODE_FLAGS "$script"
         ;;
     "/bin/sh" | "sh" | "/bin/bash" | "bash")
         exec "$@"

apps/worker/package.json

@@ -63,6 +63,7 @@
     "@faker-js/faker": "^10.4.0",
     "@mozilla/readability": "^0.6.0",
     "@platformatic/kafka": "^1.34.0",
+    "@sentry/node": "^10.56.0",
     "@t3-oss/env-core": "^0.13.11",
     "ai": "^6.0.170",
     "bullmq": "^5.76.4",

apps/worker/src/chat/worker.ts

@@ -11,6 +11,7 @@ import {
 import { type Job, Worker } from "bullmq"
 import { ensureBootstrapped } from "../lib/bootstrap"
 import { logger } from "../lib/logger"
+import { captureException, flushSentry, reportJobFailure } from "../lib/sentry"
 import { sendChatMessage, sendFlowStep } from "./handlers/send-flow-step"
 import {
   sendMessageToChannel,
@@ -25,6 +26,8 @@ async function startChatWorker() {
     logger.info("Chat worker bootstrapped successfully")
   } catch (err) {
     logger.error(err, "Failed to bootstrap chat worker")
+    captureException(err, { worker: "chat", phase: "bootstrap" })
+    await flushSentry()
     process.exit(1)
   }
 
@@ -75,6 +78,7 @@ async function startChatWorker() {
   worker.on("failed", (job, err) => {
     if (job) {
       logger.error(err, `Job ${job.id} has failed`)
+      reportJobFailure(job, err, { worker: "chat" })
     }
   })
 }

apps/worker/src/env.ts

@@ -9,6 +9,13 @@ export const env = createEnv({
   server: {
     NEXT_PUBLIC_EDITION: editionRule,
     QUOTA_SYNC_INTERVAL_SECONDS: z.coerce.number().int().min(10).default(60),
+    // Observability — all optional; omitting SENTRY_DSN disables Sentry.
+    // Read directly from process.env in instrument.ts (runs before this module).
+    SENTRY_DSN: z.url().optional(),
+    SENTRY_ENVIRONMENT: z.string().optional(),
+    SENTRY_RELEASE: z.string().optional(),
+    SENTRY_TRACES_SAMPLE_RATE: z.coerce.number().min(0).max(1).optional(),
+    WORKER_NAME: z.string().optional(),
   },
   runtimeEnv: process.env,
   skipValidation: process.env.SKIP_ENV_CHECK === "true",

apps/worker/src/instrument.ts

@@ -0,0 +1,33 @@
+/**
+ * Sentry initialization for the worker.
+ *
+ * This file MUST be loaded before any other module so Sentry can hook into
+ * the runtime early. In production it is wired via node's `--import` flag in
+ * `docker/rootfs/usr/local/bin/docker-entrypoint.sh`, so it runs ahead of every
+ * worker entrypoint.
+ *
+ * It reads configuration straight from `process.env` (not the validated `env`
+ * module) on purpose: this runs before anything else, and a missing DSN simply
+ * disables Sentry rather than throwing.
+ */
+import { init } from "@sentry/node"
+
+const dsn = process.env.SENTRY_DSN
+
+if (dsn) {
+  const tracesSampleRate = process.env.SENTRY_TRACES_SAMPLE_RATE
+    ? Number(process.env.SENTRY_TRACES_SAMPLE_RATE)
+    : 0
+
+  init({
+    dsn,
+    environment:
+      process.env.SENTRY_ENVIRONMENT ?? process.env.NODE_ENV ?? "development",
+    release: process.env.SENTRY_RELEASE,
+    // Errors are the priority. Performance tracing is sampled low (default off)
+    // to protect the Sentry quota — raise SENTRY_TRACES_SAMPLE_RATE when needed.
+    tracesSampleRate: Number.isFinite(tracesSampleRate) ? tracesSampleRate : 0,
+    // Identify which worker process emitted the event (set per container).
+    serverName: process.env.WORKER_NAME,
+  })
+}

apps/worker/src/lib/sentry.ts

@@ -0,0 +1,85 @@
+import {
+  flush,
+  getClient,
+  captureException as sentryCaptureException,
+} from "@sentry/node"
+import type { Job } from "bullmq"
+
+/**
+ * Whether Sentry was initialized (i.e. a DSN was provided). When false, every
+ * helper below is a cheap no-op, so call sites don't need their own guards.
+ */
+export function isSentryEnabled(): boolean {
+  return Boolean(getClient())
+}
+
+/**
+ * Capture an arbitrary exception with optional structured context.
+ * Safe to call even when Sentry is disabled — it just returns.
+ */
+export function captureException(
+  err: unknown,
+  context?: Record<string, unknown>,
+): void {
+  if (!isSentryEnabled()) {
+    return
+  }
+
+  sentryCaptureException(err, context ? { extra: context } : undefined)
+}
+
+/**
+ * Report a BullMQ job failure. Only the *final* attempt is sent: a job that
+ * still has retries left would otherwise emit a duplicate event on every
+ * attempt and burn through the Sentry quota during a retry storm.
+ *
+ * Job payloads can contain contact PII, so only identifiers and retry metadata
+ * are attached — never the raw `job.data`.
+ */
+export function reportJobFailure(
+  job: Job | undefined,
+  err: unknown,
+  context: { worker: string },
+): void {
+  if (!(job && isSentryEnabled())) {
+    return
+  }
+
+  const attemptsMade = job.attemptsMade ?? 0
+  const maxAttempts = job.opts?.attempts ?? 1
+
+  // Still has retries left — wait for the final attempt before reporting.
+  if (attemptsMade < maxAttempts) {
+    return
+  }
+
+  sentryCaptureException(err, (scope) => {
+    scope.setTag("worker", context.worker)
+    scope.setTag("queue", job.queueName)
+    scope.setTag("job.name", job.name)
+    scope.setContext("job", {
+      id: job.id,
+      name: job.name,
+      queue: job.queueName,
+      attemptsMade,
+      maxAttempts,
+    })
+    return scope
+  })
+}
+
+/**
+ * Flush queued Sentry events before the process exits. Best-effort: never block
+ * shutdown if Sentry is slow or unreachable.
+ */
+export async function flushSentry(timeoutMs = 2000): Promise<void> {
+  if (!isSentryEnabled()) {
+    return
+  }
+
+  try {
+    await flush(timeoutMs)
+  } catch {
+    // ignore — shutdown must not hang on Sentry
+  }
+}

apps/worker/tsdown.config.ts

@@ -3,6 +3,8 @@ import { defineConfig } from "tsdown"
 export default defineConfig({
   format: ["esm"],
   entry: [
+    // Loaded first via node --import; initializes Sentry before any worker runs.
+    "src/instrument.ts",
     "src/chat/worker.ts",
     "src/integration/worker.ts",
     "src/ai-agent/worker.ts",
@@ -29,6 +31,8 @@ export default defineConfig({
   minify: false,
   unbundle: false,
   // splitting: false,
-  sourcemap: false,
+  // Emit source maps so node --enable-source-maps and Sentry map stack traces
+  // in the bundled .mjs output back to the original TypeScript sources.
+  sourcemap: true,
   treeshake: true,
 })