From 3fe654a536e813f70367b23a0aaacbff5abf86f4 Mon Sep 17 00:00:00 2001
From: Matt Aitken <matt@mattaitken.com>
Date: Thu, 23 Apr 2026 20:53:06 +0100
Subject: [PATCH 01/17] test: add e2e webapp auth baseline and testcontainer
 infrastructure

Adds the ability to spawn the built webapp as a child process in tests,
plus a baseline of auth behaviour tests that will be used to verify the
upcoming apiBuilder RBAC migration leaves auth behaviour unchanged.

- internal-packages/testcontainers/src/webapp.ts: new helper that
  spawns build/server.js, waits for /healthcheck, and exposes a
  WebappInstance + startTestServer() convenience wrapper
- internal-packages/testcontainers/package.json: add ./webapp sub-path
  export so tests can import from @internal/testcontainers/webapp
- internal-packages/testcontainers/src/index.ts: export
  createPostgresContainer (needed by webapp.ts internally)
- apps/webapp/test/helpers/seedTestEnvironment.ts: creates a minimal
  org/project/environment row set for use in auth tests
- apps/webapp/test/api-auth.e2e.test.ts: 8 baseline tests covering
  API-key auth, JWT auth, missing/invalid credentials, and error shapes

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 apps/webapp/test/api-auth.e2e.test.ts         | 121 +++++++++++++++
 .../test/helpers/seedTestEnvironment.ts       |  43 ++++++
 internal-packages/testcontainers/package.json |   4 +
 internal-packages/testcontainers/src/index.ts |   2 +-
 .../testcontainers/src/webapp.ts              | 146 ++++++++++++++++++
 5 files changed, 315 insertions(+), 1 deletion(-)
 create mode 100644 apps/webapp/test/api-auth.e2e.test.ts
 create mode 100644 apps/webapp/test/helpers/seedTestEnvironment.ts
 create mode 100644 internal-packages/testcontainers/src/webapp.ts

diff --git a/apps/webapp/test/api-auth.e2e.test.ts b/apps/webapp/test/api-auth.e2e.test.ts
new file mode 100644
index 00000000000..e1d4963d159
--- /dev/null
+++ b/apps/webapp/test/api-auth.e2e.test.ts
@@ -0,0 +1,121 @@
+/**
+ * E2E auth baseline tests.
+ *
+ * These tests capture current auth behavior before the apiBuilder migration to RBAC.
+ * Run them before and after the migration to verify behavior is identical.
+ *
+ * Requires a pre-built webapp: pnpm run build --filter webapp
+ */
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import type { TestServer } from "@internal/testcontainers/webapp";
+import { startTestServer } from "@internal/testcontainers/webapp";
+import { generateJWT } from "@trigger.dev/core/v3/jwt";
+import { seedTestEnvironment } from "./helpers/seedTestEnvironment";
+
+vi.setConfig({ testTimeout: 180_000 });
+
+// Shared across all tests in this file — one postgres container + one webapp instance.
+let server: TestServer;
+
+beforeAll(async () => {
+  server = await startTestServer();
+}, 180_000);
+
+afterAll(async () => {
+  await server?.stop();
+}, 120_000);
+
+async function generateTestJWT(
+  environment: { id: string; apiKey: string },
+  options: { scopes?: string[] } = {}
+): Promise<string> {
+  const scopes = options.scopes ?? ["read:runs"];
+  return generateJWT({
+    secretKey: environment.apiKey,
+    payload: { pub: true, sub: environment.id, scopes },
+    expirationTime: "15m",
+  });
+}
+
+describe("API bearer auth — baseline behavior", () => {
+  it("valid API key: auth passes (404 not 401)", async () => {
+    const { apiKey } = await seedTestEnvironment(server.prisma);
+    const res = await server.webapp.fetch("/api/v1/runs/run_doesnotexist/result", {
+      headers: { Authorization: `Bearer ${apiKey}` },
+    });
+    // Auth passed — resource just doesn't exist
+    expect(res.status).not.toBe(401);
+    expect(res.status).not.toBe(403);
+  });
+
+  it("missing Authorization header: 401", async () => {
+    const res = await server.webapp.fetch("/api/v1/runs/run_doesnotexist/result");
+    expect(res.status).toBe(401);
+  });
+
+  it("invalid API key: 401", async () => {
+    const res = await server.webapp.fetch("/api/v1/runs/run_doesnotexist/result", {
+      headers: { Authorization: "Bearer tr_dev_completely_invalid_key_xyz_not_real" },
+    });
+    expect(res.status).toBe(401);
+  });
+
+  it("401 response has error field", async () => {
+    const res = await server.webapp.fetch("/api/v1/runs/run_doesnotexist/result");
+    const body = await res.json();
+    expect(body).toHaveProperty("error");
+  });
+});
+
+describe("JWT bearer auth — baseline behavior", () => {
+  it("valid JWT on JWT-enabled route: auth passes", async () => {
+    const { environment } = await seedTestEnvironment(server.prisma);
+    const jwt = await generateTestJWT(environment, { scopes: ["read:runs"] });
+
+    // /api/v1/runs has allowJWT: true with superScopes: ["read:runs", ...]
+    const res = await server.webapp.fetch("/api/v1/runs", {
+      headers: { Authorization: `Bearer ${jwt}` },
+    });
+
+    // Auth passed — 200 (empty list) or 400 (bad search params), not 401
+    expect(res.status).not.toBe(401);
+  });
+
+  it("valid JWT on non-JWT route: 401", async () => {
+    const { environment } = await seedTestEnvironment(server.prisma);
+    const jwt = await generateTestJWT(environment, { scopes: ["read:runs"] });
+
+    // /api/v1/runs/$runParam/result does NOT have allowJWT: true
+    const res = await server.webapp.fetch("/api/v1/runs/run_doesnotexist/result", {
+      headers: { Authorization: `Bearer ${jwt}` },
+    });
+
+    expect(res.status).toBe(401);
+  });
+
+  it("JWT with empty scopes on JWT-enabled route: 403", async () => {
+    const { environment } = await seedTestEnvironment(server.prisma);
+    const jwt = await generateTestJWT(environment, { scopes: [] });
+
+    const res = await server.webapp.fetch("/api/v1/runs", {
+      headers: { Authorization: `Bearer ${jwt}` },
+    });
+
+    // Empty scopes → no read:runs permission → 403
+    expect(res.status).toBe(403);
+  });
+
+  it("JWT signed with wrong key: 401", async () => {
+    const { environment } = await seedTestEnvironment(server.prisma);
+    const jwt = await generateJWT({
+      secretKey: "wrong-signing-key-that-does-not-match-environment-key",
+      payload: { pub: true, sub: environment.id, scopes: ["read:runs"] },
+    });
+
+    const res = await server.webapp.fetch("/api/v1/runs", {
+      headers: { Authorization: `Bearer ${jwt}` },
+    });
+
+    expect(res.status).toBe(401);
+  });
+});
diff --git a/apps/webapp/test/helpers/seedTestEnvironment.ts b/apps/webapp/test/helpers/seedTestEnvironment.ts
new file mode 100644
index 00000000000..92f0b543e4c
--- /dev/null
+++ b/apps/webapp/test/helpers/seedTestEnvironment.ts
@@ -0,0 +1,43 @@
+import type { PrismaClient } from "@trigger.dev/database";
+
+function randomHex(len = 12): string {
+  return Math.random().toString(16).slice(2, 2 + len).padEnd(len, "0");
+}
+
+export async function seedTestEnvironment(prisma: PrismaClient) {
+  const suffix = randomHex(8);
+  const apiKey = `tr_dev_${randomHex(24)}`;
+  const pkApiKey = `pk_dev_${randomHex(24)}`;
+
+  const organization = await prisma.organization.create({
+    data: {
+      title: `e2e-test-org-${suffix}`,
+      slug: `e2e-org-${suffix}`,
+      v3Enabled: true,
+    },
+  });
+
+  const project = await prisma.project.create({
+    data: {
+      name: `e2e-test-project-${suffix}`,
+      slug: `e2e-proj-${suffix}`,
+      externalRef: `proj_${suffix}`,
+      organizationId: organization.id,
+      engine: "V2",
+    },
+  });
+
+  const environment = await prisma.runtimeEnvironment.create({
+    data: {
+      slug: "dev",
+      type: "DEVELOPMENT",
+      apiKey,
+      pkApiKey,
+      shortcode: suffix.slice(0, 4),
+      projectId: project.id,
+      organizationId: organization.id,
+    },
+  });
+
+  return { organization, project, environment, apiKey };
+}
diff --git a/internal-packages/testcontainers/package.json b/internal-packages/testcontainers/package.json
index 3a2cee6d746..0d70ac6a3c2 100644
--- a/internal-packages/testcontainers/package.json
+++ b/internal-packages/testcontainers/package.json
@@ -4,6 +4,10 @@
   "version": "0.0.1",
   "main": "./src/index.ts",
   "types": "./src/index.ts",
+  "exports": {
+    ".": "./src/index.ts",
+    "./webapp": "./src/webapp.ts"
+  },
   "dependencies": {
     "@clickhouse/client": "^1.11.1",
     "@opentelemetry/api": "^1.9.0",
diff --git a/internal-packages/testcontainers/src/index.ts b/internal-packages/testcontainers/src/index.ts
index e9c8067de4c..f678fa01e7b 100644
--- a/internal-packages/testcontainers/src/index.ts
+++ b/internal-packages/testcontainers/src/index.ts
@@ -18,7 +18,7 @@ import { StartedClickHouseContainer } from "./clickhouse";
 import { StartedMinIOContainer, type MinIOConnectionConfig } from "./minio";
 import { ClickHouseClient, createClient } from "@clickhouse/client";
 
-export { assertNonNullable } from "./utils";
+export { assertNonNullable, createPostgresContainer } from "./utils";
 export { logCleanup };
 export type { MinIOConnectionConfig };
 
diff --git a/internal-packages/testcontainers/src/webapp.ts b/internal-packages/testcontainers/src/webapp.ts
new file mode 100644
index 00000000000..78af84c9345
--- /dev/null
+++ b/internal-packages/testcontainers/src/webapp.ts
@@ -0,0 +1,146 @@
+import { spawn } from "child_process";
+import { createServer } from "net";
+import { resolve } from "path";
+import { Network } from "testcontainers";
+import { PrismaClient } from "@trigger.dev/database";
+import { createPostgresContainer } from "./utils";
+
+const WEBAPP_ROOT = resolve(__dirname, "../../../apps/webapp");
+// pnpm hoists transitive deps to node_modules/.pnpm/node_modules but does NOT symlink them
+// to the root node_modules. We need NODE_PATH so the webapp process can find them at runtime.
+const PNPM_HOISTED_MODULES = resolve(__dirname, "../../../node_modules/.pnpm/node_modules");
+
+async function findFreePort(): Promise<number> {
+  return new Promise((res, rej) => {
+    const srv = createServer();
+    srv.listen(0, () => {
+      const port = (srv.address() as { port: number }).port;
+      srv.close((err) => (err ? rej(err) : res(port)));
+    });
+  });
+}
+
+async function waitForHealthcheck(url: string, timeoutMs = 60000): Promise<void> {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    try {
+      const res = await fetch(url);
+      if (res.ok) return;
+    } catch {}
+    await new Promise((r) => setTimeout(r, 500));
+  }
+  throw new Error(`Webapp did not become healthy at ${url} within ${timeoutMs}ms`);
+}
+
+export interface WebappInstance {
+  baseUrl: string;
+  fetch(path: string, init?: RequestInit): Promise<Response>;
+}
+
+export async function startWebapp(databaseUrl: string): Promise<{
+  instance: WebappInstance;
+  stop: () => Promise<void>;
+}> {
+  const port = await findFreePort();
+
+  // Merge NODE_PATH so transitive pnpm deps (hoisted to .pnpm/node_modules) are resolvable
+  const existingNodePath = process.env.NODE_PATH;
+  const nodePath = existingNodePath
+    ? `${PNPM_HOISTED_MODULES}:${existingNodePath}`
+    : PNPM_HOISTED_MODULES;
+
+  const proc = spawn(process.execPath, ["build/server.js"], {
+    cwd: WEBAPP_ROOT,
+    env: {
+      ...process.env,
+      NODE_ENV: "test",
+      DATABASE_URL: databaseUrl,
+      DIRECT_URL: databaseUrl,
+      PORT: String(port),
+      REMIX_APP_PORT: String(port), // override .env file value (vitest loads .env via Vite)
+      SESSION_SECRET: "test-session-secret-for-e2e-tests",
+      MAGIC_LINK_SECRET: "test-magic-link-secret-32chars!!",
+      ENCRYPTION_KEY: "test-encryption-key-for-e2e!!!!!", // exactly 32 bytes
+      CLICKHOUSE_URL: "http://localhost:19123", // dummy, auth paths never connect
+      DEPLOY_REGISTRY_HOST: "registry.example.com", // dummy, not needed for auth tests
+      ELECTRIC_ORIGIN: "http://localhost:3060",
+      NODE_PATH: nodePath,
+    },
+    stdio: ["ignore", "pipe", "pipe"],
+  });
+
+  const stderr: string[] = [];
+  proc.stderr?.on("data", (d: Buffer) => {
+    const line = d.toString();
+    stderr.push(line);
+    if (process.env.WEBAPP_TEST_VERBOSE) {
+      process.stderr.write(line);
+    }
+  });
+
+  const stdout: string[] = [];
+  proc.stdout?.on("data", (d: Buffer) => {
+    const line = d.toString();
+    stdout.push(line);
+    if (process.env.WEBAPP_TEST_VERBOSE) {
+      process.stdout.write(line);
+    }
+  });
+
+  proc.on("error", (err) => {
+    throw new Error(`Failed to start webapp: ${err.message}`);
+  });
+
+  const baseUrl = `http://localhost:${port}`;
+
+  try {
+    await waitForHealthcheck(`${baseUrl}/healthcheck`);
+  } catch (err) {
+    proc.kill("SIGTERM");
+    const output = [...stdout, ...stderr].join("\n");
+    throw new Error(`Webapp failed to start.\nOutput:\n${output}\n\nOriginal error: ${err}`);
+  }
+
+  return {
+    instance: {
+      baseUrl,
+      fetch: (path: string, init?: RequestInit) => fetch(`${baseUrl}${path}`, init),
+    },
+    stop: () =>
+      new Promise<void>((res) => {
+        const timer = setTimeout(() => {
+          proc.kill("SIGKILL");
+          res();
+        }, 10_000);
+        proc.once("exit", () => {
+          clearTimeout(timer);
+          res();
+        });
+        proc.kill("SIGTERM");
+      }),
+  };
+}
+
+export interface TestServer {
+  webapp: WebappInstance;
+  prisma: PrismaClient;
+  stop: () => Promise<void>;
+}
+
+/** Convenience helper: starts a postgres container + webapp and returns both for testing. */
+export async function startTestServer(): Promise<TestServer> {
+  const network = await new Network().start();
+  const { url: databaseUrl, container } = await createPostgresContainer(network);
+
+  const prisma = new PrismaClient({ datasources: { db: { url: databaseUrl } } });
+  const { instance: webapp, stop: stopWebapp } = await startWebapp(databaseUrl);
+
+  const stop = async () => {
+    await stopWebapp();
+    await prisma.$disconnect();
+    await container.stop();
+    await network.stop();
+  };
+
+  return { webapp, prisma, stop };
+}

From 7292806d1df0624605cd125efd7fe728302f99e5 Mon Sep 17 00:00:00 2001
From: Matt Aitken <matt@mattaitken.com>
Date: Thu, 23 Apr 2026 20:56:24 +0100
Subject: [PATCH 02/17] Potential fix for pull request finding 'CodeQL /
 Insecure randomness'

Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>
---
 apps/webapp/test/helpers/seedTestEnvironment.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/apps/webapp/test/helpers/seedTestEnvironment.ts b/apps/webapp/test/helpers/seedTestEnvironment.ts
index 92f0b543e4c..670927cc626 100644
--- a/apps/webapp/test/helpers/seedTestEnvironment.ts
+++ b/apps/webapp/test/helpers/seedTestEnvironment.ts
@@ -1,7 +1,8 @@
 import type { PrismaClient } from "@trigger.dev/database";
+import { randomBytes } from "crypto";
 
 function randomHex(len = 12): string {
-  return Math.random().toString(16).slice(2, 2 + len).padEnd(len, "0");
+  return randomBytes(Math.ceil(len / 2)).toString("hex").slice(0, len);
 }
 
 export async function seedTestEnvironment(prisma: PrismaClient) {

From b3c3f15804ca951f558d21f3033b23bec71f7728 Mon Sep 17 00:00:00 2001
From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Thu, 23 Apr 2026 20:10:26 +0000
Subject: [PATCH 03/17] fix: address Devin Review bugs in webapp testcontainer

- Capture spawn errors in a variable instead of throwing from the
  proc.on('error') listener. A synchronous throw inside an EventEmitter
  listener bypasses the surrounding try/catch and surfaces as an
  uncaughtException, tearing down the test runner.
- Wrap resource acquisition in startTestServer() in a try/catch so that
  a failure in startWebapp() (e.g. healthcheck timeout) tears down the
  postgres container, network, and PrismaClient instead of leaking them.

Co-Authored-By: Matt Aitken <matt@mattaitken.com>
---
 .../testcontainers/src/webapp.ts              | 41 +++++++++++++++----
 1 file changed, 33 insertions(+), 8 deletions(-)

diff --git a/internal-packages/testcontainers/src/webapp.ts b/internal-packages/testcontainers/src/webapp.ts
index 78af84c9345..6803acd340b 100644
--- a/internal-packages/testcontainers/src/webapp.ts
+++ b/internal-packages/testcontainers/src/webapp.ts
@@ -87,14 +87,20 @@ export async function startWebapp(databaseUrl: string): Promise<{
     }
   });
 
+  // Capture spawn errors instead of throwing from inside the listener.
+  // A synchronous throw from an EventEmitter listener bypasses the surrounding
+  // try/catch and surfaces as an uncaughtException, tearing down the test runner.
+  let spawnError: Error | undefined;
   proc.on("error", (err) => {
-    throw new Error(`Failed to start webapp: ${err.message}`);
+    spawnError = err;
   });
 
   const baseUrl = `http://localhost:${port}`;
 
   try {
+    if (spawnError) throw spawnError;
     await waitForHealthcheck(`${baseUrl}/healthcheck`);
+    if (spawnError) throw spawnError;
   } catch (err) {
     proc.kill("SIGTERM");
     const output = [...stdout, ...stderr].join("\n");
@@ -130,17 +136,36 @@ export interface TestServer {
 /** Convenience helper: starts a postgres container + webapp and returns both for testing. */
 export async function startTestServer(): Promise<TestServer> {
   const network = await new Network().start();
-  const { url: databaseUrl, container } = await createPostgresContainer(network);
 
-  const prisma = new PrismaClient({ datasources: { db: { url: databaseUrl } } });
-  const { instance: webapp, stop: stopWebapp } = await startWebapp(databaseUrl);
+  // Track each resource as we acquire it so we can tear it down if a later step fails.
+  // Without this, a healthcheck timeout in startWebapp() would leak the postgres
+  // container, network, and PrismaClient connection indefinitely.
+  let container: Awaited<ReturnType<typeof createPostgresContainer>>["container"] | undefined;
+  let prisma: PrismaClient | undefined;
+  let stopWebapp: (() => Promise<void>) | undefined;
+  let webapp: WebappInstance;
+
+  try {
+    const pg = await createPostgresContainer(network);
+    container = pg.container;
+    prisma = new PrismaClient({ datasources: { db: { url: pg.url } } });
+    const started = await startWebapp(pg.url);
+    webapp = started.instance;
+    stopWebapp = started.stop;
+  } catch (err) {
+    await stopWebapp?.().catch(() => {});
+    await prisma?.$disconnect().catch(() => {});
+    await container?.stop().catch(() => {});
+    await network.stop().catch(() => {});
+    throw err;
+  }
 
   const stop = async () => {
-    await stopWebapp();
-    await prisma.$disconnect();
-    await container.stop();
+    await stopWebapp!();
+    await prisma!.$disconnect();
+    await container!.stop();
     await network.stop();
   };
 
-  return { webapp, prisma, stop };
+  return { webapp, prisma: prisma!, stop };
 }

From ef8385339d0ac018ce2766a94a001847bebef7e9 Mon Sep 17 00:00:00 2001
From: Matt Aitken <matt@mattaitken.com>
Date: Fri, 24 Apr 2026 07:06:05 +0100
Subject: [PATCH 04/17] test: exclude *.e2e.test.ts from standard vitest shards

E2e tests require a pre-built webapp (build/server.js) which is not
present during normal CI test runs. Exclude them from the default
include pattern so the sharded test jobs don't pick them up.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 apps/webapp/vitest.config.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/apps/webapp/vitest.config.ts b/apps/webapp/vitest.config.ts
index 0c08af40ea4..2e51eb3f17d 100644
--- a/apps/webapp/vitest.config.ts
+++ b/apps/webapp/vitest.config.ts
@@ -4,6 +4,7 @@ import tsconfigPaths from "vite-tsconfig-paths";
 export default defineConfig({
   test: {
     include: ["test/**/*.test.ts"],
+    exclude: ["test/**/*.e2e.test.ts"],
     globals: true,
     pool: "forks",
   },

From 121cda1305ebc32e0bd61a44b0d606f2e47ed17a Mon Sep 17 00:00:00 2001
From: Matt Aitken <matt@mattaitken.com>
Date: Fri, 24 Apr 2026 07:33:33 +0100
Subject: [PATCH 05/17] ci: add dedicated e2e webapp job that builds first then
 runs auth tests

---
 .github/workflows/e2e-webapp.yml | 88 ++++++++++++++++++++++++++++++++
 .github/workflows/unit-tests.yml |  3 ++
 2 files changed, 91 insertions(+)
 create mode 100644 .github/workflows/e2e-webapp.yml

diff --git a/.github/workflows/e2e-webapp.yml b/.github/workflows/e2e-webapp.yml
new file mode 100644
index 00000000000..0013bff830b
--- /dev/null
+++ b/.github/workflows/e2e-webapp.yml
@@ -0,0 +1,88 @@
+name: "🧪 E2E Tests: Webapp"
+
+permissions:
+  contents: read
+
+on:
+  workflow_call:
+
+jobs:
+  e2eTests:
+    name: "🧪 E2E Tests: Webapp"
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    env:
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+    steps:
+      - name: 🔧 Disable IPv6
+        run: |
+          sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1
+          sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1
+          sudo sysctl -w net.ipv6.conf.lo.disable_ipv6=1
+
+      - name: 🔧 Configure docker address pool
+        run: |
+          CONFIG='{
+            "default-address-pools" : [
+              {
+                "base" : "172.17.0.0/12",
+                "size" : 20
+              },
+              {
+                "base" : "192.168.0.0/16",
+                "size" : 24
+              }
+            ]
+          }'
+          mkdir -p /etc/docker
+          echo "$CONFIG" | sudo tee /etc/docker/daemon.json
+
+      - name: 🔧 Restart docker daemon
+        run: sudo systemctl restart docker
+
+      - name: ⬇️ Checkout repo
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: ⎔ Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10.23.0
+
+      - name: ⎔ Setup node
+        uses: buildjet/setup-node@v4
+        with:
+          node-version: 20.20.0
+          cache: "pnpm"
+
+      # ..to avoid rate limits when pulling images
+      - name: 🐳 Login to DockerHub
+        if: ${{ env.DOCKERHUB_USERNAME }}
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: 🐳 Skipping DockerHub login (no secrets available)
+        if: ${{ !env.DOCKERHUB_USERNAME }}
+        run: echo "DockerHub login skipped because secrets are not available."
+
+      - name: 🐳 Pre-pull testcontainer images
+        if: ${{ env.DOCKERHUB_USERNAME }}
+        run: |
+          echo "Pre-pulling Docker images with authenticated session..."
+          docker pull postgres:14
+          docker pull testcontainers/ryuk:0.11.0
+          echo "Image pre-pull complete"
+
+      - name: 📥 Download deps
+        run: pnpm install --frozen-lockfile
+
+      - name: 📀 Generate Prisma Client
+        run: pnpm run generate
+
+      - name: 🏗️ Build Webapp
+        run: pnpm run build --filter webapp
+
+      - name: 🧪 Run Webapp E2E Tests
+        run: cd apps/webapp && pnpm exec vitest run test/api-auth.e2e.test.ts --reporter=default
diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
index 7c90a5a30ad..2c4276a5aa0 100644
--- a/.github/workflows/unit-tests.yml
+++ b/.github/workflows/unit-tests.yml
@@ -10,6 +10,9 @@ jobs:
   webapp:
     uses: ./.github/workflows/unit-tests-webapp.yml
     secrets: inherit
+  e2e-webapp:
+    uses: ./.github/workflows/e2e-webapp.yml
+    secrets: inherit
   packages:
     uses: ./.github/workflows/unit-tests-packages.yml
     secrets: inherit

From 8413af5c5b101786d35314228e607f73b0d6eb06 Mon Sep 17 00:00:00 2001
From: Matt Aitken <matt@mattaitken.com>
Date: Fri, 24 Apr 2026 07:35:50 +0100
Subject: [PATCH 06/17] fix: use path.delimiter instead of hard-coded colon in
 NODE_PATH join

---
 internal-packages/testcontainers/src/webapp.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/internal-packages/testcontainers/src/webapp.ts b/internal-packages/testcontainers/src/webapp.ts
index 6803acd340b..b66be927f10 100644
--- a/internal-packages/testcontainers/src/webapp.ts
+++ b/internal-packages/testcontainers/src/webapp.ts
@@ -1,6 +1,6 @@
 import { spawn } from "child_process";
 import { createServer } from "net";
-import { resolve } from "path";
+import { delimiter, resolve } from "path";
 import { Network } from "testcontainers";
 import { PrismaClient } from "@trigger.dev/database";
 import { createPostgresContainer } from "./utils";
@@ -46,7 +46,7 @@ export async function startWebapp(databaseUrl: string): Promise<{
   // Merge NODE_PATH so transitive pnpm deps (hoisted to .pnpm/node_modules) are resolvable
   const existingNodePath = process.env.NODE_PATH;
   const nodePath = existingNodePath
-    ? `${PNPM_HOISTED_MODULES}:${existingNodePath}`
+    ? `${PNPM_HOISTED_MODULES}${delimiter}${existingNodePath}`
     : PNPM_HOISTED_MODULES;
 
   const proc = spawn(process.execPath, ["build/server.js"], {

From 419a2dfdbff203858dfdd8556b7bc5abbe64380a Mon Sep 17 00:00:00 2001
From: Matt Aitken <matt@mattaitken.com>
Date: Fri, 24 Apr 2026 07:37:20 +0100
Subject: [PATCH 07/17] fix: import vi explicitly in api-auth e2e test

---
 apps/webapp/test/api-auth.e2e.test.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/webapp/test/api-auth.e2e.test.ts b/apps/webapp/test/api-auth.e2e.test.ts
index e1d4963d159..c425ca7449c 100644
--- a/apps/webapp/test/api-auth.e2e.test.ts
+++ b/apps/webapp/test/api-auth.e2e.test.ts
@@ -6,7 +6,7 @@
  *
  * Requires a pre-built webapp: pnpm run build --filter webapp
  */
-import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
 import type { TestServer } from "@internal/testcontainers/webapp";
 import { startTestServer } from "@internal/testcontainers/webapp";
 import { generateJWT } from "@trigger.dev/core/v3/jwt";

From 9e9ff329f77c6b476239904499c5854d4bfd3a60 Mon Sep 17 00:00:00 2001
From: Matt Aitken <matt@mattaitken.com>
Date: Fri, 24 Apr 2026 07:39:12 +0100
Subject: [PATCH 08/17] fix: best-effort teardown in stop() to prevent resource
 leaks

---
 internal-packages/testcontainers/src/webapp.ts | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/internal-packages/testcontainers/src/webapp.ts b/internal-packages/testcontainers/src/webapp.ts
index b66be927f10..403a01c3f9a 100644
--- a/internal-packages/testcontainers/src/webapp.ts
+++ b/internal-packages/testcontainers/src/webapp.ts
@@ -161,10 +161,10 @@ export async function startTestServer(): Promise<TestServer> {
   }
 
   const stop = async () => {
-    await stopWebapp!();
-    await prisma!.$disconnect();
-    await container!.stop();
-    await network.stop();
+    await stopWebapp!().catch((err) => console.error("stopWebapp failed:", err));
+    await prisma!.$disconnect().catch((err) => console.error("prisma.$disconnect failed:", err));
+    await container!.stop().catch((err) => console.error("container.stop failed:", err));
+    await network.stop().catch((err) => console.error("network.stop failed:", err));
   };
 
   return { webapp, prisma: prisma!, stop };

From 407a3c02c492eba2a948046d55fcb90a876c4de7 Mon Sep 17 00:00:00 2001
From: Matt Aitken <matt@mattaitken.com>
Date: Fri, 24 Apr 2026 08:06:58 +0100
Subject: [PATCH 09/17] fix: add vitest.e2e.config.ts so e2e tests aren't
 excluded in CI

---
 .github/workflows/e2e-webapp.yml |  2 +-
 apps/webapp/vitest.e2e.config.ts | 12 ++++++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)
 create mode 100644 apps/webapp/vitest.e2e.config.ts

diff --git a/.github/workflows/e2e-webapp.yml b/.github/workflows/e2e-webapp.yml
index 0013bff830b..45f48244345 100644
--- a/.github/workflows/e2e-webapp.yml
+++ b/.github/workflows/e2e-webapp.yml
@@ -85,4 +85,4 @@ jobs:
         run: pnpm run build --filter webapp
 
       - name: 🧪 Run Webapp E2E Tests
-        run: cd apps/webapp && pnpm exec vitest run test/api-auth.e2e.test.ts --reporter=default
+        run: cd apps/webapp && pnpm exec vitest run --config vitest.e2e.config.ts --reporter=default
diff --git a/apps/webapp/vitest.e2e.config.ts b/apps/webapp/vitest.e2e.config.ts
new file mode 100644
index 00000000000..0c5c1e86c84
--- /dev/null
+++ b/apps/webapp/vitest.e2e.config.ts
@@ -0,0 +1,12 @@
+import { defineConfig } from "vitest/config";
+import tsconfigPaths from "vite-tsconfig-paths";
+
+export default defineConfig({
+  test: {
+    include: ["test/**/*.e2e.test.ts"],
+    globals: true,
+    pool: "forks",
+  },
+  // @ts-ignore
+  plugins: [tsconfigPaths({ projects: ["./tsconfig.json"] })],
+});

From 90f7f13ed79d208064a5f494f80be101ed3957f5 Mon Sep 17 00:00:00 2001
From: Matt Aitken <matt@mattaitken.com>
Date: Fri, 24 Apr 2026 08:13:13 +0100
Subject: [PATCH 10/17] fix: add dummy REDIS_HOST/PORT so webapp passes
 module-level validation on startup

---
 internal-packages/testcontainers/src/webapp.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/internal-packages/testcontainers/src/webapp.ts b/internal-packages/testcontainers/src/webapp.ts
index 403a01c3f9a..dc4c7221bb1 100644
--- a/internal-packages/testcontainers/src/webapp.ts
+++ b/internal-packages/testcontainers/src/webapp.ts
@@ -64,6 +64,8 @@ export async function startWebapp(databaseUrl: string): Promise<{
       CLICKHOUSE_URL: "http://localhost:19123", // dummy, auth paths never connect
       DEPLOY_REGISTRY_HOST: "registry.example.com", // dummy, not needed for auth tests
       ELECTRIC_ORIGIN: "http://localhost:3060",
+      REDIS_HOST: "localhost", // dummy, satisfies module-level validation; auth paths never use Redis
+      REDIS_PORT: "6379",
       NODE_PATH: nodePath,
     },
     stdio: ["ignore", "pipe", "pipe"],

From b58176f6ddf81d19133c20ed6cf304a3cbbe832d Mon Sep 17 00:00:00 2001
From: Matt Aitken <matt@mattaitken.com>
Date: Fri, 24 Apr 2026 09:39:09 +0100
Subject: [PATCH 11/17] fix: spin up real Redis container in startTestServer;
 pass host/port to startWebapp

---
 .github/workflows/e2e-webapp.yml              |  1 +
 .../testcontainers/src/webapp.ts              | 32 ++++++++++++-------
 2 files changed, 21 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/e2e-webapp.yml b/.github/workflows/e2e-webapp.yml
index 45f48244345..ae977e1c608 100644
--- a/.github/workflows/e2e-webapp.yml
+++ b/.github/workflows/e2e-webapp.yml
@@ -72,6 +72,7 @@ jobs:
         run: |
           echo "Pre-pulling Docker images with authenticated session..."
           docker pull postgres:14
+          docker pull redis:7-alpine
           docker pull testcontainers/ryuk:0.11.0
           echo "Image pre-pull complete"
 
diff --git a/internal-packages/testcontainers/src/webapp.ts b/internal-packages/testcontainers/src/webapp.ts
index dc4c7221bb1..7ffab6c77bd 100644
--- a/internal-packages/testcontainers/src/webapp.ts
+++ b/internal-packages/testcontainers/src/webapp.ts
@@ -3,7 +3,7 @@ import { createServer } from "net";
 import { delimiter, resolve } from "path";
 import { Network } from "testcontainers";
 import { PrismaClient } from "@trigger.dev/database";
-import { createPostgresContainer } from "./utils";
+import { createPostgresContainer, createRedisContainer } from "./utils";
 
 const WEBAPP_ROOT = resolve(__dirname, "../../../apps/webapp");
 // pnpm hoists transitive deps to node_modules/.pnpm/node_modules but does NOT symlink them
@@ -37,7 +37,10 @@ export interface WebappInstance {
   fetch(path: string, init?: RequestInit): Promise<Response>;
 }
 
-export async function startWebapp(databaseUrl: string): Promise<{
+export async function startWebapp(
+  databaseUrl: string,
+  redis: { host: string; port: number }
+): Promise<{
   instance: WebappInstance;
   stop: () => Promise<void>;
 }> {
@@ -64,8 +67,8 @@ export async function startWebapp(databaseUrl: string): Promise<{
       CLICKHOUSE_URL: "http://localhost:19123", // dummy, auth paths never connect
       DEPLOY_REGISTRY_HOST: "registry.example.com", // dummy, not needed for auth tests
       ELECTRIC_ORIGIN: "http://localhost:3060",
-      REDIS_HOST: "localhost", // dummy, satisfies module-level validation; auth paths never use Redis
-      REDIS_PORT: "6379",
+      REDIS_HOST: redis.host,
+      REDIS_PORT: String(redis.port),
       NODE_PATH: nodePath,
     },
     stdio: ["ignore", "pipe", "pipe"],
@@ -135,29 +138,33 @@ export interface TestServer {
   stop: () => Promise<void>;
 }
 
-/** Convenience helper: starts a postgres container + webapp and returns both for testing. */
+/** Convenience helper: starts a postgres + redis container + webapp and returns both for testing. */
 export async function startTestServer(): Promise<TestServer> {
   const network = await new Network().start();
 
   // Track each resource as we acquire it so we can tear it down if a later step fails.
-  // Without this, a healthcheck timeout in startWebapp() would leak the postgres
-  // container, network, and PrismaClient connection indefinitely.
-  let container: Awaited<ReturnType<typeof createPostgresContainer>>["container"] | undefined;
+  let pgContainer: Awaited<ReturnType<typeof createPostgresContainer>>["container"] | undefined;
+  let redisContainer: Awaited<ReturnType<typeof createRedisContainer>>["container"] | undefined;
   let prisma: PrismaClient | undefined;
   let stopWebapp: (() => Promise<void>) | undefined;
   let webapp: WebappInstance;
 
   try {
     const pg = await createPostgresContainer(network);
-    container = pg.container;
+    pgContainer = pg.container;
+
+    const { container: rc } = await createRedisContainer({ network });
+    redisContainer = rc;
+
     prisma = new PrismaClient({ datasources: { db: { url: pg.url } } });
-    const started = await startWebapp(pg.url);
+    const started = await startWebapp(pg.url, { host: rc.getHost(), port: rc.getPort() });
     webapp = started.instance;
     stopWebapp = started.stop;
   } catch (err) {
     await stopWebapp?.().catch(() => {});
     await prisma?.$disconnect().catch(() => {});
-    await container?.stop().catch(() => {});
+    await pgContainer?.stop().catch(() => {});
+    await redisContainer?.stop().catch(() => {});
     await network.stop().catch(() => {});
     throw err;
   }
@@ -165,7 +172,8 @@ export async function startTestServer(): Promise<TestServer> {
   const stop = async () => {
     await stopWebapp!().catch((err) => console.error("stopWebapp failed:", err));
     await prisma!.$disconnect().catch((err) => console.error("prisma.$disconnect failed:", err));
-    await container!.stop().catch((err) => console.error("container.stop failed:", err));
+    await pgContainer!.stop().catch((err) => console.error("pgContainer.stop failed:", err));
+    await redisContainer!.stop().catch((err) => console.error("redisContainer.stop failed:", err));
     await network.stop().catch((err) => console.error("network.stop failed:", err));
   };
 

From 63030bf7ffabeb6e61d8885e05f6880343fd272b Mon Sep 17 00:00:00 2001
From: Matt Aitken <matt@mattaitken.com>
Date: Fri, 24 Apr 2026 10:24:18 +0100
Subject: [PATCH 12/17] fix: disable RUN_REPLICATION_ENABLED and WORKER_ENABLED
 in test webapp to prevent slot lock blocking test DB writes

---
 internal-packages/testcontainers/src/webapp.ts | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/internal-packages/testcontainers/src/webapp.ts b/internal-packages/testcontainers/src/webapp.ts
index 7ffab6c77bd..bac02cde510 100644
--- a/internal-packages/testcontainers/src/webapp.ts
+++ b/internal-packages/testcontainers/src/webapp.ts
@@ -69,6 +69,11 @@ export async function startWebapp(
       ELECTRIC_ORIGIN: "http://localhost:3060",
       REDIS_HOST: redis.host,
       REDIS_PORT: String(redis.port),
+      // Disable background workers and logical replication: they are irrelevant for auth
+      // tests and in CI the replication slot creation holds a snapshot lock that blocks
+      // the test process's Prisma writes to the same DB.
+      WORKER_ENABLED: "false",
+      RUN_REPLICATION_ENABLED: "0",
       NODE_PATH: nodePath,
     },
     stdio: ["ignore", "pipe", "pipe"],

From acdbbb61175ae17ebf6a4a9a81d1b5c1addb3763 Mon Sep 17 00:00:00 2001
From: Matt Aitken <matt@mattaitken.com>
Date: Fri, 24 Apr 2026 10:34:08 +0100
Subject: [PATCH 13/17] debug: enable WEBAPP_TEST_VERBOSE in CI; pre-warm
 prisma pool before tests start

---
 .github/workflows/e2e-webapp.yml               | 2 ++
 internal-packages/testcontainers/src/webapp.ts | 1 +
 2 files changed, 3 insertions(+)

diff --git a/.github/workflows/e2e-webapp.yml b/.github/workflows/e2e-webapp.yml
index ae977e1c608..ac47aa093c9 100644
--- a/.github/workflows/e2e-webapp.yml
+++ b/.github/workflows/e2e-webapp.yml
@@ -87,3 +87,5 @@ jobs:
 
       - name: 🧪 Run Webapp E2E Tests
         run: cd apps/webapp && pnpm exec vitest run --config vitest.e2e.config.ts --reporter=default
+        env:
+          WEBAPP_TEST_VERBOSE: "1"
diff --git a/internal-packages/testcontainers/src/webapp.ts b/internal-packages/testcontainers/src/webapp.ts
index bac02cde510..cc2033d419b 100644
--- a/internal-packages/testcontainers/src/webapp.ts
+++ b/internal-packages/testcontainers/src/webapp.ts
@@ -162,6 +162,7 @@ export async function startTestServer(): Promise<TestServer> {
     redisContainer = rc;
 
     prisma = new PrismaClient({ datasources: { db: { url: pg.url } } });
+    await prisma.$connect(); // pre-warm pool; surface connection failures before tests start
     const started = await startWebapp(pg.url, { host: rc.getHost(), port: rc.getPort() });
     webapp = started.instance;
     stopWebapp = started.stop;

From a846e72e60927d5851a2f2c9aed6bef962f223bf Mon Sep 17 00:00:00 2001
From: Matt Aitken <matt@mattaitken.com>
Date: Fri, 24 Apr 2026 11:29:43 +0100
Subject: [PATCH 14/17] debug: disable all workers + pg_stat_activity
 diagnostics for hang investigation

- Fix SCHEDULE_WORKER_ENABLED (was 'false', must be '0' to disable)
- Add RUN_ENGINE_WORKER_ENABLED: '0' (was missing, defaulted to '1')
- Add BATCH_QUEUE_WORKER_ENABLED: 'false' (not controlled by WORKER_ENABLED)
- Add LEGACY/COMMON/TTL worker disabling env vars for completeness
- Add pg_stat_activity polling every 10s when WEBAPP_TEST_VERBOSE=1
- Add per-step timing logs in seedTestEnvironment to identify which DB
  operation hangs

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 .../test/helpers/seedTestEnvironment.ts       |  6 +++
 .../testcontainers/src/webapp.ts              | 37 +++++++++++++++++--
 2 files changed, 39 insertions(+), 4 deletions(-)

diff --git a/apps/webapp/test/helpers/seedTestEnvironment.ts b/apps/webapp/test/helpers/seedTestEnvironment.ts
index 670927cc626..69b6773213a 100644
--- a/apps/webapp/test/helpers/seedTestEnvironment.ts
+++ b/apps/webapp/test/helpers/seedTestEnvironment.ts
@@ -10,6 +10,9 @@ export async function seedTestEnvironment(prisma: PrismaClient) {
   const apiKey = `tr_dev_${randomHex(24)}`;
   const pkApiKey = `pk_dev_${randomHex(24)}`;
 
+  const t0 = Date.now();
+  process.stderr.write(`[seed] creating organization (${suffix})...\n`);
+
   const organization = await prisma.organization.create({
     data: {
       title: `e2e-test-org-${suffix}`,
@@ -17,6 +20,7 @@ export async function seedTestEnvironment(prisma: PrismaClient) {
       v3Enabled: true,
     },
   });
+  process.stderr.write(`[seed] organization created in ${Date.now() - t0}ms\n`);
 
   const project = await prisma.project.create({
     data: {
@@ -27,6 +31,7 @@ export async function seedTestEnvironment(prisma: PrismaClient) {
       engine: "V2",
     },
   });
+  process.stderr.write(`[seed] project created in ${Date.now() - t0}ms\n`);
 
   const environment = await prisma.runtimeEnvironment.create({
     data: {
@@ -39,6 +44,7 @@ export async function seedTestEnvironment(prisma: PrismaClient) {
       organizationId: organization.id,
     },
   });
+  process.stderr.write(`[seed] environment created in ${Date.now() - t0}ms\n`);
 
   return { organization, project, environment, apiKey };
 }
diff --git a/internal-packages/testcontainers/src/webapp.ts b/internal-packages/testcontainers/src/webapp.ts
index cc2033d419b..c969c31cea6 100644
--- a/internal-packages/testcontainers/src/webapp.ts
+++ b/internal-packages/testcontainers/src/webapp.ts
@@ -69,10 +69,16 @@ export async function startWebapp(
       ELECTRIC_ORIGIN: "http://localhost:3060",
       REDIS_HOST: redis.host,
       REDIS_PORT: String(redis.port),
-      // Disable background workers and logical replication: they are irrelevant for auth
-      // tests and in CI the replication slot creation holds a snapshot lock that blocks
-      // the test process's Prisma writes to the same DB.
-      WORKER_ENABLED: "false",
+      // Disable all background workers. Each worker has its own env var and its own
+      // check idiom ("0" vs "false" vs boolean), so we set all of them explicitly.
+      WORKER_ENABLED: "false",          // disables workerQueue.initialize() (checked === "true")
+      RUN_ENGINE_WORKER_ENABLED: "0",   // disables run engine workers (checked === "0", default "1")
+      SCHEDULE_WORKER_ENABLED: "0",     // disables schedule engine worker (checked === "0")
+      BATCH_QUEUE_WORKER_ENABLED: "false", // disables batch queue consumers (BoolEnv)
+      LEGACY_RUN_ENGINE_WORKER_ENABLED: "0", // disables legacy run engine worker
+      COMMON_WORKER_ENABLED: "0",       // disables common worker
+      RUN_ENGINE_TTL_SYSTEM_DISABLED: "true",     // disables TTL expiry system (BoolEnv)
+      RUN_ENGINE_TTL_CONSUMERS_DISABLED: "true",  // disables TTL consumers (BoolEnv)
       RUN_REPLICATION_ENABLED: "0",
       NODE_PATH: nodePath,
     },
@@ -153,6 +159,7 @@ export async function startTestServer(): Promise<TestServer> {
   let prisma: PrismaClient | undefined;
   let stopWebapp: (() => Promise<void>) | undefined;
   let webapp: WebappInstance;
+  let diagInterval: ReturnType<typeof setInterval> | undefined;
 
   try {
     const pg = await createPostgresContainer(network);
@@ -163,6 +170,27 @@ export async function startTestServer(): Promise<TestServer> {
 
     prisma = new PrismaClient({ datasources: { db: { url: pg.url } } });
     await prisma.$connect(); // pre-warm pool; surface connection failures before tests start
+
+    // Periodically dump pg_stat_activity when debugging to identify hangs.
+    if (process.env.WEBAPP_TEST_VERBOSE) {
+      diagInterval = setInterval(async () => {
+        try {
+          const rows = await prisma!.$queryRawUnsafe<Record<string, unknown>[]>(`
+            SELECT pid, state, wait_event_type, wait_event,
+                   LEFT(query, 300) AS query,
+                   EXTRACT(EPOCH FROM (now() - state_change))::int AS state_age_secs,
+                   EXTRACT(EPOCH FROM (now() - query_start))::int  AS query_age_secs
+            FROM   pg_stat_activity
+            WHERE  datname = current_database() AND pid != pg_backend_pid()
+            ORDER  BY query_start NULLS LAST
+          `);
+          process.stderr.write(`[pg_stat_activity] ${JSON.stringify(rows)}\n`);
+        } catch {
+          // Diagnostic failures are non-fatal; ignore them.
+        }
+      }, 10_000);
+    }
+
     const started = await startWebapp(pg.url, { host: rc.getHost(), port: rc.getPort() });
     webapp = started.instance;
     stopWebapp = started.stop;
@@ -176,6 +204,7 @@ export async function startTestServer(): Promise<TestServer> {
   }
 
   const stop = async () => {
+    if (diagInterval) clearInterval(diagInterval);
     await stopWebapp!().catch((err) => console.error("stopWebapp failed:", err));
     await prisma!.$disconnect().catch((err) => console.error("prisma.$disconnect failed:", err));
     await pgContainer!.stop().catch((err) => console.error("pgContainer.stop failed:", err));

From 24399e33c7e434c06607370ae8fdd719a6f69612 Mon Sep 17 00:00:00 2001
From: Matt Aitken <matt@mattaitken.com>
Date: Fri, 24 Apr 2026 11:40:22 +0100
Subject: [PATCH 15/17] =?UTF-8?q?fix:=20disable=20Redis=20TLS=20for=20test?=
 =?UTF-8?q?=20webapp=20=E2=80=94=20all=20connections=20were=20timing=20out?=
 =?UTF-8?q?=20on=20TLS=20handshake?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The webapp's cacheStore and rateLimiter Redis connections default TLS to
enabled. Every request with a valid auth header used these connections,
causing connect ETIMEDOUT (TLS handshake against a non-TLS Redis server).
REDIS_TLS_DISABLED=true cascades to all 15+ *_REDIS_TLS_DISABLED env vars.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 internal-packages/testcontainers/src/webapp.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/internal-packages/testcontainers/src/webapp.ts b/internal-packages/testcontainers/src/webapp.ts
index c969c31cea6..3bcf1524288 100644
--- a/internal-packages/testcontainers/src/webapp.ts
+++ b/internal-packages/testcontainers/src/webapp.ts
@@ -69,6 +69,7 @@ export async function startWebapp(
       ELECTRIC_ORIGIN: "http://localhost:3060",
       REDIS_HOST: redis.host,
       REDIS_PORT: String(redis.port),
+      REDIS_TLS_DISABLED: "true", // all *_REDIS_TLS_DISABLED vars default to this; test Redis has no TLS
       // Disable all background workers. Each worker has its own env var and its own
       // check idiom ("0" vs "false" vs boolean), so we set all of them explicitly.
       WORKER_ENABLED: "false",          // disables workerQueue.initialize() (checked === "true")

From 19bc0161a64c41c62b4d38757a8c8eee74fce7fa Mon Sep 17 00:00:00 2001
From: Matt Aitken <matt@mattaitken.com>
Date: Fri, 24 Apr 2026 11:48:25 +0100
Subject: [PATCH 16/17] chore: remove debug diagnostics after root cause
 identified and fixed

---
 .../test/helpers/seedTestEnvironment.ts       |  6 -----
 .../testcontainers/src/webapp.ts              | 23 -------------------
 2 files changed, 29 deletions(-)

diff --git a/apps/webapp/test/helpers/seedTestEnvironment.ts b/apps/webapp/test/helpers/seedTestEnvironment.ts
index 69b6773213a..670927cc626 100644
--- a/apps/webapp/test/helpers/seedTestEnvironment.ts
+++ b/apps/webapp/test/helpers/seedTestEnvironment.ts
@@ -10,9 +10,6 @@ export async function seedTestEnvironment(prisma: PrismaClient) {
   const apiKey = `tr_dev_${randomHex(24)}`;
   const pkApiKey = `pk_dev_${randomHex(24)}`;
 
-  const t0 = Date.now();
-  process.stderr.write(`[seed] creating organization (${suffix})...\n`);
-
   const organization = await prisma.organization.create({
     data: {
       title: `e2e-test-org-${suffix}`,
@@ -20,7 +17,6 @@ export async function seedTestEnvironment(prisma: PrismaClient) {
       v3Enabled: true,
     },
   });
-  process.stderr.write(`[seed] organization created in ${Date.now() - t0}ms\n`);
 
   const project = await prisma.project.create({
     data: {
@@ -31,7 +27,6 @@ export async function seedTestEnvironment(prisma: PrismaClient) {
       engine: "V2",
     },
   });
-  process.stderr.write(`[seed] project created in ${Date.now() - t0}ms\n`);
 
   const environment = await prisma.runtimeEnvironment.create({
     data: {
@@ -44,7 +39,6 @@ export async function seedTestEnvironment(prisma: PrismaClient) {
       organizationId: organization.id,
     },
   });
-  process.stderr.write(`[seed] environment created in ${Date.now() - t0}ms\n`);
 
   return { organization, project, environment, apiKey };
 }
diff --git a/internal-packages/testcontainers/src/webapp.ts b/internal-packages/testcontainers/src/webapp.ts
index 3bcf1524288..9530f4c38fb 100644
--- a/internal-packages/testcontainers/src/webapp.ts
+++ b/internal-packages/testcontainers/src/webapp.ts
@@ -160,7 +160,6 @@ export async function startTestServer(): Promise<TestServer> {
   let prisma: PrismaClient | undefined;
   let stopWebapp: (() => Promise<void>) | undefined;
   let webapp: WebappInstance;
-  let diagInterval: ReturnType<typeof setInterval> | undefined;
 
   try {
     const pg = await createPostgresContainer(network);
@@ -171,27 +170,6 @@ export async function startTestServer(): Promise<TestServer> {
 
     prisma = new PrismaClient({ datasources: { db: { url: pg.url } } });
     await prisma.$connect(); // pre-warm pool; surface connection failures before tests start
-
-    // Periodically dump pg_stat_activity when debugging to identify hangs.
-    if (process.env.WEBAPP_TEST_VERBOSE) {
-      diagInterval = setInterval(async () => {
-        try {
-          const rows = await prisma!.$queryRawUnsafe<Record<string, unknown>[]>(`
-            SELECT pid, state, wait_event_type, wait_event,
-                   LEFT(query, 300) AS query,
-                   EXTRACT(EPOCH FROM (now() - state_change))::int AS state_age_secs,
-                   EXTRACT(EPOCH FROM (now() - query_start))::int  AS query_age_secs
-            FROM   pg_stat_activity
-            WHERE  datname = current_database() AND pid != pg_backend_pid()
-            ORDER  BY query_start NULLS LAST
-          `);
-          process.stderr.write(`[pg_stat_activity] ${JSON.stringify(rows)}\n`);
-        } catch {
-          // Diagnostic failures are non-fatal; ignore them.
-        }
-      }, 10_000);
-    }
-
     const started = await startWebapp(pg.url, { host: rc.getHost(), port: rc.getPort() });
     webapp = started.instance;
     stopWebapp = started.stop;
@@ -205,7 +183,6 @@ export async function startTestServer(): Promise<TestServer> {
   }
 
   const stop = async () => {
-    if (diagInterval) clearInterval(diagInterval);
     await stopWebapp!().catch((err) => console.error("stopWebapp failed:", err));
     await prisma!.$disconnect().catch((err) => console.error("prisma.$disconnect failed:", err));
     await pgContainer!.stop().catch((err) => console.error("pgContainer.stop failed:", err));

From dee80ae364b90454b28813fe0fd4a384fbe077b6 Mon Sep 17 00:00:00 2001
From: Matt Aitken <matt@mattaitken.com>
Date: Fri, 24 Apr 2026 11:50:54 +0100
Subject: [PATCH 17/17] fix: align Redis pre-pull image tag with
 @testcontainers/redis default (redis:7.2)

---
 .github/workflows/e2e-webapp.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/e2e-webapp.yml b/.github/workflows/e2e-webapp.yml
index ac47aa093c9..9a58aa58c7b 100644
--- a/.github/workflows/e2e-webapp.yml
+++ b/.github/workflows/e2e-webapp.yml
@@ -72,7 +72,7 @@ jobs:
         run: |
           echo "Pre-pulling Docker images with authenticated session..."
           docker pull postgres:14
-          docker pull redis:7-alpine
+          docker pull redis:7.2
           docker pull testcontainers/ryuk:0.11.0
           echo "Image pre-pull complete"