Harden Playwright foundation and persona fidelity (#5820)

* Harden Playwright lifecycle and persona fidelity

Close production-state and cleanup gaps so the settings E2E platform fails closed without sacrificing useful diagnostics.

* Serialize E2E signal cleanup ownership

Prevent opposite signals and normal finalization from racing detached cleanup or releasing its run lock.

* Preserve browser fixture failures

Report network isolation violations without masking the original Playwright test or fixture error.

* Make E2E signal handoff resilient

Keep cleanup single-flight across repeated signals and retain lock ownership when supervisor logging or startup fails.

* fix(e2e): preserve replacement org coverage

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix(e2e): derive usage from entitled coverage

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix(e2e): serialize cleanup lock ownership

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix(ci): detect safe E2E diagnostics directly

Co-authored-by: Cursor <cursoragent@cursor.com>

---------

Co-authored-by: Bill Leoutsakos <billleoutsakos@Bills-MacBook-Pro.local>
Co-authored-by: Cursor <cursoragent@cursor.com>

Author: 3 people

.github/workflows/test-build.yml

@@ -318,8 +318,21 @@ jobs:
           E2E_PG_ADMIN_URL: 'postgresql://postgres:postgres@127.0.0.1:5432/postgres'
         run: bun run test:e2e
 
-      - name: Upload E2E diagnostics
+      - name: Check E2E diagnostics eligibility
+        id: e2e_diagnostics
         if: failure()
+        shell: bash
+        run: |
+          shopt -s nullglob
+          markers=(apps/sim/e2e/.runs/*/markers/leak-scan-complete.json)
+          if (( ${#markers[@]} > 0 )); then
+            echo "safe=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "safe=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Upload E2E diagnostics
+        if: failure() && steps.e2e_diagnostics.outputs.safe == 'true'
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
         with:
           name: settings-e2e-${{ github.run_id }}

apps/sim/app/(landing)/layout.tsx

@@ -11,6 +11,7 @@ import { XPageViewTracker } from '@/app/(landing)/x-page-view-tracker'
 const HUBSPOT_SCRIPT_SRC = 'https://js-na2.hs-scripts.com/246720681.js' as const
 
 const X_PIXEL_ID = 'q5xbl' as const
+const isMarketingAnalyticsEnabled = isHosted && !process.env.E2E_PROFILE
 
 /** X (Twitter) conversion tracking base code — loads uwt.js and fires the initial PageView. */
 const X_PIXEL_BASE_CODE = `!function(e,t,n,s,u,a){e.twq||(s=e.twq=function(){s.exe?s.exe.apply(s,arguments):s.queue.push(arguments);
@@ -43,7 +44,7 @@ export default function LandingLayout({ children }: { children: ReactNode }) {
     <LandingShell>
       {children}
       {/* HubSpot + X pixel tracking — hosted only */}
-      {isHosted && (
+      {isMarketingAnalyticsEnabled && (
         <>
           <Script id='hs-script-loader' src={HUBSPOT_SCRIPT_SRC} strategy='afterInteractive' />
           <Script id='x-pixel-base' strategy='afterInteractive'>

apps/sim/app/api/users/me/settings/route.ts

@@ -60,6 +60,7 @@ export const PATCH = withRouteHandler(async (request: NextRequest) => {
       .values({
         id: generateShortId(),
         userId,
+        ...defaultUserSettings,
         ...validatedData,
         updatedAt: new Date(),
       })

apps/sim/app/layout.tsx

@@ -28,6 +28,7 @@ export const metadata: Metadata = generateBrandedMetadata()
 
 const GTM_ID = 'GTM-T7PHSRX5' as const
 const GA_ID = 'G-DR7YBE70VS' as const
+const isMarketingAnalyticsEnabled = isHosted && !process.env.E2E_PROFILE
 
 export default function RootLayout({ children }: { children: React.ReactNode }) {
   const themeCSS = generateThemeCSS()
@@ -196,7 +197,7 @@ export default function RootLayout({ children }: { children: React.ReactNode })
         <meta httpEquiv='x-ua-compatible' content='ie=edge' />
 
         {/* Google Tag Manager — hosted only */}
-        {isHosted && (
+        {isMarketingAnalyticsEnabled && (
           <Script
             id='gtm'
             strategy='afterInteractive'
@@ -211,7 +212,7 @@ j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
         )}
 
         {/* Google Analytics (gtag.js) — hosted only */}
-        {isHosted && (
+        {isMarketingAnalyticsEnabled && (
           <>
             <Script
               id='gtag-src'
@@ -232,7 +233,7 @@ j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
       </head>
       <body className={`${season.variable} font-season`} suppressHydrationWarning>
         {/* Google Tag Manager (noscript) — hosted only */}
-        {isHosted && (
+        {isMarketingAnalyticsEnabled && (
           <noscript>
             <iframe
               src={`https://www.googletagmanager.com/ns.html?id=${GTM_ID}`}

apps/sim/e2e/README.md

@@ -58,14 +58,19 @@ Server telemetry currently shares the strict loopback Stripe-fake process; the
 generic modular fake-service refactor remains scoped to the roadmap's
 `e2e/06b-enterprise-integrations` phase.
 An exclusive checkout-level orchestrator lock prevents concurrent runs from
-racing on `.next`, the shared build cache, or the fixed app/realtime ports.
+racing on `.next`, the shared build cache, or the fixed app/realtime ports. The
+lock records managed process groups, so recovery from an uncatchable
+orchestrator termination kills verified stale descendants before admitting a
+new writer.
 
 On interruption, the runner launches a detached cleanup supervisor before
 exiting. It terminates managed process groups, force-drops the guarded database,
-and removes temporary auth/cloud-config directories even if another Ctrl-C
-terminates the foreground package runner.
+and removes temporary auth/cloud-config directories. Repeated or opposite
+signals remain on the same single-flight path until the supervisor owns the
+lock, after which the foreground runner exits.
 Cleanup failures retain the lock and require the reported resources to be
 inspected and cleaned before manually removing `e2e/.cache/orchestrator.lock`.
+Failure to start the detached cleanup supervisor also retains the lock.
 
 Pass Playwright arguments after `--`:
 
@@ -160,26 +165,31 @@ the in-memory canary scans the manifest, logs, report files, and trace archives
 and fails if a synthetic password, invitation token, or runtime secret escaped
 the excluded private directories. Cancelled CI runs do not upload unscanned
 diagnostics, and an unreadable canary or incomplete archive scan causes all
-potentially unscanned diagnostic roots to be scrubbed. Storage-state session cookies are intentionally not canaried
+potentially unscanned diagnostic roots to be scrubbed. CI uploads failure
+diagnostics only when the runner wrote its successful scan marker. The fixed
+foundation smoke password is public test input and is intentionally not a
+canary, so expected login traces remain useful. Storage-state session cookies are intentionally not canaried
 because authenticated Playwright traces contain them by design; they are
 synthetic and invalid once the run database is dropped.
 Fresh-session recapture is deliberately deferred. Future membership-mutation
 coverage must explicitly restore a private credential handoff and re-review its
 access boundary rather than assuming credentials persist through Playwright.
 
 E2E builds verify the pinned Bun executable plus reviewed sandbox-bundle
-source, direct dependency, and output fingerprints and never regenerate
-committed `.cjs` files.
+source, direct dependency, and output fingerprints. They also regenerate the
+bundles into a temporary directory and require those fresh outputs to match the
+reviewed fingerprints without modifying committed `.cjs` files.
 `bun run build:sandbox-bundles:integrity` is the explicit maintenance command
 that regenerates bundles and their reviewed integrity manifest together.
-Unrelated monorepo lockfile changes do not invalidate the bundle fingerprint;
-the committed output hashes still detect any transitive change that alters a
-bundle.
+Unrelated monorepo lockfile changes do not invalidate the reviewed fingerprint;
+the fresh-output comparison detects transitive changes that alter a bundle.
 
 Reset/reseed cleanup remains deferred with keep-stack supervision. Ordinary
 runs own a unique guarded database and remove it wholesale rather than carrying
 untested row-level deletion code.
 
 Provider log scans are diagnostic tripwires, not proof of zero egress. The
 primary boundaries are the default-deny child environment, provider disabling,
-loopback-only service bindings, and guarded Stripe transport.
+loopback-only service bindings, guarded Stripe transport, disabled hosted
+marketing tags, and a browser-context allowlist that rejects every HTTP(S)
+origin outside the app and realtime service.

apps/sim/e2e/fixtures/browser-test.ts

@@ -0,0 +1,32 @@
+import { test as base } from '@playwright/test'
+import { installBrowserNetworkGuard } from '../support/browser-network'
+
+interface BrowserFixtures {
+  browserNetworkGuard: undefined
+}
+
+export const test = base.extend<BrowserFixtures>({
+  browserNetworkGuard: [
+    async ({ context }, use) => {
+      const guard = await installBrowserNetworkGuard(context)
+      const failures: unknown[] = []
+      try {
+        await use(undefined)
+      } catch (error) {
+        failures.push(error)
+      }
+      try {
+        guard.assertNoUnexpectedRequests()
+      } catch (error) {
+        failures.push(error)
+      }
+      if (failures.length === 1) throw failures[0]
+      if (failures.length > 1) {
+        throw new AggregateError(failures, 'Browser test and network isolation both failed')
+      }
+    },
+    { auto: true },
+  ],
+})
+
+export { expect } from '@playwright/test'

apps/sim/e2e/fixtures/factories/billing.ts

@@ -2,6 +2,8 @@ import { db } from '@sim/db'
 import { member, organization, subscription, user, userStats } from '@sim/db/schema'
 import { generateId } from '@sim/utils/id'
 import { and, eq, inArray } from 'drizzle-orm'
+import { hasOtherEntitledOrganizationSubscription } from '@/lib/billing/subscriptions/organization-coverage'
+import { detachOrganizationWorkspaces } from '@/lib/workspaces/organization-workspaces'
 
 const CREDITS_PER_DOLLAR = 200
 const FREE_USAGE_LIMIT_DOLLARS = '5'
@@ -66,6 +68,8 @@ export async function arrangeSubscription(input: SubscriptionArrangement): Promi
       metadata,
     })
 
+    if (status === 'canceled') return
+
     if (input.plan.startsWith('pro_')) {
       await tx
         .update(userStats)
@@ -141,6 +145,13 @@ export async function lapseOrganizationSubscription(input: {
         periodEnd: now,
       })
       .where(eq(subscription.id, input.subscriptionId))
+  })
+
+  if (await hasOtherEntitledOrganizationSubscription(input.organizationId, input.subscriptionId)) {
+    return
+  }
+
+  await db.transaction(async (tx) => {
     await tx
       .update(organization)
       .set({ orgUsageLimit: null, updatedAt: now })
@@ -157,6 +168,7 @@ export async function lapseOrganizationSubscription(input: {
         .where(inArray(userStats.userId, input.memberUserIds))
     }
   })
+  await detachOrganizationWorkspaces(input.organizationId)
 }
 
 function planCredits(plan: SubscriptionArrangement['plan']): number {

apps/sim/e2e/fixtures/persona-test.ts

@@ -1,4 +1,5 @@
 import { test as base } from '@playwright/test'
+import { type BrowserNetworkGuard, installBrowserNetworkGuard } from '../support/browser-network'
 import {
   type PersonaManifestEntry,
   readScenarioManifest,
@@ -23,7 +24,7 @@ export const test = base.extend<PersonaFixtures>({
     { scope: 'test' },
   ],
   contextForPersona: async ({ browser, contextOptions, personaManifest }, use) => {
-    const contexts = new Set<import('@playwright/test').BrowserContext>()
+    const contexts = new Map<import('@playwright/test').BrowserContext, BrowserNetworkGuard>()
     await use(async (personaKey) => {
       const persona = requirePersona(personaManifest, personaKey)
       const context = await browser.newContext({
@@ -34,10 +35,25 @@ export const test = base.extend<PersonaFixtures>({
           persona.storageStatePath
         ),
       })
-      contexts.add(context)
+      contexts.set(context, await installBrowserNetworkGuard(context))
       return context
     })
-    await Promise.all([...contexts].map((context) => context.close()))
+    const failures: unknown[] = []
+    for (const [context, guard] of contexts) {
+      try {
+        await context.close()
+      } catch (error) {
+        failures.push(error)
+      }
+      try {
+        guard.assertNoUnexpectedRequests()
+      } catch (error) {
+        failures.push(error)
+      }
+    }
+    if (failures.length > 0) {
+      throw new AggregateError(failures, 'Persona browser cleanup or network isolation failed')
+    }
   },
 })
 

apps/sim/e2e/fixtures/scenario-billing.ts

@@ -0,0 +1,58 @@
+import type { ResolvedScenario, ScenarioSubscription } from './scenario'
+
+export function isEntitledSubscription(subscription: ScenarioSubscription): boolean {
+  return subscription.status === 'active' || subscription.status === 'past_due'
+}
+
+export function sameBillingReference(
+  left: ScenarioSubscription,
+  right: ScenarioSubscription
+): boolean {
+  return billingReferenceKey(left) === billingReferenceKey(right)
+}
+
+export function initialSubscriptionStatus(
+  scenario: ResolvedScenario,
+  definition: ScenarioSubscription
+): 'active' | 'past_due' | 'canceled' {
+  if (definition.status !== 'lapsed') return definition.status
+  const hasEntitledReplacement = scenario.definition.subscriptions.some(
+    (candidate) =>
+      candidate.key !== definition.key &&
+      isEntitledSubscription(candidate) &&
+      sameBillingReference(candidate, definition)
+  )
+  return hasEntitledReplacement ? 'canceled' : 'active'
+}
+
+export function expectedUsageLimit(scenario: ResolvedScenario, userKey: string): string | null {
+  const organizationKeys = new Set(
+    scenario.definition.organizationMemberships
+      .filter((membership) => membership.userKey === userKey)
+      .map((membership) => membership.organizationKey)
+  )
+  const hasEntitledOrganizationSubscription = scenario.definition.subscriptions.some(
+    (candidate) =>
+      candidate.billingReference.kind === 'organization' &&
+      organizationKeys.has(candidate.billingReference.organizationKey) &&
+      isEntitledSubscription(candidate)
+  )
+  if (hasEntitledOrganizationSubscription) return null
+
+  const personalSubscription = scenario.definition.subscriptions.find(
+    (candidate) =>
+      candidate.billingReference.kind === 'user' &&
+      candidate.billingReference.userKey === userKey &&
+      isEntitledSubscription(candidate)
+  )
+  if (personalSubscription?.plan.startsWith('pro_')) {
+    return String(Number(personalSubscription.plan.split('_')[1]) / 200)
+  }
+  return '5'
+}
+
+export function billingReferenceKey(subscription: ScenarioSubscription): string {
+  return subscription.billingReference.kind === 'user'
+    ? `user/${subscription.billingReference.userKey}`
+    : `organization/${subscription.billingReference.organizationKey}`
+}