Skip to content

Commit c096f4e

Browse files
Bill LeoutsakosBill Leoutsakos
Bill Leoutsakos
authored and
Bill Leoutsakos
committed
Harden Playwright lifecycle and persona fidelity
Close production-state and cleanup gaps so the settings E2E platform fails closed without sacrificing useful diagnostics.
1 parent b0b0470 commit c096f4e

33 files changed

Lines changed: 17941 additions & 114 deletions

‎.github/workflows/test-build.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -319,7 +319,7 @@ jobs:
319319
run: bun run test:e2e
320320

321321
- name: Upload E2E diagnostics
322-
if: failure()
322+
if: failure() && hashFiles('apps/sim/e2e/.runs/**/markers/leak-scan-complete.json') != ''
323323
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
324324
with:
325325
name: settings-e2e-${{ github.run_id }}

‎apps/sim/app/(landing)/layout.tsx‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import { XPageViewTracker } from '@/app/(landing)/x-page-view-tracker'
1111
const HUBSPOT_SCRIPT_SRC = 'https://js-na2.hs-scripts.com/246720681.js' as const
1212

1313
const X_PIXEL_ID = 'q5xbl' as const
14+
const isMarketingAnalyticsEnabled = isHosted && !process.env.E2E_PROFILE
1415

1516
/** X (Twitter) conversion tracking base code — loads uwt.js and fires the initial PageView. */
1617
const X_PIXEL_BASE_CODE = `!function(e,t,n,s,u,a){e.twq||(s=e.twq=function(){s.exe?s.exe.apply(s,arguments):s.queue.push(arguments);
@@ -43,7 +44,7 @@ export default function LandingLayout({ children }: { children: ReactNode }) {
4344
<LandingShell>
4445
{children}
4546
{/* HubSpot + X pixel tracking — hosted only */}
46-
{isHosted && (
47+
{isMarketingAnalyticsEnabled && (
4748
<>
4849
<Script id='hs-script-loader' src={HUBSPOT_SCRIPT_SRC} strategy='afterInteractive' />
4950
<Script id='x-pixel-base' strategy='afterInteractive'>

‎apps/sim/app/api/users/me/settings/route.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ export const PATCH = withRouteHandler(async (request: NextRequest) => {
6060
.values({
6161
id: generateShortId(),
6262
userId,
63+
...defaultUserSettings,
6364
...validatedData,
6465
updatedAt: new Date(),
6566
})

‎apps/sim/app/layout.tsx‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ export const metadata: Metadata = generateBrandedMetadata()
2828

2929
const GTM_ID = 'GTM-T7PHSRX5' as const
3030
const GA_ID = 'G-DR7YBE70VS' as const
31+
const isMarketingAnalyticsEnabled = isHosted && !process.env.E2E_PROFILE
3132

3233
export default function RootLayout({ children }: { children: React.ReactNode }) {
3334
const themeCSS = generateThemeCSS()
@@ -196,7 +197,7 @@ export default function RootLayout({ children }: { children: React.ReactNode })
196197
<meta httpEquiv='x-ua-compatible' content='ie=edge' />
197198

198199
{/* Google Tag Manager — hosted only */}
199-
{isHosted && (
200+
{isMarketingAnalyticsEnabled && (
200201
<Script
201202
id='gtm'
202203
strategy='afterInteractive'
@@ -211,7 +212,7 @@ j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
211212
)}
212213

213214
{/* Google Analytics (gtag.js) — hosted only */}
214-
{isHosted && (
215+
{isMarketingAnalyticsEnabled && (
215216
<>
216217
<Script
217218
id='gtag-src'
@@ -232,7 +233,7 @@ j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
232233
</head>
233234
<body className={`${season.variable} font-season`} suppressHydrationWarning>
234235
{/* Google Tag Manager (noscript) — hosted only */}
235-
{isHosted && (
236+
{isMarketingAnalyticsEnabled && (
236237
<noscript>
237238
<iframe
238239
src={`https://www.googletagmanager.com/ns.html?id=${GTM_ID}`}

‎apps/sim/e2e/README.md‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -58,14 +58,18 @@ Server telemetry currently shares the strict loopback Stripe-fake process; the
5858
generic modular fake-service refactor remains scoped to the roadmap's
5959
`e2e/06b-enterprise-integrations` phase.
6060
An exclusive checkout-level orchestrator lock prevents concurrent runs from
61-
racing on `.next`, the shared build cache, or the fixed app/realtime ports.
61+
racing on `.next`, the shared build cache, or the fixed app/realtime ports. The
62+
lock records managed process groups, so recovery from an uncatchable
63+
orchestrator termination kills verified stale descendants before admitting a
64+
new writer.
6265

6366
On interruption, the runner launches a detached cleanup supervisor before
6467
exiting. It terminates managed process groups, force-drops the guarded database,
6568
and removes temporary auth/cloud-config directories even if another Ctrl-C
6669
terminates the foreground package runner.
6770
Cleanup failures retain the lock and require the reported resources to be
6871
inspected and cleaned before manually removing `e2e/.cache/orchestrator.lock`.
72+
Failure to start the detached cleanup supervisor also retains the lock.
6973

7074
Pass Playwright arguments after `--`:
7175

@@ -160,26 +164,31 @@ the in-memory canary scans the manifest, logs, report files, and trace archives
160164
and fails if a synthetic password, invitation token, or runtime secret escaped
161165
the excluded private directories. Cancelled CI runs do not upload unscanned
162166
diagnostics, and an unreadable canary or incomplete archive scan causes all
163-
potentially unscanned diagnostic roots to be scrubbed. Storage-state session cookies are intentionally not canaried
167+
potentially unscanned diagnostic roots to be scrubbed. CI uploads failure
168+
diagnostics only when the runner wrote its successful scan marker. The fixed
169+
foundation smoke password is public test input and is intentionally not a
170+
canary, so expected login traces remain useful. Storage-state session cookies are intentionally not canaried
164171
because authenticated Playwright traces contain them by design; they are
165172
synthetic and invalid once the run database is dropped.
166173
Fresh-session recapture is deliberately deferred. Future membership-mutation
167174
coverage must explicitly restore a private credential handoff and re-review its
168175
access boundary rather than assuming credentials persist through Playwright.
169176

170177
E2E builds verify the pinned Bun executable plus reviewed sandbox-bundle
171-
source, direct dependency, and output fingerprints and never regenerate
172-
committed `.cjs` files.
178+
source, direct dependency, and output fingerprints. They also regenerate the
179+
bundles into a temporary directory and require those fresh outputs to match the
180+
reviewed fingerprints without modifying committed `.cjs` files.
173181
`bun run build:sandbox-bundles:integrity` is the explicit maintenance command
174182
that regenerates bundles and their reviewed integrity manifest together.
175-
Unrelated monorepo lockfile changes do not invalidate the bundle fingerprint;
176-
the committed output hashes still detect any transitive change that alters a
177-
bundle.
183+
Unrelated monorepo lockfile changes do not invalidate the reviewed fingerprint;
184+
the fresh-output comparison detects transitive changes that alter a bundle.
178185

179186
Reset/reseed cleanup remains deferred with keep-stack supervision. Ordinary
180187
runs own a unique guarded database and remove it wholesale rather than carrying
181188
untested row-level deletion code.
182189

183190
Provider log scans are diagnostic tripwires, not proof of zero egress. The
184191
primary boundaries are the default-deny child environment, provider disabling,
185-
loopback-only service bindings, and guarded Stripe transport.
192+
loopback-only service bindings, guarded Stripe transport, disabled hosted
193+
marketing tags, and a browser-context allowlist that rejects every HTTP(S)
194+
origin outside the app and realtime service.

‎apps/sim/e2e/fixtures/browser-test.ts‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
import { test as base } from '@playwright/test'
2+
import { installBrowserNetworkGuard } from '../support/browser-network'
3+
4+
interface BrowserFixtures {
5+
browserNetworkGuard: undefined
6+
}
7+
8+
export const test = base.extend<BrowserFixtures>({
9+
browserNetworkGuard: [
10+
async ({ context }, use) => {
11+
const guard = await installBrowserNetworkGuard(context)
12+
try {
13+
await use(undefined)
14+
} finally {
15+
guard.assertNoUnexpectedRequests()
16+
}
17+
},
18+
{ auto: true },
19+
],
20+
})
21+
22+
export { expect } from '@playwright/test'

‎apps/sim/e2e/fixtures/factories/billing.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { db } from '@sim/db'
22
import { member, organization, subscription, user, userStats } from '@sim/db/schema'
33
import { generateId } from '@sim/utils/id'
44
import { and, eq, inArray } from 'drizzle-orm'
5+
import { detachOrganizationWorkspaces } from '@/lib/workspaces/organization-workspaces'
56

67
const CREDITS_PER_DOLLAR = 200
78
const FREE_USAGE_LIMIT_DOLLARS = '5'
@@ -157,6 +158,7 @@ export async function lapseOrganizationSubscription(input: {
157158
.where(inArray(userStats.userId, input.memberUserIds))
158159
}
159160
})
161+
await detachOrganizationWorkspaces(input.organizationId)
160162
}
161163

162164
function planCredits(plan: SubscriptionArrangement['plan']): number {

‎apps/sim/e2e/fixtures/persona-test.ts‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { test as base } from '@playwright/test'
2+
import { type BrowserNetworkGuard, installBrowserNetworkGuard } from '../support/browser-network'
23
import {
34
type PersonaManifestEntry,
45
readScenarioManifest,
@@ -23,7 +24,7 @@ export const test = base.extend<PersonaFixtures>({
2324
{ scope: 'test' },
2425
],
2526
contextForPersona: async ({ browser, contextOptions, personaManifest }, use) => {
26-
const contexts = new Set<import('@playwright/test').BrowserContext>()
27+
const contexts = new Map<import('@playwright/test').BrowserContext, BrowserNetworkGuard>()
2728
await use(async (personaKey) => {
2829
const persona = requirePersona(personaManifest, personaKey)
2930
const context = await browser.newContext({
@@ -34,10 +35,25 @@ export const test = base.extend<PersonaFixtures>({
3435
persona.storageStatePath
3536
),
3637
})
37-
contexts.add(context)
38+
contexts.set(context, await installBrowserNetworkGuard(context))
3839
return context
3940
})
40-
await Promise.all([...contexts].map((context) => context.close()))
41+
const failures: unknown[] = []
42+
for (const [context, guard] of contexts) {
43+
try {
44+
await context.close()
45+
} catch (error) {
46+
failures.push(error)
47+
}
48+
try {
49+
guard.assertNoUnexpectedRequests()
50+
} catch (error) {
51+
failures.push(error)
52+
}
53+
}
54+
if (failures.length > 0) {
55+
throw new AggregateError(failures, 'Persona browser cleanup or network isolation failed')
56+
}
4157
},
4258
})
4359

‎apps/sim/e2e/fixtures/validate-scenario.ts‎

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -459,6 +459,16 @@ function validateWorkspaces(
459459
`workspace "${workspace.key}" references missing subscription "${workspace.subscriptionKey}"`
460460
)
461461
}
462+
const entitledSubscription = definition.subscriptions.find(
463+
(candidate) =>
464+
(candidate.status === 'active' || candidate.status === 'past_due') &&
465+
billingReferenceKey(candidate) === payerReferenceKey(workspace)
466+
)
467+
if (entitledSubscription && subscription?.key !== entitledSubscription.key) {
468+
issues.push(
469+
`workspace "${workspace.key}" must reference its payer's current entitled subscription "${entitledSubscription.key}"`
470+
)
471+
}
462472

463473
if (workspace.organizationKey) {
464474
if (!organizationsByKey.has(workspace.organizationKey)) {
@@ -670,19 +680,28 @@ function validatePersonaWorkspaceExpectation(
670680
`persona "${personaKey}" owner/admin expectation for "${workspace.key}" is below admin`
671681
)
672682
}
673-
const subscription = workspace.subscriptionKey
683+
const declaredSubscription = workspace.subscriptionKey
674684
? subscriptionsByKey.get(workspace.subscriptionKey)
675685
: undefined
676-
const actualPlan = !subscription || subscription.status === 'lapsed' ? 'free' : subscription.plan
686+
const entitledSubscription = [...subscriptionsByKey.values()].find(
687+
(candidate) =>
688+
(candidate.status === 'active' || candidate.status === 'past_due') &&
689+
billingReferenceKey(candidate) === payerReferenceKey(workspace)
690+
)
691+
const actualPlan = entitledSubscription?.plan ?? 'free'
677692
const actualMembership = actual.isOwner
678693
? 'owner'
679694
: actual.organizationRole
680695
? 'member'
681696
: 'external'
697+
const actualPayerScope =
698+
workspace.organizationKey && declaredSubscription?.status === 'lapsed'
699+
? 'user'
700+
: workspace.payer.kind
682701
if (
683702
expected.hostContext.isOwner !== actual.isOwner ||
684703
expected.hostContext.hostMembership !== actualMembership ||
685-
expected.hostContext.payerScope !== workspace.payer.kind ||
704+
expected.hostContext.payerScope !== actualPayerScope ||
686705
expected.hostContext.plan !== actualPlan ||
687706
expected.hostContext.hosted !== workspace.hosted ||
688707
expected.hostContext.billingEnabled !== workspace.billingEnabled
@@ -741,6 +760,12 @@ function billingReferenceKey(subscription: ScenarioSubscription): string {
741760
: `organization/${subscription.billingReference.organizationKey}`
742761
}
743762

763+
function payerReferenceKey(workspace: ScenarioWorkspace): string {
764+
return workspace.payer.kind === 'user'
765+
? `user/${workspace.payer.userKey}`
766+
: `organization/${workspace.payer.organizationKey}`
767+
}
768+
744769
function sameSet(left: readonly string[], right: readonly string[]): boolean {
745770
return left.length === right.length && left.every((value) => right.includes(value))
746771
}

‎apps/sim/e2e/foundation/build-manifest.spec.ts‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
import { spawn } from 'node:child_process'
2+
import { once } from 'node:events'
13
import {
24
existsSync,
35
mkdirSync,
@@ -14,6 +16,7 @@ import {
1416
type BuildArtifactPaths,
1517
type BuildIdentity,
1618
clearActiveNextBuild,
19+
isNextBuildInput,
1720
pruneBuildCache,
1821
restoreCachedBuild,
1922
storeCompletedBuild,
@@ -33,6 +36,13 @@ const identity: BuildIdentity = {
3336
}
3437

3538
test.describe('verified Next build cache', () => {
39+
test('hashes build-affecting E2E harness files without hashing test artifacts', () => {
40+
expect(isNextBuildInput('apps/sim/e2e/support/stack.ts')).toBe(true)
41+
expect(isNextBuildInput('apps/sim/e2e/support/deployment-profile.ts')).toBe(true)
42+
expect(isNextBuildInput('apps/sim/e2e/settings/persona-contracts.spec.ts')).toBe(false)
43+
expect(isNextBuildInput('apps/sim/test-results/trace.zip')).toBe(false)
44+
})
45+
3646
test('stores, restores, and rejects artifact corruption', () => {
3747
withBuildPaths((paths) => {
3848
writeBuild(paths.activeNextDirectory, 'build-one', 'original')
@@ -155,6 +165,58 @@ test('orchestrator lock rejects live ownership and recovers stale descriptors',
155165
}
156166
})
157167

168+
test('stale orchestrator recovery terminates its persisted process groups', async () => {
169+
test.skip(process.platform === 'win32', 'POSIX process-group cleanup is tested on Unix')
170+
const directory = mkdtempSync(path.join(os.tmpdir(), 'sim-e2e-stale-process-group-'))
171+
const lockPath = path.join(directory, 'orchestrator.lock')
172+
const groupIdPath = path.join(directory, 'process-group-id')
173+
const launcher = spawn(
174+
process.execPath,
175+
[
176+
'-e',
177+
`
178+
const { spawn } = require('node:child_process')
179+
const { writeFileSync } = require('node:fs')
180+
const child = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1_000)'], {
181+
detached: true,
182+
stdio: 'ignore',
183+
})
184+
writeFileSync(${JSON.stringify(groupIdPath)}, String(child.pid))
185+
child.unref()
186+
`,
187+
],
188+
{
189+
stdio: 'ignore',
190+
}
191+
)
192+
await once(launcher, 'exit')
193+
const groupId = Number(readFileSync(groupIdPath, 'utf8'))
194+
try {
195+
expect(groupId).toBeGreaterThan(0)
196+
const lock = acquireE2eRunLock(lockPath)
197+
lock.setProcessGroupIds([groupId])
198+
const descriptorPath = path.join(lockPath, 'owner.json')
199+
const descriptor = JSON.parse(readFileSync(descriptorPath, 'utf8')) as Record<string, unknown>
200+
writeFileSync(
201+
descriptorPath,
202+
JSON.stringify({
203+
...descriptor,
204+
pid: 2_147_483_647,
205+
processStartIdentity: null,
206+
})
207+
)
208+
209+
const recovered = acquireE2eRunLock(lockPath)
210+
expect(() => process.kill(-groupId, 0)).toThrow()
211+
recovered.release()
212+
} finally {
213+
try {
214+
process.kill(-groupId, 'SIGKILL')
215+
} catch {}
216+
rmSync(directory, { recursive: true, force: true })
217+
}
218+
})
219+
158220
function withBuildPaths(run: (paths: BuildArtifactPaths) => void): void {
159221
const directory = mkdtempSync(path.join(os.tmpdir(), 'sim-e2e-build-cache-'))
160222
try {

0 commit comments

Comments
 (0)