From e8f3bfe86403ca3dece0fe94de76576fe79b2bbc Mon Sep 17 00:00:00 2001 From: Bill Leoutsakos Date: Mon, 20 Jul 2026 21:09:18 -0700 Subject: [PATCH 1/6] Add safe E2E build reuse and environment isolation Separate build-time inputs from per-run services so verified artifacts can accelerate local iteration without weakening fresh database or provider boundaries. --- .gitignore | 1 + apps/sim/e2e/README.md | 32 +- apps/sim/e2e/foundation/safety.spec.ts | 55 +++ apps/sim/e2e/scripts/options.ts | 22 +- apps/sim/e2e/scripts/run.ts | 45 ++- apps/sim/e2e/support/build-manifest.ts | 368 ++++++++++++++++++ apps/sim/e2e/support/deployment-profile.ts | 194 ++++++++- apps/sim/e2e/support/env.ts | 12 +- apps/sim/e2e/support/paths.ts | 2 + apps/sim/e2e/support/sandbox-bundles.ts | 61 +++ apps/sim/e2e/support/stack.ts | 51 ++- .../lib/execution/sandbox/bundles/build.ts | 31 +- .../execution/sandbox/bundles/integrity.json | 20 + 13 files changed, 839 insertions(+), 55 deletions(-) create mode 100644 apps/sim/e2e/support/build-manifest.ts create mode 100644 apps/sim/e2e/support/sandbox-bundles.ts create mode 100644 apps/sim/lib/execution/sandbox/bundles/integrity.json diff --git a/.gitignore b/.gitignore index dc42d1c8e3f..109495320d0 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ package-lock.json /apps/**/test-results/ /apps/**/e2e/.runs/ /apps/**/e2e/.auth/ +/apps/**/e2e/.cache/ # next.js /.next/ diff --git a/apps/sim/e2e/README.md b/apps/sim/e2e/README.md index 49f108e8dd7..76d0f32b640 100644 --- a/apps/sim/e2e/README.md +++ b/apps/sim/e2e/README.md @@ -65,6 +65,24 @@ bun run test:e2e -- --project=hosted-billing-chromium-navigation bun run test:e2e -- --grep "unauthenticated" ``` +For a local follow-up run, reuse only a verified build while still creating a +new database, Stripe fake, app, realtime process, and browser run: + +```bash +bun run test:e2e -- --reuse-build --project=hosted-billing-chromium-navigation +``` + +The cache lives under ignored `e2e/.cache/builds/`. A hit requires matching +source contents (including uncommitted/untracked files), build/public profile, +Node/Bun/Next versions, platform, `BUILD_ID`, and the cached artifact checksum. +Any mismatch performs and caches a fresh build. CI rejects `--reuse-build`. +`--skip-build` remains unsupported. + +Keep-stack/rerun supervision is intentionally unavailable. The initial safety +experiment requires descriptor ownership, mutation observation, state snapshots, +and teardown to ship as one unit; until all of those are proven, each invocation +uses the normal one-shot lifecycle. + Do not invoke `playwright test` directly. Raw Playwright bypasses environment, database, process, sharding, and teardown guards; the config rejects runs that were not launched by the orchestrator. Report and trace viewer commands remain @@ -91,9 +109,17 @@ Open a trace: node ../../node_modules/@playwright/test/cli.js show-trace test-results//trace.zip ``` -The runner starts every child process from a fresh environment. It allowlists -only deterministic E2E values and shadows keys found in local `.env*` files, so -developer credentials are not used as test state or written to reports. +The runner starts every child process from a fresh, purpose-specific +environment. Next build receives deterministic sentinels instead of the run +database/fake endpoint; app, realtime, migrations, future seeding/auth capture, +and Playwright each receive only their required values. Next build/start shadow +keys found in local `.env*` files, while children that cannot load those files +omit denied keys entirely. Developer credentials are not used as test state or +written to reports. + +E2E builds verify the reviewed sandbox-bundle source/dependency/output +fingerprint and never regenerate committed `.cjs` files. Regeneration remains an +explicit repository maintenance operation. Provider log scans are diagnostic tripwires, not proof of zero egress. The primary boundaries are the default-deny child environment, provider disabling, diff --git a/apps/sim/e2e/foundation/safety.spec.ts b/apps/sim/e2e/foundation/safety.spec.ts index 261180d16ed..fa7c8b0867e 100644 --- a/apps/sim/e2e/foundation/safety.spec.ts +++ b/apps/sim/e2e/foundation/safety.spec.ts @@ -5,11 +5,13 @@ import path from 'node:path' import { loadEnvConfig } from '@next/env' import { expect, test } from '@playwright/test' import { parseRunOptions } from '../scripts/options' +import { classifyE2eHashOwner } from '../support/build-manifest' import { assertLoopbackPostgresUrl, assertSafeDatabaseName, buildRunDatabaseUrl, } from '../support/database' +import { createHostedBillingProfile } from '../support/deployment-profile' import { buildChildEnvironment, discoverEnvFileKeys } from '../support/env' import { areValidE2eHostAddresses, isLoopbackAddress } from '../support/hosts' import { @@ -17,6 +19,7 @@ import { spawnManagedProcess, waitForManagedProcessReady, } from '../support/process' +import { verifySandboxBundleIntegrity } from '../support/sandbox-bundles' import { parseProcessGroupIds } from '../support/signal-cleanup' test.describe('foundation safety guards', () => { @@ -62,6 +65,38 @@ test.describe('foundation safety guards', () => { } }) + test('deployment profile projects least-privilege build and runtime environments', () => { + const profile = createHostedBillingProfile({ + runId: 'projection_test', + databaseUrl: 'postgresql://postgres:postgres@127.0.0.1:5432/sim_e2e_projection_test', + stripeApiBaseUrl: 'http://127.0.0.1:40123', + homeDirectory: path.join(os.tmpdir(), 'sim-e2e-projection'), + playwrightBrowsersPath: path.join(os.tmpdir(), 'sim-e2e-browsers'), + ci: false, + }) + const { build, app, realtime, migration, seed, authCapture, playwright } = profile.environments + + expect(build.env.DATABASE_URL).toContain('/sim_e2e_build_sentinel') + expect(build.env.DATABASE_URL).not.toBe(app.env.DATABASE_URL) + expect(build.env.STRIPE_API_BASE_URL).toBe('http://127.0.0.1:1') + expect(build.env.E2E_RUN_ID).toBe('build_sentinel') + + for (const key of Object.keys(app.env).filter((key) => key.startsWith('NEXT_PUBLIC_'))) { + expect(build.env[key], `${key} must be identical at build and runtime`).toBe(app.env[key]) + } + + expect(seed.env.ADMIN_API_KEY).toBe(app.env.ADMIN_API_KEY) + expect(seed.env.DATABASE_URL).toBe(app.env.DATABASE_URL) + expect(authCapture.env.ADMIN_API_KEY).toBeUndefined() + expect(authCapture.env.DATABASE_URL).toBeUndefined() + expect(playwright.env.ADMIN_API_KEY).toBeUndefined() + expect(playwright.env.DATABASE_URL).toBeUndefined() + expect(realtime.env.ADMIN_API_KEY).toBeUndefined() + expect(realtime.env.STRIPE_SECRET_KEY).toBeUndefined() + expect(migration.env.ADMIN_API_KEY).toBeUndefined() + expect(migration.env.MIGRATION_DATABASE_URL).toBe(app.env.DATABASE_URL) + }) + test('database guards reject shared or remote targets', () => { expect(() => assertSafeDatabaseName('simstudio')).toThrow() expect(() => assertSafeDatabaseName('sim_e2e_valid_run')).not.toThrow() @@ -109,6 +144,26 @@ test.describe('foundation safety guards', () => { /canonical/ ) expect(() => parseRunOptions(['--pass-with-no-tests'])).toThrow(/cannot override/) + expect(parseRunOptions(['--reuse-build'], { ci: false })).toEqual({ + playwrightArgs: [], + reuseBuild: true, + }) + expect(() => parseRunOptions(['--reuse-build'], { ci: true })).toThrow(/local-only/) + expect(() => parseRunOptions(['--keep-stack'], { ci: false })).toThrow(/deferred/) + }) + + test('hash ownership follows execution ownership', () => { + expect(classifyE2eHashOwner('apps/sim/e2e/settings/navigation/routes.spec.ts')).toBe('rerun') + expect(classifyE2eHashOwner('apps/sim/e2e/fixtures/scenario.ts')).toBe('scenario') + expect(classifyE2eHashOwner('apps/sim/e2e/scripts/seed-world.ts')).toBe('scenario') + expect(classifyE2eHashOwner('apps/sim/e2e/fakes/stripe/server.ts')).toBe('retained-stack') + expect(classifyE2eHashOwner('apps/sim/e2e/support/readiness.ts')).toBe('retained-stack') + expect(classifyE2eHashOwner('apps/realtime/src/index.ts')).toBe('retained-stack') + expect(classifyE2eHashOwner('apps/sim/app/page.tsx')).toBe('next-build') + }) + + test('committed sandbox bundles match the reviewed fingerprint', () => { + expect(() => verifySandboxBundleIntegrity()).not.toThrow() }) test('empty signal cleanup groups never become PID zero', () => { diff --git a/apps/sim/e2e/scripts/options.ts b/apps/sim/e2e/scripts/options.ts index fb8bdde3e0f..e33050d29f2 100644 --- a/apps/sim/e2e/scripts/options.ts +++ b/apps/sim/e2e/scripts/options.ts @@ -13,15 +13,31 @@ const FORBIDDEN_OPTIONS = [ export interface E2eRunOptions { playwrightArgs: string[] + reuseBuild: boolean } -export function parseRunOptions(argv: string[]): E2eRunOptions { +export function parseRunOptions( + argv: string[], + environment: { ci: boolean } = { ci: process.env.CI === 'true' } +): E2eRunOptions { const normalizedArgs = argv[0] === '--' ? argv.slice(1) : [...argv] if (normalizedArgs.includes('--skip-build')) { throw new Error( '--skip-build remains disabled until the planned profile/source-keyed build reuse experiment proves it safe' ) } + if (hasOption(normalizedArgs, '--keep-stack')) { + throw new Error( + '--keep-stack is unavailable: the all-or-nothing retained-stack safety experiment was deferred' + ) + } + if (normalizedArgs.some((arg) => arg.startsWith('--reuse-build='))) { + throw new Error('Use --reuse-build without a value') + } + const reuseBuild = normalizedArgs.includes('--reuse-build') + if (reuseBuild && environment.ci) { + throw new Error('--reuse-build is local-only; CI must run a fresh one-shot build') + } for (const option of FORBIDDEN_OPTIONS) { if (hasOption(normalizedArgs, option)) { throw new Error(`${option} cannot override E2E orchestration invariants`) @@ -33,7 +49,7 @@ export function parseRunOptions(argv: string[]): E2eRunOptions { if (normalizedArgs.includes('--shard')) { throw new Error('Use canonical --shard= syntax') } - const playwrightArgs = [...normalizedArgs] + const playwrightArgs = normalizedArgs.filter((arg) => arg !== '--reuse-build') const projects = getEqualsOptionValues(playwrightArgs, '--project') const unknownProject = projects.find( (project) => project !== NAVIGATION_PROJECT && project !== WORKFLOWS_PROJECT @@ -52,7 +68,7 @@ export function parseRunOptions(argv: string[]): E2eRunOptions { ) } - return { playwrightArgs } + return { playwrightArgs, reuseBuild } } function hasOption(args: string[], name: string): boolean { diff --git a/apps/sim/e2e/scripts/run.ts b/apps/sim/e2e/scripts/run.ts index 9a1f102e921..b49f9f59374 100644 --- a/apps/sim/e2e/scripts/run.ts +++ b/apps/sim/e2e/scripts/run.ts @@ -23,7 +23,7 @@ import { assertNoForbiddenProviderInitialization, assertNoForbiddenProviderTraffic, } from '../support/diagnostics' -import { E2E_OS_PASSTHROUGH_KEYS, formatRedactedEnvironmentSummary } from '../support/env' +import { formatRedactedEnvironmentSummary } from '../support/env' import { assertE2eHostResolvesToLoopback } from '../support/hosts' import { getRunDirectory, SIM_APP_DIR } from '../support/paths' import { @@ -186,27 +186,42 @@ async function main(): Promise { playwrightBrowsersPath: resolvePlaywrightBrowsersPath(), ci: process.env.CI === 'true', }) - console.info(formatRedactedEnvironmentSummary(profile.id, profile.childEnvironment)) + for (const [name, environment] of Object.entries(profile.environments)) { + console.info(formatRedactedEnvironmentSummary(`${profile.id}/${name}`, environment)) + } - const stackOptions = { + const commandOptions = { bunExecutable, nodeExecutable, - env: profile.childEnvironment.env, logsDirectory, } - await runMigrations(stackOptions) - await buildApp(stackOptions) - realtime = await startRealtime(stackOptions) - app = await startApp(stackOptions) + await runMigrations({ + ...commandOptions, + env: profile.environments.migration.env, + }) + await buildApp({ + ...commandOptions, + env: profile.environments.app.env, + buildEnvironment: profile.environments.build, + reuseBuild: options.reuseBuild, + }) + realtime = await startRealtime({ + ...commandOptions, + env: profile.environments.realtime.env, + }) + app = await startApp({ + ...commandOptions, + env: profile.environments.app.env, + }) await assertAdminApiBoundary( 'http://127.0.0.1:3000', - profile.childEnvironment.env.ADMIN_API_KEY + profile.environments.app.env.ADMIN_API_KEY ) assertNoForbiddenProviderInitialization([app.logPath, realtime.logPath]) const playwrightEnvironment = createPlaywrightEnvironment( - profile.childEnvironment.env, + profile.environments.playwright.env, runId, storageStateDirectory, markerDirectory @@ -291,19 +306,13 @@ function resolvePlaywrightBrowsersPath(): string { } function createPlaywrightEnvironment( - stackEnvironment: Record, + baseEnvironment: Record, runId: string, storageStateDirectory: string, markerDirectory: string ): Record { - const keys = [...E2E_OS_PASSTHROUGH_KEYS, 'HOME', 'NODE_OPTIONS', 'PLAYWRIGHT_BROWSERS_PATH'] - const env: Record = {} - for (const key of keys) { - const value = stackEnvironment[key] - if (value !== undefined) env[key] = value - } return { - ...env, + ...baseEnvironment, E2E_PROFILE, E2E_ORCHESTRATED: '1', E2E_RUN_ID: runId, diff --git a/apps/sim/e2e/support/build-manifest.ts b/apps/sim/e2e/support/build-manifest.ts new file mode 100644 index 00000000000..67b8c988095 --- /dev/null +++ b/apps/sim/e2e/support/build-manifest.ts @@ -0,0 +1,368 @@ +import { spawnSync } from 'node:child_process' +import { createHash } from 'node:crypto' +import { + cpSync, + existsSync, + lstatSync, + mkdirSync, + readdirSync, + readFileSync, + readlinkSync, + renameSync, + rmSync, + writeFileSync, +} from 'node:fs' +import path from 'node:path' +import type { ChildEnvironment } from './env' +import { E2E_BUILD_CACHE_DIR, REPO_ROOT, SIM_APP_DIR } from './paths' + +const BUILD_MANIFEST_VERSION = 1 +const NEXT_DIR = path.join(SIM_APP_DIR, '.next') +const ROOT_BUILD_FILES = new Set([ + 'bun.lock', + 'bunfig.toml', + 'package.json', + 'turbo.json', + 'tsconfig.json', +]) +const UNHASHED_OS_ENV_KEYS = new Set([ + 'PATH', + 'USER', + 'SHELL', + 'TMPDIR', + 'TMP', + 'TEMP', + 'SYSTEMROOT', + 'GITHUB_ACTIONS', +]) + +export type E2eHashOwner = 'next-build' | 'retained-stack' | 'scenario' | 'rerun' + +export interface BuildIdentity { + schemaVersion: typeof BUILD_MANIFEST_VERSION + nextBuildHash: string + sourceHash: string + profileHash: string + nodeVersion: string + bunVersion: string + nextVersion: string + platform: NodeJS.Platform + architecture: string +} + +interface BuildManifest extends BuildIdentity { + completed: true + createdAt: string + buildId: string + artifactHash: string +} + +export interface BuildReuseDecision { + reused: boolean + nextBuildHash: string + reason: string + cacheDirectory: string +} + +export function computeBuildIdentity(options: { + buildEnvironment: ChildEnvironment + nodeExecutable: string +}): BuildIdentity { + const sourceHash = hashRepositoryFiles(listRepositoryFiles().filter(isNextBuildInput)) + const profileHash = hashJson( + Object.fromEntries( + Object.entries(options.buildEnvironment.env) + .filter(([key]) => !UNHASHED_OS_ENV_KEYS.has(key)) + .sort(([left], [right]) => left.localeCompare(right)) + ) + ) + const nodeVersion = getExecutableVersion(options.nodeExecutable) + const bunVersion = process.versions.bun ?? 'not-bun' + const nextVersion = readPackageVersion(path.join(REPO_ROOT, 'node_modules/next/package.json')) + const stableIdentity = { + schemaVersion: BUILD_MANIFEST_VERSION, + sourceHash, + profileHash, + nodeVersion, + bunVersion, + nextVersion, + platform: process.platform, + architecture: process.arch, + } as const + + return { + ...stableIdentity, + nextBuildHash: hashJson(stableIdentity), + } +} + +export function computeExecutionHashes(nextBuildHash: string): { + retainedStackHash: string + scenarioHash: string +} { + const files = listRepositoryFiles() + return { + retainedStackHash: hashJson({ + nextBuildHash, + files: hashOwnedFiles(files, 'retained-stack'), + }), + scenarioHash: hashJson({ + files: hashOwnedFiles(files, 'scenario'), + }), + } +} + +export function classifyE2eHashOwner(relativePath: string): E2eHashOwner { + const normalized = relativePath.replaceAll(path.sep, '/') + if (isNextBuildInput(normalized)) return 'next-build' + if (normalized.startsWith('apps/realtime/')) return 'retained-stack' + if (normalized === 'apps/sim/playwright.config.ts') return 'retained-stack' + if (!normalized.startsWith('apps/sim/e2e/')) return 'retained-stack' + + const e2ePath = normalized.slice('apps/sim/e2e/'.length) + if ( + e2ePath.startsWith('fixtures/') || + e2ePath === 'settings/personas.ts' || + e2ePath === 'scripts/seed-world.ts' || + e2ePath === 'scripts/capture-auth-states.ts' + ) { + return 'scenario' + } + if ( + e2ePath.endsWith('.spec.ts') || + e2ePath.startsWith('auth/') || + e2ePath.startsWith('settings/') + ) { + return 'rerun' + } + if ( + e2ePath.startsWith('support/') || + e2ePath.startsWith('fakes/') || + e2ePath.startsWith('scripts/') + ) { + return 'retained-stack' + } + return 'retained-stack' +} + +export function restoreCachedBuild(identity: BuildIdentity): BuildReuseDecision { + const cacheDirectory = getCacheDirectory(identity.nextBuildHash) + const manifestPath = path.join(cacheDirectory, 'manifest.json') + const artifactDirectory = path.join(cacheDirectory, '.next') + const miss = (reason: string): BuildReuseDecision => ({ + reused: false, + nextBuildHash: identity.nextBuildHash, + reason, + cacheDirectory, + }) + + if (!existsSync(manifestPath) || !existsSync(artifactDirectory)) { + return miss('cache artifact or completed manifest is missing') + } + + let manifest: BuildManifest + try { + manifest = JSON.parse(readFileSync(manifestPath, 'utf8')) as BuildManifest + } catch { + return miss('cache manifest is unreadable') + } + if (!isMatchingIdentity(manifest, identity) || manifest.completed !== true) { + return miss('cache manifest identity does not match') + } + const buildIdPath = path.join(artifactDirectory, 'BUILD_ID') + if (!existsSync(buildIdPath) || readFileSync(buildIdPath, 'utf8').trim() !== manifest.buildId) { + return miss('cached BUILD_ID does not match the manifest') + } + if (hashDirectory(artifactDirectory) !== manifest.artifactHash) { + return miss('cached artifact checksum does not match the manifest') + } + + activateDirectory(artifactDirectory, NEXT_DIR) + return { + reused: true, + nextBuildHash: identity.nextBuildHash, + reason: 'verified cache hit', + cacheDirectory, + } +} + +export function storeCompletedBuild(identity: BuildIdentity): BuildReuseDecision { + const buildIdPath = path.join(NEXT_DIR, 'BUILD_ID') + if (!existsSync(buildIdPath)) { + throw new Error('Next build completed without .next/BUILD_ID') + } + + mkdirSync(E2E_BUILD_CACHE_DIR, { recursive: true }) + const cacheDirectory = getCacheDirectory(identity.nextBuildHash) + const temporaryDirectory = `${cacheDirectory}.tmp-${process.pid}-${Date.now()}` + rmSync(temporaryDirectory, { recursive: true, force: true }) + mkdirSync(temporaryDirectory, { recursive: true }) + const artifactDirectory = path.join(temporaryDirectory, '.next') + cpSync(NEXT_DIR, artifactDirectory, { recursive: true, verbatimSymlinks: true }) + + const manifest: BuildManifest = { + ...identity, + completed: true, + createdAt: new Date().toISOString(), + buildId: readFileSync(buildIdPath, 'utf8').trim(), + artifactHash: hashDirectory(artifactDirectory), + } + writeFileSync( + path.join(temporaryDirectory, 'manifest.json'), + `${JSON.stringify(manifest, null, 2)}\n`, + { mode: 0o600 } + ) + rmSync(cacheDirectory, { recursive: true, force: true }) + renameSync(temporaryDirectory, cacheDirectory) + + return { + reused: false, + nextBuildHash: identity.nextBuildHash, + reason: 'fresh build stored in verified cache', + cacheDirectory, + } +} + +export function clearActiveNextBuild(): void { + rmSync(NEXT_DIR, { recursive: true, force: true }) +} + +function listRepositoryFiles(): string[] { + const result = spawnSync( + 'git', + ['-C', REPO_ROOT, 'ls-files', '--cached', '--others', '--exclude-standard'], + { + encoding: 'utf8', + env: { NODE_ENV: 'test', PATH: process.env.PATH ?? '' }, + } + ) + if (result.status !== 0) { + throw new Error(`Unable to enumerate E2E hash inputs: ${result.stderr}`) + } + return [...new Set(result.stdout.split(/\r?\n/).filter(Boolean))].sort() +} + +function isNextBuildInput(relativePath: string): boolean { + const normalized = relativePath.replaceAll(path.sep, '/') + if (normalized.startsWith('packages/')) return true + if (ROOT_BUILD_FILES.has(normalized)) return true + if (!normalized.startsWith('apps/sim/')) return false + return !( + normalized.startsWith('apps/sim/e2e/') || + normalized.startsWith('apps/sim/.next/') || + normalized.startsWith('apps/sim/playwright-report/') || + normalized.startsWith('apps/sim/test-results/') + ) +} + +function hashOwnedFiles(files: string[], owner: E2eHashOwner): Array<[string, string]> { + return files + .filter((file) => classifyE2eHashOwner(file) === owner) + .map((file) => [file, hashRepositoryPath(file)]) +} + +function hashRepositoryFiles(files: string[]): string { + const hash = createHash('sha256') + for (const file of files) { + hash.update(file) + hash.update('\0') + hash.update(hashRepositoryPath(file)) + hash.update('\0') + } + return hash.digest('hex') +} + +function hashRepositoryPath(relativePath: string): string { + const absolutePath = path.join(REPO_ROOT, relativePath) + if (!existsSync(absolutePath)) return 'missing' + const stats = lstatSync(absolutePath) + if (stats.isSymbolicLink()) return hashText(`symlink:${readlinkSync(absolutePath)}`) + if (!stats.isFile()) return `unsupported:${stats.mode}` + return hashText(readFileSync(absolutePath)) +} + +function hashDirectory(directory: string): string { + const hash = createHash('sha256') + const visit = (current: string): void => { + for (const name of readdirSync(current).sort()) { + const absolutePath = path.join(current, name) + const relativePath = path.relative(directory, absolutePath).replaceAll(path.sep, '/') + const stats = lstatSync(absolutePath) + hash.update(relativePath) + hash.update('\0') + if (stats.isDirectory()) { + hash.update('directory\0') + visit(absolutePath) + } else if (stats.isSymbolicLink()) { + hash.update(`symlink:${readlinkSync(absolutePath)}\0`) + } else if (stats.isFile()) { + hash.update(readFileSync(absolutePath)) + hash.update('\0') + } else { + throw new Error(`Unsupported file in cached Next build: ${absolutePath}`) + } + } + } + visit(directory) + return hash.digest('hex') +} + +function activateDirectory(source: string, destination: string): void { + const temporary = `${destination}.e2e-restore-${process.pid}` + const backup = `${destination}.e2e-backup-${process.pid}` + rmSync(temporary, { recursive: true, force: true }) + rmSync(backup, { recursive: true, force: true }) + cpSync(source, temporary, { recursive: true, verbatimSymlinks: true }) + try { + if (existsSync(destination)) renameSync(destination, backup) + renameSync(temporary, destination) + rmSync(backup, { recursive: true, force: true }) + } catch (error) { + rmSync(temporary, { recursive: true, force: true }) + if (!existsSync(destination) && existsSync(backup)) renameSync(backup, destination) + throw error + } +} + +function isMatchingIdentity(manifest: BuildManifest, identity: BuildIdentity): boolean { + return ( + manifest.schemaVersion === identity.schemaVersion && + manifest.nextBuildHash === identity.nextBuildHash && + manifest.sourceHash === identity.sourceHash && + manifest.profileHash === identity.profileHash && + manifest.nodeVersion === identity.nodeVersion && + manifest.bunVersion === identity.bunVersion && + manifest.nextVersion === identity.nextVersion && + manifest.platform === identity.platform && + manifest.architecture === identity.architecture + ) +} + +function getCacheDirectory(nextBuildHash: string): string { + return path.join(E2E_BUILD_CACHE_DIR, nextBuildHash) +} + +function getExecutableVersion(executable: string): string { + const result = spawnSync(executable, ['--version'], { + encoding: 'utf8', + env: { NODE_ENV: 'test', PATH: process.env.PATH ?? '' }, + }) + if (result.status !== 0) { + throw new Error(`Unable to read ${executable} version: ${result.stderr}`) + } + return result.stdout.trim() +} + +function readPackageVersion(packageJsonPath: string): string { + const parsed = JSON.parse(readFileSync(packageJsonPath, 'utf8')) as { version?: string } + if (!parsed.version) throw new Error(`Package has no version: ${packageJsonPath}`) + return parsed.version +} + +function hashJson(value: unknown): string { + return hashText(JSON.stringify(value)) +} + +function hashText(value: string | Buffer): string { + return createHash('sha256').update(value).digest('hex') +} diff --git a/apps/sim/e2e/support/deployment-profile.ts b/apps/sim/e2e/support/deployment-profile.ts index badcd7bb054..63ffb844ec0 100644 --- a/apps/sim/e2e/support/deployment-profile.ts +++ b/apps/sim/e2e/support/deployment-profile.ts @@ -1,17 +1,17 @@ import { buildChildEnvironment, type ChildEnvironment } from './env' +import { E2E_CACHE_DIR } from './paths' export const E2E_PROFILE = 'hosted-billing-chromium' export const E2E_HOST = 'e2e.sim.ai' export const E2E_ORIGIN = `http://${E2E_HOST}:3000` export const E2E_SOCKET_ORIGIN = `http://${E2E_HOST}:3002` -const REQUIRED_KEYS = [ +const APP_REQUIRED_KEYS = [ 'NODE_ENV', 'NEXT_PUBLIC_APP_URL', 'BETTER_AUTH_URL', 'BETTER_AUTH_SECRET', 'DATABASE_URL', - 'MIGRATION_DATABASE_URL', 'ENCRYPTION_KEY', 'API_ENCRYPTION_KEY', 'INTERNAL_API_SECRET', @@ -23,7 +23,31 @@ const REQUIRED_KEYS = [ 'E2E_PROFILE', 'E2E_RUN_ID', 'HOME', - 'PLAYWRIGHT_BROWSERS_PATH', +] as const +const APP_ENVIRONMENT_KEYS = [ + ...APP_REQUIRED_KEYS, + 'NODE_OPTIONS', + 'NEXT_TELEMETRY_DISABLED', + 'XDG_CONFIG_HOME', + 'AWS_EC2_METADATA_DISABLED', + 'AWS_SHARED_CREDENTIALS_FILE', + 'AWS_CONFIG_FILE', + 'CLOUDSDK_CONFIG', + 'AZURE_CONFIG_DIR', + 'E2E_BASE_URL', + 'EMAIL_VERIFICATION_ENABLED', + 'EMAIL_PASSWORD_SIGNUP_ENABLED', + 'NEXT_PUBLIC_EMAIL_PASSWORD_SIGNUP_ENABLED', + 'DISABLE_REGISTRATION', + 'DISABLE_EMAIL_SIGNUP', + 'SIGNUP_MX_VALIDATION_ENABLED', + 'NEXT_PUBLIC_POSTHOG_ENABLED', + 'BLACKLISTED_PROVIDERS', + 'STRIPE_WEBHOOK_SECRET', + 'STRIPE_FREE_PRICE_ID', + 'SOCKET_SERVER_URL', + 'NEXT_PUBLIC_SOCKET_URL', + 'CI', ] as const const ALLOWED_SENSITIVE_KEYS = new Set([ @@ -50,7 +74,15 @@ export interface HostedBillingProfileOptions { export interface HostedBillingProfile { id: typeof E2E_PROFILE origin: typeof E2E_ORIGIN - childEnvironment: ChildEnvironment + environments: { + build: ChildEnvironment + app: ChildEnvironment + realtime: ChildEnvironment + migration: ChildEnvironment + seed: ChildEnvironment + authCapture: ChildEnvironment + playwright: ChildEnvironment + } } export function createHostedBillingProfile({ @@ -105,18 +137,162 @@ export function createHostedBillingProfile({ } validateProfileValues(values) + const buildHomeDirectory = `${E2E_CACHE_DIR}/build-home` + const buildValues = { + ...pickValues(values, APP_ENVIRONMENT_KEYS), + XDG_CONFIG_HOME: `${buildHomeDirectory}/xdg`, + AWS_EC2_METADATA_DISABLED: values.AWS_EC2_METADATA_DISABLED, + AWS_SHARED_CREDENTIALS_FILE: values.AWS_SHARED_CREDENTIALS_FILE, + AWS_CONFIG_FILE: values.AWS_CONFIG_FILE, + CLOUDSDK_CONFIG: `${buildHomeDirectory}/gcloud`, + AZURE_CONFIG_DIR: `${buildHomeDirectory}/azure`, + E2E_RUN_ID: 'build_sentinel', + HOME: buildHomeDirectory, + DATABASE_URL: 'postgresql://e2e_build:e2e_build@127.0.0.1:1/sim_e2e_build_sentinel', + STRIPE_API_BASE_URL: 'http://127.0.0.1:1', + CI: 'false', + } + validateProfileValues(buildValues) return { id: E2E_PROFILE, origin: E2E_ORIGIN, - childEnvironment: buildChildEnvironment({ - values, - required: REQUIRED_KEYS, - allowedSensitiveKeys: ALLOWED_SENSITIVE_KEYS, - }), + environments: { + build: createEnvironment(buildValues, APP_REQUIRED_KEYS), + app: createEnvironment(pickValues(values, APP_ENVIRONMENT_KEYS), APP_REQUIRED_KEYS), + realtime: createEnvironment( + pickValues(values, [ + 'NODE_ENV', + 'NODE_OPTIONS', + 'HOME', + 'DATABASE_URL', + 'BETTER_AUTH_URL', + 'BETTER_AUTH_SECRET', + 'INTERNAL_API_SECRET', + 'NEXT_PUBLIC_APP_URL', + 'E2E_RUN_ID', + 'CI', + ]), + [ + 'NODE_ENV', + 'HOME', + 'DATABASE_URL', + 'BETTER_AUTH_URL', + 'BETTER_AUTH_SECRET', + 'INTERNAL_API_SECRET', + 'NEXT_PUBLIC_APP_URL', + 'E2E_RUN_ID', + ], + false + ), + migration: createEnvironment( + pickValues(values, [ + 'NODE_ENV', + 'NODE_OPTIONS', + 'HOME', + 'MIGRATION_DATABASE_URL', + 'DATABASE_URL', + 'E2E_PROFILE', + 'E2E_RUN_ID', + 'CI', + ]), + ['NODE_ENV', 'HOME', 'MIGRATION_DATABASE_URL', 'DATABASE_URL', 'E2E_PROFILE', 'E2E_RUN_ID'], + false + ), + seed: createEnvironment( + pickValues(values, [ + 'NODE_ENV', + 'NODE_OPTIONS', + 'HOME', + 'DATABASE_URL', + 'ADMIN_API_KEY', + 'E2E_PROFILE', + 'E2E_RUN_ID', + 'E2E_BASE_URL', + 'CI', + ]), + [ + 'NODE_ENV', + 'HOME', + 'DATABASE_URL', + 'ADMIN_API_KEY', + 'E2E_PROFILE', + 'E2E_RUN_ID', + 'E2E_BASE_URL', + ], + false + ), + authCapture: createEnvironment( + pickValues(values, [ + 'NODE_ENV', + 'NODE_OPTIONS', + 'HOME', + 'PLAYWRIGHT_BROWSERS_PATH', + 'E2E_PROFILE', + 'E2E_RUN_ID', + 'E2E_BASE_URL', + 'CI', + ]), + [ + 'NODE_ENV', + 'HOME', + 'PLAYWRIGHT_BROWSERS_PATH', + 'E2E_PROFILE', + 'E2E_RUN_ID', + 'E2E_BASE_URL', + ], + false + ), + playwright: createEnvironment( + pickValues(values, [ + 'NODE_ENV', + 'NODE_OPTIONS', + 'HOME', + 'PLAYWRIGHT_BROWSERS_PATH', + 'E2E_PROFILE', + 'E2E_RUN_ID', + 'E2E_BASE_URL', + 'CI', + ]), + [ + 'NODE_ENV', + 'HOME', + 'PLAYWRIGHT_BROWSERS_PATH', + 'E2E_PROFILE', + 'E2E_RUN_ID', + 'E2E_BASE_URL', + ], + false + ), + }, } } +function createEnvironment( + values: Record, + required: readonly string[], + shadowDiscovered = true +): ChildEnvironment { + return buildChildEnvironment({ + values, + required, + allowedSensitiveKeys: ALLOWED_SENSITIVE_KEYS, + shadowDiscovered, + }) +} + +function pickValues( + values: Record, + keys: readonly string[] +): Record { + return Object.fromEntries( + keys.flatMap((key) => { + const value = values[key] + return value === undefined ? [] : [[key, value]] + }) + ) +} + function validateProfileValues(values: Record): void { if (values.NEXT_PUBLIC_APP_URL !== E2E_ORIGIN || values.BETTER_AUTH_URL !== E2E_ORIGIN) { throw new Error('E2E app and Better Auth origins must exactly match') diff --git a/apps/sim/e2e/support/env.ts b/apps/sim/e2e/support/env.ts index 7527596b41e..0ba38b20163 100644 --- a/apps/sim/e2e/support/env.ts +++ b/apps/sim/e2e/support/env.ts @@ -29,6 +29,7 @@ export interface BuildChildEnvironmentOptions { required: readonly string[] allowedSensitiveKeys: ReadonlySet envDirectory?: string + shadowDiscovered?: boolean } export function discoverEnvFileKeys(directory = SIM_APP_DIR): string[] { @@ -55,6 +56,7 @@ export function buildChildEnvironment({ required, allowedSensitiveKeys, envDirectory = SIM_APP_DIR, + shadowDiscovered = true, }: BuildChildEnvironmentOptions): ChildEnvironment { const missing = required.filter((key) => !values[key]?.trim()) if (missing.length > 0) { @@ -76,10 +78,12 @@ export function buildChildEnvironment({ const discoveredKeys = discoverEnvFileKeys(envDirectory) const shadowedKeys: string[] = [] - for (const key of discoveredKeys) { - if (key in env) continue - env[key] = '' - shadowedKeys.push(key) + if (shadowDiscovered) { + for (const key of discoveredKeys) { + if (key in env) continue + env[key] = '' + shadowedKeys.push(key) + } } return { env, discoveredKeys, shadowedKeys } diff --git a/apps/sim/e2e/support/paths.ts b/apps/sim/e2e/support/paths.ts index 44ec033e0e4..2dc124547cd 100644 --- a/apps/sim/e2e/support/paths.ts +++ b/apps/sim/e2e/support/paths.ts @@ -5,6 +5,8 @@ export const REPO_ROOT = path.resolve(SIM_APP_DIR, '../..') export const DB_PACKAGE_DIR = path.join(REPO_ROOT, 'packages/db') export const REALTIME_APP_DIR = path.join(REPO_ROOT, 'apps/realtime') export const PLAYWRIGHT_CLI = path.join(REPO_ROOT, 'node_modules/@playwright/test/cli.js') +export const E2E_CACHE_DIR = path.join(SIM_APP_DIR, 'e2e/.cache') +export const E2E_BUILD_CACHE_DIR = path.join(E2E_CACHE_DIR, 'builds') export function getRunDirectory(runId: string): string { return path.join(SIM_APP_DIR, 'e2e/.runs', runId) diff --git a/apps/sim/e2e/support/sandbox-bundles.ts b/apps/sim/e2e/support/sandbox-bundles.ts new file mode 100644 index 00000000000..66528bb4e57 --- /dev/null +++ b/apps/sim/e2e/support/sandbox-bundles.ts @@ -0,0 +1,61 @@ +import { createHash } from 'node:crypto' +import { readFileSync } from 'node:fs' +import path from 'node:path' +import { REPO_ROOT } from './paths' + +const INTEGRITY_PATH = 'apps/sim/lib/execution/sandbox/bundles/integrity.json' + +interface SandboxBundleIntegrity { + schemaVersion: 1 + bunVersion: string + sources: Record + dependencies: Record + outputs: Record +} + +export function verifySandboxBundleIntegrity(): void { + const integrity = readJson(path.join(REPO_ROOT, INTEGRITY_PATH)) + if (integrity.schemaVersion !== 1) { + throw new Error(`Unsupported sandbox bundle integrity version: ${integrity.schemaVersion}`) + } + const repositoryPackage = readJson<{ packageManager?: string }>( + path.join(REPO_ROOT, 'package.json') + ) + if (repositoryPackage.packageManager !== `bun@${integrity.bunVersion}`) { + throw new Error( + `Sandbox bundle Bun fingerprint is ${integrity.bunVersion}, but package.json declares ${repositoryPackage.packageManager ?? 'nothing'}` + ) + } + + verifyHashes('source', integrity.sources) + verifyHashes('output', integrity.outputs) + for (const [packageName, expectedVersion] of Object.entries(integrity.dependencies)) { + const packageJson = readJson<{ version?: string }>( + path.join(REPO_ROOT, 'node_modules', packageName, 'package.json') + ) + if (packageJson.version !== expectedVersion) { + throw new Error( + `Sandbox bundle dependency ${packageName} changed: expected ${expectedVersion}, received ${packageJson.version ?? 'missing'}` + ) + } + } +} + +function verifyHashes(kind: string, expected: Record): void { + for (const [relativePath, expectedHash] of Object.entries(expected)) { + const actualHash = hashFile(path.join(REPO_ROOT, relativePath)) + if (actualHash !== expectedHash) { + throw new Error( + `Sandbox bundle ${kind} fingerprint changed for ${relativePath}; regenerate bundles and review integrity.json` + ) + } + } +} + +function hashFile(filePath: string): string { + return createHash('sha256').update(readFileSync(filePath)).digest('hex') +} + +function readJson(filePath: string): T { + return JSON.parse(readFileSync(filePath, 'utf8')) as T +} diff --git a/apps/sim/e2e/support/stack.ts b/apps/sim/e2e/support/stack.ts index d1ce203d6b6..aeb51acce37 100644 --- a/apps/sim/e2e/support/stack.ts +++ b/apps/sim/e2e/support/stack.ts @@ -1,4 +1,12 @@ +import { mkdirSync, writeFileSync } from 'node:fs' import path from 'node:path' +import { + clearActiveNextBuild, + computeBuildIdentity, + restoreCachedBuild, + storeCompletedBuild, +} from './build-manifest' +import type { ChildEnvironment } from './env' import { DB_PACKAGE_DIR, PLAYWRIGHT_CLI, REALTIME_APP_DIR, REPO_ROOT, SIM_APP_DIR } from './paths' import { type ManagedProcess, @@ -7,6 +15,7 @@ import { waitForManagedProcessReady, } from './process' import { waitForHttpReady } from './readiness' +import { verifySandboxBundleIntegrity } from './sandbox-bundles' export interface StackCommandOptions { bunExecutable: string @@ -15,6 +24,11 @@ export interface StackCommandOptions { logsDirectory: string } +export interface BuildAppOptions extends StackCommandOptions { + buildEnvironment: ChildEnvironment + reuseBuild: boolean +} + export async function runMigrations(options: StackCommandOptions): Promise { await runCommand({ name: 'migrate', @@ -26,23 +40,35 @@ export async function runMigrations(options: StackCommandOptions): Promise }) } -export async function buildApp(options: StackCommandOptions): Promise { - await runCommand({ - name: 'sandbox-bundles', - command: options.bunExecutable, - args: ['--no-env-file', 'run', 'build:sandbox-bundles'], - cwd: SIM_APP_DIR, - env: options.env, - logsDirectory: options.logsDirectory, +export async function buildApp(options: BuildAppOptions): Promise { + verifySandboxBundleIntegrity() + const identity = computeBuildIdentity({ + buildEnvironment: options.buildEnvironment, + nodeExecutable: options.nodeExecutable, }) + if (options.reuseBuild) { + const reuseDecision = restoreCachedBuild(identity) + writeBuildDecision(options.logsDirectory, reuseDecision) + if (reuseDecision.reused) { + console.info(`Reused verified Next build ${identity.nextBuildHash}`) + return + } + console.info(`Next build cache miss: ${reuseDecision.reason}`) + } + + clearActiveNextBuild() + const buildHome = options.buildEnvironment.env.HOME + if (buildHome) mkdirSync(buildHome, { recursive: true }) await runCommand({ name: 'next-build', command: options.nodeExecutable, args: [path.join(REPO_ROOT, 'node_modules/next/dist/bin/next'), 'build'], cwd: SIM_APP_DIR, - env: options.env, + env: options.buildEnvironment.env, logsDirectory: options.logsDirectory, }) + const storedDecision = storeCompletedBuild(identity) + writeBuildDecision(options.logsDirectory, storedDecision) } export async function startRealtime(options: StackCommandOptions): Promise { @@ -134,3 +160,10 @@ export async function runPlaywright( logsDirectory: options.logsDirectory, }) } + +function writeBuildDecision(logsDirectory: string, decision: object): void { + writeFileSync( + path.join(logsDirectory, 'build-reuse-decision.json'), + `${JSON.stringify(decision, null, 2)}\n` + ) +} diff --git a/apps/sim/lib/execution/sandbox/bundles/build.ts b/apps/sim/lib/execution/sandbox/bundles/build.ts index 5e6ab81645d..4b6e0368028 100644 --- a/apps/sim/lib/execution/sandbox/bundles/build.ts +++ b/apps/sim/lib/execution/sandbox/bundles/build.ts @@ -11,7 +11,7 @@ */ import { mkdirSync, rmSync, writeFileSync } from 'node:fs' -import { dirname, join } from 'node:path' +import { dirname, isAbsolute, join, resolve } from 'node:path' import { fileURLToPath } from 'node:url' import { createLogger } from '@sim/logger' @@ -33,8 +33,6 @@ interface BunBuildOptions { declare const Bun: { build: (opts: BunBuildOptions) => Promise } const HERE = dirname(fileURLToPath(import.meta.url)) -const BUNDLES_DIR = HERE -const ENTRIES_DIR = join(HERE, '.entries') const APP_SIM_ROOT = join(HERE, '..', '..', '..', '..') interface BundleSpec { @@ -91,12 +89,15 @@ globalThis.__bundles['pptxgenjs'] = PptxGenJS ] async function main(): Promise { - rmSync(ENTRIES_DIR, { recursive: true, force: true }) - mkdirSync(ENTRIES_DIR, { recursive: true }) - mkdirSync(BUNDLES_DIR, { recursive: true }) + const outputDirectory = resolveOutputDirectory(process.argv.slice(2)) + const entriesDirectory = join(outputDirectory, '.entries') + + rmSync(entriesDirectory, { recursive: true, force: true }) + mkdirSync(entriesDirectory, { recursive: true }) + mkdirSync(outputDirectory, { recursive: true }) for (const spec of BUNDLES) { - const entryPath = join(ENTRIES_DIR, `${spec.name}.entry.ts`) + const entryPath = join(entriesDirectory, `${spec.name}.entry.ts`) writeFileSync(entryPath, spec.entry, 'utf-8') const result = await Bun.build({ @@ -121,11 +122,23 @@ async function main(): Promise { const code = await result.outputs[0].text() const banner = `// sandbox bundle: ${spec.name}\n// generated by apps/sim/lib/execution/sandbox/bundles/build.ts\n// do not edit by hand. run \`bun run build:sandbox-bundles\` to regenerate.\n` - writeFileSync(join(BUNDLES_DIR, spec.outFile), banner + code, 'utf-8') + writeFileSync(join(outputDirectory, spec.outFile), banner + code, 'utf-8') logger.info(`built ${spec.outFile} (${code.length.toLocaleString()} chars)`) } - rmSync(ENTRIES_DIR, { recursive: true, force: true }) + rmSync(entriesDirectory, { recursive: true, force: true }) +} + +function resolveOutputDirectory(args: string[]): string { + if (args.length === 0) return HERE + if (args.length !== 1 || !args[0].startsWith('--output-dir=')) { + throw new Error('Usage: build.ts [--output-dir=/absolute/path]') + } + const requested = args[0].slice('--output-dir='.length) + if (!requested || !isAbsolute(requested)) { + throw new Error('--output-dir must be an absolute path') + } + return resolve(requested) } main().catch((err) => { diff --git a/apps/sim/lib/execution/sandbox/bundles/integrity.json b/apps/sim/lib/execution/sandbox/bundles/integrity.json new file mode 100644 index 00000000000..2ef8966b49d --- /dev/null +++ b/apps/sim/lib/execution/sandbox/bundles/integrity.json @@ -0,0 +1,20 @@ +{ + "schemaVersion": 1, + "bunVersion": "1.3.13", + "sources": { + "apps/sim/lib/execution/sandbox/bundles/build.ts": "da99fa40ae36a694c45a5f63f901ec8132f653184f3a2de87ff39fe3ac1e7133", + "apps/sim/lib/execution/sandbox/bundles/_polyfills.ts": "131dd6e7aec37bff3acf6507d6fe7bc95aeabae07fa2e97bbf2e32aa7251d491" + }, + "dependencies": { + "pdf-lib": "1.17.1", + "docx": "9.7.1", + "pptxgenjs": "4.0.1", + "buffer": "6.0.3", + "process": "0.11.10" + }, + "outputs": { + "apps/sim/lib/execution/sandbox/bundles/pdf-lib.cjs": "936750faa220af85bf9b4edfe82bcab64ad38ee637b7f584322598f229d24036", + "apps/sim/lib/execution/sandbox/bundles/docx.cjs": "4c2aaf691ab633f272cd654df92be557e460874ace581e3fdbca4ae44c1c62ce", + "apps/sim/lib/execution/sandbox/bundles/pptxgenjs.cjs": "f83df52b4577edce2ae0808e0f6da97cd119f1cbb475374b2e4ea13571fa024b" + } +} From 48c2d4501e54a1aeb272fe48d07c5f29a3ac1bd1 Mon Sep 17 00:00:00 2001 From: Bill Leoutsakos Date: Mon, 20 Jul 2026 21:57:19 -0700 Subject: [PATCH 2/6] Add validated E2E persona world model Model settings personas as deterministic, serializable resource graphs and provide production-first factories with exact-ID cleanup so impossible or cross-world fixture states fail before browser tests run. Co-authored-by: Cursor --- apps/sim/e2e/fixtures/cleanup.ts | 172 +++++ apps/sim/e2e/fixtures/e2e-world.ts | 263 +++++++ apps/sim/e2e/fixtures/factories/billing.ts | 143 ++++ .../sim/e2e/fixtures/factories/invitations.ts | 42 ++ .../e2e/fixtures/factories/organizations.ts | 52 ++ .../fixtures/factories/permission-groups.ts | 61 ++ apps/sim/e2e/fixtures/factories/platform.ts | 16 + apps/sim/e2e/fixtures/factories/users.ts | 49 ++ apps/sim/e2e/fixtures/factories/workspaces.ts | 49 ++ apps/sim/e2e/fixtures/http-client.ts | 144 ++++ apps/sim/e2e/fixtures/namespace.ts | 86 +++ apps/sim/e2e/fixtures/scenario.ts | 191 +++++ apps/sim/e2e/fixtures/validate-scenario.ts | 683 ++++++++++++++++++ apps/sim/e2e/foundation/http-client.spec.ts | 67 ++ .../foundation/scenario-validation.spec.ts | 291 ++++++++ apps/sim/e2e/settings/personas.ts | 442 ++++++++++++ 16 files changed, 2751 insertions(+) create mode 100644 apps/sim/e2e/fixtures/cleanup.ts create mode 100644 apps/sim/e2e/fixtures/e2e-world.ts create mode 100644 apps/sim/e2e/fixtures/factories/billing.ts create mode 100644 apps/sim/e2e/fixtures/factories/invitations.ts create mode 100644 apps/sim/e2e/fixtures/factories/organizations.ts create mode 100644 apps/sim/e2e/fixtures/factories/permission-groups.ts create mode 100644 apps/sim/e2e/fixtures/factories/platform.ts create mode 100644 apps/sim/e2e/fixtures/factories/users.ts create mode 100644 apps/sim/e2e/fixtures/factories/workspaces.ts create mode 100644 apps/sim/e2e/fixtures/http-client.ts create mode 100644 apps/sim/e2e/fixtures/namespace.ts create mode 100644 apps/sim/e2e/fixtures/scenario.ts create mode 100644 apps/sim/e2e/fixtures/validate-scenario.ts create mode 100644 apps/sim/e2e/foundation/http-client.spec.ts create mode 100644 apps/sim/e2e/foundation/scenario-validation.spec.ts create mode 100644 apps/sim/e2e/settings/personas.ts diff --git a/apps/sim/e2e/fixtures/cleanup.ts b/apps/sim/e2e/fixtures/cleanup.ts new file mode 100644 index 00000000000..2c9d68756a8 --- /dev/null +++ b/apps/sim/e2e/fixtures/cleanup.ts @@ -0,0 +1,172 @@ +import { db } from '@sim/db' +import { + auditLog, + invitation, + invitationWorkspaceGrant, + member, + organization, + permissionGroup, + permissionGroupMember, + permissionGroupWorkspace, + permissions, + settings, + subscription, + user, + userStats, + workspace, +} from '@sim/db/schema' +import { inArray, or } from 'drizzle-orm' +import type { ScenarioManifest } from './e2e-world' + +/** + * Deletes only IDs recorded by a successful seed. Names and prefixes are never used as delete + * predicates. Identity verification makes a stale or hand-edited manifest fail closed. + */ +export async function cleanupSeededWorld(manifest: ScenarioManifest): Promise { + const userIdentities = Object.values(manifest.worlds).flatMap((world) => + Object.values(world.userIdentities) + ) + const organizationIdentities = Object.values(manifest.worlds).flatMap((world) => + Object.values(world.organizationIdentities) + ) + const workspaceIdentities = Object.values(manifest.worlds).flatMap((world) => + Object.values(world.workspaceIdentities) + ) + await verifyOwnership(userIdentities, organizationIdentities, workspaceIdentities) + + const userIds = userIdentities.map(({ id }) => id) + const organizationIds = organizationIdentities.map(({ id }) => id) + const workspaceIds = workspaceIdentities.map(({ id }) => id) + const resourceIds = [ + ...workspaceIds, + ...organizationIds, + ...Object.values(manifest.personas).flatMap(({ permissionGroupIds }) => permissionGroupIds), + ] + const subscriptionIds = collectValues(manifest, 'subscriptionIds') + const permissionIds = collectValues(manifest, 'permissionIds') + const permissionGroupIds = collectValues(manifest, 'permissionGroupIds') + const permissionGroupMemberIds = collectValues(manifest, 'permissionGroupMemberIds') + const invitationIds = collectValues(manifest, 'invitationIds') + const invitationGrantIds = collectValues(manifest, 'invitationGrantIds') + const organizationMemberIds = collectValues(manifest, 'organizationMemberIds') + + await db.transaction(async (tx) => { + const auditPredicates = [ + workspaceIds.length ? inArray(auditLog.workspaceId, workspaceIds) : undefined, + userIds.length ? inArray(auditLog.actorId, userIds) : undefined, + resourceIds.length ? inArray(auditLog.resourceId, resourceIds) : undefined, + ].filter((value): value is NonNullable => value !== undefined) + if (auditPredicates.length > 0) await tx.delete(auditLog).where(or(...auditPredicates)) + + if (invitationGrantIds.length) { + await tx + .delete(invitationWorkspaceGrant) + .where(inArray(invitationWorkspaceGrant.id, invitationGrantIds)) + } + if (invitationIds.length) { + await tx.delete(invitation).where(inArray(invitation.id, invitationIds)) + } + if (permissionGroupMemberIds.length) { + await tx + .delete(permissionGroupMember) + .where(inArray(permissionGroupMember.id, permissionGroupMemberIds)) + } + if (permissionGroupIds.length) { + await tx + .delete(permissionGroupWorkspace) + .where(inArray(permissionGroupWorkspace.permissionGroupId, permissionGroupIds)) + await tx.delete(permissionGroup).where(inArray(permissionGroup.id, permissionGroupIds)) + } + if (permissionIds.length) { + await tx.delete(permissions).where(inArray(permissions.id, permissionIds)) + } + if (workspaceIds.length) { + await tx.delete(workspace).where(inArray(workspace.id, workspaceIds)) + } + if (organizationMemberIds.length) { + await tx.delete(member).where(inArray(member.id, organizationMemberIds)) + } + if (subscriptionIds.length) { + await tx.delete(subscription).where(inArray(subscription.id, subscriptionIds)) + } + if (organizationIds.length) { + await tx.delete(organization).where(inArray(organization.id, organizationIds)) + } + if (userIds.length) { + await tx.delete(settings).where(inArray(settings.userId, userIds)) + await tx.delete(userStats).where(inArray(userStats.userId, userIds)) + await tx.delete(user).where(inArray(user.id, userIds)) + } + }) +} + +async function verifyOwnership( + expectedUsers: Array<{ id: string; email: string; name: string }>, + expectedOrganizations: Array<{ id: string; name: string; slug: string }>, + expectedWorkspaces: Array<{ id: string; name: string }> +): Promise { + const actualUsers = expectedUsers.length + ? await db + .select({ id: user.id, email: user.email, name: user.name }) + .from(user) + .where( + inArray( + user.id, + expectedUsers.map(({ id }) => id) + ) + ) + : [] + const actualOrganizations = expectedOrganizations.length + ? await db + .select({ id: organization.id, name: organization.name, slug: organization.slug }) + .from(organization) + .where( + inArray( + organization.id, + expectedOrganizations.map(({ id }) => id) + ) + ) + : [] + const actualWorkspaces = expectedWorkspaces.length + ? await db + .select({ id: workspace.id, name: workspace.name }) + .from(workspace) + .where( + inArray( + workspace.id, + expectedWorkspaces.map(({ id }) => id) + ) + ) + : [] + + assertExactIdentities('user', expectedUsers, actualUsers) + assertExactIdentities('organization', expectedOrganizations, actualOrganizations) + assertExactIdentities('workspace', expectedWorkspaces, actualWorkspaces) +} + +function assertExactIdentities( + label: string, + expected: T[], + actual: T[] +): void { + const actualById = new Map(actual.map((value) => [value.id, value])) + for (const value of expected) { + if (JSON.stringify(actualById.get(value.id)) !== JSON.stringify(value)) { + throw new Error(`Refusing cleanup: ${label} ownership mismatch for exact ID ${value.id}`) + } + } +} + +function collectValues( + manifest: ScenarioManifest, + key: + | 'subscriptionIds' + | 'permissionIds' + | 'permissionGroupIds' + | 'permissionGroupMemberIds' + | 'invitationIds' + | 'invitationGrantIds' + | 'organizationMemberIds' +): string[] { + return Object.values(manifest.worlds).flatMap((world) => Object.values(world[key])) +} diff --git a/apps/sim/e2e/fixtures/e2e-world.ts b/apps/sim/e2e/fixtures/e2e-world.ts new file mode 100644 index 00000000000..ffa2b05cab9 --- /dev/null +++ b/apps/sim/e2e/fixtures/e2e-world.ts @@ -0,0 +1,263 @@ +import { randomUUID } from 'node:crypto' +import { mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs' +import path from 'node:path' +import { z } from 'zod' +import type { + ResolvedScenario, + ScenarioPersona, + ScenarioWorkspace, +} from './scenario' + +export const PERSONA_MANIFEST_VERSION = 1 as const + +const workspaceExpectationSchema = z.object({ + workspaceId: z.string(), + workspaceKey: z.string(), + access: z.enum(['none', 'read', 'write', 'admin']), + roleSource: z.enum(['none', 'owner', 'explicit', 'org-admin']), + hostContext: z.object({ + isOwner: z.boolean(), + hostMembership: z.enum(['owner', 'member', 'external']), + payerScope: z.enum(['user', 'organization']), + plan: z.enum([ + 'free', + 'pro_6000', + 'pro_25000', + 'team_6000', + 'team_25000', + 'enterprise', + ]), + hosted: z.boolean(), + billingEnabled: z.boolean(), + }), +}) + +export const personaManifestEntrySchema = z.object({ + key: z.string(), + world: z.string(), + userId: z.string(), + email: z.string().email(), + name: z.string(), + expectedActiveOrganizationId: z.string().nullable(), + storageStatePath: z.string(), + canonicalRoute: z.string().startsWith('/'), + workspaces: z.array(workspaceExpectationSchema).min(1), + permissionGroupIds: z.array(z.string()), + expectedPlatformRole: z.enum(['user', 'admin']), + expectedSuperUserMode: z.boolean(), +}) + +const worldManifestSchema = z.object({ + namespace: z.object({ + run: z.string(), + world: z.string(), + prefix: z.string(), + }), + userIds: z.record(z.string(), z.string()), + userIdentities: z.record( + z.string(), + z.object({ id: z.string(), email: z.string().email(), name: z.string() }) + ), + organizationIds: z.record(z.string(), z.string()), + organizationIdentities: z.record( + z.string(), + z.object({ id: z.string(), name: z.string(), slug: z.string() }) + ), + organizationMemberIds: z.record(z.string(), z.string()), + workspaceIds: z.record(z.string(), z.string()), + workspaceIdentities: z.record( + z.string(), + z.object({ id: z.string(), name: z.string() }) + ), + subscriptionIds: z.record(z.string(), z.string()), + permissionIds: z.record(z.string(), z.string()), + permissionGroupIds: z.record(z.string(), z.string()), + permissionGroupMemberIds: z.record(z.string(), z.string()), + invitationIds: z.record(z.string(), z.string()), + invitationGrantIds: z.record(z.string(), z.string()), +}) + +export const scenarioManifestSchema = z.object({ + schemaVersion: z.literal(PERSONA_MANIFEST_VERSION), + runId: z.string(), + createdAt: z.string(), + authCaptureComplete: z.boolean(), + worlds: z.record(z.string(), worldManifestSchema), + personas: z.record(z.string(), personaManifestEntrySchema), +}) + +export const personaCredentialsSchema = z.object({ + schemaVersion: z.literal(PERSONA_MANIFEST_VERSION), + runId: z.string(), + personas: z.record( + z.string(), + z.object({ + email: z.string().email(), + password: z.string().min(12), + }) + ), +}) + +export type ScenarioManifest = z.infer +export type PersonaManifestEntry = z.infer +export type PersonaCredentials = z.infer + +export interface CreatedWorldRecords { + users: Map + organizations: Map + organizationMembers: Map + workspaces: Map + subscriptions: Map + permissions: Map + permissionGroups: Map + permissionGroupMembers: Map + invitations: Map + invitationGrants: Map +} + +export interface E2EWorld { + scenario: ResolvedScenario + records: CreatedWorldRecords +} + +export function createWorldRecords(): CreatedWorldRecords { + return { + users: new Map(), + organizations: new Map(), + organizationMembers: new Map(), + workspaces: new Map(), + subscriptions: new Map(), + permissions: new Map(), + permissionGroups: new Map(), + permissionGroupMembers: new Map(), + invitations: new Map(), + invitationGrants: new Map(), + } +} + +export function buildScenarioManifest(runId: string, worlds: E2EWorld[]): ScenarioManifest { + const manifest: ScenarioManifest = { + schemaVersion: PERSONA_MANIFEST_VERSION, + runId, + createdAt: new Date().toISOString(), + authCaptureComplete: false, + worlds: {}, + personas: {}, + } + + for (const world of worlds) { + const worldKey = world.scenario.definition.namespace.world + manifest.worlds[worldKey] = { + namespace: world.scenario.definition.namespace, + userIds: mapValues(world.records.users, ({ id }) => id), + userIdentities: Object.fromEntries(world.records.users), + organizationIds: mapValues(world.records.organizations, ({ id }) => id), + organizationIdentities: Object.fromEntries( + [...world.records.organizations].map(([key, { id, name, slug }]) => [ + key, + { id, name, slug }, + ]) + ), + organizationMemberIds: Object.fromEntries(world.records.organizationMembers), + workspaceIds: mapValues(world.records.workspaces, ({ id }) => id), + workspaceIdentities: Object.fromEntries(world.records.workspaces), + subscriptionIds: Object.fromEntries(world.records.subscriptions), + permissionIds: Object.fromEntries(world.records.permissions), + permissionGroupIds: Object.fromEntries(world.records.permissionGroups), + permissionGroupMemberIds: Object.fromEntries(world.records.permissionGroupMembers), + invitationIds: Object.fromEntries(world.records.invitations), + invitationGrantIds: Object.fromEntries(world.records.invitationGrants), + } + for (const persona of world.scenario.definition.personas) { + if (manifest.personas[persona.key]) { + throw new Error(`Duplicate cross-world persona key: ${persona.key}`) + } + manifest.personas[persona.key] = buildPersonaManifest(world, persona) + } + } + assertManifestContainsNoSecrets(manifest) + return scenarioManifestSchema.parse(manifest) +} + +export function readScenarioManifest(filePath: string): ScenarioManifest { + return scenarioManifestSchema.parse(JSON.parse(readFileSync(filePath, 'utf8'))) +} + +export function readPersonaCredentials(filePath: string): PersonaCredentials { + return personaCredentialsSchema.parse(JSON.parse(readFileSync(filePath, 'utf8'))) +} + +export function writeJsonAtomic(filePath: string, value: unknown, mode = 0o600): void { + mkdirSync(path.dirname(filePath), { recursive: true }) + const temporaryPath = `${filePath}.tmp-${process.pid}-${randomUUID()}` + writeFileSync(temporaryPath, `${JSON.stringify(value, null, 2)}\n`, { mode }) + renameSync(temporaryPath, filePath) +} + +export function assertManifestContainsNoSecrets(value: unknown): void { + const serialized = JSON.stringify(value) + for (const forbidden of [ + 'password', + 'cookie', + 'adminApiKey', + 'databaseUrl', + 'invitationToken', + 'stripeSecret', + ]) { + if (serialized.toLowerCase().includes(forbidden.toLowerCase())) { + throw new Error(`Non-secret scenario manifest contains forbidden field: ${forbidden}`) + } + } +} + +function buildPersonaManifest(world: E2EWorld, persona: ScenarioPersona): PersonaManifestEntry { + const user = required(world.records.users, persona.userKey, 'user') + const membership = world.scenario.definition.organizationMemberships.find( + ({ userKey }) => userKey === persona.userKey + ) + const organizationId = membership + ? required(world.records.organizations, membership.organizationKey, 'organization').id + : null + const canonicalWorkspace = required( + world.records.workspaces, + persona.canonicalRoute.workspaceKey, + 'canonical workspace' + ) + return { + key: persona.key, + world: world.scenario.definition.namespace.world, + userId: user.id, + email: user.email, + name: user.name, + expectedActiveOrganizationId: organizationId, + storageStatePath: persona.storageStateFilename, + canonicalRoute: `/workspace/${encodeURIComponent(canonicalWorkspace.id)}/settings/${ + persona.canonicalRoute.settingsSection + }`, + workspaces: persona.workspaces.map((expectation) => ({ + workspaceId: required(world.records.workspaces, expectation.workspaceKey, 'workspace').id, + workspaceKey: expectation.workspaceKey, + access: expectation.access, + roleSource: expectation.roleSource, + hostContext: expectation.hostContext, + })), + permissionGroupIds: persona.permissionGroupKeys.map((key) => + required(world.records.permissionGroups, key, 'permission group') + ), + expectedPlatformRole: persona.expectedPlatformRole, + expectedSuperUserMode: persona.expectedSuperUserMode, + } +} + +function mapValues( + values: ReadonlyMap, + select: (value: T) => string +): Record { + return Object.fromEntries([...values].map(([key, value]) => [key, select(value)])) +} + +function required(values: ReadonlyMap, key: string, label: string): T { + const value = values.get(key) + if (!value) throw new Error(`Missing created ${label}: ${key}`) + return value +} diff --git a/apps/sim/e2e/fixtures/factories/billing.ts b/apps/sim/e2e/fixtures/factories/billing.ts new file mode 100644 index 00000000000..b35fef2ef6b --- /dev/null +++ b/apps/sim/e2e/fixtures/factories/billing.ts @@ -0,0 +1,143 @@ +import { db } from '@sim/db' +import { organization, subscription, userStats } from '@sim/db/schema' +import { generateId } from '@sim/utils/id' +import { and, eq, inArray } from 'drizzle-orm' + +const CREDITS_PER_DOLLAR = 200 +const FREE_USAGE_LIMIT_DOLLARS = '5' + +export interface SubscriptionArrangement { + referenceId: string + plan: 'pro_6000' | 'pro_25000' | 'team_6000' | 'team_25000' | 'enterprise' + status?: 'active' | 'past_due' | 'canceled' + seats?: number + memberUserIds?: string[] + enterprise?: { + monthlyPrice: number + } +} + +export async function arrangeSubscription(input: SubscriptionArrangement): Promise<{ id: string }> { + const entitled = await db + .select({ id: subscription.id }) + .from(subscription) + .where( + and( + eq(subscription.referenceId, input.referenceId), + inArray(subscription.status, ['active', 'past_due']) + ) + ) + .limit(1) + if (entitled.length > 0 && (input.status ?? 'active') !== 'canceled') { + throw new Error(`Billing reference already has an entitled subscription: ${input.referenceId}`) + } + + const now = new Date() + const periodEnd = new Date(now.getTime() + 30 * 24 * 60 * 60 * 1_000) + const id = generateId() + const status = input.status ?? 'active' + const metadata = + input.plan === 'enterprise' + ? { + plan: 'enterprise', + referenceId: input.referenceId, + monthlyPrice: input.enterprise?.monthlyPrice ?? 500, + seats: input.seats ?? 1, + } + : null + + await db.transaction(async (tx) => { + await tx.insert(subscription).values({ + id, + plan: input.plan, + referenceId: input.referenceId, + stripeCustomerId: `cus_e2e_${id}`, + stripeSubscriptionId: `sub_e2e_${id}`, + status, + periodStart: now, + periodEnd, + cancelAtPeriodEnd: status !== 'active', + canceledAt: status === 'canceled' ? now : null, + endedAt: status === 'canceled' ? now : null, + seats: input.seats, + billingInterval: 'month', + metadata, + }) + + if (input.plan.startsWith('pro_')) { + await tx + .update(userStats) + .set({ + currentUsageLimit: String(planCredits(input.plan) / CREDITS_PER_DOLLAR), + usageLimitUpdatedAt: now, + billingBlocked: false, + billingBlockedReason: null, + }) + .where(eq(userStats.userId, input.referenceId)) + return + } + + const orgUsageLimit = + input.plan === 'enterprise' + ? String(metadata?.monthlyPrice ?? 500) + : String((planCredits(input.plan) / CREDITS_PER_DOLLAR) * (input.seats ?? 1)) + await tx + .update(organization) + .set({ orgUsageLimit, updatedAt: now }) + .where(eq(organization.id, input.referenceId)) + if (input.memberUserIds && input.memberUserIds.length > 0) { + await tx + .update(userStats) + .set({ + currentUsageLimit: null, + usageLimitUpdatedAt: now, + billingBlocked: false, + billingBlockedReason: null, + }) + .where(inArray(userStats.userId, input.memberUserIds)) + } + }) + + return { id } +} + +export async function lapseOrganizationSubscription(input: { + subscriptionId: string + organizationId: string + memberUserIds: string[] +}): Promise { + const now = new Date() + await db.transaction(async (tx) => { + await tx + .update(subscription) + .set({ + status: 'canceled', + cancelAtPeriodEnd: false, + cancelAt: now, + canceledAt: now, + endedAt: now, + periodEnd: now, + }) + .where(eq(subscription.id, input.subscriptionId)) + await tx + .update(organization) + .set({ orgUsageLimit: null, updatedAt: now }) + .where(eq(organization.id, input.organizationId)) + if (input.memberUserIds.length > 0) { + await tx + .update(userStats) + .set({ + currentUsageLimit: FREE_USAGE_LIMIT_DOLLARS, + usageLimitUpdatedAt: now, + billingBlocked: false, + billingBlockedReason: null, + }) + .where(inArray(userStats.userId, input.memberUserIds)) + } + }) +} + +function planCredits(plan: SubscriptionArrangement['plan']): number { + const match = plan.match(/_(\d+)$/) + return match ? Number(match[1]) : 0 +} diff --git a/apps/sim/e2e/fixtures/factories/invitations.ts b/apps/sim/e2e/fixtures/factories/invitations.ts new file mode 100644 index 00000000000..3ed895ad833 --- /dev/null +++ b/apps/sim/e2e/fixtures/factories/invitations.ts @@ -0,0 +1,42 @@ +import { db } from '@sim/db' +import { invitation, invitationWorkspaceGrant } from '@sim/db/schema' +import { generateId } from '@sim/utils/id' + +export async function arrangePendingInvitation(input: { + email: string + token: string + inviterId: string + organizationId: string + workspaceId: string + role: 'admin' | 'member' + permission: 'admin' | 'write' | 'read' +}): Promise<{ invitationId: string; grantId: string }> { + const invitationId = generateId() + const grantId = generateId() + const now = new Date() + await db.transaction(async (tx) => { + await tx.insert(invitation).values({ + id: invitationId, + kind: 'workspace', + email: input.email.trim().toLowerCase(), + inviterId: input.inviterId, + organizationId: input.organizationId, + membershipIntent: 'internal', + role: input.role, + status: 'pending', + token: input.token, + expiresAt: new Date(now.getTime() + 7 * 24 * 60 * 60 * 1_000), + createdAt: now, + updatedAt: now, + }) + await tx.insert(invitationWorkspaceGrant).values({ + id: grantId, + invitationId, + workspaceId: input.workspaceId, + permission: input.permission, + createdAt: now, + updatedAt: now, + }) + }) + return { invitationId, grantId } +} diff --git a/apps/sim/e2e/fixtures/factories/organizations.ts b/apps/sim/e2e/fixtures/factories/organizations.ts new file mode 100644 index 00000000000..d374b7fda49 --- /dev/null +++ b/apps/sim/e2e/fixtures/factories/organizations.ts @@ -0,0 +1,52 @@ +import { z } from 'zod' +import { E2eHttpClient } from '../http-client' + +const organizationSchema = z.object({ + id: z.string(), + name: z.string(), + slug: z.string(), + memberId: z.string(), +}) +const organizationMemberSchema = z.object({ + id: z.string(), + userId: z.string(), + organizationId: z.string(), + role: z.string(), + action: z.enum(['created', 'updated', 'already_member']), +}) + +export function createAdminClient(baseUrl: string, adminApiKey: string): E2eHttpClient { + return new E2eHttpClient({ + baseUrl, + defaultHeaders: { 'x-admin-key': adminApiKey }, + }) +} + +export async function createOrganization( + adminClient: E2eHttpClient, + input: { name: string; slug: string; ownerId: string } +): Promise> { + const response = await adminClient.request({ + method: 'POST', + path: '/api/v1/admin/organizations', + body: input, + schema: z.object({ data: organizationSchema }), + expectedStatus: 200, + }) + return response.data +} + +export async function addOrganizationMember( + adminClient: E2eHttpClient, + organizationId: string, + input: { userId: string; role: 'admin' | 'member' } +): Promise> { + const response = await adminClient.request({ + method: 'POST', + path: `/api/v1/admin/organizations/${organizationId}/members`, + body: input, + schema: z.object({ data: organizationMemberSchema }), + expectedStatus: 200, + }) + return response.data +} diff --git a/apps/sim/e2e/fixtures/factories/permission-groups.ts b/apps/sim/e2e/fixtures/factories/permission-groups.ts new file mode 100644 index 00000000000..dcdaf99b378 --- /dev/null +++ b/apps/sim/e2e/fixtures/factories/permission-groups.ts @@ -0,0 +1,61 @@ +import { z } from 'zod' +import type { E2eHttpClient } from '../http-client' + +const permissionGroupSchema = z.object({ + id: z.string(), + organizationId: z.string(), + name: z.string(), + createdBy: z.string(), + isDefault: z.boolean(), + workspaceIds: z.array(z.string()), + config: z.record(z.string(), z.unknown()), +}) + +export interface RestrictedPermissionConfig { + hideSecretsTab: boolean + hideApiKeysTab: boolean + hideInboxTab: boolean + disableMcpTools: boolean + disableCustomTools: boolean +} + +export async function createPermissionGroup( + ownerClient: E2eHttpClient, + organizationId: string, + input: { + name: string + description?: string + workspaceIds: string[] + config: RestrictedPermissionConfig + } +): Promise> { + const response = await ownerClient.request({ + method: 'POST', + path: `/api/organizations/${organizationId}/permission-groups`, + body: { ...input, isDefault: false }, + schema: z.object({ permissionGroup: permissionGroupSchema }), + expectedStatus: 201, + }) + return response.permissionGroup +} + +export async function addPermissionGroupMember( + ownerClient: E2eHttpClient, + organizationId: string, + permissionGroupId: string, + userId: string +): Promise { + const response = await ownerClient.request({ + method: 'POST', + path: `/api/organizations/${organizationId}/permission-groups/${permissionGroupId}/members`, + body: { userId }, + schema: z.object({ + member: z.object({ + id: z.string(), + userId: z.string(), + }), + }), + expectedStatus: 201, + }) + return response.member.id +} diff --git a/apps/sim/e2e/fixtures/factories/platform.ts b/apps/sim/e2e/fixtures/factories/platform.ts new file mode 100644 index 00000000000..ddb4e3f67a7 --- /dev/null +++ b/apps/sim/e2e/fixtures/factories/platform.ts @@ -0,0 +1,16 @@ +import { db } from '@sim/db' +import { settings, user } from '@sim/db/schema' +import { eq } from 'drizzle-orm' + +export async function arrangeEffectivePlatformAdmin(userId: string): Promise { + await db.transaction(async (tx) => { + await tx.update(user).set({ role: 'admin', updatedAt: new Date() }).where(eq(user.id, userId)) + await tx + .insert(settings) + .values({ id: userId, userId, superUserModeEnabled: true }) + .onConflictDoUpdate({ + target: settings.userId, + set: { superUserModeEnabled: true }, + }) + }) +} diff --git a/apps/sim/e2e/fixtures/factories/users.ts b/apps/sim/e2e/fixtures/factories/users.ts new file mode 100644 index 00000000000..e1635e49e9a --- /dev/null +++ b/apps/sim/e2e/fixtures/factories/users.ts @@ -0,0 +1,49 @@ +import { z } from 'zod' +import { E2eHttpClient } from '../http-client' + +const authUserSchema = z.object({ + id: z.string().min(1), + email: z.string().email(), + name: z.string(), +}) +const authResponseSchema = z + .object({ + user: authUserSchema, + }) + .passthrough() + +export interface SyntheticLogin { + name: string + email: string + password: string +} + +export async function createSyntheticUser( + client: E2eHttpClient, + login: SyntheticLogin +): Promise> { + const response = await client.request({ + method: 'POST', + path: '/api/auth/sign-up/email', + body: login, + schema: authResponseSchema, + expectedStatus: 200, + }) + return response.user +} + +export async function createAuthenticatedClient( + baseUrl: string, + login: SyntheticLogin, + onAttempt?: ConstructorParameters[0]['onAttempt'] +): Promise { + const client = new E2eHttpClient({ baseUrl, onAttempt }) + await client.request({ + method: 'POST', + path: '/api/auth/sign-in/email', + body: { email: login.email, password: login.password }, + schema: authResponseSchema, + expectedStatus: 200, + }) + return client +} diff --git a/apps/sim/e2e/fixtures/factories/workspaces.ts b/apps/sim/e2e/fixtures/factories/workspaces.ts new file mode 100644 index 00000000000..c078fae2bbb --- /dev/null +++ b/apps/sim/e2e/fixtures/factories/workspaces.ts @@ -0,0 +1,49 @@ +import { z } from 'zod' +import type { E2eHttpClient } from '../http-client' + +const workspaceSchema = z + .object({ + id: z.string(), + name: z.string(), + ownerId: z.string(), + organizationId: z.string().nullable(), + billedAccountUserId: z.string(), + workspaceMode: z.enum(['personal', 'organization', 'grandfathered_shared']), + }) + .passthrough() +const permissionMutationSchema = z.object({ + id: z.string(), + workspaceId: z.string(), + userId: z.string(), + permissions: z.enum(['admin', 'write', 'read']), + action: z.enum(['created', 'updated', 'already_member']), +}) + +export async function createWorkspace( + ownerClient: E2eHttpClient, + input: { name: string; color?: string } +): Promise> { + const response = await ownerClient.request({ + method: 'POST', + path: '/api/workspaces', + body: { ...input, skipDefaultWorkflow: true }, + schema: z.object({ workspace: workspaceSchema }), + expectedStatus: 200, + }) + return response.workspace +} + +export async function grantWorkspacePermission( + adminClient: E2eHttpClient, + workspaceId: string, + input: { userId: string; permissions: 'admin' | 'write' | 'read' } +): Promise> { + const response = await adminClient.request({ + method: 'POST', + path: `/api/v1/admin/workspaces/${workspaceId}/members`, + body: input, + schema: z.object({ data: permissionMutationSchema }), + expectedStatus: 200, + }) + return response.data +} diff --git a/apps/sim/e2e/fixtures/http-client.ts b/apps/sim/e2e/fixtures/http-client.ts new file mode 100644 index 00000000000..163e8e1e5ee --- /dev/null +++ b/apps/sim/e2e/fixtures/http-client.ts @@ -0,0 +1,144 @@ +import { sleep } from '@sim/utils/helpers' +import type { z } from 'zod' + +// Better Auth does not emit Retry-After for every limiter. The bounded fallback spans its +// one-minute signup window while still preferring an explicit server header when available. +const DEFAULT_RETRY_DELAYS_MS = [1_000, 2_000, 4_000, 8_000, 16_000, 30_000] as const + +export interface E2eHttpClientOptions { + baseUrl: string + defaultHeaders?: Record + fetchImplementation?: typeof fetch + sleepImplementation?: (milliseconds: number) => Promise + retryDelaysMs?: readonly number[] + onAttempt?: (attempt: { method: string; path: string; number: number; status?: number }) => void +} + +export class E2eHttpClient { + private readonly baseUrl: string + private readonly defaultHeaders: Record + private readonly fetchImplementation: typeof fetch + private readonly sleepImplementation: (milliseconds: number) => Promise + private readonly retryDelaysMs: readonly number[] + private readonly onAttempt?: E2eHttpClientOptions['onAttempt'] + private readonly cookies = new Map() + + constructor(options: E2eHttpClientOptions) { + this.baseUrl = options.baseUrl.replace(/\/$/, '') + this.defaultHeaders = options.defaultHeaders ?? {} + this.fetchImplementation = options.fetchImplementation ?? fetch + this.sleepImplementation = options.sleepImplementation ?? sleep + this.retryDelaysMs = options.retryDelaysMs ?? DEFAULT_RETRY_DELAYS_MS + this.onAttempt = options.onAttempt + } + + async request(options: { + method?: string + path: string + body?: unknown + schema: TSchema + expectedStatus?: number | readonly number[] + }): Promise> { + const method = options.method ?? 'GET' + const expectedStatuses = Array.isArray(options.expectedStatus) + ? options.expectedStatus + : [options.expectedStatus ?? 200] + + for (let attempt = 1; ; attempt += 1) { + const response = await this.fetchImplementation(`${this.baseUrl}${options.path}`, { + method, + headers: { + accept: 'application/json', + ...(options.body === undefined ? {} : { 'content-type': 'application/json' }), + ...this.defaultHeaders, + ...(this.cookies.size > 0 ? { cookie: this.serializeCookies() } : {}), + }, + body: options.body === undefined ? undefined : JSON.stringify(options.body), + redirect: 'manual', + }) + this.captureCookies(response.headers) + this.onAttempt?.({ method, path: options.path, number: attempt, status: response.status }) + + if (response.status === 429 && attempt <= this.retryDelaysMs.length) { + const delay = + parseRetryAfter(response.headers.get('retry-after')) ?? this.retryDelaysMs[attempt - 1] + await this.sleepImplementation(delay) + continue + } + + const payload = await readJsonResponse(response, method, options.path) + if (!expectedStatuses.includes(response.status)) { + throw new Error( + `${method} ${options.path} returned ${response.status}; expected ${expectedStatuses.join( + '/' + )}; response body redacted` + ) + } + const parsed = options.schema.safeParse(payload) + if (!parsed.success) { + throw new Error( + `${method} ${options.path} returned an invalid response: ${parsed.error.issues + .map((issue) => `${issue.path.join('.') || ''}: ${issue.message}`) + .join('; ')}` + ) + } + return parsed.data + } + } + + getCookieHeader(): string { + return this.serializeCookies() + } + + clearCookies(): void { + this.cookies.clear() + } + + private captureCookies(headers: Headers): void { + const values = + 'getSetCookie' in headers && typeof headers.getSetCookie === 'function' + ? headers.getSetCookie() + : splitCombinedSetCookie(headers.get('set-cookie')) + for (const value of values) { + const pair = value.split(';', 1)[0] + const separator = pair.indexOf('=') + if (separator <= 0) continue + const name = pair.slice(0, separator).trim() + const cookieValue = pair.slice(separator + 1).trim() + if (cookieValue) this.cookies.set(name, cookieValue) + else this.cookies.delete(name) + } + } + + private serializeCookies(): string { + return [...this.cookies.entries()].map(([name, value]) => `${name}=${value}`).join('; ') + } +} + +function parseRetryAfter(value: string | null): number | null { + if (!value) return null + const seconds = Number(value) + if (Number.isFinite(seconds) && seconds >= 0) return Math.ceil(seconds * 1_000) + const date = Date.parse(value) + if (!Number.isNaN(date)) return Math.max(0, date - Date.now()) + return null +} + +function splitCombinedSetCookie(value: string | null): string[] { + if (!value) return [] + return value.split(/,(?=\s*[^;,=\s]+=[^;,]*)/) +} + +async function readJsonResponse( + response: Response, + method: string, + path: string +): Promise { + const text = await response.text() + if (!text) return null + try { + return JSON.parse(text) + } catch { + throw new Error(`${method} ${path} returned non-JSON content with status ${response.status}`) + } +} diff --git a/apps/sim/e2e/fixtures/namespace.ts b/apps/sim/e2e/fixtures/namespace.ts new file mode 100644 index 00000000000..4d47df866fa --- /dev/null +++ b/apps/sim/e2e/fixtures/namespace.ts @@ -0,0 +1,86 @@ +import { createHash } from 'node:crypto' +import type { ScenarioNamespaceDescriptor } from './scenario' + +const EMAIL_DOMAIN = 'example.com' + +export interface ScenarioNamespace extends ScenarioNamespaceDescriptor { + email(label: string): string + slug(label: string): string + name(label: string): string + invitationToken(label: string): string + storageStateFilename(personaKey: string): string +} + +/** + * Namespaces only values the fixture controls. Production IDs are deliberately absent from this API. + */ +export function createScenarioNamespace(run: string, world: string): ScenarioNamespace { + const normalizedRun = normalizePart(run, 'run') + const normalizedWorld = normalizePart(world, 'world') + const digest = hash(`${run}\0${world}`).slice(0, 8) + const prefix = `e2e-${normalizedRun.slice(0, 10)}-${normalizedWorld.slice(0, 10)}-${digest}` + + return Object.freeze({ + run, + world, + prefix, + email(label: string): string { + return `${prefix}-${shortLabel(label, 11)}-${hash(label).slice(0, 8)}@${EMAIL_DOMAIN}` + }, + slug(label: string): string { + return `${prefix}-${shortLabel(label, 18)}-${hash(label).slice(0, 8)}` + }, + name(label: string): string { + return `E Two E ${humanize(world)} ${humanize(label)} ${alphabeticHash( + `${prefix}\0${label}` + )}` + }, + invitationToken(label: string): string { + const normalizedLabel = normalizeLabel(label).slice(0, 24) + return `${prefix}.invite.${normalizedLabel}.${hash(`${prefix}\0${label}`).slice(0, 12)}` + }, + storageStateFilename(personaKey: string): string { + return `${prefix}-${shortLabel(personaKey, 20)}-${hash(personaKey).slice(0, 8)}.json` + }, + }) +} + +function normalizePart(value: string, label: string): string { + const normalized = normalizeLabel(value) + if (!normalized) throw new Error(`Scenario namespace ${label} must contain a letter or number`) + return normalized +} + +function normalizeLabel(value: string): string { + const normalized = value + .normalize('NFKD') + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, '') + if (!normalized) throw new Error('Namespaced value label must contain a letter or number') + return normalized +} + +function humanize(value: string): string { + const normalized = normalizeLabel(value) + return normalized + .split('-') + .map((part) => `${part[0].toUpperCase()}${part.slice(1)}`) + .join(' ') +} + +function shortLabel(value: string, length: number): string { + return normalizeLabel(value).slice(0, length).replace(/-+$/, '') +} + +function hash(value: string): string { + return createHash('sha256').update(value).digest('hex') +} + +function alphabeticHash(value: string): string { + return hash(value) + .slice(0, 8) + .replace(/[0-9a-f]/g, (character) => + String.fromCharCode('a'.charCodeAt(0) + Number.parseInt(character, 16)) + ) +} diff --git a/apps/sim/e2e/fixtures/scenario.ts b/apps/sim/e2e/fixtures/scenario.ts new file mode 100644 index 00000000000..e68dd20c74c --- /dev/null +++ b/apps/sim/e2e/fixtures/scenario.ts @@ -0,0 +1,191 @@ +export const SCENARIO_VERSION = 1 as const + +export type ScenarioVersion = typeof SCENARIO_VERSION +export type ResourceKey = string +export type WorkspaceAccess = 'read' | 'write' | 'admin' +export type ExpectedWorkspaceAccess = WorkspaceAccess | 'none' +export type RoleSource = 'owner' | 'explicit' | 'org-admin' | 'none' +export type OrganizationRole = 'owner' | 'admin' | 'member' +export type SubscriptionPlan = + | 'pro_6000' + | 'pro_25000' + | 'team_6000' + | 'team_25000' + | 'enterprise' +export type SubscriptionStatus = 'active' | 'past_due' | 'lapsed' +export type SettingsSection = 'general' | 'secrets' | 'api-keys' | 'inbox' | 'mcp' | 'custom-tools' + +export interface ScenarioNamespaceDescriptor { + run: string + world: string + prefix: string +} + +export interface ScenarioDeployment { + hosted: boolean + billingEnabled: boolean +} + +export interface ScenarioUser { + key: ResourceKey + email: string + name: string + hosted: boolean + billingEnabled: boolean + platformRole?: 'user' | 'admin' + superUserModeEnabled?: boolean +} + +export interface ScenarioOrganization { + key: ResourceKey + name: string + slug: string + ownerUserKey: ResourceKey + hosted: boolean + billingEnabled: boolean +} + +export interface ScenarioOrganizationMembership { + organizationKey: ResourceKey + userKey: ResourceKey + role: OrganizationRole +} + +export interface EnterpriseSubscriptionMetadata { + plan: 'enterprise' + referenceId: string + monthlyPrice: number + seats: number +} + +export interface ScenarioSubscription { + key: ResourceKey + plan: SubscriptionPlan + status: SubscriptionStatus + billingReference: + | { kind: 'user'; userKey: ResourceKey } + | { kind: 'organization'; organizationKey: ResourceKey } + hosted: boolean + billingEnabled: boolean + seats?: number + enterprise?: EnterpriseSubscriptionMetadata +} + +export interface ScenarioWorkspace { + key: ResourceKey + name: string + ownerUserKey: ResourceKey + organizationKey?: ResourceKey + payer: + | { kind: 'user'; userKey: ResourceKey } + | { kind: 'organization'; organizationKey: ResourceKey } + subscriptionKey?: ResourceKey + hosted: boolean + billingEnabled: boolean +} + +export interface ScenarioWorkspaceGrant { + workspaceKey: ResourceKey + userKey: ResourceKey + access: WorkspaceAccess +} + +export interface PermissionGroupRestrictions { + hiddenSettings: readonly SettingsSection[] + disabledFeatures: readonly ('mcp' | 'custom-tools')[] +} + +export interface ScenarioPermissionGroup { + key: ResourceKey + name: string + organizationKey: ResourceKey + workspaceKeys: readonly ResourceKey[] + memberUserKeys: readonly ResourceKey[] + isDefault?: boolean + restrictions: PermissionGroupRestrictions +} + +export interface ScenarioInvitation { + key: ResourceKey + organizationKey: ResourceKey + email: string + token: string + role: Exclude + expiresAt: string + workspaceGrants: readonly { + workspaceKey: ResourceKey + access: WorkspaceAccess + }[] +} + +export interface PersonaWorkspaceExpectation { + workspaceKey: ResourceKey + access: ExpectedWorkspaceAccess + roleSource: RoleSource + hostContext: { + isOwner: boolean + hostMembership: 'owner' | 'member' | 'external' + payerScope: 'user' | 'organization' + plan: 'free' | SubscriptionPlan + hosted: boolean + billingEnabled: boolean + } +} + +export interface ScenarioPersona { + key: ResourceKey + userKey: ResourceKey + storageStateFilename: string + workspaces: readonly PersonaWorkspaceExpectation[] + permissionGroupKeys: readonly ResourceKey[] + canonicalRoute: { + workspaceKey: ResourceKey + settingsSection: SettingsSection + } + expectedPlatformRole: 'user' | 'admin' + expectedSuperUserMode: boolean +} + +/** + * Pure declaration consumed by later factories. Keys are graph-local references, never production IDs. + * Factories must record API-generated IDs separately and keep them opaque. + */ +export interface ScenarioDefinition { + version: ScenarioVersion + namespace: ScenarioNamespaceDescriptor + deployment: ScenarioDeployment + users: readonly ScenarioUser[] + organizations: readonly ScenarioOrganization[] + organizationMemberships: readonly ScenarioOrganizationMembership[] + subscriptions: readonly ScenarioSubscription[] + workspaces: readonly ScenarioWorkspace[] + workspaceGrants: readonly ScenarioWorkspaceGrant[] + permissionGroups: readonly ScenarioPermissionGroup[] + invitations: readonly ScenarioInvitation[] + personas: readonly ScenarioPersona[] +} + +export interface ResolvedPersona { + definition: ScenarioPersona + user: ScenarioUser + workspaces: readonly { + expectation: PersonaWorkspaceExpectation + workspace: ScenarioWorkspace + }[] + permissionGroups: readonly ScenarioPermissionGroup[] +} + +export interface ResolvedScenario { + definition: ScenarioDefinition + usersByKey: ReadonlyMap + organizationsByKey: ReadonlyMap + subscriptionsByKey: ReadonlyMap + workspacesByKey: ReadonlyMap + permissionGroupsByKey: ReadonlyMap + invitationsByKey: ReadonlyMap + personasByKey: ReadonlyMap +} + +export function canonicalSettingsRoute(workspaceId: string, section: SettingsSection): string { + return `/workspace/${encodeURIComponent(workspaceId)}/settings/${section}` +} diff --git a/apps/sim/e2e/fixtures/validate-scenario.ts b/apps/sim/e2e/fixtures/validate-scenario.ts new file mode 100644 index 00000000000..d6b768b794d --- /dev/null +++ b/apps/sim/e2e/fixtures/validate-scenario.ts @@ -0,0 +1,683 @@ +import type { + ExpectedWorkspaceAccess, + OrganizationRole, + ResolvedPersona, + ResolvedScenario, + ResourceKey, + ScenarioDefinition, + ScenarioOrganization, + ScenarioPermissionGroup, + ScenarioSubscription, + ScenarioUser, + ScenarioWorkspace, + ScenarioWorkspaceGrant, + WorkspaceAccess, +} from './scenario' +import { SCENARIO_VERSION } from './scenario' + +export class ScenarioValidationError extends Error { + readonly issues: readonly string[] + + constructor(issues: readonly string[]) { + super(`Invalid E2E scenario:\n- ${issues.join('\n- ')}`) + this.name = 'ScenarioValidationError' + this.issues = issues + } +} + +export function validateScenario(definition: ScenarioDefinition): ResolvedScenario { + const issues: string[] = [] + if (definition.version !== SCENARIO_VERSION) { + issues.push(`version must be ${SCENARIO_VERSION}`) + } + if (!definition.namespace.run || !definition.namespace.world || !definition.namespace.prefix) { + issues.push('namespace run, world, and prefix must be non-empty') + } + + const usersByKey = indexByKey('user', definition.users, issues) + const organizationsByKey = indexByKey('organization', definition.organizations, issues) + const subscriptionsByKey = indexByKey('subscription', definition.subscriptions, issues) + const workspacesByKey = indexByKey('workspace', definition.workspaces, issues) + const permissionGroupsByKey = indexByKey('permission group', definition.permissionGroups, issues) + const invitationsByKey = indexByKey('invitation', definition.invitations, issues) + indexByKey('persona', definition.personas, issues) + + validateControllableIdentities(definition, issues) + validateDeploymentFlags(definition, issues) + validateMemberships(definition, usersByKey, organizationsByKey, issues) + validateOrganizations(definition, usersByKey, issues) + validateSubscriptions(definition, usersByKey, organizationsByKey, issues) + validateWorkspaces(definition, usersByKey, organizationsByKey, subscriptionsByKey, issues) + validateWorkspaceGrants(definition.workspaceGrants, usersByKey, workspacesByKey, issues) + validatePermissionGroups(definition, usersByKey, organizationsByKey, workspacesByKey, issues) + validateInvitations(definition, organizationsByKey, workspacesByKey, issues) + + const resolvedPersonas = new Map() + for (const persona of definition.personas) { + const user = usersByKey.get(persona.userKey) + if (!user) { + issues.push(`persona "${persona.key}" references missing user "${persona.userKey}"`) + continue + } + if (persona.workspaces.filter(({ access }) => access !== 'none').length === 0) { + issues.push(`persona "${persona.key}" has zero accessible seeded workspaces`) + } + validateUniqueValues( + `persona "${persona.key}" workspace expectation`, + persona.workspaces.map(({ workspaceKey }) => workspaceKey), + issues + ) + validateUniqueValues( + `persona "${persona.key}" permission group`, + persona.permissionGroupKeys, + issues + ) + + const resolvedWorkspaces: ResolvedPersona['workspaces'][number][] = [] + for (const expectation of persona.workspaces) { + const workspace = workspacesByKey.get(expectation.workspaceKey) + if (!workspace) { + issues.push( + `persona "${persona.key}" references missing workspace "${expectation.workspaceKey}"` + ) + continue + } + validatePersonaWorkspaceExpectation( + definition, + persona.key, + user, + workspace, + expectation, + subscriptionsByKey, + issues + ) + resolvedWorkspaces.push({ expectation, workspace }) + } + + const routeExpectation = persona.workspaces.find( + ({ workspaceKey }) => workspaceKey === persona.canonicalRoute.workspaceKey + ) + if (!routeExpectation || routeExpectation.access === 'none') { + issues.push( + `persona "${persona.key}" canonical route must target an accessible declared workspace` + ) + } + + const expectedGroups = groupsForUser(definition, user.key) + if ( + !sameSet( + persona.permissionGroupKeys, + expectedGroups.map(({ key }) => key) + ) + ) { + issues.push(`persona "${persona.key}" permission-group expectations do not match its grants`) + } + const resolvedGroups = persona.permissionGroupKeys + .map((key) => permissionGroupsByKey.get(key)) + .filter((group): group is ScenarioPermissionGroup => group !== undefined) + + const platformRole = user.platformRole ?? 'user' + if (persona.expectedPlatformRole !== platformRole) { + issues.push(`persona "${persona.key}" has an inconsistent expected platform role`) + } + if (persona.expectedSuperUserMode !== (user.superUserModeEnabled ?? false)) { + issues.push(`persona "${persona.key}" has an inconsistent expected superuser setting`) + } + if (platformRole === 'admin' && !user.superUserModeEnabled) { + issues.push(`platform admin user "${user.key}" must enable superuser mode`) + } + if (user.superUserModeEnabled && platformRole !== 'admin') { + issues.push(`superuser user "${user.key}" must also have the platform admin role`) + } + + resolvedPersonas.set(persona.key, { + definition: persona, + user, + workspaces: resolvedWorkspaces, + permissionGroups: resolvedGroups, + }) + } + + if (issues.length > 0) throw new ScenarioValidationError(issues) + + return { + definition, + usersByKey, + organizationsByKey, + subscriptionsByKey, + workspacesByKey, + permissionGroupsByKey, + invitationsByKey, + personasByKey: resolvedPersonas, + } +} + +function indexByKey( + label: string, + values: readonly T[], + issues: string[] +): Map { + const result = new Map() + for (const value of values) { + if (!value.key.trim()) { + issues.push(`${label} key must be non-empty`) + } else if (result.has(value.key)) { + issues.push(`duplicate ${label} key "${value.key}"`) + } else { + result.set(value.key, value) + } + } + return result +} + +function validateControllableIdentities(definition: ScenarioDefinition, issues: string[]): void { + validateUniqueValues( + 'email', + [ + ...definition.users.map(({ email }) => email.toLowerCase()), + ...definition.invitations.map(({ email }) => email.toLowerCase()), + ], + issues + ) + validateUniqueValues( + 'user name', + definition.users.map(({ name }) => name), + issues + ) + validateUniqueValues( + 'organization name', + definition.organizations.map(({ name }) => name), + issues + ) + validateUniqueValues( + 'organization slug', + definition.organizations.map(({ slug }) => slug.toLowerCase()), + issues + ) + validateUniqueValues( + 'workspace name', + definition.workspaces.map(({ name }) => name), + issues + ) + validateUniqueValues( + 'permission-group name', + definition.permissionGroups.map( + ({ organizationKey, name }) => `${organizationKey}\0${name.toLowerCase()}` + ), + issues + ) + validateUniqueValues( + 'invitation token', + definition.invitations.map(({ token }) => token), + issues + ) + validateUniqueValues( + 'storage-state filename', + definition.personas.map(({ storageStateFilename }) => storageStateFilename), + issues + ) +} + +function validateUniqueValues(label: string, values: readonly string[], issues: string[]): void { + const seen = new Set() + for (const value of values) { + if (!value.trim()) { + issues.push(`${label} must be non-empty`) + } else if (seen.has(value)) { + issues.push(`duplicate ${label} "${value.replace('\0', '/')}"`) + } else { + seen.add(value) + } + } +} + +function validateDeploymentFlags(definition: ScenarioDefinition, issues: string[]): void { + const resources = [ + ...definition.users.map((resource) => ({ kind: 'user', resource })), + ...definition.organizations.map((resource) => ({ kind: 'organization', resource })), + ...definition.subscriptions.map((resource) => ({ kind: 'subscription', resource })), + ...definition.workspaces.map((resource) => ({ kind: 'workspace', resource })), + ] + for (const { kind, resource } of resources) { + if (resource.hosted !== definition.deployment.hosted) { + issues.push(`${kind} "${resource.key}" has an inconsistent hosted flag`) + } + if (resource.billingEnabled !== definition.deployment.billingEnabled) { + issues.push(`${kind} "${resource.key}" has an inconsistent billing flag`) + } + } + if (!definition.deployment.billingEnabled && definition.subscriptions.length > 0) { + issues.push('billing-disabled scenarios cannot contain subscriptions') + } +} + +function validateMemberships( + definition: ScenarioDefinition, + usersByKey: ReadonlyMap, + organizationsByKey: ReadonlyMap, + issues: string[] +): void { + const pairs = new Set() + const organizationByUser = new Map() + for (const membership of definition.organizationMemberships) { + const pair = `${membership.organizationKey}\0${membership.userKey}` + if (pairs.has(pair)) { + issues.push( + `duplicate organization membership "${membership.organizationKey}/${membership.userKey}"` + ) + } + pairs.add(pair) + if (!organizationsByKey.has(membership.organizationKey)) { + issues.push(`membership references missing organization "${membership.organizationKey}"`) + } + if (!usersByKey.has(membership.userKey)) { + issues.push(`membership references missing user "${membership.userKey}"`) + } + const previousOrganization = organizationByUser.get(membership.userKey) + if (previousOrganization && previousOrganization !== membership.organizationKey) { + issues.push(`user "${membership.userKey}" cannot belong to more than one organization`) + } + organizationByUser.set(membership.userKey, membership.organizationKey) + } +} + +function validateOrganizations( + definition: ScenarioDefinition, + usersByKey: ReadonlyMap, + issues: string[] +): void { + for (const organization of definition.organizations) { + if (!usersByKey.has(organization.ownerUserKey)) { + issues.push( + `organization "${organization.key}" references missing owner "${organization.ownerUserKey}"` + ) + } + const ownerMemberships = definition.organizationMemberships.filter( + (membership) => + membership.organizationKey === organization.key && + membership.userKey === organization.ownerUserKey && + membership.role === 'owner' + ) + if (ownerMemberships.length !== 1) { + issues.push(`organization "${organization.key}" must have exactly one owner membership`) + } + const otherOwners = definition.organizationMemberships.filter( + (membership) => + membership.organizationKey === organization.key && + membership.role === 'owner' && + membership.userKey !== organization.ownerUserKey + ) + if (otherOwners.length > 0) { + issues.push(`organization "${organization.key}" cannot have a second owner membership`) + } + } +} + +function validateSubscriptions( + definition: ScenarioDefinition, + usersByKey: ReadonlyMap, + organizationsByKey: ReadonlyMap, + issues: string[] +): void { + const entitledBillingReferences = new Set() + for (const subscription of definition.subscriptions) { + const reference = billingReferenceKey(subscription) + if (subscription.billingReference.kind === 'user') { + if (!usersByKey.has(subscription.billingReference.userKey)) { + issues.push( + `subscription "${subscription.key}" references missing user "${subscription.billingReference.userKey}"` + ) + } + if (subscription.plan.startsWith('team_') || subscription.plan === 'enterprise') { + issues.push(`subscription "${subscription.key}" has an invalid plan for a user payer`) + } + } else { + const organizationKey = subscription.billingReference.organizationKey + if (!organizationsByKey.has(organizationKey)) { + issues.push( + `subscription "${subscription.key}" references missing organization "${organizationKey}"` + ) + } + if (subscription.plan === 'pro_6000' || subscription.plan === 'pro_25000') { + issues.push( + `subscription "${subscription.key}" has an invalid plan for an organization payer` + ) + } + const memberCount = definition.organizationMemberships.filter( + (membership) => membership.organizationKey === organizationKey + ).length + if (!subscription.seats || subscription.seats < memberCount) { + issues.push(`subscription "${subscription.key}" seats do not cover organization members`) + } + } + + if (subscription.status === 'active' || subscription.status === 'past_due') { + if (entitledBillingReferences.has(reference)) { + issues.push(`billing reference "${reference}" has duplicate entitled subscriptions`) + } + entitledBillingReferences.add(reference) + } + + if (subscription.plan === 'enterprise') { + const metadata = subscription.enterprise + if ( + !metadata || + metadata.plan !== 'enterprise' || + !metadata.referenceId || + !Number.isFinite(metadata.monthlyPrice) || + metadata.monthlyPrice <= 0 || + !Number.isInteger(metadata.seats) || + metadata.seats < 1 || + metadata.seats !== subscription.seats + ) { + issues.push(`subscription "${subscription.key}" has invalid Enterprise metadata`) + } + } else if (subscription.enterprise) { + issues.push(`non-Enterprise subscription "${subscription.key}" has Enterprise metadata`) + } + } +} + +function validateWorkspaces( + definition: ScenarioDefinition, + usersByKey: ReadonlyMap, + organizationsByKey: ReadonlyMap, + subscriptionsByKey: ReadonlyMap, + issues: string[] +): void { + for (const workspace of definition.workspaces) { + if (!usersByKey.has(workspace.ownerUserKey)) { + issues.push( + `workspace "${workspace.key}" references missing owner "${workspace.ownerUserKey}"` + ) + } + const subscription = workspace.subscriptionKey + ? subscriptionsByKey.get(workspace.subscriptionKey) + : undefined + if (workspace.subscriptionKey && !subscription) { + issues.push( + `workspace "${workspace.key}" references missing subscription "${workspace.subscriptionKey}"` + ) + } + + if (workspace.organizationKey) { + if (!organizationsByKey.has(workspace.organizationKey)) { + issues.push( + `workspace "${workspace.key}" references missing organization "${workspace.organizationKey}"` + ) + } + if ( + workspace.payer.kind !== 'organization' || + workspace.payer.organizationKey !== workspace.organizationKey + ) { + issues.push(`organization workspace "${workspace.key}" has an incoherent payer`) + } + const ownerMembership = membershipFor( + definition, + workspace.organizationKey, + workspace.ownerUserKey + ) + if ( + !ownerMembership || + (ownerMembership.role !== 'owner' && ownerMembership.role !== 'admin') + ) { + issues.push(`organization workspace "${workspace.key}" owner must be an owner/admin member`) + } + if (!subscription) { + issues.push(`organization workspace "${workspace.key}" must retain its subscription record`) + } else if ( + subscription.billingReference.kind !== 'organization' || + subscription.billingReference.organizationKey !== workspace.organizationKey + ) { + issues.push(`organization workspace "${workspace.key}" has an incoherent subscription`) + } + } else { + if (workspace.payer.kind !== 'user' || workspace.payer.userKey !== workspace.ownerUserKey) { + issues.push(`personal workspace "${workspace.key}" has an incoherent payer/owner`) + } + if ( + subscription && + (subscription.billingReference.kind !== 'user' || + subscription.billingReference.userKey !== workspace.ownerUserKey) + ) { + issues.push(`personal workspace "${workspace.key}" has an incoherent subscription`) + } + } + } +} + +function validateWorkspaceGrants( + grants: readonly ScenarioWorkspaceGrant[], + usersByKey: ReadonlyMap, + workspacesByKey: ReadonlyMap, + issues: string[] +): void { + const pairs = new Set() + for (const grant of grants) { + const pair = `${grant.workspaceKey}\0${grant.userKey}` + if (pairs.has(pair)) { + issues.push(`duplicate workspace grant "${grant.workspaceKey}/${grant.userKey}"`) + } + pairs.add(pair) + if (!workspacesByKey.has(grant.workspaceKey)) { + issues.push(`workspace grant references missing workspace "${grant.workspaceKey}"`) + } + if (!usersByKey.has(grant.userKey)) { + issues.push(`workspace grant references missing user "${grant.userKey}"`) + } + } +} + +function validatePermissionGroups( + definition: ScenarioDefinition, + usersByKey: ReadonlyMap, + organizationsByKey: ReadonlyMap, + workspacesByKey: ReadonlyMap, + issues: string[] +): void { + const defaultOrganizations = new Set() + for (const group of definition.permissionGroups) { + if (!organizationsByKey.has(group.organizationKey)) { + issues.push( + `permission group "${group.key}" references missing organization "${group.organizationKey}"` + ) + } + if (group.workspaceKeys.length === 0) { + issues.push(`permission group "${group.key}" must have Enterprise workspace scope`) + } + validateUniqueValues( + `permission group "${group.key}" workspace scope`, + group.workspaceKeys, + issues + ) + validateUniqueValues(`permission group "${group.key}" member`, group.memberUserKeys, issues) + + const enterpriseSubscription = definition.subscriptions.find( + (subscription) => + subscription.billingReference.kind === 'organization' && + subscription.billingReference.organizationKey === group.organizationKey && + subscription.plan === 'enterprise' && + subscription.status === 'active' + ) + if (!enterpriseSubscription) { + issues.push(`permission group "${group.key}" requires an active Enterprise organization`) + } + for (const workspaceKey of group.workspaceKeys) { + const workspace = workspacesByKey.get(workspaceKey) + if (!workspace) { + issues.push( + `permission group "${group.key}" references missing workspace "${workspaceKey}"` + ) + } else if ( + workspace.organizationKey !== group.organizationKey || + workspace.subscriptionKey !== enterpriseSubscription?.key + ) { + issues.push( + `permission group "${group.key}" workspace "${workspaceKey}" is outside its Enterprise scope` + ) + } + } + for (const userKey of group.memberUserKeys) { + if (!usersByKey.has(userKey)) { + issues.push(`permission group "${group.key}" references missing user "${userKey}"`) + } + if (!membershipFor(definition, group.organizationKey, userKey)) { + issues.push(`permission group "${group.key}" member "${userKey}" lacks host membership`) + } + } + if (group.isDefault) { + if (defaultOrganizations.has(group.organizationKey)) { + issues.push(`organization "${group.organizationKey}" has duplicate default groups`) + } + defaultOrganizations.add(group.organizationKey) + } + } +} + +function validateInvitations( + definition: ScenarioDefinition, + organizationsByKey: ReadonlyMap, + workspacesByKey: ReadonlyMap, + issues: string[] +): void { + for (const invitation of definition.invitations) { + if (!organizationsByKey.has(invitation.organizationKey)) { + issues.push( + `invitation "${invitation.key}" references missing organization "${invitation.organizationKey}"` + ) + } + if (Number.isNaN(Date.parse(invitation.expiresAt))) { + issues.push(`invitation "${invitation.key}" has an invalid expiry`) + } + validateUniqueValues( + `invitation "${invitation.key}" workspace grant`, + invitation.workspaceGrants.map(({ workspaceKey }) => workspaceKey), + issues + ) + for (const grant of invitation.workspaceGrants) { + const workspace = workspacesByKey.get(grant.workspaceKey) + if (!workspace) { + issues.push( + `invitation "${invitation.key}" references missing workspace "${grant.workspaceKey}"` + ) + } else if (workspace.organizationKey !== invitation.organizationKey) { + issues.push(`invitation "${invitation.key}" grants a workspace outside its organization`) + } + } + } +} + +function validatePersonaWorkspaceExpectation( + definition: ScenarioDefinition, + personaKey: ResourceKey, + user: ScenarioUser, + workspace: ScenarioWorkspace, + expected: { + access: ExpectedWorkspaceAccess + roleSource: 'owner' | 'explicit' | 'org-admin' | 'none' + hostContext: { + isOwner: boolean + hostMembership: 'owner' | 'member' | 'external' + payerScope: 'user' | 'organization' + plan: 'free' | ScenarioSubscription['plan'] + hosted: boolean + billingEnabled: boolean + } + }, + subscriptionsByKey: ReadonlyMap, + issues: string[] +): void { + const actual = deriveWorkspaceAccess(definition, user.key, workspace) + if (expected.access !== actual.access || expected.roleSource !== actual.roleSource) { + issues.push( + `persona "${personaKey}" has incoherent access/roleSource for workspace "${workspace.key}"` + ) + } + if ( + (actual.isOwner || + actual.organizationRole === 'owner' || + actual.organizationRole === 'admin') && + expected.access !== 'admin' + ) { + issues.push( + `persona "${personaKey}" owner/admin expectation for "${workspace.key}" is below admin` + ) + } + const subscription = workspace.subscriptionKey + ? subscriptionsByKey.get(workspace.subscriptionKey) + : undefined + const actualPlan = !subscription || subscription.status === 'lapsed' ? 'free' : subscription.plan + const actualMembership = actual.isOwner + ? 'owner' + : actual.organizationRole + ? 'member' + : 'external' + if ( + expected.hostContext.isOwner !== actual.isOwner || + expected.hostContext.hostMembership !== actualMembership || + expected.hostContext.payerScope !== workspace.payer.kind || + expected.hostContext.plan !== actualPlan || + expected.hostContext.hosted !== workspace.hosted || + expected.hostContext.billingEnabled !== workspace.billingEnabled + ) { + issues.push(`persona "${personaKey}" has an incoherent host context for "${workspace.key}"`) + } +} + +function deriveWorkspaceAccess( + definition: ScenarioDefinition, + userKey: ResourceKey, + workspace: ScenarioWorkspace +): { + access: ExpectedWorkspaceAccess + roleSource: 'owner' | 'explicit' | 'org-admin' | 'none' + isOwner: boolean + organizationRole?: OrganizationRole +} { + const isOwner = workspace.ownerUserKey === userKey + const organizationRole = workspace.organizationKey + ? membershipFor(definition, workspace.organizationKey, userKey)?.role + : undefined + if (isOwner) return { access: 'admin', roleSource: 'owner', isOwner, organizationRole } + if (organizationRole === 'owner' || organizationRole === 'admin') { + return { access: 'admin', roleSource: 'org-admin', isOwner, organizationRole } + } + const grant = definition.workspaceGrants.find( + (candidate) => candidate.workspaceKey === workspace.key && candidate.userKey === userKey + ) + if (grant) { + return { access: grant.access, roleSource: 'explicit', isOwner, organizationRole } + } + return { access: 'none', roleSource: 'none', isOwner, organizationRole } +} + +function groupsForUser( + definition: ScenarioDefinition, + userKey: ResourceKey +): ScenarioPermissionGroup[] { + return definition.permissionGroups.filter((group) => group.memberUserKeys.includes(userKey)) +} + +function membershipFor( + definition: ScenarioDefinition, + organizationKey: ResourceKey, + userKey: ResourceKey +): ScenarioDefinition['organizationMemberships'][number] | undefined { + return definition.organizationMemberships.find( + (membership) => membership.organizationKey === organizationKey && membership.userKey === userKey + ) +} + +function billingReferenceKey(subscription: ScenarioSubscription): string { + return subscription.billingReference.kind === 'user' + ? `user/${subscription.billingReference.userKey}` + : `organization/${subscription.billingReference.organizationKey}` +} + +function sameSet(left: readonly string[], right: readonly string[]): boolean { + return left.length === right.length && left.every((value) => right.includes(value)) +} + +export function accessRank(access: WorkspaceAccess): number { + return { read: 1, write: 2, admin: 3 }[access] +} diff --git a/apps/sim/e2e/foundation/http-client.spec.ts b/apps/sim/e2e/foundation/http-client.spec.ts new file mode 100644 index 00000000000..4dd36a01b4c --- /dev/null +++ b/apps/sim/e2e/foundation/http-client.spec.ts @@ -0,0 +1,67 @@ +import { expect, test } from '@playwright/test' +import { z } from 'zod' +import { E2eHttpClient } from '../fixtures/http-client' + +test.describe('E2E HTTP client', () => { + test('honors Retry-After only for 429 responses and records attempts', async () => { + const delays: number[] = [] + const attempts: number[] = [] + let requestCount = 0 + const client = new E2eHttpClient({ + baseUrl: 'http://127.0.0.1:1', + fetchImplementation: async () => { + requestCount += 1 + return requestCount === 1 + ? new Response(JSON.stringify({ error: 'limited' }), { + status: 429, + headers: { 'content-type': 'application/json', 'retry-after': '0.01' }, + }) + : Response.json({ ok: true }) + }, + sleepImplementation: async (milliseconds) => { + delays.push(milliseconds) + }, + onAttempt: ({ number }) => attempts.push(number), + }) + + await expect( + client.request({ + path: '/retry', + schema: z.object({ ok: z.literal(true) }), + }) + ).resolves.toEqual({ ok: true }) + expect(delays).toEqual([10]) + expect(attempts).toEqual([1, 2]) + }) + + test('keeps independent cookie jars and redacts failed response bodies', async () => { + const cookieHeaders: Array = [] + const createFetch = + (secret: string): typeof fetch => + async (_input, init) => { + cookieHeaders.push(new Headers(init?.headers).get('cookie')) + if (cookieHeaders.length <= 2) { + return new Response(JSON.stringify({ ok: true }), { + headers: { 'set-cookie': `session=${secret}; Path=/; HttpOnly` }, + }) + } + return new Response(JSON.stringify({ error: `do not print ${secret}` }), { status: 400 }) + } + const fetchImplementation = createFetch('synthetic-secret') + const first = new E2eHttpClient({ baseUrl: 'http://127.0.0.1:1', fetchImplementation }) + const second = new E2eHttpClient({ baseUrl: 'http://127.0.0.1:1', fetchImplementation }) + const schema = z.object({ ok: z.literal(true) }) + + await first.request({ path: '/login-a', schema }) + await second.request({ path: '/login-b', schema }) + expect(first.getCookieHeader()).toBe('session=synthetic-secret') + expect(second.getCookieHeader()).toBe('session=synthetic-secret') + + await expect(first.request({ path: '/failure', schema })).rejects.not.toThrow( + /synthetic-secret/ + ) + expect(cookieHeaders[0]).toBeNull() + expect(cookieHeaders[1]).toBeNull() + expect(cookieHeaders[2]).toBe('session=synthetic-secret') + }) +}) diff --git a/apps/sim/e2e/foundation/scenario-validation.spec.ts b/apps/sim/e2e/foundation/scenario-validation.spec.ts new file mode 100644 index 00000000000..a6c366cd403 --- /dev/null +++ b/apps/sim/e2e/foundation/scenario-validation.spec.ts @@ -0,0 +1,291 @@ +import { expect, test } from '@playwright/test' +import { createScenarioNamespace } from '../fixtures/namespace' +import { + canonicalSettingsRoute, + type ScenarioDefinition, + type ScenarioOrganizationMembership, + type ScenarioPermissionGroup, + type ScenarioWorkspaceGrant, +} from '../fixtures/scenario' +import { ScenarioValidationError, validateScenario } from '../fixtures/validate-scenario' +import { + createPrimarySettingsScenario, + createSettingsPersonaScenarios, + SETTINGS_PERSONA_KEYS, +} from '../settings/personas' + +test.describe('pure scenario validation', () => { + test('resolves the literal settings cast and minimal isolation twin', () => { + const scenarios = createSettingsPersonaScenarios('run-42') + const primary = validateScenario(scenarios.primary) + const twin = validateScenario(scenarios.isolationTwin) + + expect([...primary.personasByKey.keys()]).toEqual(SETTINGS_PERSONA_KEYS) + expect(twin.personasByKey.size).toBe(1) + expect(twin.workspacesByKey.size).toBe(1) + expect(twin.organizationsByKey.size).toBe(0) + + expect(workspaceExpectation(primary, 'personalPaidOwner').hostContext.plan).toBe('pro_6000') + expect(workspaceExpectation(primary, 'personalMaxOwner').hostContext.plan).toBe('pro_25000') + expect(workspaceExpectation(primary, 'workspaceReadMember')).toMatchObject({ + access: 'read', + roleSource: 'explicit', + hostContext: { hostMembership: 'member', plan: 'team_6000' }, + }) + expect(workspaceExpectation(primary, 'workspaceWriteMember').access).toBe('write') + expect(workspaceExpectation(primary, 'workspaceAdminMember')).toMatchObject({ + access: 'admin', + roleSource: 'explicit', + hostContext: { hostMembership: 'member' }, + }) + expect(workspaceExpectation(primary, 'externalWorkspaceAdmin')).toMatchObject({ + access: 'admin', + roleSource: 'explicit', + hostContext: { hostMembership: 'external' }, + }) + expect( + primary.definition.organizationMemberships.some( + ({ userKey }) => userKey === 'external-workspace-admin' + ) + ).toBe(false) + + expect(workspaceExpectation(primary, 'enterpriseOrganizationAdmin')).toMatchObject({ + access: 'admin', + roleSource: 'org-admin', + hostContext: { isOwner: false, plan: 'enterprise' }, + }) + expect( + primary.definition.workspaceGrants.some( + ({ userKey }) => userKey === 'enterprise-organization-admin' + ) + ).toBe(false) + expect(workspaceExpectation(primary, 'freeOrganizationOwner').hostContext.plan).toBe('free') + expect(primary.subscriptionsByKey.get('lapsed-team-subscription')?.status).toBe('lapsed') + + const restricted = primary.personasByKey.get('permissionGroupRestricted') + expect(restricted?.permissionGroups[0]?.restrictions).toEqual({ + hiddenSettings: ['secrets', 'api-keys', 'inbox'], + disabledFeatures: ['mcp', 'custom-tools'], + }) + expect(primary.personasByKey.get('platformAdmin')?.user).toMatchObject({ + platformRole: 'admin', + superUserModeEnabled: true, + }) + for (const persona of primary.personasByKey.values()) { + expect(persona.workspaces.some(({ expectation }) => expectation.access !== 'none')).toBe(true) + expect(persona.definition.canonicalRoute.settingsSection).toBe('general') + } + expect(workspaceExpectation(primary, 'personalFreeOwner', 'team-workspace')).toMatchObject({ + access: 'none', + roleSource: 'none', + }) + }) + + test('namespaces controllable values deterministically without manufacturing API IDs', () => { + const first = createScenarioNamespace('Run 42/Alpha', 'Primary World') + const again = createScenarioNamespace('Run 42/Alpha', 'Primary World') + const otherWorld = createScenarioNamespace('Run 42/Alpha', 'Twin World') + + expect(first).toMatchObject({ + run: 'Run 42/Alpha', + world: 'Primary World', + }) + expect(first.email('Owner')).toBe(again.email('Owner')) + expect(first.slug('Organization')).toBe(again.slug('Organization')) + expect(first.name('Workspace')).toBe(again.name('Workspace')) + expect(first.invitationToken('Invite')).toBe(again.invitationToken('Invite')) + expect(first.storageStateFilename('personalFreeOwner')).toBe( + again.storageStateFilename('personalFreeOwner') + ) + expect(first.email('Owner')).not.toBe(otherWorld.email('Owner')) + expect(first.storageStateFilename('personalFreeOwner')).toMatch(/\.json$/) + expect(Object.keys(first).some((key) => /(^|api)id$/i.test(key))).toBe(false) + expect(canonicalSettingsRoute('opaque/id:generated-by-api', 'api-keys')).toBe( + '/workspace/opaque%2Fid%3Agenerated-by-api/settings/api-keys' + ) + }) + + test('rejects duplicate organization memberships', () => { + const scenario = validScenario() + scenario.organizationMemberships = [ + ...scenario.organizationMemberships, + scenario.organizationMemberships[0], + ] as ScenarioOrganizationMembership[] + expectInvalid(scenario, /duplicate organization membership/) + }) + + test('rejects incoherent organization workspace payer and subscription identity', () => { + const scenario = validScenario() + scenario.workspaces = scenario.workspaces.map((workspace) => + workspace.key === 'team-workspace' + ? { + ...workspace, + payer: { kind: 'organization', organizationKey: 'enterprise-organization' }, + } + : workspace + ) + expectInvalid(scenario, /incoherent payer/) + }) + + test('rejects owner or organization-admin expectations below admin', () => { + const scenario = validScenario() + scenario.personas = scenario.personas.map((persona) => + persona.key === 'enterpriseOrganizationAdmin' + ? { + ...persona, + workspaces: persona.workspaces.map((expectation) => ({ + ...expectation, + access: 'read', + })), + } + : persona + ) + expectInvalid(scenario, /below admin/) + }) + + test('rejects invalid and misplaced Enterprise metadata', () => { + const invalidEnterprise = validScenario() + invalidEnterprise.subscriptions = invalidEnterprise.subscriptions.map((subscription) => + subscription.key === 'enterprise-subscription' + ? { ...subscription, enterprise: { ...subscription.enterprise!, monthlyPrice: 0 } } + : subscription + ) + expectInvalid(invalidEnterprise, /invalid Enterprise metadata/) + + const metadataOnTeam = validScenario() + metadataOnTeam.subscriptions = metadataOnTeam.subscriptions.map((subscription) => + subscription.key === 'team-subscription' + ? { + ...subscription, + enterprise: { + plan: 'enterprise', + referenceId: 'wrong-plan', + monthlyPrice: 1, + seats: 4, + }, + } + : subscription + ) + expectInvalid(metadataOnTeam, /non-Enterprise subscription/) + }) + + test('rejects permission groups without active Enterprise organization/workspace scope', () => { + const scenario = validScenario() + scenario.permissionGroups = scenario.permissionGroups.map((group) => ({ + ...group, + organizationKey: 'team-organization', + workspaceKeys: ['team-workspace'], + })) + expectInvalid(scenario, /requires an active Enterprise organization/) + }) + + test('rejects duplicate explicit grants and default groups', () => { + const duplicateGrant = validScenario() + duplicateGrant.workspaceGrants = [ + ...duplicateGrant.workspaceGrants, + duplicateGrant.workspaceGrants[0], + ] as ScenarioWorkspaceGrant[] + expectInvalid(duplicateGrant, /duplicate workspace grant/) + + const duplicateDefault = validScenario() + const existing = duplicateDefault.permissionGroups[0] + duplicateDefault.permissionGroups = [ + { ...existing, isDefault: true }, + { + ...existing, + key: 'second-default-enterprise-group', + name: `${existing.name} Second`, + isDefault: true, + }, + ] as ScenarioPermissionGroup[] + expectInvalid(duplicateDefault, /duplicate default groups/) + }) + + test('rejects inconsistent hosted and billing flags', () => { + const hosted = validScenario() + hosted.workspaces = hosted.workspaces.map((workspace, index) => + index === 0 ? { ...workspace, hosted: false } : workspace + ) + expectInvalid(hosted, /inconsistent hosted flag/) + + const billing = validScenario() + billing.users = billing.users.map((user, index) => + index === 0 ? { ...user, billingEnabled: false } : user + ) + expectInvalid(billing, /inconsistent billing flag/) + }) + + test('rejects zero-workspace personas and none-only foreign denial declarations', () => { + const empty = validScenario() + empty.personas = empty.personas.map((persona) => + persona.key === 'personalPaidOwner' ? { ...persona, workspaces: [] } : persona + ) + expectInvalid(empty, /zero accessible seeded workspaces/) + + const noneOnly = validScenario() + noneOnly.personas = noneOnly.personas.map((persona) => + persona.key === 'personalFreeOwner' + ? { + ...persona, + workspaces: persona.workspaces.filter(({ access }) => access === 'none'), + canonicalRoute: { workspaceKey: 'team-workspace', settingsSection: 'general' }, + } + : persona + ) + expectInvalid(noneOnly, /zero accessible seeded workspaces/) + }) + + test('rejects invalid references and duplicate controllable identities', () => { + const badReference = validScenario() + badReference.workspaceGrants = badReference.workspaceGrants.map((grant, index) => + index === 0 ? { ...grant, userKey: 'missing-user' } : grant + ) + expectInvalid(badReference, /references missing user/) + + const duplicateEmail = validScenario() + duplicateEmail.users = duplicateEmail.users.map((user, index, users) => + index === 1 ? { ...user, email: users[0].email.toUpperCase() } : user + ) + expectInvalid(duplicateEmail, /duplicate email/) + + const duplicateStorageState = validScenario() + duplicateStorageState.personas = duplicateStorageState.personas.map( + (persona, index, personas) => + index === 1 + ? { ...persona, storageStateFilename: personas[0].storageStateFilename } + : persona + ) + expectInvalid(duplicateStorageState, /duplicate storage-state filename/) + }) +}) + +function validScenario(): MutableScenario { + return structuredClone( + createPrimarySettingsScenario(createScenarioNamespace('validation-run', 'primary')) + ) as MutableScenario +} + +type MutableScenario = { + -readonly [Key in keyof ScenarioDefinition]: ScenarioDefinition[Key] extends readonly (infer Item)[] + ? Item[] + : ScenarioDefinition[Key] +} + +function expectInvalid(scenario: ScenarioDefinition, message: RegExp): void { + expect(() => validateScenario(scenario)).toThrow(ScenarioValidationError) + expect(() => validateScenario(scenario)).toThrow(message) +} + +function workspaceExpectation( + scenario: ReturnType, + personaKey: string, + workspaceKey?: string +) { + const persona = scenario.personasByKey.get(personaKey) + if (!persona) throw new Error(`Missing persona "${personaKey}"`) + const expectation = workspaceKey + ? persona.definition.workspaces.find((candidate) => candidate.workspaceKey === workspaceKey) + : persona.definition.workspaces.find((candidate) => candidate.access !== 'none') + if (!expectation) throw new Error(`Missing workspace expectation for "${personaKey}"`) + return expectation +} diff --git a/apps/sim/e2e/settings/personas.ts b/apps/sim/e2e/settings/personas.ts new file mode 100644 index 00000000000..f7783a89bda --- /dev/null +++ b/apps/sim/e2e/settings/personas.ts @@ -0,0 +1,442 @@ +import { createScenarioNamespace, type ScenarioNamespace } from '../fixtures/namespace' +import { + type ExpectedWorkspaceAccess, + type PersonaWorkspaceExpectation, + type RoleSource, + SCENARIO_VERSION, + type ScenarioDefinition, + type ScenarioPersona, + type ScenarioUser, + type SubscriptionPlan, +} from '../fixtures/scenario' + +const HOSTED_BILLING = { hosted: true, billingEnabled: true } as const + +export const SETTINGS_PERSONA_KEYS = [ + 'personalFreeOwner', + 'personalPaidOwner', + 'personalMaxOwner', + 'paidOrganizationOwner', + 'workspaceReadMember', + 'workspaceWriteMember', + 'workspaceAdminMember', + 'externalWorkspaceAdmin', + 'enterpriseOrganizationAdmin', + 'freeOrganizationOwner', + 'permissionGroupRestricted', + 'platformAdmin', +] as const + +export type SettingsPersonaKey = (typeof SETTINGS_PERSONA_KEYS)[number] + +export interface SettingsPersonaScenarios { + primary: ScenarioDefinition + isolationTwin: ScenarioDefinition +} + +export function createSettingsPersonaScenarios(run: string): SettingsPersonaScenarios { + return { + primary: createPrimarySettingsScenario(createScenarioNamespace(run, 'settings-primary')), + isolationTwin: createIsolationTwinScenario( + createScenarioNamespace(run, 'settings-isolation-twin') + ), + } +} + +export function createPrimarySettingsScenario(namespace: ScenarioNamespace): ScenarioDefinition { + const userKeys = [ + 'personal-free-owner', + 'personal-paid-owner', + 'personal-max-owner', + 'paid-organization-owner', + 'workspace-read-member', + 'workspace-write-member', + 'workspace-admin-member', + 'external-workspace-admin', + 'enterprise-organization-owner', + 'enterprise-organization-admin', + 'free-organization-owner', + 'permission-group-restricted', + 'platform-admin', + ] as const + const users: ScenarioUser[] = userKeys.map((key) => user(namespace, key)) + const platformAdmin = users.find(({ key }) => key === 'platform-admin') + if (!platformAdmin) throw new Error('Platform admin declaration is missing') + platformAdmin.platformRole = 'admin' + platformAdmin.superUserModeEnabled = true + + const organizations = [ + { + key: 'team-organization', + name: namespace.name('team-organization'), + slug: namespace.slug('team-organization'), + ownerUserKey: 'paid-organization-owner', + ...HOSTED_BILLING, + }, + { + key: 'enterprise-organization', + name: namespace.name('enterprise-organization'), + slug: namespace.slug('enterprise-organization'), + ownerUserKey: 'enterprise-organization-owner', + ...HOSTED_BILLING, + }, + { + key: 'lapsed-organization', + name: namespace.name('lapsed-organization'), + slug: namespace.slug('lapsed-organization'), + ownerUserKey: 'free-organization-owner', + ...HOSTED_BILLING, + }, + ] as const + + const organizationMemberships = [ + membership('team-organization', 'paid-organization-owner', 'owner'), + membership('team-organization', 'workspace-read-member', 'member'), + membership('team-organization', 'workspace-write-member', 'member'), + membership('team-organization', 'workspace-admin-member', 'member'), + membership('enterprise-organization', 'enterprise-organization-owner', 'owner'), + membership('enterprise-organization', 'enterprise-organization-admin', 'admin'), + membership('enterprise-organization', 'permission-group-restricted', 'member'), + membership('lapsed-organization', 'free-organization-owner', 'owner'), + ] as const + + const subscriptions = [ + { + key: 'personal-paid-subscription', + plan: 'pro_6000', + status: 'active', + billingReference: { kind: 'user', userKey: 'personal-paid-owner' }, + ...HOSTED_BILLING, + }, + { + key: 'personal-max-subscription', + plan: 'pro_25000', + status: 'active', + billingReference: { kind: 'user', userKey: 'personal-max-owner' }, + ...HOSTED_BILLING, + }, + { + key: 'team-subscription', + plan: 'team_6000', + status: 'active', + billingReference: { kind: 'organization', organizationKey: 'team-organization' }, + seats: 4, + ...HOSTED_BILLING, + }, + { + key: 'enterprise-subscription', + plan: 'enterprise', + status: 'active', + billingReference: { + kind: 'organization', + organizationKey: 'enterprise-organization', + }, + seats: 3, + enterprise: { + plan: 'enterprise', + referenceId: namespace.slug('enterprise-contract'), + monthlyPrice: 12_000, + seats: 3, + }, + ...HOSTED_BILLING, + }, + { + key: 'lapsed-team-subscription', + plan: 'team_6000', + status: 'lapsed', + billingReference: { kind: 'organization', organizationKey: 'lapsed-organization' }, + seats: 1, + ...HOSTED_BILLING, + }, + ] as const + + const workspaces = [ + personalWorkspace(namespace, 'personal-free-workspace', 'personal-free-owner'), + personalWorkspace( + namespace, + 'personal-paid-workspace', + 'personal-paid-owner', + 'personal-paid-subscription' + ), + personalWorkspace( + namespace, + 'personal-max-workspace', + 'personal-max-owner', + 'personal-max-subscription' + ), + { + key: 'team-workspace', + name: namespace.name('team-workspace'), + ownerUserKey: 'paid-organization-owner', + organizationKey: 'team-organization', + payer: { kind: 'organization', organizationKey: 'team-organization' }, + subscriptionKey: 'team-subscription', + ...HOSTED_BILLING, + }, + { + key: 'enterprise-workspace', + name: namespace.name('enterprise-workspace'), + ownerUserKey: 'enterprise-organization-owner', + organizationKey: 'enterprise-organization', + payer: { kind: 'organization', organizationKey: 'enterprise-organization' }, + subscriptionKey: 'enterprise-subscription', + ...HOSTED_BILLING, + }, + { + key: 'lapsed-organization-workspace', + name: namespace.name('lapsed-organization-workspace'), + ownerUserKey: 'free-organization-owner', + organizationKey: 'lapsed-organization', + payer: { kind: 'organization', organizationKey: 'lapsed-organization' }, + subscriptionKey: 'lapsed-team-subscription', + ...HOSTED_BILLING, + }, + personalWorkspace(namespace, 'platform-admin-workspace', 'platform-admin'), + ] as const + + const workspaceGrants = [ + grant('team-workspace', 'workspace-read-member', 'read'), + grant('team-workspace', 'workspace-write-member', 'write'), + grant('team-workspace', 'workspace-admin-member', 'admin'), + grant('team-workspace', 'external-workspace-admin', 'admin'), + grant('enterprise-workspace', 'permission-group-restricted', 'read'), + ] as const + + const permissionGroups = [ + { + key: 'restricted-enterprise-group', + name: namespace.name('restricted-enterprise-group'), + organizationKey: 'enterprise-organization', + workspaceKeys: ['enterprise-workspace'], + memberUserKeys: ['permission-group-restricted'], + restrictions: { + hiddenSettings: ['secrets', 'api-keys', 'inbox'], + disabledFeatures: ['mcp', 'custom-tools'], + }, + }, + ] as const + + const invitations = [ + { + key: 'pending-team-invitation', + organizationKey: 'team-organization', + email: namespace.email('pending-team-invitee'), + token: namespace.invitationToken('pending-team-invitation'), + role: 'member', + expiresAt: '2099-01-01T00:00:00.000Z', + workspaceGrants: [{ workspaceKey: 'team-workspace', access: 'read' }], + }, + ] as const + + const personas: ScenarioPersona[] = [ + persona(namespace, 'personalFreeOwner', 'personal-free-owner', [ + expected('personal-free-workspace', 'admin', 'owner', 'owner', 'user', 'free', true), + expected('team-workspace', 'none', 'none', 'external', 'organization', 'team_6000', false), + ]), + persona(namespace, 'personalPaidOwner', 'personal-paid-owner', [ + expected('personal-paid-workspace', 'admin', 'owner', 'owner', 'user', 'pro_6000', true), + ]), + persona(namespace, 'personalMaxOwner', 'personal-max-owner', [ + expected('personal-max-workspace', 'admin', 'owner', 'owner', 'user', 'pro_25000', true), + ]), + persona(namespace, 'paidOrganizationOwner', 'paid-organization-owner', [ + expected('team-workspace', 'admin', 'owner', 'owner', 'organization', 'team_6000', true), + ]), + persona(namespace, 'workspaceReadMember', 'workspace-read-member', [ + expected('team-workspace', 'read', 'explicit', 'member', 'organization', 'team_6000', false), + ]), + persona(namespace, 'workspaceWriteMember', 'workspace-write-member', [ + expected('team-workspace', 'write', 'explicit', 'member', 'organization', 'team_6000', false), + ]), + persona(namespace, 'workspaceAdminMember', 'workspace-admin-member', [ + expected('team-workspace', 'admin', 'explicit', 'member', 'organization', 'team_6000', false), + ]), + persona(namespace, 'externalWorkspaceAdmin', 'external-workspace-admin', [ + expected( + 'team-workspace', + 'admin', + 'explicit', + 'external', + 'organization', + 'team_6000', + false + ), + ]), + persona(namespace, 'enterpriseOrganizationAdmin', 'enterprise-organization-admin', [ + expected( + 'enterprise-workspace', + 'admin', + 'org-admin', + 'member', + 'organization', + 'enterprise', + false + ), + ]), + persona(namespace, 'freeOrganizationOwner', 'free-organization-owner', [ + expected( + 'lapsed-organization-workspace', + 'admin', + 'owner', + 'owner', + 'organization', + 'free', + true + ), + ]), + persona( + namespace, + 'permissionGroupRestricted', + 'permission-group-restricted', + [ + expected( + 'enterprise-workspace', + 'read', + 'explicit', + 'member', + 'organization', + 'enterprise', + false + ), + ], + ['restricted-enterprise-group'] + ), + persona( + namespace, + 'platformAdmin', + 'platform-admin', + [expected('platform-admin-workspace', 'admin', 'owner', 'owner', 'user', 'free', true)], + [], + 'admin', + true + ), + ] + + return { + version: SCENARIO_VERSION, + namespace: namespaceDescriptor(namespace), + deployment: HOSTED_BILLING, + users, + organizations, + organizationMemberships, + subscriptions, + workspaces, + workspaceGrants, + permissionGroups, + invitations, + personas, + } +} + +export function createIsolationTwinScenario(namespace: ScenarioNamespace): ScenarioDefinition { + return { + version: SCENARIO_VERSION, + namespace: namespaceDescriptor(namespace), + deployment: HOSTED_BILLING, + users: [user(namespace, 'isolation-twin-owner')], + organizations: [], + organizationMemberships: [], + subscriptions: [], + workspaces: [personalWorkspace(namespace, 'isolation-twin-workspace', 'isolation-twin-owner')], + workspaceGrants: [], + permissionGroups: [], + invitations: [], + personas: [ + persona(namespace, 'isolationTwinOwner', 'isolation-twin-owner', [ + expected('isolation-twin-workspace', 'admin', 'owner', 'owner', 'user', 'free', true), + ]), + ], + } +} + +function user(namespace: ScenarioNamespace, key: string): ScenarioUser { + return { + key, + email: namespace.email(key), + name: namespace.name(key), + platformRole: 'user', + superUserModeEnabled: false, + ...HOSTED_BILLING, + } +} + +function namespaceDescriptor(namespace: ScenarioNamespace) { + return { + run: namespace.run, + world: namespace.world, + prefix: namespace.prefix, + } +} + +function membership(organizationKey: string, userKey: string, role: 'owner' | 'admin' | 'member') { + return { organizationKey, userKey, role } as const +} + +function grant(workspaceKey: string, userKey: string, access: 'read' | 'write' | 'admin') { + return { workspaceKey, userKey, access } as const +} + +function personalWorkspace( + namespace: ScenarioNamespace, + key: string, + ownerUserKey: string, + subscriptionKey?: string +) { + return { + key, + name: namespace.name(key), + ownerUserKey, + payer: { kind: 'user', userKey: ownerUserKey }, + ...(subscriptionKey ? { subscriptionKey } : {}), + ...HOSTED_BILLING, + } as const +} + +function persona( + namespace: ScenarioNamespace, + key: string, + userKey: string, + workspaces: readonly PersonaWorkspaceExpectation[], + permissionGroupKeys: readonly string[] = [], + expectedPlatformRole: 'user' | 'admin' = 'user', + expectedSuperUserMode = false +): ScenarioPersona { + const canonicalWorkspace = workspaces.find(({ access }) => access !== 'none') + if (!canonicalWorkspace) throw new Error(`Persona "${key}" needs an accessible workspace`) + return { + key, + userKey, + storageStateFilename: namespace.storageStateFilename(key), + workspaces, + permissionGroupKeys, + canonicalRoute: { + workspaceKey: canonicalWorkspace.workspaceKey, + settingsSection: 'general', + }, + expectedPlatformRole, + expectedSuperUserMode, + } +} + +function expected( + workspaceKey: string, + access: ExpectedWorkspaceAccess, + roleSource: RoleSource, + hostMembership: 'owner' | 'member' | 'external', + payerScope: 'user' | 'organization', + plan: 'free' | SubscriptionPlan, + isOwner: boolean +): PersonaWorkspaceExpectation { + return { + workspaceKey, + access, + roleSource, + hostContext: { + isOwner, + hostMembership, + payerScope, + plan, + ...HOSTED_BILLING, + }, + } +} From f7d10c7c509ebc1c04afeb55ab318af6c90d2a31 Mon Sep 17 00:00:00 2001 From: Bill Leoutsakos Date: Mon, 20 Jul 2026 21:57:34 -0700 Subject: [PATCH 3/6] Seed and verify E2E settings personas Provision the validated world through real product boundaries, capture isolated sessions through the login UI, and assert persona contracts plus two-worker cross-world isolation without exposing credentials to Playwright. Co-authored-by: Cursor --- apps/sim/e2e/README.md | 23 +- apps/sim/e2e/fakes/stripe/server.ts | 7 + apps/sim/e2e/fixtures/persona-test.ts | 56 ++ apps/sim/e2e/foundation/safety.spec.ts | 10 +- apps/sim/e2e/foundation/stripe-fake.spec.ts | 13 + apps/sim/e2e/scripts/capture-auth-states.ts | 128 +++++ apps/sim/e2e/scripts/options.ts | 14 +- apps/sim/e2e/scripts/run.ts | 39 +- apps/sim/e2e/scripts/seed-world.ts | 505 ++++++++++++++++++ .../e2e/settings/persona-contracts.spec.ts | 97 ++++ .../e2e/settings/persona-isolation.spec.ts | 38 ++ apps/sim/e2e/support/deployment-profile.ts | 3 + apps/sim/e2e/support/stack.ts | 22 + apps/sim/playwright.config.ts | 14 +- 14 files changed, 953 insertions(+), 16 deletions(-) create mode 100644 apps/sim/e2e/fixtures/persona-test.ts create mode 100644 apps/sim/e2e/scripts/capture-auth-states.ts create mode 100644 apps/sim/e2e/scripts/seed-world.ts create mode 100644 apps/sim/e2e/settings/persona-contracts.spec.ts create mode 100644 apps/sim/e2e/settings/persona-isolation.spec.ts diff --git a/apps/sim/e2e/README.md b/apps/sim/e2e/README.md index 76d0f32b640..5f708125ba3 100644 --- a/apps/sim/e2e/README.md +++ b/apps/sim/e2e/README.md @@ -50,8 +50,10 @@ E2E_PG_ADMIN_URL=postgresql://postgres:postgres@127.0.0.1:5432/postgres \ ``` The runner creates a unique `sim_e2e_` database, migrates it, starts the -Stripe fake and realtime, builds and starts Next.js, runs Playwright on Node 22, -then stops services and drops only that guarded database. +Stripe fake and realtime, builds and starts Next.js, seeds the validated persona +world through production APIs plus narrow trusted arrangements, captures each +persona through the real login UI, runs Playwright on Node 22, then stops +services and drops only that guarded database. On interruption, the runner launches a detached cleanup supervisor before exiting. It terminates managed process groups, force-drops the guarded database, @@ -89,13 +91,17 @@ were not launched by the orchestrator. Report and trace viewer commands remain safe because they do not execute tests. Sharding is supported only for the navigation project. The runner rejects -`--shard` for `hosted-billing-chromium-workflows`. +`--shard` for workflows, persona contracts, and the dedicated two-worker +cross-world isolation project. ## Diagnostics - HTML report: `playwright-report/` - Traces and screenshots: `test-results/` -- App, realtime, migration, and fake logs: `e2e/.runs//logs/` +- App, realtime, migration, seed, auth-capture, and fake logs: + `e2e/.runs//logs/` +- Non-secret persona manifest and post-login auth-capture screenshots: + `e2e/.runs//` Open the report: @@ -111,11 +117,14 @@ node ../../node_modules/@playwright/test/cli.js show-trace test-results//t The runner starts every child process from a fresh, purpose-specific environment. Next build receives deterministic sentinels instead of the run -database/fake endpoint; app, realtime, migrations, future seeding/auth capture, -and Playwright each receive only their required values. Next build/start shadow +database/fake endpoint; app, realtime, migrations, seeding, auth capture, and +Playwright each receive only their required values. Playwright receives the +non-secret manifest and storage-state directory, never passwords, the admin API +key, or the database URL. Next build/start shadow keys found in local `.env*` files, while children that cannot load those files omit denied keys entirely. Developer credentials are not used as test state or -written to reports. +written to reports. Synthetic persona passwords live in a private run-home file +and are removed with the auth/cloud-config directory during teardown. E2E builds verify the reviewed sandbox-bundle source/dependency/output fingerprint and never regenerate committed `.cjs` files. Regeneration remains an diff --git a/apps/sim/e2e/fakes/stripe/server.ts b/apps/sim/e2e/fakes/stripe/server.ts index 8f89c48102e..d87e465c780 100644 --- a/apps/sim/e2e/fakes/stripe/server.ts +++ b/apps/sim/e2e/fakes/stripe/server.ts @@ -6,6 +6,7 @@ export const STRIPE_FAKE_ENDPOINTS = { health: '/health', requestLog: '/__control/requests', reset: '/__control/reset', + telemetry: '/v1/traces', } as const const DEFAULT_MAX_BODY_BYTES = 64 * 1024 @@ -63,6 +64,7 @@ function isExpectedStripeRequest(method: string, path: string): boolean { ((method === 'GET' || method === 'POST') && path === '/v1/customers/search') || (method === 'GET' && path === '/v1/customers') || (method === 'POST' && path === '/v1/customers') || + (method === 'POST' && path === STRIPE_FAKE_ENDPOINTS.telemetry) || (method === 'GET' && /^\/v1\/customers\/cus_e2e_[a-f0-9]+$/.test(path)) ) } @@ -326,6 +328,11 @@ export function createStripeFakeServer(options: StripeFakeServerOptions): Stripe unexpected: !expected, }) + if (method === 'POST' && url.pathname === STRIPE_FAKE_ENDPOINTS.telemetry) { + sendJson(response, 200, { partialSuccess: {} }, requestId) + return + } + if (!secureEqual(request.headers.authorization, expectedAuthorization)) { sendStripeError( response, diff --git a/apps/sim/e2e/fixtures/persona-test.ts b/apps/sim/e2e/fixtures/persona-test.ts new file mode 100644 index 00000000000..4ba3df9f2cb --- /dev/null +++ b/apps/sim/e2e/fixtures/persona-test.ts @@ -0,0 +1,56 @@ +import path from 'node:path' +import { test as base } from '@playwright/test' +import { + type PersonaManifestEntry, + readScenarioManifest, + type ScenarioManifest, +} from './e2e-world' + +interface PersonaFixtures { + personaManifest: ScenarioManifest + contextForPersona: ( + personaKey: string + ) => Promise +} + +export const test = base.extend({ + personaManifest: [ + async ({}, use) => { + const manifest = readScenarioManifest(requiredEnv('E2E_MANIFEST_PATH')) + if (!manifest.authCaptureComplete) { + throw new Error('Persona storage states were not captured successfully') + } + await use(manifest) + }, + { scope: 'test' }, + ], + contextForPersona: async ({ browser, personaManifest }, use) => { + const contexts = new Set() + await use(async (personaKey) => { + const persona = requirePersona(personaManifest, personaKey) + const context = await browser.newContext({ + storageState: path.join(requiredEnv('E2E_STORAGE_STATE_DIR'), persona.storageStatePath), + }) + contexts.add(context) + return context + }) + await Promise.all([...contexts].map((context) => context.close())) + }, +}) + +export { expect } from '@playwright/test' + +export function requirePersona( + manifest: ScenarioManifest, + personaKey: string +): PersonaManifestEntry { + const persona = manifest.personas[personaKey] + if (!persona) throw new Error(`Unknown persona: ${personaKey}`) + return persona +} + +function requiredEnv(key: string): string { + const value = process.env[key] + if (!value) throw new Error(`Missing persona fixture environment value: ${key}`) + return value +} diff --git a/apps/sim/e2e/foundation/safety.spec.ts b/apps/sim/e2e/foundation/safety.spec.ts index fa7c8b0867e..5e0ad40e696 100644 --- a/apps/sim/e2e/foundation/safety.spec.ts +++ b/apps/sim/e2e/foundation/safety.spec.ts @@ -79,6 +79,8 @@ test.describe('foundation safety guards', () => { expect(build.env.DATABASE_URL).toContain('/sim_e2e_build_sentinel') expect(build.env.DATABASE_URL).not.toBe(app.env.DATABASE_URL) expect(build.env.STRIPE_API_BASE_URL).toBe('http://127.0.0.1:1') + expect(build.env.TELEMETRY_ENDPOINT).toBe('http://127.0.0.1:1/v1/traces') + expect(app.env.TELEMETRY_ENDPOINT).toMatch(/^http:\/\/127\.0\.0\.1:\d+\/v1\/traces$/) expect(build.env.E2E_RUN_ID).toBe('build_sentinel') for (const key of Object.keys(app.env).filter((key) => key.startsWith('NEXT_PUBLIC_'))) { @@ -134,7 +136,13 @@ test.describe('foundation safety guards', () => { ).not.toThrow() expect(() => parseRunOptions(['--project=hosted-billing-chromium-workflows', '--shard=1/2']) - ).toThrow(/coupled workflows must remain unsharded/) + ).toThrow(/coupled E2E projects must remain unsharded/) + expect(() => + parseRunOptions(['--project=hosted-billing-chromium-personas']) + ).not.toThrow() + expect(() => + parseRunOptions(['--project=hosted-billing-chromium-persona-isolation', '--shard=1/2']) + ).toThrow(/coupled E2E projects must remain unsharded/) }) test('Playwright CLI arguments cannot override orchestration invariants', () => { diff --git a/apps/sim/e2e/foundation/stripe-fake.spec.ts b/apps/sim/e2e/foundation/stripe-fake.spec.ts index 8eec71d6870..ff5e75342bf 100644 --- a/apps/sim/e2e/foundation/stripe-fake.spec.ts +++ b/apps/sim/e2e/foundation/stripe-fake.spec.ts @@ -36,6 +36,19 @@ test('Stripe fake records allowlisted calls and rejects unknown routes', async ( ) expect(search.status).toBe(200) + const telemetry = await fetch(`${baseUrl}/v1/traces`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ resourceSpans: [] }), + }) + expect(telemetry.status).toBe(200) + expect( + fake.requestLog.some( + ({ method, path, unexpected }) => + method === 'POST' && path === '/v1/traces' && !unexpected + ) + ).toBe(true) + const unsupportedSearch = await fetch( `${baseUrl}/v1/customers/search?${new URLSearchParams({ query: 'name:"Foundation"', diff --git a/apps/sim/e2e/scripts/capture-auth-states.ts b/apps/sim/e2e/scripts/capture-auth-states.ts new file mode 100644 index 00000000000..ef212a8b8d4 --- /dev/null +++ b/apps/sim/e2e/scripts/capture-auth-states.ts @@ -0,0 +1,128 @@ +import { mkdirSync } from 'node:fs' +import path from 'node:path' +import { chromium, expect } from '@playwright/test' +import { + readPersonaCredentials, + readScenarioManifest, + scenarioManifestSchema, + writeJsonAtomic, +} from '../fixtures/e2e-world' + +const baseUrl = requiredEnv('E2E_BASE_URL') +const manifestPath = requiredEnv('E2E_MANIFEST_PATH') +const credentialsPath = requiredEnv('E2E_CREDENTIALS_PATH') +const storageStateDirectory = requiredEnv('E2E_STORAGE_STATE_DIR') +const screenshotsDirectory = requiredEnv('E2E_AUTH_SCREENSHOT_DIR') +const UI_RETRY_DELAYS_MS = [1_000, 2_000, 4_000, 8_000, 16_000, 30_000] as const + +async function main(): Promise { + const manifest = readScenarioManifest(manifestPath) + const credentials = readPersonaCredentials(credentialsPath) + if (manifest.runId !== credentials.runId || manifest.runId !== requiredEnv('E2E_RUN_ID')) { + throw new Error('Persona auth inputs belong to different E2E runs') + } + mkdirSync(storageStateDirectory, { recursive: true }) + mkdirSync(screenshotsDirectory, { recursive: true }) + + const browser = await chromium.launch({ + args: ['--host-resolver-rules=MAP e2e.sim.ai 127.0.0.1'], + }) + try { + for (const [personaKey, persona] of Object.entries(manifest.personas)) { + const login = credentials.personas[personaKey] + if (!login) throw new Error(`Missing private login for persona: ${personaKey}`) + const context = await browser.newContext({ baseURL: baseUrl }) + try { + const page = await context.newPage() + await page.goto('/login') + await expect(page.getByRole('heading', { name: 'Sign in' })).toBeVisible() + await page.getByLabel('Email').fill(login.email) + await page.getByRole('textbox', { name: 'Password' }).fill(login.password) + await signInThroughUi(page, personaKey) + + const sessionResponse = await context.request.get('/api/auth/get-session') + if (!sessionResponse.ok()) { + throw new Error(`Session probe failed for ${personaKey}: ${sessionResponse.status()}`) + } + const session = (await sessionResponse.json()) as { + user?: { id?: string; email?: string } + session?: { activeOrganizationId?: string | null } + } + if (session.user?.id !== persona.userId || session.user.email !== persona.email) { + throw new Error(`Session identity mismatch for persona: ${personaKey}`) + } + if ( + (session.session?.activeOrganizationId ?? null) !== persona.expectedActiveOrganizationId + ) { + throw new Error(`Active organization mismatch for persona: ${personaKey}`) + } + + await page.goto(persona.canonicalRoute) + await expect(page).toHaveURL(/\/settings\/general$/) + await expect(page.getByRole('heading', { name: 'General', level: 1 })).toBeVisible() + await page.screenshot({ + path: path.join(screenshotsDirectory, `${personaKey}.png`), + fullPage: false, + }) + const storageStatePath = path.join(storageStateDirectory, persona.storageStatePath) + await context.storageState({ path: storageStatePath }) + } finally { + await context.close() + } + } + + manifest.authCaptureComplete = true + writeJsonAtomic(manifestPath, scenarioManifestSchema.parse(manifest)) + } finally { + await browser.close() + } +} + +function requiredEnv(key: string): string { + const value = process.env[key] + if (!value) throw new Error(`Missing auth capture environment value: ${key}`) + return value +} + +async function signInThroughUi( + page: import('@playwright/test').Page, + personaKey: string +): Promise { + for (let attempt = 0; attempt <= UI_RETRY_DELAYS_MS.length; attempt += 1) { + const responsePromise = page.waitForResponse((response) => { + const url = new URL(response.url()) + return url.pathname === '/api/auth/sign-in/email' + }) + await page.getByRole('button', { name: 'Sign in' }).click() + const response = await responsePromise + if (response.status() === 200) { + try { + await page.waitForURL(/\/workspace(?:\/|$)/, { timeout: 30_000 }) + } catch { + throw new Error(`UI sign-in did not redirect for persona: ${personaKey}`) + } + return + } + if (response.status() !== 429 || attempt === UI_RETRY_DELAYS_MS.length) { + throw new Error( + `UI sign-in failed for persona ${personaKey} with status ${response.status()}; response body redacted` + ) + } + await new Promise((resolve) => setTimeout(resolve, UI_RETRY_DELAYS_MS[attempt])) + } +} + +main().catch((error) => { + const raw = error instanceof Error ? (error.stack ?? error.message) : String(error) + let redacted = raw + try { + const credentials = readPersonaCredentials(credentialsPath) + for (const { password } of Object.values(credentials.personas)) { + redacted = redacted.replaceAll(password, '[REDACTED]') + } + } catch { + redacted = 'Auth capture failed; credentials unavailable for safe diagnostic formatting' + } + console.error(redacted) + process.exitCode = 1 +}) diff --git a/apps/sim/e2e/scripts/options.ts b/apps/sim/e2e/scripts/options.ts index e33050d29f2..e87fe6fb940 100644 --- a/apps/sim/e2e/scripts/options.ts +++ b/apps/sim/e2e/scripts/options.ts @@ -1,5 +1,13 @@ const NAVIGATION_PROJECT = 'hosted-billing-chromium-navigation' const WORKFLOWS_PROJECT = 'hosted-billing-chromium-workflows' +const PERSONAS_PROJECT = 'hosted-billing-chromium-personas' +const PERSONA_ISOLATION_PROJECT = 'hosted-billing-chromium-persona-isolation' +const E2E_PROJECTS = new Set([ + NAVIGATION_PROJECT, + WORKFLOWS_PROJECT, + PERSONAS_PROJECT, + PERSONA_ISOLATION_PROJECT, +]) const FORBIDDEN_OPTIONS = [ '--config', '-c', @@ -51,9 +59,7 @@ export function parseRunOptions( } const playwrightArgs = normalizedArgs.filter((arg) => arg !== '--reuse-build') const projects = getEqualsOptionValues(playwrightArgs, '--project') - const unknownProject = projects.find( - (project) => project !== NAVIGATION_PROJECT && project !== WORKFLOWS_PROJECT - ) + const unknownProject = projects.find((project) => !E2E_PROJECTS.has(project)) if (unknownProject) throw new Error(`Unknown E2E Playwright project: ${unknownProject}`) const hasShard = hasOption(playwrightArgs, '--shard') @@ -64,7 +70,7 @@ export function parseRunOptions( projects.some((project) => project !== NAVIGATION_PROJECT)) ) { throw new Error( - `--shard is supported only with --project=${NAVIGATION_PROJECT}; coupled workflows must remain unsharded` + `--shard is supported only with --project=${NAVIGATION_PROJECT}; coupled E2E projects must remain unsharded` ) } diff --git a/apps/sim/e2e/scripts/run.ts b/apps/sim/e2e/scripts/run.ts index b49f9f59374..8dfafc87f44 100644 --- a/apps/sim/e2e/scripts/run.ts +++ b/apps/sim/e2e/scripts/run.ts @@ -37,7 +37,15 @@ import { type ManagedProcess, stopAllManagedProcesses, } from '../support/process' -import { buildApp, runMigrations, runPlaywright, startApp, startRealtime } from '../support/stack' +import { + buildApp, + capturePersonaAuthStates, + runMigrations, + runPlaywright, + seedE2eWorld, + startApp, + startRealtime, +} from '../support/stack' import { parseRunOptions } from './options' const DEFAULT_ADMIN_DATABASE_URL = 'postgresql://postgres:postgres@127.0.0.1:5432/postgres' @@ -51,6 +59,9 @@ async function main(): Promise { const storageStateDirectory = path.join(runDirectory, 'auth') const markerDirectory = path.join(runDirectory, 'markers') const homeDirectory = path.join(runDirectory, 'home') + const manifestPath = path.join(runDirectory, 'persona-manifest.json') + const credentialsPath = path.join(homeDirectory, 'persona-credentials.json') + const authScreenshotsDirectory = path.join(runDirectory, 'auth-capture-screenshots') mkdirSync(logsDirectory, { recursive: true }) mkdirSync(storageStateDirectory, { recursive: true }) mkdirSync(markerDirectory, { recursive: true }) @@ -220,11 +231,31 @@ async function main(): Promise { ) assertNoForbiddenProviderInitialization([app.logPath, realtime.logPath]) + await seedE2eWorld({ + ...commandOptions, + env: { + ...profile.environments.seed.env, + E2E_MANIFEST_PATH: manifestPath, + E2E_CREDENTIALS_PATH: credentialsPath, + }, + }) + await capturePersonaAuthStates({ + ...commandOptions, + env: { + ...profile.environments.authCapture.env, + E2E_MANIFEST_PATH: manifestPath, + E2E_CREDENTIALS_PATH: credentialsPath, + E2E_STORAGE_STATE_DIR: storageStateDirectory, + E2E_AUTH_SCREENSHOT_DIR: authScreenshotsDirectory, + }, + }) + const playwrightEnvironment = createPlaywrightEnvironment( profile.environments.playwright.env, runId, storageStateDirectory, - markerDirectory + markerDirectory, + manifestPath ) await runPlaywright( { @@ -309,7 +340,8 @@ function createPlaywrightEnvironment( baseEnvironment: Record, runId: string, storageStateDirectory: string, - markerDirectory: string + markerDirectory: string, + manifestPath: string ): Record { return { ...baseEnvironment, @@ -319,6 +351,7 @@ function createPlaywrightEnvironment( E2E_BASE_URL: E2E_ORIGIN, E2E_STORAGE_STATE_DIR: storageStateDirectory, E2E_MARKER_DIR: markerDirectory, + E2E_MANIFEST_PATH: manifestPath, } } diff --git a/apps/sim/e2e/scripts/seed-world.ts b/apps/sim/e2e/scripts/seed-world.ts new file mode 100644 index 00000000000..9ab77afcd3d --- /dev/null +++ b/apps/sim/e2e/scripts/seed-world.ts @@ -0,0 +1,505 @@ +import { randomBytes } from 'node:crypto' +import { db } from '@sim/db' +import { member, permissions, subscription } from '@sim/db/schema' +import { and, eq, inArray } from 'drizzle-orm' +import { z } from 'zod' +import { + buildScenarioManifest, + createWorldRecords, + type E2EWorld, + PERSONA_MANIFEST_VERSION, + type PersonaCredentials, + writeJsonAtomic, +} from '../fixtures/e2e-world' +import { arrangeSubscription, lapseOrganizationSubscription } from '../fixtures/factories/billing' +import { arrangePendingInvitation } from '../fixtures/factories/invitations' +import { + addOrganizationMember, + createAdminClient, + createOrganization, +} from '../fixtures/factories/organizations' +import { + addPermissionGroupMember, + createPermissionGroup, +} from '../fixtures/factories/permission-groups' +import { arrangeEffectivePlatformAdmin } from '../fixtures/factories/platform' +import { + createAuthenticatedClient, + createSyntheticUser, + type SyntheticLogin, +} from '../fixtures/factories/users' +import { + createWorkspace, + grantWorkspacePermission, +} from '../fixtures/factories/workspaces' +import { E2eHttpClient } from '../fixtures/http-client' +import type { ResolvedScenario, ScenarioSubscription } from '../fixtures/scenario' +import { validateScenario } from '../fixtures/validate-scenario' +import { createSettingsPersonaScenarios } from '../settings/personas' + +const requiredEnvSchema = z.object({ + E2E_RUN_ID: z.string().min(1), + E2E_BASE_URL: z.string().url(), + ADMIN_API_KEY: z.string().min(1), + DATABASE_URL: z.string().url(), + E2E_MANIFEST_PATH: z.string().min(1), + E2E_CREDENTIALS_PATH: z.string().min(1), +}) + +async function main(): Promise { + const env = requiredEnvSchema.parse(process.env) + const definitions = createSettingsPersonaScenarios(env.E2E_RUN_ID) + const scenarios = [validateScenario(definitions.primary), validateScenario(definitions.isolationTwin)] + const adminClient = createAdminClient(env.E2E_BASE_URL, env.ADMIN_API_KEY) + const attemptCounts = new Map() + const worlds: E2EWorld[] = [] + const loginsByWorldAndUser = new Map() + const ownerClients = new Map() + + for (const scenario of scenarios) { + const world: E2EWorld = { scenario, records: createWorldRecords() } + await createUsers(world, env.E2E_BASE_URL, attemptCounts, loginsByWorldAndUser) + await createOrganizationsAndSubscriptions(world, adminClient) + await addMemberships(world, adminClient) + await createScenarioWorkspaces( + world, + env.E2E_BASE_URL, + loginsByWorldAndUser, + ownerClients, + attemptCounts + ) + await lapsePlannedSubscriptions(world) + await createGrantsAndPermissionGroups(world, adminClient, ownerClients) + await arrangePlatformAdminsAndInvitations(world) + await recordProductionPermissionIds(world) + await assertTrustedWorldInvariants(world) + worlds.push(world) + } + + await assertSecondOrganizationIsRejected(worlds, adminClient) + const manifest = buildScenarioManifest(env.E2E_RUN_ID, worlds) + const credentials = buildPersonaCredentials(env.E2E_RUN_ID, worlds, loginsByWorldAndUser) + writeJsonAtomic(env.E2E_MANIFEST_PATH, manifest) + writeJsonAtomic(env.E2E_CREDENTIALS_PATH, credentials) + + console.info( + `Seeded ${Object.keys(manifest.personas).length} personas across ${worlds.length} isolated worlds` + ) + console.info( + `Auth attempts: ${[...attemptCounts.entries()] + .map(([operation, count]) => `${operation}=${count}`) + .join(', ')}` + ) +} + +async function createUsers( + world: E2EWorld, + baseUrl: string, + attemptCounts: Map, + logins: Map +): Promise { + for (const user of world.scenario.definition.users) { + const login = { + name: user.name, + email: user.email, + password: randomBytes(24).toString('base64url'), + } + const created = await createSyntheticUser( + new E2eHttpClient({ + baseUrl, + onAttempt: ({ path }) => increment(attemptCounts, `signup:${path}`), + }), + login + ) + world.records.users.set(user.key, created) + logins.set(worldUserKey(world.scenario, user.key), login) + } +} + +async function createOrganizationsAndSubscriptions( + world: E2EWorld, + adminClient: E2eHttpClient +): Promise { + for (const organization of world.scenario.definition.organizations) { + const owner = required(world.records.users, organization.ownerUserKey, 'organization owner') + const created = await createOrganization(adminClient, { + name: organization.name, + slug: organization.slug, + ownerId: owner.id, + }) + world.records.organizations.set(organization.key, { + id: created.id, + memberId: created.memberId, + name: created.name, + slug: created.slug, + }) + world.records.organizationMembers.set( + `${organization.key}:${organization.ownerUserKey}`, + created.memberId + ) + } + + for (const subscription of world.scenario.definition.subscriptions) { + const referenceId = resolveBillingReference(world, subscription) + const organizationKey = + subscription.billingReference.kind === 'organization' + ? subscription.billingReference.organizationKey + : undefined + const memberUserIds = organizationKey + ? world.scenario.definition.organizationMemberships + .filter((membership) => membership.organizationKey === organizationKey) + .map(({ userKey }) => required(world.records.users, userKey, 'subscription member').id) + : undefined + const created = await arrangeSubscription({ + referenceId, + plan: subscription.plan, + status: subscription.status === 'lapsed' ? 'active' : subscription.status, + seats: subscription.seats, + memberUserIds, + enterprise: subscription.enterprise + ? { monthlyPrice: subscription.enterprise.monthlyPrice } + : undefined, + }) + world.records.subscriptions.set(subscription.key, created.id) + } +} + +async function addMemberships(world: E2EWorld, adminClient: E2eHttpClient): Promise { + for (const membership of world.scenario.definition.organizationMemberships) { + if (membership.role === 'owner') continue + const organization = required( + world.records.organizations, + membership.organizationKey, + 'organization' + ) + const user = required(world.records.users, membership.userKey, 'organization member') + const created = await addOrganizationMember(adminClient, organization.id, { + userId: user.id, + role: membership.role, + }) + world.records.organizationMembers.set( + `${membership.organizationKey}:${membership.userKey}`, + created.id + ) + } +} + +async function createScenarioWorkspaces( + world: E2EWorld, + baseUrl: string, + logins: Map, + ownerClients: Map, + attemptCounts: Map +): Promise { + for (const workspace of world.scenario.definition.workspaces) { + const login = required( + logins, + worldUserKey(world.scenario, workspace.ownerUserKey), + 'owner login' + ) + const clientKey = worldUserKey(world.scenario, workspace.ownerUserKey) + let client = ownerClients.get(clientKey) + if (!client) { + client = await createAuthenticatedClient(baseUrl, login, ({ path }) => + increment(attemptCounts, `signin:${path}`) + ) + ownerClients.set(clientKey, client) + } + const created = await createWorkspace(client, { name: workspace.name }) + assertCreatedWorkspace(world, workspace, created) + world.records.workspaces.set(workspace.key, { id: created.id, name: created.name }) + } +} + +async function lapsePlannedSubscriptions(world: E2EWorld): Promise { + for (const subscription of world.scenario.definition.subscriptions) { + if (subscription.status !== 'lapsed' || subscription.billingReference.kind !== 'organization') { + continue + } + const organizationKey = subscription.billingReference.organizationKey + const organizationId = required( + world.records.organizations, + organizationKey, + 'lapsed organization' + ).id + const memberUserIds = world.scenario.definition.organizationMemberships + .filter((membership) => membership.organizationKey === organizationKey) + .map(({ userKey }) => required(world.records.users, userKey, 'lapsed member').id) + await lapseOrganizationSubscription({ + subscriptionId: required(world.records.subscriptions, subscription.key, 'subscription'), + organizationId, + memberUserIds, + }) + } +} + +async function createGrantsAndPermissionGroups( + world: E2EWorld, + adminClient: E2eHttpClient, + ownerClients: Map +): Promise { + for (const grant of world.scenario.definition.workspaceGrants) { + const created = await grantWorkspacePermission( + adminClient, + required(world.records.workspaces, grant.workspaceKey, 'workspace').id, + { + userId: required(world.records.users, grant.userKey, 'permission user').id, + permissions: grant.access, + } + ) + world.records.permissions.set(`${grant.workspaceKey}:${grant.userKey}`, created.id) + } + + for (const group of world.scenario.definition.permissionGroups) { + const organization = required( + world.records.organizations, + group.organizationKey, + 'permission group organization' + ) + const ownerKey = required( + world.scenario.organizationsByKey, + group.organizationKey, + 'permission group organization definition' + ).ownerUserKey + const ownerClient = required( + ownerClients, + worldUserKey(world.scenario, ownerKey), + 'permission group owner session' + ) + const created = await createPermissionGroup(ownerClient, organization.id, { + name: group.name, + workspaceIds: group.workspaceKeys.map( + (key) => required(world.records.workspaces, key, 'permission group workspace').id + ), + config: { + hideSecretsTab: group.restrictions.hiddenSettings.includes('secrets'), + hideApiKeysTab: group.restrictions.hiddenSettings.includes('api-keys'), + hideInboxTab: group.restrictions.hiddenSettings.includes('inbox'), + disableMcpTools: group.restrictions.disabledFeatures.includes('mcp'), + disableCustomTools: group.restrictions.disabledFeatures.includes('custom-tools'), + }, + }) + world.records.permissionGroups.set(group.key, created.id) + for (const userKey of group.memberUserKeys) { + const memberId = await addPermissionGroupMember( + ownerClient, + organization.id, + created.id, + required(world.records.users, userKey, 'permission group member').id + ) + world.records.permissionGroupMembers.set(`${group.key}:${userKey}`, memberId) + } + } +} + +async function arrangePlatformAdminsAndInvitations(world: E2EWorld): Promise { + for (const user of world.scenario.definition.users) { + if (user.platformRole === 'admin' && user.superUserModeEnabled) { + await arrangeEffectivePlatformAdmin( + required(world.records.users, user.key, 'platform admin').id + ) + } + } + for (const invitation of world.scenario.definition.invitations) { + const organization = required( + world.records.organizations, + invitation.organizationKey, + 'invitation organization' + ) + const organizationDefinition = required( + world.scenario.organizationsByKey, + invitation.organizationKey, + 'invitation organization definition' + ) + for (const grant of invitation.workspaceGrants) { + const created = await arrangePendingInvitation({ + email: invitation.email, + token: invitation.token, + inviterId: required( + world.records.users, + organizationDefinition.ownerUserKey, + 'inviter' + ).id, + organizationId: organization.id, + workspaceId: required(world.records.workspaces, grant.workspaceKey, 'invited workspace').id, + role: invitation.role, + permission: grant.access, + }) + world.records.invitations.set(invitation.key, created.invitationId) + world.records.invitationGrants.set( + `${invitation.key}:${grant.workspaceKey}`, + created.grantId + ) + } + } +} + +async function recordProductionPermissionIds(world: E2EWorld): Promise { + for (const [workspaceKey, workspaceRecord] of world.records.workspaces) { + const rows = await db + .select({ id: permissions.id, userId: permissions.userId }) + .from(permissions) + .where( + and( + eq(permissions.entityType, 'workspace'), + eq(permissions.entityId, workspaceRecord.id) + ) + ) + for (const row of rows) { + const userKey = [...world.records.users].find(([, user]) => user.id === row.userId)?.[0] + if (userKey) world.records.permissions.set(`${workspaceKey}:${userKey}`, row.id) + } + } +} + +async function assertTrustedWorldInvariants(world: E2EWorld): Promise { + const adminUser = world.records.users.get('enterprise-organization-admin') + const enterpriseWorkspace = world.records.workspaces.get('enterprise-workspace') + if (adminUser && enterpriseWorkspace) { + const explicitAdminRows = await db + .select({ id: permissions.id }) + .from(permissions) + .where( + and( + eq(permissions.entityType, 'workspace'), + eq(permissions.entityId, enterpriseWorkspace.id), + eq(permissions.userId, adminUser.id) + ) + ) + if (explicitAdminRows.length !== 0) { + throw new Error('Enterprise organization admin unexpectedly received an explicit grant') + } + } + + const externalUser = world.records.users.get('external-workspace-admin') + if (externalUser) { + const memberships = await db + .select({ id: member.id }) + .from(member) + .where(eq(member.userId, externalUser.id)) + if (memberships.length !== 0) { + throw new Error('External workspace admin unexpectedly received organization membership') + } + } + + const subscriptionIds = [...world.records.subscriptions.values()] + if (subscriptionIds.length > 0) { + const rows = await db + .select({ + id: subscription.id, + plan: subscription.plan, + status: subscription.status, + referenceId: subscription.referenceId, + }) + .from(subscription) + .where(inArray(subscription.id, subscriptionIds)) + if (rows.length !== subscriptionIds.length) { + throw new Error('Seeded subscription registry does not match persisted rows') + } + for (const definition of world.scenario.definition.subscriptions) { + const id = required(world.records.subscriptions, definition.key, 'subscription') + const row = rows.find((candidate) => candidate.id === id) + if ( + !row || + row.plan !== definition.plan || + row.referenceId !== resolveBillingReference(world, definition) || + row.status !== (definition.status === 'lapsed' ? 'canceled' : definition.status) + ) { + throw new Error(`Persisted subscription does not match scenario: ${definition.key}`) + } + } + } +} + +async function assertSecondOrganizationIsRejected( + worlds: E2EWorld[], + adminClient: E2eHttpClient +): Promise { + const primary = worlds[0] + const teamMember = required(primary.records.users, 'workspace-read-member', 'constraint user') + const enterprise = required( + primary.records.organizations, + 'enterprise-organization', + 'constraint organization' + ) + await adminClient.request({ + method: 'POST', + path: `/api/v1/admin/organizations/${enterprise.id}/members`, + body: { userId: teamMember.id, role: 'member' }, + schema: z.object({ error: z.unknown() }), + expectedStatus: 400, + }) +} + +function buildPersonaCredentials( + runId: string, + worlds: E2EWorld[], + logins: Map +): PersonaCredentials { + const credentials: PersonaCredentials = { + schemaVersion: PERSONA_MANIFEST_VERSION, + runId, + personas: {}, + } + for (const world of worlds) { + for (const persona of world.scenario.definition.personas) { + const login = required( + logins, + worldUserKey(world.scenario, persona.userKey), + 'persona login' + ) + credentials.personas[persona.key] = { email: login.email, password: login.password } + } + } + return credentials +} + +function resolveBillingReference(world: E2EWorld, subscription: ScenarioSubscription): string { + return subscription.billingReference.kind === 'user' + ? required(world.records.users, subscription.billingReference.userKey, 'subscription user').id + : required( + world.records.organizations, + subscription.billingReference.organizationKey, + 'subscription organization' + ).id +} + +function assertCreatedWorkspace( + world: E2EWorld, + expected: (typeof world.scenario.definition.workspaces)[number], + created: { + organizationId: string | null + ownerId: string + billedAccountUserId: string + workspaceMode: string + } +): void { + const ownerId = required(world.records.users, expected.ownerUserKey, 'workspace owner').id + const organizationId = expected.organizationKey + ? required(world.records.organizations, expected.organizationKey, 'workspace organization').id + : null + if ( + created.ownerId !== ownerId || + created.billedAccountUserId !== ownerId || + created.organizationId !== organizationId || + created.workspaceMode !== (organizationId ? 'organization' : 'personal') + ) { + throw new Error(`Production workspace policy created an unexpected shape for ${expected.key}`) + } +} + +function worldUserKey(scenario: ResolvedScenario, userKey: string): string { + return `${scenario.definition.namespace.world}:${userKey}` +} + +function increment(values: Map, key: string): void { + values.set(key, (values.get(key) ?? 0) + 1) +} + +function required(values: ReadonlyMap, key: K, label: string): V { + const value = values.get(key) + if (!value) throw new Error(`Missing ${label}: ${String(key)}`) + return value +} + +await main() diff --git a/apps/sim/e2e/settings/persona-contracts.spec.ts b/apps/sim/e2e/settings/persona-contracts.spec.ts new file mode 100644 index 00000000000..b8000df2efb --- /dev/null +++ b/apps/sim/e2e/settings/persona-contracts.spec.ts @@ -0,0 +1,97 @@ +import { SETTINGS_PERSONA_KEYS } from './personas' +import { expect, requirePersona, test } from '../fixtures/persona-test' + +for (const personaKey of SETTINGS_PERSONA_KEYS) { + test(`${personaKey} matches its real API contract`, async ({ + contextForPersona, + personaManifest, + }) => { + const persona = requirePersona(personaManifest, personaKey) + const context = await contextForPersona(personaKey) + + const sessionResponse = await context.request.get('/api/auth/get-session') + expect(sessionResponse.status()).toBe(200) + const session = (await sessionResponse.json()) as { + user?: { id?: string; email?: string; role?: string } + session?: { activeOrganizationId?: string | null } + } + expect(session.user).toMatchObject({ + id: persona.userId, + email: persona.email, + role: persona.expectedPlatformRole, + }) + expect(session.session?.activeOrganizationId ?? null).toBe( + persona.expectedActiveOrganizationId + ) + + const workspacesResponse = await context.request.get('/api/workspaces?scope=all') + expect(workspacesResponse.status()).toBe(200) + const workspacePayload = (await workspacesResponse.json()) as { + workspaces?: Array<{ id: string; permissions: string; role: string }> + } + const expectedAccessible = persona.workspaces + .filter(({ access }) => access !== 'none') + .map(({ workspaceId }) => workspaceId) + .sort() + expect(workspacePayload.workspaces?.map(({ id }) => id).sort()).toEqual(expectedAccessible) + + for (const expected of persona.workspaces) { + const response = await context.request.get( + `/api/workspaces/${encodeURIComponent(expected.workspaceId)}/host-context` + ) + if (expected.access === 'none') { + expect(response.status()).toBe(403) + continue + } + expect(response.status()).toBe(200) + const host = (await response.json()) as { + workspace?: { id?: string } + ownerBilling?: { plan?: string; isOrgScoped?: boolean } + viewer?: { + permission?: string + isHostOrganizationMember?: boolean + isHostOrganizationAdmin?: boolean + } + } + expect(host.workspace?.id).toBe(expected.workspaceId) + expect(host.viewer?.permission).toBe(expected.access) + expect(host.ownerBilling?.plan).toBe(expected.hostContext.plan) + expect(host.ownerBilling?.isOrgScoped).toBe( + expected.hostContext.payerScope === 'organization' + ) + expect(host.viewer?.isHostOrganizationMember).toBe( + expected.hostContext.payerScope === 'organization' && + expected.hostContext.hostMembership !== 'external' + ) + expect(host.viewer?.isHostOrganizationAdmin).toBe( + expected.hostContext.payerScope === 'organization' && + (expected.roleSource === 'owner' || expected.roleSource === 'org-admin') + ) + } + + const accessibleWorkspace = persona.workspaces.find(({ access }) => access !== 'none') + expect(accessibleWorkspace).toBeTruthy() + const groupResponse = await context.request.get( + `/api/permission-groups/user?workspaceId=${encodeURIComponent( + accessibleWorkspace?.workspaceId ?? '' + )}` + ) + expect(groupResponse.status()).toBe(200) + const group = (await groupResponse.json()) as { + permissionGroupId?: string | null + config?: Record | null + } + if (persona.permissionGroupIds.length > 0) { + expect(group.permissionGroupId).toBe(persona.permissionGroupIds[0]) + expect(group.config).toMatchObject({ + hideSecretsTab: true, + hideApiKeysTab: true, + hideInboxTab: true, + disableMcpTools: true, + disableCustomTools: true, + }) + } else { + expect(group.permissionGroupId).toBeNull() + } + }) +} diff --git a/apps/sim/e2e/settings/persona-isolation.spec.ts b/apps/sim/e2e/settings/persona-isolation.spec.ts new file mode 100644 index 00000000000..24ec2cfb8ec --- /dev/null +++ b/apps/sim/e2e/settings/persona-isolation.spec.ts @@ -0,0 +1,38 @@ +import { expect, requirePersona, test } from '../fixtures/persona-test' + +test.describe.configure({ mode: 'parallel' }) + +test('primary world cannot discover or access the isolation twin', async ({ + contextForPersona, + personaManifest, +}) => { + await assertWorldCannotSee( + await contextForPersona('personalFreeOwner'), + requirePersona(personaManifest, 'isolationTwinOwner').workspaces[0].workspaceId + ) +}) + +test('isolation twin cannot discover or access the primary world', async ({ + contextForPersona, + personaManifest, +}) => { + await assertWorldCannotSee( + await contextForPersona('isolationTwinOwner'), + requirePersona(personaManifest, 'personalFreeOwner').workspaces[0].workspaceId + ) +}) + +async function assertWorldCannotSee( + context: import('@playwright/test').BrowserContext, + foreignWorkspaceId: string +): Promise { + const listResponse = await context.request.get('/api/workspaces?scope=all') + expect(listResponse.status()).toBe(200) + const payload = (await listResponse.json()) as { workspaces?: Array<{ id: string }> } + expect(payload.workspaces?.map(({ id }) => id)).not.toContain(foreignWorkspaceId) + + const response = await context.request.get( + `/api/workspaces/${encodeURIComponent(foreignWorkspaceId)}/host-context` + ) + expect(response.status()).toBe(403) +} diff --git a/apps/sim/e2e/support/deployment-profile.ts b/apps/sim/e2e/support/deployment-profile.ts index 63ffb844ec0..34f2fca490e 100644 --- a/apps/sim/e2e/support/deployment-profile.ts +++ b/apps/sim/e2e/support/deployment-profile.ts @@ -20,6 +20,7 @@ const APP_REQUIRED_KEYS = [ 'NEXT_PUBLIC_BILLING_ENABLED', 'STRIPE_SECRET_KEY', 'STRIPE_API_BASE_URL', + 'TELEMETRY_ENDPOINT', 'E2E_PROFILE', 'E2E_RUN_ID', 'HOME', @@ -131,6 +132,7 @@ export function createHostedBillingProfile({ STRIPE_WEBHOOK_SECRET: 'whsec_sim_e2e_foundation', STRIPE_FREE_PRICE_ID: 'price_e2e_free', STRIPE_API_BASE_URL: stripeApiBaseUrl, + TELEMETRY_ENDPOINT: `${stripeApiBaseUrl}/v1/traces`, SOCKET_SERVER_URL: 'http://127.0.0.1:3002', NEXT_PUBLIC_SOCKET_URL: E2E_SOCKET_ORIGIN, CI: ci ? 'true' : 'false', @@ -150,6 +152,7 @@ export function createHostedBillingProfile({ HOME: buildHomeDirectory, DATABASE_URL: 'postgresql://e2e_build:e2e_build@127.0.0.1:1/sim_e2e_build_sentinel', STRIPE_API_BASE_URL: 'http://127.0.0.1:1', + TELEMETRY_ENDPOINT: 'http://127.0.0.1:1/v1/traces', CI: 'false', } validateProfileValues(buildValues) diff --git a/apps/sim/e2e/support/stack.ts b/apps/sim/e2e/support/stack.ts index aeb51acce37..1832474e551 100644 --- a/apps/sim/e2e/support/stack.ts +++ b/apps/sim/e2e/support/stack.ts @@ -40,6 +40,28 @@ export async function runMigrations(options: StackCommandOptions): Promise }) } +export async function seedE2eWorld(options: StackCommandOptions): Promise { + await runCommand({ + name: 'seed-world', + command: options.bunExecutable, + args: ['--no-env-file', path.join(SIM_APP_DIR, 'e2e/scripts/seed-world.ts')], + cwd: SIM_APP_DIR, + env: options.env, + logsDirectory: options.logsDirectory, + }) +} + +export async function capturePersonaAuthStates(options: StackCommandOptions): Promise { + await runCommand({ + name: 'capture-auth-states', + command: options.nodeExecutable, + args: ['--import', 'tsx', path.join(SIM_APP_DIR, 'e2e/scripts/capture-auth-states.ts')], + cwd: SIM_APP_DIR, + env: options.env, + logsDirectory: options.logsDirectory, + }) +} + export async function buildApp(options: BuildAppOptions): Promise { verifySandboxBundleIntegrity() const identity = computeBuildIdentity({ diff --git a/apps/sim/playwright.config.ts b/apps/sim/playwright.config.ts index 70cb2641644..da61f3ac305 100644 --- a/apps/sim/playwright.config.ts +++ b/apps/sim/playwright.config.ts @@ -15,7 +15,7 @@ export default defineConfig({ fullyParallel: true, forbidOnly: isCI, retries: 0, - workers: 1, + workers: 2, timeout: 60_000, expect: { timeout: 10_000, @@ -54,5 +54,17 @@ export default defineConfig({ fullyParallel: false, workers: 1, }, + { + name: 'hosted-billing-chromium-personas', + testMatch: '**/settings/persona-contracts.spec.ts', + fullyParallel: false, + workers: 1, + }, + { + name: 'hosted-billing-chromium-persona-isolation', + testMatch: '**/settings/persona-isolation.spec.ts', + fullyParallel: true, + workers: 2, + }, ], }) From b4a0ff2c14ebab4e90f37d29ae7b7b1b7ce5848a Mon Sep 17 00:00:00 2001 From: Bill Leoutsakos Date: Tue, 21 Jul 2026 13:12:56 -0700 Subject: [PATCH 4/6] Harden E2E persona orchestration and remove deferred cleanup Generate runtime secrets per run, fail closed when diagnostics cannot be scanned, and harden cache and process cleanup while removing the unused exact-ID cleanup module until retained-stack support has a real consumer. --- .github/workflows/test-build.yml | 12 +- apps/sim/e2e/README.md | 70 ++- apps/sim/e2e/fakes/stripe/server.ts | 1 + apps/sim/e2e/fixtures/cleanup.ts | 172 ------- apps/sim/e2e/fixtures/e2e-world.ts | 52 +- apps/sim/e2e/fixtures/factories/billing.ts | 30 +- .../sim/e2e/fixtures/factories/invitations.ts | 35 +- .../fixtures/factories/permission-groups.ts | 3 +- apps/sim/e2e/fixtures/http-client.ts | 14 +- apps/sim/e2e/fixtures/persona-test.ts | 17 +- apps/sim/e2e/fixtures/scenario.ts | 9 +- apps/sim/e2e/fixtures/validate-scenario.ts | 93 +++- .../sim/e2e/foundation/build-manifest.spec.ts | 174 +++++++ apps/sim/e2e/foundation/http-client.spec.ts | 23 + apps/sim/e2e/foundation/leak-canary.spec.ts | 84 ++++ apps/sim/e2e/foundation/safety.spec.ts | 133 ++++-- .../foundation/scenario-validation.spec.ts | 70 ++- apps/sim/e2e/foundation/stripe-fake.spec.ts | 3 +- apps/sim/e2e/scripts/capture-auth-states.ts | 57 ++- apps/sim/e2e/scripts/options.ts | 43 ++ apps/sim/e2e/scripts/run.ts | 148 +++++- apps/sim/e2e/scripts/seed-world.ts | 448 +++++++++++++++--- apps/sim/e2e/scripts/signal-cleanup.ts | 45 +- .../e2e/settings/persona-contracts.spec.ts | 80 +++- .../e2e/settings/persona-isolation.spec.ts | 39 +- apps/sim/e2e/settings/personas.ts | 24 +- .../e2e/settings/smoke/authenticated.spec.ts | 10 +- apps/sim/e2e/support/build-manifest.ts | 160 ++++--- apps/sim/e2e/support/deployment-profile.ts | 142 +++--- apps/sim/e2e/support/env.ts | 2 - apps/sim/e2e/support/leak-canary.ts | 132 ++++++ apps/sim/e2e/support/paths.ts | 2 +- apps/sim/e2e/support/probes.ts | 34 ++ apps/sim/e2e/support/process.ts | 76 ++- apps/sim/e2e/support/run-lock.ts | 168 +++++++ apps/sim/e2e/support/runtime-secrets.ts | 29 ++ apps/sim/e2e/support/sandbox-bundles.ts | 8 +- apps/sim/e2e/support/seed-safety.ts | 33 ++ apps/sim/e2e/support/signal-cleanup.ts | 11 + apps/sim/e2e/support/stack.ts | 37 +- .../lib/execution/sandbox/bundles/build.ts | 86 +++- .../execution/sandbox/bundles/integrity.json | 2 +- apps/sim/package.json | 2 + apps/sim/playwright.config.ts | 3 + bun.lock | 72 ++- 45 files changed, 2250 insertions(+), 638 deletions(-) delete mode 100644 apps/sim/e2e/fixtures/cleanup.ts create mode 100644 apps/sim/e2e/foundation/build-manifest.spec.ts create mode 100644 apps/sim/e2e/foundation/leak-canary.spec.ts create mode 100644 apps/sim/e2e/support/leak-canary.ts create mode 100644 apps/sim/e2e/support/run-lock.ts create mode 100644 apps/sim/e2e/support/runtime-secrets.ts create mode 100644 apps/sim/e2e/support/seed-safety.ts diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 4d3fa2dbde4..579c4fc0d48 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -300,14 +300,6 @@ jobs: key: ${{ github.repository }}-playwright-browsers-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} path: ~/.cache/ms-playwright - - name: Restore E2E Next.js build cache - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 - with: - path: ./apps/sim/.next/cache - key: ${{ runner.os }}-nextjs-e2e-hosted-billing-${{ hashFiles('bun.lock') }}-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-nextjs-e2e-hosted-billing-${{ hashFiles('bun.lock') }}- - - name: Install dependencies run: bun install --frozen-lockfile @@ -327,7 +319,7 @@ jobs: run: bun run test:e2e - name: Upload E2E diagnostics - if: failure() || cancelled() + if: failure() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: settings-e2e-${{ github.run_id }} @@ -336,6 +328,8 @@ jobs: apps/sim/test-results/ apps/sim/e2e/.runs/ !apps/sim/e2e/.runs/**/auth/** + !apps/sim/e2e/.runs/**/private/** + !apps/sim/e2e/.runs/**/homes/** if-no-files-found: ignore include-hidden-files: true retention-days: 7 \ No newline at end of file diff --git a/apps/sim/e2e/README.md b/apps/sim/e2e/README.md index 5f708125ba3..de632d015f9 100644 --- a/apps/sim/e2e/README.md +++ b/apps/sim/e2e/README.md @@ -54,11 +54,18 @@ Stripe fake and realtime, builds and starts Next.js, seeds the validated persona world through production APIs plus narrow trusted arrangements, captures each persona through the real login UI, runs Playwright on Node 22, then stops services and drops only that guarded database. +Server telemetry currently shares the strict loopback Stripe-fake process; the +generic modular fake-service refactor remains scoped to the roadmap's +`e2e/06b-enterprise-integrations` phase. +An exclusive checkout-level orchestrator lock prevents concurrent runs from +racing on `.next`, the shared build cache, or the fixed app/realtime ports. On interruption, the runner launches a detached cleanup supervisor before exiting. It terminates managed process groups, force-drops the guarded database, and removes temporary auth/cloud-config directories even if another Ctrl-C terminates the foreground package runner. +Cleanup failures retain the lock and require the reported resources to be +inspected and cleaned before manually removing `e2e/.cache/orchestrator.lock`. Pass Playwright arguments after `--`: @@ -67,6 +74,19 @@ bun run test:e2e -- --project=hosted-billing-chromium-navigation bun run test:e2e -- --grep "unauthenticated" ``` +Projects form a dependency chain to keep shared boundaries serialized. Selecting +the personas project therefore also runs navigation and workflows by default, +and an upstream failure skips its dependents. For focused local iteration, +`--no-deps` is explicitly supported: + +```bash +bun run test:e2e -- --project=hosted-billing-chromium-personas --no-deps +``` + +`--no-deps` requires exactly one explicit canonical project. It skips only +Playwright project dependencies; the guarded one-shot stack still performs full +seed and auth setup. Do not use it for full verification; the runner rejects it in CI. + For a local follow-up run, reuse only a verified build while still creating a new database, Stripe fake, app, realtime process, and browser run: @@ -77,8 +97,13 @@ bun run test:e2e -- --reuse-build --project=hosted-billing-chromium-navigation The cache lives under ignored `e2e/.cache/builds/`. A hit requires matching source contents (including uncommitted/untracked files), build/public profile, Node/Bun/Next versions, platform, `BUILD_ID`, and the cached artifact checksum. -Any mismatch performs and caches a fresh build. CI rejects `--reuse-build`. -`--skip-build` remains unsupported. +Any mismatch performs and caches a fresh local build; only the most recent cache +entry is retained because this application's production artifact is several +gigabytes. CI rejects `--reuse-build` and does not copy or hash a disposable +cache artifact. Plain local runs also skip cache identity computation, +multi-gigabyte copying, and cache population; only an explicit `--reuse-build` +request pays those costs. Cache restore also removes abandoned temporary stores +from interrupted writes before accepting a hit. `--skip-build` remains unsupported. Keep-stack/rerun supervision is intentionally unavailable. The initial safety experiment requires descriptor ownership, mutation observation, state snapshots, @@ -92,7 +117,9 @@ safe because they do not execute tests. Sharding is supported only for the navigation project. The runner rejects `--shard` for workflows, persona contracts, and the dedicated two-worker -cross-world isolation project. +cross-world isolation project. Project dependencies serialize navigation, +workflows, and persona contracts before the isolation project opens its +two-worker pool. ## Diagnostics @@ -100,7 +127,7 @@ cross-world isolation project. - Traces and screenshots: `test-results/` - App, realtime, migration, seed, auth-capture, and fake logs: `e2e/.runs//logs/` -- Non-secret persona manifest and post-login auth-capture screenshots: +- Non-secret persona manifest and auth-capture failure screenshots: `e2e/.runs//` Open the report: @@ -123,12 +150,35 @@ non-secret manifest and storage-state directory, never passwords, the admin API key, or the database URL. Next build/start shadow keys found in local `.env*` files, while children that cannot load those files omit denied keys entirely. Developer credentials are not used as test state or -written to reports. Synthetic persona passwords live in a private run-home file -and are removed with the auth/cloud-config directory during teardown. - -E2E builds verify the reviewed sandbox-bundle source/dependency/output -fingerprint and never regenerate committed `.cjs` files. Regeneration remains an -explicit repository maintenance operation. +written to reports. Named persona credentials and a separate all-synthetic-user +canary list live outside every child `HOME` in a private run directory. Auth +capture receives only the persona file, which is deleted immediately after +capture; storage states are mode `0600` and excluded from CI artifacts. Captured +passwords are loaded into orchestrator memory, then both secret files are +deleted before Playwright starts. After managed processes stop and logs flush, +the in-memory canary scans the manifest, logs, report files, and trace archives +and fails if a synthetic password, invitation token, or runtime secret escaped +the excluded private directories. Cancelled CI runs do not upload unscanned +diagnostics, and an unreadable canary or incomplete archive scan causes all +potentially unscanned diagnostic roots to be scrubbed. Storage-state session cookies are intentionally not canaried +because authenticated Playwright traces contain them by design; they are +synthetic and invalid once the run database is dropped. +Fresh-session recapture is deliberately deferred. Future membership-mutation +coverage must explicitly restore a private credential handoff and re-review its +access boundary rather than assuming credentials persist through Playwright. + +E2E builds verify the pinned Bun executable plus reviewed sandbox-bundle +source, direct dependency, and output fingerprints and never regenerate +committed `.cjs` files. +`bun run build:sandbox-bundles:integrity` is the explicit maintenance command +that regenerates bundles and their reviewed integrity manifest together. +Unrelated monorepo lockfile changes do not invalidate the bundle fingerprint; +the committed output hashes still detect any transitive change that alters a +bundle. + +Reset/reseed cleanup remains deferred with keep-stack supervision. Ordinary +runs own a unique guarded database and remove it wholesale rather than carrying +untested row-level deletion code. Provider log scans are diagnostic tripwires, not proof of zero egress. The primary boundaries are the default-deny child environment, provider disabling, diff --git a/apps/sim/e2e/fakes/stripe/server.ts b/apps/sim/e2e/fakes/stripe/server.ts index d87e465c780..9fe5d07826c 100644 --- a/apps/sim/e2e/fakes/stripe/server.ts +++ b/apps/sim/e2e/fakes/stripe/server.ts @@ -6,6 +6,7 @@ export const STRIPE_FAKE_ENDPOINTS = { health: '/health', requestLog: '/__control/requests', reset: '/__control/reset', + // Co-locate the only other server-side external boundary so the E2E stack needs one loopback fake. telemetry: '/v1/traces', } as const diff --git a/apps/sim/e2e/fixtures/cleanup.ts b/apps/sim/e2e/fixtures/cleanup.ts deleted file mode 100644 index 2c9d68756a8..00000000000 --- a/apps/sim/e2e/fixtures/cleanup.ts +++ /dev/null @@ -1,172 +0,0 @@ -import { db } from '@sim/db' -import { - auditLog, - invitation, - invitationWorkspaceGrant, - member, - organization, - permissionGroup, - permissionGroupMember, - permissionGroupWorkspace, - permissions, - settings, - subscription, - user, - userStats, - workspace, -} from '@sim/db/schema' -import { inArray, or } from 'drizzle-orm' -import type { ScenarioManifest } from './e2e-world' - -/** - * Deletes only IDs recorded by a successful seed. Names and prefixes are never used as delete - * predicates. Identity verification makes a stale or hand-edited manifest fail closed. - */ -export async function cleanupSeededWorld(manifest: ScenarioManifest): Promise { - const userIdentities = Object.values(manifest.worlds).flatMap((world) => - Object.values(world.userIdentities) - ) - const organizationIdentities = Object.values(manifest.worlds).flatMap((world) => - Object.values(world.organizationIdentities) - ) - const workspaceIdentities = Object.values(manifest.worlds).flatMap((world) => - Object.values(world.workspaceIdentities) - ) - await verifyOwnership(userIdentities, organizationIdentities, workspaceIdentities) - - const userIds = userIdentities.map(({ id }) => id) - const organizationIds = organizationIdentities.map(({ id }) => id) - const workspaceIds = workspaceIdentities.map(({ id }) => id) - const resourceIds = [ - ...workspaceIds, - ...organizationIds, - ...Object.values(manifest.personas).flatMap(({ permissionGroupIds }) => permissionGroupIds), - ] - const subscriptionIds = collectValues(manifest, 'subscriptionIds') - const permissionIds = collectValues(manifest, 'permissionIds') - const permissionGroupIds = collectValues(manifest, 'permissionGroupIds') - const permissionGroupMemberIds = collectValues(manifest, 'permissionGroupMemberIds') - const invitationIds = collectValues(manifest, 'invitationIds') - const invitationGrantIds = collectValues(manifest, 'invitationGrantIds') - const organizationMemberIds = collectValues(manifest, 'organizationMemberIds') - - await db.transaction(async (tx) => { - const auditPredicates = [ - workspaceIds.length ? inArray(auditLog.workspaceId, workspaceIds) : undefined, - userIds.length ? inArray(auditLog.actorId, userIds) : undefined, - resourceIds.length ? inArray(auditLog.resourceId, resourceIds) : undefined, - ].filter((value): value is NonNullable => value !== undefined) - if (auditPredicates.length > 0) await tx.delete(auditLog).where(or(...auditPredicates)) - - if (invitationGrantIds.length) { - await tx - .delete(invitationWorkspaceGrant) - .where(inArray(invitationWorkspaceGrant.id, invitationGrantIds)) - } - if (invitationIds.length) { - await tx.delete(invitation).where(inArray(invitation.id, invitationIds)) - } - if (permissionGroupMemberIds.length) { - await tx - .delete(permissionGroupMember) - .where(inArray(permissionGroupMember.id, permissionGroupMemberIds)) - } - if (permissionGroupIds.length) { - await tx - .delete(permissionGroupWorkspace) - .where(inArray(permissionGroupWorkspace.permissionGroupId, permissionGroupIds)) - await tx.delete(permissionGroup).where(inArray(permissionGroup.id, permissionGroupIds)) - } - if (permissionIds.length) { - await tx.delete(permissions).where(inArray(permissions.id, permissionIds)) - } - if (workspaceIds.length) { - await tx.delete(workspace).where(inArray(workspace.id, workspaceIds)) - } - if (organizationMemberIds.length) { - await tx.delete(member).where(inArray(member.id, organizationMemberIds)) - } - if (subscriptionIds.length) { - await tx.delete(subscription).where(inArray(subscription.id, subscriptionIds)) - } - if (organizationIds.length) { - await tx.delete(organization).where(inArray(organization.id, organizationIds)) - } - if (userIds.length) { - await tx.delete(settings).where(inArray(settings.userId, userIds)) - await tx.delete(userStats).where(inArray(userStats.userId, userIds)) - await tx.delete(user).where(inArray(user.id, userIds)) - } - }) -} - -async function verifyOwnership( - expectedUsers: Array<{ id: string; email: string; name: string }>, - expectedOrganizations: Array<{ id: string; name: string; slug: string }>, - expectedWorkspaces: Array<{ id: string; name: string }> -): Promise { - const actualUsers = expectedUsers.length - ? await db - .select({ id: user.id, email: user.email, name: user.name }) - .from(user) - .where( - inArray( - user.id, - expectedUsers.map(({ id }) => id) - ) - ) - : [] - const actualOrganizations = expectedOrganizations.length - ? await db - .select({ id: organization.id, name: organization.name, slug: organization.slug }) - .from(organization) - .where( - inArray( - organization.id, - expectedOrganizations.map(({ id }) => id) - ) - ) - : [] - const actualWorkspaces = expectedWorkspaces.length - ? await db - .select({ id: workspace.id, name: workspace.name }) - .from(workspace) - .where( - inArray( - workspace.id, - expectedWorkspaces.map(({ id }) => id) - ) - ) - : [] - - assertExactIdentities('user', expectedUsers, actualUsers) - assertExactIdentities('organization', expectedOrganizations, actualOrganizations) - assertExactIdentities('workspace', expectedWorkspaces, actualWorkspaces) -} - -function assertExactIdentities( - label: string, - expected: T[], - actual: T[] -): void { - const actualById = new Map(actual.map((value) => [value.id, value])) - for (const value of expected) { - if (JSON.stringify(actualById.get(value.id)) !== JSON.stringify(value)) { - throw new Error(`Refusing cleanup: ${label} ownership mismatch for exact ID ${value.id}`) - } - } -} - -function collectValues( - manifest: ScenarioManifest, - key: - | 'subscriptionIds' - | 'permissionIds' - | 'permissionGroupIds' - | 'permissionGroupMemberIds' - | 'invitationIds' - | 'invitationGrantIds' - | 'organizationMemberIds' -): string[] { - return Object.values(manifest.worlds).flatMap((world) => Object.values(world[key])) -} diff --git a/apps/sim/e2e/fixtures/e2e-world.ts b/apps/sim/e2e/fixtures/e2e-world.ts index ffa2b05cab9..fe7014f611b 100644 --- a/apps/sim/e2e/fixtures/e2e-world.ts +++ b/apps/sim/e2e/fixtures/e2e-world.ts @@ -2,13 +2,16 @@ import { randomUUID } from 'node:crypto' import { mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs' import path from 'node:path' import { z } from 'zod' -import type { - ResolvedScenario, - ScenarioPersona, - ScenarioWorkspace, -} from './scenario' +import type { ResolvedScenario, ScenarioPersona } from './scenario' export const PERSONA_MANIFEST_VERSION = 1 as const +const SAFE_IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]*$/ +const STORAGE_STATE_FILENAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*\.json$/ +const safeIdentifierSchema = z.string().regex(SAFE_IDENTIFIER_PATTERN) +const storageStateFilenameSchema = z + .string() + .regex(STORAGE_STATE_FILENAME_PATTERN) + .refine((value) => !value.includes('..'), 'storage-state filename must not contain ".."') const workspaceExpectationSchema = z.object({ workspaceId: z.string(), @@ -19,27 +22,20 @@ const workspaceExpectationSchema = z.object({ isOwner: z.boolean(), hostMembership: z.enum(['owner', 'member', 'external']), payerScope: z.enum(['user', 'organization']), - plan: z.enum([ - 'free', - 'pro_6000', - 'pro_25000', - 'team_6000', - 'team_25000', - 'enterprise', - ]), + plan: z.enum(['free', 'pro_6000', 'pro_25000', 'team_6000', 'team_25000', 'enterprise']), hosted: z.boolean(), billingEnabled: z.boolean(), }), }) export const personaManifestEntrySchema = z.object({ - key: z.string(), - world: z.string(), + key: safeIdentifierSchema, + world: safeIdentifierSchema, userId: z.string(), email: z.string().email(), name: z.string(), expectedActiveOrganizationId: z.string().nullable(), - storageStatePath: z.string(), + storageStatePath: storageStateFilenameSchema, canonicalRoute: z.string().startsWith('/'), workspaces: z.array(workspaceExpectationSchema).min(1), permissionGroupIds: z.array(z.string()), @@ -65,10 +61,7 @@ const worldManifestSchema = z.object({ ), organizationMemberIds: z.record(z.string(), z.string()), workspaceIds: z.record(z.string(), z.string()), - workspaceIdentities: z.record( - z.string(), - z.object({ id: z.string(), name: z.string() }) - ), + workspaceIdentities: z.record(z.string(), z.object({ id: z.string(), name: z.string() })), subscriptionIds: z.record(z.string(), z.string()), permissionIds: z.record(z.string(), z.string()), permissionGroupIds: z.record(z.string(), z.string()), @@ -82,15 +75,15 @@ export const scenarioManifestSchema = z.object({ runId: z.string(), createdAt: z.string(), authCaptureComplete: z.boolean(), - worlds: z.record(z.string(), worldManifestSchema), - personas: z.record(z.string(), personaManifestEntrySchema), + worlds: z.record(safeIdentifierSchema, worldManifestSchema), + personas: z.record(safeIdentifierSchema, personaManifestEntrySchema), }) export const personaCredentialsSchema = z.object({ schemaVersion: z.literal(PERSONA_MANIFEST_VERSION), runId: z.string(), personas: z.record( - z.string(), + safeIdentifierSchema, z.object({ email: z.string().email(), password: z.string().min(12), @@ -147,6 +140,9 @@ export function buildScenarioManifest(runId: string, worlds: E2EWorld[]): Scenar for (const world of worlds) { const worldKey = world.scenario.definition.namespace.world + if (manifest.worlds[worldKey]) { + throw new Error(`Duplicate world namespace: ${worldKey}`) + } manifest.worlds[worldKey] = { namespace: world.scenario.definition.namespace, userIds: mapValues(world.records.users, ({ id }) => id), @@ -187,6 +183,16 @@ export function readPersonaCredentials(filePath: string): PersonaCredentials { return personaCredentialsSchema.parse(JSON.parse(readFileSync(filePath, 'utf8'))) } +export function resolveStorageStatePath(directory: string, filename: string): string { + const safeFilename = storageStateFilenameSchema.parse(filename) + const root = path.resolve(directory) + const resolved = path.resolve(root, safeFilename) + if (path.dirname(resolved) !== root) { + throw new Error(`Storage-state path escapes its run directory: ${filename}`) + } + return resolved +} + export function writeJsonAtomic(filePath: string, value: unknown, mode = 0o600): void { mkdirSync(path.dirname(filePath), { recursive: true }) const temporaryPath = `${filePath}.tmp-${process.pid}-${randomUUID()}` diff --git a/apps/sim/e2e/fixtures/factories/billing.ts b/apps/sim/e2e/fixtures/factories/billing.ts index b35fef2ef6b..be6507c0c87 100644 --- a/apps/sim/e2e/fixtures/factories/billing.ts +++ b/apps/sim/e2e/fixtures/factories/billing.ts @@ -1,5 +1,5 @@ import { db } from '@sim/db' -import { organization, subscription, userStats } from '@sim/db/schema' +import { member, organization, subscription, user, userStats } from '@sim/db/schema' import { generateId } from '@sim/utils/id' import { and, eq, inArray } from 'drizzle-orm' @@ -36,6 +36,7 @@ export async function arrangeSubscription(input: SubscriptionArrangement): Promi const periodEnd = new Date(now.getTime() + 30 * 24 * 60 * 60 * 1_000) const id = generateId() const status = input.status ?? 'active' + const stripeCustomerId = await resolveCanonicalStripeCustomerId(input.referenceId) const metadata = input.plan === 'enterprise' ? { @@ -51,12 +52,13 @@ export async function arrangeSubscription(input: SubscriptionArrangement): Promi id, plan: input.plan, referenceId: input.referenceId, - stripeCustomerId: `cus_e2e_${id}`, - stripeSubscriptionId: `sub_e2e_${id}`, + stripeCustomerId, + // Stripe lifecycle calls are outside Step 2; do not invent a remote subscription identity. + stripeSubscriptionId: null, status, periodStart: now, periodEnd, - cancelAtPeriodEnd: status !== 'active', + cancelAtPeriodEnd: false, canceledAt: status === 'canceled' ? now : null, endedAt: status === 'canceled' ? now : null, seats: input.seats, @@ -101,6 +103,26 @@ export async function arrangeSubscription(input: SubscriptionArrangement): Promi return { id } } +async function resolveCanonicalStripeCustomerId(referenceId: string): Promise { + const direct = await db + .select({ stripeCustomerId: user.stripeCustomerId }) + .from(user) + .where(eq(user.id, referenceId)) + .limit(1) + const directCustomerId = direct[0]?.stripeCustomerId + if (directCustomerId) return directCustomerId + + const owner = await db + .select({ stripeCustomerId: user.stripeCustomerId }) + .from(member) + .innerJoin(user, eq(user.id, member.userId)) + .where(and(eq(member.organizationId, referenceId), eq(member.role, 'owner'))) + .limit(1) + const ownerCustomerId = owner[0]?.stripeCustomerId + if (ownerCustomerId) return ownerCustomerId + throw new Error(`Billing reference has no canonical Stripe customer: ${referenceId}`) +} + export async function lapseOrganizationSubscription(input: { subscriptionId: string organizationId: string diff --git a/apps/sim/e2e/fixtures/factories/invitations.ts b/apps/sim/e2e/fixtures/factories/invitations.ts index 3ed895ad833..4eda69144b0 100644 --- a/apps/sim/e2e/fixtures/factories/invitations.ts +++ b/apps/sim/e2e/fixtures/factories/invitations.ts @@ -7,12 +7,15 @@ export async function arrangePendingInvitation(input: { token: string inviterId: string organizationId: string - workspaceId: string role: 'admin' | 'member' - permission: 'admin' | 'write' | 'read' -}): Promise<{ invitationId: string; grantId: string }> { + expiresAt: Date + workspaceGrants: Array<{ + workspaceId: string + permission: 'admin' | 'write' | 'read' + }> +}): Promise<{ invitationId: string; grantIds: string[] }> { const invitationId = generateId() - const grantId = generateId() + const grantIds = input.workspaceGrants.map(() => generateId()) const now = new Date() await db.transaction(async (tx) => { await tx.insert(invitation).values({ @@ -25,18 +28,22 @@ export async function arrangePendingInvitation(input: { role: input.role, status: 'pending', token: input.token, - expiresAt: new Date(now.getTime() + 7 * 24 * 60 * 60 * 1_000), - createdAt: now, - updatedAt: now, - }) - await tx.insert(invitationWorkspaceGrant).values({ - id: grantId, - invitationId, - workspaceId: input.workspaceId, - permission: input.permission, + expiresAt: input.expiresAt, createdAt: now, updatedAt: now, }) + if (input.workspaceGrants.length > 0) { + await tx.insert(invitationWorkspaceGrant).values( + input.workspaceGrants.map((grant, index) => ({ + id: grantIds[index], + invitationId, + workspaceId: grant.workspaceId, + permission: grant.permission, + createdAt: now, + updatedAt: now, + })) + ) + } }) - return { invitationId, grantId } + return { invitationId, grantIds } } diff --git a/apps/sim/e2e/fixtures/factories/permission-groups.ts b/apps/sim/e2e/fixtures/factories/permission-groups.ts index dcdaf99b378..25615fc1f97 100644 --- a/apps/sim/e2e/fixtures/factories/permission-groups.ts +++ b/apps/sim/e2e/fixtures/factories/permission-groups.ts @@ -27,12 +27,13 @@ export async function createPermissionGroup( description?: string workspaceIds: string[] config: RestrictedPermissionConfig + isDefault: boolean } ): Promise> { const response = await ownerClient.request({ method: 'POST', path: `/api/organizations/${organizationId}/permission-groups`, - body: { ...input, isDefault: false }, + body: input, schema: z.object({ permissionGroup: permissionGroupSchema }), expectedStatus: 201, }) diff --git a/apps/sim/e2e/fixtures/http-client.ts b/apps/sim/e2e/fixtures/http-client.ts index 163e8e1e5ee..54d7c153a83 100644 --- a/apps/sim/e2e/fixtures/http-client.ts +++ b/apps/sim/e2e/fixtures/http-client.ts @@ -4,6 +4,8 @@ import type { z } from 'zod' // Better Auth does not emit Retry-After for every limiter. The bounded fallback spans its // one-minute signup window while still preferring an explicit server header when available. const DEFAULT_RETRY_DELAYS_MS = [1_000, 2_000, 4_000, 8_000, 16_000, 30_000] as const +const MAX_RETRY_DELAY_MS = 30_000 +const MAX_TOTAL_RETRY_DELAY_MS = 90_000 export interface E2eHttpClientOptions { baseUrl: string @@ -43,6 +45,7 @@ export class E2eHttpClient { const expectedStatuses = Array.isArray(options.expectedStatus) ? options.expectedStatus : [options.expectedStatus ?? 200] + let totalRetryDelay = 0 for (let attempt = 1; ; attempt += 1) { const response = await this.fetchImplementation(`${this.baseUrl}${options.path}`, { @@ -60,10 +63,15 @@ export class E2eHttpClient { this.onAttempt?.({ method, path: options.path, number: attempt, status: response.status }) if (response.status === 429 && attempt <= this.retryDelaysMs.length) { - const delay = + const requestedDelay = parseRetryAfter(response.headers.get('retry-after')) ?? this.retryDelaysMs[attempt - 1] - await this.sleepImplementation(delay) - continue + const remainingDelayBudget = MAX_TOTAL_RETRY_DELAY_MS - totalRetryDelay + if (remainingDelayBudget > 0) { + const delay = Math.min(requestedDelay, MAX_RETRY_DELAY_MS, remainingDelayBudget) + totalRetryDelay += delay + await this.sleepImplementation(delay) + continue + } } const payload = await readJsonResponse(response, method, options.path) diff --git a/apps/sim/e2e/fixtures/persona-test.ts b/apps/sim/e2e/fixtures/persona-test.ts index 4ba3df9f2cb..1bfe4d84e3f 100644 --- a/apps/sim/e2e/fixtures/persona-test.ts +++ b/apps/sim/e2e/fixtures/persona-test.ts @@ -1,21 +1,19 @@ -import path from 'node:path' import { test as base } from '@playwright/test' import { type PersonaManifestEntry, readScenarioManifest, + resolveStorageStatePath, type ScenarioManifest, } from './e2e-world' interface PersonaFixtures { personaManifest: ScenarioManifest - contextForPersona: ( - personaKey: string - ) => Promise + contextForPersona: (personaKey: string) => Promise } export const test = base.extend({ personaManifest: [ - async ({}, use) => { + async ({ browserName: _browserName }, use) => { const manifest = readScenarioManifest(requiredEnv('E2E_MANIFEST_PATH')) if (!manifest.authCaptureComplete) { throw new Error('Persona storage states were not captured successfully') @@ -24,12 +22,17 @@ export const test = base.extend({ }, { scope: 'test' }, ], - contextForPersona: async ({ browser, personaManifest }, use) => { + contextForPersona: async ({ browser, contextOptions, personaManifest }, use) => { const contexts = new Set() await use(async (personaKey) => { const persona = requirePersona(personaManifest, personaKey) const context = await browser.newContext({ - storageState: path.join(requiredEnv('E2E_STORAGE_STATE_DIR'), persona.storageStatePath), + ...contextOptions, + baseURL: requiredEnv('E2E_BASE_URL'), + storageState: resolveStorageStatePath( + requiredEnv('E2E_STORAGE_STATE_DIR'), + persona.storageStatePath + ), }) contexts.add(context) return context diff --git a/apps/sim/e2e/fixtures/scenario.ts b/apps/sim/e2e/fixtures/scenario.ts index e68dd20c74c..9367c70979b 100644 --- a/apps/sim/e2e/fixtures/scenario.ts +++ b/apps/sim/e2e/fixtures/scenario.ts @@ -6,12 +6,7 @@ export type WorkspaceAccess = 'read' | 'write' | 'admin' export type ExpectedWorkspaceAccess = WorkspaceAccess | 'none' export type RoleSource = 'owner' | 'explicit' | 'org-admin' | 'none' export type OrganizationRole = 'owner' | 'admin' | 'member' -export type SubscriptionPlan = - | 'pro_6000' - | 'pro_25000' - | 'team_6000' - | 'team_25000' - | 'enterprise' +export type SubscriptionPlan = 'pro_6000' | 'pro_25000' | 'team_6000' | 'team_25000' | 'enterprise' export type SubscriptionStatus = 'active' | 'past_due' | 'lapsed' export type SettingsSection = 'general' | 'secrets' | 'api-keys' | 'inbox' | 'mcp' | 'custom-tools' @@ -53,7 +48,6 @@ export interface ScenarioOrganizationMembership { export interface EnterpriseSubscriptionMetadata { plan: 'enterprise' - referenceId: string monthlyPrice: number seats: number } @@ -101,7 +95,6 @@ export interface ScenarioPermissionGroup { organizationKey: ResourceKey workspaceKeys: readonly ResourceKey[] memberUserKeys: readonly ResourceKey[] - isDefault?: boolean restrictions: PermissionGroupRestrictions } diff --git a/apps/sim/e2e/fixtures/validate-scenario.ts b/apps/sim/e2e/fixtures/validate-scenario.ts index d6b768b794d..0d2ac3e7698 100644 --- a/apps/sim/e2e/fixtures/validate-scenario.ts +++ b/apps/sim/e2e/fixtures/validate-scenario.ts @@ -11,7 +11,6 @@ import type { ScenarioUser, ScenarioWorkspace, ScenarioWorkspaceGrant, - WorkspaceAccess, } from './scenario' import { SCENARIO_VERSION } from './scenario' @@ -25,6 +24,9 @@ export class ScenarioValidationError extends Error { } } +const RESOURCE_KEY_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]*$/ +const STORAGE_STATE_FILENAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*\.json$/ + export function validateScenario(definition: ScenarioDefinition): ResolvedScenario { const issues: string[] = [] if (definition.version !== SCENARIO_VERSION) { @@ -152,6 +154,48 @@ export function validateScenario(definition: ScenarioDefinition): ResolvedScenar } } +export function validateScenarioSet(scenarios: readonly ResolvedScenario[]): void { + const issues: string[] = [] + validateUniqueValues( + 'world namespace', + scenarios.map(({ definition }) => definition.namespace.world), + issues + ) + validateUniqueValues( + 'world namespace prefix', + scenarios.map(({ definition }) => definition.namespace.prefix), + issues + ) + validateUniqueValues( + 'cross-world email', + scenarios.flatMap(({ definition }) => [ + ...definition.users.map(({ email }) => email.toLowerCase()), + ...definition.invitations.map(({ email }) => email.toLowerCase()), + ]), + issues + ) + validateUniqueValues( + 'cross-world organization slug', + scenarios.flatMap(({ definition }) => + definition.organizations.map(({ slug }) => slug.toLowerCase()) + ), + issues + ) + validateUniqueValues( + 'cross-world persona key', + scenarios.flatMap(({ definition }) => definition.personas.map(({ key }) => key)), + issues + ) + validateUniqueValues( + 'cross-world storage-state filename', + scenarios.flatMap(({ definition }) => + definition.personas.map(({ storageStateFilename }) => storageStateFilename) + ), + issues + ) + if (issues.length > 0) throw new ScenarioValidationError(issues) +} + function indexByKey( label: string, values: readonly T[], @@ -161,6 +205,8 @@ function indexByKey( for (const value of values) { if (!value.key.trim()) { issues.push(`${label} key must be non-empty`) + } else if (!RESOURCE_KEY_PATTERN.test(value.key)) { + issues.push(`${label} key "${value.key}" must be a safe identifier`) } else if (result.has(value.key)) { issues.push(`duplicate ${label} key "${value.key}"`) } else { @@ -216,6 +262,16 @@ function validateControllableIdentities(definition: ScenarioDefinition, issues: definition.personas.map(({ storageStateFilename }) => storageStateFilename), issues ) + for (const { storageStateFilename } of definition.personas) { + if ( + !STORAGE_STATE_FILENAME_PATTERN.test(storageStateFilename) || + storageStateFilename.includes('..') + ) { + issues.push( + `storage-state filename "${storageStateFilename}" must be a separator-free JSON basename` + ) + } + } } function validateUniqueValues(label: string, values: readonly string[], issues: string[]): void { @@ -331,6 +387,11 @@ function validateSubscriptions( if (subscription.plan.startsWith('team_') || subscription.plan === 'enterprise') { issues.push(`subscription "${subscription.key}" has an invalid plan for a user payer`) } + if (subscription.status === 'lapsed') { + issues.push( + `subscription "${subscription.key}" uses unsupported lapsed status for a user payer` + ) + } } else { const organizationKey = subscription.billingReference.organizationKey if (!organizationsByKey.has(organizationKey)) { @@ -363,7 +424,6 @@ function validateSubscriptions( if ( !metadata || metadata.plan !== 'enterprise' || - !metadata.referenceId || !Number.isFinite(metadata.monthlyPrice) || metadata.monthlyPrice <= 0 || !Number.isInteger(metadata.seats) || @@ -417,11 +477,15 @@ function validateWorkspaces( workspace.organizationKey, workspace.ownerUserKey ) + const organization = organizationsByKey.get(workspace.organizationKey) if ( !ownerMembership || - (ownerMembership.role !== 'owner' && ownerMembership.role !== 'admin') + ownerMembership.role !== 'owner' || + organization?.ownerUserKey !== workspace.ownerUserKey ) { - issues.push(`organization workspace "${workspace.key}" owner must be an owner/admin member`) + issues.push( + `organization workspace "${workspace.key}" creator must be the organization owner` + ) } if (!subscription) { issues.push(`organization workspace "${workspace.key}" must retain its subscription record`) @@ -431,6 +495,11 @@ function validateWorkspaces( ) { issues.push(`organization workspace "${workspace.key}" has an incoherent subscription`) } + if (subscription?.status === 'past_due') { + issues.push( + `organization workspace "${workspace.key}" cannot be provisioned from a past-due subscription` + ) + } } else { if (workspace.payer.kind !== 'user' || workspace.payer.userKey !== workspace.ownerUserKey) { issues.push(`personal workspace "${workspace.key}" has an incoherent payer/owner`) @@ -475,7 +544,6 @@ function validatePermissionGroups( workspacesByKey: ReadonlyMap, issues: string[] ): void { - const defaultOrganizations = new Set() for (const group of definition.permissionGroups) { if (!organizationsByKey.has(group.organizationKey)) { issues.push( @@ -485,6 +553,11 @@ function validatePermissionGroups( if (group.workspaceKeys.length === 0) { issues.push(`permission group "${group.key}" must have Enterprise workspace scope`) } + if (group.memberUserKeys.length === 0) { + issues.push( + `permission group "${group.key}" must name explicit members; default and all-member groups are not modeled` + ) + } validateUniqueValues( `permission group "${group.key}" workspace scope`, group.workspaceKeys, @@ -525,12 +598,6 @@ function validatePermissionGroups( issues.push(`permission group "${group.key}" member "${userKey}" lacks host membership`) } } - if (group.isDefault) { - if (defaultOrganizations.has(group.organizationKey)) { - issues.push(`organization "${group.organizationKey}" has duplicate default groups`) - } - defaultOrganizations.add(group.organizationKey) - } } } @@ -677,7 +744,3 @@ function billingReferenceKey(subscription: ScenarioSubscription): string { function sameSet(left: readonly string[], right: readonly string[]): boolean { return left.length === right.length && left.every((value) => right.includes(value)) } - -export function accessRank(access: WorkspaceAccess): number { - return { read: 1, write: 2, admin: 3 }[access] -} diff --git a/apps/sim/e2e/foundation/build-manifest.spec.ts b/apps/sim/e2e/foundation/build-manifest.spec.ts new file mode 100644 index 00000000000..b993bbd56cd --- /dev/null +++ b/apps/sim/e2e/foundation/build-manifest.spec.ts @@ -0,0 +1,174 @@ +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + utimesSync, + writeFileSync, +} from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { expect, test } from '@playwright/test' +import { + type BuildArtifactPaths, + type BuildIdentity, + clearActiveNextBuild, + pruneBuildCache, + restoreCachedBuild, + storeCompletedBuild, +} from '../support/build-manifest' +import { acquireE2eRunLock } from '../support/run-lock' + +const identity: BuildIdentity = { + schemaVersion: 1, + nextBuildHash: 'next-build-hash', + sourceHash: 'source-hash', + profileHash: 'profile-hash', + nodeVersion: 'v22.0.0', + bunVersion: '1.0.0', + nextVersion: '16.0.0', + platform: process.platform, + architecture: process.arch, +} + +test.describe('verified Next build cache', () => { + test('stores, restores, and rejects artifact corruption', () => { + withBuildPaths((paths) => { + writeBuild(paths.activeNextDirectory, 'build-one', 'original') + storeCompletedBuild(identity, paths) + clearActiveNextBuild(paths.activeNextDirectory) + const staleRestore = `${paths.activeNextDirectory}.e2e-restore-123` + const staleBackup = `${paths.activeNextDirectory}.e2e-backup-123` + const interruptedStore = path.join(paths.buildCacheDirectory, 'other-hash.tmp-123') + mkdirSync(staleRestore) + mkdirSync(staleBackup) + mkdirSync(interruptedStore) + + expect(restoreCachedBuild(identity, paths)).toMatchObject({ + reused: true, + reason: 'verified cache hit', + }) + expect(readFileSync(path.join(paths.activeNextDirectory, 'server.js'), 'utf8')).toBe( + 'original' + ) + expect(existsSync(staleRestore)).toBe(false) + expect(existsSync(staleBackup)).toBe(false) + expect(existsSync(interruptedStore)).toBe(false) + + writeFileSync( + path.join(paths.buildCacheDirectory, identity.nextBuildHash, '.next', 'server.js'), + 'tampered' + ) + expect(restoreCachedBuild(identity, paths)).toMatchObject({ + reused: false, + reason: 'cached artifact checksum does not match the manifest', + }) + }) + }) + + test('rejects manifest identity and BUILD_ID corruption', () => { + withBuildPaths((paths) => { + writeBuild(paths.activeNextDirectory, 'build-one', 'original') + storeCompletedBuild(identity, paths) + const cacheDirectory = path.join(paths.buildCacheDirectory, identity.nextBuildHash) + const manifestPath = path.join(cacheDirectory, 'manifest.json') + const manifest = JSON.parse(readFileSync(manifestPath, 'utf8')) as Record + writeFileSync(manifestPath, JSON.stringify({ ...manifest, sourceHash: 'wrong' })) + expect(restoreCachedBuild(identity, paths).reason).toBe( + 'cache manifest identity does not match' + ) + + writeFileSync(manifestPath, JSON.stringify(manifest)) + writeFileSync(path.join(cacheDirectory, '.next', 'BUILD_ID'), 'wrong') + expect(restoreCachedBuild(identity, paths).reason).toBe( + 'cached BUILD_ID does not match the manifest' + ) + }) + }) + + test('prunes old build entries while retaining the selected cache', () => { + withBuildPaths((paths) => { + const now = Date.now() / 1_000 + for (const [index, name] of ['oldest', 'newer', identity.nextBuildHash].entries()) { + const directory = path.join(paths.buildCacheDirectory, name) + mkdirSync(directory, { recursive: true }) + utimesSync(directory, now + index, now + index) + } + const abandonedStore = path.join(paths.buildCacheDirectory, 'abandoned.tmp-123') + mkdirSync(abandonedStore) + expect(pruneBuildCache(identity.nextBuildHash, 2, paths.buildCacheDirectory)).toEqual([ + 'oldest', + ]) + expect(existsSync(abandonedStore)).toBe(false) + expect(restoreCachedBuild(identity, paths).reason).toBe( + 'cache artifact or completed manifest is missing' + ) + }) + }) +}) + +test('orchestrator lock rejects live ownership and recovers stale descriptors', () => { + const directory = mkdtempSync(path.join(os.tmpdir(), 'sim-e2e-run-lock-')) + const lockPath = path.join(directory, 'orchestrator.lock') + try { + const lock = acquireE2eRunLock(lockPath) + expect(() => acquireE2eRunLock(lockPath)).toThrow(/Another E2E orchestrator/) + lock.transfer(process.pid) + expect(() => acquireE2eRunLock(lockPath)).toThrow(/Another E2E orchestrator/) + lock.retain('manual cleanup required') + lock.transfer(process.pid) + expect(() => acquireE2eRunLock(lockPath)).toThrow(/manual cleanup required/) + lock.release() + + mkdirSync(lockPath) + expect(() => acquireE2eRunLock(lockPath)).toThrow(/is acquiring/) + rmSync(lockPath, { recursive: true }) + + mkdirSync(lockPath) + writeFileSync( + path.join(lockPath, 'owner.json'), + JSON.stringify({ + pid: 2_147_483_647, + token: 'stale', + startedAt: '2000-01-01T00:00:00Z', + processStartIdentity: null, + }) + ) + const recovered = acquireE2eRunLock(lockPath) + recovered.release() + + mkdirSync(lockPath) + writeFileSync( + path.join(lockPath, 'owner.json'), + JSON.stringify({ + pid: process.pid, + token: 'reused-pid', + startedAt: '2000-01-01T00:00:00Z', + processStartIdentity: 'not-the-current-process-start', + }) + ) + const reusedPidRecovered = acquireE2eRunLock(lockPath) + reusedPidRecovered.release() + } finally { + rmSync(directory, { recursive: true, force: true }) + } +}) + +function withBuildPaths(run: (paths: BuildArtifactPaths) => void): void { + const directory = mkdtempSync(path.join(os.tmpdir(), 'sim-e2e-build-cache-')) + try { + run({ + activeNextDirectory: path.join(directory, '.next'), + buildCacheDirectory: path.join(directory, 'cache'), + }) + } finally { + rmSync(directory, { recursive: true, force: true }) + } +} + +function writeBuild(directory: string, buildId: string, contents: string): void { + mkdirSync(directory, { recursive: true }) + writeFileSync(path.join(directory, 'BUILD_ID'), buildId) + writeFileSync(path.join(directory, 'server.js'), contents) +} diff --git a/apps/sim/e2e/foundation/http-client.spec.ts b/apps/sim/e2e/foundation/http-client.spec.ts index 4dd36a01b4c..c9d2f04a704 100644 --- a/apps/sim/e2e/foundation/http-client.spec.ts +++ b/apps/sim/e2e/foundation/http-client.spec.ts @@ -34,6 +34,29 @@ test.describe('E2E HTTP client', () => { expect(attempts).toEqual([1, 2]) }) + test('clamps oversized Retry-After values', async () => { + const delays: number[] = [] + let requestCount = 0 + const client = new E2eHttpClient({ + baseUrl: 'http://127.0.0.1:1', + retryDelaysMs: [1], + fetchImplementation: async () => { + requestCount += 1 + return requestCount === 1 + ? new Response(JSON.stringify({ error: 'limited' }), { + status: 429, + headers: { 'content-type': 'application/json', 'retry-after': '3600' }, + }) + : Response.json({ ok: true }) + }, + sleepImplementation: async (milliseconds) => { + delays.push(milliseconds) + }, + }) + await client.request({ path: '/retry', schema: z.object({ ok: z.literal(true) }) }) + expect(delays).toEqual([30_000]) + }) + test('keeps independent cookie jars and redacts failed response bodies', async () => { const cookieHeaders: Array = [] const createFetch = diff --git a/apps/sim/e2e/foundation/leak-canary.spec.ts b/apps/sim/e2e/foundation/leak-canary.spec.ts new file mode 100644 index 00000000000..0a1014437ac --- /dev/null +++ b/apps/sim/e2e/foundation/leak-canary.spec.ts @@ -0,0 +1,84 @@ +import { existsSync, mkdtempSync, rmSync, unlinkSync, writeFileSync } from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { expect, test } from '@playwright/test' +import JSZip from 'jszip' +import { writeJsonAtomic } from '../fixtures/e2e-world' +import { + assertNoSyntheticSecretLeaks, + loadSyntheticSecretCanaryForScan, + readSyntheticSecretCanarySecrets, + scrubUnscannableArtifacts, + writeSyntheticSecretCanary, +} from '../support/leak-canary' + +test('credential leak canary scans artifacts but excludes its private source', async () => { + const directory = mkdtempSync(path.join(os.tmpdir(), 'sim-e2e-leak-canary-')) + const secretsPath = path.join(directory, 'private', 'synthetic-secrets.json') + const artifactsPath = path.join(directory, 'artifacts') + const password = 'known-synthetic-password' + try { + writeSyntheticSecretCanary(secretsPath, 'leak-canary', [password, 'setup-only-password']) + expect(loadSyntheticSecretCanaryForScan([], secretsPath)).toEqual([ + password, + 'setup-only-password', + ]) + expect(loadSyntheticSecretCanaryForScan(['already-loaded'], secretsPath)).toEqual([ + 'already-loaded', + password, + 'setup-only-password', + ]) + writeJsonAtomic(path.join(artifactsPath, 'clean.json'), { status: 'clean' }) + + await expect( + assertNoSyntheticSecretLeaks({ + secrets: readSyntheticSecretCanarySecrets(secretsPath), + roots: [directory], + excludedPaths: [path.dirname(secretsPath)], + }) + ).resolves.toBeUndefined() + + const tracePath = path.join(artifactsPath, 'trace.txt') + writeFileSync(tracePath, 'request body: setup-only-password') + await expect( + assertNoSyntheticSecretLeaks({ + secrets: readSyntheticSecretCanarySecrets(secretsPath), + roots: [directory], + excludedPaths: [path.dirname(secretsPath)], + }) + ).rejects.toThrow(/secret leaked/) + + writeFileSync(tracePath, `request body: ${password}`) + const zip = new JSZip() + zip.file('trace.txt', `request body: ${password}`) + writeFileSync( + path.join(artifactsPath, 'trace.zip'), + await zip.generateAsync({ type: 'nodebuffer' }) + ) + unlinkSync(tracePath) + await expect( + assertNoSyntheticSecretLeaks({ + secrets: readSyntheticSecretCanarySecrets(secretsPath), + roots: [directory], + excludedPaths: [path.dirname(secretsPath)], + }) + ).rejects.toThrow(/secret leaked/) + } finally { + rmSync(directory, { recursive: true, force: true }) + } +}) + +test('malformed canaries fail closed by scrubbing unscannable diagnostics', () => { + const directory = mkdtempSync(path.join(os.tmpdir(), 'sim-e2e-malformed-canary-')) + const secretsPath = path.join(directory, 'synthetic-secrets.json') + const diagnosticsPath = path.join(directory, 'diagnostics') + try { + writeFileSync(secretsPath, '{"schemaVersion":1,"secrets":["truncated') + writeJsonAtomic(path.join(diagnosticsPath, 'report.json'), { status: 'unscanned' }) + expect(() => loadSyntheticSecretCanaryForScan(['runtime-secret'], secretsPath)).toThrow() + scrubUnscannableArtifacts([diagnosticsPath]) + expect(existsSync(diagnosticsPath)).toBe(false) + } finally { + rmSync(directory, { recursive: true, force: true }) + } +}) diff --git a/apps/sim/e2e/foundation/safety.spec.ts b/apps/sim/e2e/foundation/safety.spec.ts index 5e0ad40e696..a971f624d88 100644 --- a/apps/sim/e2e/foundation/safety.spec.ts +++ b/apps/sim/e2e/foundation/safety.spec.ts @@ -5,7 +5,6 @@ import path from 'node:path' import { loadEnvConfig } from '@next/env' import { expect, test } from '@playwright/test' import { parseRunOptions } from '../scripts/options' -import { classifyE2eHashOwner } from '../support/build-manifest' import { assertLoopbackPostgresUrl, assertSafeDatabaseName, @@ -19,8 +18,10 @@ import { spawnManagedProcess, waitForManagedProcessReady, } from '../support/process' +import { createE2eRuntimeSecrets } from '../support/runtime-secrets' import { verifySandboxBundleIntegrity } from '../support/sandbox-bundles' -import { parseProcessGroupIds } from '../support/signal-cleanup' +import { assertSafeSeedEnvironment } from '../support/seed-safety' +import { isProcessGroupAlive, parseProcessGroupIds } from '../support/signal-cleanup' test.describe('foundation safety guards', () => { test('discovers env keys without leaking values and shadows unknown keys', () => { @@ -68,10 +69,14 @@ test.describe('foundation safety guards', () => { test('deployment profile projects least-privilege build and runtime environments', () => { const profile = createHostedBillingProfile({ runId: 'projection_test', - databaseUrl: 'postgresql://postgres:postgres@127.0.0.1:5432/sim_e2e_projection_test', + databaseUrl: 'postgresql://127.0.0.1:5432/sim_e2e_projection_test', stripeApiBaseUrl: 'http://127.0.0.1:40123', - homeDirectory: path.join(os.tmpdir(), 'sim-e2e-projection'), + runtimeHomeDirectory: path.join(os.tmpdir(), 'sim-e2e-projection-runtime'), + setupHomeDirectory: path.join(os.tmpdir(), 'sim-e2e-projection-setup'), + authCaptureHomeDirectory: path.join(os.tmpdir(), 'sim-e2e-projection-auth'), + playwrightHomeDirectory: path.join(os.tmpdir(), 'sim-e2e-projection-playwright'), playwrightBrowsersPath: path.join(os.tmpdir(), 'sim-e2e-browsers'), + runtimeSecrets: createE2eRuntimeSecrets(), ci: false, }) const { build, app, realtime, migration, seed, authCapture, playwright } = profile.environments @@ -82,6 +87,17 @@ test.describe('foundation safety guards', () => { expect(build.env.TELEMETRY_ENDPOINT).toBe('http://127.0.0.1:1/v1/traces') expect(app.env.TELEMETRY_ENDPOINT).toMatch(/^http:\/\/127\.0\.0\.1:\d+\/v1\/traces$/) expect(build.env.E2E_RUN_ID).toBe('build_sentinel') + for (const key of [ + 'BETTER_AUTH_SECRET', + 'ENCRYPTION_KEY', + 'API_ENCRYPTION_KEY', + 'INTERNAL_API_SECRET', + 'ADMIN_API_KEY', + 'STRIPE_SECRET_KEY', + 'STRIPE_WEBHOOK_SECRET', + ]) { + expect(build.env[key], `${key} must use a build-only sentinel`).not.toBe(app.env[key]) + } for (const key of Object.keys(app.env).filter((key) => key.startsWith('NEXT_PUBLIC_'))) { expect(build.env[key], `${key} must be identical at build and runtime`).toBe(app.env[key]) @@ -97,28 +113,50 @@ test.describe('foundation safety guards', () => { expect(realtime.env.STRIPE_SECRET_KEY).toBeUndefined() expect(migration.env.ADMIN_API_KEY).toBeUndefined() expect(migration.env.MIGRATION_DATABASE_URL).toBe(app.env.DATABASE_URL) + expect(app.env.HOME).not.toBe(seed.env.HOME) + expect(seed.env.HOME).not.toBe(authCapture.env.HOME) + expect(authCapture.env.HOME).not.toBe(playwright.env.HOME) }) test('database guards reject shared or remote targets', () => { expect(() => assertSafeDatabaseName('simstudio')).toThrow() expect(() => assertSafeDatabaseName('sim_e2e_valid_run')).not.toThrow() + expect(() => assertLoopbackPostgresUrl('postgresql://example.com/postgres')).toThrow() + expect(() => assertLoopbackPostgresUrl('postgresql://localhost/postgres')).toThrow() expect(() => - assertLoopbackPostgresUrl('postgresql://postgres:postgres@example.com/postgres') - ).toThrow() - expect(() => - assertLoopbackPostgresUrl('postgresql://postgres:postgres@localhost/postgres') - ).toThrow() - expect(() => - assertLoopbackPostgresUrl('postgresql://postgres:postgres@127.0.0.1/postgres?sslmode=disable') + assertLoopbackPostgresUrl('postgresql://127.0.0.1/postgres?sslmode=disable') ).toThrow() expect( - buildRunDatabaseUrl( - 'postgresql://postgres:postgres@127.0.0.1:5432/postgres', - 'sim_e2e_valid_run' - ) + buildRunDatabaseUrl('postgresql://127.0.0.1:5432/postgres', 'sim_e2e_valid_run') ).toContain('/sim_e2e_valid_run') }) + test('standalone seeding requires its exact guarded run target', () => { + const environment = { + E2E_ORCHESTRATED: '1', + E2E_PROFILE: 'hosted-billing-chromium', + E2E_RUN_ID: 'seed_guard', + E2E_BASE_URL: 'http://e2e.sim.ai:3000', + DATABASE_URL: 'postgresql://127.0.0.1:5432/sim_e2e_seed_guard', + } + expect(() => assertSafeSeedEnvironment(environment)).not.toThrow() + expect(() => assertSafeSeedEnvironment({ ...environment, E2E_ORCHESTRATED: '0' })).toThrow( + /guarded E2E orchestrator/ + ) + expect(() => + assertSafeSeedEnvironment({ + ...environment, + DATABASE_URL: 'postgresql://127.0.0.1:5432/simstudio', + }) + ).toThrow(/unsafe E2E database/) + expect(() => + assertSafeSeedEnvironment({ + ...environment, + DATABASE_URL: 'postgresql://192.0.2.1:5432/sim_e2e_seed_guard', + }) + ).toThrow(/must be loopback/) + }) + test('loopback detection rejects public addresses', () => { expect(isLoopbackAddress('127.0.0.1')).toBe(true) expect(isLoopbackAddress('127.10.20.30')).toBe(true) @@ -137,9 +175,21 @@ test.describe('foundation safety guards', () => { expect(() => parseRunOptions(['--project=hosted-billing-chromium-workflows', '--shard=1/2']) ).toThrow(/coupled E2E projects must remain unsharded/) + expect(() => parseRunOptions(['--project=hosted-billing-chromium-personas'])).not.toThrow() expect(() => - parseRunOptions(['--project=hosted-billing-chromium-personas']) + parseRunOptions(['--project=hosted-billing-chromium-personas', '--no-deps']) ).not.toThrow() + expect(() => + parseRunOptions(['--project=hosted-billing-chromium-personas', '--no-deps'], { ci: true }) + ).toThrow(/--no-deps is local-only/) + expect(() => parseRunOptions(['--no-deps'])).toThrow(/exactly one explicit canonical/) + expect(() => + parseRunOptions([ + '--project=hosted-billing-chromium-navigation', + '--project=hosted-billing-chromium-personas', + '--no-deps', + ]) + ).toThrow(/exactly one explicit canonical/) expect(() => parseRunOptions(['--project=hosted-billing-chromium-persona-isolation', '--shard=1/2']) ).toThrow(/coupled E2E projects must remain unsharded/) @@ -147,11 +197,21 @@ test.describe('foundation safety guards', () => { test('Playwright CLI arguments cannot override orchestration invariants', () => { expect(() => parseRunOptions(['--workers=8'])).toThrow(/cannot override/) + expect(() => parseRunOptions(['-j8'])).toThrow(/cannot override/) expect(() => parseRunOptions(['--config=other.config.ts'])).toThrow(/cannot override/) + expect(() => parseRunOptions(['-cother.config.ts'])).toThrow(/cannot override/) expect(() => parseRunOptions(['--project', 'hosted-billing-chromium-navigation'])).toThrow( /canonical/ ) expect(() => parseRunOptions(['--pass-with-no-tests'])).toThrow(/cannot override/) + expect(() => parseRunOptions(['--output=/tmp/elsewhere'])).toThrow(/cannot override/) + expect(() => parseRunOptions(['--reporter=line'])).toThrow(/cannot override/) + expect(() => parseRunOptions(['--update-snapshots'])).toThrow(/not a supported/) + expect(parseRunOptions(['--grep', 'persona contract', '--headed']).playwrightArgs).toEqual([ + '--grep', + 'persona contract', + '--headed', + ]) expect(parseRunOptions(['--reuse-build'], { ci: false })).toEqual({ playwrightArgs: [], reuseBuild: true, @@ -160,18 +220,11 @@ test.describe('foundation safety guards', () => { expect(() => parseRunOptions(['--keep-stack'], { ci: false })).toThrow(/deferred/) }) - test('hash ownership follows execution ownership', () => { - expect(classifyE2eHashOwner('apps/sim/e2e/settings/navigation/routes.spec.ts')).toBe('rerun') - expect(classifyE2eHashOwner('apps/sim/e2e/fixtures/scenario.ts')).toBe('scenario') - expect(classifyE2eHashOwner('apps/sim/e2e/scripts/seed-world.ts')).toBe('scenario') - expect(classifyE2eHashOwner('apps/sim/e2e/fakes/stripe/server.ts')).toBe('retained-stack') - expect(classifyE2eHashOwner('apps/sim/e2e/support/readiness.ts')).toBe('retained-stack') - expect(classifyE2eHashOwner('apps/realtime/src/index.ts')).toBe('retained-stack') - expect(classifyE2eHashOwner('apps/sim/app/page.tsx')).toBe('next-build') - }) - test('committed sandbox bundles match the reviewed fingerprint', () => { - expect(() => verifySandboxBundleIntegrity()).not.toThrow() + expect(() => verifySandboxBundleIntegrity({ runningBunVersion: '1.3.13' })).not.toThrow() + expect(() => verifySandboxBundleIntegrity({ runningBunVersion: '1.3.14' })).toThrow( + /require Bun 1\.3\.13/ + ) }) test('empty signal cleanup groups never become PID zero', () => { @@ -213,6 +266,32 @@ test.describe('foundation safety guards', () => { } }) + test('cleanup retains and terminates descendants after their group leader exits', async () => { + test.skip(process.platform === 'win32', 'Windows does not expose POSIX process groups') + const logsDirectory = mkdtempSync(path.join(os.tmpdir(), 'sim-e2e-descendant-')) + try { + const managed = spawnManagedProcess({ + name: 'exited-group-leader', + command: process.execPath, + args: [ + '-e', + "const { spawn } = require('node:child_process'); const child = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: 'ignore' }); child.unref();", + ], + cwd: logsDirectory, + env: { NODE_ENV: 'test', PATH: process.env.PATH ?? '' }, + logsDirectory, + }) + const groupId = managed.child.pid + expect(groupId).toBeTruthy() + await managed.completion + expect(isProcessGroupAlive(groupId ?? 0)).toBe(true) + await managed.stop() + expect(isProcessGroupAlive(groupId ?? 0)).toBe(false) + } finally { + rmSync(logsDirectory, { recursive: true, force: true }) + } + }) + test('process exit after readiness does not create an unhandled rejection', async () => { const logsDirectory = mkdtempSync(path.join(os.tmpdir(), 'sim-e2e-ready-')) let unhandled: unknown diff --git a/apps/sim/e2e/foundation/scenario-validation.spec.ts b/apps/sim/e2e/foundation/scenario-validation.spec.ts index a6c366cd403..019a4a7f059 100644 --- a/apps/sim/e2e/foundation/scenario-validation.spec.ts +++ b/apps/sim/e2e/foundation/scenario-validation.spec.ts @@ -1,13 +1,17 @@ import { expect, test } from '@playwright/test' +import { resolveStorageStatePath } from '../fixtures/e2e-world' import { createScenarioNamespace } from '../fixtures/namespace' import { canonicalSettingsRoute, type ScenarioDefinition, type ScenarioOrganizationMembership, - type ScenarioPermissionGroup, type ScenarioWorkspaceGrant, } from '../fixtures/scenario' -import { ScenarioValidationError, validateScenario } from '../fixtures/validate-scenario' +import { + ScenarioValidationError, + validateScenario, + validateScenarioSet, +} from '../fixtures/validate-scenario' import { createPrimarySettingsScenario, createSettingsPersonaScenarios, @@ -24,6 +28,7 @@ test.describe('pure scenario validation', () => { expect(twin.personasByKey.size).toBe(1) expect(twin.workspacesByKey.size).toBe(1) expect(twin.organizationsByKey.size).toBe(0) + expect(() => validateScenarioSet([primary, twin])).not.toThrow() expect(workspaceExpectation(primary, 'personalPaidOwner').hostContext.plan).toBe('pro_6000') expect(workspaceExpectation(primary, 'personalMaxOwner').hostContext.plan).toBe('pro_25000') @@ -81,6 +86,26 @@ test.describe('pure scenario validation', () => { }) }) + test('rejects unsafe storage paths and cross-world identity collisions', () => { + const unsafe = validScenario() + unsafe.personas = unsafe.personas.map((persona, index) => + index === 0 ? { ...persona, storageStateFilename: '../../escaped.json' } : persona + ) + expectInvalid(unsafe, /separator-free JSON basename/) + expect(() => resolveStorageStatePath('/tmp/e2e-auth', '../../escaped.json')).toThrow() + + const definitions = createSettingsPersonaScenarios('cross-world') + const primary = validateScenario(definitions.primary) + const twin = validateScenario({ + ...definitions.isolationTwin, + namespace: { + ...definitions.isolationTwin.namespace, + world: definitions.primary.namespace.world, + }, + }) + expect(() => validateScenarioSet([primary, twin])).toThrow(/duplicate world namespace/) + }) + test('namespaces controllable values deterministically without manufacturing API IDs', () => { const first = createScenarioNamespace('Run 42/Alpha', 'Primary World') const again = createScenarioNamespace('Run 42/Alpha', 'Primary World') @@ -159,7 +184,6 @@ test.describe('pure scenario validation', () => { ...subscription, enterprise: { plan: 'enterprise', - referenceId: 'wrong-plan', monthlyPrice: 1, seats: 4, }, @@ -169,6 +193,26 @@ test.describe('pure scenario validation', () => { expectInvalid(metadataOnTeam, /non-Enterprise subscription/) }) + test('rejects unsupported lapsed personal subscriptions before seeding', () => { + const scenario = validScenario() + scenario.subscriptions = scenario.subscriptions.map((subscription) => + subscription.key === 'personal-paid-subscription' + ? { ...subscription, status: 'lapsed' as const } + : subscription + ) + expectInvalid(scenario, /unsupported lapsed status for a user payer/) + }) + + test('rejects organization workspace provisioning from past-due subscriptions', () => { + const scenario = validScenario() + scenario.subscriptions = scenario.subscriptions.map((subscription) => + subscription.key === 'team-subscription' + ? { ...subscription, status: 'past_due' as const } + : subscription + ) + expectInvalid(scenario, /cannot be provisioned from a past-due subscription/) + }) + test('rejects permission groups without active Enterprise organization/workspace scope', () => { const scenario = validScenario() scenario.permissionGroups = scenario.permissionGroups.map((group) => ({ @@ -179,7 +223,7 @@ test.describe('pure scenario validation', () => { expectInvalid(scenario, /requires an active Enterprise organization/) }) - test('rejects duplicate explicit grants and default groups', () => { + test('rejects duplicate explicit grants and unsupported all-member groups', () => { const duplicateGrant = validScenario() duplicateGrant.workspaceGrants = [ ...duplicateGrant.workspaceGrants, @@ -187,18 +231,12 @@ test.describe('pure scenario validation', () => { ] as ScenarioWorkspaceGrant[] expectInvalid(duplicateGrant, /duplicate workspace grant/) - const duplicateDefault = validScenario() - const existing = duplicateDefault.permissionGroups[0] - duplicateDefault.permissionGroups = [ - { ...existing, isDefault: true }, - { - ...existing, - key: 'second-default-enterprise-group', - name: `${existing.name} Second`, - isDefault: true, - }, - ] as ScenarioPermissionGroup[] - expectInvalid(duplicateDefault, /duplicate default groups/) + const allMemberGroup = validScenario() + allMemberGroup.permissionGroups = allMemberGroup.permissionGroups.map((group) => ({ + ...group, + memberUserKeys: [], + })) + expectInvalid(allMemberGroup, /default and all-member groups are not modeled/) }) test('rejects inconsistent hosted and billing flags', () => { diff --git a/apps/sim/e2e/foundation/stripe-fake.spec.ts b/apps/sim/e2e/foundation/stripe-fake.spec.ts index ff5e75342bf..8a0b839164b 100644 --- a/apps/sim/e2e/foundation/stripe-fake.spec.ts +++ b/apps/sim/e2e/foundation/stripe-fake.spec.ts @@ -44,8 +44,7 @@ test('Stripe fake records allowlisted calls and rejects unknown routes', async ( expect(telemetry.status).toBe(200) expect( fake.requestLog.some( - ({ method, path, unexpected }) => - method === 'POST' && path === '/v1/traces' && !unexpected + ({ method, path, unexpected }) => method === 'POST' && path === '/v1/traces' && !unexpected ) ).toBe(true) diff --git a/apps/sim/e2e/scripts/capture-auth-states.ts b/apps/sim/e2e/scripts/capture-auth-states.ts index ef212a8b8d4..da1b773921a 100644 --- a/apps/sim/e2e/scripts/capture-auth-states.ts +++ b/apps/sim/e2e/scripts/capture-auth-states.ts @@ -1,9 +1,11 @@ -import { mkdirSync } from 'node:fs' +import { chmodSync, mkdirSync, statSync } from 'node:fs' import path from 'node:path' import { chromium, expect } from '@playwright/test' +import { sleep } from '@sim/utils/helpers' import { readPersonaCredentials, readScenarioManifest, + resolveStorageStatePath, scenarioManifestSchema, writeJsonAtomic, } from '../fixtures/e2e-world' @@ -32,8 +34,8 @@ async function main(): Promise { const login = credentials.personas[personaKey] if (!login) throw new Error(`Missing private login for persona: ${personaKey}`) const context = await browser.newContext({ baseURL: baseUrl }) + const page = await context.newPage() try { - const page = await context.newPage() await page.goto('/login') await expect(page.getByRole('heading', { name: 'Sign in' })).toBeVisible() await page.getByLabel('Email').fill(login.email) @@ -56,16 +58,53 @@ async function main(): Promise { ) { throw new Error(`Active organization mismatch for persona: ${personaKey}`) } + // Deliberately duplicated by persona contracts: capture must reject lazy workspace creation + // before persisting auth, while the later contract independently verifies the saved session. + const workspaceResponse = await context.request.get('/api/workspaces?scope=all') + if (!workspaceResponse.ok()) { + throw new Error( + `Workspace inventory failed for ${personaKey}: ${workspaceResponse.status()}` + ) + } + const workspacePayload = (await workspaceResponse.json()) as { + workspaces?: Array<{ id?: string }> + } + const actualWorkspaceIds = (workspacePayload.workspaces ?? []) + .map(({ id }) => id) + .filter((id): id is string => Boolean(id)) + .sort() + const expectedWorkspaceIds = persona.workspaces + .filter(({ access }) => access !== 'none') + .map(({ workspaceId }) => workspaceId) + .sort() + if (JSON.stringify(actualWorkspaceIds) !== JSON.stringify(expectedWorkspaceIds)) { + throw new Error(`Workspace inventory mismatch for persona: ${personaKey}`) + } await page.goto(persona.canonicalRoute) - await expect(page).toHaveURL(/\/settings\/general$/) + await expect(page).toHaveURL(new URL(persona.canonicalRoute, baseUrl).toString()) await expect(page.getByRole('heading', { name: 'General', level: 1 })).toBeVisible() - await page.screenshot({ - path: path.join(screenshotsDirectory, `${personaKey}.png`), - fullPage: false, - }) - const storageStatePath = path.join(storageStateDirectory, persona.storageStatePath) + const storageStatePath = resolveStorageStatePath( + storageStateDirectory, + persona.storageStatePath + ) await context.storageState({ path: storageStatePath }) + chmodSync(storageStatePath, 0o600) + if ((statSync(storageStatePath).mode & 0o777) !== 0o600) { + throw new Error(`Storage state permissions are not private for persona: ${personaKey}`) + } + } catch (error) { + await page + .getByRole('textbox', { name: 'Password' }) + .fill('') + .catch(() => {}) + await page + .screenshot({ + path: path.join(screenshotsDirectory, `${personaKey}.failure.png`), + fullPage: true, + }) + .catch(() => {}) + throw error } finally { await context.close() } @@ -108,7 +147,7 @@ async function signInThroughUi( `UI sign-in failed for persona ${personaKey} with status ${response.status()}; response body redacted` ) } - await new Promise((resolve) => setTimeout(resolve, UI_RETRY_DELAYS_MS[attempt])) + await sleep(UI_RETRY_DELAYS_MS[attempt]) } } diff --git a/apps/sim/e2e/scripts/options.ts b/apps/sim/e2e/scripts/options.ts index e87fe6fb940..ae4dc26150d 100644 --- a/apps/sim/e2e/scripts/options.ts +++ b/apps/sim/e2e/scripts/options.ts @@ -17,7 +17,16 @@ const FORBIDDEN_OPTIONS = [ '--fully-parallel', '--pass-with-no-tests', '--list', + '--output', + '--output-dir', + '--reporter', + '--trace', + '--timeout', + '--debug', + '--ui', ] as const +const SAFE_BOOLEAN_OPTIONS = new Set(['--no-deps', '--headed', '--quiet']) +const SAFE_VALUE_OPTIONS = new Set(['--grep', '--grep-invert', '-g']) export interface E2eRunOptions { playwrightArgs: string[] @@ -46,6 +55,12 @@ export function parseRunOptions( if (reuseBuild && environment.ci) { throw new Error('--reuse-build is local-only; CI must run a fresh one-shot build') } + if (hasOption(normalizedArgs, '--no-deps') && environment.ci) { + throw new Error('--no-deps is local-only; CI must run the complete project dependency chain') + } + if (normalizedArgs.some((arg) => arg.startsWith('--no-deps='))) { + throw new Error('Use --no-deps without a value') + } for (const option of FORBIDDEN_OPTIONS) { if (hasOption(normalizedArgs, option)) { throw new Error(`${option} cannot override E2E orchestration invariants`) @@ -58,9 +73,13 @@ export function parseRunOptions( throw new Error('Use canonical --shard= syntax') } const playwrightArgs = normalizedArgs.filter((arg) => arg !== '--reuse-build') + assertSupportedPlaywrightArgs(playwrightArgs) const projects = getEqualsOptionValues(playwrightArgs, '--project') const unknownProject = projects.find((project) => !E2E_PROJECTS.has(project)) if (unknownProject) throw new Error(`Unknown E2E Playwright project: ${unknownProject}`) + if (normalizedArgs.includes('--no-deps') && projects.length !== 1) { + throw new Error('--no-deps requires exactly one explicit canonical --project=') + } const hasShard = hasOption(playwrightArgs, '--shard') if ( @@ -77,7 +96,31 @@ export function parseRunOptions( return { playwrightArgs, reuseBuild } } +function assertSupportedPlaywrightArgs(args: string[]): void { + for (let index = 0; index < args.length; index += 1) { + const argument = args[index] + if (!argument.startsWith('-')) continue + if (SAFE_BOOLEAN_OPTIONS.has(argument)) continue + if (argument.startsWith('--project=') || argument.startsWith('--shard=')) continue + const equalsName = argument.includes('=') ? argument.slice(0, argument.indexOf('=')) : argument + if (SAFE_VALUE_OPTIONS.has(equalsName) && argument.includes('=')) continue + if (SAFE_VALUE_OPTIONS.has(argument)) { + if (args[index + 1] === undefined) { + throw new Error(`${argument} requires a value`) + } + index += 1 + continue + } + throw new Error( + `${argument} is not a supported E2E Playwright option; artifact and orchestration overrides are denied` + ) + } +} + function hasOption(args: string[], name: string): boolean { + if (name.startsWith('-') && !name.startsWith('--')) { + return args.some((arg) => arg === name || arg.startsWith(name)) + } return args.some((arg) => arg === name || arg.startsWith(`${name}=`)) } diff --git a/apps/sim/e2e/scripts/run.ts b/apps/sim/e2e/scripts/run.ts index 8dfafc87f44..c6b6924934c 100644 --- a/apps/sim/e2e/scripts/run.ts +++ b/apps/sim/e2e/scripts/run.ts @@ -25,9 +25,15 @@ import { } from '../support/diagnostics' import { formatRedactedEnvironmentSummary } from '../support/env' import { assertE2eHostResolvesToLoopback } from '../support/hosts' +import { + assertNoSyntheticSecretLeaks, + loadSyntheticSecretCanaryForScan, + scrubUnscannableArtifacts, +} from '../support/leak-canary' import { getRunDirectory, SIM_APP_DIR } from '../support/paths' import { assertAdminApiBoundary, + assertManifestWorkspaceIdentities, type FoundationProvisioningResult, inspectFoundationUsers, } from '../support/probes' @@ -37,6 +43,12 @@ import { type ManagedProcess, stopAllManagedProcesses, } from '../support/process' +import { acquireE2eRunLock } from '../support/run-lock' +import { + createE2eRuntimeSecrets, + FOUNDATION_TEST_PASSWORD, + runtimeSecretValues, +} from '../support/runtime-secrets' import { buildApp, capturePersonaAuthStates, @@ -49,7 +61,6 @@ import { import { parseRunOptions } from './options' const DEFAULT_ADMIN_DATABASE_URL = 'postgresql://postgres:postgres@127.0.0.1:5432/postgres' -const STRIPE_TEST_KEY = 'sk_test_sim_e2e_foundation' async function main(): Promise { const options = parseRunOptions(process.argv.slice(2)) @@ -58,18 +69,39 @@ async function main(): Promise { const logsDirectory = path.join(runDirectory, 'logs') const storageStateDirectory = path.join(runDirectory, 'auth') const markerDirectory = path.join(runDirectory, 'markers') - const homeDirectory = path.join(runDirectory, 'home') + const privateDirectory = path.join(runDirectory, 'private') + const homesDirectory = path.join(runDirectory, 'homes') + const runtimeHomeDirectory = path.join(homesDirectory, 'runtime') + const setupHomeDirectory = path.join(homesDirectory, 'setup') + const authCaptureHomeDirectory = path.join(homesDirectory, 'auth-capture') + const playwrightHomeDirectory = path.join(homesDirectory, 'playwright') const manifestPath = path.join(runDirectory, 'persona-manifest.json') - const credentialsPath = path.join(homeDirectory, 'persona-credentials.json') + const credentialsPath = path.join(privateDirectory, 'persona-credentials.json') + const canarySecretsPath = path.join(privateDirectory, 'synthetic-secrets.json') const authScreenshotsDirectory = path.join(runDirectory, 'auth-capture-screenshots') + const diagnosticRoots = [ + runDirectory, + path.join(SIM_APP_DIR, 'playwright-report'), + path.join(SIM_APP_DIR, 'test-results'), + ] mkdirSync(logsDirectory, { recursive: true }) - mkdirSync(storageStateDirectory, { recursive: true }) + mkdirSync(storageStateDirectory, { recursive: true, mode: 0o700 }) mkdirSync(markerDirectory, { recursive: true }) - mkdirSync(homeDirectory, { recursive: true }) + mkdirSync(privateDirectory, { recursive: true, mode: 0o700 }) + for (const directory of [ + runtimeHomeDirectory, + setupHomeDirectory, + authCaptureHomeDirectory, + playwrightHomeDirectory, + ]) { + mkdirSync(directory, { recursive: true, mode: 0o700 }) + } const nodeExecutable = resolveNode22() const bunExecutable = resolveBunExecutable() const adminDatabaseUrl = process.env.E2E_PG_ADMIN_URL ?? DEFAULT_ADMIN_DATABASE_URL + const runtimeSecrets = createE2eRuntimeSecrets() + const runLock = acquireE2eRunLock() let runDatabase: RunDatabase | null = null let databaseCreationComplete = false @@ -77,6 +109,12 @@ async function main(): Promise { let realtime: ManagedProcess | null = null let app: ManagedProcess | null = null let cleanupPromise: Promise | null = null + let leakCanarySecrets: string[] = [ + FOUNDATION_TEST_PASSWORD, + ...runtimeSecretValues(runtimeSecrets), + ] + let canaryCoverageComplete = true + let diagnosticsRetained = true let failed = false const cleanup = (): Promise => { @@ -105,9 +143,12 @@ async function main(): Promise { } } - for (const sensitiveDirectory of [storageStateDirectory, homeDirectory]) { + for (const sensitiveDirectory of [storageStateDirectory, privateDirectory, homesDirectory]) { try { rmSync(sensitiveDirectory, { recursive: true, force: true }) + if (existsSync(sensitiveDirectory)) { + throw new Error(`Sensitive directory still exists: ${sensitiveDirectory}`) + } } catch (error) { failures.push(error) } @@ -139,6 +180,7 @@ async function main(): Promise { ) } catch {} } + let lockTransferred = false if (runDatabase) { const cleanupLogFd = openSync(path.join(logsDirectory, 'signal-cleanup.log'), 'a') const cleanupProcess = spawn( @@ -150,20 +192,31 @@ async function main(): Promise { env: { NODE_ENV: 'test', PATH: process.env.PATH ?? '', - HOME: homeDirectory, + HOME: setupHomeDirectory, E2E_PG_ADMIN_URL: adminDatabaseUrl, E2E_DATABASE_NAME: runDatabase.name, E2E_DATABASE_CREATION_COMPLETE: String(databaseCreationComplete), E2E_CLEANUP_PROCESS_GROUPS: getActiveManagedProcessGroupIds().join(','), - E2E_CLEANUP_DIRECTORIES: JSON.stringify([storageStateDirectory, homeDirectory]), + E2E_CLEANUP_DIRECTORIES: JSON.stringify([ + storageStateDirectory, + privateDirectory, + homesDirectory, + ]), + E2E_RUN_LOCK_PATH: runLock.path, + E2E_RUN_LOCK_TOKEN: runLock.token, }, stdio: ['ignore', cleanupLogFd, cleanupLogFd], } ) + if (cleanupProcess.pid) { + runLock.transfer(cleanupProcess.pid) + lockTransferred = true + } cleanupProcess.unref() closeSync(cleanupLogFd) console.error(`Detached E2E cleanup supervisor started as PID ${cleanupProcess.pid}`) } + if (!lockTransferred) runLock.release() process.exit(exitCode) } // `once` is intentional: a second signal uses the OS default force termination. @@ -183,7 +236,7 @@ async function main(): Promise { await createRunDatabase(adminDatabaseUrl, runDatabaseName) databaseCreationComplete = true stripeFake = await startStripeFakeServer({ - apiKey: STRIPE_TEST_KEY, + apiKey: runtimeSecrets.stripeSecretKey, hostname: '127.0.0.1', port: 0, }) @@ -193,8 +246,12 @@ async function main(): Promise { runId, databaseUrl: runDatabase.url, stripeApiBaseUrl: stripeFake.baseUrl, - homeDirectory, + runtimeHomeDirectory, + setupHomeDirectory, + authCaptureHomeDirectory, + playwrightHomeDirectory, playwrightBrowsersPath: resolvePlaywrightBrowsersPath(), + runtimeSecrets, ci: process.env.CI === 'true', }) for (const [name, environment] of Object.entries(profile.environments)) { @@ -213,9 +270,9 @@ async function main(): Promise { }) await buildApp({ ...commandOptions, - env: profile.environments.app.env, buildEnvironment: profile.environments.build, reuseBuild: options.reuseBuild, + ci: process.env.CI === 'true', }) realtime = await startRealtime({ ...commandOptions, @@ -235,8 +292,10 @@ async function main(): Promise { ...commandOptions, env: { ...profile.environments.seed.env, + E2E_ORCHESTRATED: '1', E2E_MANIFEST_PATH: manifestPath, E2E_CREDENTIALS_PATH: credentialsPath, + E2E_CANARY_SECRETS_PATH: canarySecretsPath, }, }) await capturePersonaAuthStates({ @@ -249,6 +308,9 @@ async function main(): Promise { E2E_AUTH_SCREENSHOT_DIR: authScreenshotsDirectory, }, }) + leakCanarySecrets = loadSyntheticSecretCanaryForScan(leakCanarySecrets, canarySecretsPath) + rmSync(credentialsPath, { force: true }) + rmSync(canarySecretsPath, { force: true }) const playwrightEnvironment = createPlaywrightEnvironment( profile.environments.playwright.env, @@ -276,25 +338,81 @@ async function main(): Promise { console.error(error) process.exitCode = 1 } finally { + if (runDatabase && existsSync(manifestPath)) { + try { + await assertManifestWorkspaceIdentities(runDatabase.url, manifestPath) + } catch (error) { + failed = true + process.exitCode = 1 + console.error(error) + } + } try { - assertNoForbiddenProviderTraffic( - [app?.logPath, realtime?.logPath].filter((value): value is string => Boolean(value)) - ) + leakCanarySecrets = loadSyntheticSecretCanaryForScan(leakCanarySecrets, canarySecretsPath) } catch (error) { + canaryCoverageComplete = false failed = true process.exitCode = 1 console.error(error) + } finally { + rmSync(credentialsPath, { force: true }) + rmSync(canarySecretsPath, { force: true }) } + let cleanupSucceeded = false try { await cleanup() + cleanupSucceeded = true } catch (error) { failed = true process.exitCode = 1 console.error(error) } + try { + assertNoForbiddenProviderTraffic( + [app?.logPath, realtime?.logPath].filter((value): value is string => Boolean(value)) + ) + } catch (error) { + failed = true + process.exitCode = 1 + console.error(error) + } + if (canaryCoverageComplete && leakCanarySecrets.length > 0) { + try { + await assertNoSyntheticSecretLeaks({ + secrets: leakCanarySecrets, + roots: diagnosticRoots, + excludedPaths: [privateDirectory, storageStateDirectory], + }) + } catch (error) { + failed = true + process.exitCode = 1 + console.error(error) + diagnosticsRetained = scrubDiagnostics(diagnosticRoots) + if (diagnosticsRetained) cleanupSucceeded = false + } + } else if (!canaryCoverageComplete) { + diagnosticsRetained = scrubDiagnostics(diagnosticRoots) + if (diagnosticsRetained) cleanupSucceeded = false + } process.off('SIGINT', handleSignal) process.off('SIGTERM', handleSignal) - console.info(`E2E ${failed ? 'failed' : 'completed'}; diagnostics: ${runDirectory}`) + if (cleanupSucceeded) runLock.release() + else runLock.retain('normal cleanup failed; inspect diagnostics and clean resources manually') + console.info( + diagnosticsRetained + ? `E2E ${failed ? 'failed' : 'completed'}; diagnostics: ${runDirectory}` + : 'E2E failed; diagnostics were scrubbed because complete secret scanning was impossible' + ) + } +} + +function scrubDiagnostics(roots: string[]): boolean { + try { + scrubUnscannableArtifacts(roots) + return false + } catch (error) { + console.error(error) + return true } } diff --git a/apps/sim/e2e/scripts/seed-world.ts b/apps/sim/e2e/scripts/seed-world.ts index 9ab77afcd3d..4f27bf3f669 100644 --- a/apps/sim/e2e/scripts/seed-world.ts +++ b/apps/sim/e2e/scripts/seed-world.ts @@ -1,6 +1,14 @@ import { randomBytes } from 'node:crypto' import { db } from '@sim/db' -import { member, permissions, subscription } from '@sim/db/schema' +import { + invitation, + invitationWorkspaceGrant, + member, + permissions, + subscription, + user, + userStats, +} from '@sim/db/schema' import { and, eq, inArray } from 'drizzle-orm' import { z } from 'zod' import { @@ -28,33 +36,48 @@ import { createSyntheticUser, type SyntheticLogin, } from '../fixtures/factories/users' -import { - createWorkspace, - grantWorkspacePermission, -} from '../fixtures/factories/workspaces' +import { createWorkspace, grantWorkspacePermission } from '../fixtures/factories/workspaces' import { E2eHttpClient } from '../fixtures/http-client' import type { ResolvedScenario, ScenarioSubscription } from '../fixtures/scenario' -import { validateScenario } from '../fixtures/validate-scenario' +import { validateScenario, validateScenarioSet } from '../fixtures/validate-scenario' import { createSettingsPersonaScenarios } from '../settings/personas' +import { writeSyntheticSecretCanary } from '../support/leak-canary' +import { assertSafeSeedEnvironment } from '../support/seed-safety' const requiredEnvSchema = z.object({ E2E_RUN_ID: z.string().min(1), + E2E_ORCHESTRATED: z.literal('1'), + E2E_PROFILE: z.string().min(1), E2E_BASE_URL: z.string().url(), ADMIN_API_KEY: z.string().min(1), DATABASE_URL: z.string().url(), E2E_MANIFEST_PATH: z.string().min(1), E2E_CREDENTIALS_PATH: z.string().min(1), + E2E_CANARY_SECRETS_PATH: z.string().min(1), }) async function main(): Promise { const env = requiredEnvSchema.parse(process.env) + assertSafeSeedEnvironment(env) const definitions = createSettingsPersonaScenarios(env.E2E_RUN_ID) - const scenarios = [validateScenario(definitions.primary), validateScenario(definitions.isolationTwin)] + const scenarios = [ + validateScenario(definitions.primary), + validateScenario(definitions.isolationTwin), + ] + validateScenarioSet(scenarios) const adminClient = createAdminClient(env.E2E_BASE_URL, env.ADMIN_API_KEY) const attemptCounts = new Map() const worlds: E2EWorld[] = [] - const loginsByWorldAndUser = new Map() + const loginsByWorldAndUser = buildSyntheticLogins(scenarios) const ownerClients = new Map() + writeSyntheticSecretCanary(env.E2E_CANARY_SECRETS_PATH, env.E2E_RUN_ID, [ + ...[...loginsByWorldAndUser.values()].map(({ password }) => password), + ...scenarios.flatMap(({ definition }) => definition.invitations.map(({ token }) => token)), + ]) + writeJsonAtomic( + env.E2E_CREDENTIALS_PATH, + buildPersonaCredentials(env.E2E_RUN_ID, scenarios, loginsByWorldAndUser) + ) for (const scenario of scenarios) { const world: E2EWorld = { scenario, records: createWorldRecords() } @@ -78,9 +101,7 @@ async function main(): Promise { await assertSecondOrganizationIsRejected(worlds, adminClient) const manifest = buildScenarioManifest(env.E2E_RUN_ID, worlds) - const credentials = buildPersonaCredentials(env.E2E_RUN_ID, worlds, loginsByWorldAndUser) writeJsonAtomic(env.E2E_MANIFEST_PATH, manifest) - writeJsonAtomic(env.E2E_CREDENTIALS_PATH, credentials) console.info( `Seeded ${Object.keys(manifest.personas).length} personas across ${worlds.length} isolated worlds` @@ -99,11 +120,7 @@ async function createUsers( logins: Map ): Promise { for (const user of world.scenario.definition.users) { - const login = { - name: user.name, - email: user.email, - password: randomBytes(24).toString('base64url'), - } + const login = required(logins, worldUserKey(world.scenario, user.key), 'synthetic login') const created = await createSyntheticUser( new E2eHttpClient({ baseUrl, @@ -111,8 +128,10 @@ async function createUsers( }), login ) + if (created.email !== user.email || created.name !== user.name) { + throw new Error(`Production signup returned an unexpected identity for ${user.key}`) + } world.records.users.set(user.key, created) - logins.set(worldUserKey(world.scenario, user.key), login) } } @@ -127,6 +146,9 @@ async function createOrganizationsAndSubscriptions( slug: organization.slug, ownerId: owner.id, }) + if (created.name !== organization.name || created.slug !== organization.slug) { + throw new Error(`Production organization identity mismatch for ${organization.key}`) + } world.records.organizations.set(organization.key, { id: created.id, memberId: created.memberId, @@ -278,7 +300,11 @@ async function createGrantsAndPermissionGroups( disableMcpTools: group.restrictions.disabledFeatures.includes('mcp'), disableCustomTools: group.restrictions.disabledFeatures.includes('custom-tools'), }, + isDefault: false, }) + if (created.isDefault) { + throw new Error(`Permission-group default state does not match scenario: ${group.key}`) + } world.records.permissionGroups.set(group.key, created.id) for (const userKey of group.memberUserKeys) { const memberId = await addPermissionGroupMember( @@ -311,26 +337,25 @@ async function arrangePlatformAdminsAndInvitations(world: E2EWorld): Promise ({ workspaceId: required(world.records.workspaces, grant.workspaceKey, 'invited workspace').id, - role: invitation.role, permission: grant.access, - }) - world.records.invitations.set(invitation.key, created.invitationId) + })), + }) + world.records.invitations.set(invitation.key, created.invitationId) + invitation.workspaceGrants.forEach((grant, index) => { world.records.invitationGrants.set( `${invitation.key}:${grant.workspaceKey}`, - created.grantId + created.grantIds[index] ) - } + }) } } @@ -340,10 +365,7 @@ async function recordProductionPermissionIds(world: E2EWorld): Promise { .select({ id: permissions.id, userId: permissions.userId }) .from(permissions) .where( - and( - eq(permissions.entityType, 'workspace'), - eq(permissions.entityId, workspaceRecord.id) - ) + and(eq(permissions.entityType, 'workspace'), eq(permissions.entityId, workspaceRecord.id)) ) for (const row of rows) { const userKey = [...world.records.users].find(([, user]) => user.id === row.userId)?.[0] @@ -353,32 +375,113 @@ async function recordProductionPermissionIds(world: E2EWorld): Promise { } async function assertTrustedWorldInvariants(world: E2EWorld): Promise { - const adminUser = world.records.users.get('enterprise-organization-admin') - const enterpriseWorkspace = world.records.workspaces.get('enterprise-workspace') - if (adminUser && enterpriseWorkspace) { - const explicitAdminRows = await db - .select({ id: permissions.id }) - .from(permissions) - .where( - and( - eq(permissions.entityType, 'workspace'), - eq(permissions.entityId, enterpriseWorkspace.id), - eq(permissions.userId, adminUser.id) - ) + for (const persona of world.scenario.definition.personas) { + const userId = required(world.records.users, persona.userKey, 'invariant user').id + for (const expectation of persona.workspaces) { + if (expectation.access === 'none') continue + const workspaceDefinition = required( + world.scenario.workspacesByKey, + expectation.workspaceKey, + 'invariant workspace definition' ) - if (explicitAdminRows.length !== 0) { - throw new Error('Enterprise organization admin unexpectedly received an explicit grant') + const workspaceId = required( + world.records.workspaces, + expectation.workspaceKey, + 'invariant workspace' + ).id + if (expectation.roleSource === 'org-admin') { + const explicitRows = await db + .select({ id: permissions.id }) + .from(permissions) + .where( + and( + eq(permissions.entityType, 'workspace'), + eq(permissions.entityId, workspaceId), + eq(permissions.userId, userId) + ) + ) + if (explicitRows.length !== 0) { + throw new Error( + `Organization-derived admin unexpectedly received an explicit grant: ${persona.key}/${expectation.workspaceKey}` + ) + } + } + if ( + expectation.hostContext.hostMembership === 'external' && + workspaceDefinition.organizationKey + ) { + const organizationId = required( + world.records.organizations, + workspaceDefinition.organizationKey, + 'external host organization' + ).id + const memberships = await db + .select({ id: member.id }) + .from(member) + .where(and(eq(member.userId, userId), eq(member.organizationId, organizationId))) + if (memberships.length !== 0) { + throw new Error( + `External workspace persona unexpectedly received host membership: ${persona.key}` + ) + } + } + } + } + + const userIds = [...world.records.users.values()].map(({ id }) => id) + const persistedUsers = await db + .select({ + id: user.id, + stripeCustomerId: user.stripeCustomerId, + currentUsageLimit: userStats.currentUsageLimit, + billingBlocked: userStats.billingBlocked, + billingBlockedReason: userStats.billingBlockedReason, + }) + .from(user) + .innerJoin(userStats, eq(userStats.userId, user.id)) + .where(inArray(user.id, userIds)) + if (persistedUsers.length !== userIds.length) { + throw new Error('Every synthetic user must retain exactly one user_stats row') + } + for (const definition of world.scenario.definition.users) { + const userId = required(world.records.users, definition.key, 'persisted user').id + const row = persistedUsers.find((candidate) => candidate.id === userId) + if ( + !row?.stripeCustomerId?.startsWith('cus_e2e_') || + row.currentUsageLimit !== expectedUsageLimit(world.scenario, definition.key) || + row.billingBlocked || + row.billingBlockedReason !== null + ) { + throw new Error(`Persisted user billing state does not match scenario: ${definition.key}`) } } - const externalUser = world.records.users.get('external-workspace-admin') - if (externalUser) { - const memberships = await db - .select({ id: member.id }) + const organizationIds = [...world.records.organizations.values()].map(({ id }) => id) + if (organizationIds.length > 0) { + const persistedMemberships = await db + .select({ + organizationId: member.organizationId, + userId: member.userId, + role: member.role, + }) .from(member) - .where(eq(member.userId, externalUser.id)) - if (memberships.length !== 0) { - throw new Error('External workspace admin unexpectedly received organization membership') + .where(inArray(member.organizationId, organizationIds)) + const expectedMemberships = world.scenario.definition.organizationMemberships + .map((definition) => ({ + organizationId: required( + world.records.organizations, + definition.organizationKey, + 'membership organization' + ).id, + userId: required(world.records.users, definition.userKey, 'membership user').id, + role: definition.role, + })) + .sort(byMembership) + if ( + JSON.stringify(persistedMemberships.sort(byMembership)) !== + JSON.stringify(expectedMemberships) + ) { + throw new Error('Persisted organization membership set does not match scenario') } } @@ -390,6 +493,15 @@ async function assertTrustedWorldInvariants(world: E2EWorld): Promise { plan: subscription.plan, status: subscription.status, referenceId: subscription.referenceId, + stripeCustomerId: subscription.stripeCustomerId, + stripeSubscriptionId: subscription.stripeSubscriptionId, + periodStart: subscription.periodStart, + periodEnd: subscription.periodEnd, + cancelAtPeriodEnd: subscription.cancelAtPeriodEnd, + canceledAt: subscription.canceledAt, + endedAt: subscription.endedAt, + seats: subscription.seats, + metadata: subscription.metadata, }) .from(subscription) .where(inArray(subscription.id, subscriptionIds)) @@ -403,37 +515,139 @@ async function assertTrustedWorldInvariants(world: E2EWorld): Promise { !row || row.plan !== definition.plan || row.referenceId !== resolveBillingReference(world, definition) || - row.status !== (definition.status === 'lapsed' ? 'canceled' : definition.status) + row.status !== (definition.status === 'lapsed' ? 'canceled' : definition.status) || + !row.stripeCustomerId?.startsWith('cus_e2e_') || + row.stripeSubscriptionId !== null || + !row.periodStart || + !row.periodEnd || + row.cancelAtPeriodEnd || + row.seats !== (definition.seats ?? null) || + (definition.status === 'lapsed' + ? !row.canceledAt || !row.endedAt + : row.canceledAt !== null || row.endedAt !== null) || + !matchesEnterpriseMetadata(row.metadata, definition, row.referenceId) ) { throw new Error(`Persisted subscription does not match scenario: ${definition.key}`) } } } + + for (const definition of world.scenario.definition.invitations) { + const invitationId = required(world.records.invitations, definition.key, 'invitation') + const [row] = await db + .select({ + kind: invitation.kind, + email: invitation.email, + inviterId: invitation.inviterId, + organizationId: invitation.organizationId, + membershipIntent: invitation.membershipIntent, + role: invitation.role, + status: invitation.status, + expiresAt: invitation.expiresAt, + token: invitation.token, + }) + .from(invitation) + .where(eq(invitation.id, invitationId)) + .limit(1) + if ( + !row || + row.kind !== 'workspace' || + row.email !== definition.email || + row.inviterId !== + required( + world.records.users, + required( + world.scenario.organizationsByKey, + definition.organizationKey, + 'invitation organization definition' + ).ownerUserKey, + 'invitation inviter' + ).id || + row.organizationId !== + required(world.records.organizations, definition.organizationKey, 'invitation organization') + .id || + row.role !== definition.role || + row.status !== 'pending' || + row.membershipIntent !== 'internal' || + row.token !== definition.token || + row.expiresAt.toISOString() !== new Date(definition.expiresAt).toISOString() + ) { + throw new Error(`Persisted invitation does not match scenario: ${definition.key}`) + } + const grants = await db + .select({ + id: invitationWorkspaceGrant.id, + workspaceId: invitationWorkspaceGrant.workspaceId, + permission: invitationWorkspaceGrant.permission, + }) + .from(invitationWorkspaceGrant) + .where(eq(invitationWorkspaceGrant.invitationId, invitationId)) + const expectedGrants = definition.workspaceGrants.map((grant) => ({ + id: required( + world.records.invitationGrants, + `${definition.key}:${grant.workspaceKey}`, + 'invitation grant' + ), + workspaceId: required(world.records.workspaces, grant.workspaceKey, 'invited workspace').id, + permission: grant.access, + })) + if (JSON.stringify(grants.sort(byId)) !== JSON.stringify(expectedGrants.sort(byId))) { + throw new Error(`Persisted invitation grants do not match scenario: ${definition.key}`) + } + } } async function assertSecondOrganizationIsRejected( worlds: E2EWorld[], adminClient: E2eHttpClient ): Promise { - const primary = worlds[0] - const teamMember = required(primary.records.users, 'workspace-read-member', 'constraint user') - const enterprise = required( - primary.records.organizations, - 'enterprise-organization', - 'constraint organization' - ) - await adminClient.request({ - method: 'POST', - path: `/api/v1/admin/organizations/${enterprise.id}/members`, - body: { userId: teamMember.id, role: 'member' }, - schema: z.object({ error: z.unknown() }), - expectedStatus: 400, - }) + for (const world of worlds) { + for (const membership of world.scenario.definition.organizationMemberships) { + const target = world.scenario.definition.organizations.find( + (organization) => + organization.key !== membership.organizationKey && + !world.scenario.definition.organizationMemberships.some( + (candidate) => + candidate.organizationKey === organization.key && + candidate.userKey === membership.userKey + ) + ) + if (!target) continue + const user = required(world.records.users, membership.userKey, 'constraint user') + const organization = required( + world.records.organizations, + target.key, + 'constraint organization' + ) + await adminClient.request({ + method: 'POST', + path: `/api/v1/admin/organizations/${organization.id}/members`, + body: { userId: user.id, role: 'member' }, + schema: z.object({ + error: z.object({ + code: z.literal('BAD_REQUEST'), + message: z.string().min(1), + }), + }), + expectedStatus: 400, + }) + const unexpectedMembership = await db + .select({ id: member.id }) + .from(member) + .where(and(eq(member.organizationId, organization.id), eq(member.userId, user.id))) + .limit(1) + if (unexpectedMembership.length > 0) { + throw new Error('Rejected second-organization request still created a membership') + } + return + } + } + throw new Error('Scenario does not contain a cross-organization membership constraint probe') } function buildPersonaCredentials( runId: string, - worlds: E2EWorld[], + scenarios: ResolvedScenario[], logins: Map ): PersonaCredentials { const credentials: PersonaCredentials = { @@ -441,19 +655,29 @@ function buildPersonaCredentials( runId, personas: {}, } - for (const world of worlds) { - for (const persona of world.scenario.definition.personas) { - const login = required( - logins, - worldUserKey(world.scenario, persona.userKey), - 'persona login' - ) + for (const scenario of scenarios) { + for (const persona of scenario.definition.personas) { + const login = required(logins, worldUserKey(scenario, persona.userKey), 'persona login') credentials.personas[persona.key] = { email: login.email, password: login.password } } } return credentials } +function buildSyntheticLogins(scenarios: ResolvedScenario[]): Map { + const logins = new Map() + for (const scenario of scenarios) { + for (const user of scenario.definition.users) { + logins.set(worldUserKey(scenario, user.key), { + name: user.name, + email: user.email, + password: randomBytes(24).toString('base64url'), + }) + } + } + return logins +} + function resolveBillingReference(world: E2EWorld, subscription: ScenarioSubscription): string { return subscription.billingReference.kind === 'user' ? required(world.records.users, subscription.billingReference.userKey, 'subscription user').id @@ -464,10 +688,60 @@ function resolveBillingReference(world: E2EWorld, subscription: ScenarioSubscrip ).id } +function expectedUsageLimit(scenario: ResolvedScenario, userKey: string): string | null { + const membership = scenario.definition.organizationMemberships.find( + (candidate) => candidate.userKey === userKey + ) + if (membership) { + const organizationSubscription = scenario.definition.subscriptions.find( + (candidate) => + candidate.billingReference.kind === 'organization' && + candidate.billingReference.organizationKey === membership.organizationKey + ) + if ( + organizationSubscription?.status === 'active' || + organizationSubscription?.status === 'past_due' + ) { + return null + } + } + const personalSubscription = scenario.definition.subscriptions.find( + (candidate) => + candidate.billingReference.kind === 'user' && + candidate.billingReference.userKey === userKey && + (candidate.status === 'active' || candidate.status === 'past_due') + ) + if (personalSubscription?.plan.startsWith('pro_')) { + return String(Number(personalSubscription.plan.split('_')[1]) / 200) + } + return '5' +} + +function matchesEnterpriseMetadata( + metadata: unknown, + definition: ScenarioSubscription, + referenceId: string +): boolean { + if (definition.plan !== 'enterprise') return metadata === null + const persisted = metadata as { + plan?: unknown + referenceId?: unknown + monthlyPrice?: unknown + seats?: unknown + } | null + return ( + persisted?.plan === 'enterprise' && + persisted.referenceId === referenceId && + persisted.monthlyPrice === definition.enterprise?.monthlyPrice && + persisted.seats === definition.enterprise?.seats + ) +} + function assertCreatedWorkspace( world: E2EWorld, expected: (typeof world.scenario.definition.workspaces)[number], created: { + name: string organizationId: string | null ownerId: string billedAccountUserId: string @@ -479,6 +753,7 @@ function assertCreatedWorkspace( ? required(world.records.organizations, expected.organizationKey, 'workspace organization').id : null if ( + created.name !== expected.name || created.ownerId !== ownerId || created.billedAccountUserId !== ownerId || created.organizationId !== organizationId || @@ -496,6 +771,21 @@ function increment(values: Map, key: string): void { values.set(key, (values.get(key) ?? 0) + 1) } +function byId(left: { id: string }, right: { id: string }): number { + return left.id.localeCompare(right.id) +} + +function byMembership( + left: { organizationId: string; userId: string; role: string }, + right: { organizationId: string; userId: string; role: string } +): number { + return ( + left.organizationId.localeCompare(right.organizationId) || + left.userId.localeCompare(right.userId) || + left.role.localeCompare(right.role) + ) +} + function required(values: ReadonlyMap, key: K, label: string): V { const value = values.get(key) if (!value) throw new Error(`Missing ${label}: ${String(key)}`) diff --git a/apps/sim/e2e/scripts/signal-cleanup.ts b/apps/sim/e2e/scripts/signal-cleanup.ts index 5d24f82df5a..dfa9559ae68 100644 --- a/apps/sim/e2e/scripts/signal-cleanup.ts +++ b/apps/sim/e2e/scripts/signal-cleanup.ts @@ -1,22 +1,42 @@ -import { rmSync } from 'node:fs' +import { existsSync, rmSync } from 'node:fs' import { sleep } from '@sim/utils/helpers' import { dropRunDatabase, dropRunDatabaseWithRetries } from '../support/database' -import { parseProcessGroupIds } from '../support/signal-cleanup' +import { releaseE2eRunLock, retainE2eRunLock } from '../support/run-lock' +import { isProcessGroupAlive, parseProcessGroupIds } from '../support/signal-cleanup' const adminUrl = process.env.E2E_PG_ADMIN_URL const databaseName = process.env.E2E_DATABASE_NAME const processGroupIds = parseProcessGroupIds(process.env.E2E_CLEANUP_PROCESS_GROUPS) const sensitiveDirectories = JSON.parse(process.env.E2E_CLEANUP_DIRECTORIES ?? '[]') as string[] const databaseCreationComplete = process.env.E2E_DATABASE_CREATION_COMPLETE === 'true' +const runLockPath = process.env.E2E_RUN_LOCK_PATH +const runLockToken = process.env.E2E_RUN_LOCK_TOKEN if (!adminUrl || !databaseName) { console.error('Signal cleanup requires E2E_PG_ADMIN_URL and E2E_DATABASE_NAME') + retainRunLock('signal cleanup was missing database coordinates') process.exit(1) } const failures: unknown[] = [] signalProcessGroups(processGroupIds, 'SIGTERM', failures) await sleep(500) +signalProcessGroups(processGroupIds, 'SIGKILL', failures) +await sleep(250) +for (const groupId of processGroupIds) { + if (isProcessGroupAlive(groupId)) { + failures.push(new Error(`Managed process group still exists after SIGKILL: ${groupId}`)) + } +} + +for (const directory of sensitiveDirectories) { + try { + rmSync(directory, { recursive: true, force: true }) + if (existsSync(directory)) throw new Error(`Sensitive directory still exists: ${directory}`) + } catch (error) { + failures.push(error) + } +} try { if (databaseCreationComplete) { @@ -28,19 +48,22 @@ try { failures.push(error) } -signalProcessGroups(processGroupIds, 'SIGKILL', failures) -await sleep(250) +for (const error of failures) console.error(error) +if (failures.length === 0) releaseRunLock() +else retainRunLock(`signal cleanup failed with ${failures.length} error(s)`) +process.exit(failures.length > 0 ? 1 : 0) -for (const directory of sensitiveDirectories) { - try { - rmSync(directory, { recursive: true, force: true }) - } catch (error) { - failures.push(error) +function releaseRunLock(): void { + if (runLockPath && runLockToken) { + releaseE2eRunLock(runLockPath, runLockToken) } } -for (const error of failures) console.error(error) -process.exit(failures.length > 0 ? 1 : 0) +function retainRunLock(reason: string): void { + if (runLockPath && runLockToken) { + retainE2eRunLock(runLockPath, runLockToken, reason) + } +} function signalProcessGroups( groupIds: number[], diff --git a/apps/sim/e2e/settings/persona-contracts.spec.ts b/apps/sim/e2e/settings/persona-contracts.spec.ts index b8000df2efb..c898ba8c140 100644 --- a/apps/sim/e2e/settings/persona-contracts.spec.ts +++ b/apps/sim/e2e/settings/persona-contracts.spec.ts @@ -1,5 +1,27 @@ -import { SETTINGS_PERSONA_KEYS } from './personas' +import { existsSync } from 'node:fs' +import path from 'node:path' import { expect, requirePersona, test } from '../fixtures/persona-test' +import { SETTINGS_PERSONA_KEYS } from './personas' + +test('persona workers receive no privileged setup values', () => { + expect(process.env.ADMIN_API_KEY).toBeUndefined() + expect(process.env.DATABASE_URL).toBeUndefined() + expect(process.env.E2E_CREDENTIALS_PATH).toBeUndefined() + for (const key of [ + 'BETTER_AUTH_SECRET', + 'ENCRYPTION_KEY', + 'API_ENCRYPTION_KEY', + 'INTERNAL_API_SECRET', + 'STRIPE_SECRET_KEY', + 'STRIPE_WEBHOOK_SECRET', + ]) { + expect(process.env[key]).toBeUndefined() + } + expect(process.env.HOME).toMatch(/[\\/]homes[\\/]playwright$/) + const privateDirectory = path.join(path.dirname(requiredEnv('E2E_MANIFEST_PATH')), 'private') + expect(existsSync(path.join(privateDirectory, 'persona-credentials.json'))).toBe(false) + expect(existsSync(path.join(privateDirectory, 'synthetic-secrets.json'))).toBe(false) +}) for (const personaKey of SETTINGS_PERSONA_KEYS) { test(`${personaKey} matches its real API contract`, async ({ @@ -8,6 +30,27 @@ for (const personaKey of SETTINGS_PERSONA_KEYS) { }) => { const persona = requirePersona(personaManifest, personaKey) const context = await contextForPersona(personaKey) + const page = await context.newPage() + const canonicalUrl = new URL(persona.canonicalRoute, requiredEnv('E2E_BASE_URL')).toString() + await page.goto(canonicalUrl) + await expect(page).toHaveURL(canonicalUrl) + await expect(page.getByRole('heading', { name: 'General', level: 1 })).toBeVisible() + if (personaKey === 'permissionGroupRestricted') { + await expect(page.getByRole('button', { name: 'General', exact: true })).toBeVisible() + for (const label of ['Secrets', 'Sim API keys', 'Sim mailer', 'MCP tools', 'Custom tools']) { + await expect(page.getByRole('button', { name: label, exact: true })).toHaveCount(0) + } + const workspaceId = persona.workspaces.find(({ access }) => access !== 'none')?.workspaceId + expect(workspaceId).toBeTruthy() + for (const section of ['secrets', 'api-keys', 'inbox', 'mcp', 'custom-tools']) { + const deniedUrl = new URL( + `/workspace/${encodeURIComponent(workspaceId ?? '')}/settings/${section}`, + requiredEnv('E2E_BASE_URL') + ).toString() + const response = await page.goto(deniedUrl) + expect(response?.status(), `${section} must be denied by the server route gate`).toBe(404) + } + } const sessionResponse = await context.request.get('/api/auth/get-session') expect(sessionResponse.status()).toBe(200) @@ -20,9 +63,13 @@ for (const personaKey of SETTINGS_PERSONA_KEYS) { email: persona.email, role: persona.expectedPlatformRole, }) - expect(session.session?.activeOrganizationId ?? null).toBe( - persona.expectedActiveOrganizationId - ) + expect(session.session?.activeOrganizationId ?? null).toBe(persona.expectedActiveOrganizationId) + const settingsResponse = await context.request.get('/api/users/me/settings') + expect(settingsResponse.status()).toBe(200) + const userSettings = (await settingsResponse.json()) as { + data?: { superUserModeEnabled?: boolean } + } + expect(userSettings.data?.superUserModeEnabled ?? false).toBe(persona.expectedSuperUserMode) const workspacesResponse = await context.request.get('/api/workspaces?scope=all') expect(workspacesResponse.status()).toBe(200) @@ -67,6 +114,25 @@ for (const personaKey of SETTINGS_PERSONA_KEYS) { expected.hostContext.payerScope === 'organization' && (expected.roleSource === 'owner' || expected.roleSource === 'org-admin') ) + + const permissionsResponse = await context.request.get( + `/api/workspaces/${encodeURIComponent(expected.workspaceId)}/permissions` + ) + expect(permissionsResponse.status()).toBe(200) + const permissionPayload = (await permissionsResponse.json()) as { + users?: Array<{ + userId?: string + permissionType?: string + roleSource?: string + isExternal?: boolean + }> + } + const viewer = permissionPayload.users?.find(({ userId }) => userId === persona.userId) + expect(viewer).toMatchObject({ + permissionType: expected.access, + roleSource: expected.roleSource, + isExternal: expected.hostContext.hostMembership === 'external', + }) } const accessibleWorkspace = persona.workspaces.find(({ access }) => access !== 'none') @@ -95,3 +161,9 @@ for (const personaKey of SETTINGS_PERSONA_KEYS) { } }) } + +function requiredEnv(key: string): string { + const value = process.env[key] + if (!value) throw new Error(`Missing persona contract environment value: ${key}`) + return value +} diff --git a/apps/sim/e2e/settings/persona-isolation.spec.ts b/apps/sim/e2e/settings/persona-isolation.spec.ts index 24ec2cfb8ec..a68cc52ab92 100644 --- a/apps/sim/e2e/settings/persona-isolation.spec.ts +++ b/apps/sim/e2e/settings/persona-isolation.spec.ts @@ -6,8 +6,10 @@ test('primary world cannot discover or access the isolation twin', async ({ contextForPersona, personaManifest, }) => { + const ownPersona = requirePersona(personaManifest, 'personalFreeOwner') await assertWorldCannotSee( await contextForPersona('personalFreeOwner'), + ownPersona, requirePersona(personaManifest, 'isolationTwinOwner').workspaces[0].workspaceId ) }) @@ -16,23 +18,58 @@ test('isolation twin cannot discover or access the primary world', async ({ contextForPersona, personaManifest, }) => { + const ownPersona = requirePersona(personaManifest, 'isolationTwinOwner') await assertWorldCannotSee( await contextForPersona('isolationTwinOwner'), + ownPersona, requirePersona(personaManifest, 'personalFreeOwner').workspaces[0].workspaceId ) }) async function assertWorldCannotSee( context: import('@playwright/test').BrowserContext, + ownPersona: import('../fixtures/e2e-world').PersonaManifestEntry, foreignWorkspaceId: string ): Promise { + const page = await context.newPage() + const canonicalUrl = new URL(ownPersona.canonicalRoute, requiredEnv('E2E_BASE_URL')).toString() + await page.goto(canonicalUrl) + await expect(page).toHaveURL(canonicalUrl) + await expect(page.getByRole('heading', { name: 'General', level: 1 })).toBeVisible() + + const sessionResponse = await context.request.get('/api/auth/get-session') + expect(sessionResponse.status()).toBe(200) + const session = (await sessionResponse.json()) as { user?: { id?: string; email?: string } } + expect(session.user).toMatchObject({ id: ownPersona.userId, email: ownPersona.email }) + const listResponse = await context.request.get('/api/workspaces?scope=all') expect(listResponse.status()).toBe(200) const payload = (await listResponse.json()) as { workspaces?: Array<{ id: string }> } - expect(payload.workspaces?.map(({ id }) => id)).not.toContain(foreignWorkspaceId) + const listedWorkspaceIds = payload.workspaces?.map(({ id }) => id) + expect(listedWorkspaceIds).toContain(ownPersona.workspaces[0].workspaceId) + expect(listedWorkspaceIds).not.toContain(foreignWorkspaceId) + + const ownHostContext = await context.request.get( + `/api/workspaces/${encodeURIComponent(ownPersona.workspaces[0].workspaceId)}/host-context` + ) + expect(ownHostContext.status()).toBe(200) const response = await context.request.get( `/api/workspaces/${encodeURIComponent(foreignWorkspaceId)}/host-context` ) expect(response.status()).toBe(403) + + const patchResponse = await context.request.patch( + `/api/workspaces/${encodeURIComponent(foreignWorkspaceId)}`, + { + data: { name: 'E Two E Foreign Mutation Must Fail' }, + } + ) + expect([403, 404]).toContain(patchResponse.status()) +} + +function requiredEnv(key: string): string { + const value = process.env[key] + if (!value) throw new Error(`Missing persona isolation environment value: ${key}`) + return value } diff --git a/apps/sim/e2e/settings/personas.ts b/apps/sim/e2e/settings/personas.ts index f7783a89bda..71e49a8ec62 100644 --- a/apps/sim/e2e/settings/personas.ts +++ b/apps/sim/e2e/settings/personas.ts @@ -134,7 +134,6 @@ export function createPrimarySettingsScenario(namespace: ScenarioNamespace): Sce seats: 3, enterprise: { plan: 'enterprise', - referenceId: namespace.slug('enterprise-contract'), monthlyPrice: 12_000, seats: 3, }, @@ -173,6 +172,15 @@ export function createPrimarySettingsScenario(namespace: ScenarioNamespace): Sce subscriptionKey: 'team-subscription', ...HOSTED_BILLING, }, + { + key: 'team-invitation-workspace', + name: namespace.name('team-invitation-workspace'), + ownerUserKey: 'paid-organization-owner', + organizationKey: 'team-organization', + payer: { kind: 'organization', organizationKey: 'team-organization' }, + subscriptionKey: 'team-subscription', + ...HOSTED_BILLING, + }, { key: 'enterprise-workspace', name: namespace.name('enterprise-workspace'), @@ -224,7 +232,10 @@ export function createPrimarySettingsScenario(namespace: ScenarioNamespace): Sce token: namespace.invitationToken('pending-team-invitation'), role: 'member', expiresAt: '2099-01-01T00:00:00.000Z', - workspaceGrants: [{ workspaceKey: 'team-workspace', access: 'read' }], + workspaceGrants: [ + { workspaceKey: 'team-workspace', access: 'read' }, + { workspaceKey: 'team-invitation-workspace', access: 'write' }, + ], }, ] as const @@ -241,6 +252,15 @@ export function createPrimarySettingsScenario(namespace: ScenarioNamespace): Sce ]), persona(namespace, 'paidOrganizationOwner', 'paid-organization-owner', [ expected('team-workspace', 'admin', 'owner', 'owner', 'organization', 'team_6000', true), + expected( + 'team-invitation-workspace', + 'admin', + 'owner', + 'owner', + 'organization', + 'team_6000', + true + ), ]), persona(namespace, 'workspaceReadMember', 'workspace-read-member', [ expected('team-workspace', 'read', 'explicit', 'member', 'organization', 'team_6000', false), diff --git a/apps/sim/e2e/settings/smoke/authenticated.spec.ts b/apps/sim/e2e/settings/smoke/authenticated.spec.ts index a64c7e9a1db..51392426fb1 100644 --- a/apps/sim/e2e/settings/smoke/authenticated.spec.ts +++ b/apps/sim/e2e/settings/smoke/authenticated.spec.ts @@ -2,9 +2,11 @@ import { createHash } from 'node:crypto' import { writeFileSync } from 'node:fs' import path from 'node:path' import { expect, test } from '@playwright/test' +import { FOUNDATION_TEST_PASSWORD } from '../../support/runtime-secrets' test('billing-enabled signup, login, and settings use real Sim boundaries', async ({ browser, + contextOptions, page, }, testInfo) => { test.slow() @@ -15,7 +17,7 @@ test('billing-enabled signup, login, and settings use real Sim boundaries', asyn .digest('hex') .slice(0, 16) const email = `e2e-foundation-${runId}-${testIdentity}@example.com` - const password = 'E2eFoundation1!' + const password = FOUNDATION_TEST_PASSWORD const storageStatePath = path.join(requiredEnv('E2E_STORAGE_STATE_DIR'), `${testIdentity}.json`) await page.goto('/signup') @@ -44,7 +46,11 @@ test('billing-enabled signup, login, and settings use real Sim boundaries', asyn await expect(page).toHaveURL(/\/workspace(?:\/|$)/) await page.context().storageState({ path: storageStatePath }) - const restoredContext = await browser.newContext({ storageState: storageStatePath }) + const restoredContext = await browser.newContext({ + ...contextOptions, + baseURL: requiredEnv('E2E_BASE_URL'), + storageState: storageStatePath, + }) try { const restoredPage = await restoredContext.newPage() await restoredPage.goto(`${requiredEnv('E2E_BASE_URL')}${settingsPath}`) diff --git a/apps/sim/e2e/support/build-manifest.ts b/apps/sim/e2e/support/build-manifest.ts index 67b8c988095..b7859beb013 100644 --- a/apps/sim/e2e/support/build-manifest.ts +++ b/apps/sim/e2e/support/build-manifest.ts @@ -36,8 +36,6 @@ const UNHASHED_OS_ENV_KEYS = new Set([ 'GITHUB_ACTIONS', ]) -export type E2eHashOwner = 'next-build' | 'retained-stack' | 'scenario' | 'rerun' - export interface BuildIdentity { schemaVersion: typeof BUILD_MANIFEST_VERSION nextBuildHash: string @@ -64,6 +62,16 @@ export interface BuildReuseDecision { cacheDirectory: string } +export interface BuildArtifactPaths { + activeNextDirectory: string + buildCacheDirectory: string +} + +const DEFAULT_ARTIFACT_PATHS: BuildArtifactPaths = { + activeNextDirectory: NEXT_DIR, + buildCacheDirectory: E2E_BUILD_CACHE_DIR, +} + export function computeBuildIdentity(options: { buildEnvironment: ChildEnvironment nodeExecutable: string @@ -96,57 +104,12 @@ export function computeBuildIdentity(options: { } } -export function computeExecutionHashes(nextBuildHash: string): { - retainedStackHash: string - scenarioHash: string -} { - const files = listRepositoryFiles() - return { - retainedStackHash: hashJson({ - nextBuildHash, - files: hashOwnedFiles(files, 'retained-stack'), - }), - scenarioHash: hashJson({ - files: hashOwnedFiles(files, 'scenario'), - }), - } -} - -export function classifyE2eHashOwner(relativePath: string): E2eHashOwner { - const normalized = relativePath.replaceAll(path.sep, '/') - if (isNextBuildInput(normalized)) return 'next-build' - if (normalized.startsWith('apps/realtime/')) return 'retained-stack' - if (normalized === 'apps/sim/playwright.config.ts') return 'retained-stack' - if (!normalized.startsWith('apps/sim/e2e/')) return 'retained-stack' - - const e2ePath = normalized.slice('apps/sim/e2e/'.length) - if ( - e2ePath.startsWith('fixtures/') || - e2ePath === 'settings/personas.ts' || - e2ePath === 'scripts/seed-world.ts' || - e2ePath === 'scripts/capture-auth-states.ts' - ) { - return 'scenario' - } - if ( - e2ePath.endsWith('.spec.ts') || - e2ePath.startsWith('auth/') || - e2ePath.startsWith('settings/') - ) { - return 'rerun' - } - if ( - e2ePath.startsWith('support/') || - e2ePath.startsWith('fakes/') || - e2ePath.startsWith('scripts/') - ) { - return 'retained-stack' - } - return 'retained-stack' -} - -export function restoreCachedBuild(identity: BuildIdentity): BuildReuseDecision { - const cacheDirectory = getCacheDirectory(identity.nextBuildHash) +export function restoreCachedBuild( + identity: BuildIdentity, + paths: BuildArtifactPaths = DEFAULT_ARTIFACT_PATHS +): BuildReuseDecision { + clearInterruptedBuildStores(paths.buildCacheDirectory) + const cacheDirectory = getCacheDirectory(identity.nextBuildHash, paths) const manifestPath = path.join(cacheDirectory, 'manifest.json') const artifactDirectory = path.join(cacheDirectory, '.next') const miss = (reason: string): BuildReuseDecision => ({ @@ -177,7 +140,7 @@ export function restoreCachedBuild(identity: BuildIdentity): BuildReuseDecision return miss('cached artifact checksum does not match the manifest') } - activateDirectory(artifactDirectory, NEXT_DIR) + activateDirectory(artifactDirectory, paths.activeNextDirectory) return { reused: true, nextBuildHash: identity.nextBuildHash, @@ -186,19 +149,25 @@ export function restoreCachedBuild(identity: BuildIdentity): BuildReuseDecision } } -export function storeCompletedBuild(identity: BuildIdentity): BuildReuseDecision { - const buildIdPath = path.join(NEXT_DIR, 'BUILD_ID') +export function storeCompletedBuild( + identity: BuildIdentity, + paths: BuildArtifactPaths = DEFAULT_ARTIFACT_PATHS +): BuildReuseDecision { + const buildIdPath = path.join(paths.activeNextDirectory, 'BUILD_ID') if (!existsSync(buildIdPath)) { throw new Error('Next build completed without .next/BUILD_ID') } - mkdirSync(E2E_BUILD_CACHE_DIR, { recursive: true }) - const cacheDirectory = getCacheDirectory(identity.nextBuildHash) + mkdirSync(paths.buildCacheDirectory, { recursive: true }) + const cacheDirectory = getCacheDirectory(identity.nextBuildHash, paths) const temporaryDirectory = `${cacheDirectory}.tmp-${process.pid}-${Date.now()}` rmSync(temporaryDirectory, { recursive: true, force: true }) mkdirSync(temporaryDirectory, { recursive: true }) const artifactDirectory = path.join(temporaryDirectory, '.next') - cpSync(NEXT_DIR, artifactDirectory, { recursive: true, verbatimSymlinks: true }) + cpSync(paths.activeNextDirectory, artifactDirectory, { + recursive: true, + verbatimSymlinks: true, + }) const manifest: BuildManifest = { ...identity, @@ -223,14 +192,57 @@ export function storeCompletedBuild(identity: BuildIdentity): BuildReuseDecision } } -export function clearActiveNextBuild(): void { - rmSync(NEXT_DIR, { recursive: true, force: true }) +export function pruneBuildCache( + retainedHash: string, + maxEntries = 1, + buildCacheDirectory = E2E_BUILD_CACHE_DIR +): string[] { + if (!existsSync(buildCacheDirectory)) return [] + clearInterruptedBuildStores(buildCacheDirectory) + const entries = readdirSync(buildCacheDirectory) + const candidates = entries + .filter((name) => !name.includes('.tmp-')) + .map((name) => { + const directory = path.join(buildCacheDirectory, name) + const stats = lstatSync(directory) + return stats.isDirectory() && !name.includes('.tmp-') + ? { name, directory, modifiedAt: stats.mtimeMs } + : null + }) + .filter( + (candidate): candidate is { name: string; directory: string; modifiedAt: number } => + candidate !== null + ) + .sort((left, right) => { + if (left.name === retainedHash) return -1 + if (right.name === retainedHash) return 1 + return right.modifiedAt - left.modifiedAt + }) + const removed = candidates.slice(Math.max(1, maxEntries)) + for (const candidate of removed) { + rmSync(candidate.directory, { recursive: true, force: true }) + } + return removed.map(({ name }) => name) +} + +function clearInterruptedBuildStores(buildCacheDirectory: string): void { + if (!existsSync(buildCacheDirectory)) return + for (const name of readdirSync(buildCacheDirectory).filter((entry) => entry.includes('.tmp-'))) { + rmSync(path.join(buildCacheDirectory, name), { recursive: true, force: true }) + } +} + +export function clearActiveNextBuild( + activeNextDirectory = DEFAULT_ARTIFACT_PATHS.activeNextDirectory +): void { + rmSync(activeNextDirectory, { recursive: true, force: true }) + clearStaleActivationDirectories(activeNextDirectory) } function listRepositoryFiles(): string[] { const result = spawnSync( 'git', - ['-C', REPO_ROOT, 'ls-files', '--cached', '--others', '--exclude-standard'], + ['-C', REPO_ROOT, 'ls-files', '-z', '--cached', '--others', '--exclude-standard'], { encoding: 'utf8', env: { NODE_ENV: 'test', PATH: process.env.PATH ?? '' }, @@ -239,7 +251,7 @@ function listRepositoryFiles(): string[] { if (result.status !== 0) { throw new Error(`Unable to enumerate E2E hash inputs: ${result.stderr}`) } - return [...new Set(result.stdout.split(/\r?\n/).filter(Boolean))].sort() + return [...new Set(result.stdout.split('\0').filter(Boolean))].sort() } function isNextBuildInput(relativePath: string): boolean { @@ -255,12 +267,6 @@ function isNextBuildInput(relativePath: string): boolean { ) } -function hashOwnedFiles(files: string[], owner: E2eHashOwner): Array<[string, string]> { - return files - .filter((file) => classifyE2eHashOwner(file) === owner) - .map((file) => [file, hashRepositoryPath(file)]) -} - function hashRepositoryFiles(files: string[]): string { const hash = createHash('sha256') for (const file of files) { @@ -308,6 +314,7 @@ function hashDirectory(directory: string): string { } function activateDirectory(source: string, destination: string): void { + clearStaleActivationDirectories(destination) const temporary = `${destination}.e2e-restore-${process.pid}` const backup = `${destination}.e2e-backup-${process.pid}` rmSync(temporary, { recursive: true, force: true }) @@ -324,6 +331,17 @@ function activateDirectory(source: string, destination: string): void { } } +function clearStaleActivationDirectories(destination: string): void { + const parent = path.dirname(destination) + if (!existsSync(parent)) return + const base = path.basename(destination) + for (const name of readdirSync(parent)) { + if (name.startsWith(`${base}.e2e-restore-`) || name.startsWith(`${base}.e2e-backup-`)) { + rmSync(path.join(parent, name), { recursive: true, force: true }) + } + } +} + function isMatchingIdentity(manifest: BuildManifest, identity: BuildIdentity): boolean { return ( manifest.schemaVersion === identity.schemaVersion && @@ -338,8 +356,8 @@ function isMatchingIdentity(manifest: BuildManifest, identity: BuildIdentity): b ) } -function getCacheDirectory(nextBuildHash: string): string { - return path.join(E2E_BUILD_CACHE_DIR, nextBuildHash) +function getCacheDirectory(nextBuildHash: string, paths: BuildArtifactPaths): string { + return path.join(paths.buildCacheDirectory, nextBuildHash) } function getExecutableVersion(executable: string): string { diff --git a/apps/sim/e2e/support/deployment-profile.ts b/apps/sim/e2e/support/deployment-profile.ts index 34f2fca490e..2dc238edb42 100644 --- a/apps/sim/e2e/support/deployment-profile.ts +++ b/apps/sim/e2e/support/deployment-profile.ts @@ -1,11 +1,22 @@ import { buildChildEnvironment, type ChildEnvironment } from './env' import { E2E_CACHE_DIR } from './paths' +import type { E2eRuntimeSecrets } from './runtime-secrets' export const E2E_PROFILE = 'hosted-billing-chromium' export const E2E_HOST = 'e2e.sim.ai' export const E2E_ORIGIN = `http://${E2E_HOST}:3000` export const E2E_SOCKET_ORIGIN = `http://${E2E_HOST}:3002` +const BUILD_SECRET_SENTINELS: E2eRuntimeSecrets = { + betterAuthSecret: 'build-sentinel-better-auth-secret-00000000000000000000000000000000', + encryptionKey: 'aa'.repeat(32), + apiEncryptionKey: 'bb'.repeat(32), + internalApiSecret: 'build-sentinel-internal-api-secret-000000000000000000000000000000', + adminApiKey: 'build-sentinel-admin-api-key-00000000000000000000000000000000', + stripeSecretKey: 'sk_test_build_sentinel', + stripeWebhookSecret: 'whsec_build_sentinel', +} + const APP_REQUIRED_KEYS = [ 'NODE_ENV', 'NEXT_PUBLIC_APP_URL', @@ -67,8 +78,12 @@ export interface HostedBillingProfileOptions { runId: string databaseUrl: string stripeApiBaseUrl: string - homeDirectory: string + runtimeHomeDirectory: string + setupHomeDirectory: string + authCaptureHomeDirectory: string + playwrightHomeDirectory: string playwrightBrowsersPath: string + runtimeSecrets: E2eRuntimeSecrets ci: boolean } @@ -90,34 +105,38 @@ export function createHostedBillingProfile({ runId, databaseUrl, stripeApiBaseUrl, - homeDirectory, + runtimeHomeDirectory, + setupHomeDirectory, + authCaptureHomeDirectory, + playwrightHomeDirectory, playwrightBrowsersPath, + runtimeSecrets, ci, }: HostedBillingProfileOptions): HostedBillingProfile { const values: Record = { NODE_ENV: 'production', NODE_OPTIONS: '--no-warnings --max-old-space-size=8192 --dns-result-order=ipv4first', NEXT_TELEMETRY_DISABLED: '1', - HOME: homeDirectory, - XDG_CONFIG_HOME: `${homeDirectory}/xdg`, + HOME: runtimeHomeDirectory, + XDG_CONFIG_HOME: `${runtimeHomeDirectory}/xdg`, AWS_EC2_METADATA_DISABLED: 'true', AWS_SHARED_CREDENTIALS_FILE: '/dev/null', AWS_CONFIG_FILE: '/dev/null', - CLOUDSDK_CONFIG: `${homeDirectory}/gcloud`, - AZURE_CONFIG_DIR: `${homeDirectory}/azure`, + CLOUDSDK_CONFIG: `${runtimeHomeDirectory}/gcloud`, + AZURE_CONFIG_DIR: `${runtimeHomeDirectory}/azure`, PLAYWRIGHT_BROWSERS_PATH: playwrightBrowsersPath, E2E_PROFILE, E2E_RUN_ID: runId, E2E_BASE_URL: E2E_ORIGIN, NEXT_PUBLIC_APP_URL: E2E_ORIGIN, BETTER_AUTH_URL: E2E_ORIGIN, - BETTER_AUTH_SECRET: 'e2e-better-auth-secret-at-least-32-characters-long', + BETTER_AUTH_SECRET: runtimeSecrets.betterAuthSecret, DATABASE_URL: databaseUrl, MIGRATION_DATABASE_URL: databaseUrl, - ENCRYPTION_KEY: '11'.repeat(32), - API_ENCRYPTION_KEY: '22'.repeat(32), - INTERNAL_API_SECRET: 'e2e-internal-api-secret-at-least-32-characters', - ADMIN_API_KEY: 'e2e-admin-api-key-at-least-32-characters-long', + ENCRYPTION_KEY: runtimeSecrets.encryptionKey, + API_ENCRYPTION_KEY: runtimeSecrets.apiEncryptionKey, + INTERNAL_API_SECRET: runtimeSecrets.internalApiSecret, + ADMIN_API_KEY: runtimeSecrets.adminApiKey, BILLING_ENABLED: 'true', NEXT_PUBLIC_BILLING_ENABLED: 'true', EMAIL_VERIFICATION_ENABLED: 'false', @@ -128,8 +147,8 @@ export function createHostedBillingProfile({ SIGNUP_MX_VALIDATION_ENABLED: 'false', NEXT_PUBLIC_POSTHOG_ENABLED: 'false', BLACKLISTED_PROVIDERS: 'ollama,ollama-cloud,vllm,litellm,openrouter,together,fireworks,baseten', - STRIPE_SECRET_KEY: 'sk_test_sim_e2e_foundation', - STRIPE_WEBHOOK_SECRET: 'whsec_sim_e2e_foundation', + STRIPE_SECRET_KEY: runtimeSecrets.stripeSecretKey, + STRIPE_WEBHOOK_SECRET: runtimeSecrets.stripeWebhookSecret, STRIPE_FREE_PRICE_ID: 'price_e2e_free', STRIPE_API_BASE_URL: stripeApiBaseUrl, TELEMETRY_ENDPOINT: `${stripeApiBaseUrl}/v1/traces`, @@ -150,7 +169,14 @@ export function createHostedBillingProfile({ AZURE_CONFIG_DIR: `${buildHomeDirectory}/azure`, E2E_RUN_ID: 'build_sentinel', HOME: buildHomeDirectory, + BETTER_AUTH_SECRET: BUILD_SECRET_SENTINELS.betterAuthSecret, + ENCRYPTION_KEY: BUILD_SECRET_SENTINELS.encryptionKey, + API_ENCRYPTION_KEY: BUILD_SECRET_SENTINELS.apiEncryptionKey, + INTERNAL_API_SECRET: BUILD_SECRET_SENTINELS.internalApiSecret, + ADMIN_API_KEY: BUILD_SECRET_SENTINELS.adminApiKey, DATABASE_URL: 'postgresql://e2e_build:e2e_build@127.0.0.1:1/sim_e2e_build_sentinel', + STRIPE_SECRET_KEY: BUILD_SECRET_SENTINELS.stripeSecretKey, + STRIPE_WEBHOOK_SECRET: BUILD_SECRET_SENTINELS.stripeWebhookSecret, STRIPE_API_BASE_URL: 'http://127.0.0.1:1', TELEMETRY_ENDPOINT: 'http://127.0.0.1:1/v1/traces', CI: 'false', @@ -189,31 +215,35 @@ export function createHostedBillingProfile({ false ), migration: createEnvironment( - pickValues(values, [ - 'NODE_ENV', - 'NODE_OPTIONS', - 'HOME', - 'MIGRATION_DATABASE_URL', - 'DATABASE_URL', - 'E2E_PROFILE', - 'E2E_RUN_ID', - 'CI', - ]), + { + ...pickValues(values, [ + 'NODE_ENV', + 'NODE_OPTIONS', + 'MIGRATION_DATABASE_URL', + 'DATABASE_URL', + 'E2E_PROFILE', + 'E2E_RUN_ID', + 'CI', + ]), + HOME: setupHomeDirectory, + }, ['NODE_ENV', 'HOME', 'MIGRATION_DATABASE_URL', 'DATABASE_URL', 'E2E_PROFILE', 'E2E_RUN_ID'], false ), seed: createEnvironment( - pickValues(values, [ - 'NODE_ENV', - 'NODE_OPTIONS', - 'HOME', - 'DATABASE_URL', - 'ADMIN_API_KEY', - 'E2E_PROFILE', - 'E2E_RUN_ID', - 'E2E_BASE_URL', - 'CI', - ]), + { + ...pickValues(values, [ + 'NODE_ENV', + 'NODE_OPTIONS', + 'DATABASE_URL', + 'ADMIN_API_KEY', + 'E2E_PROFILE', + 'E2E_RUN_ID', + 'E2E_BASE_URL', + 'CI', + ]), + HOME: setupHomeDirectory, + }, [ 'NODE_ENV', 'HOME', @@ -226,16 +256,18 @@ export function createHostedBillingProfile({ false ), authCapture: createEnvironment( - pickValues(values, [ - 'NODE_ENV', - 'NODE_OPTIONS', - 'HOME', - 'PLAYWRIGHT_BROWSERS_PATH', - 'E2E_PROFILE', - 'E2E_RUN_ID', - 'E2E_BASE_URL', - 'CI', - ]), + { + ...pickValues(values, [ + 'NODE_ENV', + 'NODE_OPTIONS', + 'PLAYWRIGHT_BROWSERS_PATH', + 'E2E_PROFILE', + 'E2E_RUN_ID', + 'E2E_BASE_URL', + 'CI', + ]), + HOME: authCaptureHomeDirectory, + }, [ 'NODE_ENV', 'HOME', @@ -247,16 +279,18 @@ export function createHostedBillingProfile({ false ), playwright: createEnvironment( - pickValues(values, [ - 'NODE_ENV', - 'NODE_OPTIONS', - 'HOME', - 'PLAYWRIGHT_BROWSERS_PATH', - 'E2E_PROFILE', - 'E2E_RUN_ID', - 'E2E_BASE_URL', - 'CI', - ]), + { + ...pickValues(values, [ + 'NODE_ENV', + 'NODE_OPTIONS', + 'PLAYWRIGHT_BROWSERS_PATH', + 'E2E_PROFILE', + 'E2E_RUN_ID', + 'E2E_BASE_URL', + 'CI', + ]), + HOME: playwrightHomeDirectory, + }, [ 'NODE_ENV', 'HOME', diff --git a/apps/sim/e2e/support/env.ts b/apps/sim/e2e/support/env.ts index 0ba38b20163..227e2ce40ec 100644 --- a/apps/sim/e2e/support/env.ts +++ b/apps/sim/e2e/support/env.ts @@ -102,5 +102,3 @@ export function formatRedactedEnvironmentSummary( `Shadowed local keys: ${childEnvironment.shadowedKeys.join(', ') || '(none)'}`, ].join('\n') } - -export const E2E_OS_PASSTHROUGH_KEYS = OS_PASSTHROUGH_KEYS diff --git a/apps/sim/e2e/support/leak-canary.ts b/apps/sim/e2e/support/leak-canary.ts new file mode 100644 index 00000000000..4c4f48c1864 --- /dev/null +++ b/apps/sim/e2e/support/leak-canary.ts @@ -0,0 +1,132 @@ +import { existsSync, lstatSync, readdirSync, readFileSync, rmSync } from 'node:fs' +import path from 'node:path' +import JSZip from 'jszip' +import { writeJsonAtomic } from '../fixtures/e2e-world' + +const BINARY_EXTENSIONS = new Set(['.gif', '.jpeg', '.jpg', '.mp4', '.png', '.webp']) + +interface SyntheticSecretCanary { + schemaVersion: 1 + runId: string + secrets: string[] +} + +export function writeSyntheticSecretCanary( + filePath: string, + runId: string, + secrets: string[] +): void { + if (secrets.length === 0 || secrets.some((secret) => !secret)) { + throw new Error('Synthetic secret canary requires non-empty secrets') + } + writeJsonAtomic(filePath, { + schemaVersion: 1, + runId, + secrets: [...new Set(secrets)], + } satisfies SyntheticSecretCanary) +} + +export function readSyntheticSecretCanarySecrets(filePath: string): string[] { + return readSyntheticSecretCanary(filePath).secrets +} + +export function loadSyntheticSecretCanaryForScan( + inMemorySecrets: string[], + filePath: string +): string[] { + if (!existsSync(filePath)) return inMemorySecrets + return [...new Set([...inMemorySecrets, ...readSyntheticSecretCanarySecrets(filePath)])] +} + +export function scrubUnscannableArtifacts(roots: string[]): void { + const failures: unknown[] = [] + for (const root of roots) { + try { + rmPath(root) + } catch (error) { + failures.push(error) + } + } + if (failures.length > 0) { + throw new AggregateError(failures, 'Unable to scrub unscannable E2E artifacts') + } +} + +export async function assertNoSyntheticSecretLeaks(options: { + secrets: string[] + roots: string[] + excludedPaths?: string[] +}): Promise { + if (options.secrets.length === 0 || options.secrets.some((secret) => !secret)) { + throw new Error('Synthetic secret leak scan requires non-empty secrets') + } + const secrets = [...new Set(options.secrets)] + const excluded = new Set((options.excludedPaths ?? []).map((value) => path.resolve(value))) + const violations: string[] = [] + + for (const root of options.roots) { + if (!existsSync(root)) continue + for (const filePath of listFiles(path.resolve(root), excluded)) { + if (BINARY_EXTENSIONS.has(path.extname(filePath).toLowerCase())) continue + const contents = + path.extname(filePath).toLowerCase() === '.zip' + ? await readZipContents(filePath) + : readFileSync(filePath) + if (secrets.some((secret) => contents.includes(Buffer.from(secret)))) { + violations.push(filePath) + } + } + } + + if (violations.length > 0) { + throw new Error( + `Synthetic E2E secret leaked outside private artifacts:\n${violations + .map((filePath) => `- ${filePath}`) + .join('\n')}` + ) + } +} + +function readSyntheticSecretCanary(filePath: string): SyntheticSecretCanary { + const parsed = JSON.parse(readFileSync(filePath, 'utf8')) as Partial + if ( + parsed.schemaVersion !== 1 || + typeof parsed.runId !== 'string' || + !Array.isArray(parsed.secrets) || + parsed.secrets.length === 0 || + parsed.secrets.some((secret) => typeof secret !== 'string' || !secret) + ) { + throw new Error('Invalid synthetic secret canary artifact') + } + return parsed as SyntheticSecretCanary +} + +function listFiles(currentPath: string, excluded: ReadonlySet = new Set()): string[] { + if (excluded.has(currentPath)) return [] + const stats = lstatSync(currentPath) + if (stats.isSymbolicLink()) return [] + if (stats.isFile()) return [currentPath] + if (!stats.isDirectory()) return [] + return readdirSync(currentPath).flatMap((name) => + listFiles(path.join(currentPath, name), excluded) + ) +} + +function rmPath(filePath: string): void { + rmSync(filePath, { recursive: true, force: true }) + if (existsSync(filePath)) throw new Error(`Unscannable E2E artifact still exists: ${filePath}`) +} + +async function readZipContents(filePath: string): Promise { + try { + const archive = await JSZip.loadAsync(readFileSync(filePath)) + const contents = await Promise.all( + Object.values(archive.files) + .filter((entry) => !entry.dir) + .map(async (entry) => Buffer.from(await entry.async('uint8array'))) + ) + return Buffer.concat(contents) + } catch { + throw new Error(`Unable to inspect E2E diagnostic archive: ${filePath}`) + } +} diff --git a/apps/sim/e2e/support/paths.ts b/apps/sim/e2e/support/paths.ts index 2dc124547cd..2fe4fbe299a 100644 --- a/apps/sim/e2e/support/paths.ts +++ b/apps/sim/e2e/support/paths.ts @@ -1,6 +1,6 @@ import path from 'node:path' -export const SIM_APP_DIR = path.resolve(process.cwd()) +export const SIM_APP_DIR = path.resolve(__dirname, '../..') export const REPO_ROOT = path.resolve(SIM_APP_DIR, '../..') export const DB_PACKAGE_DIR = path.join(REPO_ROOT, 'packages/db') export const REALTIME_APP_DIR = path.join(REPO_ROOT, 'apps/realtime') diff --git a/apps/sim/e2e/support/probes.ts b/apps/sim/e2e/support/probes.ts index 40f4e42bae1..4dbc0b7140c 100644 --- a/apps/sim/e2e/support/probes.ts +++ b/apps/sim/e2e/support/probes.ts @@ -1,4 +1,5 @@ import postgres from 'postgres' +import { readScenarioManifest } from '../fixtures/e2e-world' export async function assertAdminApiBoundary(origin: string, adminKey: string): Promise { const endpoint = `${origin}/api/v1/admin/users?limit=1&offset=0` @@ -45,3 +46,36 @@ export async function inspectFoundationUsers( await sql.end() } } + +export async function assertManifestWorkspaceIdentities( + databaseUrl: string, + manifestPath: string +): Promise { + const manifest = readScenarioManifest(manifestPath) + const expected = Object.values(manifest.worlds).flatMap((world) => + Object.values(world.workspaceIdentities) + ) + const sql = postgres(databaseUrl, { max: 1, connect_timeout: 10 }) + try { + const ids = expected.map(({ id }) => id) + const rows = + ids.length === 0 + ? [] + : await sql>` + SELECT id, name + FROM workspace + WHERE id = ANY(${ids}) + ` + const actualById = new Map(rows.map((row) => [row.id, row.name])) + for (const workspace of expected) { + if (actualById.get(workspace.id) !== workspace.name) { + throw new Error(`Manifest workspace changed or disappeared: ${workspace.id}`) + } + } + if (rows.length !== expected.length) { + throw new Error('Manifest workspace inventory contains unexpected duplicates') + } + } finally { + await sql.end() + } +} diff --git a/apps/sim/e2e/support/process.ts b/apps/sim/e2e/support/process.ts index 406502275af..bf270e15e39 100644 --- a/apps/sim/e2e/support/process.ts +++ b/apps/sim/e2e/support/process.ts @@ -1,7 +1,9 @@ -import { type ChildProcess, spawn, spawnSync } from 'node:child_process' +import { type ChildProcess, spawn } from 'node:child_process' import { closeSync, mkdirSync, openSync } from 'node:fs' import { createServer } from 'node:net' import path from 'node:path' +import { sleep } from '@sim/utils/helpers' +import { isProcessGroupAlive } from './signal-cleanup' export interface CommandOptions { name: string @@ -53,7 +55,7 @@ export function spawnManagedProcess(options: CommandOptions): ManagedProcess { const finalize = (result: ProcessCompletion): void => { if (finalized) return finalized = true - activeProcesses.delete(managed) + if (!child.pid || !isProcessGroupAlive(child.pid)) activeProcesses.delete(managed) closeSync(logFd) resolveCompletion(result) } @@ -66,6 +68,7 @@ export function spawnManagedProcess(options: CommandOptions): ManagedProcess { export async function runCommand(options: CommandOptions): Promise { const managed = spawnManagedProcess(options) const result = await managed.completion + await managed.stop() if (result.error) { throw new Error(`${options.name} failed to spawn. See ${managed.logPath}`, { cause: result.error, @@ -144,26 +147,35 @@ async function stopProcess(managed: ManagedProcess): Promise { const { child } = managed if (!child.pid) { await managed.completion + activeProcesses.delete(managed) + return + } + if (!isProcessGroupAlive(child.pid)) { + activeProcesses.delete(managed) return } - if (child.exitCode !== null || child.signalCode !== null) return const processIds = [child.pid] sendSignalToPids(processIds, 'SIGTERM') - const stopped = await Promise.race([ - managed.completion.then(() => true), - new Promise((resolve) => setTimeout(() => resolve(false), 5_000)), - ]) - if (stopped) return - - sendSignalToPids(processIds.filter(isPidRunning), 'SIGKILL') - const killed = await Promise.race([ - managed.completion.then(() => true), - new Promise((resolve) => setTimeout(() => resolve(false), 2_000)), - ]) - if (!killed) { - throw new Error(`Managed process ${managed.name} did not exit after SIGKILL`) + if (await waitForProcessGroupExit(child.pid, 5_000)) { + activeProcesses.delete(managed) + return + } + + sendSignalToPids(processIds, 'SIGKILL') + if (!(await waitForProcessGroupExit(child.pid, 2_000))) { + throw new Error(`Managed process group ${managed.name} survived SIGKILL`) + } + activeProcesses.delete(managed) +} + +async function waitForProcessGroupExit(groupId: number, timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + if (!isProcessGroupAlive(groupId)) return true + await sleep(50) } + return !isProcessGroupAlive(groupId) } function sendSignalToPids(processIds: number[], signal: NodeJS.Signals): void { @@ -188,35 +200,3 @@ function sendSignalToPids(processIds: number[], signal: NodeJS.Signals): void { } } } - -function isPidRunning(processId: number): boolean { - return getRunningPids([processId]).length > 0 -} - -function getRunningPids(processIds: number[]): number[] { - if (processIds.length === 0) return [] - if (process.platform === 'win32') { - return processIds.filter((processId) => { - try { - process.kill(processId, 0) - return true - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ESRCH') return false - throw error - } - }) - } - - const status = spawnSync('ps', ['-o', 'pid=,stat=', '-p', processIds.join(',')], { - encoding: 'utf8', - env: { NODE_ENV: 'test', PATH: process.env.PATH ?? '' }, - }) - if (status.status !== 0 && status.status !== 1) { - throw new Error(`Unable to inspect managed E2E processes: ${status.stderr}`) - } - return status.stdout.split('\n').flatMap((line) => { - const [pidText, processStatus] = line.trim().split(/\s+/) - const pid = Number(pidText) - return Number.isInteger(pid) && !processStatus?.startsWith('Z') ? [pid] : [] - }) -} diff --git a/apps/sim/e2e/support/run-lock.ts b/apps/sim/e2e/support/run-lock.ts new file mode 100644 index 00000000000..4b67424e988 --- /dev/null +++ b/apps/sim/e2e/support/run-lock.ts @@ -0,0 +1,168 @@ +import { spawnSync } from 'node:child_process' +import { randomUUID } from 'node:crypto' +import { + existsSync, + mkdirSync, + readFileSync, + renameSync, + rmSync, + statSync, + writeFileSync, +} from 'node:fs' +import path from 'node:path' +import { E2E_CACHE_DIR } from './paths' + +const OWNER_FILE = 'owner.json' +const ACQUISITION_GRACE_MS = 2_000 +const ACQUISITION_RETRIES = 10 +const ACQUISITION_RETRY_MS = 10 + +interface RunLockDescriptor { + pid: number + token: string + startedAt: string + processStartIdentity: string | null + retainedFailure?: string +} + +export interface E2eRunLock { + path: string + token: string + transfer(pid: number): void + retain(reason: string): void + release(): void +} + +export function acquireE2eRunLock( + lockPath = path.join(E2E_CACHE_DIR, 'orchestrator.lock') +): E2eRunLock { + mkdirSync(path.dirname(lockPath), { recursive: true }) + const descriptor: RunLockDescriptor = { + pid: process.pid, + token: randomUUID(), + startedAt: new Date().toISOString(), + processStartIdentity: readProcessStartIdentity(process.pid), + } + + for (let attempt = 0; attempt < ACQUISITION_RETRIES; attempt += 1) { + try { + mkdirSync(lockPath, { mode: 0o700 }) + writeDescriptor(lockPath, descriptor) + return { + path: lockPath, + token: descriptor.token, + transfer(pid: number): void { + const current = readDescriptor(lockPath) + if (current?.token !== descriptor.token) return + writeDescriptor(lockPath, { + ...current, + pid, + processStartIdentity: readProcessStartIdentity(pid), + }) + }, + retain(reason: string): void { + retainE2eRunLock(lockPath, descriptor.token, reason) + }, + release(): void { + releaseE2eRunLock(lockPath, descriptor.token) + }, + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error + const existing = readDescriptor(lockPath) + if (existing?.retainedFailure) { + throw new Error( + `Previous E2E cleanup failed and retained ${lockPath}: ${existing.retainedFailure}. Remove the lock only after manual cleanup.` + ) + } + if (existing && isLockOwnerAlive(existing)) { + throw new Error( + `Another E2E orchestrator owns ${lockPath} (PID ${existing.pid}, started ${existing.startedAt})` + ) + } + if (!existing && lockAgeMs(lockPath) < ACQUISITION_GRACE_MS) { + if (attempt === ACQUISITION_RETRIES - 1) { + throw new Error(`Another E2E orchestrator is acquiring ${lockPath}`) + } + sleepSync(ACQUISITION_RETRY_MS) + continue + } + rmSync(lockPath, { recursive: true, force: true }) + } + } + throw new Error(`Unable to acquire E2E orchestrator lock: ${lockPath}`) +} + +export function releaseE2eRunLock(lockPath: string, token: string): void { + if (!existsSync(lockPath)) return + const current = readDescriptor(lockPath) + if (current?.token === token) { + rmSync(lockPath, { recursive: true, force: true }) + } +} + +export function retainE2eRunLock(lockPath: string, token: string, reason: string): void { + const current = readDescriptor(lockPath) + if (current?.token !== token) return + writeDescriptor(lockPath, { ...current, retainedFailure: reason }) +} + +function readDescriptor(lockPath: string): RunLockDescriptor | null { + try { + const parsed = JSON.parse( + readFileSync(path.join(lockPath, OWNER_FILE), 'utf8') + ) as Partial + return typeof parsed.pid === 'number' && + typeof parsed.token === 'string' && + typeof parsed.startedAt === 'string' && + (typeof parsed.processStartIdentity === 'string' || parsed.processStartIdentity === null) && + (parsed.retainedFailure === undefined || typeof parsed.retainedFailure === 'string') + ? (parsed as RunLockDescriptor) + : null + } catch { + return null + } +} + +function writeDescriptor(lockPath: string, descriptor: RunLockDescriptor): void { + const temporary = path.join(lockPath, `${OWNER_FILE}.tmp-${process.pid}`) + writeFileSync(temporary, `${JSON.stringify(descriptor)}\n`, { mode: 0o600 }) + renameSync(temporary, path.join(lockPath, OWNER_FILE)) +} + +function lockAgeMs(lockPath: string): number { + try { + return Date.now() - statSync(lockPath).mtimeMs + } catch { + return 0 + } +} + +function sleepSync(milliseconds: number): void { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, milliseconds) +} + +function isProcessAlive(pid: number): boolean { + try { + process.kill(pid, 0) + return true + } catch (error) { + return (error as NodeJS.ErrnoException).code === 'EPERM' + } +} + +function isLockOwnerAlive(descriptor: RunLockDescriptor): boolean { + if (!isProcessAlive(descriptor.pid)) return false + if (!descriptor.processStartIdentity) return true + return readProcessStartIdentity(descriptor.pid) === descriptor.processStartIdentity +} + +function readProcessStartIdentity(pid: number): string | null { + const result = spawnSync('ps', ['-o', 'lstart=', '-p', String(pid)], { + encoding: 'utf8', + env: { NODE_ENV: 'test', PATH: process.env.PATH ?? '' }, + }) + if (result.status !== 0) return null + const identity = result.stdout.trim() + return identity || null +} diff --git a/apps/sim/e2e/support/runtime-secrets.ts b/apps/sim/e2e/support/runtime-secrets.ts new file mode 100644 index 00000000000..9cc5aeaed8e --- /dev/null +++ b/apps/sim/e2e/support/runtime-secrets.ts @@ -0,0 +1,29 @@ +import { randomBytes } from 'node:crypto' + +export const FOUNDATION_TEST_PASSWORD = 'E2eFoundation1!' + +export interface E2eRuntimeSecrets { + betterAuthSecret: string + encryptionKey: string + apiEncryptionKey: string + internalApiSecret: string + adminApiKey: string + stripeSecretKey: string + stripeWebhookSecret: string +} + +export function createE2eRuntimeSecrets(): E2eRuntimeSecrets { + return { + betterAuthSecret: randomBytes(32).toString('hex'), + encryptionKey: randomBytes(32).toString('hex'), + apiEncryptionKey: randomBytes(32).toString('hex'), + internalApiSecret: randomBytes(32).toString('hex'), + adminApiKey: randomBytes(32).toString('hex'), + stripeSecretKey: `sk_test_sim_e2e_${randomBytes(24).toString('hex')}`, + stripeWebhookSecret: `whsec_sim_e2e_${randomBytes(24).toString('hex')}`, + } +} + +export function runtimeSecretValues(secrets: E2eRuntimeSecrets): string[] { + return Object.values(secrets) +} diff --git a/apps/sim/e2e/support/sandbox-bundles.ts b/apps/sim/e2e/support/sandbox-bundles.ts index 66528bb4e57..eef0abf023d 100644 --- a/apps/sim/e2e/support/sandbox-bundles.ts +++ b/apps/sim/e2e/support/sandbox-bundles.ts @@ -13,7 +13,7 @@ interface SandboxBundleIntegrity { outputs: Record } -export function verifySandboxBundleIntegrity(): void { +export function verifySandboxBundleIntegrity(options?: { runningBunVersion?: string }): void { const integrity = readJson(path.join(REPO_ROOT, INTEGRITY_PATH)) if (integrity.schemaVersion !== 1) { throw new Error(`Unsupported sandbox bundle integrity version: ${integrity.schemaVersion}`) @@ -26,6 +26,12 @@ export function verifySandboxBundleIntegrity(): void { `Sandbox bundle Bun fingerprint is ${integrity.bunVersion}, but package.json declares ${repositoryPackage.packageManager ?? 'nothing'}` ) } + const runningBunVersion = options?.runningBunVersion ?? process.versions.bun + if (runningBunVersion !== integrity.bunVersion) { + throw new Error( + `Sandbox bundles require Bun ${integrity.bunVersion}, but verification is running under ${runningBunVersion ?? 'Node'}` + ) + } verifyHashes('source', integrity.sources) verifyHashes('output', integrity.outputs) diff --git a/apps/sim/e2e/support/seed-safety.ts b/apps/sim/e2e/support/seed-safety.ts new file mode 100644 index 00000000000..a5edf8d43ba --- /dev/null +++ b/apps/sim/e2e/support/seed-safety.ts @@ -0,0 +1,33 @@ +import { + assertLoopbackPostgresUrl, + assertSafeDatabaseName, + createRunDatabaseName, +} from './database' +import { E2E_ORIGIN, E2E_PROFILE } from './deployment-profile' + +export function assertSafeSeedEnvironment(environment: { + E2E_ORCHESTRATED: string + E2E_PROFILE: string + E2E_RUN_ID: string + E2E_BASE_URL: string + DATABASE_URL: string +}): void { + if (environment.E2E_ORCHESTRATED !== '1') { + throw new Error('seed-world must run under the guarded E2E orchestrator') + } + if (environment.E2E_PROFILE !== E2E_PROFILE) { + throw new Error(`seed-world requires profile ${E2E_PROFILE}`) + } + if (environment.E2E_BASE_URL !== E2E_ORIGIN) { + throw new Error(`seed-world requires origin ${E2E_ORIGIN}`) + } + const databaseUrl = assertLoopbackPostgresUrl(environment.DATABASE_URL) + const databaseName = decodeURIComponent(databaseUrl.pathname.replace(/^\//, '')) + assertSafeDatabaseName(databaseName) + const expectedName = createRunDatabaseName(environment.E2E_RUN_ID) + if (databaseName !== expectedName) { + throw new Error( + `seed-world database ${databaseName} does not match guarded run ${environment.E2E_RUN_ID}` + ) + } +} diff --git a/apps/sim/e2e/support/signal-cleanup.ts b/apps/sim/e2e/support/signal-cleanup.ts index 18fcfe1e0d8..d98e08b105c 100644 --- a/apps/sim/e2e/support/signal-cleanup.ts +++ b/apps/sim/e2e/support/signal-cleanup.ts @@ -6,3 +6,14 @@ export function parseProcessGroupIds(rawValue: string | undefined): number[] { .map(Number) .filter((value) => Number.isInteger(value) && value > 0) } + +export function isProcessGroupAlive(groupId: number): boolean { + if (!Number.isInteger(groupId) || groupId <= 0) return false + try { + if (process.platform !== 'win32') process.kill(-groupId, 0) + else process.kill(groupId, 0) + return true + } catch (error) { + return (error as NodeJS.ErrnoException).code === 'EPERM' + } +} diff --git a/apps/sim/e2e/support/stack.ts b/apps/sim/e2e/support/stack.ts index 1832474e551..222aafd4eb7 100644 --- a/apps/sim/e2e/support/stack.ts +++ b/apps/sim/e2e/support/stack.ts @@ -1,8 +1,9 @@ -import { mkdirSync, writeFileSync } from 'node:fs' +import { mkdirSync, rmSync, writeFileSync } from 'node:fs' import path from 'node:path' import { clearActiveNextBuild, computeBuildIdentity, + pruneBuildCache, restoreCachedBuild, storeCompletedBuild, } from './build-manifest' @@ -24,9 +25,10 @@ export interface StackCommandOptions { logsDirectory: string } -export interface BuildAppOptions extends StackCommandOptions { +export interface BuildAppOptions extends Omit { buildEnvironment: ChildEnvironment reuseBuild: boolean + ci: boolean } export async function runMigrations(options: StackCommandOptions): Promise { @@ -64,11 +66,15 @@ export async function capturePersonaAuthStates(options: StackCommandOptions): Pr export async function buildApp(options: BuildAppOptions): Promise { verifySandboxBundleIntegrity() - const identity = computeBuildIdentity({ - buildEnvironment: options.buildEnvironment, - nodeExecutable: options.nodeExecutable, - }) + const identity = + options.ci || !options.reuseBuild + ? null + : computeBuildIdentity({ + buildEnvironment: options.buildEnvironment, + nodeExecutable: options.nodeExecutable, + }) if (options.reuseBuild) { + if (!identity) throw new Error('CI cannot restore a local E2E build cache') const reuseDecision = restoreCachedBuild(identity) writeBuildDecision(options.logsDirectory, reuseDecision) if (reuseDecision.reused) { @@ -80,7 +86,10 @@ export async function buildApp(options: BuildAppOptions): Promise { clearActiveNextBuild() const buildHome = options.buildEnvironment.env.HOME - if (buildHome) mkdirSync(buildHome, { recursive: true }) + if (buildHome) { + rmSync(buildHome, { recursive: true, force: true }) + mkdirSync(buildHome, { recursive: true, mode: 0o700 }) + } await runCommand({ name: 'next-build', command: options.nodeExecutable, @@ -89,8 +98,18 @@ export async function buildApp(options: BuildAppOptions): Promise { env: options.buildEnvironment.env, logsDirectory: options.logsDirectory, }) - const storedDecision = storeCompletedBuild(identity) - writeBuildDecision(options.logsDirectory, storedDecision) + if (identity) { + const storedDecision = storeCompletedBuild(identity) + pruneBuildCache(identity.nextBuildHash) + writeBuildDecision(options.logsDirectory, storedDecision) + } else { + writeBuildDecision(options.logsDirectory, { + reused: false, + reason: options.ci + ? 'CI build cache population is disabled' + : 'local cache population requires --reuse-build', + }) + } } export async function startRealtime(options: StackCommandOptions): Promise { diff --git a/apps/sim/lib/execution/sandbox/bundles/build.ts b/apps/sim/lib/execution/sandbox/bundles/build.ts index 4b6e0368028..b10a90296dc 100644 --- a/apps/sim/lib/execution/sandbox/bundles/build.ts +++ b/apps/sim/lib/execution/sandbox/bundles/build.ts @@ -10,7 +10,8 @@ * Run via: `bun run build:sandbox-bundles`. */ -import { mkdirSync, rmSync, writeFileSync } from 'node:fs' +import { createHash } from 'node:crypto' +import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { dirname, isAbsolute, join, resolve } from 'node:path' import { fileURLToPath } from 'node:url' import { createLogger } from '@sim/logger' @@ -34,6 +35,10 @@ declare const Bun: { build: (opts: BunBuildOptions) => Promise } const HERE = dirname(fileURLToPath(import.meta.url)) const APP_SIM_ROOT = join(HERE, '..', '..', '..', '..') +const REPO_ROOT = join(APP_SIM_ROOT, '..', '..') +const INTEGRITY_PATH = join(HERE, 'integrity.json') +const INTEGRITY_SOURCES = [join(HERE, 'build.ts'), join(HERE, '_polyfills.ts')] as const +const INTEGRITY_DEPENDENCIES = ['pdf-lib', 'docx', 'pptxgenjs', 'buffer', 'process'] as const interface BundleSpec { /** Key on `globalThis.__bundles`. */ @@ -89,7 +94,7 @@ globalThis.__bundles['pptxgenjs'] = PptxGenJS ] async function main(): Promise { - const outputDirectory = resolveOutputDirectory(process.argv.slice(2)) + const { outputDirectory, writeIntegrity } = resolveOptions(process.argv.slice(2)) const entriesDirectory = join(outputDirectory, '.entries') rmSync(entriesDirectory, { recursive: true, force: true }) @@ -127,18 +132,81 @@ async function main(): Promise { } rmSync(entriesDirectory, { recursive: true, force: true }) + if (writeIntegrity) { + if (outputDirectory !== HERE) { + throw new Error('--write-integrity requires the committed bundle output directory') + } + writeIntegrityManifest() + logger.info('updated integrity.json') + } } -function resolveOutputDirectory(args: string[]): string { - if (args.length === 0) return HERE - if (args.length !== 1 || !args[0].startsWith('--output-dir=')) { - throw new Error('Usage: build.ts [--output-dir=/absolute/path]') +function resolveOptions(args: string[]): { + outputDirectory: string + writeIntegrity: boolean +} { + const outputOption = args.find((arg) => arg.startsWith('--output-dir=')) + const unknown = args.filter( + (arg) => arg !== '--write-integrity' && !arg.startsWith('--output-dir=') + ) + if ( + unknown.length > 0 || + args.filter((arg) => arg === '--write-integrity').length > 1 || + args.filter((arg) => arg.startsWith('--output-dir=')).length > 1 + ) { + throw new Error('Usage: build.ts [--output-dir=/absolute/path] [--write-integrity]') } - const requested = args[0].slice('--output-dir='.length) - if (!requested || !isAbsolute(requested)) { + const requested = outputOption?.slice('--output-dir='.length) + if (requested !== undefined && (!requested || !isAbsolute(requested))) { throw new Error('--output-dir must be an absolute path') } - return resolve(requested) + return { + outputDirectory: requested ? resolve(requested) : HERE, + writeIntegrity: args.includes('--write-integrity'), + } +} + +function writeIntegrityManifest(): void { + const repositoryPackage = readJson<{ packageManager?: string }>(join(REPO_ROOT, 'package.json')) + const declaredBunVersion = repositoryPackage.packageManager?.match(/^bun@(.+)$/)?.[1] + if (!declaredBunVersion) throw new Error('package.json must declare a Bun packageManager version') + if (process.versions.bun !== declaredBunVersion) { + throw new Error( + `Sandbox bundle integrity must be generated with Bun ${declaredBunVersion}; running ${process.versions.bun ?? 'Node'}` + ) + } + + const relative = (filePath: string): string => + filePath.slice(REPO_ROOT.length + 1).replaceAll('\\', '/') + const manifest = { + schemaVersion: 1, + bunVersion: declaredBunVersion, + sources: Object.fromEntries( + INTEGRITY_SOURCES.map((filePath) => [relative(filePath), hashFile(filePath)]) + ), + dependencies: Object.fromEntries( + INTEGRITY_DEPENDENCIES.map((packageName) => [ + packageName, + readJson<{ version: string }>(join(REPO_ROOT, 'node_modules', packageName, 'package.json')) + .version, + ]) + ), + outputs: Object.fromEntries( + BUNDLES.map(({ outFile }) => { + const filePath = join(HERE, outFile) + return [relative(filePath), hashFile(filePath)] + }) + ), + } + writeFileSync(INTEGRITY_PATH, `${JSON.stringify(manifest, null, 2)}\n`) +} + +function hashFile(filePath: string): string { + return createHash('sha256').update(readFileSync(filePath)).digest('hex') +} + +function readJson(filePath: string): T { + return JSON.parse(readFileSync(filePath, 'utf8')) as T } main().catch((err) => { diff --git a/apps/sim/lib/execution/sandbox/bundles/integrity.json b/apps/sim/lib/execution/sandbox/bundles/integrity.json index 2ef8966b49d..c2c5d055dd1 100644 --- a/apps/sim/lib/execution/sandbox/bundles/integrity.json +++ b/apps/sim/lib/execution/sandbox/bundles/integrity.json @@ -2,7 +2,7 @@ "schemaVersion": 1, "bunVersion": "1.3.13", "sources": { - "apps/sim/lib/execution/sandbox/bundles/build.ts": "da99fa40ae36a694c45a5f63f901ec8132f653184f3a2de87ff39fe3ac1e7133", + "apps/sim/lib/execution/sandbox/bundles/build.ts": "5ac7773ab65451f908fe54ba609978469117fcaa61cff47be973d7ba528a9982", "apps/sim/lib/execution/sandbox/bundles/_polyfills.ts": "131dd6e7aec37bff3acf6507d6fe7bc95aeabae07fa2e97bbf2e32aa7251d491" }, "dependencies": { diff --git a/apps/sim/package.json b/apps/sim/package.json index 288bf38b3c7..8e4308f079e 100644 --- a/apps/sim/package.json +++ b/apps/sim/package.json @@ -19,6 +19,7 @@ "load:workflow:isolation": "BASE_URL=${BASE_URL:-http://localhost:3000} ISOLATION_DURATION=${ISOLATION_DURATION:-30} TOTAL_RATE=${TOTAL_RATE:-9} WORKSPACE_A_WEIGHT=${WORKSPACE_A_WEIGHT:-8} WORKSPACE_B_WEIGHT=${WORKSPACE_B_WEIGHT:-1} bunx artillery run scripts/load/workflow-isolation.yml", "build": "bun run build:sandbox-bundles && NODE_OPTIONS='--max-old-space-size=8192' next build", "build:sandbox-bundles": "bun run ./lib/execution/sandbox/bundles/build.ts", + "build:sandbox-bundles:integrity": "bun run ./lib/execution/sandbox/bundles/build.ts --write-integrity", "start": "next start", "prepare": "cd ../.. && bun husky", "test": "vitest run", @@ -250,6 +251,7 @@ "postcss": "^8", "react-email": "4.3.2", "tailwindcss": "^3.4.1", + "tsx": "4.23.1", "typescript": "^7.0.2", "vite-tsconfig-paths": "^5.1.4", "vitest": "^4.1.0" diff --git a/apps/sim/playwright.config.ts b/apps/sim/playwright.config.ts index da61f3ac305..8ea60e16de9 100644 --- a/apps/sim/playwright.config.ts +++ b/apps/sim/playwright.config.ts @@ -51,18 +51,21 @@ export default defineConfig({ { name: 'hosted-billing-chromium-workflows', testMatch: ['**/settings/smoke/authenticated.spec.ts', '**/settings/workflows/**/*.spec.ts'], + dependencies: ['hosted-billing-chromium-navigation'], fullyParallel: false, workers: 1, }, { name: 'hosted-billing-chromium-personas', testMatch: '**/settings/persona-contracts.spec.ts', + dependencies: ['hosted-billing-chromium-workflows'], fullyParallel: false, workers: 1, }, { name: 'hosted-billing-chromium-persona-isolation', testMatch: '**/settings/persona-isolation.spec.ts', + dependencies: ['hosted-billing-chromium-personas'], fullyParallel: true, workers: 2, }, diff --git a/bun.lock b/bun.lock index 51ef90840cb..8e9da57756b 100644 --- a/bun.lock +++ b/bun.lock @@ -1,5 +1,6 @@ { "lockfileVersion": 1, + "configVersion": 0, "workspaces": { "": { "name": "simstudio", @@ -316,6 +317,7 @@ "postcss": "^8", "react-email": "4.3.2", "tailwindcss": "^3.4.1", + "tsx": "4.23.1", "typescript": "^7.0.2", "vite-tsconfig-paths": "^5.1.4", "vitest": "^4.1.0", @@ -3088,7 +3090,7 @@ "lru.min": ["lru.min@1.1.4", "", {}, "sha512-DqC6n3QQ77zdFpCMASA1a3Jlb64Hv2N2DciFGkO/4L9+q/IpIAuRlKOvCXabtRW6cQf8usbmM6BE/TOPysCdIA=="], - "lucide-react": ["lucide-react@1.23.0", "", { "peerDependencies": { "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-38BpJcD0JhFosxHApP/BYsBetLpQFRoTRzEzstM/XCc3jsAG7wqaY1lgVwxiUe3xqYE+lNxo2PkCmYwXWrwwIw=="], + "lucide-react": ["lucide-react@0.479.0", "", { "peerDependencies": { "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-aBhNnveRhorBOK7uA4gDjgaf+YlHMdMhQ/3cupk6exM10hWlEU+2QtWYOfhXhjAsmdb6LeKR+NZnow4UxRRiTQ=="], "magic-string": ["magic-string@0.30.21", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" } }, "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ=="], @@ -3982,7 +3984,7 @@ "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], - "tsx": ["tsx@4.22.4", "", { "dependencies": { "esbuild": "~0.28.0" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "bin": { "tsx": "dist/cli.mjs" } }, "sha512-X8EX+XV4QR5xCsrgxaED954zTDfY8KqlDtskKEL0cHhyS/P8b4IFOvGDQpsC9Q1XnLq915wEfwwY/zzskCtmhg=="], + "tsx": ["tsx@4.23.1", "", { "dependencies": { "esbuild": "~0.28.0" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "bin": { "tsx": "dist/cli.mjs" } }, "sha512-GQHnkIfxyx1wYCOS/wonik5MVRZU9hi1TEZmzGZSCJB1y9YgoZ8H6itNE/u4suE+yLmOzuE4E5S4TZ/ZX2wcWQ=="], "tunnel-agent": ["tunnel-agent@0.6.0", "", { "dependencies": { "safe-buffer": "^5.0.1" } }, "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w=="], @@ -4468,10 +4470,6 @@ "@sim/db/@types/node": ["@types/node@22.19.21", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-VMeFBSCKQKmm2swI2kW51SFusDqekC6q9trBCvJ/JliDchFSuoYYKN7yVNjPthP1HKZcx3U1gI/wTcEBjEFKTA=="], - "@sim/emcn/lucide-react": ["lucide-react@0.479.0", "", { "peerDependencies": { "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-aBhNnveRhorBOK7uA4gDjgaf+YlHMdMhQ/3cupk6exM10hWlEU+2QtWYOfhXhjAsmdb6LeKR+NZnow4UxRRiTQ=="], - - "@sim/workflow-renderer/lucide-react": ["lucide-react@0.479.0", "", { "peerDependencies": { "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-aBhNnveRhorBOK7uA4gDjgaf+YlHMdMhQ/3cupk6exM10hWlEU+2QtWYOfhXhjAsmdb6LeKR+NZnow4UxRRiTQ=="], - "@smithy/middleware-compression/fflate": ["fflate@0.8.1", "", {}, "sha512-/exOvEuc+/iaUm105QIiOt4LpBdMTWsXxqR0HDF35vx3fmaKzw7354gTilCh5rkzEt8WYyG//ku3h3nRmd7CHQ=="], "@socket.io/redis-adapter/debug": ["debug@4.3.7", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ=="], @@ -4626,6 +4624,8 @@ "dom-serializer/entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="], + "drizzle-kit/tsx": ["tsx@4.22.4", "", { "dependencies": { "esbuild": "~0.28.0" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "bin": { "tsx": "dist/cli.mjs" } }, "sha512-X8EX+XV4QR5xCsrgxaED954zTDfY8KqlDtskKEL0cHhyS/P8b4IFOvGDQpsC9Q1XnLq915wEfwwY/zzskCtmhg=="], + "duplexify/readable-stream": ["readable-stream@3.6.2", "", { "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", "util-deprecate": "^1.0.1" } }, "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA=="], "e2b/glob": ["glob@11.1.0", "", { "dependencies": { "foreground-child": "^3.3.1", "jackspeak": "^4.1.1", "minimatch": "^10.1.1", "minipass": "^7.1.2", "package-json-from-dist": "^1.0.0", "path-scurry": "^2.0.0" }, "bin": { "glob": "dist/esm/bin.mjs" } }, "sha512-vuNwKSaKiqm7g0THUBu2x7ckSs3XJLXE+2ssL7/MfTGPLLcrJQ/4Uq1CjPTtO5cCIiRxqvN6Twy1qOwhL0Xjcw=="], @@ -4670,10 +4670,14 @@ "fumadocs-openapi/ajv": ["ajv@8.20.0", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA=="], + "fumadocs-openapi/lucide-react": ["lucide-react@1.23.0", "", { "peerDependencies": { "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-38BpJcD0JhFosxHApP/BYsBetLpQFRoTRzEzstM/XCc3jsAG7wqaY1lgVwxiUe3xqYE+lNxo2PkCmYwXWrwwIw=="], + "fumadocs-openapi/tailwind-merge": ["tailwind-merge@3.6.0", "", {}, "sha512-uxL7qAVQriqRQPAyK3pj66VqskWqoZ37PW94jwOTwNfq/z9oyu1V+eqrZqtR2+fCiXdYOZe/Modt8GtvqNzu+w=="], "fumadocs-ui/@radix-ui/react-slot": ["@radix-ui/react-slot@1.3.0", "", { "dependencies": { "@radix-ui/react-compose-refs": "1.1.3" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-MojKku4U/miO8Av4Dkb+ctMAQx7JmY96LmtDQlAarCRtd7rN52QCSzBF+XAvr5S6coSVj9HEPBgHAHKEJVk/WA=="], + "fumadocs-ui/lucide-react": ["lucide-react@1.23.0", "", { "peerDependencies": { "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-38BpJcD0JhFosxHApP/BYsBetLpQFRoTRzEzstM/XCc3jsAG7wqaY1lgVwxiUe3xqYE+lNxo2PkCmYwXWrwwIw=="], + "fumadocs-ui/tailwind-merge": ["tailwind-merge@3.6.0", "", {}, "sha512-uxL7qAVQriqRQPAyK3pj66VqskWqoZ37PW94jwOTwNfq/z9oyu1V+eqrZqtR2+fCiXdYOZe/Modt8GtvqNzu+w=="], "gaxios/node-fetch": ["node-fetch@2.7.0", "", { "dependencies": { "whatwg-url": "^5.0.0" }, "peerDependencies": { "encoding": "^0.1.0" }, "optionalPeers": ["encoding"] }, "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A=="], @@ -4830,8 +4834,6 @@ "sharp/semver": ["semver@7.8.4", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-rUCObTnP32Q08R2uuIrt7r9PlEonuTmtuXYcW6s5kjdlj3xbnwe+21yXptAUYcMAABLkYYTtnmzb3w3EDZfueA=="], - "sim/lucide-react": ["lucide-react@0.479.0", "", { "peerDependencies": { "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-aBhNnveRhorBOK7uA4gDjgaf+YlHMdMhQ/3cupk6exM10hWlEU+2QtWYOfhXhjAsmdb6LeKR+NZnow4UxRRiTQ=="], - "sim/tailwindcss": ["tailwindcss@3.4.19", "", { "dependencies": { "@alloc/quick-lru": "^5.2.0", "arg": "^5.0.2", "chokidar": "^3.6.0", "didyoumean": "^1.2.2", "dlv": "^1.1.3", "fast-glob": "^3.3.2", "glob-parent": "^6.0.2", "is-glob": "^4.0.3", "jiti": "^1.21.7", "lilconfig": "^3.1.3", "micromatch": "^4.0.8", "normalize-path": "^3.0.0", "object-hash": "^3.0.0", "picocolors": "^1.1.1", "postcss": "^8.4.47", "postcss-import": "^15.1.0", "postcss-js": "^4.0.1", "postcss-load-config": "^4.0.2 || ^5.0 || ^6.0", "postcss-nested": "^6.2.0", "postcss-selector-parser": "^6.1.2", "resolve": "^1.22.8", "sucrase": "^3.35.0" }, "bin": { "tailwind": "lib/cli.js", "tailwindcss": "lib/cli.js" } }, "sha512-3ofp+LL8E+pK/JuPLPggVAIaEuhvIz4qNcf3nA1Xn2o/7fb7s/TYpHhwGDv1ZU3PkBluUVaF8PyCHcm48cKLWQ=="], "simstudio/@types/node": ["@types/node@20.19.43", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-6oYBAi5ikg4Pl+kGsoYtawUMBT2zZMCvPNF7pVLnHZfd1zf38DRiWn/gT01RYCdUqkv7Fhr+C9ot4/tb+2sVvA=="], @@ -5096,6 +5098,8 @@ "docx/@types/node/undici-types": ["undici-types@7.24.6", "", {}, "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg=="], + "drizzle-kit/tsx/esbuild": ["esbuild@0.28.1", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.28.1", "@esbuild/android-arm": "0.28.1", "@esbuild/android-arm64": "0.28.1", "@esbuild/android-x64": "0.28.1", "@esbuild/darwin-arm64": "0.28.1", "@esbuild/darwin-x64": "0.28.1", "@esbuild/freebsd-arm64": "0.28.1", "@esbuild/freebsd-x64": "0.28.1", "@esbuild/linux-arm": "0.28.1", "@esbuild/linux-arm64": "0.28.1", "@esbuild/linux-ia32": "0.28.1", "@esbuild/linux-loong64": "0.28.1", "@esbuild/linux-mips64el": "0.28.1", "@esbuild/linux-ppc64": "0.28.1", "@esbuild/linux-riscv64": "0.28.1", "@esbuild/linux-s390x": "0.28.1", "@esbuild/linux-x64": "0.28.1", "@esbuild/netbsd-arm64": "0.28.1", "@esbuild/netbsd-x64": "0.28.1", "@esbuild/openbsd-arm64": "0.28.1", "@esbuild/openbsd-x64": "0.28.1", "@esbuild/openharmony-arm64": "0.28.1", "@esbuild/sunos-x64": "0.28.1", "@esbuild/win32-arm64": "0.28.1", "@esbuild/win32-ia32": "0.28.1", "@esbuild/win32-x64": "0.28.1" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw=="], + "duplexify/readable-stream/string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="], "engine.io/@types/node/undici-types": ["undici-types@7.24.6", "", {}, "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg=="], @@ -5392,6 +5396,58 @@ "cli-truncate/string-width/strip-ansi/ansi-regex": ["ansi-regex@6.2.2", "", {}, "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg=="], + "drizzle-kit/tsx/esbuild/@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.28.1", "", { "os": "aix", "cpu": "ppc64" }, "sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ=="], + + "drizzle-kit/tsx/esbuild/@esbuild/android-arm": ["@esbuild/android-arm@0.28.1", "", { "os": "android", "cpu": "arm" }, "sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ=="], + + "drizzle-kit/tsx/esbuild/@esbuild/android-arm64": ["@esbuild/android-arm64@0.28.1", "", { "os": "android", "cpu": "arm64" }, "sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg=="], + + "drizzle-kit/tsx/esbuild/@esbuild/android-x64": ["@esbuild/android-x64@0.28.1", "", { "os": "android", "cpu": "x64" }, "sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng=="], + + "drizzle-kit/tsx/esbuild/@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.28.1", "", { "os": "darwin", "cpu": "arm64" }, "sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q=="], + + "drizzle-kit/tsx/esbuild/@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.28.1", "", { "os": "darwin", "cpu": "x64" }, "sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ=="], + + "drizzle-kit/tsx/esbuild/@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.28.1", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw=="], + + "drizzle-kit/tsx/esbuild/@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.28.1", "", { "os": "freebsd", "cpu": "x64" }, "sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ=="], + + "drizzle-kit/tsx/esbuild/@esbuild/linux-arm": ["@esbuild/linux-arm@0.28.1", "", { "os": "linux", "cpu": "arm" }, "sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ=="], + + "drizzle-kit/tsx/esbuild/@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.28.1", "", { "os": "linux", "cpu": "arm64" }, "sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g=="], + + "drizzle-kit/tsx/esbuild/@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.28.1", "", { "os": "linux", "cpu": "ia32" }, "sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w=="], + + "drizzle-kit/tsx/esbuild/@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.28.1", "", { "os": "linux", "cpu": "none" }, "sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg=="], + + "drizzle-kit/tsx/esbuild/@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.28.1", "", { "os": "linux", "cpu": "none" }, "sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ=="], + + "drizzle-kit/tsx/esbuild/@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.28.1", "", { "os": "linux", "cpu": "ppc64" }, "sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ=="], + + "drizzle-kit/tsx/esbuild/@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.28.1", "", { "os": "linux", "cpu": "none" }, "sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ=="], + + "drizzle-kit/tsx/esbuild/@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.28.1", "", { "os": "linux", "cpu": "s390x" }, "sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag=="], + + "drizzle-kit/tsx/esbuild/@esbuild/linux-x64": ["@esbuild/linux-x64@0.28.1", "", { "os": "linux", "cpu": "x64" }, "sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA=="], + + "drizzle-kit/tsx/esbuild/@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.28.1", "", { "os": "none", "cpu": "arm64" }, "sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw=="], + + "drizzle-kit/tsx/esbuild/@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.28.1", "", { "os": "none", "cpu": "x64" }, "sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg=="], + + "drizzle-kit/tsx/esbuild/@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.28.1", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q=="], + + "drizzle-kit/tsx/esbuild/@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.28.1", "", { "os": "openbsd", "cpu": "x64" }, "sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw=="], + + "drizzle-kit/tsx/esbuild/@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.28.1", "", { "os": "none", "cpu": "arm64" }, "sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg=="], + + "drizzle-kit/tsx/esbuild/@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.28.1", "", { "os": "sunos", "cpu": "x64" }, "sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ=="], + + "drizzle-kit/tsx/esbuild/@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.28.1", "", { "os": "win32", "cpu": "arm64" }, "sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA=="], + + "drizzle-kit/tsx/esbuild/@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.28.1", "", { "os": "win32", "cpu": "ia32" }, "sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg=="], + + "drizzle-kit/tsx/esbuild/@esbuild/win32-x64": ["@esbuild/win32-x64@0.28.1", "", { "os": "win32", "cpu": "x64" }, "sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A=="], + "gaxios/node-fetch/whatwg-url/tr46": ["tr46@0.0.3", "", {}, "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="], "gaxios/node-fetch/whatwg-url/webidl-conversions": ["webidl-conversions@3.0.1", "", {}, "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="], From 27878e07ce7e6e10727a77e9ffabd9c08f097fa8 Mon Sep 17 00:00:00 2001 From: Bill Leoutsakos Date: Tue, 21 Jul 2026 13:39:52 -0700 Subject: [PATCH 5/6] Fix CI option policy assertions Make local-only parser scenarios explicit so the safety suite tests the intended policy instead of inheriting the GitHub runner environment. --- apps/sim/e2e/foundation/safety.spec.ts | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/apps/sim/e2e/foundation/safety.spec.ts b/apps/sim/e2e/foundation/safety.spec.ts index a971f624d88..131ff2ed4cf 100644 --- a/apps/sim/e2e/foundation/safety.spec.ts +++ b/apps/sim/e2e/foundation/safety.spec.ts @@ -177,18 +177,23 @@ test.describe('foundation safety guards', () => { ).toThrow(/coupled E2E projects must remain unsharded/) expect(() => parseRunOptions(['--project=hosted-billing-chromium-personas'])).not.toThrow() expect(() => - parseRunOptions(['--project=hosted-billing-chromium-personas', '--no-deps']) + parseRunOptions(['--project=hosted-billing-chromium-personas', '--no-deps'], { ci: false }) ).not.toThrow() expect(() => parseRunOptions(['--project=hosted-billing-chromium-personas', '--no-deps'], { ci: true }) ).toThrow(/--no-deps is local-only/) - expect(() => parseRunOptions(['--no-deps'])).toThrow(/exactly one explicit canonical/) + expect(() => parseRunOptions(['--no-deps'], { ci: false })).toThrow( + /exactly one explicit canonical/ + ) expect(() => - parseRunOptions([ - '--project=hosted-billing-chromium-navigation', - '--project=hosted-billing-chromium-personas', - '--no-deps', - ]) + parseRunOptions( + [ + '--project=hosted-billing-chromium-navigation', + '--project=hosted-billing-chromium-personas', + '--no-deps', + ], + { ci: false } + ) ).toThrow(/exactly one explicit canonical/) expect(() => parseRunOptions(['--project=hosted-billing-chromium-persona-isolation', '--shard=1/2']) From 428a54b9eb3b1ed0ed7ceff98b4e6ba22a500658 Mon Sep 17 00:00:00 2001 From: Bill Leoutsakos Date: Tue, 21 Jul 2026 13:55:16 -0700 Subject: [PATCH 6/6] Fix organization invitation fixture semantics Match production organization invitations even when workspace grants are attached, and remove an unused cookie-reset method from the E2E client. --- apps/sim/e2e/fixtures/factories/invitations.ts | 2 +- apps/sim/e2e/fixtures/http-client.ts | 4 ---- apps/sim/e2e/scripts/seed-world.ts | 2 +- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/apps/sim/e2e/fixtures/factories/invitations.ts b/apps/sim/e2e/fixtures/factories/invitations.ts index 4eda69144b0..76904948baa 100644 --- a/apps/sim/e2e/fixtures/factories/invitations.ts +++ b/apps/sim/e2e/fixtures/factories/invitations.ts @@ -20,7 +20,7 @@ export async function arrangePendingInvitation(input: { await db.transaction(async (tx) => { await tx.insert(invitation).values({ id: invitationId, - kind: 'workspace', + kind: 'organization', email: input.email.trim().toLowerCase(), inviterId: input.inviterId, organizationId: input.organizationId, diff --git a/apps/sim/e2e/fixtures/http-client.ts b/apps/sim/e2e/fixtures/http-client.ts index 54d7c153a83..42cd8034659 100644 --- a/apps/sim/e2e/fixtures/http-client.ts +++ b/apps/sim/e2e/fixtures/http-client.ts @@ -98,10 +98,6 @@ export class E2eHttpClient { return this.serializeCookies() } - clearCookies(): void { - this.cookies.clear() - } - private captureCookies(headers: Headers): void { const values = 'getSetCookie' in headers && typeof headers.getSetCookie === 'function' diff --git a/apps/sim/e2e/scripts/seed-world.ts b/apps/sim/e2e/scripts/seed-world.ts index 4f27bf3f669..530658b991e 100644 --- a/apps/sim/e2e/scripts/seed-world.ts +++ b/apps/sim/e2e/scripts/seed-world.ts @@ -551,7 +551,7 @@ async function assertTrustedWorldInvariants(world: E2EWorld): Promise { .limit(1) if ( !row || - row.kind !== 'workspace' || + row.kind !== 'organization' || row.email !== definition.email || row.inviterId !== required(