From 6ca0929aa2899c574e2d0111b808ef8edde13ae9 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Tue, 15 Sep 2026 01:00:24 -0700 Subject: [PATCH 1/2] Reconcile the membership mirror from WorkOS events --- .changeset/member-directory-reconciler.md | 7 + .../drizzle/0020_workos_sync_drained_at.sql | 1 + apps/cloud/drizzle/meta/0020_snapshot.json | 1760 +++++++++++++++++ apps/cloud/drizzle/meta/_journal.json | 9 +- apps/cloud/package.json | 2 + apps/cloud/scripts/backfill-workos-mirror.ts | 14 +- apps/cloud/scripts/drain-workos-events.ts | 128 ++ .../account/org-api-key-revoke.node.test.ts | 4 +- .../src/auth/mirror-feeders.node.test.ts | 98 +- apps/cloud/src/auth/user-store.ts | 95 +- .../auth/workos-callback-state.node.test.ts | 4 +- apps/cloud/src/auth/workos-events-replay.ts | 556 ++++++ apps/cloud/src/auth/workos-events-runner.ts | 56 + .../src/auth/workos-events-sync.node.test.ts | 1268 ++++++++++++ apps/cloud/src/auth/workos-events-sync.ts | 107 + apps/cloud/src/auth/workos-mirror-backfill.ts | 70 +- apps/cloud/src/auth/workos-mirror-store.ts | 345 +++- .../cloud/src/auth/workos-mirror.node.test.ts | 88 +- apps/cloud/src/auth/workos-mirror.ts | 2 + apps/cloud/src/auth/workos-webhook.ts | 94 + apps/cloud/src/auth/workos.ts | 64 +- apps/cloud/src/db/schema.ts | 47 +- apps/cloud/src/env-augment.d.ts | 8 + apps/cloud/src/extensions/routes.ts | 16 + apps/cloud/src/server.ts | 15 + apps/cloud/wrangler.jsonc | 7 + 26 files changed, 4669 insertions(+), 196 deletions(-) create mode 100644 .changeset/member-directory-reconciler.md create mode 100644 apps/cloud/drizzle/0020_workos_sync_drained_at.sql create mode 100644 apps/cloud/drizzle/meta/0020_snapshot.json create mode 100644 apps/cloud/scripts/drain-workos-events.ts create mode 100644 apps/cloud/src/auth/workos-events-replay.ts create mode 100644 apps/cloud/src/auth/workos-events-runner.ts create mode 100644 apps/cloud/src/auth/workos-events-sync.node.test.ts create mode 100644 apps/cloud/src/auth/workos-events-sync.ts create mode 100644 apps/cloud/src/auth/workos-webhook.ts diff --git a/.changeset/member-directory-reconciler.md b/.changeset/member-directory-reconciler.md new file mode 100644 index 0000000000..b73bbf8786 --- /dev/null +++ b/.changeset/member-directory-reconciler.md @@ -0,0 +1,7 @@ +--- +"@executor-js/cloud": patch +--- + +The cloud membership mirror is now reconciled from the WorkOS Events API: an every-minute cron replays user, organization-membership, and organization events from a persisted cursor, so changes made in the WorkOS dashboard (a removed member, a role edit, a profile update) reach the mirror without anyone signing in. A signed webhook at `/api/webhooks/workos` pokes the same reconciler so those changes land in seconds, and `bun run --cwd apps/cloud db:drain-workos-events:prod` runs the same replay out-of-band until the stream is drained. + +**Ops steps (cloud):** set the webhook signing secret with `wrangler secret put WORKOS_WEBHOOK_SECRET`, then register `https://executor.sh/api/webhooks/workos` as a webhook endpoint in the WorkOS dashboard for the `user.*`, `organization_membership.*`, `organization.updated`, and `organization.deleted` events. Until the secret is set the route answers 503 and the cron alone keeps the mirror current. diff --git a/apps/cloud/drizzle/0020_workos_sync_drained_at.sql b/apps/cloud/drizzle/0020_workos_sync_drained_at.sql new file mode 100644 index 0000000000..f1e1b34c42 --- /dev/null +++ b/apps/cloud/drizzle/0020_workos_sync_drained_at.sql @@ -0,0 +1 @@ +ALTER TABLE "workos_sync" ADD COLUMN "drained_at" timestamp with time zone; \ No newline at end of file diff --git a/apps/cloud/drizzle/meta/0020_snapshot.json b/apps/cloud/drizzle/meta/0020_snapshot.json new file mode 100644 index 0000000000..886f0a7849 --- /dev/null +++ b/apps/cloud/drizzle/meta/0020_snapshot.json @@ -0,0 +1,1760 @@ +{ + "id": "88f2845b-be28-4ad3-92b2-2cac819478e5", + "prevId": "26e5a445-9146-40bb-afaf-0f26bfb00818", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.accounts": { + "name": "accounts", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "first_name": { + "name": "first_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_name": { + "name": "last_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "workos_updated_at": { + "name": "workos_updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_sign_in_at": { + "name": "last_sign_in_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "accounts_email_lower_idx": { + "name": "accounts_email_lower_idx", + "columns": [ + { + "expression": "lower(\"email\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.membership_tombstones": { + "name": "membership_tombstones", + "schema": "", + "columns": { + "membership_id": { + "name": "membership_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "membership_tombstones_organization_id_idx": { + "name": "membership_tombstones_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "membership_tombstones_account_id_accounts_id_fk": { + "name": "membership_tombstones_account_id_accounts_id_fk", + "tableFrom": "membership_tombstones", + "tableTo": "accounts", + "columnsFrom": ["account_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "membership_tombstones_organization_id_organizations_id_fk": { + "name": "membership_tombstones_organization_id_organizations_id_fk", + "tableFrom": "membership_tombstones", + "tableTo": "organizations", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.memberships": { + "name": "memberships", + "schema": "", + "columns": { + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "membership_id": { + "name": "membership_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'member'" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "workos_updated_at": { + "name": "workos_updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "memberships_membership_id_unique": { + "name": "memberships_membership_id_unique", + "columns": [ + { + "expression": "membership_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "memberships_organization_id_idx": { + "name": "memberships_organization_id_idx", + "columns": [ + { + "expression": "organization_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "memberships_account_id_accounts_id_fk": { + "name": "memberships_account_id_accounts_id_fk", + "tableFrom": "memberships", + "tableTo": "accounts", + "columnsFrom": ["account_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "memberships_organization_id_organizations_id_fk": { + "name": "memberships_organization_id_organizations_id_fk", + "tableFrom": "memberships", + "tableTo": "organizations", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "memberships_account_id_organization_id_pk": { + "name": "memberships_account_id_organization_id_pk", + "columns": ["account_id", "organization_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organizations": { + "name": "organizations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "backfilled_at": { + "name": "backfilled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "workos_updated_at": { + "name": "workos_updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "organizations_slug_unique": { + "name": "organizations_slug_unique", + "columns": [ + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.workos_sync": { + "name": "workos_sync", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "cursor": { + "name": "cursor", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "range_start": { + "name": "range_start", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "backfill_completed_at": { + "name": "backfill_completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "drained_at": { + "name": "drained_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.artifact": { + "name": "artifact", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "code": { + "name": "code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "bindings": { + "name": "bindings", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "preview": { + "name": "preview", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "artifact_uidx": { + "name": "artifact_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.blob": { + "name": "blob", + "schema": "", + "columns": { + "namespace": { + "name": "namespace", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "blob_id_uidx": { + "name": "blob_id_uidx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.connection": { + "name": "connection", + "schema": "", + "columns": { + "integration": { + "name": "integration", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "item_ids": { + "name": "item_ids", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "credential_write": { + "name": "credential_write", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "identity_label": { + "name": "identity_label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_health": { + "name": "last_health", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "tools_synced_at": { + "name": "tools_synced_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "oauth_client": { + "name": "oauth_client", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oauth_client_owner": { + "name": "oauth_client_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_item_id": { + "name": "refresh_item_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "oauth_scope": { + "name": "oauth_scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oauth_token_url": { + "name": "oauth_token_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_state": { + "name": "provider_state", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "connection_uidx": { + "name": "connection_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "integration", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.definition": { + "name": "definition", + "schema": "", + "columns": { + "integration": { + "name": "integration", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "connection": { + "name": "connection", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "schema": { + "name": "schema", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "definition_uidx": { + "name": "definition_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "integration", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "connection", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration": { + "name": "integration", + "schema": "", + "columns": { + "slug": { + "name": "slug", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "config": { + "name": "config", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "health_check": { + "name": "health_check", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "config_revised_at": { + "name": "config_revised_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "can_remove": { + "name": "can_remove", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "can_refresh": { + "name": "can_refresh", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "integration_uidx": { + "name": "integration_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client": { + "name": "oauth_client", + "schema": "", + "columns": { + "slug": { + "name": "slug", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "authorization_url": { + "name": "authorization_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_url": { + "name": "token_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "grant": { + "name": "grant", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_secret_item_id": { + "name": "client_secret_item_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "credential_write": { + "name": "credential_write", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "token_endpoint_auth_method": { + "name": "token_endpoint_auth_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resource": { + "name": "resource", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_kind": { + "name": "origin_kind", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_integration": { + "name": "origin_integration", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_issuer": { + "name": "origin_issuer", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_redirect_uri": { + "name": "origin_redirect_uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauth_client_uidx": { + "name": "oauth_client_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_session": { + "name": "oauth_session", + "schema": "", + "columns": { + "state": { + "name": "state", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "client_slug": { + "name": "client_slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "integration": { + "name": "integration", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "redirect_url": { + "name": "redirect_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pkce_verifier": { + "name": "pkce_verifier", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "identity_label": { + "name": "identity_label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauth_session_uidx": { + "name": "oauth_session_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.plugin_storage": { + "name": "plugin_storage", + "schema": "", + "columns": { + "plugin_id": { + "name": "plugin_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "collection": { + "name": "collection", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "plugin_storage_uidx": { + "name": "plugin_storage_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "plugin_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "collection", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.private_executor_cloud_settings": { + "name": "private_executor_cloud_settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "version": { + "name": "version", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "default": "'1.0.0'" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.subject": { + "name": "subject", + "schema": "", + "columns": { + "external_id": { + "name": "external_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "subject_uidx": { + "name": "subject_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tool": { + "name": "tool", + "schema": "", + "columns": { + "integration": { + "name": "integration", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "connection": { + "name": "connection", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "input_schema": { + "name": "input_schema", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "output_schema": { + "name": "output_schema", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "annotations": { + "name": "annotations", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "tool_uidx": { + "name": "tool_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "integration", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "connection", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tool_policy": { + "name": "tool_policy", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "pattern": { + "name": "pattern", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "position": { + "name": "position", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "tool_policy_uidx": { + "name": "tool_policy_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/apps/cloud/drizzle/meta/_journal.json b/apps/cloud/drizzle/meta/_journal.json index 97d6c17734..73842e4d59 100644 --- a/apps/cloud/drizzle/meta/_journal.json +++ b/apps/cloud/drizzle/meta/_journal.json @@ -141,6 +141,13 @@ "when": 1789571259533, "tag": "0019_workos_mirror_sync_state", "breakpoints": true + }, + { + "idx": 20, + "version": "7", + "when": 1789575639971, + "tag": "0020_workos_sync_drained_at", + "breakpoints": true } ] -} \ No newline at end of file +} diff --git a/apps/cloud/package.json b/apps/cloud/package.json index 0c93031685..328d096b52 100644 --- a/apps/cloud/package.json +++ b/apps/cloud/package.json @@ -34,6 +34,8 @@ "db:backfill-subjects:dev": "op run --env-file=.env.op -- bun run scripts/backfill-subjects.ts", "db:backfill-workos-mirror:prod": "op run --env-file=.env.production -- bun run scripts/backfill-workos-mirror.ts", "db:backfill-workos-mirror:dev": "op run --env-file=.env.op -- bun run scripts/backfill-workos-mirror.ts", + "db:drain-workos-events:prod": "op run --env-file=.env.production -- bun run scripts/drain-workos-events.ts", + "db:drain-workos-events:dev": "op run --env-file=.env.op -- bun run scripts/drain-workos-events.ts", "routes:gen": "bun scripts/gen-routes.ts", "vendor-wasm": "bun run scripts/vendor-quickjs-wasm.ts" }, diff --git a/apps/cloud/scripts/backfill-workos-mirror.ts b/apps/cloud/scripts/backfill-workos-mirror.ts index 712138585d..945de792b2 100644 --- a/apps/cloud/scripts/backfill-workos-mirror.ts +++ b/apps/cloud/scripts/backfill-workos-mirror.ts @@ -24,13 +24,13 @@ // // DEPLOY ORDER: run this against production BEFORE deploying the builds that // reconcile from the Events API and read seat counts from the mirror, so no -// request pays for an on-demand scan. The FIRST completed run records the -// events replay boundary (the reconciler's first run reads from it; without -// one it waits); later runs keep it, since only the events stream covers the -// org renames and user deletions between two runs. A run that fails part-way -// keeps the marks of the orgs it finished, records no boundary, and is safe -// to repeat. Verify the printed membership count against the WorkOS -// dashboard. +// request pays for an on-demand scan. The FIRST run records the events +// replay boundary BEFORE it lists anything (the reconciler's first run reads +// from it; without one it waits); later runs — a retry included — keep it, +// since only the events stream covers the org renames and user deletions +// after that instant. A run that fails part-way keeps the marks of the orgs +// it finished and the boundary it recorded, and is safe to repeat. Verify +// the printed membership count against the WorkOS dashboard. // --------------------------------------------------------------------------- import { asc, isNull } from "drizzle-orm"; diff --git a/apps/cloud/scripts/drain-workos-events.ts b/apps/cloud/scripts/drain-workos-events.ts new file mode 100644 index 0000000000..b5cfc6f3db --- /dev/null +++ b/apps/cloud/scripts/drain-workos-events.ts @@ -0,0 +1,128 @@ +// --------------------------------------------------------------------------- +// Out-of-band reconciler run: replay the WorkOS Events API into the +// membership mirror from the persisted cursor until the stream is drained, +// over a plain postgres.js connection under bun — the SAME replay the +// Worker's every-minute cron runs (`src/auth/workos-events-replay.ts`). +// +// bun run db:drain-workos-events:prod # op run --env-file=.env.production +// +// Exists for the deploy gate (`scripts/ensure-workos-mirror-ready.ts`): the +// build that authorizes from the mirror trusts it only once the reconciler +// has drained the stream recently, and the gate must be able to MAKE that +// true itself rather than wait for a cron that may not be deployed yet — +// otherwise the reconciler build could only ever ship ahead of the gated +// one, by hand. Safe to run beside a live cron: a page is applied under the +// cursor's compare-and-set, so whichever run loses the stream writes +// nothing and stops. Runs until the stream is drained or another run owns +// it; a page budget bounds one pass, so a long backlog takes several. Exits +// 0 on a drain, 1 otherwise, with the reason. +// --------------------------------------------------------------------------- + +import { drizzle } from "drizzle-orm/postgres-js"; +import { Effect, Option } from "effect"; +import postgres from "postgres"; +import { WorkOS } from "@workos-inc/node"; + +import { makeUserStore } from "../src/auth/user-store"; +import { replayWorkOsEvents, type WorkOsEventsSyncReport } from "../src/auth/workos-events-replay"; +import { makeWorkOsMirrorStore } from "../src/auth/workos-mirror-store"; + +const connectionString = process.env.DATABASE_URL; +if (!connectionString) { + console.error("DATABASE_URL is not set"); + process.exit(1); +} +const apiKey = process.env.WORKOS_API_KEY; +if (!apiKey) { + console.error("WORKOS_API_KEY is not set"); + process.exit(1); +} + +const usesLocalDatabase = + connectionString.includes("127.0.0.1") || connectionString.includes("localhost"); + +const sql = postgres(connectionString, { + max: 1, + prepare: false, + ...(usesLocalDatabase ? {} : { ssl: "require" as const }), +}); +const db = drizzle(sql); +const workos = new WorkOS(apiKey); +const users = makeUserStore(db); + +// The script boundary: raw SDK / driver promises lifted once, here. Only a +// 404 is the deterministic "gone" the replay acts on; every other failure +// fails the pass, as in the Worker (`src/auth/workos-events-sync.ts`). +const fromPromise = (fn: () => Promise) => + Effect.tryPromise({ try: fn, catch: (cause) => cause }); + +const isNotFound = (cause: unknown): boolean => + typeof cause === "object" && + cause !== null && + "status" in cause && + (cause as { readonly status: unknown }).status === 404; + +const noneWhenGone = (fn: () => Promise) => + Effect.tryPromise({ try: fn, catch: (cause) => cause }).pipe( + Effect.map(Option.some), + Effect.catch((cause) => + isNotFound(cause) ? Effect.succeed(Option.none()) : Effect.fail(cause), + ), + ); + +// One pass is bounded by the replay's page budget; loop until the stream +// is drained, another run owns it, or the backfill has not run. +const MAX_PASSES = 50; + +const drain = Effect.gen(function* () { + const deps = { + source: { + listEvents: (options: Parameters[0]) => + fromPromise(async () => { + const page = await workos.events.listEvents({ + ...options, + events: [...options.events], + }); + return { data: page.data, after: page.listMetadata.after ?? null }; + }), + getOrganization: (organizationId: string) => + noneWhenGone(() => workos.organizations.getOrganization(organizationId)), + getUser: (userId: string) => noneWhenGone(() => workos.userManagement.getUser(userId)), + }, + store: { + getOrganization: (organizationId: string) => + fromPromise(() => users.getOrganization(organizationId)), + upsertOrganization: (organization: Parameters[0]) => + fromPromise(() => users.upsertOrganization(organization)), + getAccount: (accountId: string) => fromPromise(() => users.getAccount(accountId)), + }, + mirror: makeWorkOsMirrorStore(db), + }; + let last: WorkOsEventsSyncReport | null = null; + for (let pass = 0; pass < MAX_PASSES; pass++) { + const report = yield* replayWorkOsEvents(deps); + console.log( + `[drain-events] pass ${pass + 1}: ${report.pages} page(s), ${report.events} event(s), ` + + `${report.applied} applied, ${report.stale} stale, ${report.absent} absent — ${report.stopped}`, + ); + last = report; + if (report.stopped !== "page_budget") break; + } + return last; +}); + +const report = await Effect.runPromise( + drain.pipe(Effect.ensuring(Effect.promise(() => sql.end({ timeout: 5 })))), +); + +if (report === null || report.stopped !== "drained") { + console.error( + `[drain-events] the events stream was not drained: ${report?.stopped ?? "no pass ran"}` + + (report?.stopped === "awaiting_backfill" + ? " (run scripts/backfill-workos-mirror.ts first)" + : report?.stopped === "cursor_contended" + ? " (another run owns the stream; rerun once it finishes)" + : ""), + ); + process.exit(1); +} diff --git a/apps/cloud/src/account/org-api-key-revoke.node.test.ts b/apps/cloud/src/account/org-api-key-revoke.node.test.ts index 6682280a09..10353ccecd 100644 --- a/apps/cloud/src/account/org-api-key-revoke.node.test.ts +++ b/apps/cloud/src/account/org-api-key-revoke.node.test.ts @@ -136,12 +136,14 @@ const stubMirror = Layer.succeed(WorkOsMirror)({ deleteMembership: () => Effect.die("revoke does not write the membership mirror"), deleteUser: () => Effect.die("revoke does not write the membership mirror"), getCursor: () => Effect.die("revoke does not read the events cursor"), - setCursor: () => Effect.die("revoke does not move the events cursor"), + applyPage: () => Effect.die("revoke does not move the events cursor"), applyOrganizationScan: () => Effect.die("revoke does not run the backfill"), replayBoundary: () => Effect.die("revoke does not run the reconciler"), setReplayBoundary: () => Effect.die("revoke does not run the backfill"), backfillCompletedAt: () => Effect.die("revoke does not check mirror readiness"), markBackfillCompleted: () => Effect.die("revoke does not run the backfill"), + drainedAt: () => Effect.die("revoke does not check mirror readiness"), + markDrained: () => Effect.die("revoke does not run the reconciler"), organizationBackfilledAt: () => Effect.die("revoke does not report seats"), }); diff --git a/apps/cloud/src/auth/mirror-feeders.node.test.ts b/apps/cloud/src/auth/mirror-feeders.node.test.ts index 338ce41391..3381943557 100644 --- a/apps/cloud/src/auth/mirror-feeders.node.test.ts +++ b/apps/cloud/src/auth/mirror-feeders.node.test.ts @@ -21,11 +21,12 @@ // writes nothing on a dry run, converges on a re-run, tombstones a // membership WorkOS no longer lists — but never one written after its // listing was taken — marks each org backfilled as of its listing, and -// records the events replay boundary only when it completes and only +// records the events replay boundary BEFORE its first listing and only // ONCE: a run that fails part-way keeps the marks of the orgs it -// finished and records no boundary, and a later completed run keeps the -// first boundary (the org renames and user deletions between two runs -// are the events stream's to replay) +// finished and the boundary it recorded, and its retry (like any later +// run) keeps that first boundary — so a user deleted between the failed +// attempt and the retry is still inside the events replay, and the +// reconciler clears their profile // - two scans of one org that overlap cannot resurrect a membership: a scan // that listed it, stalled, and resumed after a later listing (which no // longer had it) was applied is refused whole @@ -736,7 +737,7 @@ describe("backfill", () => { const backfilledAt = (org: string) => withMirror((mirror) => mirror.organizationBackfilledAt(org)); - it("records the replay boundary on first completion, marks and mirrors every organization's members, counts the writes, converges and repairs on a re-run", async () => { + it("records the replay boundary at its start, marks and mirrors every organization's members, counts the writes, converges and repairs on a re-run", async () => { const orgA = freshId("org"); const orgB = freshId("org"); await seedOrganization(orgA); @@ -785,6 +786,10 @@ describe("backfill", () => { expect(firstCompletion, "and that every organization is now covered").not.toBeNull(); expect(firstCompletion!.getTime()).toBeGreaterThanOrEqual(after!.getTime()); expect(after!.getTime()).toBeGreaterThanOrEqual(startedAt); + expect( + after!.getTime(), + "the boundary is the instant the run began reading, before any listing", + ).toBeLessThanOrEqual(startedAt + 60 * 1000); for (const org of [orgA, orgB]) { const marked = await backfilledAt(org); expect(marked, "each scanned organization is marked as of its listing").not.toBeNull(); @@ -918,31 +923,26 @@ describe("backfill", () => { ); }); - it("keeps the marks of the organizations it finished but records no replay boundary when a run fails part-way", async () => { + it("keeps the boundary a run that fails part-way recorded, so a user deleted before the retry is still the reconciler's to clear", async () => { const orgA = freshId("org"); const orgB = freshId("org"); await seedOrganization(orgA); await seedOrganization(orgB); - const member = freshId("user"); - const orgs = new Map([ - [orgA, [workosMembership(member, orgA)]], - [orgB, [workosMembership(member, orgB)]], + const staying = freshId("user"); + const deletedMeanwhile = freshId("user"); + await clearEventsRow(); + + // Attempt A mirrors both users of orgA, then fails on orgB's listing. + const attemptA = new Map([ + [orgA, [workosMembership(staying, orgA), workosMembership(deletedMeanwhile, orgA)]], + [orgB, [workosMembership(staying, orgB)]], ]); - // A completed run first, so there IS a boundary to protect. - await runBackfill(orgs, false); - const completed = await syncState(); - expect(completed).not.toBeNull(); - const markedA = await backfilledAt(orgA); - - // A re-run whose second organization fails on a WorkOS read: the first - // org was written, but the run as a whole did not complete — and even a - // completed one would keep the first boundary. const failing = { - ...source(orgs, []), + ...source(attemptA, []), listOrgMembers: (organizationId: string) => organizationId === orgB ? Effect.fail(new WorkOSError({ status: 503 })) - : Effect.succeed(orgs.get(organizationId) ?? []), + : Effect.succeed(attemptA.get(organizationId) ?? []), }; const exit = await withMirror((mirror) => Effect.exit( @@ -953,12 +953,58 @@ describe("backfill", () => { ), ); expect(Exit.isFailure(exit), "the run fails rather than skipping the org").toBe(true); - expect(await syncState(), "the completed run's boundary stands").toEqual(completed); - expect(await completedAt(), "and so does its completion mark").not.toBeNull(); + const boundary = await syncState(); + expect(boundary, "the failed attempt already fixed the replay boundary").not.toBeNull(); + expect(await backfilledAt(orgA), "the org it finished is marked").not.toBeNull(); + expect(await backfilledAt(orgB), "the org it did not reach is not").toBeNull(); + expect( + await completedAt(), + "no completion mark: the failed attempt did not cover every organization", + ).toBeNull(); + expect( + (await readMembers(orgA)).find((m) => m.accountId === deletedMeanwhile)?.email, + "the user's profile is mirrored", + ).toBe(`${deletedMeanwhile}@placeholder.test`); + + // WorkOS deletes `deletedMeanwhile` between the attempts. Its + // `user.deleted` event is stamped AFTER the boundary attempt A recorded. + const deletedAt = new Date(boundary!.getTime() + 1); + + // Retry B lists WorkOS without the deleted user and succeeds. + const attemptB = new Map([ + [orgA, [workosMembership(staying, orgA)]], + [orgB, [workosMembership(staying, orgB)]], + ]); + const retried = await runBackfill(attemptB, false); + expect(retried).toMatchObject({ organizations: 2, membershipsTombstoned: 1 }); + expect( + await syncState(), + "the retry keeps the first attempt's boundary instead of taking a later one", + ).toEqual(boundary); + expect( + await backfilledAt(orgB), + "and finishes the org the first attempt did not", + ).not.toBeNull(); + expect( + await completedAt(), + "the retry is the first run to cover every organization, so it records the completion", + ).not.toBeNull(); + // The scan tombstoned the membership, but the account profile is not + // the scan's to clear: that is the `user.deleted` event's job, which is + // exactly why the boundary must not move past it. + const tombstoned = (await readMembers(orgA)).find((m) => m.accountId === deletedMeanwhile); + expect(tombstoned?.status).toBe("inactive"); + expect(tombstoned?.email, "the profile is still there for the event to clear").not.toBeNull(); + + // The reconciler's first run reads from the kept boundary, so the + // deletion (stamped after it) is inside the replay and clears the row. + expect(deletedAt.getTime()).toBeGreaterThan(boundary!.getTime()); + const cleared = await withMirror((mirror) => mirror.deleteUser(deletedMeanwhile, deletedAt)); + expect(cleared).toBe(true); expect( - (await backfilledAt(orgA))!.getTime(), - "the org the failed run did finish is marked as of its new listing", - ).toBeGreaterThanOrEqual(markedA!.getTime()); + (await readMembers(orgA)).find((m) => m.accountId === deletedMeanwhile)?.email, + "the deleted user's profile is gone from the directory", + ).toBeNull(); }); it("scans one organization on demand and marks only that one", async () => { diff --git a/apps/cloud/src/auth/user-store.ts b/apps/cloud/src/auth/user-store.ts index 931d4fcc80..1dfdb7d022 100644 --- a/apps/cloud/src/auth/user-store.ts +++ b/apps/cloud/src/auth/user-store.ts @@ -41,12 +41,33 @@ export interface OrganizationPayload { export const organizationAcceptsName = (updatedAt: Date) => or(isNull(organizations.workosUpdatedAt), lte(organizations.workosUpdatedAt, updatedAt)); -export const makeUserStore = (db: DrizzleDb) => { - const getOrganization = async (id: string) => { - const rows = await db.select().from(organizations).where(eq(organizations.id, id)); - return rows[0] ?? null; - }; +const readOrganization = async (db: DrizzleDb, id: string) => { + const rows = await db.select().from(organizations).where(eq(organizations.id, id)); + return rows[0] ?? null; +}; +/** + * Insert the organization row for `row.id` with a freshly minted URL slug, + * and return the row now held for that id: the one inserted, or the one a + * concurrent writer minted first. THE single mint point for slugs: every + * organization row is born with one, so there is no nullable window and no + * self-healing. With `deletedAt` set this mints a TOMBSTONE — the row an + * organization deleted in WorkOS before the mirror ever saw it leaves + * behind, so a feeder still holding a membership of it cannot mint it live + * (`upsertOrganization` returns a marked row untouched). + * + * `ON CONFLICT DO NOTHING` (no target) absorbs BOTH unique violations + * without throwing: an id collision (the org was mirrored concurrently), + * which resolves to the row now held, and a slug collision (the candidate + * was claimed by a different org), which retries with a fresh candidate. + * + * @throws when slug minting exhausts its retries — `isTaken` is broken; + * surfacing loudly beats a silently unslugged organization. + */ +export const insertOrganization = async ( + db: DrizzleDb, + row: Pick, +): Promise => { const slugTaken = async (slug: string) => { const rows = await db .select({ id: organizations.id }) @@ -54,36 +75,36 @@ export const makeUserStore = (db: DrizzleDb) => { .where(eq(organizations.slug, slug)); return rows.length > 0; }; - - // Insert a brand-new org row carrying a freshly-minted slug. `ON CONFLICT DO - // NOTHING` (no target) absorbs BOTH unique violations without throwing: an - // id collision (the org was mirrored concurrently) and a slug collision (the - // candidate was claimed by a different org). Returns the inserted row, or - // null when either conflict swallowed the insert — the caller decides whether - // to re-read (id race) or retry with a new candidate (slug race). - const tryInsertOrg = async (org: OrganizationPayload, slug: string) => { - const [row] = await db + for (let attempt = 0; attempt < 4; attempt++) { + const slug = await generateOrgSlug(row.name, slugTaken); + const [inserted] = await db .insert(organizations) - .values({ - id: org.id, - name: org.name, - slug, - workosUpdatedAt: org.updatedAt, - }) + .values({ ...row, slug }) .onConflictDoNothing() .returning(); - return row ?? null; - }; + if (inserted) return inserted; + // The insert was swallowed by a conflict. If the id now exists, a + // concurrent writer mirrored it — return that row. Otherwise the slug + // candidate collided; loop and mint a fresh one. + const held = await readOrganization(db, row.id); + if (held) return held; + } + // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: slug minting exhausted retries; surfacing loudly beats a silently unslugged org + throw new Error(`unable to mint a slug for organization ${row.id}`); +}; - // Every new org row is born with a slug — there is no nullable window and no - // self-healing. Existing rows keep their slug (stable across renames, so org - // URLs survive) and only refresh their name — and only from a payload at - // least as new as the one that last named it (`organizationAcceptsName`): - // a sign-in whose membership list was fetched before a rename would +export const makeUserStore = (db: DrizzleDb) => { + const getOrganization = (id: string) => readOrganization(db, id); + + // Existing rows keep their slug (stable across renames, so org URLs + // survive) and only refresh their name — and only from a payload at least + // as new as the one that last named it (`organizationAcceptsName`): a + // sign-in whose membership list was fetched before a rename would // otherwise revert the rename after it landed. A row marked deleted is // returned as it is: the organization is gone, and nothing a feeder still // holds about it (a name, a membership fetched before the deletion) is - // written — never re-minted live, never renamed. + // written — never re-minted live, never renamed. A row the mirror does not + // hold is minted live (`insertOrganization`). const upsertOrganization = async (org: OrganizationPayload) => { const existing = await getOrganization(org.id); if (existing) { @@ -95,18 +116,12 @@ export const makeUserStore = (db: DrizzleDb) => { .returning(); return updated ?? existing; } - for (let attempt = 0; attempt < 4; attempt++) { - const slug = await generateOrgSlug(org.name, slugTaken); - const inserted = await tryInsertOrg(org, slug); - if (inserted) return inserted; - // The insert was swallowed by a conflict. If the id now exists, a - // concurrent request mirrored it — return that row. Otherwise the slug - // candidate collided; loop and mint a fresh one. - const fresh = await getOrganization(org.id); - if (fresh) return fresh; - } - // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: slug minting exhausted retries; surfacing loudly beats a silently unslugged org - throw new Error(`unable to mint a slug for organization ${org.id}`); + return insertOrganization(db, { + id: org.id, + name: org.name, + workosUpdatedAt: org.updatedAt, + deletedAt: null, + }); }; return { diff --git a/apps/cloud/src/auth/workos-callback-state.node.test.ts b/apps/cloud/src/auth/workos-callback-state.node.test.ts index 44152dda4e..a62f840b67 100644 --- a/apps/cloud/src/auth/workos-callback-state.node.test.ts +++ b/apps/cloud/src/auth/workos-callback-state.node.test.ts @@ -116,12 +116,14 @@ const stubMirror = Layer.succeed(WorkOsMirror)({ deleteMembership: () => Effect.die("the callback does not delete memberships"), deleteUser: () => Effect.die("the callback does not delete users"), getCursor: () => Effect.die("the callback does not read the events cursor"), - setCursor: () => Effect.die("the callback does not move the events cursor"), + applyPage: () => Effect.die("the callback does not move the events cursor"), applyOrganizationScan: () => Effect.die("the callback does not run the backfill"), replayBoundary: () => Effect.die("the callback does not run the reconciler"), setReplayBoundary: () => Effect.die("the callback does not run the backfill"), backfillCompletedAt: () => Effect.die("the callback does not check mirror readiness"), markBackfillCompleted: () => Effect.die("the callback does not run the backfill"), + drainedAt: () => Effect.die("the callback does not check mirror readiness"), + markDrained: () => Effect.die("the callback does not run the reconciler"), organizationBackfilledAt: () => Effect.die("the callback does not report seats"), }); diff --git a/apps/cloud/src/auth/workos-events-replay.ts b/apps/cloud/src/auth/workos-events-replay.ts new file mode 100644 index 0000000000..52032d7480 --- /dev/null +++ b/apps/cloud/src/auth/workos-events-replay.ts @@ -0,0 +1,556 @@ +// --------------------------------------------------------------------------- +// The membership mirror's RECONCILER, as a pure function over its ports: +// replays the WorkOS Events API into the mirror store so changes made +// outside Executor — a member removed in the WorkOS dashboard, a role edited +// there, a profile updated, an SSO just-in-time join — land in the mirror +// without anyone signing in. +// +// Kept free of `cloudflare:workers` (no `DbService`, no `env`, no +// `WorkOSClient`), like `workos-mirror-store.ts` and +// `workos-mirror-backfill.ts`, so the SAME replay runs in three places: the +// Worker's every-minute cron and the signed webhook poke +// (`workos-events-sync.ts` binds the ports to the request-scoped services), +// and the deploy gate (`scripts/ensure-workos-mirror-ready.ts`, through +// `scripts/drain-workos-events.ts`) that must bring the mirror up to date +// BEFORE the build that authorizes from it goes live — and cannot wait on a +// cron that may not be deployed yet. The ports are the WorkOS reads a replay +// makes (`WorkOsEventsSource`), the organization and account rows it +// consults (`WorkOsEventsStore`), and the mirror store it writes. +// +// The Events API is the ONLY source this applies. It is ordered and +// replayable from an event id, so the mirror persists the id of the last +// event it applied (`workos_sync.cursor`) and resumes from there; a webhook +// delivery only pokes a run (`workos-webhook.ts`), it is never applied +// itself, because a webhook is unordered and at-least-once. Two runs may +// overlap (the every-minute cron, a webhook poke, the deploy gate), so a +// page is applied and its cursor advanced in ONE transaction that +// compare-and-sets the cursor first (`WorkOsMirrorShape.applyPage`): the run +// that lost the stream writes nothing. The `updatedAt` guard on upserts is +// not enough on its own — a lagging run replaying `membership.updated` after +// the leading run applied that membership's `deleted` would re-insert the +// revoked row. There is no first-run history replay: the one-off backfill +// (`scripts/backfill-workos-mirror.ts`) covers history, and its first run +// records the instant it began reading WorkOS as the REPLAY BOUNDARY +// (`workos_sync.range_start`) before it lists anything. A run with no cursor +// reads the stream from that boundary — never from a wall-clock guess, which +// would silently drop every revocation older than the guess — and with no +// boundary either it does nothing but warn: the backfill has not run, and +// there is no honest place to start. The boundary never moves: a backfill +// retry or re-run refreshes memberships only, so the organization renames +// and user deletions after the first boundary are this stream's alone to +// apply. +// +// A deletion event tombstones its row as of the event's own `createdAt`, +// not the payload's `updatedAt` (which predates the delete): the tombstone +// must be newer than every payload a feeder could have fetched before the +// delete, so none of them can reinstate the row. +// +// A page is PLANNED before its transaction opens: every event becomes a +// mirror write, and that planning is where the only WorkOS reads happen — +// resolving an organization the mirror has never seen, and reading the +// profile of a member the mirror has never seen. A membership event carries +// no profile, and the `user.created` that would have carried it may predate +// the replay boundary: a user who existed before the mirror shipped and +// joins an organization the backfill has already scanned gets a bare +// account row from the membership write, and nothing in the stream would +// ever fill it — the member would be unsearchable by name or email until an +// unrelated profile update or sign-in. So a membership created or updated +// for an account the mirror holds no profile for (no row, or the bare row a +// membership write mints) is planned WITH the profile (`UpsertMember`), one +// `getUser` per such member, never per event. A deterministic answer to +// either read ("WorkOS no longer has this organization / user") does not +// fail the run — a failed run re-reads the same page from the same cursor +// next tick, so one such event would freeze the whole mirror, including +// revocations in every other org — but it is not dropped either: a gone +// organization MARKS the organization deleted (below), the same write its +// own `organization.deleted` further down the stream makes; a gone user is +// mirrored without a profile, and their own `user.deleted` follows. +// +// `organization.deleted` MARKS the organization deleted +// (`organizations.deleted_at`, the same mark cloud's own deletion flow sets +// and its purge keeps as a tombstone): the mirror is the membership read +// path, so an org deleted in the WorkOS dashboard must stop authorizing its +// members' sessions here, and this event is the only way that reaches the +// mirror. The mark is written HERE, in the first build that consumes the +// event, so no `organization.deleted` is ever drained from the stream +// without effect — an event consumed before the mark existed could never be +// replayed. For the same reason an organization the mirror has never seen +// gets a TOMBSTONE row minted: with no row, a login that fetched its +// memberships before the deletion and stalled would mint the organization +// live afterwards, and nothing left in the stream would ever revoke it. It +// never PURGES: deleting tenant data and secrets is cloud's own flow +// (`db/org-deletion.ts`), sequenced with billing and confirmed by an admin, +// and an event must not do it. An org already marked (by cloud's own flow, +// or a replay) is `absent` and nothing changes. `organization.updated` renames an +// organization the mirror already holds — never inserts one, so a rename +// replayed after cloud purged the org cannot resurrect it with a fresh slug +// — under the same name guard every feeder applies, so a rename event and +// a sign-in's name order each other by their stamps however they arrive. +// --------------------------------------------------------------------------- + +import { Clock, Effect, Match, Option } from "effect"; +import type { Event as WorkOSEvent } from "@workos-inc/node/worker"; + +import type { Account, Organization, OrganizationPayload } from "./user-store"; +import type { WorkOSListEventsOptions } from "./workos"; +import { + WorkOsMirrorWrite, + mirrorMembershipFromWorkOs, + mirrorUserFromWorkOs, + type WorkOsMembershipPayload, + type WorkOsMirrorShape, + type WorkOsMirrorUser, + type WorkOsMirrorWriteOutcome, + type WorkOsUserPayload, +} from "./workos-mirror-store"; + +/** + * The event types the mirror follows. Invitations are not mirrored (they + * stay a live WorkOS read), and `organization.created` is not needed: an org + * is mirrored lazily the first time a membership or a session names it. + */ +export const MIRRORED_EVENT_NAMES = [ + "user.created", + "user.updated", + "user.deleted", + "organization_membership.created", + "organization_membership.updated", + "organization_membership.deleted", + "organization.updated", + "organization.deleted", +] as const; + +export type WorkOsMirroredEventName = (typeof MIRRORED_EVENT_NAMES)[number]; + +/** The SDK events the reconciler applies, narrowed to the followed types. */ +export type WorkOsMirroredEvent = Extract; + +const mirroredEventNames: ReadonlySet = new Set(MIRRORED_EVENT_NAMES); + +/** Whether an event from the stream is one the mirror follows. */ +export const isMirroredEvent = (event: WorkOSEvent): event is WorkOsMirroredEvent => + mirroredEventNames.has(event.event); + +/** + * What one event did to the mirror: + * - `applied`: a row was written, marked, or tombstoned; + * - `stale`: the `updatedAt` guard refused an older payload (a replay or a + * late event behind a fresher write); + * - `absent`: a delete found its row already tombstoned or superseded by a + * newer membership (a replayed delete), a rename found no live + * organization row — the mirror has never seen it, or it is marked + * deleted — or a deletion mark found the organization already marked. + */ +export type WorkOsEventOutcome = WorkOsMirrorWriteOutcome; + +/** One page of the Events API stream, as the source hands it to the replay. */ +export interface WorkOsEventsPage { + readonly data: readonly WorkOSEvent[]; + /** The id to resume after, or `null` at the end of the stream. */ + readonly after: string | null; +} + +/** The WorkOS organization fields the replay reads to mint an org row. */ +export interface WorkOsOrganizationPayload { + readonly id: string; + readonly name: string; + readonly updatedAt: string; +} + +/** + * The WorkOS reads one replay makes, over whatever client the caller wires: + * the Events API page, and — only for a membership event whose organization + * or member the mirror has never seen — the organization or user resource. + * The two lookups answer `None` when WorkOS no longer has the resource (a + * 404): that is a deterministic answer the replay acts on, not a failure. + * Every other failure (401/403, 429, 5xx, no answer) is `E` and fails the + * run, so the event is retried once the cause is fixed rather than skipped. + */ +export interface WorkOsEventsSource { + readonly listEvents: (options: WorkOSListEventsOptions) => Effect.Effect; + readonly getOrganization: ( + organizationId: string, + ) => Effect.Effect, E>; + readonly getUser: (userId: string) => Effect.Effect, E>; +} + +/** + * The organization and account rows a replay consults while planning a + * page: the org row a membership's foreign key needs (minted from WorkOS + * through `upsertOrganization` when the mirror has never seen it — the one + * slug mint point) and the account row that says whether the member's + * profile is already held. + */ +export interface WorkOsEventsStore { + readonly getOrganization: (organizationId: string) => Effect.Effect; + readonly upsertOrganization: ( + organization: OrganizationPayload, + ) => Effect.Effect; + readonly getAccount: (accountId: string) => Effect.Effect; +} + +/** Everything one replay reads and writes. */ +export interface WorkOsEventsReplayDeps { + readonly source: WorkOsEventsSource; + readonly store: WorkOsEventsStore; + readonly mirror: WorkOsMirrorShape; +} + +/** A membership event's payload: the SDK's `OrganizationMembership`, which names its organization. */ +interface WorkOsMembershipEventPayload extends WorkOsMembershipPayload { + readonly organizationName: string; +} + +// The organization row a membership event needs: the mirror's, or — for an +// org the mirror has never seen (created and populated in the WorkOS +// dashboard before anyone signed in) — minted from the WorkOS organization +// so the membership's foreign key holds. `None` when WorkOS no longer has +// the organization. +const resolveOrganization = ( + deps: WorkOsEventsReplayDeps, + organizationId: string, +): Effect.Effect, E> => + Effect.gen(function* () { + const existing = yield* deps.store.getOrganization(organizationId); + if (existing) return Option.some(existing); + const fresh = yield* deps.source.getOrganization(organizationId); + if (Option.isNone(fresh)) return Option.none(); + const minted = yield* deps.store.upsertOrganization({ + id: fresh.value.id, + name: fresh.value.name, + updatedAt: new Date(fresh.value.updatedAt), + }); + return Option.some(minted); + }); + +// A membership event carries only the organization's id, so an org the +// mirror has never seen is mirrored first (`resolveOrganization`) so the +// membership's foreign key holds. That goes for a DELETE too: it leaves a +// tombstone behind even when the mirror has never seen the membership (so +// the backfill's older payload cannot insert it live), and the tombstone row +// needs the org as much as a live one. An org WorkOS no longer has (deleted +// there, or through Executor, after this event was emitted) is MARKED +// deleted instead — minting its tombstone row when the mirror has never +// seen it — so a login still holding a membership of it cannot mint it +// live: its own `organization.deleted` follows in the stream and finds the +// mark already there, and the membership itself is not written, there is +// nothing live to hold it. +const planMembershipWrite = ( + deps: WorkOsEventsReplayDeps, + membership: WorkOsMembershipEventPayload, + event: { readonly id: string; readonly createdAt: string }, + write: () => Effect.Effect, +): Effect.Effect => + Effect.gen(function* () { + const organization = yield* resolveOrganization(deps, membership.organizationId); + if (Option.isNone(organization)) { + yield* Effect.logWarning( + "workos_events: membership for an organization WorkOS no longer has; marking the org deleted instead", + { organizationId: membership.organizationId, eventId: event.id }, + ); + return WorkOsMirrorWrite.MarkOrganizationDeleted({ + organizationId: membership.organizationId, + name: membership.organizationName, + deletedAt: new Date(event.createdAt), + }); + } + return yield* write(); + }); + +// Whether the mirror holds a profile for the account: none at all, or only +// the bare row a membership write mints (`ensureAccount`: no email, no +// stamp), calls for a WorkOS read. A deletion tombstone (no email, stamped +// by `deleteUser`) does not: WorkOS never reuses a user id, and the +// membership write refuses the account anyway. +const holdsNoProfile = (account: Account | null): boolean => + account === null || (account.email === null && account.workosUpdatedAt === null); + +/** + * The user ids whose profile an earlier event of the SAME page has already + * planned a write for (`user.created` / `user.updated`, or a profile read + * for a membership). A page is planned in full before it is applied, so the + * mirror does not yet hold what the page's own earlier events carry; this + * is what keeps a `user.created` followed by that user's membership in one + * page from reading the profile WorkOS just streamed. + */ +export type PlannedProfiles = Set; + +// The member's profile from WorkOS, when neither the mirror nor an earlier +// event of the page holds one (see the header); `null` when one does, or +// when WorkOS no longer has the user (the user's own `user.deleted` follows +// in the stream, or has been applied). +const planMemberProfile = ( + deps: WorkOsEventsReplayDeps, + userId: string, + event: { readonly id: string }, + profiled: PlannedProfiles, +): Effect.Effect => + Effect.gen(function* () { + if (profiled.has(userId)) return null; + const account = yield* deps.store.getAccount(userId); + if (!holdsNoProfile(account)) return null; + const user = yield* deps.source.getUser(userId); + if (Option.isNone(user)) { + yield* Effect.logWarning( + "workos_events: membership for a user WorkOS no longer has; mirrored without a profile", + { eventId: event.id }, + ); + return null; + } + profiled.add(userId); + return mirrorUserFromWorkOs(user.value); + }); + +const planMembershipUpsert = ( + deps: WorkOsEventsReplayDeps, + membership: WorkOsMembershipEventPayload, + event: { readonly id: string; readonly createdAt: string }, + profiled: PlannedProfiles, +) => + planMembershipWrite(deps, membership, event, () => + Effect.map(planMemberProfile(deps, membership.userId, event, profiled), (user) => + user === null + ? WorkOsMirrorWrite.UpsertMembership({ + membership: mirrorMembershipFromWorkOs(membership), + }) + : WorkOsMirrorWrite.UpsertMember({ + user, + membership: mirrorMembershipFromWorkOs(membership), + }), + ), + ); + +/** + * Translate one event into the mirror write it calls for. Every followed + * event yields a write: none is drained from the stream without effect. + * This is the only step that may read WorkOS (an organization the mirror + * has never seen, a member it holds no profile for); it runs before the + * page's transaction opens. Fails on a store failure or a WorkOS failure + * that a retry could clear — the run stops before the page is applied, so + * the event is retried next run. `profiled` is the page's running set of + * users whose profile is already planned (see {@link PlannedProfiles}); one + * set per page. + */ +export const planWorkOsEvent = ( + deps: WorkOsEventsReplayDeps, + event: WorkOsMirroredEvent, + profiled: PlannedProfiles = new Set(), +): Effect.Effect => { + const userWrite = (data: WorkOsUserPayload) => + Effect.sync(() => { + profiled.add(data.id); + return WorkOsMirrorWrite.UpsertUser({ user: mirrorUserFromWorkOs(data) }); + }); + return Match.value(event).pipe( + Match.discriminatorsExhaustive("event")({ + "user.created": ({ data }) => userWrite(data), + "user.updated": ({ data }) => userWrite(data), + "user.deleted": ({ data }) => + Effect.succeed( + WorkOsMirrorWrite.DeleteUser({ + accountId: data.id, + deletedAt: new Date(event.createdAt), + }), + ), + "organization_membership.created": ({ data }) => + planMembershipUpsert(deps, data, event, profiled), + "organization_membership.updated": ({ data }) => + planMembershipUpsert(deps, data, event, profiled), + "organization_membership.deleted": ({ data }) => + planMembershipWrite(deps, data, event, () => + Effect.succeed( + WorkOsMirrorWrite.DeleteMembership({ + membership: { + id: data.id, + accountId: data.userId, + organizationId: data.organizationId, + }, + deletedAt: new Date(event.createdAt), + }), + ), + ), + "organization.updated": ({ data }) => + Effect.succeed( + WorkOsMirrorWrite.RenameOrganization({ + organizationId: data.id, + name: data.name, + updatedAt: new Date(data.updatedAt), + }), + ), + "organization.deleted": ({ data }) => + Effect.logWarning( + "workos_events: organization.deleted received; marking the org deleted locally — tenant data is kept (purging is cloud's own flow, db/org-deletion.ts)", + { organizationId: data.id, eventId: event.id }, + ).pipe( + Effect.as( + WorkOsMirrorWrite.MarkOrganizationDeleted({ + organizationId: data.id, + name: data.name, + deletedAt: new Date(event.createdAt), + }), + ), + ), + }), + ); +}; + +// One page is one WorkOS read and one cursor advance. 100 is the API's +// maximum; the page budget bounds a single run (a backlog after an outage +// drains over successive runs, each committing what it applied) so a cron +// invocation stays well inside the Worker's wall-clock limits. +const PAGE_SIZE = 100; +const MAX_PAGES_PER_RUN = 20; + +export interface WorkOsEventsSyncReport { + readonly pages: number; + readonly events: number; + readonly applied: number; + readonly stale: number; + readonly absent: number; + /** + * Why the run ended: the stream was read to its end (`drained`), another + * run moved the cursor first (`cursor_contended`), the page budget for one + * run was spent with more to read (`page_budget`), or there is neither a + * cursor nor a replay boundary to start from — the backfill has not run — + * so nothing was read (`awaiting_backfill`). + */ + readonly stopped: "drained" | "cursor_contended" | "page_budget" | "awaiting_backfill"; + /** The cursor this run left behind (the last event id it committed). */ + readonly cursor: string | null; +} + +/** + * One reconciler run: read the cursor (or, before the first page was ever + * committed, the backfill's replay boundary), page the Events API from it + * (oldest first), plan every event, and apply each page with its cursor + * advance in one transaction. Stops as soon as that transaction finds the + * cursor moved — another run owns the stream, and nothing from the page was + * written — and fails (before the page is applied) on the first source, + * store, or mirror failure, so nothing is skipped: the next run resumes + * from the last committed page. With neither cursor nor boundary it reads + * nothing and reports `awaiting_backfill`. A run that reads the stream to + * its end records the drain (`markDrained`) as of its own start. + */ +export const replayWorkOsEvents = (deps: WorkOsEventsReplayDeps) => + Effect.gen(function* () { + const { source, mirror } = deps; + + // Taken before the first read, so the drained mark below cannot + // post-date an event this run never saw. + const startedAt = new Date(yield* Clock.currentTimeMillis); + let cursor = yield* mirror.getCursor(); + const counts = { + pages: 0, + events: 0, + applied: 0, + stale: 0, + absent: 0, + }; + let stopped: WorkOsEventsSyncReport["stopped"] = "page_budget"; + + // Where the next page starts: after the last committed event id, or — + // for the very first read, which has no id to resume from — at the + // backfill's replay boundary, the only instant known to be covered. + let resume: { readonly after: string } | { readonly rangeStart: string }; + if (cursor === null) { + const boundary = yield* mirror.replayBoundary(); + if (boundary === null) { + yield* Effect.logWarning( + "workos_events: no cursor and no replay boundary — the mirror backfill has not run (db:backfill-workos-mirror:prod); nothing read", + ); + const report: WorkOsEventsSyncReport = { + ...counts, + stopped: "awaiting_backfill", + cursor, + }; + return report; + } + resume = { rangeStart: boundary.toISOString() }; + } else { + resume = { after: cursor }; + } + + while (counts.pages < MAX_PAGES_PER_RUN) { + const page = yield* source.listEvents({ + events: MIRRORED_EVENT_NAMES, + limit: PAGE_SIZE, + order: "asc", + ...resume, + }); + counts.pages += 1; + if (page.data.length === 0) { + stopped = "drained"; + break; + } + + // Plan first (the WorkOS reads), then apply under the cursor lock. + let lastEventId = cursor; + const profiled: PlannedProfiles = new Set(); + const planned: { + readonly event: WorkOsMirroredEvent; + readonly write: WorkOsMirrorWrite; + }[] = []; + for (const event of page.data) { + counts.events += 1; + lastEventId = event.id; + if (!isMirroredEvent(event)) { + // The request named the followed types; anything else is a WorkOS + // change of contract worth seeing, not a reason to stop the stream. + yield* Effect.logWarning("workos_events: unrequested event type skipped", { + event: event.event, + eventId: event.id, + }); + continue; + } + planned.push({ event, write: yield* planWorkOsEvent(deps, event, profiled) }); + } + + // `lastEventId` is an event id here: the page was non-empty. + if (lastEventId === null) break; + const outcomes = yield* mirror.applyPage( + cursor, + lastEventId, + planned.map((p) => p.write), + ); + if (Option.isNone(outcomes)) { + yield* Effect.logWarning("workos_events: cursor moved by another run; stopping", { + expected: cursor, + }); + stopped = "cursor_contended"; + break; + } + for (const [index, outcome] of outcomes.value.entries()) { + counts[outcome] += 1; + if (outcome === "absent") { + // Normal for a replayed delete; for a rename it means the org was + // never mirrored or is marked deleted, and for a deletion mark + // that it is already marked — either way nothing to do. + yield* Effect.logInfo("workos_events: event targets a row the mirror does not hold", { + event: planned[index]?.event.event, + eventId: planned[index]?.event.id, + }); + } + } + cursor = lastEventId; + resume = { after: cursor }; + if (page.after === null) { + stopped = "drained"; + break; + } + } + + if (stopped === "drained") { + // The stream was read to its end: everything WorkOS had emitted by + // the time this run began is now in the mirror. Recorded as of the + // run's START, not its end — an event emitted while the run was + // reading may still be ahead of the last page it saw — so the mark + // never claims more than was covered. This is what the authorization + // path reads to tell a caught-up mirror from one whose reconciler has + // stalled. + yield* mirror.markDrained(startedAt); + } + + const report: WorkOsEventsSyncReport = { ...counts, stopped, cursor }; + yield* Effect.logInfo("workos_events: sync run finished", report); + return report; + }).pipe(Effect.withSpan("workos_events.replay")); diff --git a/apps/cloud/src/auth/workos-events-runner.ts b/apps/cloud/src/auth/workos-events-runner.ts new file mode 100644 index 0000000000..1e1c860e7c --- /dev/null +++ b/apps/cloud/src/auth/workos-events-runner.ts @@ -0,0 +1,56 @@ +// --------------------------------------------------------------------------- +// Runs one reconciler pass (`syncWorkOsEvents`) from a Worker entry that is +// not an HTTP request handled by the Effect app: the every-minute cron +// (`scheduled` in server.ts) and the webhook poke (`workos-webhook.ts`, +// detached past the response with `waitUntil`). +// +// Both entries build the request-scoped services FRESH for the run — the +// same reason `mcp/auth.ts` does: a postgres socket belongs to one Workers +// invocation, and the webhook route's own per-request layer is closed the +// moment its response is returned, so a detached run cannot borrow it. The +// run is its own scope; the socket is released when it ends. +// +// A failing run is captured (Sentry + structured log) and swallowed here: +// neither entry has a caller to report to, and the run is retried by the +// next cron tick from the last committed cursor. +// --------------------------------------------------------------------------- + +import { Effect, Layer } from "effect"; + +import { captureCauseEffect } from "../observability"; +import { WorkerTelemetryLive } from "../observability/telemetry"; +import { makeDbLayer } from "../db/db"; +import { makeUserStoreLayer } from "./context"; +import { CoreSharedServices } from "./workos"; +import { syncWorkOsEvents } from "./workos-events-sync"; +import { makeWorkOsMirrorLayer } from "./workos-mirror"; + +const makeSyncServices = () => { + const dbLive = makeDbLayer(); + return Layer.mergeAll( + makeUserStoreLayer().pipe(Layer.provide(dbLive)), + makeWorkOsMirrorLayer().pipe(Layer.provide(dbLive)), + CoreSharedServices, + ); +}; + +/** + * One reconciler pass over fresh request-scoped services. Resolves when the + * pass ends, whether it drained the stream, stopped at the page budget, + * yielded to another run, or failed (a failure is reported, never thrown). + */ +export const runWorkOsEventsSync = (): Promise => + Effect.runPromise( + syncWorkOsEvents().pipe( + Effect.asVoid, + Effect.provide(makeSyncServices()), + Effect.scoped, + Effect.catchCause((cause) => + Effect.gen(function* () { + yield* Effect.logError("workos_events: sync run failed", cause); + yield* captureCauseEffect(cause); + }), + ), + Effect.provide(WorkerTelemetryLive), + ), + ); diff --git a/apps/cloud/src/auth/workos-events-sync.node.test.ts b/apps/cloud/src/auth/workos-events-sync.node.test.ts new file mode 100644 index 0000000000..d0a465a09d --- /dev/null +++ b/apps/cloud/src/auth/workos-events-sync.node.test.ts @@ -0,0 +1,1268 @@ +// --------------------------------------------------------------------------- +// The membership mirror's RECONCILER (`workos-events-sync.ts`) and the +// webhook that pokes it (`workos-webhook.ts`), against the real PGlite +// Postgres every cloud unit test runs on (scripts/test-globalsetup.ts). +// WorkOS is a fake `WorkOSClient` for the Events API (the emulator has no +// events route); the signature check runs the REAL client's verifier over a +// locally computed HMAC, because that check is the webhook's only +// authentication. +// +// What this pins: +// - every followed event type lands in the mirror: user created/updated/ +// deleted, membership created/updated/deleted, organization renamed +// - an older event never regresses a newer row (`stale`) — an older +// organization rename included — a replayed delete is `absent`, and +// `organization.deleted` MARKS the org deleted without purging anything; +// replayed, or after cloud's own flow marked it first, it is `absent` +// - `organization.deleted` for an org the mirror has never seen MINTS a +// tombstone row, so a login that fetched a membership of it before the +// deletion cannot mint the org or the membership afterwards +// - a delete TOMBSTONES its row as of the event's `createdAt`, so an older +// payload replayed after it is `stale` and the row stays inactive — even +// when the delete arrives before the mirror has ever seen the membership +// (the reconciler ahead of the backfill): the tombstone is minted, with +// its organization, so the backfill cannot insert the row live +// - a membership created or updated for a member the mirror holds no +// profile for reads the profile from WorkOS (one `getUser`, only then), +// so a pre-boundary user who joins a scanned org is searchable by name +// and email; a member WorkOS no longer has is mirrored bare, and a +// transient failure fails the run +// - a membership for an organization the mirror has never seen mirrors +// the org first (one WorkOS read), so the foreign key holds; one whose +// org WorkOS no longer has marks the org deleted instead (minting the +// tombstone) and the cursor still advances, while a transient WorkOS +// failure still fails the run +// - `organization.updated` never inserts an org the mirror does not hold +// - a run with no cursor reads from the backfill's replay boundary, and +// with no boundary either reads nothing (the backfill has not run) +// - a run pages from the persisted cursor, commits after every page, and +// STOPS when another run moves the cursor under it — with NOTHING from +// the contended page written (a lagging run cannot resurrect a +// membership the leading run already deleted) +// - a run that reads the stream to its end records the drain as of its +// start (the authorization path's caught-up check); a run that read +// nothing, or yielded the stream, records none +// - the webhook accepts only a genuinely signed delivery, never applies +// it, and refuses everything when no signing secret is configured +// --------------------------------------------------------------------------- + +import { createHmac } from "node:crypto"; + +import { describe, expect, it } from "@effect/vitest"; +import { sql } from "drizzle-orm"; +import { Effect, Exit, Layer, Option } from "effect"; +import { HttpRouter, HttpServer } from "effect/unstable/http"; +import type { Organization, OrganizationMembership, User } from "@workos-inc/node/worker"; + +import { MemberDirectory, type MemberStatus } from "@executor-js/api/server"; + +import { DbService } from "../db/db"; +import { UserStoreService } from "./context"; +import { WorkOSError } from "./errors"; +import { cloudMemberDirectoryLayer } from "./member-directory"; +import { mirrorSignIn } from "./mirror-feeders"; +import { WorkOSClient, type WorkOSClientService, type WorkOSListEventsOptions } from "./workos"; +import { + planEvent, + syncWorkOsEvents, + type WorkOsEventOutcome, + type WorkOsEventsSyncReport, + type WorkOsMirroredEvent, +} from "./workos-events-sync"; +import { WorkOsMirror, WorkOsMirrorWrite, mirrorMembershipFromWorkOs } from "./workos-mirror"; +import { WORKOS_WEBHOOK_PATH, makeWorkOsWebhookRoute } from "./workos-webhook"; + +const T1 = "2026-01-01T00:00:00.000Z"; +const T2 = "2026-01-02T00:00:00.000Z"; +const T3 = "2026-01-03T00:00:00.000Z"; + +// Synthetic identities only; every test mints its own ids so the shared test +// database never couples two tests. +const freshId = (prefix: string) => `${prefix}_${crypto.randomUUID().replaceAll("-", "")}`; + +const workosUser = (id: string, overrides: Partial = {}): User => ({ + object: "user", + id, + email: `${id}@placeholder.test`, + emailVerified: true, + firstName: "Ada", + lastName: "Placeholder", + profilePictureUrl: null, + lastSignInAt: T1, + locale: null, + createdAt: T1, + updatedAt: T1, + externalId: null, + metadata: {}, + ...overrides, +}); + +const workosMembership = ( + userId: string, + organizationId: string, + overrides: Partial = {}, +): OrganizationMembership => ({ + object: "organization_membership", + id: `om_${userId}_${organizationId}`, + userId, + organizationId, + organizationName: `Org ${organizationId}`, + status: "active", + directoryManaged: false, + createdAt: T1, + updatedAt: T1, + customAttributes: {}, + role: { slug: "member" }, + ...overrides, +}); + +const workosOrganization = (id: string, name: string, updatedAt = T1): Organization => ({ + object: "organization", + id, + name, + allowProfilesOutsideOrganization: false, + domains: [], + createdAt: T1, + updatedAt, + externalId: null, + metadata: {}, +}); + +const userEvent = ( + event: "user.created" | "user.updated" | "user.deleted", + data: User, + id = freshId("event"), + createdAt = T1, +): WorkOsMirroredEvent => ({ + id, + event, + data, + createdAt, + context: undefined, +}); + +const membershipEvent = ( + event: + | "organization_membership.created" + | "organization_membership.updated" + | "organization_membership.deleted", + data: OrganizationMembership, + id = freshId("event"), + createdAt = T1, +): WorkOsMirroredEvent => ({ + id, + event, + data, + createdAt, + context: undefined, +}); + +const organizationEvent = ( + event: "organization.updated" | "organization.deleted", + data: Organization, + id = freshId("event"), + createdAt = T1, +): WorkOsMirroredEvent => ({ + id, + event, + data, + createdAt, + context: undefined, +}); + +/** + * A `WorkOSClient` whose every method is one of `methods`; anything else is + * an unexpected call and dies, so a reconciler that silently adds a WorkOS + * read fails the test instead of passing on a fake. + */ +const stubWorkOS = (methods: Partial) => + Layer.succeed( + WorkOSClient, + new Proxy({} as WorkOSClientService, { + get: (_target, prop) => + (methods as Record)[prop] ?? + (() => Effect.die(`unexpected WorkOSClient.${String(prop)} call`)), + }), + ); + +/** + * `getUser` for every member a test's membership events name: the + * reconciler reads a profile for a member the mirror holds none for, and + * the strict stub above would die on it. Records each read in `reads`. + */ +const profiles = (reads: string[] = []): Partial => ({ + getUser: (userId) => + Effect.sync(() => { + reads.push(userId); + return workosUser(userId); + }), +}); + +const DbLive = DbService.Live; +const MirrorServices = Layer.mergeAll( + WorkOsMirror.Live, + UserStoreService.Live, + cloudMemberDirectoryLayer, +).pipe(Layer.provideMerge(DbLive)); + +type Services = WorkOsMirror | UserStoreService | MemberDirectory | DbService | WorkOSClient; + +const run = ( + body: Effect.Effect, + workos: Layer.Layer = stubWorkOS({}), +) => + Effect.runPromise( + body.pipe(Effect.provide(Layer.mergeAll(MirrorServices, workos)), Effect.scoped), + ); + +const seedOrganization = (id: string) => + Effect.flatMap(UserStoreService.asEffect(), (users) => + users.use("upsertOrganization", (s) => + s.upsertOrganization({ id, name: `Org ${id}`, updatedAt: new Date(T1) }), + ), + ); + +const readOrganization = (id: string) => + Effect.flatMap(UserStoreService.asEffect(), (users) => + users.use("getOrganization", (s) => s.getOrganization(id)), + ); + +const readMembership = ( + accountId: string, + organizationId: string, + statuses?: readonly MemberStatus[], +) => + Effect.flatMap(MemberDirectory.asEffect(), (directory) => + directory.membership(accountId, organizationId, statuses), + ); + +/** + * Apply one event the way a run does — plan it, then apply it as a one-event + * page under the cursor CAS — and report the event's outcome. The cursor is + * instance-wide; each apply moves it to a fresh id, which is what a run does. + */ +const applyEvent = (event: WorkOsMirroredEvent) => + Effect.gen(function* () { + const mirror = yield* WorkOsMirror; + const write = yield* planEvent(event); + const prev = yield* mirror.getCursor(); + const outcomes = yield* mirror.applyPage(prev, freshId("event"), [write]); + expect(Option.isSome(outcomes), "no other run contends in a single-event apply").toBe(true); + const outcome: WorkOsEventOutcome | undefined = Option.getOrElse(outcomes, () => [])[0]; + expect(outcome, "one write, one outcome").toBeDefined(); + return outcome ?? "absent"; + }); + +describe("applyEvent", () => { + it("mirrors a user, refreshes it, refuses an older update, and deletes it", async () => { + const org = freshId("org"); + const userId = freshId("user"); + const result = await run( + Effect.gen(function* () { + yield* seedOrganization(org); + yield* applyEvent( + membershipEvent("organization_membership.created", workosMembership(userId, org)), + ); + + const created = yield* applyEvent( + userEvent("user.created", workosUser(userId, { firstName: "Grace", updatedAt: T2 })), + ); + const afterCreate = yield* readMembership(userId, org); + const updated = yield* applyEvent( + userEvent("user.updated", workosUser(userId, { firstName: "Newer", updatedAt: T3 })), + ); + const afterUpdate = yield* readMembership(userId, org); + const stale = yield* applyEvent( + userEvent("user.updated", workosUser(userId, { firstName: "Stale", updatedAt: T1 })), + ); + const afterStale = yield* readMembership(userId, org); + // The delete is stamped with the EVENT's time (T3), after every + // payload above; the SDK payload's own `updatedAt` predates it. + const deleted = yield* applyEvent( + userEvent("user.deleted", workosUser(userId), freshId("event"), T3), + ); + const afterDelete = yield* readMembership(userId, org); + const tombstoned = yield* readMembership(userId, org, ["inactive"]); + // A profile update that happened before the delete but lands after it. + const lateUpdate = yield* applyEvent( + userEvent("user.updated", workosUser(userId, { firstName: "Late", updatedAt: T2 })), + ); + const afterLate = yield* readMembership(userId, org, ["inactive"]); + return { + created, + afterCreate, + updated, + afterUpdate, + stale, + afterStale, + deleted, + afterDelete, + tombstoned, + lateUpdate, + afterLate, + }; + }), + stubWorkOS(profiles()), + ); + expect(result.created).toBe("applied"); + expect(result.afterCreate?.name).toBe("Grace Placeholder"); + expect(result.updated).toBe("applied"); + expect(result.afterUpdate?.name).toBe("Newer Placeholder"); + expect(result.stale, "an event older than the stored row is reported stale").toBe("stale"); + expect(result.afterStale?.name, "and leaves the newer row untouched").toBe("Newer Placeholder"); + expect(result.deleted).toBe("applied"); + expect(result.afterDelete, "deleting the user tombstones its membership").toBeNull(); + expect(result.tombstoned, "the row stays, inactive, with the profile cleared").toMatchObject({ + status: "inactive", + name: null, + email: null, + }); + expect(result.lateUpdate, "a payload older than the deletion is refused").toBe("stale"); + expect(result.afterLate).toMatchObject({ status: "inactive", name: null }); + }); + + it("mirrors a membership, updates its role, refuses an older update, and deletes it", async () => { + const org = freshId("org"); + const userId = freshId("user"); + const result = await run( + Effect.gen(function* () { + yield* seedOrganization(org); + const created = yield* applyEvent( + membershipEvent( + "organization_membership.created", + workosMembership(userId, org, { status: "pending" }), + ), + ); + const afterCreate = yield* readMembership(userId, org); + const updated = yield* applyEvent( + membershipEvent( + "organization_membership.updated", + workosMembership(userId, org, { + role: { slug: "admin" }, + updatedAt: T3, + }), + ), + ); + const afterUpdate = yield* readMembership(userId, org); + const stale = yield* applyEvent( + membershipEvent( + "organization_membership.updated", + workosMembership(userId, org, { + role: { slug: "member" }, + status: "inactive", + updatedAt: T2, + }), + ), + ); + const afterStale = yield* readMembership(userId, org); + // The delete is stamped with the EVENT's time (T3), after every + // payload above; the SDK payload's own `updatedAt` predates it. + const deleted = yield* applyEvent( + membershipEvent( + "organization_membership.deleted", + workosMembership(userId, org), + freshId("event"), + T3, + ), + ); + const afterDelete = yield* readMembership(userId, org); + const deletedAgain = yield* applyEvent( + membershipEvent( + "organization_membership.deleted", + workosMembership(userId, org), + freshId("event"), + T3, + ), + ); + // A membership update that happened before the delete but lands + // after it (a lagging feeder, an out-of-order delivery): refused, the + // tombstone stands. + const lateUpdate = yield* applyEvent( + membershipEvent( + "organization_membership.updated", + workosMembership(userId, org, { + role: { slug: "admin" }, + updatedAt: T2, + }), + ), + ); + const afterLate = yield* readMembership(userId, org, ["inactive"]); + return { + created, + afterCreate, + updated, + afterUpdate, + stale, + afterStale, + deleted, + afterDelete, + deletedAgain, + lateUpdate, + afterLate, + }; + }), + stubWorkOS(profiles()), + ); + expect(result.created).toBe("applied"); + expect(result.afterCreate).toMatchObject({ + membershipId: `om_${userId}_${org}`, + status: "pending", + role: "member", + }); + expect(result.updated).toBe("applied"); + expect(result.afterUpdate).toMatchObject({ + status: "active", + role: "admin", + }); + expect(result.stale).toBe("stale"); + expect(result.afterStale).toMatchObject({ + status: "active", + role: "admin", + }); + expect(result.deleted).toBe("applied"); + expect(result.afterDelete, "a tombstone reads as no membership").toBeNull(); + expect(result.deletedAgain, "a replayed delete changes nothing").toBe("absent"); + expect(result.lateUpdate, "a payload older than the deletion is refused").toBe("stale"); + expect(result.afterLate).toMatchObject({ + status: "inactive", + role: "admin", + }); + }); + + it("tombstones a membership the mirror has never seen, mirroring its organization first, so the backfill cannot insert it live", async () => { + const org = freshId("org"); + const userId = freshId("user"); + const reads: string[] = []; + const result = await run( + Effect.gen(function* () { + // The deletion lands before any feeder wrote the membership or the + // org; the org is read from WorkOS so the tombstone row can exist. + const deleted = yield* applyEvent( + membershipEvent( + "organization_membership.deleted", + workosMembership(userId, org), + freshId("event"), + T2, + ), + ); + const organization = yield* readOrganization(org); + const tombstone = yield* readMembership(userId, org, ["inactive"]); + // The backfill, listing WorkOS as it was before the deletion, writes + // the membership afterwards: refused, the tombstone stands. + const mirror = yield* WorkOsMirror; + const backfilled = yield* mirror.upsertMembership( + mirrorMembershipFromWorkOs(workosMembership(userId, org)), + ); + const afterBackfill = yield* readMembership(userId, org); + return { deleted, organization, tombstone, backfilled, afterBackfill }; + }), + stubWorkOS({ + getOrganization: (id) => { + reads.push(id); + return Effect.succeed(workosOrganization(id, "Dashboard Org")); + }, + }), + ); + expect(reads, "exactly one WorkOS read, for the unknown org").toEqual([org]); + expect(result.deleted, "the delete leaves a tombstone behind").toBe("applied"); + expect(result.organization?.name).toBe("Dashboard Org"); + expect(result.tombstone).toMatchObject({ + membershipId: `om_${userId}_${org}`, + status: "inactive", + }); + expect(result.backfilled, "the pre-deletion payload is refused").toBe(false); + expect(result.afterBackfill, "and the member is not live").toBeNull(); + }); + + it("mirrors the organization first when a membership names one the mirror has never seen", async () => { + const org = freshId("org"); + const userId = freshId("user"); + const reads: string[] = []; + const result = await run( + Effect.gen(function* () { + const outcome = yield* applyEvent( + membershipEvent("organization_membership.created", workosMembership(userId, org)), + ); + const organization = yield* readOrganization(org); + const membership = yield* readMembership(userId, org); + return { outcome, organization, membership }; + }), + stubWorkOS({ + ...profiles(reads), + getOrganization: (id) => { + reads.push(id); + return Effect.succeed(workosOrganization(id, "Dashboard Org")); + }, + }), + ); + expect(reads, "one WorkOS read for the unknown org, one for the unknown member").toEqual([ + org, + userId, + ]); + expect(result.outcome).toBe("applied"); + expect(result.organization?.name).toBe("Dashboard Org"); + expect(result.membership?.membershipId).toBe(`om_${userId}_${org}`); + }); + + it("reads the member's profile from WorkOS when the mirror holds none, never when it does, and mirrors a member WorkOS no longer has bare", async () => { + const org = freshId("org"); + const joiner = freshId("user"); + const known = freshId("user"); + const gone = freshId("user"); + const reads: string[] = []; + const result = await run( + Effect.gen(function* () { + yield* seedOrganization(org); + // `known` signed in before: the stream's own `user.created` for them + // is behind the replay boundary, but the mirror holds their profile. + yield* applyEvent( + userEvent("user.created", workosUser(known, { firstName: "Known", updatedAt: T1 })), + ); + const knownJoins = yield* applyEvent( + membershipEvent("organization_membership.created", workosMembership(known, org)), + ); + const knownRow = yield* readMembership(known, org); + // `joiner` predates the mirror: no row, no profile event in the + // stream, the org already scanned. The membership event alone + // would leave them nameless. + const joins = yield* applyEvent( + membershipEvent("organization_membership.created", workosMembership(joiner, org)), + ); + const joinerRow = yield* readMembership(joiner, org); + // A later role change for the now-profiled member reads nothing. + const promoted = yield* applyEvent( + membershipEvent( + "organization_membership.updated", + workosMembership(joiner, org, { role: { slug: "admin" }, updatedAt: T2 }), + ), + ); + // A member WorkOS no longer has (their `user.deleted` is further down + // the stream): mirrored without a profile, the run goes on. + const goneJoins = yield* applyEvent( + membershipEvent("organization_membership.created", workosMembership(gone, org)), + ); + const goneRow = yield* readMembership(gone, org); + return { knownJoins, knownRow, joins, joinerRow, promoted, goneJoins, goneRow }; + }), + stubWorkOS({ + getUser: (userId) => + Effect.suspend(() => { + reads.push(userId); + return userId === gone + ? Effect.fail(new WorkOSError({ status: 404 })) + : Effect.succeed(workosUser(userId, { firstName: "Fetched" })); + }), + }), + ); + expect(result.knownJoins).toBe("applied"); + expect(result.knownRow?.name, "a profiled member keeps the profile the mirror holds").toBe( + "Known Placeholder", + ); + expect(result.joins).toBe("applied"); + expect(result.joinerRow?.name, "an unprofiled member is mirrored WITH the profile").toBe( + "Fetched Placeholder", + ); + expect(result.joinerRow?.email).toBe(`${joiner}@placeholder.test`); + expect(result.promoted).toBe("applied"); + expect(result.goneJoins, "a member WorkOS no longer has is still mirrored").toBe("applied"); + expect(result.goneRow).toMatchObject({ membershipId: `om_${gone}_${org}`, name: null }); + expect(reads, "one read per unprofiled member, none for a profiled one").toEqual([ + joiner, + gone, + ]); + + // A transient failure reading the profile fails the run (the event is + // retried), exactly as for the organization read. + const blip = await Effect.runPromiseExit( + Effect.exit( + planEvent( + membershipEvent( + "organization_membership.created", + workosMembership(freshId("user"), org), + ), + ), + ).pipe( + Effect.provide( + Layer.mergeAll( + MirrorServices, + stubWorkOS({ getUser: () => Effect.fail(new WorkOSError({ status: 503 })) }), + ), + ), + Effect.scoped, + ), + ); + expect(Exit.isSuccess(blip) && Exit.isFailure(blip.value), "a 5xx keeps the event").toBe(true); + }); + + it("marks the organization deleted for a membership whose organization WorkOS no longer has, but fails on a transient WorkOS failure", async () => { + const org = freshId("org"); + const userId = freshId("user"); + const event = membershipEvent( + "organization_membership.created", + workosMembership(userId, org, { organizationName: "Gone Org" }), + freshId("event"), + T2, + ); + const gone = await run( + Effect.gen(function* () { + const outcome = yield* applyEvent(event); + const organization = yield* readOrganization(org); + const membership = yield* readMembership(userId, org); + // The org's own deletion event, further down the stream, finds the + // mark already there. + const deleted = yield* applyEvent( + organizationEvent( + "organization.deleted", + workosOrganization(org, "Gone Org"), + freshId("event"), + T3, + ), + ); + return { outcome, organization, membership, deleted }; + }), + stubWorkOS({ + getOrganization: () => Effect.fail(new WorkOSError({ status: 404 })), + }), + ); + expect( + gone.outcome, + "an org WorkOS has deleted marks the org deleted, not a failed run and not a dropped event", + ).toBe("applied"); + expect(gone.organization, "a tombstone row is minted for it").toMatchObject({ + name: "Gone Org", + deletedAt: new Date(T2), + }); + expect(gone.membership, "and the membership is not written").toBeNull(); + expect(gone.deleted, "its own deletion event finds the mark").toBe("absent"); + + // A transient failure resolving an org the mirror does not hold (the + // tombstone above would answer the read locally). + const unresolved = membershipEvent( + "organization_membership.created", + workosMembership(userId, freshId("org")), + ); + const blip = await Effect.runPromiseExit( + Effect.exit(planEvent(unresolved)).pipe( + Effect.provide( + Layer.mergeAll( + MirrorServices, + stubWorkOS({ + getOrganization: () => Effect.fail(new WorkOSError({ status: 503 })), + }), + ), + ), + Effect.scoped, + ), + ); + const unreachable = await Effect.runPromiseExit( + Effect.exit(planEvent(unresolved)).pipe( + Effect.provide( + Layer.mergeAll( + MirrorServices, + stubWorkOS({ + getOrganization: () => Effect.fail(new WorkOSError({})), + }), + ), + ), + Effect.scoped, + ), + ); + expect( + Exit.isSuccess(blip) && Exit.isFailure(blip.value), + "a 5xx keeps the event for retry", + ).toBe(true); + expect( + Exit.isSuccess(unreachable) && Exit.isFailure(unreachable.value), + "a network failure keeps the event for retry", + ).toBe(true); + }); + + it("does not create an organization row from organization.updated for an org the mirror has never seen", async () => { + const org = freshId("org"); + const result = await run( + Effect.gen(function* () { + const outcome = yield* applyEvent( + organizationEvent("organization.updated", workosOrganization(org, "Purged Org")), + ); + const organization = yield* readOrganization(org); + return { outcome, organization }; + }), + ); + expect(result.outcome, "a rename of an unmirrored org is reported absent").toBe("absent"); + expect( + result.organization, + "and mints no row (no resurrection after cloud's purge)", + ).toBeNull(); + }); + + it("mints a tombstone on organization.deleted for an org the mirror has never seen, so a delayed login cannot create it", async () => { + const org = freshId("org"); + const userId = freshId("user"); + const result = await run( + Effect.gen(function* () { + // The org was created, populated, and deleted in the WorkOS + // dashboard before anyone signed in: the mirror has no row for it. + const deleted = yield* applyEvent( + organizationEvent( + "organization.deleted", + workosOrganization(org, "Never Mirrored"), + freshId("event"), + T2, + ), + ); + const tombstone = yield* readOrganization(org); + // A login that fetched its membership list at T1, before the + // deletion, and stalled past it now writes what it holds. + yield* mirrorSignIn( + workosUser(userId), + [workosMembership(userId, org, { organizationName: "Never Mirrored" })], + new Date(T1), + ); + const afterLogin = yield* readOrganization(org); + const membership = yield* readMembership(userId, org, ["active", "pending", "inactive"]); + const replayed = yield* applyEvent( + organizationEvent( + "organization.deleted", + workosOrganization(org, "Never Mirrored"), + freshId("event"), + T3, + ), + ); + return { deleted, tombstone, afterLogin, membership, replayed }; + }), + ); + expect(result.deleted, "the deletion is applied, not dropped for want of a row").toBe( + "applied", + ); + expect(result.tombstone).toMatchObject({ + name: "Never Mirrored", + deletedAt: new Date(T2), + }); + expect(result.tombstone?.slug, "the tombstone is a slugged row like any other").toMatch( + /^never-mirrored/, + ); + expect(result.afterLogin?.deletedAt, "the delayed login does not revive the org").toEqual( + new Date(T2), + ); + expect(result.membership, "nor write the membership").toBeNull(); + expect(result.replayed, "a replayed deletion changes nothing").toBe("absent"); + }); + + it("renames the organization on organization.updated, refuses an older rename, and marks it deleted on organization.deleted, purging nothing", async () => { + const org = freshId("org"); + const userId = freshId("user"); + const result = await run( + Effect.gen(function* () { + const seeded = yield* seedOrganization(org); + yield* applyEvent( + membershipEvent("organization_membership.created", workosMembership(userId, org)), + ); + const renamed = yield* applyEvent( + organizationEvent("organization.updated", workosOrganization(org, "Renamed Org", T2)), + ); + const afterRename = yield* readOrganization(org); + // A rename event older than the name the row holds (replayed, or + // behind a sign-in that already carried the newer name). + const olderRename = yield* applyEvent( + organizationEvent("organization.updated", workosOrganization(org, "Older Name", T1)), + ); + const afterOlderRename = yield* readOrganization(org); + const deleted = yield* applyEvent( + organizationEvent( + "organization.deleted", + workosOrganization(org, "Renamed Org"), + freshId("event"), + T2, + ), + ); + const orgAfterDelete = yield* readOrganization(org); + const membershipAfterDelete = yield* readMembership(userId, org); + const deletedAgain = yield* applyEvent( + organizationEvent( + "organization.deleted", + workosOrganization(org, "Renamed Org"), + freshId("event"), + T3, + ), + ); + const renamedAfterDelete = yield* applyEvent( + organizationEvent("organization.updated", workosOrganization(org, "Late Rename", T3)), + ); + const orgAfterReplay = yield* readOrganization(org); + return { + seeded, + renamed, + afterRename, + olderRename, + afterOlderRename, + deleted, + orgAfterDelete, + membershipAfterDelete, + deletedAgain, + renamedAfterDelete, + orgAfterReplay, + }; + }), + stubWorkOS(profiles()), + ); + expect(result.renamed).toBe("applied"); + expect(result.afterRename?.name).toBe("Renamed Org"); + expect(result.afterRename?.slug, "the slug is stable across renames").toBe(result.seeded.slug); + expect(result.olderRename, "an older rename is refused").toBe("stale"); + expect(result.afterOlderRename?.name).toBe("Renamed Org"); + expect(result.deleted, "organization.deleted marks the org").toBe("applied"); + expect(result.orgAfterDelete?.deletedAt, "as of the event").toEqual(new Date(T2)); + expect(result.orgAfterDelete?.name, "the row is kept, not purged").toBe("Renamed Org"); + expect(result.membershipAfterDelete, "and so is the membership row").not.toBeNull(); + expect(result.deletedAgain, "a replayed deletion changes nothing").toBe("absent"); + expect(result.renamedAfterDelete, "a deleted org is never renamed").toBe("absent"); + expect(result.orgAfterReplay?.deletedAt, "the first mark stands").toEqual(new Date(T2)); + expect(result.orgAfterReplay?.name).toBe("Renamed Org"); + }); +}); + +describe("syncWorkOsEvents", () => { + /** Pin the instance-wide cursor to a fresh known value, whatever it was. */ + const pinCursor = (value: string) => + Effect.gen(function* () { + const mirror = yield* WorkOsMirror; + const before = yield* mirror.getCursor(); + const moved = yield* mirror.applyPage(before, value, []); + expect(Option.isSome(moved)).toBe(true); + return value; + }); + + type Page = { + readonly data: readonly WorkOsMirroredEvent[]; + readonly after: string | null; + }; + + /** + * A fake Events API serving `pages` in order, recording every request's + * paging options; `onPage` runs before the nth page is returned (the CAS + * contention test moves the cursor from there). `methods` adds any other + * WorkOS call the run under test is allowed to make. + */ + const eventsApi = ( + pages: readonly Page[], + requests: WorkOSListEventsOptions[], + onPage: (index: number) => Effect.Effect = () => Effect.void, + methods: Partial = {}, + ) => + Effect.map(WorkOsMirror.asEffect(), (mirror) => + stubWorkOS({ + ...methods, + listEvents: (options) => + Effect.gen(function* () { + const index = requests.length; + requests.push(options); + yield* onPage(index).pipe(Effect.provideService(WorkOsMirror, mirror), Effect.orDie); + const page = pages[index] ?? { data: [], after: null }; + return { + object: "list" as const, + data: [...page.data], + listMetadata: { before: null, after: page.after }, + }; + }), + }), + ); + + const sync = ( + workos: Layer.Layer, + ): Effect.Effect => + syncWorkOsEvents().pipe(Effect.provide(workos)); + + it("pages from the persisted cursor, applies every event, and commits the last id of each page", async () => { + const org = freshId("org"); + const a = freshId("user"); + const b = freshId("user"); + const requests: WorkOSListEventsOptions[] = []; + const profileReads: string[] = []; + const result = await run( + Effect.gen(function* () { + yield* seedOrganization(org); + const start = yield* pinCursor(freshId("event")); + const startedAt = Date.now(); + const pages: Page[] = [ + { + data: [ + userEvent("user.created", workosUser(a), `${start}_1`), + membershipEvent( + "organization_membership.created", + workosMembership(a, org), + `${start}_2`, + ), + ], + after: `${start}_2`, + }, + { + data: [ + membershipEvent( + "organization_membership.created", + workosMembership(b, org), + `${start}_3`, + ), + organizationEvent( + "organization.deleted", + workosOrganization(org, "Gone"), + `${start}_4`, + ), + ], + after: null, + }, + ]; + const workos = yield* eventsApi(pages, requests, () => Effect.void, profiles(profileReads)); + const report = yield* sync(workos); + const mirror = yield* WorkOsMirror; + const cursor = yield* mirror.getCursor(); + const members = yield* Effect.flatMap(MemberDirectory.asEffect(), (d) => d.members(org)); + const drainedAt = yield* mirror.drainedAt(); + return { start, startedAt, report, cursor, members, drainedAt }; + }), + ); + expect(requests.map((r) => r.after)).toEqual([result.start, `${result.start}_2`]); + expect(requests[0]).toMatchObject({ order: "asc", limit: 100 }); + expect(requests[0]?.rangeStart, "a run with a cursor never sends rangeStart").toBeUndefined(); + expect(result.report).toMatchObject({ + pages: 2, + events: 4, + applied: 4, + stopped: "drained", + cursor: `${result.start}_4`, + }); + expect(result.cursor).toBe(`${result.start}_4`); + expect(result.members.map((m) => m.accountId).sort()).toEqual([a, b].sort()); + expect( + profileReads, + "only the member whose profile the stream did not carry is read from WorkOS", + ).toEqual([b]); + expect( + result.drainedAt, + "a run that reads the stream to its end records the drain", + ).not.toBeNull(); + expect( + result.drainedAt!.getTime(), + "as of the run's start, so it never post-dates an event the run did not see", + ).toBeGreaterThanOrEqual(result.startedAt - 1000); + expect(result.drainedAt!.getTime()).toBeLessThanOrEqual(Date.now()); + }); + + /** The sync row is instance-wide: clear it so the run under test is a first run. */ + const clearSyncRow = Effect.flatMap(DbService.asEffect(), ({ db }) => + Effect.promise(() => db.execute(sql`delete from workos_sync where id = 'events'`)), + ); + + it("starts from the backfill's replay boundary when no cursor exists", async () => { + const requests: WorkOSListEventsOptions[] = []; + const result = await run( + Effect.gen(function* () { + const mirror = yield* WorkOsMirror; + yield* clearSyncRow; + yield* mirror.setReplayBoundary(new Date(T2)); + const start = freshId("event"); + const workos = yield* eventsApi( + [ + { + data: [userEvent("user.created", workosUser(freshId("user")), start)], + after: null, + }, + ], + requests, + ); + const report = yield* sync(workos); + const cursor = yield* mirror.getCursor(); + return { report, cursor, start }; + }), + ); + expect(requests).toHaveLength(1); + expect(requests[0]?.after).toBeUndefined(); + expect( + requests[0]?.rangeStart, + "the first read starts exactly where the backfill began reading WorkOS", + ).toBe(T2); + expect(result.report.stopped).toBe("drained"); + expect(result.cursor, "the first run mints the cursor").toBe(result.start); + }); + + it("reads nothing while neither a cursor nor a replay boundary exists", async () => { + const requests: WorkOSListEventsOptions[] = []; + const result = await run( + Effect.gen(function* () { + const mirror = yield* WorkOsMirror; + yield* clearSyncRow; + const workos = yield* eventsApi( + [ + { + data: [userEvent("user.created", workosUser(freshId("user")))], + after: null, + }, + ], + requests, + ); + const report = yield* sync(workos); + const cursor = yield* mirror.getCursor(); + const drainedAt = yield* mirror.drainedAt(); + return { report, cursor, drainedAt }; + }), + ); + expect(requests, "no wall-clock guess is ever sent to WorkOS").toHaveLength(0); + expect(result.report).toMatchObject({ + pages: 0, + events: 0, + stopped: "awaiting_backfill", + }); + expect(result.cursor, "and no cursor is minted").toBeNull(); + expect(result.drainedAt, "nor is a drain recorded: nothing was read").toBeNull(); + }); + + it("stops when another run moves the cursor under it, writing nothing from the contended page", async () => { + const org = freshId("org"); + const userId = freshId("user"); + const requests: WorkOSListEventsOptions[] = []; + const result = await run( + Effect.gen(function* () { + yield* seedOrganization(org); + const mirror = yield* WorkOsMirror; + const start = yield* pinCursor(freshId("event")); + const intruder = `${start}_intruder`; + // This run's page carries a membership update that the leading run + // has already applied AND deleted (the member was revoked). If the + // lagging run's page landed, the revoked member would be back. + const pages: Page[] = [ + { + data: [ + membershipEvent( + "organization_membership.updated", + workosMembership(userId, org, { role: { slug: "admin" } }), + `${start}_1`, + ), + ], + after: `${start}_1`, + }, + { + data: [userEvent("user.created", workosUser(freshId("user")), `${start}_2`)], + after: null, + }, + ]; + // While this run is reading its first page, "another run" applies the + // same page, then the membership's deletion, and commits both. + const workos = yield* eventsApi( + pages, + requests, + (index) => + index === 0 + ? Effect.gen(function* () { + const leading = yield* WorkOsMirror; + yield* leading.applyPage(start, `${start}_1`, [ + WorkOsMirrorWrite.UpsertMembership({ + membership: mirrorMembershipFromWorkOs(workosMembership(userId, org)), + }), + ]); + yield* leading.applyPage(`${start}_1`, intruder, [ + WorkOsMirrorWrite.DeleteMembership({ + membership: { + id: `om_${userId}_${org}`, + accountId: userId, + organizationId: org, + }, + deletedAt: new Date(T2), + }), + ]); + }) + : Effect.void, + profiles(), + ); + const drainedBefore = yield* mirror.drainedAt(); + const report = yield* sync(workos); + const cursor = yield* mirror.getCursor(); + const membership = yield* readMembership(userId, org); + const drainedAfter = yield* mirror.drainedAt(); + return { report, cursor, intruder, membership, drainedBefore, drainedAfter }; + }), + ); + expect(requests, "the second page is never read").toHaveLength(1); + expect(result.report).toMatchObject({ + pages: 1, + events: 1, + applied: 0, + stopped: "cursor_contended", + }); + expect(result.cursor, "the other run's cursor stands").toBe(result.intruder); + expect(result.membership, "the revoked membership is not resurrected").toBeNull(); + expect( + result.drainedAfter, + "a run that yielded the stream drained nothing and records no drain", + ).toEqual(result.drainedBefore); + }); + + it("advances the cursor past a membership event whose organization WorkOS no longer has, marking the org deleted", async () => { + const org = freshId("org"); + const userId = freshId("user"); + const requests: WorkOSListEventsOptions[] = []; + const result = await run( + Effect.gen(function* () { + const start = yield* pinCursor(freshId("event")); + const workos = yield* eventsApi( + [ + { + data: [ + membershipEvent( + "organization_membership.created", + workosMembership(userId, org), + `${start}_1`, + ), + organizationEvent( + "organization.deleted", + workosOrganization(org, "Gone"), + `${start}_2`, + ), + ], + after: null, + }, + ], + requests, + () => Effect.void, + { + getOrganization: () => Effect.fail(new WorkOSError({ status: 404 })), + }, + ); + const report = yield* sync(workos); + const mirror = yield* WorkOsMirror; + const cursor = yield* mirror.getCursor(); + const organization = yield* readOrganization(org); + return { start, report, cursor, organization }; + }), + ); + expect(result.report).toMatchObject({ + pages: 1, + events: 2, + applied: 1, + absent: 1, + stopped: "drained", + cursor: `${result.start}_2`, + }); + expect(result.cursor, "the stream is not stalled on the gone org").toBe(`${result.start}_2`); + expect(result.organization?.deletedAt, "the org is left as a tombstone").not.toBeNull(); + }); +}); + +describe("workos webhook", () => { + const SECRET = "whsec_placeholder_signing_secret"; + + const handlerFor = (deps: { + readonly secret: string | undefined; + readonly detached: Promise[]; + readonly synced: number[]; + }) => + HttpRouter.toWebHandler( + makeWorkOsWebhookRoute({ + secret: deps.secret, + detach: (work) => { + deps.detached.push(work); + }, + sync: () => { + deps.synced.push(1); + return Promise.resolve(); + }, + }).pipe( + // The REAL client: its `webhooks.constructEvent` is the signature check + // under test. The api key / client id it reads are the vitest env's. + Layer.provideMerge(WorkOSClient.Default), + Layer.provideMerge(HttpServer.layerServices), + ), + { disableLogger: true }, + ).handler; + + const delivery = { + id: "event_placeholder", + event: "user.created", + created_at: T1, + context: {}, + data: { + object: "user", + id: "user_placeholder", + email: "member@placeholder.test", + email_verified: true, + first_name: "Ada", + last_name: "Placeholder", + profile_picture_url: null, + last_sign_in_at: T1, + locale: null, + created_at: T1, + updated_at: T1, + external_id: null, + metadata: {}, + }, + }; + + /** The `WorkOS-Signature` header WorkOS sends: `t=, v1=`. */ + const signature = (body: string, secret: string, timestamp = Date.now()) => + `t=${timestamp}, v1=${createHmac("sha256", secret).update(`${timestamp}.${body}`).digest("hex")}`; + + const post = (body: string, headers: Record) => + new Request(`http://test.local${WORKOS_WEBHOOK_PATH}`, { + method: "POST", + headers: { "content-type": "application/json", ...headers }, + body, + }); + + const deps = (secret: string | undefined) => ({ + secret, + detached: [] as Promise[], + synced: [] as number[], + }); + + it("accepts a genuinely signed delivery and pokes the reconciler past the response", async () => { + const d = deps(SECRET); + const body = JSON.stringify(delivery); + const response = await handlerFor(d)( + post(body, { "workos-signature": signature(body, SECRET) }), + ); + expect(response.status).toBe(200); + expect(d.synced, "one reconciler pass").toHaveLength(1); + expect(d.detached, "handed to the platform, not awaited in the response").toHaveLength(1); + }); + + it("rejects a delivery signed with another secret, a tampered body, and a missing header", async () => { + const d = deps(SECRET); + const handler = handlerFor(d); + const body = JSON.stringify(delivery); + + const wrongSecret = await handler( + post(body, { "workos-signature": signature(body, "whsec_other") }), + ); + const tampered = await handler( + post(body.replace("user_placeholder", "user_tampered"), { + "workos-signature": signature(body, SECRET), + }), + ); + const unsigned = await handler(post(body, {})); + const notJson = await handler( + post("not json", { "workos-signature": signature("not json", SECRET) }), + ); + const expired = await handler( + post(body, { + "workos-signature": signature(body, SECRET, Date.now() - 10 * 60 * 1000), + }), + ); + + expect([ + wrongSecret.status, + tampered.status, + unsigned.status, + notJson.status, + expired.status, + ]).toEqual([400, 400, 400, 400, 400]); + expect(d.synced, "nothing is poked").toEqual([]); + }); + + it("refuses every delivery while no signing secret is configured", async () => { + const d = deps(undefined); + const body = JSON.stringify(delivery); + const response = await handlerFor(d)( + post(body, { "workos-signature": signature(body, SECRET) }), + ); + expect(response.status).toBe(503); + expect(d.synced).toEqual([]); + }); +}); diff --git a/apps/cloud/src/auth/workos-events-sync.ts b/apps/cloud/src/auth/workos-events-sync.ts new file mode 100644 index 0000000000..d404d5fbfa --- /dev/null +++ b/apps/cloud/src/auth/workos-events-sync.ts @@ -0,0 +1,107 @@ +// --------------------------------------------------------------------------- +// The membership mirror's reconciler, bound to the Worker's services: the +// replay itself is `workos-events-replay.ts` (a pure function over its +// ports, so the deploy gate can run the same replay under bun); this file +// wires those ports to `WorkOSClient`, `UserStoreService`, and +// `WorkOsMirror` for the every-minute cron and the signed webhook poke +// (`workos-events-runner.ts`). +// +// The only translation here is the WorkOS answer "gone": the replay wants +// `None` for an organization or user WorkOS no longer has, and only a 404 +// says that. A 401/403 is a credentials or permissions problem with THIS +// deployment, and 429/5xx/no status is a blip: all of those stay failures +// so the run stops and the event is retried once fixed, not skipped and +// lost. +// --------------------------------------------------------------------------- + +import { Effect, Option } from "effect"; + +import { UserStoreService } from "./context"; +import type { UserStoreError, WorkOSError } from "./errors"; +import { WorkOSClient } from "./workos"; +import { + planWorkOsEvent, + replayWorkOsEvents, + type PlannedProfiles, + type WorkOsEventsReplayDeps, + type WorkOsMirroredEvent, +} from "./workos-events-replay"; +import { WorkOsMirror, type WorkOsMirrorError } from "./workos-mirror"; + +export { + MIRRORED_EVENT_NAMES, + isMirroredEvent, + type PlannedProfiles, + type WorkOsEventOutcome, + type WorkOsEventsSyncReport, + type WorkOsMirroredEvent, + type WorkOsMirroredEventName, +} from "./workos-events-replay"; + +type SyncFailure = WorkOSError | UserStoreError | WorkOsMirrorError; + +// A 404 is the deterministic "gone" the replay acts on; everything else +// fails the run (see the header). +const noneWhenGone = ( + read: Effect.Effect, +): Effect.Effect, WorkOSError> => + read.pipe( + Effect.map(Option.some), + Effect.catchTag("WorkOSError", (error) => + error.status === 404 ? Effect.succeed(Option.none()) : Effect.fail(error), + ), + ); + +const replayDeps: Effect.Effect< + WorkOsEventsReplayDeps, + never, + WorkOSClient | UserStoreService | WorkOsMirror +> = Effect.gen(function* () { + const workos = yield* WorkOSClient; + const users = yield* UserStoreService; + const mirror = yield* WorkOsMirror; + return { + source: { + listEvents: (options) => + Effect.map(workos.listEvents(options), (page) => ({ + data: page.data, + after: page.listMetadata.after ?? null, + })), + getOrganization: (organizationId) => noneWhenGone(workos.getOrganization(organizationId)), + getUser: (userId) => noneWhenGone(workos.getUser(userId)), + }, + store: { + getOrganization: (organizationId) => + users.use("getOrganization", (s) => s.getOrganization(organizationId)), + upsertOrganization: (organization) => + users.use("upsertOrganization", (s) => s.upsertOrganization(organization)), + getAccount: (accountId) => users.use("getAccount", (s) => s.getAccount(accountId)), + }, + mirror, + }; +}); + +/** + * Translate one event into the mirror write it calls for, over the Worker's + * services. See `planWorkOsEvent` for the contract. + */ +export const planEvent = Effect.fn("workos_events.plan")(function* ( + event: WorkOsMirroredEvent, + profiled: PlannedProfiles = new Set(), +) { + yield* Effect.annotateCurrentSpan({ + "workos.event": event.event, + "workos.event_id": event.id, + }); + const deps = yield* replayDeps; + return yield* planWorkOsEvent(deps, event, profiled); +}); + +/** + * One reconciler run over the Worker's services. See `replayWorkOsEvents` + * for the contract. + */ +export const syncWorkOsEvents = Effect.fn("workos_events.sync")(function* () { + const deps = yield* replayDeps; + return yield* replayWorkOsEvents(deps); +}); diff --git a/apps/cloud/src/auth/workos-mirror-backfill.ts b/apps/cloud/src/auth/workos-mirror-backfill.ts index adcd7c11f7..d6875241b2 100644 --- a/apps/cloud/src/auth/workos-mirror-backfill.ts +++ b/apps/cloud/src/auth/workos-mirror-backfill.ts @@ -42,17 +42,18 @@ // caller's own membership) starts unmarked and is scanned before any count // read from the mirror is trusted. A run over every organization // (`backfillWorkOsMirror`) additionally records the Events API replay -// boundary — the instant it began reading WorkOS, taken BEFORE anything is -// listed so no change can fall between the boundary and a listing — once -// every organization was written, and only if no boundary is recorded yet. -// A run that fails part-way records nothing, and a later completed run -// keeps the first boundary: a scan refreshes memberships and tombstones, -// not organization names or deleted users' profiles, so an -// `organization.updated` or `user.deleted` between two runs is covered only -// by the events stream — advancing the boundary past it would skip it for -// good. The reconciler's own cursor takes over from the boundary after its -// first page, so the boundary's only job is to name where that first page -// starts. +// boundary — the instant it began reading WorkOS — BEFORE anything is +// listed, and only if no boundary is recorded yet. Recording it first, not +// on completion, is what makes a failed run safe to retry: a run that +// fails part-way has already fixed the boundary at its start, and its +// retry reads that boundary back instead of taking a fresh, later one. A +// scan refreshes memberships and tombstones, not organization names or +// deleted users' profiles, so an `organization.updated` or `user.deleted` +// that lands between the attempts (or between two runs) is covered only by +// the events stream — a boundary taken by the retry would fall after it and +// skip it for good, leaving the deleted user's profile in the mirror. The +// reconciler's own cursor takes over from the boundary after its first +// page, so the boundary's only job is to name where that first page starts. // // The mark also orders every OTHER membership write against the scan: the // mirror refuses a membership payload stamped before the organization's @@ -196,11 +197,12 @@ export const backfillOrganization = ( ); /** - * Run the full backfill: scan every organization the mirror knows, then - * record the Events API replay boundary if none is recorded yet. Fails on - * the first source or mirror failure — the organizations scanned so far stay - * marked (each was covered in full), the boundary is not recorded, and the - * run is safe to repeat. + * Run the full backfill: record the Events API replay boundary if none is + * recorded yet, then scan every organization the mirror knows. Fails on the + * first source or mirror failure — the organizations scanned so far stay + * marked (each was covered in full), the boundary recorded at the start + * stands, and the run is safe to repeat: the retry keeps that boundary, so + * every change since the first attempt began is the reconciler's to replay. */ export const backfillWorkOsMirror = ( source: WorkOsMirrorBackfillSource, @@ -208,8 +210,21 @@ export const backfillWorkOsMirror = ( options: WorkOsMirrorBackfillOptions, ) => Effect.gen(function* () { - // Taken before the first listing; recorded only once the run completes. + // The replay boundary: taken AND recorded before anything is listed, so + // every change from this instant on is the events stream's to apply — + // one that lands while this run is still listing, or between this run + // failing part-way and its retry. Kept only when none is recorded yet + // (`setReplayBoundary`): a retry or a later run reads the first one + // back instead of moving it. A dry run records nothing. const boundary = yield* now(); + if (!options.dryRun) { + const recorded = yield* mirror.setReplayBoundary(boundary); + options.log( + recorded + ? `events replay boundary set to ${boundary.toISOString()}` + : "events replay boundary already recorded by an earlier run; kept (the events reconciler replays every change since it)", + ); + } const organizationIds = yield* source.listOrganizationIds(); let memberships = 0; @@ -246,22 +261,11 @@ export const backfillWorkOsMirror = ( : `${counts.organizations} organization(s), ${counts.memberships} membership(s): wrote ${counts.usersWritten} user(s), ${counts.membershipsWritten} membership(s), tombstoned ${counts.membershipsTombstoned}`, ); if (!options.dryRun) { - // Every organization was read and written without failure (a failure - // above fails the whole run) — or refused in favour of a listing taken - // later still — so everything before `boundary` is now covered: record - // it for the reconciler — unless an earlier run already did, in which - // case the events between the two runs are the reconciler's to replay - // and the earlier boundary stands. - const recorded = yield* mirror.setReplayBoundary(boundary); - options.log( - recorded - ? `events replay boundary set to ${boundary.toISOString()}` - : "events replay boundary already recorded by an earlier run; kept (the events reconciler replays every change since it)", - ); - // And every live organization is now covered: the mirror is complete - // enough to authorize from (once the reconciler has caught up too), - // which the authorization path reads as the first half of readiness. - // Once: a re-run keeps the first completion. + // Every live organization is now covered (a failure above fails the + // whole run): the mirror is complete enough to authorize from, once + // the reconciler has caught up too — the first half of the readiness + // the authorization path checks. Once: a re-run keeps the first + // completion. const completedAt = yield* now(); const marked = yield* mirror.markBackfillCompleted(completedAt); options.log( diff --git a/apps/cloud/src/auth/workos-mirror-store.ts b/apps/cloud/src/auth/workos-mirror-store.ts index e9c7e544b3..1a5886283a 100644 --- a/apps/cloud/src/auth/workos-mirror-store.ts +++ b/apps/cloud/src/auth/workos-mirror-store.ts @@ -45,14 +45,17 @@ // cleared, stamped with the deletion), and no membership naming that account // is written again, however the payload is stamped — WorkOS never reuses a // user id, and a membership the mirror had not seen has no row of its own -// for a guard to refuse the insert against. The cursor advances only by -// compare-and-set, so two reconciler runs cannot both own the stream. +// for a guard to refuse the insert against. The reconciler applies a page +// of events and advances the cursor in ONE transaction that +// compare-and-sets the cursor first (`applyPage`), so a run that has lost +// the stream to another run writes nothing — the `updatedAt` guard alone +// cannot stop it re-applying a page the leading run has moved past. // // A backfill SCAN of one organization (its full membership listing, taken -// at one instant) is applied in ONE transaction that first compare-and-sets -// the organization's `backfilled_at` to the listing's instant -// (`applyOrganizationScan`): the row stays locked until commit, so two -// overlapping scans serialize on it, and the one whose listing is older +// at one instant) is applied the same way: ONE transaction that first +// compare-and-sets the organization's `backfilled_at` to the listing's +// instant (`applyOrganizationScan`), so the row stays locked until commit, +// two overlapping scans serialize on it, and the one whose listing is older // than the recorded one writes NOTHING. The `updatedAt` guard alone cannot // order two scans: a scan that listed a membership, stalled, and resumed // after a later scan had found it gone would insert it live — the later scan @@ -72,15 +75,16 @@ // listing that sets the mark. // // The events replay boundary (`workos_sync.range_start`) is written ONCE, by -// the first completed backfill, and never advanced: a later backfill -// refreshes memberships only, so an organization rename or user deletion -// between two runs is covered by the events stream alone, and moving the -// boundary past it would skip it for good. +// the first backfill run BEFORE its first listing, and never advanced: a +// scan refreshes memberships only, so an organization rename or user +// deletion after that instant — between two runs, or between a run that +// failed part-way and its retry — is covered by the events stream alone, +// and moving the boundary past it would skip it for good. // --------------------------------------------------------------------------- import { and, eq, isNotNull, isNull, lt, ne, notInArray, or, sql } from "drizzle-orm"; import type { AnyPgColumn } from "drizzle-orm/pg-core"; -import { Effect, Option } from "effect"; +import { Data, Effect, Option } from "effect"; import type { MemberStatus } from "@executor-js/api/server"; @@ -92,6 +96,7 @@ import { workosSync, } from "../db/schema"; import type { DrizzleDb } from "../db/db"; +import { insertOrganization, organizationAcceptsName } from "./user-store"; import { WorkOsMirrorError, tryPromiseService, @@ -163,6 +168,70 @@ export interface WorkOsOrganizationScanWrites { readonly membershipsTombstoned: number; } +/** + * One write of a reconciler page, applied by `applyPage` inside the page's + * transaction. The reconciler plans a page into these BEFORE the transaction + * opens, so every WorkOS read (resolving an organization the mirror has never + * seen) is done by then: the transaction holds the mirror's single connection + * and must not wait on the network. + */ +export type WorkOsMirrorWrite = Data.TaggedEnum<{ + readonly UpsertUser: { readonly user: WorkOsMirrorUser }; + readonly UpsertMembership: { readonly membership: WorkOsMirrorMembership }; + /** + * A membership together with its member's profile, read from WorkOS at + * plan time because the mirror held no profile for the account + * (`workos-events-sync.ts`): the user is upserted first, under the usual + * guard, then the membership. The outcome is the membership's. + */ + readonly UpsertMember: { + readonly user: WorkOsMirrorUser; + readonly membership: WorkOsMirrorMembership; + }; + /** Tombstone a membership as of `deletedAt` (the event's `createdAt`). */ + readonly DeleteMembership: { + readonly membership: WorkOsMirrorMembershipRef; + readonly deletedAt: Date; + }; + /** Tombstone a user and their memberships as of `deletedAt`. */ + readonly DeleteUser: { readonly accountId: string; readonly deletedAt: Date }; + /** + * Rename an organization the mirror already holds — never inserts one — + * from a payload stamped `updatedAt` (the WorkOS organization's own), under + * the same name guard every feeder applies (`organizationAcceptsName`). + */ + readonly RenameOrganization: { + readonly organizationId: string; + readonly name: string; + readonly updatedAt: Date; + }; + /** + * Mark an organization deleted as of `deletedAt` (the event's + * `createdAt`) — minting the row as a TOMBSTONE, named `name`, when the + * mirror has never seen the organization, so a feeder still holding a + * membership of it (a login that stalled across the deletion) finds the + * tombstone and cannot mint the organization live. Never purges tenant + * data (that is cloud's own flow, `db/org-deletion.ts`). An earlier mark + * stands (`absent`). + */ + readonly MarkOrganizationDeleted: { + readonly organizationId: string; + readonly name: string; + readonly deletedAt: Date; + }; +}>; +export const WorkOsMirrorWrite = Data.taggedEnum(); + +/** + * What one write did: a row was written, renamed, or tombstoned (`applied`); + * the `updatedAt` guard refused an older payload (`stale`); or a delete found + * its row already tombstoned or superseded by a newer membership (a + * replayed delete), or a rename found no live organization row to change — + * the mirror has never seen it, or it is marked deleted — or a deletion + * mark found the organization already marked (`absent`). + */ +export type WorkOsMirrorWriteOutcome = "applied" | "stale" | "absent"; + export interface WorkOsMirrorShape { /** * Insert or refresh a user row. `false` when the payload was refused and @@ -250,14 +319,20 @@ export interface WorkOsMirrorShape { /** The id of the last WorkOS event applied, or `null` before the first run. */ readonly getCursor: () => Effect.Effect; /** - * Compare-and-set the cursor: advance to `next` only if it still reads - * `prev` (`null` = no cursor yet). `false` means another run moved it first - * — the caller must stop, it no longer owns the stream. + * Apply one reconciler page atomically: in a single transaction, + * compare-and-set the cursor from `prev` (`null` = no cursor yet) to + * `next`, and only if that succeeded apply `writes` in order. The cursor + * row stays locked until commit, so two runs applying pages serialize on + * it and the one whose `prev` is stale sees the moved cursor and writes + * nothing: `None` means another run owns the stream and the caller must + * stop. `Some` carries one outcome per write, in order. An empty `writes` + * is a bare cursor advance. */ - readonly setCursor: ( + readonly applyPage: ( prev: string | null, next: string, - ) => Effect.Effect; + writes: readonly WorkOsMirrorWrite[], + ) => Effect.Effect, WorkOsMirrorError>; /** * Apply one organization's backfill scan — the memberships (with their * users) a WorkOS listing taken at `listedAt` contained — atomically: in a @@ -283,24 +358,25 @@ export interface WorkOsMirrorShape { scan: WorkOsOrganizationScan, ) => Effect.Effect, WorkOsMirrorError>; /** - * The Events API replay boundary: the instant the FIRST completed one-off - * backfill began reading WorkOS, or `null` if none has completed. The + * The Events API replay boundary: the instant the FIRST one-off backfill + * run began reading WorkOS, or `null` if none has started. The * reconciler's first run (no cursor yet) reads the stream from here — the * backfill covers everything before it — and without a boundary it must * not guess. */ readonly replayBoundary: () => Effect.Effect; /** - * Record the replay boundary, ONCE: `at` is the instant a completed - * backfill began reading WorkOS, and it is kept only when no boundary is - * recorded yet — `true` when this call recorded it. A later completed run - * never moves it: the backfill refreshes memberships and tombstones only, - * not organization names or deleted users' profiles, so a change between - * two runs is covered only by the events stream, which must still be read - * from the first boundary. Written only after every organization has been - * written, so a run that fails part-way records nothing. Never touches the - * cursor: a stream already being followed keeps its position, and the - * boundary is then unused. + * Record the replay boundary, ONCE: `at` is the instant a backfill run + * began reading WorkOS, and it is kept only when no boundary is recorded + * yet — `true` when this call recorded it. A later run never moves it: the + * backfill refreshes memberships and tombstones only, not organization + * names or deleted users' profiles, so a change after the first boundary + * is covered only by the events stream, which must still be read from + * there. Written BEFORE the run's first listing, so a run that fails + * part-way leaves the boundary standing and its retry keeps it — a + * `user.deleted` between the attempts stays inside the replay. Never + * touches the cursor: a stream already being followed keeps its position, + * and the boundary is then unused. */ readonly setReplayBoundary: (at: Date) => Effect.Effect; /** @@ -323,6 +399,22 @@ export interface WorkOsMirrorShape { * boundary. */ readonly markBackfillCompleted: (at: Date) => Effect.Effect; + /** + * When a reconciler run last read the events stream to its end + * (`workos_sync.drained_at`), or `null` if none has. The second half of + * the mirror's readiness for authorization: a mirror whose reconciler + * has not caught up within the lag budget may still hold a membership + * WorkOS has since revoked. + */ + readonly drainedAt: () => Effect.Effect; + /** + * Record that a reconciler run read the stream to its end at `at`. Moves + * the mark forward only — a run that finished after a later one keeps the + * later mark — and only on the row a run already owns: the events row is + * minted by the boundary or the first cursor advance, so a missing row + * means nothing was drained and nothing is written (`false`). + */ + readonly markDrained: (at: Date) => Effect.Effect; /** * When the organization's membership list was last FULLY scanned from * WorkOS (`backfillOrganization` in workos-mirror-backfill.ts), or `null` @@ -502,9 +594,9 @@ const membershipDeletableBy = (id: string) => and(or(isNull(memberships.membershipId), eq(memberships.membershipId, id)), notDeleted); // The write queries, over `db` or over a transaction handle (drizzle's is a -// `PgDatabase` too): the one `applyOrganizationScan` opens, or the one the -// store opens per `upsertMembership`. Each answers whether it wrote a row; -// the public shape translates that. +// `PgDatabase` too): the one `applyPage` or `applyOrganizationScan` opens, +// or the one the store opens per `upsertMembership`. Each answers whether +// it wrote a row; the public shape and the transactions translate that. const makeWrites = (db: DrizzleDb) => { const ensureAccount = (id: string) => db.insert(accounts).values({ id }).onConflictDoNothing({ target: accounts.id }); @@ -742,6 +834,82 @@ const makeWrites = (db: DrizzleDb) => { return cleared.length > 0; }, + // An UPDATE, never an insert: the slug is minted only by + // `upsertOrganization` (auth/user-store.ts), and an org purged by cloud's + // own deletion flow must not come back — with a fresh slug and no members + // — because a rename that preceded the deletion is replayed after it. + // Only of a LIVE row, and only from a payload at least as new as the one + // that last named it: the same guard `upsertOrganization` applies, so + // an event rename and a sign-in's name (stamped by its fetch) order + // each other however they arrive. + renameOrganization: async ( + organizationId: string, + name: string, + updatedAt: Date, + ): Promise => { + const renamed = await db + .update(organizations) + .set({ name, workosUpdatedAt: updatedAt }) + .where( + and( + eq(organizations.id, organizationId), + isNull(organizations.deletedAt), + organizationAcceptsName(updatedAt), + ), + ) + .returning({ id: organizations.id }); + if (renamed.length > 0) return "applied"; + // Refused: tell a live row the guard held back (`stale`) from a row + // the mirror does not hold or holds as deleted (`absent`). + const live = await db + .select({ id: organizations.id }) + .from(organizations) + .where(and(eq(organizations.id, organizationId), isNull(organizations.deletedAt))); + return live.length > 0 ? "stale" : "absent"; + }, + + // Marks a live row, or MINTS a tombstone row when the mirror has never + // seen the organization: an org created, populated, and deleted in the + // WorkOS dashboard before anyone signed in leaves no row behind + // otherwise, and a login that fetched its memberships before the + // deletion (and stalled) would then mint the org live, with nothing + // left in the stream to revoke it — this event is consumed. A row + // already marked is left alone: cloud's own deletion flow marks the org + // before deleting it in WorkOS, so the event that follows finds the + // mark already there and changes nothing — `false`, as for a replayed + // event. Minted through the one slug mint point (`insertOrganization`), + // so a tombstone is a routable, unique-slugged row like any other; the + // mark alone is what refuses it. + markOrganizationDeleted: async ( + organizationId: string, + name: string, + deletedAt: Date, + ): Promise => { + const mark = async () => { + const marked = await db + .update(organizations) + .set({ deletedAt }) + .where(and(eq(organizations.id, organizationId), isNull(organizations.deletedAt))) + .returning({ id: organizations.id }); + return marked.length > 0; + }; + if (await mark()) return true; + const held = await db + .select({ id: organizations.id }) + .from(organizations) + .where(eq(organizations.id, organizationId)); + if (held.length > 0) return false; + const minted = await insertOrganization(db, { + id: organizationId, + name, + workosUpdatedAt: null, + deletedAt, + }); + // A concurrent feeder may have minted the row LIVE between the read + // and the insert (the insert then yields its row): mark that one. + return minted.deletedAt !== null || mark(); + }, + // Tombstone (at `listedAt`) every membership of the organization that a // listing taken at `listedAt` did not contain. Only inside a scan's // transaction, after its `backfilled_at` CAS: on its own this could @@ -800,6 +968,57 @@ const makeWrites = (db: DrizzleDb) => { }; }; +type Writes = ReturnType; + +const applyWrite = (writes: Writes, write: WorkOsMirrorWrite): Promise => + WorkOsMirrorWrite.$match(write, { + UpsertUser: async ({ user }) => ((await writes.upsertUser(user)) ? "applied" : "stale"), + UpsertMembership: async ({ membership }) => + (await writes.upsertMembership(membership)) ? "applied" : "stale", + UpsertMember: async ({ user, membership }) => { + await writes.upsertUser(user); + return (await writes.upsertMembership(membership)) ? "applied" : "stale"; + }, + DeleteMembership: async ({ membership, deletedAt }) => + (await writes.deleteMembership(membership, deletedAt)) ? "applied" : "absent", + DeleteUser: async ({ accountId, deletedAt }) => + (await writes.deleteUser(accountId, deletedAt)) ? "applied" : "absent", + RenameOrganization: ({ organizationId, name, updatedAt }) => + writes.renameOrganization(organizationId, name, updatedAt), + MarkOrganizationDeleted: async ({ organizationId, name, deletedAt }) => + (await writes.markOrganizationDeleted(organizationId, name, deletedAt)) + ? "applied" + : "absent", + }); + +// Compare-and-set the events cursor. Run inside a transaction this also +// LOCKS the cursor row until commit: a concurrent run's CAS waits here, then +// re-reads the moved cursor and matches nothing. +const advanceCursor = async (db: DrizzleDb, prev: string | null, next: string) => { + const now = new Date(); + if (prev === null) { + // First advance: mint the row, or claim an existing row that still has + // no cursor. A row that already carries one belongs to another run and + // is left alone. + const written = await db + .insert(workosSync) + .values({ id: WORKOS_EVENTS_STREAM_ID, cursor: next, updatedAt: now }) + .onConflictDoUpdate({ + target: workosSync.id, + set: { cursor: next, updatedAt: now }, + setWhere: isNull(workosSync.cursor), + }) + .returning({ id: workosSync.id }); + return written.length > 0; + } + const written = await db + .update(workosSync) + .set({ cursor: next, updatedAt: now }) + .where(and(eq(workosSync.id, WORKOS_EVENTS_STREAM_ID), eq(workosSync.cursor, prev))) + .returning({ id: workosSync.id }); + return written.length > 0; +}; + // Claim the organization for a scan listed at `listedAt`: move its // `backfilled_at` forward to `listedAt` if the recorded mark is older (or // missing). Run inside a transaction this also LOCKS the organization row @@ -875,31 +1094,21 @@ export const makeWorkOsMirrorStore = (db: DrizzleDb): WorkOsMirrorShape => { return rows[0]?.cursor ?? null; }), - setCursor: (prev, next) => - run("setCursor", async () => { - const now = new Date(); - if (prev === null) { - // First advance: mint the row, or claim an existing row that still - // has no cursor. A row that already carries one belongs to another - // run and is left alone. - const written = await db - .insert(workosSync) - .values({ id: WORKOS_EVENTS_STREAM_ID, cursor: next, updatedAt: now }) - .onConflictDoUpdate({ - target: workosSync.id, - set: { cursor: next, updatedAt: now }, - setWhere: isNull(workosSync.cursor), - }) - .returning({ id: workosSync.id }); - return written.length > 0; - } - const written = await db - .update(workosSync) - .set({ cursor: next, updatedAt: now }) - .where(and(eq(workosSync.id, WORKOS_EVENTS_STREAM_ID), eq(workosSync.cursor, prev))) - .returning({ id: workosSync.id }); - return written.length > 0; - }), + applyPage: (prev, next, pageWrites) => + run("applyPage", () => + db.transaction(async (tx) => { + // The CAS comes FIRST so the lock is held for every write below; + // a run that lost the stream commits an empty transaction. + const owned = await advanceCursor(tx, prev, next); + if (!owned) return Option.none(); + const txWrites = makeWrites(tx); + const outcomes: WorkOsMirrorWriteOutcome[] = []; + for (const write of pageWrites) { + outcomes.push(await applyWrite(txWrites, write)); + } + return Option.some(outcomes); + }), + ), applyOrganizationScan: (scan) => run("applyOrganizationScan", () => @@ -987,6 +1196,30 @@ export const makeWorkOsMirrorStore = (db: DrizzleDb): WorkOsMirrorShape => { return recorded.length > 0; }), + drainedAt: () => + run("drainedAt", async () => { + const rows = await db + .select({ drainedAt: workosSync.drainedAt }) + .from(workosSync) + .where(eq(workosSync.id, WORKOS_EVENTS_STREAM_ID)); + return rows[0]?.drainedAt ?? null; + }), + + markDrained: (at) => + run("markDrained", async () => { + const moved = await db + .update(workosSync) + .set({ drainedAt: at }) + .where( + and( + eq(workosSync.id, WORKOS_EVENTS_STREAM_ID), + or(isNull(workosSync.drainedAt), lt(workosSync.drainedAt, at)), + ), + ) + .returning({ id: workosSync.id }); + return moved.length > 0; + }), + organizationBackfilledAt: (organizationId) => run("organizationBackfilledAt", async () => { const rows = await db diff --git a/apps/cloud/src/auth/workos-mirror.node.test.ts b/apps/cloud/src/auth/workos-mirror.node.test.ts index 25e493385b..f70c208169 100644 --- a/apps/cloud/src/auth/workos-mirror.node.test.ts +++ b/apps/cloud/src/auth/workos-mirror.node.test.ts @@ -26,7 +26,8 @@ // - a delete with no WorkOS instant keeps the row's own WorkOS stamp, so // a replacement membership WorkOS created meanwhile is not refused, // while the removed membership's own payload still is -// - the cursor advances only by compare-and-set (one owner per stream) +// - the cursor advances only by compare-and-set (one owner per stream), +// and a page that loses the CAS writes nothing // - a backfill scan is applied only if its listing is newer than the one // already applied to the organization (one owner per listing instant), // so an older listing cannot insert a membership the newer one lacked; @@ -52,7 +53,12 @@ import { DbService, makeDbLayer } from "../db/db"; import { accounts, organizations } from "../db/schema"; import { cloudMemberDirectoryLayer } from "./member-directory"; import { UserStoreService } from "./context"; -import { WorkOsMirror, type WorkOsMirrorMembership, type WorkOsMirrorUser } from "./workos-mirror"; +import { + WorkOsMirror, + WorkOsMirrorWrite, + type WorkOsMirrorMembership, + type WorkOsMirrorUser, +} from "./workos-mirror"; import { makeWorkOsMirrorStore } from "./workos-mirror-store"; const DbLive = DbService.Live; @@ -776,31 +782,67 @@ describe("WorkOsMirror upserts", () => { }); describe("WorkOsMirror cursor", () => { - it("advances only by compare-and-set", async () => { + it("advances only by compare-and-set, and a page that loses the CAS writes nothing", async () => { const result = await run( Effect.gen(function* () { const mirror = yield* WorkOsMirror; + const directory = yield* MemberDirectory; + const org = yield* freshOrg(); + const id = `user_${crypto.randomUUID()}`; // The cursor is instance-wide; read whatever a previous test left so // this test's expectations are relative, not absolute. const before = yield* mirror.getCursor(); - const first = yield* mirror.setCursor(before, "event_1"); - const wrongPrev = yield* mirror.setCursor(before === null ? "event_0" : null, "event_x"); + const first = yield* mirror.applyPage(before, "event_1", []); + const wrongPrev = yield* mirror.applyPage(before === null ? "event_0" : null, "event_x", [ + WorkOsMirrorWrite.UpsertUser({ user: user(id) }), + WorkOsMirrorWrite.UpsertMembership({ + membership: membership(org, id), + }), + ]); const afterWrong = yield* mirror.getCursor(); - const right = yield* mirror.setCursor("event_1", "event_2"); + const notWritten = yield* directory.membership(id, org); + const right = yield* mirror.applyPage("event_1", "event_2", [ + WorkOsMirrorWrite.UpsertUser({ user: user(id) }), + WorkOsMirrorWrite.UpsertMembership({ + membership: membership(org, id), + }), + // A rename of an org the mirror has never seen: nothing to write. + WorkOsMirrorWrite.RenameOrganization({ + organizationId: "org_nobody", + name: "Nobody", + updatedAt: T1, + }), + ]); const after = yield* mirror.getCursor(); - return { first, wrongPrev, afterWrong, right, after }; + const written = yield* directory.membership(id, org); + return { + id, + org, + first, + wrongPrev, + afterWrong, + notWritten, + right, + after, + written, + }; }), ); - expect(result.first).toBe(true); - expect(result.wrongPrev, "a run holding a stale prev cannot move the cursor").toBe(false); + expect(Option.isSome(result.first)).toBe(true); + expect( + Option.isNone(result.wrongPrev), + "a run holding a stale prev cannot move the cursor", + ).toBe(true); expect(result.afterWrong).toBe("event_1"); - expect(result.right).toBe(true); + expect(result.notWritten, "and none of its page's writes land").toBeNull(); + expect(result.right).toEqual(Option.some(["applied", "applied", "absent"])); expect(result.after).toBe("event_2"); + expect(result.written?.membershipId).toBe(`om_${result.id}_${result.org}`); }); }); describe("WorkOsMirror backfill sync state", () => { - it("records the replay boundary and the backfill completion once each, without touching the cursor", async () => { + it("records the replay boundary and the backfill completion once each, and the drained mark forward only, without touching the cursor", async () => { const result = await run( Effect.gen(function* () { const mirror = yield* WorkOsMirror; @@ -808,6 +850,8 @@ describe("WorkOsMirror backfill sync state", () => { // empty test database, other tests may have written it): start from // no row, as a database that has never been backfilled has. yield* clearEventsRow; + // No events row yet: nothing has been drained, and nothing is minted. + const drainedWithoutRow = yield* mirror.markDrained(T1); const first = yield* mirror.setReplayBoundary(T2); const boundary = yield* mirror.replayBoundary(); const cursorAfterBoundary = yield* mirror.getCursor(); @@ -816,7 +860,7 @@ describe("WorkOsMirror backfill sync state", () => { const afterAgain = yield* mirror.replayBoundary(); // Nor once the stream is being followed. const cursorBefore = yield* mirror.getCursor(); - yield* mirror.setCursor(cursorBefore, "event_boundary"); + yield* mirror.applyPage(cursorBefore, "event_boundary", []); const afterCursor = yield* mirror.setReplayBoundary(T1); const boundaryWithCursor = yield* mirror.replayBoundary(); const cursor = yield* mirror.getCursor(); @@ -828,7 +872,19 @@ describe("WorkOsMirror backfill sync state", () => { const completedAt = yield* mirror.backfillCompletedAt(); const boundaryAfterCompletion = yield* mirror.replayBoundary(); const cursorAfterCompletion = yield* mirror.getCursor(); + // The drained mark moves forward only, on the row the stream owns. + const notDrained = yield* mirror.drainedAt(); + const drainedFirst = yield* mirror.markDrained(T3); + const drainedBackwards = yield* mirror.markDrained(T2); + const drainedForward = yield* mirror.markDrained(T4); + const drainedAt = yield* mirror.drainedAt(); return { + drainedWithoutRow, + notDrained, + drainedFirst, + drainedBackwards, + drainedForward, + drainedAt, first, boundary, cursorAfterBoundary, @@ -860,6 +916,14 @@ describe("WorkOsMirror backfill sync state", () => { expect(result.completedAt, "the first stands").toEqual(T3); expect(result.boundaryAfterCompletion, "the boundary is untouched").toEqual(T2); expect(result.cursorAfterCompletion, "and so is the cursor").toBe("event_boundary"); + expect(result.drainedWithoutRow, "no row, nothing drained: nothing written").toBe(false); + expect(result.notDrained, "no drain recorded until a run drains").toBeNull(); + expect(result.drainedFirst).toBe(true); + expect(result.drainedBackwards, "an earlier run finishing later cannot move it back").toBe( + false, + ); + expect(result.drainedForward).toBe(true); + expect(result.drainedAt).toEqual(T4); }); it("refuses a membership payload stamped before the organization's last scan, and accepts one stamped at or after it", async () => { diff --git a/apps/cloud/src/auth/workos-mirror.ts b/apps/cloud/src/auth/workos-mirror.ts index 6898facdb5..70ea8bf366 100644 --- a/apps/cloud/src/auth/workos-mirror.ts +++ b/apps/cloud/src/auth/workos-mirror.ts @@ -23,6 +23,7 @@ import { makeWorkOsMirrorStore, type WorkOsMirrorShape } from "./workos-mirror-s export { WorkOsMirrorError } from "./errors"; export { + WorkOsMirrorWrite, mirrorMembershipFromWorkOs, mirrorUserFromWorkOs, type WorkOsMembershipPayload, @@ -30,6 +31,7 @@ export { type WorkOsMirrorMembershipRef, type WorkOsMirrorShape, type WorkOsMirrorUser, + type WorkOsMirrorWriteOutcome, type WorkOsOrganizationScan, type WorkOsOrganizationScanWrites, type WorkOsScannedMember, diff --git a/apps/cloud/src/auth/workos-webhook.ts b/apps/cloud/src/auth/workos-webhook.ts new file mode 100644 index 0000000000..5309cdc581 --- /dev/null +++ b/apps/cloud/src/auth/workos-webhook.ts @@ -0,0 +1,94 @@ +// --------------------------------------------------------------------------- +// `POST /api/webhooks/workos` — the WorkOS webhook endpoint, which only +// POKES the reconciler. It verifies the delivery's signature and, when it +// is genuine, starts one `syncWorkOsEvents` pass past the response. It +// never applies the webhook's own payload: webhooks are unordered and +// at-least-once, while the Events API the reconciler reads is ordered and +// replayable from the persisted cursor. The webhook's only job is to turn +// "within a minute" (the cron) into "within seconds" for dashboard-side +// changes such as a revoked membership. +// +// Unauthenticated by design (WorkOS holds no session); the signature IS the +// authentication. Nothing about the payload is reflected in the response. +// --------------------------------------------------------------------------- + +import { Effect, Option, Schema } from "effect"; +import { Headers, HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http"; + +import { WorkOSClient } from "./workos"; + +export const WORKOS_WEBHOOK_PATH = "/api/webhooks/workos"; + +const SIGNATURE_HEADER = "workos-signature"; + +// The SDK verifies the signature over `JSON.stringify(payload)`, so the body +// must be a JSON object; an array or scalar can never be a WorkOS delivery. +const WebhookPayload = Schema.Record(Schema.String, Schema.Unknown); +const decodeWebhookPayload = Schema.decodeUnknownOption(WebhookPayload); + +export interface WorkOsWebhookDeps { + /** + * The endpoint's signing secret (`WORKOS_WEBHOOK_SECRET`). `undefined` + * when the deployment has not configured one: every delivery is then + * refused with 503, never accepted unverified. + */ + readonly secret: string | undefined; + /** + * Hand the reconciler pass to the platform so it outlives the response + * (`waitUntil` from `cloudflare:workers`). The promise never rejects: the + * runner reports its own failures. + */ + readonly detach: (work: Promise) => void; + /** One reconciler pass over fresh services (`runWorkOsEventsSync`). */ + readonly sync: () => Promise; +} + +/** + * The webhook route. 200 for a verified delivery (a sync pass has been + * detached), 400 for a missing or invalid signature or a body that is not a + * JSON object, 503 when no signing secret is configured. + */ +export const makeWorkOsWebhookRoute = (deps: WorkOsWebhookDeps) => + HttpRouter.add( + "POST", + WORKOS_WEBHOOK_PATH, + Effect.gen(function* () { + if (deps.secret === undefined) { + yield* Effect.logError( + "workos_webhook: WORKOS_WEBHOOK_SECRET is not set; refusing the delivery", + ); + return HttpServerResponse.empty({ status: 503 }); + } + const secret = deps.secret; + const request = yield* HttpServerRequest.HttpServerRequest; + const sigHeader = Headers.get(request.headers, SIGNATURE_HEADER); + if (Option.isNone(sigHeader)) { + return HttpServerResponse.empty({ status: 400 }); + } + const body = yield* request.json.pipe(Effect.option); + const payload = Option.flatMap(body, decodeWebhookPayload); + if (Option.isNone(payload)) { + return HttpServerResponse.empty({ status: 400 }); + } + + const workos = yield* WorkOSClient; + const verified = yield* workos + .constructWebhookEvent({ + payload: payload.value, + sigHeader: sigHeader.value, + secret, + }) + .pipe(Effect.option); + if (Option.isNone(verified)) { + yield* Effect.logWarning("workos_webhook: signature rejected"); + return HttpServerResponse.empty({ status: 400 }); + } + + yield* Effect.logInfo("workos_webhook: verified delivery; poking the reconciler", { + event: verified.value.event, + eventId: verified.value.id, + }); + deps.detach(deps.sync()); + return HttpServerResponse.empty({ status: 200 }); + }), + ); diff --git a/apps/cloud/src/auth/workos.ts b/apps/cloud/src/auth/workos.ts index 821ea90278..dcaaa723da 100644 --- a/apps/cloud/src/auth/workos.ts +++ b/apps/cloud/src/auth/workos.ts @@ -4,7 +4,12 @@ import { env } from "cloudflare:workers"; import { Context, Data, Effect, Layer, Option, Predicate, Schema } from "effect"; -import { GeneratePortalLinkIntent, WorkOS } from "@workos-inc/node/worker"; +import { + GeneratePortalLinkIntent, + WorkOS, + type Event as WorkOSEvent, + type EventName as WorkOSEventName, +} from "@workos-inc/node/worker"; import { defaults as ironDefaults, unseal as unsealIron } from "iron-webcrypto"; import { decodeJwt, jwtVerify } from "jose"; import { workosAccessTokenOptions } from "./access-token-options"; @@ -16,6 +21,7 @@ import { tryPromiseService, withServiceLogging, workosErrorFromFailure, + type WorkOSError, } from "./errors"; const COOKIE_NAME = "wos-session"; @@ -48,6 +54,20 @@ type WorkOSAutoPaginatable = { readonly autoPagination: () => Promise; }; +/** + * One read of the WorkOS Events API stream. `events` names the types to + * return; `after` resumes from an event id (exclusive), `rangeStart` (ISO) + * bounds a first read that has no cursor yet. Mirrors the SDK's + * `ListEventOptions` with readonly inputs. + */ +export type WorkOSListEventsOptions = { + readonly events: readonly WorkOSEventName[]; + readonly after?: string; + readonly rangeStart?: string; + readonly limit?: number; + readonly order?: "asc" | "desc"; +}; + export type WorkOSCollectedList = { readonly object: "list"; readonly data: Resource[]; @@ -753,6 +773,48 @@ const make = Effect.gen(function* () { wos.organizations.listOrganizationRoles({ organizationId }), ), + /** + * One page of the Events API stream, oldest first when `order` is `asc`. + * The reconciler (`workos-events-sync.ts`) is the only consumer: it pages + * by `after` = the last event id it applied, so the stream is replayable + * from the persisted cursor. Returns the SDK page as-is (`data` + + * `listMetadata.after`); paging is the caller's loop, not + * `collectWorkOSList`, because each page is committed before the next is + * read. + */ + listEvents: (options: WorkOSListEventsOptions) => + use("events.listEvents", (wos) => + wos.events.listEvents({ + events: [...options.events], + ...(options.after === undefined ? {} : { after: options.after }), + ...(options.rangeStart === undefined ? {} : { rangeStart: options.rangeStart }), + ...(options.limit === undefined ? {} : { limit: options.limit }), + ...(options.order === undefined ? {} : { order: options.order }), + }), + ), + + /** + * Verify a webhook delivery against `secret` (the endpoint's signing + * secret from the WorkOS dashboard) and decode its event. A local HMAC + * check, no network: it fails with a status-less `WorkOSError` when the + * `WorkOS-Signature` header is missing its parts, older than the SDK's + * tolerance, or does not match `payload`. The decoded event is returned + * for the caller to inspect; the webhook route deliberately does NOT + * apply it (the Events API is the only source the mirror replays from). + */ + constructWebhookEvent: (params: { + readonly payload: Record; + readonly sigHeader: string; + readonly secret: string; + }): Effect.Effect => + use("webhooks.constructEvent", (wos) => + wos.webhooks.constructEvent({ + payload: params.payload, + sigHeader: params.sigHeader, + secret: params.secret, + }), + ), + /** Get an organization (includes domains). */ getOrganization: (organizationId: string) => use("organizations.getOrganization", (wos) => diff --git a/apps/cloud/src/db/schema.ts b/apps/cloud/src/db/schema.ts index 9bbd73290b..89e5c3cdb9 100644 --- a/apps/cloud/src/db/schema.ts +++ b/apps/cloud/src/db/schema.ts @@ -83,13 +83,14 @@ export const organizations = pgTable( backfilledAt: timestamp("backfilled_at", { withTimezone: true }), /** * When this organization was deleted, or null while it is live. Set by - * cloud's own deletion flow and by the `organization.deleted` event, and - * KEPT by the local purge (`db/org-deletion.ts`), which removes the - * organization's memberships and tenant data but leaves this row as a - * tombstone: a feeder that fetched a membership before the deletion and - * writes it after (a login that stalled across the purge) finds the - * tombstone and does not re-mint the organization live. A marked - * organization is never renamed and authorizes nobody. + * cloud's own deletion flow and by the `organization.deleted` event — + * which MINTS the row as a tombstone when the mirror has never seen the + * organization — and KEPT by the local purge (`db/org-deletion.ts`), + * which removes the organization's memberships and tenant data but + * leaves this row as a tombstone: a feeder that fetched a membership + * before the deletion and writes it after (a login that stalled across + * the deletion) finds the tombstone and does not mint the organization + * live. A marked organization is never renamed and authorizes nobody. */ deletedAt: timestamp("deleted_at", { withTimezone: true }), /** @@ -201,16 +202,16 @@ export const membershipTombstones = pgTable( * stops. * * `range_start` on the `"events"` row is the REPLAY BOUNDARY: the instant the - * FIRST completed one-off backfill (`scripts/backfill-workos-mirror.ts`) began - * reading WorkOS. Everything before it is covered by that backfill; the - * reconciler's first run (no cursor yet) reads the events stream from here, - * so a revocation between the backfill and the first run is never skipped. - * Written once: a backfill that fails part-way records nothing, and a later - * completed one keeps it, because the backfill does not refresh everything - * the events stream carries (organization renames, deleted users' - * profiles) — those between two runs are replayed from the first boundary. - * Without a cursor or a boundary the reconciler does not guess; it waits for - * the backfill. + * FIRST one-off backfill run (`scripts/backfill-workos-mirror.ts`) began + * reading WorkOS, recorded BEFORE its first listing. Everything before it is + * covered by that backfill; the reconciler's first run (no cursor yet) reads + * the events stream from here, so a revocation between the backfill and the + * first run is never skipped. Written once: a run that fails part-way leaves + * it standing and its retry keeps it, and a later run keeps it too, because + * the backfill does not refresh everything the events stream carries + * (organization renames, deleted users' profiles) — those after the first + * boundary are replayed from it. Without a cursor or a boundary the + * reconciler does not guess; it waits for the backfill. * * `backfill_completed_at` is when a backfill run first wrote EVERY live * organization (`scripts/backfill-workos-mirror.ts` completing, or refusing @@ -223,13 +224,23 @@ export const membershipTombstones = pgTable( * completeness for the seat gates is tracked separately * (`organizations.backfilled_at`). * + * `drained_at` is when a reconciler run last read the events stream to its + * END (an empty page, or a page with nothing after it) — the second half of + * the readiness mark: a mirror whose reconciler has not caught up recently + * may still grant a member WorkOS already revoked, so the authorization + * path trusts the mirror only while this is within its lag budget. Moved + * forward by every draining run; never cleared. A run that stops at its + * page budget or yields to another run leaves it as it was. + * * Migration 0019 seeds the boundary and the completion mark on a database - * with no organizations, where there is nothing to backfill. + * with no organizations, where there is nothing to backfill; `drained_at` + * is left for the reconciler's first run to set (migration 0020). */ export const workosSync = pgTable("workos_sync", { id: text("id").primaryKey(), cursor: text("cursor"), rangeStart: timestamp("range_start", { withTimezone: true }), backfillCompletedAt: timestamp("backfill_completed_at", { withTimezone: true }), + drainedAt: timestamp("drained_at", { withTimezone: true }), updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow(), }); diff --git a/apps/cloud/src/env-augment.d.ts b/apps/cloud/src/env-augment.d.ts index 773ca4d468..715991f394 100644 --- a/apps/cloud/src/env-augment.d.ts +++ b/apps/cloud/src/env-augment.d.ts @@ -111,6 +111,14 @@ declare global { /** Optional WorkOS base-URL override (WorkOS emulator in tests/dev). */ WORKOS_API_URL?: string; + /** + * Signing secret of the WorkOS webhook endpoint that pokes the + * membership-mirror reconciler (`/api/webhooks/workos`). Set with + * `wrangler secret put WORKOS_WEBHOOK_SECRET`; while unset the route + * refuses every delivery (503) and the every-minute cron alone keeps + * the mirror current. + */ + WORKOS_WEBHOOK_SECRET?: string; // MCP EXECUTOR_MCP_DEBUG?: string; diff --git a/apps/cloud/src/extensions/routes.ts b/apps/cloud/src/extensions/routes.ts index e362b62d5a..bd4b1d4c12 100644 --- a/apps/cloud/src/extensions/routes.ts +++ b/apps/cloud/src/extensions/routes.ts @@ -9,6 +9,8 @@ // - Swagger UI + the OpenAPI JSON for the full cloud spec. // - the Autumn billing proxy (`/api/billing/*`) — billing-as-extension (the // `extensions.routes` SEAM, but served under `/api` like everything else). +// - the WorkOS webhook (`/api/webhooks/workos`) — signature-verified poke of +// the membership-mirror reconciler. // - the global request-failure logging middleware. // // They all serve UNDER the `/api` prefix (the same namespace the protected + @@ -19,6 +21,7 @@ // so the postgres.js socket lives in the request fiber's scope). // --------------------------------------------------------------------------- +import { env, waitUntil } from "cloudflare:workers"; import { Effect, Layer } from "effect"; import { HttpRouter, HttpServerResponse } from "effect/unstable/http"; import { HttpApiBuilder } from "effect/unstable/httpapi"; @@ -36,6 +39,8 @@ import { } from "../auth/handlers"; import { CloudAuthApi, CloudAuthPublicApi } from "../auth/api"; import { SessionAuthLive } from "../auth/middleware-live"; +import { runWorkOsEventsSync } from "../auth/workos-events-runner"; +import { makeWorkOsWebhookRoute } from "../auth/workos-webhook"; import { makeCloudAdminUsersRoutes } from "../admin/admin-users-api"; import { OrgApi, OrgHttpApi } from "../org/api"; import { orgAuthMiddleware } from "../org/auth-middleware"; @@ -113,12 +118,23 @@ export const makeCloudExtensionRoutes = ( // org key (or an admin session) and builds a subject-less platform view. const AdminUsersRoutes = makeCloudAdminUsersRoutes(rsLive, { router: apiPrefixedRouter }); + // The WorkOS webhook needs no per-request DB layer: it verifies the + // signature with the boot `WorkOSClient` and detaches a reconciler pass + // that builds its own fresh services (the route's request scope is gone by + // the time the pass runs). `waitUntil` binds to the in-flight invocation. + const WebhookRoutes = makeWorkOsWebhookRoute({ + secret: env.WORKOS_WEBHOOK_SECRET, + detach: waitUntil, + sync: runWorkOsEventsSync, + }); + return [ SessionRoutes, OrgRoutes, AdminUsersRoutes, DocsRoutes, BillingRoutes, + WebhookRoutes, ApiErrorLoggingLive, ] as const; }; diff --git a/apps/cloud/src/server.ts b/apps/cloud/src/server.ts index 4f5bf76289..fc9c146f3a 100644 --- a/apps/cloud/src/server.ts +++ b/apps/cloud/src/server.ts @@ -13,6 +13,7 @@ import handler from "@tanstack/react-start/server-entry"; import { isAppOwnedPath, servedByAppPlane } from "./app-paths"; import { marketingProxyRequest } from "./edge/marketing"; import { passthroughResponse } from "./edge/passthrough"; +import { runWorkOsEventsSync } from "./auth/workos-events-runner"; import { makeCloudMcpAgentHandler } from "./mcp/agent-handler"; import { classifyMcpPath, prepareMcpOrgScope } from "./mcp/mount"; import { parseTraceparent } from "./mcp/traceparent"; @@ -433,6 +434,20 @@ const cloudflareHandler: ExportedHandler = { }, ); }, + + // Cron: the membership-mirror reconciler (wrangler.jsonc `triggers.crons`, + // every minute). One pass over the WorkOS Events API from the persisted + // cursor, on fresh request-scoped services. `Sentry.withSentry` instruments + // `scheduled` alongside `fetch` (`instrumentExportedHandlerScheduled`), so + // a failing pass reports like a failing request. The tracer is installed + // here as on the fetch path — a scheduled invocation may be the isolate's + // first — and flushed past the pass so the run's spans export before the + // isolate goes idle. + scheduled: async (_controller, _env, ctx) => { + installTracerProvider(); + await runWorkOsEventsSync(); + ctx.waitUntil(flushTracerProvider()); + }, }; export default Sentry.withSentry(cloudSentryOptions, cloudflareHandler); diff --git a/apps/cloud/wrangler.jsonc b/apps/cloud/wrangler.jsonc index 3578de6d25..6d92064589 100644 --- a/apps/cloud/wrangler.jsonc +++ b/apps/cloud/wrangler.jsonc @@ -18,6 +18,13 @@ "limits": { "cpu_ms": 30000, }, + // Every minute: replay the WorkOS Events API into the membership mirror + // (`scheduled` in src/server.ts → auth/workos-events-sync.ts). Changes made + // in the WorkOS dashboard reach the mirror within this interval; the + // signed webhook at /api/webhooks/workos shortens it to seconds. + "triggers": { + "crons": ["* * * * *"], + }, "observability": { "enabled": true, }, From 787569e0a871c663a2d79f63bb7937cf1d618ee3 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Mon, 14 Sep 2026 22:33:04 -0700 Subject: [PATCH 2/2] Read members from the directory and add email search --- .changeset/member-directory-readers.md | 10 + apps/cloud/src/account/account-api.ts | 8 +- .../account/org-api-key-revoke.node.test.ts | 11 +- .../src/account/workos-account-service.ts | 100 +++-- apps/cloud/src/admin/admin-users-api.ts | 166 ++------ .../src/admin/admin-users-email.node.test.ts | 127 ------ apps/cloud/src/api/layers.ts | 16 +- apps/cloud/src/api/router.ts | 8 +- .../src/auth/mirror-feeders.node.test.ts | 371 +++++++++++++++++- apps/cloud/src/auth/mirror-feeders.ts | 112 +++++- .../auth/workos-callback-state.node.test.ts | 32 +- apps/cloud/src/auth/workos.ts | 26 +- .../src/extensions/billing/member-seats.ts | 78 ++-- apps/cloud/src/extensions/routes.ts | 4 +- .../account/better-auth-account-provider.ts | 2 +- .../src/admin/admin-users-api.ts | 97 +---- apps/host-selfhost/src/app.ts | 13 +- apps/host-selfhost/src/auth/index.ts | 15 +- packages/core/api/src/account/api.ts | 9 +- .../core/api/src/admin/admin-users.test.ts | 163 +++++++- packages/core/api/src/admin/api.ts | 10 + packages/core/api/src/admin/handlers.ts | 19 +- .../core/api/src/admin/member-directory.ts | 23 +- packages/core/api/src/admin/reads.ts | 112 +++++- packages/core/api/src/admin/service.ts | 7 +- packages/core/api/src/server.ts | 1 + packages/core/sdk/src/executor.ts | 18 + packages/core/sdk/src/platform-view.test.ts | 55 +++ packages/react/src/api/admin-atoms.tsx | 19 +- packages/react/src/pages/admin-users.tsx | 119 +++++- packages/react/src/pages/org.tsx | 36 +- 31 files changed, 1278 insertions(+), 509 deletions(-) create mode 100644 .changeset/member-directory-readers.md delete mode 100644 apps/cloud/src/admin/admin-users-email.node.test.ts diff --git a/.changeset/member-directory-readers.md b/.changeset/member-directory-readers.md new file mode 100644 index 0000000000..6cc71965cc --- /dev/null +++ b/.changeset/member-directory-readers.md @@ -0,0 +1,10 @@ +--- +"@executor-js/cloud": patch +"@executor-js/api": patch +"@executor-js/react": patch +"@executor-js/sdk": patch +--- + +Member lists, the admin users page, and seat counts on cloud now read from the local membership mirror through the shared `MemberDirectory` seam instead of fanning out one WorkOS read per member. The admin users page gains an email/name search. + +**Deploy prerequisite (cloud):** `bun run --cwd apps/cloud db:backfill-workos-mirror:prod` must complete before this build is deployed, and its printed membership count should match WorkOS. Until the backfill has stamped the mirror's marker, seat reporting to Autumn is skipped with a warning (never a partial count) and member lists show only members who have signed in since the mirror shipped. diff --git a/apps/cloud/src/account/account-api.ts b/apps/cloud/src/account/account-api.ts index 1e8075c866..a8b631f170 100644 --- a/apps/cloud/src/account/account-api.ts +++ b/apps/cloud/src/account/account-api.ts @@ -5,6 +5,7 @@ import { AccountProvider, makeAccountApiLayer, requestScopedMiddleware, + type MemberDirectory, } from "@executor-js/api/server"; import { ApiKeyService } from "../auth/api-keys"; @@ -46,7 +47,8 @@ import { AccountCaller, workosAccountProvider } from "./workos-account-service"; // Builds the WorkOS `AccountProvider` per request, providing it to the handler. // Long-lived `WorkOSClient | AutumnService` come from the surrounding context // (Autumn provided by `makeAccountApiLive` for the seat-gate); the per-request -// `UserStoreService` is supplied by the combined `rsLive` layer. +// `UserStoreService` / `WorkOsMirror` / `MemberDirectory` are supplied by the +// combined `rsLive` layer. // `ApiKeyService.WorkOS` is built here on top of the boot `WorkOSClient`. const AccountProviderMiddleware = HttpRouter.middleware<{ provides: AccountProvider }>()( Effect.gen(function* () { @@ -97,11 +99,11 @@ const AccountProviderMiddleware = HttpRouter.middleware<{ provides: AccountProvi * (the seat-gate) stays a residual requirement, satisfied by the app `boot`. */ export const workosAccountMiddleware = ( - rsLive: Layer.Layer, + rsLive: Layer.Layer, ) => AccountProviderMiddleware.combine(requestScopedMiddleware(rsLive)).layer; export const makeAccountApiLive = ( - rsLive: Layer.Layer, + rsLive: Layer.Layer, ) => { // Cloud builds the WorkOS `AccountProvider` INSIDE the request body (so it // closes over the per-request postgres socket), so it can't be a self- diff --git a/apps/cloud/src/account/org-api-key-revoke.node.test.ts b/apps/cloud/src/account/org-api-key-revoke.node.test.ts index 10353ccecd..db322ba1bb 100644 --- a/apps/cloud/src/account/org-api-key-revoke.node.test.ts +++ b/apps/cloud/src/account/org-api-key-revoke.node.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from "@effect/vitest"; import { Effect, Layer } from "effect"; -import { AccountProvider } from "@executor-js/api/server"; +import { AccountProvider, MemberDirectory } from "@executor-js/api/server"; import { AccountError, AccountForbidden } from "@executor-js/api"; import { ApiKeyService, OrgApiKeyNotFound } from "../auth/api-keys"; @@ -147,6 +147,14 @@ const stubMirror = Layer.succeed(WorkOsMirror)({ organizationBackfilledAt: () => Effect.die("revoke does not report seats"), }); +// Revoke lists no members either. +const stubDirectory = Layer.succeed(MemberDirectory)({ + membership: () => Effect.die("revoke does not read the member directory"), + members: () => Effect.die("revoke does not read the member directory"), + membersById: () => Effect.die("revoke does not read the member directory"), + findByEmail: () => Effect.die("revoke does not read the member directory"), +}); + const stubAutumn = Layer.succeed(AutumnService)({ use: () => Effect.die("revoke does not touch billing"), ensureCustomer: () => Effect.die("revoke does not touch billing"), @@ -185,6 +193,7 @@ const providerWith = (accountId: string) => { stubWorkOS, stubUsers, stubMirror, + stubDirectory, stubApiKeys, stubAutumn, Layer.succeed(AccountCaller)({ session: session(accountId) }), diff --git a/apps/cloud/src/account/workos-account-service.ts b/apps/cloud/src/account/workos-account-service.ts index beb9ef86a3..5fadfb938f 100644 --- a/apps/cloud/src/account/workos-account-service.ts +++ b/apps/cloud/src/account/workos-account-service.ts @@ -1,6 +1,6 @@ import { Context, Effect, Layer } from "effect"; -import { AccountProvider, type AccountHeaders } from "@executor-js/api/server"; +import { AccountProvider, MemberDirectory, type AccountHeaders } from "@executor-js/api/server"; import { AccountError, AccountForbidden, @@ -12,6 +12,7 @@ import { ApiKeyService } from "../auth/api-keys"; import { UserStoreService } from "../auth/context"; import type { Session } from "../auth/middleware"; import { WorkOSClient } from "../auth/workos"; +import { ensureOrganizationBackfilled, mirrorInvitedMember } from "../auth/mirror-feeders"; import { WorkOsMirror, mirrorMembershipFromWorkOs } from "../auth/workos-mirror"; import { ORG_SELECTOR_HEADER, authorizeOrganizationSelector } from "../auth/organization"; import { AutumnService } from "../extensions/billing/service"; @@ -66,7 +67,13 @@ const toAccountError = () => Effect.fail(new AccountError({ message: "Account re export const workosAccountProvider: Layer.Layer< AccountProvider, never, - WorkOSClient | UserStoreService | WorkOsMirror | ApiKeyService | AutumnService | AccountCaller + | WorkOSClient + | UserStoreService + | WorkOsMirror + | MemberDirectory + | ApiKeyService + | AutumnService + | AccountCaller > = Layer.effect(AccountProvider)( Effect.gen(function* () { const workos = yield* WorkOSClient; @@ -77,6 +84,10 @@ export const workosAccountProvider: Layer.Layer< // written through to the local mirror so the member list and the seat // count read the change without waiting for the Events reconciler. const mirror = yield* WorkOsMirror; + // Membership READS come from the mirror through the shared directory: the + // member list and the seat count are one local query each, never a + // WorkOS read per member. + const directory = yield* MemberDirectory; // The caller, resolved once per request by the cookie-only session // middleware (account-api.ts) — the same credential `SessionAuthLive` @@ -85,10 +96,12 @@ export const workosAccountProvider: Layer.Layer< const caller = yield* AccountCaller; // Capture the resolved service context once so the method bodies — which - // call `authorizeOrganization` (yields `WorkOSClient` + `UserStoreService`) — - // can be erased to `R = never`, as the neutral AccountProvider shape - // requires. Provided per method below. - const ctx = yield* Effect.context(); + // call `authorizeOrganization` (yields `WorkOSClient` + `UserStoreService`), + // the mirror feeders, and the seat reporter — can be erased to `R = never`, + // as the neutral AccountProvider shape requires. Provided per method below. + const ctx = yield* Effect.context< + WorkOSClient | UserStoreService | AutumnService | MemberDirectory | WorkOsMirror + >(); // Unauthenticated (missing/invalid session) => AccountUnauthorized, exactly // as the old inline `requireSession` did. @@ -145,7 +158,14 @@ export const workosAccountProvider: Layer.Layer< return membership; }); - // Mirror of org/handlers `getMemberSeats` — live seat usage from WorkOS. + // Seat usage: memberships from the local directory (active + pending, the + // `members` default), pending invitations live from WorkOS — invitations + // are not mirrored. The directory is trusted for a COUNT only once this + // organization's membership list has been scanned from WorkOS in full: + // login and write-through record single memberships, so an organization + // the one-off backfill did not cover holds a partial list, and counting + // it would admit invitations past the plan limit. The scan runs here, + // once, when the organization's mark is missing. const getMemberSeats = (organizationId: string) => Effect.gen(function* () { const customer = yield* autumn.use((client) => @@ -154,15 +174,16 @@ export const workosAccountProvider: Layer.Layer< const planId = selectActiveMemberLimitPlan(customer.subscriptions); const limit = getMemberLimitForPlan(planId); - // `listOrgMembers` returns active members AND pending memberships (an - // invited user shows up as status "pending"); `listPendingInvitations` + yield* ensureOrganizationBackfilled(organizationId).pipe(Effect.provideContext(ctx)); + // The directory reports active members AND pending memberships (an + // invited user is mirrored with status "pending"); `listPendingInvitations` // returns the same invited users again. `countSeatsUsed` dedupes them // so an outstanding invite is not counted twice. - const memberships = yield* workos.listOrgMembers(organizationId); + const memberships = yield* directory.members(organizationId); const invitations = yield* workos.listPendingInvitations(organizationId); return { - used: countSeatsUsed(memberships.data, invitations.data.length), + used: countSeatsUsed(memberships, invitations.data.length), granted: limit ?? 0, unlimited: limit === null, }; @@ -328,29 +349,23 @@ export const workosAccountProvider: Layer.Layer< Effect.catchCause(() => Effect.succeed({ used: 0, granted: 0, unlimited: false })), ); - const memberships = yield* workos - .listOrgMembers(org.id) - .pipe(Effect.catchTag("WorkOSError", toAccountError)); - - const members = yield* Effect.all( - memberships.data.map((m) => - Effect.gen(function* () { - const user = yield* workos.getUser(m.userId); - return { - id: m.id, - userId: m.userId, - email: user.email, - name: [user.firstName, user.lastName].filter(Boolean).join(" ") || null, - avatarUrl: user.profilePictureUrl ?? null, - role: m.role?.slug ?? "member", - status: m.status, - lastActiveAt: user.lastSignInAt ?? null, - isCurrentUser: m.userId === session.accountId, - }; - }), - ), - { concurrency: 5 }, - ).pipe(Effect.catchTag("WorkOSError", toAccountError)); + // One directory read (active + pending, ordered by email) with the + // profile already joined — no per-member WorkOS user fetch. + const directoryMembers = yield* directory + .members(org.id) + .pipe(Effect.catchTag("MemberDirectoryError", toAccountError)); + + const members = directoryMembers.map((m) => ({ + id: m.membershipId, + userId: m.accountId, + email: m.email, + name: m.name, + avatarUrl: m.avatarUrl, + role: m.role, + status: m.status, + lastActiveAt: m.lastActiveAt === null ? null : new Date(m.lastActiveAt).toISOString(), + isCurrentUser: m.accountId === session.accountId, + })); return { members, seats }; }), @@ -378,6 +393,23 @@ export const workosAccountProvider: Layer.Layer< ...(body.roleSlug ? { roleSlug: body.roleSlug } : {}), }) .pipe(Effect.catchTag("WorkOSError", toAccountError)); + // Write-through: WorkOS creates a PENDING membership for the invitee + // alongside the invitation, and the member list (the "Invited" row + // and its revoke button) reads memberships from the mirror only, so + // the row must land now — the Events reconciler is not on this path. + const mirrored = yield* mirrorInvitedMember(org.id, invitation.email).pipe( + Effect.provideContext(ctx), + Effect.catchTags({ + WorkOSError: toAccountError, + WorkOsMirrorError: toAccountError, + }), + ); + if (!mirrored) { + yield* Effect.logWarning("inviteMember: no pending membership for the invitee yet", { + organizationId: org.id, + invitationId: invitation.id, + }); + } return { id: invitation.id, email: invitation.email }; }), diff --git a/apps/cloud/src/admin/admin-users-api.ts b/apps/cloud/src/admin/admin-users-api.ts index cae66fcb5f..00c9a0e4d9 100644 --- a/apps/cloud/src/admin/admin-users-api.ts +++ b/apps/cloud/src/admin/admin-users-api.ts @@ -25,28 +25,24 @@ // every query by that tenant. // --------------------------------------------------------------------------- -import { env } from "cloudflare:workers"; import { HttpRouter } from "effect/unstable/http"; -import { Context, Effect, Layer, Option } from "effect"; +import { Effect, Layer } from "effect"; import { AdminUsersProvider, DbProvider, HostConfig, + MemberDirectory, PluginsProvider, + adminUserDirectoryFromMembers, getAdminUser, listAdminUserConnections, listAdminUsers, listAdminUsersWithConnections, makeAdminUsersApiLayer, makePlatformExecutor, - normalizeAdminUserEmail, platformViewOf, requestScopedMiddleware, - type AdminEmailResolver, - type AdminIdentityDirectory, - type AdminUserDirectory, - type AdminUserIdentity, type AdminUsersHeaders, } from "@executor-js/api/server"; import { @@ -122,124 +118,6 @@ const authorizeTenant = ( return org.id; }); -/** - * How many user-detail reads run at once. Matches the account plane's own - * member listing (`workos-account-service.ts`), which fans out the same way for - * the same reason. - */ -const IDENTITY_CONCURRENCY = 5; - -/** - * Cloud's member directory: `externalId` → email/name. - * - * THE JOIN KEY is the membership's `userId` — the WorkOS `user_...` that - * `workos-auth-provider.ts` binds as `accountId` on every credential path, and - * therefore what the subject table records in `external_id`. The membership's - * own `id` is an `om_...` row id and joins to nothing. - * - * WHY THIS IS TWO CALLS AND NOT ONE. The membership list is read once per - * request and is the authority on who belongs to the org, but WorkOS's - * `listOrganizationMemberships` carries no user detail and offers no - * include/expand — email and name only exist on the user resource. The SDK does - * expose a batched `listUsers({ organizationId })`, but the pinned - * `@executor-js/emulate` WorkOS emulator serves only `GET - * /user_management/users/:id`, so taking that path would leave every cloud e2e - * user unnamed. So: ONE membership read per request, then user detail fetched - * only for the ids ON THIS PAGE — never for the whole org, and never once per - * row of some larger list. An id that is not an active/pending member is not - * fetched at all and reports absent identity, which is the honest answer for a - * member who left while their connections remain. - */ -const identityDirectory = - (organizationId: string, context: Context.Context): AdminIdentityDirectory => - (externalIds) => - Effect.gen(function* () { - const workos = yield* WorkOSClient; - const memberships = yield* workos.listOrgMembers(organizationId); - const wanted = new Set(externalIds); - const memberIds = memberships.data - .map((membership) => membership.userId) - .filter((userId) => wanted.has(userId)); - - const resolved = yield* Effect.all( - memberIds.map((userId) => - workos.getUser(userId).pipe( - Effect.map( - (user) => - [ - userId, - { - email: user.email, - displayName: [user.firstName, user.lastName].filter(Boolean).join(" ") || null, - }, - ] as const, - ), - // One unreadable user must not cost the whole page its names. - Effect.catchCause(() => Effect.succeed(null)), - ), - ), - { concurrency: IDENTITY_CONCURRENCY }, - ); - - const identities = new Map(); - for (const entry of resolved) if (entry) identities.set(entry[0], entry[1]); - return identities; - }).pipe(Effect.provideContext(context)); - -/** - * Cloud's REVERSE directory lookup: email → the WorkOS `user_...` id. - * - * Production asks WorkOS for the email AND organization in one request. Both - * filters matter: email makes the lookup indexed rather than one `getUser` - * request per member, while organization keeps the reverse lookup bound to the - * same tenant as the platform view. - * - * The pinned `@executor-js/emulate` WorkOS emulator has no list-users route. - * `WORKOS_API_URL` is the explicit test/dev emulator override, so that path - * retains the membership scan until the emulator supports the production - * query. The fallback still starts from the tenant's membership list and can - * never return a user from another organization. - * - * CASING: WorkOS preserves whatever casing an email was created with (and the - * emulator compares byte-exact), so the directory value is normalized here - * before comparison, against an argument the seam already normalized. - */ -export const emailResolver = - (organizationId: string, context: Context.Context): AdminEmailResolver => - (email) => - Effect.gen(function* () { - const workos = yield* WorkOSClient; - - if (!env.WORKOS_API_URL) { - const users = yield* workos.listUsers({ email, organizationId }); - return users.data[0]?.id ?? null; - } - - const memberships = yield* workos.listOrgMembers(organizationId); - const userIds = memberships.data.map((membership) => membership.userId); - - // Emulator compatibility only. Short-circuit once the normalized email - // matches so the fallback makes as few unsupported-detail reads as it can. - const match = yield* Effect.findFirst(userIds, (userId) => - workos.getUser(userId).pipe( - Effect.map((user) => normalizeAdminUserEmail(user.email ?? "") === email), - // One unreadable user must not fail the whole lookup — it simply - // cannot be the match. - Effect.catchCause(() => Effect.succeed(false)), - ), - ); - return Option.getOrNull(match); - }).pipe(Effect.provideContext(context)); - -/** Both directions of cloud's directory, built once per authorized request. */ -const userDirectory = ( - organizationId: string, - context: Context.Context, -): AdminUserDirectory => ({ - identities: identityDirectory(organizationId, context), - resolveEmail: emailResolver(organizationId, context), -}); - /** * Authorize, then run `body` against the tenant's platform view. * @@ -264,8 +142,9 @@ const withPlatformView = new AdminUsersError({ message: "Failed to open the platform view" })), ); - // The authorized tenant is handed to the body so an identity join reads the - // SAME org the reads are scoped to — never one named by client input. + // The authorized tenant is handed to the body so the directory reads the + // SAME org the storage reads are scoped to — never one named by client + // input. return yield* Effect.ensuring( body(executor, organizationId), executor.close().pipe(Effect.ignore), @@ -275,22 +154,40 @@ const withPlatformView = = Layer.effect(AdminUsersProvider)( Effect.gen(function* () { const context = yield* Effect.context< WorkOSClient | ApiKeyService | UserStoreService | DbProvider | PluginsProvider | HostConfig >(); + const directory = yield* MemberDirectory; + // The authorized tenant is what scopes the directory, so every read below + // asks the same org the platform view was opened for. + const userDirectory = (organizationId: string) => + adminUserDirectoryFromMembers(directory, organizationId); return AdminUsersProvider.of({ listUsers: (headers, options) => withPlatformView(headers, (executor, organizationId) => platformViewOf(executor).pipe( Effect.flatMap((admin) => - listAdminUsers(admin, options, userDirectory(organizationId, context)), + listAdminUsers(admin, options, userDirectory(organizationId)), ), ), ).pipe(Effect.provideContext(context)), @@ -298,7 +195,7 @@ export const workosAdminUsersProvider: Layer.Layer< withPlatformView(headers, (executor, organizationId) => platformViewOf(executor).pipe( Effect.flatMap((admin) => - listAdminUsersWithConnections(admin, options, userDirectory(organizationId, context)), + listAdminUsersWithConnections(admin, options, userDirectory(organizationId)), ), ), ).pipe(Effect.provideContext(context)), @@ -312,7 +209,7 @@ export const workosAdminUsersProvider: Layer.Layer< withPlatformView(headers, (executor, organizationId) => platformViewOf(executor).pipe( Effect.flatMap((admin) => - getAdminUser(admin, identifier, userDirectory(organizationId, context)), + getAdminUser(admin, identifier, userDirectory(organizationId)), ), ), ).pipe(Effect.provideContext(context)), @@ -322,8 +219,9 @@ export const workosAdminUsersProvider: Layer.Layer< // Builds the provider per request, providing it to the handlers. Long-lived // `WorkOSClient | ApiKeyService` come from the surrounding boot context; the -// per-request `DbService`/`UserStoreService` (and the execution seams built -// over them) are supplied by the combined `requestScopedMiddleware`. +// per-request `DbService`/`UserStoreService`/`MemberDirectory` (and the +// execution seams built over them) are supplied by the combined +// `requestScopedMiddleware`. const AdminUsersProviderMiddleware = HttpRouter.middleware<{ provides: AdminUsersProvider }>()( Effect.gen(function* () { const longLived = yield* Effect.context(); @@ -348,7 +246,7 @@ const AdminUsersProviderMiddleware = HttpRouter.middleware<{ provides: AdminUser * `/api` prefix as the rest of the cloud router. */ export const makeCloudAdminUsersRoutes = ( - rsLive: Layer.Layer, + rsLive: Layer.Layer, options: Parameters[1] = {}, ) => makeAdminUsersApiLayer( diff --git a/apps/cloud/src/admin/admin-users-email.node.test.ts b/apps/cloud/src/admin/admin-users-email.node.test.ts deleted file mode 100644 index 463c83df09..0000000000 --- a/apps/cloud/src/admin/admin-users-email.node.test.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { env } from "cloudflare:workers"; -import { expect, it } from "@effect/vitest"; -import { Data, Effect, Layer } from "effect"; - -import { WorkOSClient, type WorkOSClientService } from "../auth/workos"; -import { emailResolver } from "./admin-users-api"; - -// Cloud's REVERSE directory lookup: email -> the WorkOS `user_...` id that the -// subject table records in `external_id`. Production resolves it with one -// tenant-scoped list-users query. The WorkOS emulator lacks that route, so -// tests/dev retain the membership-backed scan exercised below. - -const ORG = "org_placeholder"; -const OTHER_ORG = "org_other"; - -class WorkOSUnavailable extends Data.TaggedError("WorkOSUnavailable")<{ - readonly userId: string; -}> {} - -const DIRECTORY = [ - // Same email in another tenant must never win either lookup path. - { id: "user_foreign", email: "ada@placeholder.test", organizationId: OTHER_ORG }, - // WorkOS preserves submitted casing, while the resolver seam is normalized. - { id: "user_ada", email: "Ada@Placeholder.test", organizationId: ORG }, - { id: "user_grace", email: "grace@placeholder.test", organizationId: ORG }, - { id: "user_nameless", email: null, organizationId: ORG }, -] as const; - -const stubWorkOS = (calls: string[], unreadableUserIds: ReadonlySet) => - Layer.succeed( - WorkOSClient, - new Proxy({} as WorkOSClientService, { - get: (_target, prop) => { - if (prop === "listUsers") { - return (params: { email: string; organizationId: string }) => { - calls.push(`listUsers:${params.organizationId}:${params.email}`); - return Effect.succeed({ - data: DIRECTORY.filter( - (user) => - user.organizationId === params.organizationId && - user.email?.toLowerCase() === params.email, - ), - }); - }; - } - if (prop === "listOrgMembers") { - return (organizationId: string) => { - calls.push(`listOrgMembers:${organizationId}`); - return Effect.succeed({ - data: DIRECTORY.filter((user) => user.organizationId === organizationId).map( - (user) => ({ userId: user.id, organizationId }), - ), - }); - }; - } - if (prop === "getUser") { - return (userId: string) => { - calls.push(`getUser:${userId}`); - if (unreadableUserIds.has(userId)) { - return Effect.fail(new WorkOSUnavailable({ userId })); - } - const user = DIRECTORY.find((candidate) => candidate.id === userId); - if (!user) return Effect.die(`unexpected user ${userId}`); - return Effect.succeed(user); - }; - } - return () => Effect.die(`unexpected WorkOSClient.${String(prop)} call`); - }, - }), - ); - -const resolve = ( - email: string, - calls: string[], - emulator = false, - unreadableUserIds: ReadonlySet = new Set(), -) => { - const previousApiUrl = env.WORKOS_API_URL; - return Effect.gen(function* () { - yield* Effect.sync(() => - Object.assign(env, { - WORKOS_API_URL: emulator ? "http://workos-emulator.invalid" : undefined, - }), - ); - const context = yield* Effect.context(); - return yield* emailResolver(ORG, context)(email); - }).pipe( - Effect.provide(stubWorkOS(calls, unreadableUserIds)), - Effect.ensuring(Effect.sync(() => Object.assign(env, { WORKOS_API_URL: previousApiUrl }))), - ); -}; - -it.effect("resolves an email with one tenant-scoped WorkOS query", () => - Effect.gen(function* () { - const calls: string[] = []; - expect(yield* resolve("ada@placeholder.test", calls)).toBe("user_ada"); - expect(calls).toEqual([`listUsers:${ORG}:ada@placeholder.test`]); - }), -); - -it.effect("returns null from one query when the organization has no matching email", () => - Effect.gen(function* () { - const calls: string[] = []; - expect(yield* resolve("nobody@placeholder.test", calls)).toBeNull(); - expect(calls).toEqual([`listUsers:${ORG}:nobody@placeholder.test`]); - }), -); - -it.effect("keeps the emulator fallback tenant-scoped and case-insensitive", () => - Effect.gen(function* () { - const calls: string[] = []; - expect(yield* resolve("ada@placeholder.test", calls, true)).toBe("user_ada"); - expect(calls).toEqual([`listOrgMembers:${ORG}`, "getUser:user_ada"]); - expect(calls).not.toContain("getUser:user_foreign"); - expect(calls.some((call) => call.startsWith("listUsers:"))).toBe(false); - }), -); - -it.effect("lets the emulator fallback continue past one unreadable member", () => - Effect.gen(function* () { - const calls: string[] = []; - expect(yield* resolve("grace@placeholder.test", calls, true, new Set(["user_ada"]))).toBe( - "user_grace", - ); - expect(calls).toEqual([`listOrgMembers:${ORG}`, "getUser:user_ada", "getUser:user_grace"]); - }), -); diff --git a/apps/cloud/src/api/layers.ts b/apps/cloud/src/api/layers.ts index bcc5c5221d..f0875cc1dc 100644 --- a/apps/cloud/src/api/layers.ts +++ b/apps/cloud/src/api/layers.ts @@ -2,10 +2,15 @@ import { HttpApiBuilder } from "effect/unstable/httpapi"; import { HttpServer } from "effect/unstable/http"; import { Layer } from "effect"; -import { makeProtectedApiLayer, requestScopedMiddleware } from "@executor-js/api/server"; +import { + makeProtectedApiLayer, + requestScopedMiddleware, + type MemberDirectory, +} from "@executor-js/api/server"; import { SessionAuthLive } from "../auth/middleware-live"; import { UserStoreService } from "../auth/context"; +import { cloudMemberDirectoryLayer } from "../auth/member-directory"; import { WorkOsMirror } from "../auth/workos-mirror"; import { CloudAuthPublicHandlers, @@ -27,12 +32,17 @@ import { CoreSharedServices } from "../auth/workos"; const DbLive = DbService.Live; const UserStoreLive = UserStoreService.Live.pipe(Layer.provide(DbLive)); const WorkOsMirrorLive = WorkOsMirror.Live.pipe(Layer.provide(DbLive)); +// The shared `MemberDirectory` read seam over the membership mirror — the +// same per-request socket the mirror writes through. +const MemberDirectoryLive = cloudMemberDirectoryLayer.pipe(Layer.provide(DbLive)); // Per-request layer. Anything that opens an I/O object (postgres.js socket, // fetch stream readers, anything backed by a `Writable`) MUST live here — // `provideRequestScoped` rebuilds it per request so Cloudflare Workers' // I/O isolation is satisfied. See `api.request-scope.test.ts`. -export const RequestScopedServicesLive = Layer.mergeAll(DbLive, UserStoreLive, WorkOsMirrorLive); +export const RequestScopedServicesLive: Layer.Layer< + DbService | UserStoreService | WorkOsMirror | MemberDirectory +> = Layer.mergeAll(DbLive, UserStoreLive, WorkOsMirrorLive, MemberDirectoryLive); // Boot-scoped layer. Built once at worker boot, reused across requests. // Safe for config, in-memory caches, the global tracer provider, and @@ -57,7 +67,7 @@ export const BootSharedServices = Layer.mergeAll( // handler reads it for the free-organizations-per-user limit gate — one of the // few app-only billing touchpoints. (It is NOT on the neutral boot core.) export const makeNonProtectedApiLive = ( - rsLive: Layer.Layer, + rsLive: Layer.Layer, ) => HttpApiBuilder.layer(NonProtectedApi).pipe( Layer.provide(Layer.mergeAll(CloudAuthPublicHandlers, CloudSessionAuthHandlers)), diff --git a/apps/cloud/src/api/router.ts b/apps/cloud/src/api/router.ts index 227dc20b22..8c80825ef2 100644 --- a/apps/cloud/src/api/router.ts +++ b/apps/cloud/src/api/router.ts @@ -1,7 +1,11 @@ import { Layer } from "effect"; import { HttpRouter } from "effect/unstable/http"; -import { RouterConfigLive, requestScopedMiddleware } from "@executor-js/api/server"; +import { + RouterConfigLive, + requestScopedMiddleware, + type MemberDirectory, +} from "@executor-js/api/server"; import { UserStoreService } from "../auth/context"; import { WorkOsMirror } from "../auth/workos-mirror"; @@ -31,7 +35,7 @@ import { makeProtectedApiLive } from "./protected"; // assert per-request semantics — see // `apps/cloud/src/api.request-scope.node.test.ts`. export const makeApiLive = ( - requestScopedLive: Layer.Layer, + requestScopedLive: Layer.Layer, ) => { const BillingRoutesLive = AutumnRoutesLive.pipe( Layer.provide(requestScopedMiddleware(requestScopedLive).layer), diff --git a/apps/cloud/src/auth/mirror-feeders.node.test.ts b/apps/cloud/src/auth/mirror-feeders.node.test.ts index 3381943557..a344f4553c 100644 --- a/apps/cloud/src/auth/mirror-feeders.node.test.ts +++ b/apps/cloud/src/auth/mirror-feeders.node.test.ts @@ -12,11 +12,20 @@ // - the callback picks the landing org from that same list: a returnTo // slug or last-org cookie lands only in an ACTIVE membership, an unknown // or pending one falls through +// - `inviteMember` mirrors the PENDING membership WorkOS created for the +// invitee (found by email among the org's pending memberships), so the +// member list shows the invite and can revoke it // - `removeMember` tombstones the mirror row after the WorkOS delete, // stamped with the membership's last WorkOS state (never a local clock), // so a replay of the membership as it was before the delete cannot // restore it while a replacement WorkOS created meanwhile is accepted // - `updateMemberRole` writes the role WorkOS returned +// - the seat gate trusts the mirror's count only for an organization whose +// membership list was scanned from WorkOS in full: an unmarked one is +// scanned first (once), so a partial mirror never admits an invite past +// the plan limit +// - the seat reporter scans an unmarked organization before counting and +// never re-scans a marked one // - the backfill mirrors every org's members and counts what it wrote, // writes nothing on a dry run, converges on a re-run, tombstones a // membership WorkOS no longer lists — but never one written after its @@ -39,10 +48,11 @@ import { describe, expect, it } from "@effect/vitest"; import { sql } from "drizzle-orm"; -import { Effect, Exit, Fiber, Latch, Layer } from "effect"; +import { Effect, Exit, Fiber, Latch, Layer, Option } from "effect"; import { HttpRouter, HttpServer } from "effect/unstable/http"; import { HttpApiBuilder } from "effect/unstable/httpapi"; +import { AccountForbidden } from "@executor-js/api"; import { AccountProvider, MemberDirectory, @@ -53,6 +63,7 @@ import { import { AccountCaller, workosAccountProvider } from "../account/workos-account-service"; import { RequestScopedServicesLive } from "../api/layers"; import { DbService } from "../db/db"; +import { forkReportMemberSeats } from "../extensions/billing/member-seats"; import { AutumnService } from "../extensions/billing/service"; import { ApiKeyService } from "./api-keys"; import { UserStoreService } from "./context"; @@ -210,13 +221,21 @@ describe("login callback", () => { listMetadata: { before: null, after: null }, }); }, - // The forked seat recount after login. - listOrgMembers: () => - Effect.succeed({ + // The landing org's seat recount scans the org from WorkOS the first + // time it is counted (its per-org backfill mark is missing); the + // scan lists the org's members and fetches each user. + listOrgMembers: (organizationId) => { + calls.push(`listOrgMembers:${organizationId}`); + return Effect.succeed({ object: "list" as const, - data: [] as never[], + data: listed.filter((m) => m.organizationId === organizationId) as never[], listMetadata: { before: null, after: null }, - }), + }); + }, + getUser: (id) => { + calls.push(`getUser:${id}`); + return Effect.succeed(workosUser(id) as never); + }, refreshSession: (_sealed, organizationId) => { refreshedInto.push(organizationId); return Effect.succeed("sealed-refreshed"); @@ -268,7 +287,17 @@ describe("login callback", () => { const response = await handler(callbackRequest({})); expect(response.status).toBe(302); - expect(calls, "one membership list for the whole callback").toEqual([ + expect( + calls, + "one membership list for the callback itself; the landing org, never scanned, is scanned once for its seat count", + ).toEqual([ + `listUserMemberships:${userId}`, + `listOrgMembers:${activeOrg}`, + `getUser:${userId}`, + ]); + calls.length = 0; + expect((await handler(callbackRequest({}))).status).toBe(302); + expect(calls, "a second sign-in lists memberships only: the org is now marked").toEqual([ `listUserMemberships:${userId}`, ]); @@ -542,7 +571,14 @@ describe("account service writes through to the mirror", () => { * Provided around the WHOLE test body so the postgres socket outlives the * provider call under test. */ - const providerLayer = (org: string, deleted: string[]) => { + const providerLayer = ( + org: string, + deleted: string[], + options: { + readonly workos?: Partial; + readonly autumn?: Layer.Layer; + } = {}, + ) => { const list = (data: readonly unknown[]) => Effect.succeed({ object: "list" as const, @@ -550,6 +586,7 @@ describe("account service writes through to the mirror", () => { listMetadata: { before: null, after: null }, }); const workos = stubWorkOS({ + ...options.workos, listUserMemberships: (userId) => list([workosMembership(userId, org)]), getUserOrgMembership: (organizationId, userId) => Effect.succeed( @@ -571,7 +608,6 @@ describe("account service writes through to the mirror", () => { updatedAt: T2, }) as never, ), - listOrgMembers: () => list([]), }); // The test database serves ONE connection at a time, so the seed, the // provider, and the directory read all share this layer's socket. @@ -585,7 +621,7 @@ describe("account service writes through to the mirror", () => { Layer.mergeAll( workos, stubApiKeys, - stubAutumn, + options.autumn ?? stubAutumn, Layer.succeed(AccountCaller)({ session: session(ADMIN) }), ), ), @@ -595,7 +631,13 @@ describe("account service writes through to the mirror", () => { }; // TARGET as an existing member of `org`, seeded through the live mirror. - const seedTarget = (org: string) => + // The org is marked backfilled (as the one-off backfill leaves every org) + // unless a test wants the unscanned state, so a seat count reads the mirror + // rather than scanning WorkOS. + const seedTarget = ( + org: string, + options: { readonly backfilled: boolean } = { backfilled: true }, + ) => Effect.gen(function* () { const users = yield* UserStoreService; const mirror = yield* WorkOsMirror; @@ -614,11 +656,175 @@ describe("account service writes through to the mirror", () => { status: "active", updatedAt: new Date(T1), }); + if (options.backfilled) { + // An empty listing at T1 (nothing to tombstone: TARGET's row is + // stamped T1, not before it) marks the org scanned as of T1. + yield* mirror.applyOrganizationScan({ + organizationId: org, + listedAt: new Date(T1), + members: [], + }); + } }); const membersOf = (org: string) => Effect.flatMap(MemberDirectory.asEffect(), (directory) => directory.members(org)); + it.effect("inviteMember mirrors the pending membership WorkOS created for the invitee", () => { + const org = freshId("org"); + // Two people are already invited; the new invitee is a third pending + // membership, and only their user carries the invited address — with + // different casing than the admin typed, as WorkOS may store it. + const earlier = [freshId("user"), freshId("user")]; + const invitee = freshId("user"); + const invitedEmail = `${invitee}@placeholder.test`; + const userCalls: string[] = []; + // The plan gate reads the customer's plan before inviting: an unlimited + // plan so the seat cap never interferes with what is under test. + const teamAutumn = Layer.succeed(AutumnService)({ + use: () => + Effect.succeed({ + subscriptions: [{ planId: "team", status: "active" }], + } as never), + ensureCustomer: () => Effect.void, + checkExecutionBalance: () => Effect.die("invite does not check balances"), + trackExecution: () => Effect.void, + setMemberSeats: () => Effect.void, + }); + const layer = providerLayer(org, [], { + autumn: teamAutumn, + workos: { + listPendingInvitations: () => + Effect.succeed({ + object: "list" as const, + data: [] as never[], + listMetadata: { before: null, after: null }, + }), + sendInvitation: ({ email }) => + Effect.succeed({ + id: `invitation_${invitee}`, + email: email.toUpperCase(), + } as never), + listOrgMembers: (organizationId, statuses) => { + expect(organizationId).toBe(org); + expect(statuses, "only the pending set is listed").toEqual(["pending"]); + return Effect.succeed({ + object: "list" as const, + data: [...earlier, invitee].map((userId) => + workosMembership(userId, org, { status: "pending" }), + ) as never[], + listMetadata: { before: null, after: null }, + }); + }, + getUser: (userId) => + Effect.sync(() => { + userCalls.push(userId); + return workosUser(userId, { + firstName: "Invited", + lastName: "Person", + }) as never; + }), + }, + }); + return Effect.gen(function* () { + yield* seedTarget(org); + const account = yield* AccountProvider; + + const result = yield* account.inviteMember( + { [ORG_SELECTOR_HEADER]: org }, + { email: invitedEmail }, + ); + + expect(result.id).toBe(`invitation_${invitee}`); + const members = yield* membersOf(org); + const pending = members.find((m) => m.status === "pending"); + expect(pending, "the invitee appears as a pending member").toMatchObject({ + accountId: invitee, + membershipId: `om_${invitee}_${org}`, + email: invitedEmail, + name: "Invited Person", + role: "member", + }); + expect( + members.filter((m) => m.status === "pending"), + "only the invitee's pending membership is mirrored, not the other pending ones", + ).toHaveLength(1); + expect( + userCalls.sort(), + "one getUser per pending membership, bounded to the pending set", + ).toEqual([...earlier, invitee].sort()); + }).pipe(Effect.provide(layer)); + }); + + it.effect( + "inviteMember scans an organization the backfill never covered before counting its seats, once", + () => { + const org = freshId("org"); + const listed: string[] = []; + // A free plan (limit 3). The mirror holds ONE member of the org (TARGET) + // and the org is unmarked; WorkOS lists three. Only a count taken after + // the scan refuses the invite. + const freeAutumn = Layer.succeed(AutumnService)({ + use: () => Effect.succeed({ subscriptions: [] } as never), + ensureCustomer: () => Effect.void, + checkExecutionBalance: () => Effect.die("invite does not check balances"), + trackExecution: () => Effect.void, + setMemberSeats: () => Effect.void, + }); + const others = [freshId("user"), freshId("user")]; + const layer = providerLayer(org, [], { + autumn: freeAutumn, + workos: { + listPendingInvitations: () => + Effect.succeed({ + object: "list" as const, + data: [] as never[], + listMetadata: { before: null, after: null }, + }), + listOrgMembers: (organizationId, statuses) => { + listed.push(organizationId); + expect(statuses, "the scan lists every status, inactive included").toEqual([ + "active", + "pending", + "inactive", + ]); + return Effect.succeed({ + object: "list" as const, + data: [TARGET, ...others].map((userId) => workosMembership(userId, org)) as never[], + listMetadata: { before: null, after: null }, + }); + }, + getUser: (userId) => Effect.succeed(workosUser(userId) as never), + sendInvitation: () => + Effect.die("the plan gate refuses before WorkOS is asked to invite"), + }, + }); + return Effect.gen(function* () { + yield* seedTarget(org, { backfilled: false }); + const account = yield* AccountProvider; + const invite = () => + Effect.flip( + account.inviteMember({ [ORG_SELECTOR_HEADER]: org }, { email: "new@placeholder.test" }), + ); + + const error = yield* invite(); + expect(error).toBeInstanceOf(AccountForbidden); + expect(error).toMatchObject({ + message: expect.stringContaining("Your plan includes 3 members"), + }); + expect(listed, "the org was scanned from WorkOS before it was counted").toEqual([org]); + expect( + (yield* membersOf(org)).map((m) => m.accountId).sort(), + "and the scan filled the mirror", + ).toEqual([TARGET, ...others].sort()); + + const again = yield* invite(); + expect(again).toBeInstanceOf(AccountForbidden); + expect(listed, "a marked org is never scanned again").toEqual([org]); + }).pipe(Effect.provide(layer)); + }, + ); + it.effect("removeMember tombstones the mirror row after the WorkOS delete", () => { const org = freshId("org"); const deleted: string[] = []; @@ -688,6 +894,149 @@ describe("account service writes through to the mirror", () => { }); }); +describe("seat reporter", () => { + /** + * A `WorkOsMirror` answering the per-org backfill mark and recording the + * scan a reporter applies; every other operation is out of its reach. + */ + const recordingMirror = (backfilledAt: Date | null, writes: string[]) => + Layer.succeed(WorkOsMirror)({ + upsertUser: () => Effect.die("the seat reporter scans, it does not upsert one by one"), + upsertMembership: () => Effect.die("the seat reporter scans, it does not upsert one by one"), + deleteMembership: () => Effect.die("the seat reporter does not delete"), + deleteUser: () => Effect.die("the seat reporter does not delete"), + getCursor: () => Effect.die("the seat reporter does not read the cursor"), + applyPage: () => Effect.die("the seat reporter does not move the cursor"), + applyOrganizationScan: (scan) => + Effect.sync(() => { + writes.push( + `applyOrganizationScan:${scan.organizationId}:${scan.members + .map((member) => member.membership.id) + .join(",")}`, + ); + return Option.some({ + usersWritten: scan.members.length, + membershipsWritten: scan.members.length, + membershipsTombstoned: 0, + }); + }), + replayBoundary: () => Effect.die("the seat reporter does not run the reconciler"), + setReplayBoundary: () => Effect.die("the seat reporter does not record the boundary"), + backfillCompletedAt: () => Effect.die("the seat reporter does not check mirror readiness"), + markBackfillCompleted: () => Effect.die("the seat reporter does not record the completion"), + drainedAt: () => Effect.die("the seat reporter does not check mirror readiness"), + markDrained: () => Effect.die("the seat reporter does not run the reconciler"), + organizationBackfilledAt: () => Effect.succeed(backfilledAt), + } satisfies WorkOsMirrorShape); + + /** A directory holding `active` active members and one pending one. */ + const directoryWith = (org: string, active: number) => + Layer.succeed(MemberDirectory)({ + membership: () => Effect.die("the seat reporter lists, it does not look up"), + membersById: () => Effect.die("the seat reporter lists, it does not look up"), + findByEmail: () => Effect.die("the seat reporter lists, it does not look up"), + members: (organizationId, query) => { + expect(organizationId).toBe(org); + expect(query?.statuses, "billed seats are active members only").toEqual(["active"]); + return Effect.succeed( + Array.from({ length: active }, (_, i) => ({ + accountId: `user_${i}`, + membershipId: `om_${i}`, + organizationId, + email: null, + name: null, + avatarUrl: null, + role: "member", + status: "active" as const, + lastActiveAt: null, + })), + ); + }, + }); + + const report = ( + org: string, + backfilledAt: Date | null, + active: number, + workos: Partial = {}, + ) => + Effect.gen(function* () { + const reported: { organizationId: string; seats: number }[] = []; + const writes: string[] = []; + const recording = Layer.succeed(AutumnService)({ + use: () => Effect.die("the seat reporter sets seats, it does not read"), + ensureCustomer: () => Effect.void, + checkExecutionBalance: () => Effect.die("the seat reporter does not check balances"), + trackExecution: () => Effect.void, + setMemberSeats: (organizationId, seats) => + Effect.sync(() => { + reported.push({ organizationId, seats }); + }), + }); + yield* forkReportMemberSeats(org).pipe( + Effect.provide( + Layer.mergeAll( + recordingMirror(backfilledAt, writes), + directoryWith(org, active), + recording, + stubWorkOS(workos), + ), + ), + ); + // The Autumn call is forked; it is synchronous here, so it has landed. + return { reported, writes }; + }); + + it.effect( + "sets the active member count of a scanned organization without touching WorkOS", + () => { + const org = freshId("org"); + return Effect.gen(function* () { + const { reported, writes } = yield* report(org, new Date(T1), 3); + expect(reported).toEqual([{ organizationId: org, seats: 3 }]); + expect(writes, "a marked organization is not scanned").toEqual([]); + }); + }, + ); + + it.effect("scans an organization the backfill never covered before counting it", () => { + const org = freshId("org"); + const member = freshId("user"); + return Effect.gen(function* () { + const { reported, writes } = yield* report(org, null, 2, { + listOrgMembers: (organizationId, statuses) => { + expect(organizationId).toBe(org); + // Inactive memberships included: a scan that skipped them would + // tombstone them under their ids and refuse their reactivation. + expect(statuses).toEqual(["active", "pending", "inactive"]); + return Effect.succeed({ + object: "list" as const, + data: [workosMembership(member, org)] as never[], + listMetadata: { before: null, after: null }, + }); + }, + getUser: (userId) => Effect.succeed(workosUser(userId) as never), + }); + expect( + writes, + "the scan fills the mirror and marks the organization, then the count is read", + ).toEqual([`applyOrganizationScan:${org}:om_${member}_${org}`]); + expect(reported).toEqual([{ organizationId: org, seats: 2 }]); + }); + }); + + it.effect("pushes no count when the scan fails: a partial count is never billed", () => { + const org = freshId("org"); + return Effect.gen(function* () { + const { reported, writes } = yield* report(org, null, 2, { + listOrgMembers: () => Effect.fail(new WorkOSError({ status: 503 })), + }); + expect(reported).toEqual([]); + expect(writes, "nothing is marked").toEqual([]); + }); + }); +}); + describe("backfill", () => { /** A fake WorkOS holding `orgs` → members, counting `getUser` calls. */ const source = (orgs: ReadonlyMap, userCalls: string[]) => ({ diff --git a/apps/cloud/src/auth/mirror-feeders.ts b/apps/cloud/src/auth/mirror-feeders.ts index 0dc825e9b3..3b7ab3ec11 100644 --- a/apps/cloud/src/auth/mirror-feeders.ts +++ b/apps/cloud/src/auth/mirror-feeders.ts @@ -5,14 +5,21 @@ // Each feeder takes the WorkOS payload the caller ALREADY holds (the // authenticated user, the membership list the callback fetches to pick a // landing org, the membership a write returned) so feeding the mirror never -// adds a WorkOS read. Mirror failures fail the request: the mirror is the -// membership read path, so a login that could not record its memberships is -// not a login that finished. +// adds a WorkOS read — except the two writes whose WorkOS response is not the +// membership they changed: invitation acceptance (`auth/handlers.ts` reads +// the activated membership back) and sending an invitation +// (`mirrorInvitedMember` below reads the pending one WorkOS created). Both +// are rare, admin-driven paths. Mirror failures fail the request: the mirror +// is the membership read path, so a login that could not record its +// memberships is not a login that finished. // --------------------------------------------------------------------------- import { Effect } from "effect"; +import { normalizeAdminUserEmail } from "@executor-js/api/server"; + import { UserStoreService } from "./context"; +import { WorkOSClient } from "./workos"; import { WorkOsMirror, mirrorMembershipFromWorkOs, @@ -20,6 +27,7 @@ import { type WorkOsMembershipPayload, type WorkOsUserPayload, } from "./workos-mirror"; +import { backfillOrganization } from "./workos-mirror-backfill"; /** * A membership as WorkOS lists it for a user: carries the organization's name, @@ -78,3 +86,101 @@ export const mirrorMembership = (membership: WorkOsMembershipPayload) => Effect.flatMap(WorkOsMirror.asEffect(), (mirror) => mirror.upsertMembership(mirrorMembershipFromWorkOs(membership)), ); + +// Bounded fan-out for the per-invitee `getUser` calls, matching the backfill: +// enough to overlap WorkOS round-trips, low enough to stay clear of its rate +// limit. +const USER_FETCH_CONCURRENCY = 5; + +/** + * Record the PENDING membership WorkOS creates for an invitee the moment an + * organization invites them — the row the member list shows as "Invited" and + * the admin revokes an outstanding invite through. `sendInvitation` returns + * the invitation, not that membership, so this reads it back: it lists the + * organization's pending memberships (WorkOS has no lookup by email that the + * emulator serves) and fetches their users, five at a time, until one carries + * the invited email. Bounded by the pending set, so an organization with + * many active members pays nothing per member. + * + * `false` when no pending membership carried the email — WorkOS created none + * (the address may already hold a membership) or has not yet — which the + * caller treats as a warning, not a failure: the Events reconciler lands + * whatever WorkOS did create. + */ +export const mirrorInvitedMember = Effect.fn("workos_mirror.invitedMember")(function* ( + organizationId: string, + invitedEmail: string, +) { + const workos = yield* WorkOSClient; + const mirror = yield* WorkOsMirror; + const wanted = normalizeAdminUserEmail(invitedEmail); + const pending = yield* workos.listOrgMembers(organizationId, ["pending"]); + for (let start = 0; start < pending.data.length; start += USER_FETCH_CONCURRENCY) { + const batch = pending.data.slice(start, start + USER_FETCH_CONCURRENCY); + const candidates = yield* Effect.forEach( + batch, + (membership) => + Effect.map(workos.getUser(membership.userId), (user) => ({ + membership, + user, + })), + { concurrency: USER_FETCH_CONCURRENCY }, + ); + const match = candidates.find( + (candidate) => normalizeAdminUserEmail(candidate.user.email) === wanted, + ); + if (match === undefined) continue; + yield* mirror.upsertUser(mirrorUserFromWorkOs(match.user)); + yield* mirror.upsertMembership(mirrorMembershipFromWorkOs(match.membership)); + return true; + } + return false; +}); + +/** + * Make sure the organization's membership list has been scanned from WorkOS + * in full before a COUNT read from the mirror is trusted. Login records only + * the caller's own memberships and write-through only the one it changed, + * so an organization the one-off backfill did not cover — mirrored lazily + * by a request, or created after the backfill ran — holds a partial list + * until it is scanned. The per-organization mark + * (`organizations.backfilled_at`) says whether that scan has happened; when + * it is missing, this runs the scan now (`backfillOrganization`: one + * membership listing plus one `getUser` per member, then the mark), so the + * caller's count is complete. Returns `true` when a scan ran. A scan that + * fails marks nothing, so the next count tries again. + */ +export const ensureOrganizationBackfilled = Effect.fn("workos_mirror.ensureOrganizationBackfilled")( + function* (organizationId: string) { + const mirror = yield* WorkOsMirror; + const backfilledAt = yield* mirror.organizationBackfilledAt(organizationId); + if (backfilledAt !== null) return false; + const workos = yield* WorkOSClient; + yield* Effect.logInfo( + "workos_mirror: organization not yet backfilled; scanning it from WorkOS", + { + organizationId, + }, + ); + yield* backfillOrganization( + { + // EVERY status, as the scan source requires: the scan tombstones + // whatever its listing lacks, and a tombstone is keyed to the + // membership id for good — so a listing that skipped the inactive + // ones (the wrapper's active + pending default, the seat-occupying + // set) would tombstone a membership WorkOS merely deactivated and + // refuse its reactivation under the same id forever. + listOrgMembers: (id) => + Effect.map( + workos.listOrgMembers(id, ["active", "pending", "inactive"]), + (list) => list.data, + ), + getUser: (id) => workos.getUser(id), + }, + mirror, + organizationId, + { dryRun: false }, + ); + return true; + }, +); diff --git a/apps/cloud/src/auth/workos-callback-state.node.test.ts b/apps/cloud/src/auth/workos-callback-state.node.test.ts index a62f840b67..a80d4a710b 100644 --- a/apps/cloud/src/auth/workos-callback-state.node.test.ts +++ b/apps/cloud/src/auth/workos-callback-state.node.test.ts @@ -16,6 +16,8 @@ import { HttpRouter, HttpServer } from "effect/unstable/http"; import { HttpApiBuilder } from "effect/unstable/httpapi"; import { HttpApi } from "effect/unstable/httpapi"; +import { MemberDirectory } from "@executor-js/api/server"; + import { CloudAuthPublicHandlers } from "./handlers"; import { CloudAuthPublicApi } from "./api"; import { UserStoreService } from "./context"; @@ -50,9 +52,6 @@ const stubWorkOS = Layer.succeed( if (prop === "listUserMemberships") { return () => Effect.succeed({ data: [] }); } - if (prop === "listOrgMembers") { - return () => Effect.succeed({ data: [{ status: "active" }] }); - } return () => Effect.die(`unexpected WorkOSClient.${String(prop)} call`); }, }), @@ -109,7 +108,9 @@ const stubUsers = Layer.succeed(UserStoreService)({ }); // The callback records the sign-in (user + memberships) in the membership -// mirror; every other mirror operation is out of this route's reach. +// mirror, and its forked seat recount reads the backfill marker and the +// landed org's active members from it; every other operation is out of this +// route's reach. const stubMirror = Layer.succeed(WorkOsMirror)({ upsertUser: () => Effect.succeed(true), upsertMembership: () => Effect.succeed(true), @@ -124,7 +125,27 @@ const stubMirror = Layer.succeed(WorkOsMirror)({ markBackfillCompleted: () => Effect.die("the callback does not run the backfill"), drainedAt: () => Effect.die("the callback does not check mirror readiness"), markDrained: () => Effect.die("the callback does not run the reconciler"), - organizationBackfilledAt: () => Effect.die("the callback does not report seats"), + organizationBackfilledAt: () => Effect.succeed(new Date()), +}); + +const stubDirectory = Layer.succeed(MemberDirectory)({ + membership: () => Effect.die("the callback does not look up one membership"), + membersById: () => Effect.die("the callback does not batch members"), + findByEmail: () => Effect.die("the callback does not resolve emails"), + members: (organizationId) => + Effect.succeed([ + { + accountId: STUB_USER_ID, + membershipId: `om_${STUB_USER_ID}`, + organizationId, + email: null, + name: null, + avatarUrl: null, + role: "member", + status: "active" as const, + lastActiveAt: null, + }, + ]), }); // Only the public group is under test; the session group (and its SessionAuth @@ -137,6 +158,7 @@ const App = HttpApiBuilder.layer(PublicApi).pipe( Layer.provide(stubWorkOS), Layer.provide(stubUsers), Layer.provide(stubMirror), + Layer.provide(stubDirectory), Layer.provide(AutumnService.Default), Layer.provide(HttpServer.layerServices), ); diff --git a/apps/cloud/src/auth/workos.ts b/apps/cloud/src/auth/workos.ts index dcaaa723da..e8bf57c569 100644 --- a/apps/cloud/src/auth/workos.ts +++ b/apps/cloud/src/auth/workos.ts @@ -9,6 +9,7 @@ import { WorkOS, type Event as WorkOSEvent, type EventName as WorkOSEventName, + type OrganizationMembershipStatus, } from "@workos-inc/node/worker"; import { defaults as ironDefaults, unseal as unsealIron } from "iron-webcrypto"; import { decodeJwt, jwtVerify } from "jose"; @@ -665,13 +666,21 @@ const make = Effect.gen(function* () { deleteApiKey: (id: string) => use("apiKeys.deleteApiKey", (wos) => wos.apiKeys.deleteApiKey(id)), - /** List organization memberships with user details. */ - listOrgMembers: (organizationId: string) => + /** + * An organization's memberships, all pages. Defaults to active + pending + * (the seat-occupying set); pass `statuses` to narrow — the invite + * write-through lists only `pending` to find the membership WorkOS + * created for the invitee. + */ + listOrgMembers: ( + organizationId: string, + statuses: readonly OrganizationMembershipStatus[] = ["active", "pending"], + ) => use("userManagement.listOrganizationMemberships", async (wos) => collectWorkOSList( await wos.userManagement.listOrganizationMemberships({ organizationId, - statuses: ["active", "pending"], + statuses: [...statuses], }), ), ), @@ -691,17 +700,6 @@ const make = Effect.gen(function* () { getUser: (userId: string) => use("userManagement.getUser", (wos) => wos.userManagement.getUser(userId)), - /** List users matching an email within one organization. */ - listUsers: (params: { email: string; organizationId: string }) => - use("userManagement.listUsers", async (wos) => - collectWorkOSList( - await wos.userManagement.listUsers({ - email: params.email, - organizationId: params.organizationId, - }), - ), - ), - /** Send an organization invitation. */ sendInvitation: (params: { email: string; organizationId: string; roleSlug?: string }) => use("userManagement.sendInvitation", (wos) => diff --git a/apps/cloud/src/extensions/billing/member-seats.ts b/apps/cloud/src/extensions/billing/member-seats.ts index 75c1d7e00d..7ed5f2ab19 100644 --- a/apps/cloud/src/extensions/billing/member-seats.ts +++ b/apps/cloud/src/extensions/billing/member-seats.ts @@ -1,11 +1,16 @@ // --------------------------------------------------------------------------- -// Seat-count reporting — the WorkOS → Autumn reconciliation for seat billing +// Seat-count reporting — the membership mirror → Autumn reconciliation for +// seat billing // --------------------------------------------------------------------------- import { Effect } from "effect"; import { waitUntil } from "cloudflare:workers"; -import { WorkOSClient } from "../../auth/workos"; +import { MemberDirectory } from "@executor-js/api/server"; + +import { ensureOrganizationBackfilled } from "../../auth/mirror-feeders"; +import type { WorkOSClient } from "../../auth/workos"; +import type { WorkOsMirror } from "../../auth/workos-mirror"; import { AutumnService } from "./service"; /** @@ -16,39 +21,54 @@ import { AutumnService } from "./service"; * Seats change through paths the app never sees a mutation for (invitation * acceptance in AuthKit, SSO JIT provisioning, join by domain, WorkOS * dashboard edits), so this reconciles from a full recount rather than - * tracking deltas. It runs after in-app membership mutations AND on every - * login callback, so drift from out-of-band changes heals on the next - * sign-in. Fire-and-forget-safe: errors are logged, never surfaced. + * tracking deltas. The count comes from the local membership mirror through + * the shared `MemberDirectory`: every in-app membership mutation writes + * through to the mirror BEFORE calling this, and out-of-band changes land via + * login and the Events reconciler, so the recount reads the change on the + * next sign-in exactly as it did against WorkOS — without a WorkOS read. + * + * The Autumn call runs off the calling request's critical path: Cloudflare + * owns its promise through `waitUntil`, so the recount can finish after the + * response, and billing never stalls or fails a user-facing request. Errors + * are logged, never surfaced. + * + * The count is a PARTIAL one until THIS organization's membership list has + * been scanned from WorkOS in full (the one-off backfill, or the on-demand + * scan below): before that, the mirror holds only the members who signed in + * or were changed since the mirror shipped. Because the Autumn write is an + * authoritative SET, pushing a partial count would under-bill the + * organization, so the recount first makes sure the organization is + * backfilled (`ensureOrganizationBackfilled`: a scan runs now when its + * per-organization mark is missing) and only then counts. The plan gate + * (`reserveMemberSlot`) goes through the same step, so it never admits an + * invite past the plan limit on a partial mirror. + * + * The COUNT is read inline, not in the fork: `MemberDirectory` is per-request + * (it holds the request's postgres socket, which Cloudflare Workers' I/O + * isolation ties to the request), so a forked fiber reading it could outlive + * the socket. One indexed local query is cheap enough to pay inline; only the + * Autumn call — over the boot-scoped `AutumnService` — is forked, so the + * forked fiber captures nothing request-scoped. */ -export const reportMemberSeats = ( +export const forkReportMemberSeats = ( organizationId: string, -): Effect.Effect => +): Effect.Effect => Effect.gen(function* () { - const workos = yield* WorkOSClient; + const directory = yield* MemberDirectory; const autumn = yield* AutumnService; - const memberships = yield* workos.listOrgMembers(organizationId); - const seats = memberships.data.filter((m) => m.status === "active").length; - yield* autumn.setMemberSeats(organizationId, seats); + yield* ensureOrganizationBackfilled(organizationId); + const seats = yield* directory + .members(organizationId, { statuses: ["active"] }) + .pipe(Effect.map((members) => members.length)); + yield* Effect.sync(() => { + waitUntil(Effect.runPromise(autumn.setMemberSeats(organizationId, seats))); + }); }).pipe( Effect.catch((error) => - Effect.logWarning("reportMemberSeats: seat recount failed", { organizationId, error }), + Effect.logWarning("reportMemberSeats: seat recount failed", { + organizationId, + error, + }), ), Effect.withSpan("billing.reportMemberSeats"), ); - -/** - * Fork `reportMemberSeats` off the calling request, mirroring how execution - * tracking is forked: billing must never stall or fail a user-facing - * request. Cloudflare owns the promise through waitUntil, so the recount can - * finish after the response. Only boot-scoped WorkOS and Autumn services are - * captured. - */ -export const forkReportMemberSeats = ( - organizationId: string, -): Effect.Effect => - Effect.gen(function* () { - const ctx = yield* Effect.context(); - yield* Effect.sync(() => { - waitUntil(Effect.runPromiseWith(ctx)(reportMemberSeats(organizationId))); - }); - }); diff --git a/apps/cloud/src/extensions/routes.ts b/apps/cloud/src/extensions/routes.ts index bd4b1d4c12..2f30bceaab 100644 --- a/apps/cloud/src/extensions/routes.ts +++ b/apps/cloud/src/extensions/routes.ts @@ -28,7 +28,7 @@ import { HttpApiBuilder } from "effect/unstable/httpapi"; import { HttpApiSwagger, OpenApi } from "effect/unstable/httpapi"; import { AccountApi, AdminUsersApi } from "@executor-js/api"; -import { requestScopedMiddleware } from "@executor-js/api/server"; +import { requestScopedMiddleware, type MemberDirectory } from "@executor-js/api/server"; import { UserStoreService } from "../auth/context"; import { WorkOsMirror } from "../auth/workos-mirror"; @@ -79,7 +79,7 @@ const spec = OpenApi.fromApi(CloudOpenApi); * core. */ export const makeCloudExtensionRoutes = ( - rsLive: Layer.Layer, + rsLive: Layer.Layer, ) => { // Session routes (login / callback / me / switch-org / …). Handlers yield // `UserStoreService` directly; the per-request DB combine keeps the postgres diff --git a/apps/host-selfhost/src/account/better-auth-account-provider.ts b/apps/host-selfhost/src/account/better-auth-account-provider.ts index c7ed75a30e..885ed74eee 100644 --- a/apps/host-selfhost/src/account/better-auth-account-provider.ts +++ b/apps/host-selfhost/src/account/better-auth-account-provider.ts @@ -145,7 +145,7 @@ export const betterAuthAccountProvider: Layer.Layer ({ id: member.id, userId: member.userId, - email: member.user?.email ?? "", + email: member.user?.email ?? null, name: member.user?.name ?? null, avatarUrl: member.user?.image ?? null, role: member.role, diff --git a/apps/host-selfhost/src/admin/admin-users-api.ts b/apps/host-selfhost/src/admin/admin-users-api.ts index 38f767a168..8a4d458c52 100644 --- a/apps/host-selfhost/src/admin/admin-users-api.ts +++ b/apps/host-selfhost/src/admin/admin-users-api.ts @@ -16,7 +16,11 @@ // // The READ half is identical to cloud's: a subject-less, tenant-reach executor // from `makePlatformExecutor`, projected by the shared `admin/reads`. Self-host -// is single-tenant, so the tenant is always the boot-seeded org. +// is single-tenant, so the tenant is always the boot-seeded org. Identity +// (email/name per row), the `?email=` resolver and the `?search=` match all +// come from the shared `MemberDirectory` — here Better Auth's `member` + `user` +// tables through its own adapter (`auth/member-directory.ts`), the SAME read +// the MCP plane makes, so no plane keeps its own join. // --------------------------------------------------------------------------- import { HttpRouter } from "effect/unstable/http"; @@ -26,18 +30,17 @@ import { AdminUsersProvider, DbProvider, HostConfig, + MemberDirectory, PluginsProvider, + adminUserDirectoryFromMembers, getAdminUser, listAdminUserConnections, listAdminUsers, listAdminUsersWithConnections, makeAdminUsersApiLayer, makePlatformExecutor, - normalizeAdminUserEmail, platformViewOf, requestScopedMiddleware, - type AdminUserDirectory, - type AdminUserIdentity, type AdminUsersHeaders, } from "@executor-js/api/server"; import { @@ -67,70 +70,6 @@ const requireAdmin = (headers: AdminUsersHeaders) => ), ); -/** - * Self-host's member directory: `externalId` → email/name. - * - * THE JOIN KEY is `member.userId`, the Better Auth `user.id` — precisely what - * `auth/identity.ts` binds as `accountId` and therefore what the subject table - * records in `external_id`. `member.id` is the organization `member` ROW id and - * joins to nothing; the two look alike, so the choice is pinned here and in the - * node test rather than left to a reader. - * - * One `listMembers` call per request: Better Auth's organization plugin already - * attaches the `user` row to each member, so email and name arrive with the - * membership and no per-user lookup is needed. The requested ids are not passed - * to the call — the plugin offers no id filter, and a single-instance member - * list is small — but the caller only reads the ids it asked for. - * - * Runs as the CALLER, using their own admin headers, so this reads exactly the - * directory that session is already entitled to on `/account/members`. - */ -const listMembers = (auth: BetterAuthHandle["auth"], headers: AdminUsersHeaders) => - Effect.tryPromise(() => auth.api.listMembers({ headers: new Headers(headers) })); - -/** - * Both directions of self-host's directory, over the SAME single `listMembers` - * read. - * - * The reverse (email → `user.id`) needs no extra call and no new permission: - * the organization plugin already attaches the `user` row to each member, so - * the email is sitting beside the id the forward join uses. Better Auth - * lower-cases every email it writes, but the directory value is normalized - * anyway so this host cannot answer differently from cloud if that ever - * changes. - * - * A member with no `user.email` cannot match — `null` is not an address, and - * coercing it to "" would let an empty `?email=` select an arbitrary row. - */ -const userDirectory = ( - auth: BetterAuthHandle["auth"], - headers: AdminUsersHeaders, -): AdminUserDirectory => ({ - identities: () => - listMembers(auth, headers).pipe( - Effect.map((result) => { - const identities = new Map(); - for (const member of result.members) { - identities.set(member.userId, { - email: member.user?.email ?? null, - displayName: member.user?.name ?? null, - }); - } - return identities; - }), - ), - resolveEmail: (email) => - listMembers(auth, headers).pipe( - Effect.map( - (result) => - result.members.find((member) => { - const stored = member.user?.email; - return stored != null && normalizeAdminUserEmail(stored) === email; - })?.userId ?? null, - ), - ), -}); - const withPlatformView = ( headers: AdminUsersHeaders, organizationId: string, @@ -153,24 +92,25 @@ const withPlatformView = = Layer.effect(AdminUsersProvider)( Effect.gen(function* () { const context = yield* Effect.context(); - const { auth, organizationId } = yield* BetterAuth; + const { organizationId } = yield* BetterAuth; + // Scoped to the INSTANCE's org — the same one the platform view is opened + // for, never the caller's `activeOrganizationId` (see require-admin.ts). + const directory = adminUserDirectoryFromMembers(yield* MemberDirectory, organizationId); return AdminUsersProvider.of({ listUsers: (headers, options) => withPlatformView(headers, organizationId, (executor) => platformViewOf(executor).pipe( - Effect.flatMap((admin) => listAdminUsers(admin, options, userDirectory(auth, headers))), + Effect.flatMap((admin) => listAdminUsers(admin, options, directory)), ), ).pipe(Effect.provideContext(context)), listUsersWithConnections: (headers, options) => withPlatformView(headers, organizationId, (executor) => platformViewOf(executor).pipe( - Effect.flatMap((admin) => - listAdminUsersWithConnections(admin, options, userDirectory(auth, headers)), - ), + Effect.flatMap((admin) => listAdminUsersWithConnections(admin, options, directory)), ), ).pipe(Effect.provideContext(context)), listUserConnections: (headers, externalId) => @@ -182,9 +122,7 @@ export const betterAuthAdminUsersProvider: Layer.Layer< getUser: (headers, identifier) => withPlatformView(headers, organizationId, (executor) => platformViewOf(executor).pipe( - Effect.flatMap((admin) => - getAdminUser(admin, identifier, userDirectory(auth, headers)), - ), + Effect.flatMap((admin) => getAdminUser(admin, identifier, directory)), ), ).pipe(Effect.provideContext(context)), }); @@ -193,6 +131,9 @@ export const betterAuthAdminUsersProvider: Layer.Layer< export interface SelfHostAdminUsersApiDeps { readonly betterAuth: BetterAuthHandle; + /** The boot-built `MemberDirectory` (see `resolveAuthProviders`), so this + * plane reads the same directory instance every other plane does. */ + readonly memberDirectory: Layer.Layer; readonly db: SelfHostDbHandle; readonly mountPrefix: `/${string}`; } @@ -206,6 +147,7 @@ export interface SelfHostAdminUsersApiDeps { */ export const makeSelfHostAdminUsersApiLayer = ({ betterAuth, + memberDirectory, db, mountPrefix, }: SelfHostAdminUsersApiDeps) => { @@ -214,6 +156,7 @@ export const makeSelfHostAdminUsersApiLayer = ({ ); const provider = betterAuthAdminUsersProvider.pipe( Layer.provide(Layer.succeed(BetterAuth)(betterAuth)), + Layer.provide(memberDirectory), Layer.provide(SelfHostDbProvider), Layer.provide(SelfHostPluginsProvider), Layer.provide(SelfHostHostConfig), diff --git a/apps/host-selfhost/src/app.ts b/apps/host-selfhost/src/app.ts index a2341702a8..18bcdf9fc2 100644 --- a/apps/host-selfhost/src/app.ts +++ b/apps/host-selfhost/src/app.ts @@ -73,7 +73,8 @@ export const makeSelfHostApp = async (options: MakeSelfHostAppOptions = {}) => { // ---- auth providers --------------------------------------------------- // Better Auth: cookie/bearer/api-key identity + /api/auth handler + account // API + MCP OAuth seam, all over the shared libSQL handle. - const { identityLayer, authHandler, betterAuth } = await resolveAuthProviders(dbHandle); + const { identityLayer, memberDirectoryLayer, authHandler, betterAuth } = + await resolveAuthProviders(dbHandle); // ---- the in-process MCP serving seams (+ shutdown hook) ---------------- const mcp = makeSelfHostMcpSeams(dbHandle, betterAuth, config); @@ -130,7 +131,12 @@ export const makeSelfHostApp = async (options: MakeSelfHostAppOptions = {}) => { // Tenant-wide admin users API (/api/admin/users*): the owner's view of // who uses this instance and what they've connected. Owner/admin-gated, // same as the invite routes above. - makeSelfHostAdminUsersApiLayer({ betterAuth, db: dbHandle, mountPrefix: "/api" }), + makeSelfHostAdminUsersApiLayer({ + betterAuth, + memberDirectory: memberDirectoryLayer, + db: dbHandle, + mountPrefix: "/api", + }), // Public system API: /api/health + /api/setup-status (unauthenticated). makeSelfHostSystemApiLayer({ betterAuth, db: dbHandle, mountPrefix: "/api" }), // Swagger UI at /docs, over the /api-prefixed spec (matches the served paths). @@ -141,11 +147,14 @@ export const makeSelfHostApp = async (options: MakeSelfHostAppOptions = {}) => { // The boot-scoped context provideMerge'd under everything: the long-lived DB // handle (read by the DbProvider seam, Better Auth, and the MCP store) + the // resolved identity (captured once by the execution middleware + MCP auth) + // + the member directory (the shared membership read seam, boot-scoped + // beside identity because Better Auth's handle is an app singleton) // + the artifact-usage observer (this HTTP plane is the console UI's data // layer, so operations it serves file as `via: "ui"`). boot: Layer.mergeAll( Layer.succeed(SelfHostDb)(dbHandle), identityLayer, + memberDirectoryLayer, Layer.succeed(ArtifactUsageObserver)((action) => selfHostAnalytics.record(`artifact_${action}`, { via: "ui" }), ), diff --git a/apps/host-selfhost/src/auth/index.ts b/apps/host-selfhost/src/auth/index.ts index bf9a4b5839..1005970c21 100644 --- a/apps/host-selfhost/src/auth/index.ts +++ b/apps/host-selfhost/src/auth/index.ts @@ -1,24 +1,28 @@ import { Layer } from "effect"; -import { IdentityProvider } from "@executor-js/api/server"; +import { IdentityProvider, MemberDirectory } from "@executor-js/api/server"; import { loadConfig } from "../config"; import type { SelfHostDbHandle } from "../db/self-host-db"; import { BetterAuth, buildBetterAuth, type BetterAuthHandle } from "./better-auth"; import { betterAuthIdentityLayer } from "./identity"; +import { betterAuthMemberDirectoryLayer } from "./member-directory"; import { consentRedirectClientId, withClientName, withForcedMcpConsent } from "./force-mcp-consent"; import { rewriteInvalidOrigin } from "./invalid-origin-help"; export { BetterAuth, buildBetterAuth, type BetterAuthHandle } from "./better-auth"; export { betterAuthIdentityLayer } from "./identity"; +export { betterAuthMemberDirectoryLayer } from "./member-directory"; // --------------------------------------------------------------------------- // Resolve the self-host auth providers. // // Build the Better Auth instance over the shared libSQL file, expose its -// `IdentityProvider` (cookie/bearer/api-key) and its web handler (mounted at -// /api/auth/*). Returns the live `BetterAuthHandle` so the composition root can -// build the account API and the Better Auth MCP OAuth seam. +// `IdentityProvider` (cookie/bearer/api-key), its `MemberDirectory` (the +// shared membership read seam over the org plugin's tables) and its web +// handler (mounted at /api/auth/*). Returns the live `BetterAuthHandle` so the +// composition root can build the account API and the Better Auth MCP OAuth +// seam. // // This is the one and only production auth path. Tests that need a fake identity // (single-admin / header-driven) compose `ExecutorApp.make` directly through @@ -29,6 +33,8 @@ export { betterAuthIdentityLayer } from "./identity"; export interface ResolvedAuthProviders { /** The resolved Better Auth `IdentityProvider` seam (cookie/bearer/api-key). */ readonly identityLayer: Layer.Layer; + /** The resolved Better Auth `MemberDirectory` seam (org members + users). */ + readonly memberDirectoryLayer: Layer.Layer; /** Better Auth's web handler (`/api/auth/*`). */ readonly authHandler: (request: Request) => Promise; /** The live Better Auth handle (account API + Better Auth MCP OAuth seam). */ @@ -78,6 +84,7 @@ export const resolveAuthProviders = async ( return { identityLayer: betterAuthIdentityLayer.pipe(Layer.provide(betterAuthLayer)), + memberDirectoryLayer: betterAuthMemberDirectoryLayer.pipe(Layer.provide(betterAuthLayer)), authHandler, betterAuth, }; diff --git a/packages/core/api/src/account/api.ts b/packages/core/api/src/account/api.ts index 01581b783a..87ad648b9e 100644 --- a/packages/core/api/src/account/api.ts +++ b/packages/core/api/src/account/api.ts @@ -109,10 +109,17 @@ export const OrgApiKeysResponse = Schema.Struct({ apiKeys: Schema.Array(ApiKeySummary), }); +/** + * One member of the caller's organization, as the host's member directory + * reports them. `email` is nullable: a host can hold a membership whose + * profile it has not yet learned (cloud mirrors the membership before the + * user record lands), and reporting `""` for that would let the UI render an + * empty address as if it were one. + */ export const OrgMember = Schema.Struct({ id: Schema.String, userId: Schema.String, - email: Schema.String, + email: Schema.NullOr(Schema.String), name: Schema.NullOr(Schema.String), avatarUrl: Schema.NullOr(Schema.String), role: Schema.String, diff --git a/packages/core/api/src/admin/admin-users.test.ts b/packages/core/api/src/admin/admin-users.test.ts index aec566c295..b2c73c8e7f 100644 --- a/packages/core/api/src/admin/admin-users.test.ts +++ b/packages/core/api/src/admin/admin-users.test.ts @@ -395,6 +395,7 @@ const A1_EMAIL = "a1@users.test"; const stubUserDirectory = (options: { readonly seen?: string[][]; readonly resolved?: string[]; + readonly searched?: string[]; }): AdminUserDirectory => ({ identities: (externalIds) => { options.seen?.push([...externalIds]); @@ -405,6 +406,13 @@ const stubUserDirectory = (options: { // Compares a NORMALIZED stored value, the rule both real hosts follow. return Effect.succeed(A1_EMAIL_STORED.toLowerCase() === email ? USER_A1 : null); }, + search: (term) => { + options.searched?.push(term); + // The one member the directory knows, matched on the normalized email or + // the display name — the substring rule both real hosts apply. + const haystack = [A1_EMAIL_STORED.toLowerCase(), "user a1"]; + return Effect.succeed(haystack.some((value) => value.includes(term)) ? [USER_A1] : []); + }, }); /** The failure a host's directory raises — WorkOS or Better Auth being @@ -1022,6 +1030,102 @@ describe("admin users API", () => { ), ); + // ── ?search= ────────────────────────────────────────────────────────────── + + it.effect("filters the bulk lists by a name or email substring, case-insensitively", () => + withDb((db) => + Effect.gen(function* () { + yield* seed(db); + const searched: string[] = []; + const web = yield* webHandlerFor( + stubProvider((tenant) => platformExecutorFor(db, tenant), headerAuthorize, { + ...stubUserDirectory({ searched }), + }), + ); + + // Part of the address, typed in the wrong case and with stray spaces: + // the handler normalizes it before the directory sees it. + const byEmail = yield* jsonOf( + yield* get(web, `/admin/users?search=${encodeURIComponent(" A1@USERS ")}`, ORG_A), + ); + expect(byEmail.users.map((user) => user.externalId)).toEqual([USER_A1]); + expect(byEmail.users[0]?.email, "the page still carries identity").toBe(A1_EMAIL_STORED); + + // Part of the name, on the joined view. + const byName = yield* jsonOf( + yield* get(web, "/admin/users/with-connections?search=User%20a1", ORG_A), + ); + expect(byName.users.map((user) => user.externalId)).toEqual([USER_A1]); + expect(byName.users[0]?.connections.map((c) => c.integration)).toEqual(["github"]); + + // No match is an empty page, never the unfiltered tenant. + const nobody = yield* jsonOf( + yield* get(web, "/admin/users?search=nobody", ORG_A), + ); + expect(nobody.users).toEqual([]); + + expect(searched, "one directory search per request, normalized").toEqual([ + "a1@users", + "user a1", + "nobody", + ]); + }), + ), + ); + + it.effect("a blank search is no filter at all", () => + withDb((db) => + Effect.gen(function* () { + yield* seed(db); + const searched: string[] = []; + const web = yield* webHandlerFor( + stubProvider( + (tenant) => platformExecutorFor(db, tenant), + headerAuthorize, + stubUserDirectory({ searched }), + ), + ); + + const body = yield* jsonOf(yield* get(web, "/admin/users?search=%20%20", ORG_A)); + expect(body.users.map((user) => user.externalId)).toEqual([USER_A1, USER_A2]); + expect(searched, "the directory is never asked to match whitespace").toEqual([]); + }), + ), + ); + + it.effect("returns an empty page for a search no host directory can answer", () => + withDb((db) => + Effect.gen(function* () { + yield* seed(db); + // A directory with identities only: it cannot search, so a search + // filter must select nothing rather than hand back the whole tenant. + const web = yield* webHandlerFor( + stubProvider((tenant) => platformExecutorFor(db, tenant), headerAuthorize, { + identities: stubDirectory([]), + }), + ); + + const body = yield* jsonOf(yield* get(web, "/admin/users?search=a1", ORG_A)); + expect(body.users).toEqual([]); + }), + ), + ); + + it.effect("500s when the directory search fails, rather than reporting no match", () => + withDb((db) => + Effect.gen(function* () { + yield* seed(db); + const web = yield* webHandlerFor( + stubProvider((tenant) => platformExecutorFor(db, tenant), headerAuthorize, { + search: () => Effect.fail(new DirectoryUnavailable({ message: "down" })), + }), + ); + + expect((yield* get(web, "/admin/users?search=a1", ORG_A)).status).toBe(500); + }), + ), + ); + // A resolver OUTAGE must not read as "no such user": that is a wrong answer an // operator would act on. Contrast with the identity join, which degrades to // unnamed rows precisely because it is decoration. @@ -1096,8 +1200,8 @@ const A_SUBJECT: AdminSubject = { /** An `ExecutorAdmin` that answers everything and records the reads it was * asked for, so a test can assert the call the filter chose. */ const recordingAdmin = (calls: string[]): ExecutorAdmin => ({ - listSubjects: () => { - calls.push("listSubjects"); + listSubjects: (options) => { + calls.push(`listSubjects:${options?.externalIds?.join(",") ?? "*"}`); return Effect.succeed([A_SUBJECT]); }, getSubject: () => { @@ -1108,8 +1212,8 @@ const recordingAdmin = (calls: string[]): ExecutorAdmin => ({ calls.push("listSubjectConnections"); return Effect.succeed([]); }, - listSubjectsWithConnections: () => { - calls.push("listSubjectsWithConnections"); + listSubjectsWithConnections: (options) => { + calls.push(`listSubjectsWithConnections:${options?.externalIds?.join(",") ?? "*"}`); return Effect.succeed([{ ...A_SUBJECT, connections: [] }]); }, getSubjectWithConnections: () => { @@ -1187,7 +1291,56 @@ describe("admin users reads — the ?email= filter is applied before the read", const calls: string[] = []; yield* listUsersWithConnections(recordingAdmin(calls), { limit: 50 }, stubUserDirectory({})); - expect(calls).toEqual(["listSubjectsWithConnections"]); + expect(calls).toEqual(["listSubjectsWithConnections:*"]); + }), + ); +}); + +// --------------------------------------------------------------------------- +// `?search=` is FILTER-THEN-PAGE through storage: the directory names the +// matching principals, and the paged read carries exactly that set as its +// `externalIds` filter — never a page scan that is filtered afterwards. +// --------------------------------------------------------------------------- + +describe("admin users reads — the ?search= filter pages the directory's matches", () => { + it.effect("hands the matched ids to the paged read, on both views", () => + Effect.gen(function* () { + const calls: string[] = []; + const admin = recordingAdmin(calls); + + yield* listUsers(admin, { search: "a1" }, stubUserDirectory({})); + yield* listUsersWithConnections(admin, { search: "user", limit: 10 }, stubUserDirectory({})); + + expect(calls).toEqual([`listSubjects:${USER_A1}`, `listSubjectsWithConnections:${USER_A1}`]); + }), + ); + + it.effect("issues NO storage read when the directory matches nobody", () => + Effect.gen(function* () { + const calls: string[] = []; + const body = yield* listUsersWithConnections( + recordingAdmin(calls), + { search: "nobody" }, + stubUserDirectory({}), + ); + + expect(calls).toEqual([]); + expect(body.users).toEqual([]); + }), + ); + + it.effect("lets an exact email win over a search term", () => + Effect.gen(function* () { + const calls: string[] = []; + const searched: string[] = []; + yield* listUsers( + recordingAdmin(calls), + { email: A1_EMAIL, search: "anything" }, + stubUserDirectory({ searched }), + ); + + expect(calls, "the keyed read, not a search").toEqual(["getSubject"]); + expect(searched).toEqual([]); }), ); }); diff --git a/packages/core/api/src/admin/api.ts b/packages/core/api/src/admin/api.ts index 69e76db94d..acda949904 100644 --- a/packages/core/api/src/admin/api.ts +++ b/packages/core/api/src/admin/api.ts @@ -261,6 +261,15 @@ const AdminUserIdentifierParams = { identifier: Schema.String }; // handler seam (`normalizeEmail`), which is also where the single-user path // parameter is normalized, so both entry points share ONE rule rather than a // schema transform on one and hand-rolled code on the other. +// +// `search` is the SUBSTRING counterpart: a case-insensitive match over each +// member's email and name in the host's directory, for the operator who knows +// a person's name or part of an address rather than the exact one. Like +// `email` it narrows the fixed list shape and is applied BEFORE paging (the +// directory names the matching principals; storage pages that set), so a +// window on a searched list is a window on the matches. A blank term is no +// filter. When both filters are present `email` wins: it names one principal, +// and there is nothing left for a search to narrow. const AdminListQuery = Schema.Struct({ limit: Schema.optional( Schema.FiniteFromString.check(Schema.isInt(), Schema.isBetween({ minimum: 1, maximum: 500 })), @@ -272,6 +281,7 @@ const AdminListQuery = Schema.Struct({ ), ), email: Schema.optional(Schema.String), + search: Schema.optional(Schema.String), }); // --------------------------------------------------------------------------- diff --git a/packages/core/api/src/admin/handlers.ts b/packages/core/api/src/admin/handlers.ts index f5d6ccc8b9..88625b7551 100644 --- a/packages/core/api/src/admin/handlers.ts +++ b/packages/core/api/src/admin/handlers.ts @@ -2,6 +2,7 @@ import { HttpApiBuilder } from "effect/unstable/httpapi"; import { HttpServerRequest } from "effect/unstable/http"; import { Effect } from "effect"; +import { normalizeMemberSearch } from "../server/member-directory"; import { AdminUsersHttpApi } from "./api"; import { normalizeEmail } from "./reads"; import { AdminUsersProvider, type AdminUsersHeaders, type AdminUsersListOptions } from "./service"; @@ -24,15 +25,23 @@ const requestHeaders = Effect.map( // than an explicit `undefined` overriding them. // `email` is normalized here rather than in the contract schema, so the filter // and the single-user path parameter share ONE rule (`normalizeEmail`). +// `search` gets the directory's own rule (`normalizeMemberSearch`: the same +// trim + lower-case, and a blank term is no filter at all — dropped here so a +// provider never sees `search: ""`). const listOptions = (query: { readonly limit?: number | undefined; readonly offset?: number | undefined; readonly email?: string | undefined; -}): AdminUsersListOptions => ({ - ...(query.limit === undefined ? {} : { limit: query.limit }), - ...(query.offset === undefined ? {} : { offset: query.offset }), - ...(query.email === undefined ? {} : { email: normalizeEmail(query.email) }), -}); + readonly search?: string | undefined; +}): AdminUsersListOptions => { + const search = normalizeMemberSearch(query.search); + return { + ...(query.limit === undefined ? {} : { limit: query.limit }), + ...(query.offset === undefined ? {} : { offset: query.offset }), + ...(query.email === undefined ? {} : { email: normalizeEmail(query.email) }), + ...(search === undefined ? {} : { search }), + }; +}; export const AdminUsersHandlers = HttpApiBuilder.group( AdminUsersHttpApi, diff --git a/packages/core/api/src/admin/member-directory.ts b/packages/core/api/src/admin/member-directory.ts index e90ad89346..df7c9beb37 100644 --- a/packages/core/api/src/admin/member-directory.ts +++ b/packages/core/api/src/admin/member-directory.ts @@ -10,21 +10,26 @@ import { MemberStatus, type MemberDirectoryShape } from "../server/member-direct import type { AdminUserDirectory, AdminUserIdentity } from "./reads"; /** - * Both directions of the admin plane's directory over one org's + * Every direction of the admin plane's directory over one org's * {@link MemberDirectoryShape}. * * `identities` is one batched `membersById` read for the page of ids (never a * lookup per user); a member the org does not hold reports absent identity. * `resolveEmail` receives the already-normalized email the contract promises * and answers with the host principal id, or `null` when no member has it. + * `search` is one `members` read for the term, answering with the matching + * principal ids in directory order. * - * Both read ANY membership status, not the directory's active + pending - * default: this plane reports footprint, not current access. A member who was - * removed while their connections remain must still be named on the users - * page and findable by the address an operator has for them. + * Every direction reads ANY membership status — the same reach `membersById` + * and `findByEmail` have by contract, and `search` asks for explicitly rather + * than taking `members`' active + pending default. This plane reports + * footprint, not current access: a member who was deactivated while their + * connections remain must still be findable by the address or name an + * operator has for them, exactly as `?email=` already finds them. * - * Both fail with `MemberDirectoryError`, which the shared reads treat as a - * decorative-join outage (identities) or surface as a failed read (resolve). + * All fail with `MemberDirectoryError`, which the shared reads treat as a + * decorative-join outage (identities) or surface as a failed read (resolve, + * search). */ export const adminUserDirectoryFromMembers = ( directory: MemberDirectoryShape, @@ -47,4 +52,8 @@ export const adminUserDirectoryFromMembers = ( directory .findByEmail(organizationId, email, MemberStatus.literals) .pipe(Effect.map((member) => (member === null ? null : member.accountId))), + search: (term) => + directory + .members(organizationId, { search: term, statuses: MemberStatus.literals }) + .pipe(Effect.map((members) => members.map((member) => member.accountId))), }); diff --git a/packages/core/api/src/admin/reads.ts b/packages/core/api/src/admin/reads.ts index 96b6f2bbdc..6110686268 100644 --- a/packages/core/api/src/admin/reads.ts +++ b/packages/core/api/src/admin/reads.ts @@ -16,6 +16,7 @@ import { Effect } from "effect"; import type { AdminConnection, + AdminListSubjectsOptions, AdminSubject, AdminSubjectWithConnections, Executor, @@ -114,12 +115,27 @@ export type AdminIdentityDirectory = ( */ export type AdminEmailResolver = (email: string) => Effect.Effect; -/** Both directions of a host's member directory. Optional as a whole (a host - * with no directory reports unnamed rows and cannot resolve emails), and - * optional per direction. */ +/** + * The directory's SEARCH: a normalized term (trimmed + lower-cased, the same + * rule `normalizeEmail` applies) → the host-auth principal ids of every member + * whose email or name contains it, in the directory's own order. + * + * Unlike `resolveEmail` this names a SET, and the reads page that set through + * storage rather than in memory: the ids go into the SDK's `externalIds` + * filter and the caller's `limit`/`offset` apply there. An empty result means + * no member matches, and costs no storage read. Failures are the caller's to + * interpret on the same terms as `resolveEmail` — a search that cannot run + * must not quietly become "nobody matches". + */ +export type AdminMemberSearch = (term: string) => Effect.Effect; + +/** Every direction of a host's member directory. Optional as a whole (a host + * with no directory reports unnamed rows and cannot resolve emails or search), + * and optional per direction. */ export interface AdminUserDirectory { readonly identities?: AdminIdentityDirectory; readonly resolveEmail?: AdminEmailResolver; + readonly search?: AdminMemberSearch; } /** Identity is decoration on an operator view, not part of the answer: a @@ -292,6 +308,72 @@ const selectByEmail = ( return row === null ? [] : pageOf([row], options); }); +/** + * The `?search=` read: FILTER by the directory, then PAGE through storage. + * + * The term names a SET of principals rather than one, so unlike `?email=` it + * cannot become a keyed read — but it still must not become a page-then-filter + * scan, which on a large tenant would page past every unmatched subject before + * finding the first match. So the directory answers with the matching ids and + * storage pages exactly that set (`externalIds` + the caller's window), which + * keeps "filter, then page" as the one paging rule every filtered list here + * follows. + * + * A host with no search direction answers nothing, for the same reason an + * unanswerable `?email=` does: a filter no host can apply must return an empty + * page, never an unfiltered one. A search FAILURE is a 500 on the same terms as + * a resolver failure. + */ +const selectBySearch = ( + directory: AdminUserDirectory, + term: string, + read: (externalIds: readonly string[]) => Effect.Effect, +): Effect.Effect => + Effect.gen(function* () { + const search = directory.search; + if (!search) return []; + const wanted = yield* search(term).pipe( + Effect.mapError(() => new AdminUsersError({ message: "Failed to search the directory" })), + ); + // Nobody matches: an empty page, and no storage read for an `in ()` that + // could not match anyway. + if (wanted.length === 0) return []; + return yield* read(wanted); + }); + +/** + * Which filtered read a list request takes. `email` names ONE principal and + * wins when both are present: a keyed read is the more specific answer, and + * a search term beside an exact address has nothing left to narrow. + */ +const selectSubjects = ( + directory: AdminUserDirectory, + options: AdminUsersListOptions, + reads: { + readonly page: ( + paging: AdminListSubjectsOptions, + ) => Effect.Effect; + readonly one: (externalId: string) => Effect.Effect; + }, +): Effect.Effect => { + if (options.email !== undefined) { + return selectByEmail(directory, options.email, options, reads.one); + } + if (options.search !== undefined) { + return selectBySearch(directory, options.search, (externalIds) => + reads.page({ ...pagingOf(options), externalIds }), + ); + } + return reads.page(pagingOf(options)); +}; + +/** Only the paging window — never the filters — reaches the SDK: the filters + * are resolved here, and the SDK's own `externalIds` is set by this file. */ +const pagingOf = (options: AdminUsersListOptions): AdminListSubjectsOptions => ({ + ...(options.limit === undefined ? {} : { limit: options.limit }), + ...(options.offset === undefined ? {} : { offset: options.offset }), +}); + export const listUsers = ( admin: ExecutorAdmin, options: AdminUsersListOptions, @@ -299,12 +381,10 @@ export const listUsers = ( ): Effect.Effect => Effect.gen(function* () { const dir = asDirectory(directory); - const subjects = - options.email === undefined - ? yield* admin.listSubjects(options).pipe(Effect.mapError(readFailed("users"))) - : yield* selectByEmail(dir, options.email, options, (externalId) => - admin.getSubject(externalId).pipe(Effect.mapError(readFailed("users"))), - ); + const subjects = yield* selectSubjects(dir, options, { + page: (paging) => admin.listSubjects(paging).pipe(Effect.mapError(readFailed("users"))), + one: (externalId) => admin.getSubject(externalId).pipe(Effect.mapError(readFailed("users"))), + }); // One directory read for the page that was actually returned, joined in // memory — never a lookup per user. const identities = yield* resolveIdentities( @@ -321,14 +401,12 @@ export const listUsersWithConnections = ( ): Effect.Effect => Effect.gen(function* () { const dir = asDirectory(directory); - const subjects = - options.email === undefined - ? yield* admin - .listSubjectsWithConnections(options) - .pipe(Effect.mapError(readFailed("users"))) - : yield* selectByEmail(dir, options.email, options, (externalId) => - admin.getSubjectWithConnections(externalId).pipe(Effect.mapError(readFailed("users"))), - ); + const subjects = yield* selectSubjects(dir, options, { + page: (paging) => + admin.listSubjectsWithConnections(paging).pipe(Effect.mapError(readFailed("users"))), + one: (externalId) => + admin.getSubjectWithConnections(externalId).pipe(Effect.mapError(readFailed("users"))), + }); const identities = yield* resolveIdentities( dir.identities, subjects.map((subject) => subject.externalId), diff --git a/packages/core/api/src/admin/service.ts b/packages/core/api/src/admin/service.ts index d314e12d3b..97bfac3854 100644 --- a/packages/core/api/src/admin/service.ts +++ b/packages/core/api/src/admin/service.ts @@ -31,12 +31,15 @@ import { export type AdminUsersHeaders = Record; /** Paging and filtering, mirroring the SDK's `AdminListSubjectsOptions` plus - * the contract's `?email=`. The email arrives already trimmed and lower-cased - * by the contract schema, so a provider never re-normalizes it. */ + * the contract's `?email=` and `?search=`. Both filters arrive already + * trimmed and lower-cased by the handler seam (a blank search is omitted + * entirely), so a provider never re-normalizes them. `email` names ONE + * principal and wins when both are present. */ export interface AdminUsersListOptions { readonly limit?: number; readonly offset?: number; readonly email?: string; + readonly search?: string; } type User = typeof AdminUserResponse.Type; diff --git a/packages/core/api/src/server.ts b/packages/core/api/src/server.ts index b5974de98d..bba228e10f 100644 --- a/packages/core/api/src/server.ts +++ b/packages/core/api/src/server.ts @@ -40,6 +40,7 @@ export { normalizeEmail as normalizeAdminUserEmail, type AdminEmailResolver, type AdminIdentityDirectory, + type AdminMemberSearch, type AdminUserDirectory, type AdminUserIdentity, } from "./admin/reads"; diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index b01c62dbc7..cd89ab9075 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -593,6 +593,14 @@ export interface AdminSubjectWithConnections extends AdminSubject { export interface AdminListSubjectsOptions { readonly limit?: number; readonly offset?: number; + /** + * Keep only subjects whose `external_id` is in this set — the host's answer + * to a directory search (name or email), paged through storage rather than + * in memory. An EMPTY set matches nothing; `undefined` is no filter. Paging + * applies to the filtered set: "filter, then page", the same order the + * `?email=` read follows. + */ + readonly externalIds?: readonly string[]; } /** @@ -7006,8 +7014,18 @@ export const createExecutor = b("external_id", "in", [...externalIds]) }), // Oldest first, ties broken on the unique key so the order is // total and paging can't repeat or skip a row. orderBy: [ diff --git a/packages/core/sdk/src/platform-view.test.ts b/packages/core/sdk/src/platform-view.test.ts index 12cee13c8b..9c5ef821f3 100644 --- a/packages/core/sdk/src/platform-view.test.ts +++ b/packages/core/sdk/src/platform-view.test.ts @@ -424,6 +424,61 @@ const expectWriteRefused = ( Effect.orDie, ); +describe("platform view — admin.listSubjects externalIds filter", () => { + it.effect("keeps only the named ids, still ordered and paged through storage", () => + withDb((db) => + Effect.gen(function* () { + yield* seed(db); + const admin = yield* requireAdmin(yield* makePlatformExecutor(db)); + + const only = yield* admin.listSubjects({ externalIds: [SUBJECT_B] }); + expect(only.map((entry) => entry.externalId)).toEqual([SUBJECT_B]); + + // Ids the tenant does not hold are simply absent — including another + // tenant's subject, which the policy keeps out regardless of the filter. + const mixed = yield* admin.listSubjects({ + externalIds: [SUBJECT_B, "user_nobody", "user_elsewhere", SUBJECT_A], + }); + expect(mixed.map((entry) => entry.externalId).sort()).toEqual([SUBJECT_A, SUBJECT_B]); + + // "Filter, then page": the window applies to the filtered set. + const all = yield* admin.listSubjects(); + const second = yield* admin.listSubjects({ + externalIds: [SUBJECT_A, SUBJECT_B], + limit: 1, + offset: 1, + }); + expect(second.map((entry) => entry.externalId)).toEqual([all[1]?.externalId]); + }), + ), + ); + + it.effect("an empty id set matches nothing, on both list reads", () => + withDb((db) => + Effect.gen(function* () { + yield* seed(db); + const admin = yield* requireAdmin(yield* makePlatformExecutor(db)); + + expect(yield* admin.listSubjects({ externalIds: [] })).toEqual([]); + expect(yield* admin.listSubjectsWithConnections({ externalIds: [] })).toEqual([]); + }), + ), + ); + + it.effect("the joined read filters the same way and still joins connections", () => + withDb((db) => + Effect.gen(function* () { + yield* seed(db); + const admin = yield* requireAdmin(yield* makePlatformExecutor(db)); + + const rows = yield* admin.listSubjectsWithConnections({ externalIds: [SUBJECT_A] }); + expect(rows.map((entry) => entry.externalId)).toEqual([SUBJECT_A]); + expect(rows[0]?.connections.length).toBeGreaterThan(0); + }), + ), + ); +}); + describe("platform view — read-only across every surface", () => { it.effect("refuses org-row writes through policies and oauth", () => withDb((db) => diff --git a/packages/react/src/api/admin-atoms.tsx b/packages/react/src/api/admin-atoms.tsx index e72bac7812..3f97740edf 100644 --- a/packages/react/src/api/admin-atoms.tsx +++ b/packages/react/src/api/admin-atoms.tsx @@ -10,10 +10,11 @@ import { ReactivityKey } from "./reactivity-keys"; // rejects writes at tenant reach), so there are no mutations here and every // atom carries the same reactivity key. // -// Paging is part of the atom identity, so each page is its own cache entry and -// stepping back to a visited page is instant. `Atom.family` (not a bare arrow) -// because the page component re-derives the key object on every render — a -// fresh atom per render would refetch in a loop. +// Paging and the search term are part of the atom identity, so each page of +// each search is its own cache entry and stepping back to a visited page is +// instant. `Atom.family` (not a bare arrow) because the page component +// re-derives the key object on every render — a fresh atom per render would +// refetch in a loop. // --------------------------------------------------------------------------- /** How many users one page of the list shows. Well inside the contract's @@ -24,6 +25,10 @@ export const ADMIN_USERS_PAGE_SIZE = 25; export interface AdminUsersPage { readonly limit: number; readonly offset: number; + /** The `?search=` term (name or email substring), already debounced by the + * page. `""` is no filter and is sent as no param at all, so the unfiltered + * list keeps one cache identity regardless of how the term was cleared. */ + readonly search: string; } /** @@ -35,7 +40,11 @@ export interface AdminUsersPage { */ export const adminUsersWithConnectionsAtom = Atom.family((page: AdminUsersPage) => AdminApiClient.query("adminUsers", "listUsersWithConnections", { - query: { limit: page.limit + 1, offset: page.offset }, + query: { + limit: page.limit + 1, + offset: page.offset, + ...(page.search === "" ? {} : { search: page.search }), + }, timeToLive: "30 seconds", reactivityKeys: [ReactivityKey.adminUsers], }), diff --git a/packages/react/src/pages/admin-users.tsx b/packages/react/src/pages/admin-users.tsx index 49fae31d68..d081606325 100644 --- a/packages/react/src/pages/admin-users.tsx +++ b/packages/react/src/pages/admin-users.tsx @@ -1,6 +1,7 @@ -import { useState } from "react"; +import { useEffect, useState } from "react"; import { useAtomRefresh, useAtomValue } from "@effect/atom-react"; import { useParams } from "@tanstack/react-router"; +import { SearchIcon, XIcon } from "lucide-react"; import * as AsyncResult from "effect/unstable/reactivity/AsyncResult"; import * as Cause from "effect/Cause"; import * as Option from "effect/Option"; @@ -18,6 +19,7 @@ import { ownerLabel } from "../api/owner-display"; import { Button } from "../components/button"; import { CopyButton } from "../components/copy-button"; import { ErrorState } from "../components/error-state"; +import { Input } from "../components/input"; import { IntegrationFavicon, integrationInferredUrl, @@ -521,14 +523,94 @@ function UserDetail(props: { }); } +// ── Search ────────────────────────────────────────────────────────────────── + +/** How long the typed term settles before it becomes a request. Long enough + * that a typed name is one query rather than one per keystroke, short enough + * to read as immediate. */ +const SEARCH_DEBOUNCE_MS = 250; + +/** + * The search box: what is typed, and the settled term the list actually asks + * for. Two values because the request is debounced, and the input must keep + * echoing keystrokes while the term catches up. Clearing bypasses the debounce + * — an emptied box should show everyone at once, not after a pause. + */ +const useDebouncedSearch = (): { + readonly typed: string; + readonly term: string; + readonly setTyped: (value: string) => void; + readonly clear: () => void; +} => { + const [typed, setTypedState] = useState(""); + const [term, setTerm] = useState(""); + + useEffect(() => { + if (typed === term) return; + const handle = setTimeout(() => setTerm(typed), SEARCH_DEBOUNCE_MS); + return () => clearTimeout(handle); + }, [typed, term]); + + return { + typed, + term, + setTyped: setTypedState, + clear: () => { + setTypedState(""); + setTerm(""); + }, + }; +}; + +function UserSearch(props: { + readonly value: string; + readonly onChange: (value: string) => void; + readonly onClear: () => void; +}) { + return ( +
+ + props.onChange((event.target as HTMLInputElement).value)} + onKeyDown={(event) => { + if (event.key === "Escape" && props.value !== "") props.onClear(); + }} + placeholder="Search by name or email" + aria-label="Search users by name or email" + className="h-9 pl-9 pr-9 text-sm [&::-webkit-search-cancel-button]:hidden" + /> + {props.value !== "" && ( + + )} +
+ ); +} + // ── Page ──────────────────────────────────────────────────────────────────── export function AdminUsersPage() { useExecutorDocumentTitle("Users"); const [offset, setOffset] = useState(0); const [selected, setSelected] = useState(null); + const search = useDebouncedSearch(); - const page = { limit: ADMIN_USERS_PAGE_SIZE, offset }; + // A new term is a new list, so it starts on its first page: an offset kept + // from a broader list would land past the end of a narrower one. + const page = { limit: ADMIN_USERS_PAGE_SIZE, offset, search: search.term }; const result = useAtomValue(adminUsersWithConnectionsAtom(page)); const refresh = useAtomRefresh(adminUsersWithConnectionsAtom(page)); const catalog = useCatalogRows(); @@ -555,10 +637,24 @@ export function AdminUsersPage() { ); + const searching = search.term !== ""; + return ( {header} + { + search.setTyped(value); + setOffset(0); + }} + onClear={() => { + search.clear(); + setOffset(0); + }} + /> + {isAsyncResultLoading(result) ? loading : AsyncResult.match(result, { @@ -572,6 +668,25 @@ export function AdminUsersPage() { onSuccess: ({ value }) => { const { rows, hasNext } = splitPage(value.users, ADMIN_USERS_PAGE_SIZE); + if (rows.length === 0 && searching && offset === 0) { + return ( +
+

No users match

+

+ Nobody in this workspace has a name or email containing “ + {search.term}”. Only people who have reached the workspace or connected + an account are listed. +

+ +
+ ); + } + if (rows.length === 0) { return (
diff --git a/packages/react/src/pages/org.tsx b/packages/react/src/pages/org.tsx index 69e7a5aad0..b23ec30cf0 100644 --- a/packages/react/src/pages/org.tsx +++ b/packages/react/src/pages/org.tsx @@ -69,7 +69,7 @@ import { isAsyncResultLoading } from "../lib/async-result"; type MemberData = { id: string; - email: string; + email: string | null; name: string | null; avatarUrl: string | null; role: string; @@ -80,6 +80,23 @@ type MemberData = { type RoleData = { slug: string; name: string }; +/** What a member row is called: name, else email, else the one thing every + * member has — a membership id — so a profile the host has not learned yet + * still renders as a row an admin can act on. */ +const memberLabel = (member: MemberData): string => member.name ?? member.email ?? member.id; + +const memberInitials = (member: MemberData): string => { + if (member.name) { + return member.name + .split(" ") + .map((n: string) => n[0]) + .join("") + .slice(0, 2) + .toUpperCase(); + } + return (member.email?.[0] ?? "?").toUpperCase(); +}; + type InviteState = { email: string; roleSlug: string; @@ -314,7 +331,7 @@ export function OrgPage(props: { const filtered = search ? members.filter( (m: MemberData) => - m.email.toLowerCase().includes(search.toLowerCase()) || + (m.email?.toLowerCase().includes(search.toLowerCase()) ?? false) || (m.name?.toLowerCase().includes(search.toLowerCase()) ?? false), ) : members; @@ -338,21 +355,14 @@ export function OrgPage(props: { ) : (
- {member.name - ? member.name - .split(" ") - .map((n: string) => n[0]) - .join("") - .slice(0, 2) - .toUpperCase() - : member.email[0]!.toUpperCase()} + {memberInitials(member)}
)}

- {member.name ?? member.email} + {memberLabel(member)}

{member.isCurrentUser && ( You @@ -361,7 +371,7 @@ export function OrgPage(props: { Invited )}
- {member.name && ( + {member.name && member.email && (

{member.email}

@@ -421,7 +431,7 @@ export function OrgPage(props: { onClick={() => setRemovingMember({ id: member.id, - name: member.name ?? member.email, + name: memberLabel(member), }) } >