diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 31452734..13d7658e 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,4 +1,4 @@ -# This workflow is used to release a new version of @devrev/ts-adaas package to +# This workflow is used to release a new version of @devrev/airsync-sdk package to # npm registry and generate release notes using softprops/action-gh-release # action. It consists of two jobs: # @@ -93,7 +93,7 @@ jobs: id: version run: | # Get the latest version including prereleases - LATEST_VERSION=$(npm view @devrev/ts-adaas versions --json 2>/dev/null | jq -r '.[-1]' || echo "0.0.0") + LATEST_VERSION=$(npm view @devrev/airsync-sdk versions --json 2>/dev/null | jq -r '.[-1]' || echo "0.0.0") echo "Latest published version: $LATEST_VERSION" echo "LATEST_PUBLISHED_VERSION=$LATEST_VERSION" >> $GITHUB_ENV diff --git a/MIGRATION.md b/MIGRATION.md new file mode 100644 index 00000000..f3a00e31 --- /dev/null +++ b/MIGRATION.md @@ -0,0 +1,1028 @@ +# Migrating connectors from v1 (`@devrev/ts-adaas`) to v2 (`@devrev/airsync-sdk`) + +Covers every connector-facing breaking change between `@devrev/ts-adaas` v1.x +and `@devrev/airsync-sdk` v2.0.0, with before/after examples from real +connectors. Self-contained: the numbered sections, applied **in order** (see +execution plan), are the complete migration — executable top to bottom without +any other reference. It ships in the npm package, so after installing v2 the +version-matched copy is at `node_modules/@devrev/airsync-sdk/MIGRATION.md`. + +**The wire protocol is unchanged.** Event payloads, API routes, headers, the +artifact/upload flow, and every surviving event-type **string value** are +byte-for-byte identical to v1. Only the connector-facing **TypeScript API** +changed. + +> **Why a hard break with no deprecation shims?** v2 is a single bundled-pain +> major: the package was renamed, so there are no existing consumers to keep +> backward-compatible. Everything changed at once in one guide. + +--- + +## TL;DR — what breaks + +| # | Change | Who it affects | +|---|--------|----------------| +| 1 | Package renamed `@devrev/ts-adaas` → `@devrev/airsync-sdk` | every import | +| 2 | `AirdropEvent` → `AirSyncEvent`, `AirdropMessage` → `AirSyncMessage` | type annotations (all connectors) | +| 3 | `processTask` → `processExtractionTask` / `processLoadingTask` | every worker file | +| 4 | `adapter.emit(...)` is gone — tasks **return** a `TaskResult` instead | every worker file | +| 5 | `WorkerAdapter` **class** removed → `ExtractionAdapter` / `LoadingAdapter` | helper signatures | +| 6 | `loadItemTypes` / `loadAttachments` / `streamAttachments` now **return** a `TaskResult` | loading + attachment workers | +| 7 | `EventData.external_sync_units` **removed** — external sync units must be pushed to a repo | ESU workers | +| 8 | `adapter.state` is connector-state only; SDK fields readable via `adapter.sdkState`. `lastSyncStarted` / `lastSuccessfulSyncStarted` **removed** | connectors reading/writing SDK bookkeeping | +| 9 | `axios`, `axiosClient`, `formatAxiosError`, `serializeAxiosError`, `HTTPResponse` no longer exported | anyone importing the SDK's HTTP/axios surface | +| 10 | `Mappers` methods return the unwrapped body (`Promise`), not `Promise>` | loaders calling `adapter.mappers.*` | +| 11 | Deprecated v1 modules deleted (`Adapter`, `createAdapter`, `DemoExtractor`, `HTTPClient`, `defaultResponse`, deprecated `Uploader`) | only legacy code | +| 12 | Deprecated event-type enum **members** deleted; deprecated types/enums (`ExtractionMode`, `EventContextIn`/`Out`, `DomainObjectState`, `ErrorLevel`, `LogRecord`, `AdapterUpdateParams`, `AdapterState`) **removed**; internal worker-IPC types (`WorkerEvent`, `WorkerMessageSubject`, `WorkerMessage*`, `WorkerData`, `GetWorkerPathInterface`) off the root barrel | only if you used the old members/types or the internal IPC types | +| 13 | Several deep `dist/**` import paths moved/removed | deep-importers | +| 14 | Legacy `string[]` attachment-dedup migration dropped | in-flight attachment syncs started on SDK **< 1.15.2** | +| 15 | `spawn`'s deprecated `workerPath` option **removed** — use `baseWorkerPath: __dirname` | connectors still passing `workerPath` | +| 16 | Jest mocks of the SDK hardcode the v1 shape; emit-called assertions become return-value assertions | every test suite mocking the SDK | + +(Also: `EventData.progress` was removed — a no-op since v1; the backend computes +progress. Covered with the other `EventData` removals in §12.) + +(Also: emitted logs no longer carry the `is_sdk_log` field — an +observability-only change that breaks no connector code; see §12.) + +What does **not** change: `spawn(...)` and its surviving options (`initialState`, +`initialDomainMapping`, `workerPathOverrides`, `baseWorkerPath`, +`options.batchSize`, `timeout`, `isLocalDevelopment`, …) — the one exception is +the long-deprecated `workerPath`, removed (§15); default worker paths +(`/workers/data-extraction`, etc.); repos (`initializeRepos`, `getRepo`, +`push`); the normalization interfaces (`NormalizedItem`, `NormalizedAttachment`, +`RepoInterface`, `ExternalSyncUnit`); `installInitialDomainMapping`; +`createMockEvent` / `MockServer`; HTTP retry behavior; +`event_context.extract_from` / `extract_to`; and every surviving event-type +string value on the wire. + +### Execution plan (for migrating a whole connector in one pass) + +Apply sections **in numbered order** — later ones assume earlier ones ran (e.g. +§3–§6 rewrite emit sites that §5's adapter types must already annotate). For each +section: search the whole connector — source **and** tests — apply every hit, +then move on. Verify at the end, not per-section (a half-applied section never +compiles). Two search rules: + +- **After §1, the old specifier is gone.** Every later search must target the new + `@devrev/airsync-sdk` specifier or a bare symbol name (`\bWorkerAdapter\b`, + `\baxiosClient\b`, `\.emit\(`, `lastSuccessfulSyncStarted`, …) — searching + `@devrev/ts-adaas` after §1 finds nothing and makes a section look like a false + no-op. +- **A section with zero hits is a verified no-op** — confirm and move on; many + connectors already use `baseWorkerPath`, push ESUs to a repo, and have no deep + imports. +- **Don't narrate the migration in code.** The `// v1` / `// v2` labels here are + reading aids — never copy them into the connector. Apply each edit cleanly, no + comment explaining what changed. Add a comment only where the resulting logic + is genuinely non-obvious independent of this migration. + +Final verification, from the connector's package dir: `npm install` (regenerates +the lockfile), `npx tsc --noEmit`, `npm run lint` (unused imports/locals orphaned +by rewrites fail here — clean them up), `npm run build`, and the repo's test +command if tests exist. A leftover `adapter.emit`, a removed symbol, a missing +`delaySeconds`, a destructured `{ reports }`, or an un-renamed test fixture is +the usual failure cause. + +--- + +## 1. Package rename + +```bash +npm uninstall @devrev/ts-adaas +npm install @devrev/airsync-sdk@beta +``` + +> Until GA ships, install via the **`beta` dist-tag** — it always resolves to the +> newest published 2.x beta. A plain `npm install @devrev/airsync-sdk` may resolve +> an older beta, and `@2.0.0` does not exist yet and 404s. Once GA is published, +> plain install (`@latest`) is correct. + +> **Start from the latest stable v1.** This guide describes the delta from the +> latest stable `@devrev/ts-adaas` (1.x). If your connector is on an older 1.x +> that skipped an intra-v1 migration, do that first so the examples match your +> code. Check **https://github.com/devrev/adaas-sdk/releases** for any release +> between your version and latest whose notes call out a migration or breaking +> change (the last such were **1.18.0**, **1.17.0**, **1.16.0**; **1.19.0 → +> 1.20.0** were migration-free). Bring the connector to a green latest-stable v1 +> build before starting. + +Then global-replace the import specifier **everywhere the string appears**: every +import, every `jest.mock('...')` / `jest.requireActual('...')` first argument, and +any jest `moduleNameMapper`. This also rewrites deep-import path prefixes +(`@devrev/ts-adaas/dist/x` → `@devrev/airsync-sdk/dist/x`); §13 handles the paths +that moved. Do not rename any symbols in this step. + +```ts +// v1 +import { spawn, EventType } from '@devrev/ts-adaas'; +// v2 +import { spawn, EventType } from '@devrev/airsync-sdk'; +``` + +> **Deep imports** like `@devrev/ts-adaas/dist/...` are fragile — several paths +> moved or were removed in v2 (§13). Prefer root imports; the v2 barrel now exports +> several symbols that previously required a deep import (`Mappers`, `Item`, +> `ItemTypeToLoad`). + +## 2. Type renames: `AirdropEvent` → `AirSyncEvent` + +Hard rename, no compatibility alias. The payload **shape** is identical — only the +type name changed. + +| v1 | v2 | +|----|----| +| `AirdropEvent` | `AirSyncEvent` | +| `AirdropMessage` | `AirSyncMessage` | + +E.g. `(events: AirdropEvent[])` → `(events: AirSyncEvent[])`. + +No other public type was renamed — `ConnectionData`, `EventContext`, `EventData`, +`ExtractorEvent`, and `ExternalSyncUnit` keep their v1 names. Platform-owned +strings (`/internal/airdrop.*` routes, the `'ADaaS'` external system type, the +`adaas_library_version` metadata key, `airdrop_*` mapping enum values) are +intentionally unchanged. + +### `AirSyncEvent.context` gained identity fields + +`AirSyncEvent.context` now declares the identity fields the platform already +sends, in addition to `secrets`, `snap_in_id`, `snap_in_version_id`: + +```ts +// v2 — AirSyncEvent.context +context: { + secrets: { service_account_token: string }; + snap_in_version_id: string; + snap_in_id: string; + user_id: string; // new + dev_oid: string; // new + source_id: string; // new + service_account_id: string; // new +}; +``` + +If you previously extended the event to read these (e.g. a hand-rolled +`CustomAirdropEvent` that added `user_id`), drop the extension and read +`adapter.event.context.user_id` directly. (`snap_in_id` was already in v1; only +the four fields above are new.) + +> Note: these four fields live on the **top-level** `AirSyncEvent.context`, not on +> the `EventContext` inside `payload.event_context` (a different, unchanged +> object). + +## 3 + 4. The new worker contract: **return a `TaskResult`** instead of emitting + +The core change of v2. In v1 the connector decided *which event* to emit and +called `adapter.emit(...)`. In v2 it only reports *how the phase ended* by +**returning** a `TaskResult`; the SDK maps it to the correct platform event for +the current phase and emits it exactly once. + +```ts +// the exact union (exported from @devrev/airsync-sdk) +export type TaskResult = + | { status: 'success' } + | { status: 'progress' } + | { status: 'delay'; delaySeconds: number } // note: delaySeconds, not delay + | { status: 'error'; error: ErrorRecord }; // ErrorRecord = { message: string } +``` + +`processTask` is split into two typed entry points — pick the one matching the +worker's phase. Phase is per-file: workers under `extraction/` (or whose body uses +`initializeRepos`/`getRepo`/`streamAttachments`) are extraction; workers under +`loading/` (or using `loadItemTypes`/`loadAttachments`/`mappers`) are loading. + +### Before (v1) + +```ts +import { processTask, ExtractorEventType } from '@devrev/ts-adaas'; + +processTask({ + task: async ({ adapter }) => { + // ... extract ... + await adapter.emit(ExtractorEventType.DataExtractionDone); + }, + onTimeout: async ({ adapter }) => { + await adapter.postState(); + await adapter.emit(ExtractorEventType.DataExtractionProgress, { progress: 50 }); + }, +}); +``` + +### After (v2) + +```ts +import { processExtractionTask } from '@devrev/airsync-sdk'; + +processExtractionTask({ + task: async ({ adapter }) => { + // ... extract ... + return { status: 'success' }; + }, + // onTimeout can be omitted entirely for resumable phases: + // the SDK emits a progress (continuation) result by default. +}); +``` + +Loading workers use `processLoadingTask` the same way. + +> `adapter.emit(...)` is now **`protected`** — calling `adapter.emit(...)` or +> `this.adapter.emit(...)` is a hard **compile error** in v2, not a deprecation. +> Every emit site must be converted. + +### `emit()` → `return` translation table + +| v1 emit call | v2 return | +|--------------|-----------| +| `await adapter.emit(XxxDone)` | `return { status: 'success' }` | +| `await adapter.emit(XxxProgress, { progress })` | `return { status: 'progress' }` | +| `await adapter.emit(XxxDelayed, { delay })` | `return { status: 'delay', delaySeconds: delay }` | +| `await adapter.emit(XxxError, { error })` | `return { status: 'error', error }` | +| `await adapter.emit(ExternalSyncUnitExtractionDone, { external_sync_units })` | push to repo + `return { status: 'success' }` — see §7 | + +`progress` (`{ progress: n }`) carried no semantic value in v1 beyond "not done +yet" and is dropped; the platform tracks progress itself. + +> **A `Done` emit that carried a non-fatal error summary** (v1 +> `emit(XxxDone, { error })` — "finished, but here's what went wrong") maps to +> `return { status: 'success' }` plus surfacing the error some other way (a report +> entry or `console.warn`) — **not** to `{ status: 'error' }`, which would emit +> `*Error` and flip a successful phase to failed. + +> **`ProcessTaskInterface` / `TaskAdapterInterface` changed their generic.** Both +> survive, but the type parameter now names the **adapter**, not the connector +> state: v1 `ProcessTaskInterface` / `TaskAdapterInterface` → v2 +> `ProcessTaskInterface>` (or `LoadingAdapter`), +> same for `TaskAdapterInterface`. A bare `` compiles in loosely-typed +> spots but silently types `adapter` as the connector state. `onTimeout` is now +> optional, and both callbacks return `Promise` not `Promise`. + +### Status → emitted event, per phase + +The SDK picks the platform event from the **current phase** and the returned +status: + +| status | Resumable phases — data/attachment **extraction**, data/attachment **loading** | Non-resumable — external sync units, metadata, state deletion | +|--------|------------------------------------------------------------------------------|---------------------------------------------------------------| +| `'success'` | `*Done` | `*Done` | +| `'progress'` | `*Progress` (continuation) | **`*Error`** (illegal for these phases; emitted with a generated message) | +| `'delay'` | `*Delayed` (with `delaySeconds`) | **`*Error`** (illegal) | +| `'error'` | `*Error` (with the error record) | `*Error` | + +### Emits buried inside helper functions + +A common v1 pattern: emit deep inside a helper, return a boolean telling the +caller to stop. Since only the **task's return value** reaches the SDK in v2, the +helper must **bubble the outcome up**. + +A clean way (used by the migrated google-drive connector): store the terminal +result on a field and return it once the loop unwinds: + +```ts +// v1 — helper emits, returns false to abort +private async handlePermissionDeniedError(error: unknown) { + await this.adapter.emit(ExtractorEventType.DataExtractionError, { + error: { message: '...' }, + }); +} +``` + +```ts +// v2 — helper records the result; the task returns it +private result: TaskResult | undefined; + +private handlePermissionDeniedError(error: unknown) { + this.result = { status: 'error', error: { message: '...' } }; +} + +// ... and where the loop ends: +private finalResult(): TaskResult { + // a stored error/delay set by a helper, else progress (continuation) + return this.result ?? { status: 'progress' }; +} +``` + +For a simple "stop iterating" signal, return the `TaskResult` directly up the call +chain: + +```ts +// v2 +async function extractList(adapter: ExtractionAdapter): Promise { + if (rateLimited) return { status: 'delay', delaySeconds: retryAfter }; + // ... + return null; // keep going +} + +processExtractionTask({ + task: async ({ adapter }) => { + for (const itemType of itemTypes) { + const stop = await extractList(adapter); + if (stop) return stop; + } + return { status: 'success' }; + }, +}); +``` + +### Timeout handling + +Checking `adapter.isTimeout` in your extraction loop still works as in v1 — but +instead of emitting progress and exiting, **return** progress: + +```ts +if (adapter.isTimeout) { + return { status: 'progress' }; // platform sends CONTINUE_* next +} +``` + +Two behaviors matter when migrating: + +- **The timeout outcome always wins.** Once the soft timeout fires, the SDK emits + the `onTimeout` result (or its default) and **ignores whatever the task + returned** — a phase that ran out of time must hand off for continuation, not + report itself complete. +- **Omit `onTimeout` unless it does real work.** Its default is phase-aware: + `progress` (continuation) for resumable phases, a timeout **error** for + non-resumable ones (external sync units, metadata, state deletion) where + continuation is impossible. So a v1 `onTimeout` that only emitted the + phase-appropriate event (optionally after `postState()`) is now redundant — + **delete it**. Keep an explicit `onTimeout` only when its v1 body did real work + that must survive a timeout (cancelling rate limiting, clearing upload + bookkeeping, a custom error message) — preserve that body, replacing the + trailing emit with a returned `TaskResult`: + + ```ts + onTimeout: async () => ({ + status: 'error', + error: { message: 'Custom timeout message.' }, + }), + ``` + +> Do **not** call `process.exit()` yourself in v2 — the SDK owns the single worker +> exit after it emits your `TaskResult`. Likewise drop `adapter.postState()` inside +> `onTimeout`; the SDK persists state around the timeout emit. + +## 5. `WorkerAdapter` → `ExtractionAdapter` / `LoadingAdapter` + +The `WorkerAdapter` **class** is gone. Replace the type annotation on your helpers +with the mode-specific adapter: + +```ts +// v1 +async function extractList(adapter: WorkerAdapter) { ... } +// v2 +import { ExtractionAdapter } from '@devrev/airsync-sdk'; +async function extractList(adapter: ExtractionAdapter) { ... } +``` + +| Surface | Lives on | +|---------|----------| +| `initializeRepos`, `getRepo`, `streamAttachments`, `shouldExtract`, `artifacts` | `ExtractionAdapter` | +| `loadItemTypes`, `loadAttachments`, `mappers`, `reports`, `processedFiles` | `LoadingAdapter` | +| `event`, `state`, `sdkState`, `postState`, `isTimeout`, `extractionScope` | both (shared `BaseAdapter`) | + +> Only the **class** was removed. The **types** `WorkerAdapterInterface` and +> `WorkerAdapterOptions` still exist and are still exported — don't blindly rename +> every `WorkerAdapter` token; replace the `WorkerAdapter` annotations and +> `new WorkerAdapter(...)` constructions only. + +### Hand-constructed adapters (integration tests) + +Some connectors construct the adapter directly in integration tests +(`new WorkerAdapter({ event, adapterState })`). The same construction works with +the phase-specific classes — but `adapterState` must be a v2 state class, which +replaced the v1 `State` class / `createAdapterState` factory (both removed; §13 +for the import path): + +```ts +// v1 +const adapterState = new State({ event, initialState }); +const adapter = new WorkerAdapter({ event, adapterState }); + +// v2 — same shape, phase-specific classes +import { LoadingState } from '@devrev/airsync-sdk/dist/state/state'; +const adapterState = new LoadingState({ event, initialState }); +const adapter = new LoadingAdapter({ event, adapterState }); +``` + +Sync `new State(...)` maps to sync `new ExtractionState(...)` / +`new LoadingState(...)` (same `{ event, initialState, initialDomainMapping?, +options? }` params). The async `createAdapterState(...)` (fetched persisted state) +maps to the async `createExtractionState(...)` / `createLoadingState(...)` +factories at the same import path. A test that poked SDK fields via +`adapterState.state = { fromDevRev: ... }` must move them onto the `sdkState` +envelope (§8). + +> **`mappers` / `reports` / `processedFiles` moved to `LoadingAdapter` only.** In +> v1 they were on the single `WorkerAdapter`, reachable in any phase. In v2 they +> are not on `ExtractionAdapter`. Code that touched `adapter.mappers` (etc.) +> during an extraction phase must move to the loading path. + +## 6. Loading & attachment methods return a `TaskResult` + +`loadItemTypes`, `loadAttachments`, and `streamAttachments` no longer emit or exit +mid-flight — they **return** a `TaskResult` you pass straight through. Rate limits +(→ `delay`), timeouts (→ `progress`), errors (→ `error`), completion (→ `success`) +are all encoded in the result; `reports` / `processed_files` (loading) and +artifacts (extraction) are attached to the emitted event automatically. + +### Loading — before (v1) + +```ts +import { LoaderEventType, processTask } from '@devrev/ts-adaas'; + +processTask({ + task: async ({ adapter }) => { + await adapter.loadItemTypes({ itemTypesToLoad }); + await adapter.emit(LoaderEventType.DataLoadingDone); + }, + onTimeout: async ({ adapter }) => { + await adapter.postState(); + await adapter.emit(LoaderEventType.DataLoadingProgress); + }, +}); +``` + +### Loading — after (v2) + +```ts +import { processLoadingTask } from '@devrev/airsync-sdk'; + +processLoadingTask({ + task: async ({ adapter }) => { + return adapter.loadItemTypes({ itemTypesToLoad }); + }, +}); +``` + +### Attachment streaming — before (v1) + +```ts +const response = await adapter.streamAttachments({ stream: getFileStream, batchSize: 50 }); +if (response?.delay) { + await adapter.emit(ExtractorEventType.AttachmentExtractionDelayed, { delay: response.delay }); +} else if (response?.error) { + await adapter.emit(ExtractorEventType.AttachmentExtractionError, { error: response.error }); +} else { + await adapter.emit(ExtractorEventType.AttachmentExtractionDone); +} +``` + +### Attachment streaming — after (v2) + +```ts +return adapter.streamAttachments({ stream: getFileStream, batchSize: 50 }); +``` + +> **Never destructure the returned `TaskResult`.** The union has **no** `reports` / +> `processed_files` members, so v1 code like +> `const { reports, processed_files } = await adapter.loadAttachments(...)` — or +> pushing a synthetic report into the returned array — is a compile error. A +> connector that augmented the reports pushes onto the **live getter before** +> returning instead: +> +> ```ts +> adapter.reports.push(buildNotesReport(...)); +> return await adapter.loadAttachments({ create }); +> ``` +> +> A defensive outer try/catch with bespoke rate-limit handling can stay: +> `return await adapter.loadItemTypes(...)` in the `try`, map escaped throws to +> `{ status: 'delay' | 'error' }` in the `catch`. + +Custom attachment processors (reducer/iterator) keep the same call signatures; +only their `adapter` parameter type changes from `WorkerAdapter` to +`ExtractionAdapter` (§5). The `getAttachmentStream` function you implement still +returns `{ httpStream }` / `{ delay }` / `{ error }` — but see §9 for the +`httpStream` type change. + +## 7. External sync units go through a repo + +In v1 the SDK accepted `external_sync_units` in emit data and uploaded them +internally. With emit gone, push them to the `EXTERNAL_SYNC_UNITS` repo yourself. +The `EventData.external_sync_units` field — deprecated in v1 — is **removed +entirely** in v2: there is no inline ESU path, and any code still referencing +`external_sync_units` in emit data is now a compile error. External sync units +leave the worker only as repo artifacts. + +### Before (v1) + +```ts +await adapter.emit(ExtractorEventType.ExternalSyncUnitExtractionDone, { + external_sync_units: externalSyncUnits, +}); +``` + +### After (v2) + +```ts +import { AirSyncDefaultItemTypes, processExtractionTask } from '@devrev/airsync-sdk'; + +adapter.initializeRepos([ + { + itemType: AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS, + // mirror the batching the v1 SDK used internally for ESUs + overridenOptions: { batchSize: 25000, skipConfirmation: true }, + }, +]); +await adapter.getRepo(AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS)?.push(externalSyncUnits); + +return { status: 'success' }; +``` + +The repo is uploaded automatically before the `Done` event is emitted. + +> If your connector already pushed ESUs to a repo in v1 (some do), this section is +> a no-op — just convert the trailing emit to `return`. + +## 8. State split: `adapter.state` vs `adapter.sdkState` + +In v1, connector state and SDK bookkeeping lived in one flat blob and +`adapter.state` exposed both. In v2: + +- **`adapter.state` is connector state only** — exactly the shape of the + `initialState` you pass to `spawn`. Getter and setter both survive, so in-place + reads/writes work unchanged: + + ```ts + // works identically in v1 and v2 + adapter.state[itemType].cursor = nextCursor; + adapter.state[itemType].complete = true; + ``` + +- **SDK bookkeeping** (`workersOldest`/`workersNewest`, `pendingWorkers*`, + `toDevRev`, `fromDevRev`, `snapInVersionId`) moved to **`adapter.sdkState`** + (read-only getter). +- On disk, state is persisted as a `{ connectorState, sdkState }` envelope. The SDK + **migrates a v1 flat blob automatically on first read** (recognized SDK keys + split into `sdkState`, the rest into `connectorState`), so in-flight syncs + survive the upgrade. + +> **`lastSyncStarted` / `lastSuccessfulSyncStarted` are gone.** Both were +> `@deprecated` in v1 and are **removed from `SdkState` in v2** — the SDK no longer +> sets, reads, or declares them. They were wall-clock sync-start timestamps; the +> SDK now resolves the incremental window from extraction-data boundaries instead +> (`workersOldest`/`workersNewest` and the resolved `extract_from`/`extract_to`). +> The two old key names are still recognized as SDK-owned by the v1-blob migration, +> so they route into `sdkState` (where they sit unused) rather than leaking into +> your connector state. + +### If your v1 connector mixed SDK fields into its own `State` interface + +Some connectors declared SDK-owned fields (e.g. `toDevRev`, +`lastSuccessfulSyncStarted`) on their hand-written `State` type and seeded them in +`getInitialState()`. Remove those — `toDevRev`/`fromDevRev`/`workers*` are +SDK-managed now, and `lastSyncStarted`/`lastSuccessfulSyncStarted` no longer exist +at all: + +```ts +// v1 — State interface mixing SDK fields (remove these) +export interface State { + // ... your fields ... + lastSyncStarted?: string; // removed in v2 — no replacement field + lastSuccessfulSyncStarted?: string; // removed in v2 — read workersNewest / extract_to instead + toDevRev?: ToDevRev; // also drop the dist/state/state.interfaces import +} +``` + +### Cursor decision rule: rename vs repoint + +For `lastSyncStarted` / `lastSuccessfulSyncStarted` specifically, what to do +depends on **whether the field is declared in your own `State` interface / +`getInitialState`** — getting this backwards compiles but silently breaks +incremental sync: + +- **Not declared** (only accessed loosely as `adapter.state.`, absent from + your interface): it was SDK-supplied via the removed v1 `AdapterState`, and + the access is now a hard type error. **Delete** a bare write; **repoint** a read + to `adapter.event.payload.event_context.extract_from` (the resolved window start + — usually already destructured nearby). There is no field to preserve, so do not + rename. +- **Declared** (your connector writes and reads it as its own cross-sync cursor, + often consuming `extract_from` separately): **rename** it off the reserved SDK + key (e.g. `lastSuccessfulSyncStarted` → `lastSuccessfulWindowStart`) throughout + the interface, `getInitialState`, all read/write sites, and test fixtures. The + rename is mandatory because the v1-blob auto-migration routes the reserved key + into `sdkState` — a connector-owned cursor left on that name would be stripped + from your state. Substituting `extract_from` here would destroy the cursor + mechanism. + +### Reading the incremental-sync window + +In v1 you read `lastSuccessfulSyncStarted` to decide an incremental cursor. That +field is gone (above). Read the window the SDK resolves for you instead: + +```ts +// preferred — the resolved window for this invocation, on the event context +const { extract_from, extract_to } = adapter.event.payload.event_context; + +// the persisted cross-sync high-water mark (committed at end of each cycle) +const lastExtractedTo = adapter.sdkState.workersNewest; +``` + +`extract_from`/`extract_to` are the per-invocation window the SDK computes from the +platform's `extraction_start_time`/`extraction_end_time`; +`workersOldest`/`workersNewest` are the persisted boundaries committed at the end +of a completed cycle — the true replacement for the old "last successful sync" +resume point. + +> **`AdapterState` is removed.** The deprecated flat +> `ConnectorState & SdkState` alias no longer exists. If you annotated anything +> `AdapterState`, drop the import — use your own connector `State` type for +> `adapter.state`, and read SDK fields from `adapter.sdkState` (or the event +> context). The v2 on-disk shape is the `AdapterStateEnvelope` +> (`{ connectorState, sdkState }`), which the SDK manages for you. + +> **Edge regression (old in-flight incremental syncs):** the SDK used to fall back +> to `lastSuccessfulSyncStarted` when resolving a `WORKERS_NEWEST` window on state +> predating `workersNewest` (SDK **< 1.17.1**). With the field removed, that +> fallback is gone: such a window now resolves to the unbounded epoch, so an +> incremental sync still mid-flight across the upgrade re-extracts from the +> beginning **once**. The next completed cycle commits `workersNewest` and the sync +> self-heals; the platform deduplicates downstream. New syncs and any sync whose +> first full cycle completed on ≥ 1.17.1 are unaffected. Drain in-flight +> incremental syncs before upgrading if the one-time re-extract matters. + +> **Edge case:** if your v1 connector state had a top-level key literally named +> `connectorState` or `sdkState`, the auto-migration mis-reads it (the envelope is +> detected by those key names). Rename such a key **before** upgrading. + +## 9. HTTP / axios surface removed from the SDK + +The SDK no longer re-exports any axios surface from its public entry point: + +| v1 export | v2 | +|-----------|----| +| `axios` (raw instance) | **removed** — import `axios` in your connector | +| `axiosClient` (retry-wrapped instance) | **removed from the public API** (still exists internally, but not exported and not at the old deep path) | +| `formatAxiosError` | **removed** (deleted from source) | +| `serializeAxiosError` | **removed from the public API** — use `serializeError` | +| `HTTPResponse` | **removed** | + +`axios` and `axios-retry` are still runtime **dependencies** of the SDK (so +they're available transitively), but you must construct your own client: + +### Before (v1) + +```ts +import { axios, axiosClient } from '@devrev/ts-adaas'; + +const res = await axiosClient.get(url, { responseType: 'stream' }); +``` + +### After (v2) + +```ts +import axios from 'axios'; +import axiosRetry from 'axios-retry'; + +const axiosClient = axios.create(); +axiosRetry(axiosClient, { retries: 5, retryDelay: axiosRetry.exponentialDelay }); + +const res = await axiosClient.get(url, { responseType: 'stream' }); +``` + +Add `axios` and `axios-retry` to your `package.json` **dependencies** if missing — +`axios-retry` in particular is usually not yet a direct dependency. The minimal +client above does not replicate three behaviors of v1's public `axiosClient`: +(1) its retry condition excluded 429s (letting your rate-limit handling see them); +(2) it stripped the `Authorization` header from logged errors after max retries; +(3) it used a **1000 ms** exponential-backoff base +(`exponentialDelay(retryCount, error, 1000)` → ~2s, 4s, 8s, 16s, 32s), whereas +`axios-retry`'s bare `exponentialDelay` defaults to a 100 ms factor (~0.2s … 3.2s) +— 10× more aggressive. To match v1's backoff, pass the base explicitly: + +```ts +axiosRetry(axiosClient, { + retries: 5, + retryDelay: (retryCount, error) => + axiosRetry.exponentialDelay(retryCount, error, 1000), +}); +``` + +Replicate any of these only if your connector relied on the SDK client's behavior. + +> If you previously deep-imported `@devrev/ts-adaas/dist/http/axios-client`, that +> path **no longer exists** (the file was renamed to `http/client` and is +> internal). Bring your own axios instance as above. + +For error logging, replace `formatAxiosError` / `serializeAxiosError` with the +exported `serializeError`: + +```ts +import { serializeError } from '@devrev/airsync-sdk'; +console.error(serializeError(error)); +``` + +> **`serializeAxiosError` is conditional on how you used the result.** +> `serializeError` returns a **string**; `serializeAxiosError` returned an +> **object**. Where the result was used as a string (logging, concatenation), swap +> to `serializeError`. Where it was **spread or property-accessed as an object** +> (`{ ...serializeAxiosError(e) }`, `serializeAxiosError(e).message`), swapping +> produces a TS2698 "spread types" error — instead keep the function via its deep +> path (it still exists internally and still returns the object): +> `import { serializeAxiosError } from '@devrev/airsync-sdk/dist/logger/logger';` + +### `httpStream` type changed + +The connector-implemented attachment-stream function returns +`ExternalSystemAttachmentStreamingResponse`, whose `httpStream` field changed from +axios's `AxiosResponse` to the new public `HttpStreamResponse` +(`{ data: any; headers: Record }`). An axios stream response still +satisfies it structurally, but if you annotated the stream with `AxiosResponse` +imported *from the SDK*, switch to `HttpStreamResponse` (or import `AxiosResponse` +from `axios` directly). + +> **`error.statusCode` dropped from the streaming response.** v1.20 briefly widened +> `ExternalSystemAttachmentStreamingResponse.error` to +> `ErrorRecord & { statusCode?: number }`; v2 narrows it back to a plain +> `ErrorRecord`. The SDK never read `statusCode`, so this only affects a +> `getAttachmentStream` that returned `{ error: { message, statusCode } }` — drop +> the extra property (TS excess-property checking will flag it). Encode retry +> timing via `{ delay }` instead. + +## 10. `Mappers` methods return the unwrapped body + +`Mappers.getByTargetId` / `getByExternalId` / `create` / `update` changed their +return type from `Promise>` to `Promise` — they now return the +response **body** directly. Drop the `.data` access: + +### Before (v1) + +```ts +const mapperResponse = await this.mappers.getByTargetId({ sync_unit, target }); +const resolvedId = mapperResponse.data.sync_mapper_record.external_ids[0]; +``` + +### After (v2) + +```ts +const mapperResponse = await this.mappers.getByTargetId({ sync_unit, target }); +const resolvedId = mapperResponse.sync_mapper_record.external_ids[0]; +``` + +A **silent** change for code that reads `.data` (fails to type-check, or reads +`undefined` if loosely typed). `Mappers` is now also exported from the package +root, so you no longer need to deep-import it from `dist/mappers/mappers`. + +Three places must change **together**, or the failure is silent at runtime: + +1. the read site (drop `.data`, above); +2. any hand-rolled structural param type that wrapped the body in `data?:`; +3. **mapper test doubles** — `jest.fn().mockResolvedValue({ data: { + sync_mapper_record: ... } })` must become `mockResolvedValue({ + sync_mapper_record: ... })`, or the migrated source reads `undefined` and the + test fails. These test files often import nothing from the SDK, so a + specifier-based search misses them — grep for `sync_mapper_record`. + +> `mappers` lives on `LoadingAdapter` only (§5). Extraction-phase code that used +> `adapter.mappers` must construct its own instance — +> `new Mappers({ event: adapter.event })` (hoist it out of per-item loops). + +## 11. Deleted legacy modules + +Everything under the v1 `deprecated/` tree is gone: + +| Removed | Replacement | +|---------|-------------| +| `Adapter`, `createAdapter` | `ExtractionAdapter` / `LoadingAdapter` + `processExtractionTask` / `processLoadingTask` | +| `DemoExtractor` | — (reference implementation only) | +| `HTTPClient`, `defaultResponse` | your own axios client (§9) | +| deprecated `Uploader` | repos (`initializeRepos` / `getRepo` / `push`) | + +(The SDK has an internal `Uploader` class, but it was never part of the public API +in either version — the *public* v1 `Uploader` was the deprecated one.) + +## 12. Deleted deprecated enum members, types & enums + +The old/new duplicate enum members were collapsed; only the modern names remain. +**The string values of the surviving members are byte-identical to v1**, so nothing +changes on the wire — only the TypeScript member names. + +> ⚠️ The **deleted** `EventType` / `ExtractorEventType` members carried +> *different, older* string values than their replacements (e.g. v1 +> `ExtractionDataStart = 'EXTRACTION_DATA_START'` vs the surviving +> `StartExtractingData = 'START_EXTRACTING_DATA'`). The modern members already +> existed in v1 with the modern values, so survivors are wire-compatible — but a +> deleted member and its replacement did **not** share a value. + +**`EventType` (incoming):** + +| Deleted (v1 deprecated) | Use instead | +|--------------------------|-------------| +| `ExtractionExternalSyncUnitsStart` | `StartExtractingExternalSyncUnits` | +| `ExtractionMetadataStart` | `StartExtractingMetadata` | +| `ExtractionDataStart` | `StartExtractingData` | +| `ExtractionDataContinue` | `ContinueExtractingData` | +| `ExtractionDataDelete` | `StartDeletingExtractorState` | +| `ExtractionAttachmentsStart` | `StartExtractingAttachments` | +| `ExtractionAttachmentsContinue` | `ContinueExtractingAttachments` | +| `ExtractionAttachmentsDelete` | `StartDeletingExtractorAttachmentsState` | + +**`ExtractorEventType` (outgoing):** the `Extraction*`-prefixed members +(`ExtractionDataDone`, `ExtractionDataDelay`, `ExtractionAttachmentsProgress`, …) +are deleted; use the `*Extraction*` members (`DataExtractionDone`, +`DataExtractionDelayed`, `AttachmentExtractionProgress`, …). In practice you'll +rarely reference `ExtractorEventType` at all in v2 — see §4. + +**`LoaderEventType`:** the duplicate members `DataLoadingDelay` and +`AttachmentsLoading*` (plural) are deleted; use `DataLoadingDelayed` and +`AttachmentLoading*` (singular). These duplicates shared their survivors' string +value, so removing them is a pure source-name change. + +> **No more incoming-event-type translation.** v1 shipped an `event-type-translation` +> module mapping legacy platform strings onto the modern enum members (and +> translating outgoing types). v2 removed it entirely and passes incoming +> `payload.event_type` through untouched. Any connector importing +> `translateIncomingEventType` / `translateOutgoingEventType` / +> `translateExtractorEventType` / `translateLoaderEventType` from the SDK will fail +> to compile — drop them; the platform sends modern strings. + +### Removed `UnknownEventType` members + +`UnknownEventType = 'UNKNOWN_EVENT_TYPE'` was declared on three enums in v1 +(`EventType`, `ExtractorEventType`, `LoaderEventType`). All three copies are +**removed** in v2; the SDK's "unrecognized event" sentinel is now an internal, +un-exported constant with the same string value. If you matched on any enum +member, compare against the raw `'UNKNOWN_EVENT_TYPE'` string instead (the wire +value is unchanged). + +### Removed deprecated types & enums + +These were `@deprecated` in v1 and are **deleted from the public API** in v2. None +are referenced by the modern worker contract; each row gives the replacement (or +"no replacement" where the concept is gone): + +| Removed | Replacement | +|---------|-------------| +| `ExtractionMode` (enum) | `SyncMode` (adds `LOADING` alongside `INITIAL`/`INCREMENTAL`) | +| `EventContextIn` (interface) | `EventContext` (the single, current event-context type) | +| `EventContextOut` (interface) | `EventContext` | +| `DomainObjectState` (interface) | — (no replacement; was an unused per-object state shape) | +| `ErrorLevel` (enum) | — (logger uses its own internal log level) | +| `LogRecord` (interface) | — (unused) | +| `AdapterUpdateParams` (interface) | — (unused) | +| `AdapterState` (type alias) | your connector `State` for `adapter.state`; `adapter.sdkState` for SDK fields (§8) | + +Also removed from `EventData`: the deprecated `external_sync_units` field (§7) and +the deprecated `progress` field (a no-op since v1 — the backend computes progress). +The `artifacts` field on `EventData` is **kept** — it is how the SDK attaches +uploaded repo artifacts (including external sync units) to the emitted event. + +### Worker-thread IPC types no longer on the root barrel + +The worker↔main-thread plumbing types are no longer exported from the package root. +They were never part of the connector-authoring surface — they describe the SDK's +internal `parentPort` message protocol — but v1's barrel re-exported them, so a +connector (usually a test simulating the protocol) could import them from +`@devrev/ts-adaas`. Now removed from the root: `WorkerEvent`, +`WorkerMessageSubject` (enums), `WorkerMessage`, `WorkerMessageEmitted`, +`WorkerMessageExit`, `WorkerMessageLog`, `WorkerMessageFailed`, `WorkerData`, +`GetWorkerPathInterface`. The declarations still exist internally (deep-importable +via `@devrev/airsync-sdk/dist/types/workers`), but treat them as SDK-internal — a +worker-protocol test should assert on the `TaskResult` your task returns (§4), not +on raw IPC messages. (`WorkerMessageLog` also dropped its `isSdkLog` field.) + +> **Emitted logs dropped the `is_sdk_log` field (observability only).** v1 tagged +> every log line with `is_sdk_log: true | false` (SDK vs connector origin), driven +> by an `AsyncLocalStorage` log-context layer that v2 removed. v2's log JSON no +> longer contains `is_sdk_log`. **No connector code changes** — not a compile or +> runtime break — but any platform dashboard, monitor, or saved query filtering +> logs on `@is_sdk_log` silently stops matching. Update those filters (drop the +> facet, or distinguish origin another way) after connectors upgrade. + +## 13. Deep-import paths that moved or broke + +The compiled `dist/` mirrors `src/` 1:1, so a deep import works only if the source +file still lives at the same relative path. Status of the paths real connectors +used: + +| Deep import | Status in v2 | +|-------------|--------------| +| `dist/http/axios-client` (`axiosClient`) | ❌ **broken** — file renamed to `http/client` and made internal. Bring your own axios (§9). | +| `dist/state/state` (`State`, `createAdapterState`) | ❌ **both symbols removed** — the module now exports `BaseState`, `ExtractionState`/`createExtractionState`, `LoadingState`/`createLoadingState`. Map sync `new State(...)` → sync `new ExtractionState(...)`/`new LoadingState(...)`; async `createAdapterState(...)` → the async `create*State(...)` factory (§5). | +| `dist/state/state.interfaces` (`ToDevRev`) | ⚠️ still resolves, but `ToDevRev` is SDK-internal now (§8) — **drop** the import, don't repoint it | +| `dist/mappers/mappers.interface` (singular) | ❌ **broken** — file renamed to `mappers.interfaces` (plural). The four `*Params` interfaces and the two `SyncMapperRecord*` enums are on the root barrel (prefer root); the four `*Response` interfaces, `SyncMapperRecord`, `SyncMapperRecordExternalVersion`, `UpdateSyncMapperRecordParams`, and `MappersFactoryInterface` are **not** — repoint those to `dist/mappers/mappers.interfaces`. | +| `dist/logger/logger` (`serializeAxiosError`) | ⚠️ resolves, symbol kept internally — see the conditional rule in §9 (keep the deep import for object use; root `serializeError` for string use) | +| `dist/repo/repo.interfaces` (`Item`) | ✅ resolves — but `Item` is now on the root barrel, prefer the root import | +| `dist/types/loading` (`ItemTypeToLoad`) | ✅ resolves — also now on the root barrel | +| `dist/mappers/mappers` (`Mappers`) | ✅ resolves — also now on the root barrel (and note the return-type change, §10) | +| `dist/types/extraction` (`InitialSyncScope`) | ✅ resolves — also on the root barrel | + +**Recommended:** replace all deep `dist/**` imports with root imports. If a symbol +you need isn't on the root barrel, request it rather than deep-importing. + +## 14. Edge regression: very old in-flight attachment syncs + +In v1 the SDK migrated the legacy `string[]` form of the processed-attachments +dedup list (`lastProcessedAttachmentsIdsList`) to the current `{ id, parent_id }[]` +form on read. v2 removed that conversion. + +The `string[]` form only exists in state written by SDK **< 1.15.2**. If an +attachment-extraction phase started on a pre-1.15.2 SDK and is **still mid-flight** +when the connector upgrades to v2, the v2 dedup check (`it.id === …`) won't match +the bare-string entries, so attachments already downloaded in that sync get +re-uploaded once. New syncs — and any sync started on ≥ 1.15.2 — are unaffected; +the platform deduplicates downstream, so the only cost is the wasted +re-download/upload on that one continuation. + +If this matters for your deployment, drain in-flight attachment syncs before +upgrading. + +## 15. `spawn`'s deprecated `workerPath` option removed + +`SpawnFactoryInterface.workerPath` was `@deprecated` in v1 and is **removed** in v2. +Point `spawn` at your worker directory with `baseWorkerPath: __dirname` — the SDK +resolves the per-event worker file from there (`workerPathOverrides` still works +for custom paths). + +### Before (v1) + +```ts +spawn({ event, initialState, workerPath: __dirname + '/workers/data-extraction' }); +``` + +### After (v2) + +```ts +spawn({ event, initialState, baseWorkerPath: __dirname }); +``` + +> **Indirected `workerPath`** (a variable fed by a `switch`/dispatcher helper like +> `getWorkerPerLoadingPhase(event)`): replace with `baseWorkerPath: __dirname` +> **and delete the now-dead dispatcher function**, its locals, and any imports it +> orphaned (`EventType`, …) — otherwise `noUnusedLocals`/lint fails. +> `workerPathOverrides` is unrelated and kept. + +## 16. Jest mocks & test assertions + +Tests are first-class migration targets — a source-only migration passes `tsc` but +fails the test gate. SDK-module mocks hardcode the v1 shape. + +**Mechanical changes per test file:** + +- The module specifier in `jest.mock('...')` / `jest.requireActual('...')` / + `moduleNameMapper` was already renamed in §1. In mock factories: `processTask` → + `processExtractionTask` / `processLoadingTask` (matching the worker under test, + §3), including the `X as jest.Mock` capture. +- Drop `WorkerAdapter: {}` and `axiosClient: {}` keys from mock factories; remove + vestigial `emit: jest.fn()` from mock adapters. +- `AirdropEvent` → `AirSyncEvent` in fixtures/annotations (§2). Keep + `jest.requireActual` only for enums/constants still referenced. +- Propagate every state-field rename and `AdapterState` → your-State-type + replacement into fixtures (§8) — fixtures are the most-missed target. + +**Semantic changes:** + +- `expect(adapter.emit).toHaveBeenCalledWith(EventType.X)` becomes an assertion on + the awaited **return** of the captured task/onTimeout function: + + ```ts + // v1 + await mockProcessTask.mock.calls[0][0].task({ adapter }); + expect(adapter.emit).toHaveBeenCalledWith(ExtractorEventType.DataExtractionDone); + // v2 + const result = await mockProcessTask.mock.calls[0][0].task({ adapter }); + expect(result).toEqual({ status: 'success' }); + ``` + + The same rewrite applies to tests that invoke a **helper directly** (the helper + now returns a `TaskResult` per §4) — these lack the `mockProcessTask` capture + shape and are easy to miss. +- Expected delay objects use `delaySeconds`, not `delay`. +- Pass-through loaders (§6): assert the task returns the + `loadItemTypes`/`streamAttachments` mock's value. +- Source that now does `new Mappers({ event })` (§10) needs a `Mappers` mock in the + factory; source that moved to `import axios from 'axios'` (§9) needs + `jest.mock('axios')` (plus `jest.mock('axios-retry')` if you built a retry + client) instead of mocking the SDK's axios. +- Unwrap mapper test doubles (§10). Hand-constructed adapters in integration tests + follow §5. + +--- + +## Migration checklist + +1. `npm uninstall @devrev/ts-adaas && npm install @devrev/airsync-sdk@beta`; replace the import specifier everywhere — imports, `jest.mock`/`jest.requireActual`, `moduleNameMapper` (§1). +2. Rename `AirdropEvent` → `AirSyncEvent`, `AirdropMessage` → `AirSyncMessage`. Drop any `CustomAirdropEvent` cast that only added `user_id`/`dev_oid`/`source_id`/`service_account_id` (§2). +3. Split workers: extraction files use `processExtractionTask`, loading files use `processLoadingTask` (§3). Rewrite `ProcessTaskInterface`/`TaskAdapterInterface` annotations to take the adapter type (§3–4). +4. Convert **every** `adapter.emit(...)` into a returned `TaskResult`; bubble outcomes up from helpers and class methods; `delay` → `delaySeconds` (§4). +5. Replace `WorkerAdapter` annotations with `ExtractionAdapter` / `LoadingAdapter`, and `new WorkerAdapter(...)`/`new State(...)`/`createAdapterState(...)` constructions per §5. Move any `mappers`/`reports`/`processedFiles` access into the loading path. +6. Pass the `TaskResult` straight through from `loadItemTypes` / `loadAttachments` / `streamAttachments`; a connector that augmented `reports` pushes onto `adapter.reports` before returning (§6). +7. ESU workers: push external sync units to the `EXTERNAL_SYNC_UNITS` repo; remove any `external_sync_units` (and `progress`) from emit data — those fields are gone from `EventData` (§7, §12). +8. Remove SDK-owned fields from your `State` interface and `getInitialState`; apply the cursor decision rule for `lastSyncStarted`/`lastSuccessfulSyncStarted` — rename a connector-declared cursor, repoint an SDK-supplied read to `event_context.extract_from` (§8). +9. Replace `axios` / `axiosClient` / `formatAxiosError` SDK imports with your own axios client + `serializeError`; apply the conditional `serializeAxiosError` rule (§9). +10. Drop `.data` from `adapter.mappers.*` result reads — source, hand-rolled types, and test doubles together (§10). +11. Remove usage of deleted legacy modules and event-type-translation helpers (§11, §12). +12. Replace deleted enum members with their modern names; drop any use of the removed deprecated types (`ExtractionMode`, `EventContextIn`/`Out`, `DomainObjectState`, `ErrorLevel`, `LogRecord`, `AdapterUpdateParams`, `AdapterState` — replace an `AdapterState` annotation with your own State type, don't just delete it) (§12). +13. Replace deep `dist/**` imports with root imports; drop the now-internal `ToDevRev` import (§13). +14. Drop each `onTimeout` that only emitted the phase-appropriate event — the SDK default covers it (progress for resumable phases, a timeout error for ESU / metadata / state-deletion). Keep one only when its v1 body did real work (cleanup, a custom error message), preserving the body (§4). +15. Replace `spawn({ workerPath })` with `spawn({ baseWorkerPath: __dirname })`; delete any dead worker-path dispatcher (§15). +16. Migrate jest mocks and assertions — emit-called assertions become return-value assertions (§16). + +## A note on the early betas + +The betas `2.0.0-beta.0` through `2.0.0-beta.3` still re-exported `axios` / +`axiosClient`; they were removed in `2.0.0-beta.4` (and stay removed in GA — see §9). +If you migrated a connector against any beta before `beta.4` and imported either +symbol from `@devrev/airsync-sdk`, it will fail to compile once you move to +`beta.4`/GA — apply §9. diff --git a/README.md b/README.md index 11f7b2c5..84898bf6 100644 --- a/README.md +++ b/README.md @@ -17,9 +17,5 @@ It provides features such as: ## Installation ```bash -npm install @devrev/ts-adaas +npm install @devrev/airsync-sdk ``` - -## Reference - -Please refer to the [REFERENCE.md](./REFERENCE.md) file for more information on the types, interfaces and functions used in the library. diff --git a/REFERENCE.md b/REFERENCE.md deleted file mode 100644 index b949a3b4..00000000 --- a/REFERENCE.md +++ /dev/null @@ -1,1337 +0,0 @@ -## Reference - -### `SdkState` interface - -Defines the base state structure used by the Airdrop SDK. - -'SdkState' is an internal member that is not exported. - -#### Properties - -- _lastSyncStarted_ - - Optional. A **string** representing the timestamp when the last sync operation started. **Deprecated** - use `extract_from` and `extract_to` from the event context instead, which are automatically resolved by the SDK from `extraction_start_time` and `extraction_end_time`. - -- _lastSuccessfulSyncStarted_ - - Optional. A **string** representing the timestamp when the last successful sync operation started. **Deprecated** - use `extract_from` and `extract_to` from the event context instead, which are automatically resolved by the SDK from `extraction_start_time` and `extraction_end_time`. - -- _pendingWorkersOldest_ - - Optional. A **string** representing the pending (not yet committed) oldest extraction boundary as an ISO 8601 timestamp. Set on `StartExtractingData`, reused across subsequent phases, cleared on `AttachmentExtractionDone`. - -- _pendingWorkersNewest_ - - Optional. A **string** representing the pending (not yet committed) newest extraction boundary as an ISO 8601 timestamp. Set on `StartExtractingData`, reused across subsequent phases, cleared on `AttachmentExtractionDone`. - -- _workersOldest_ - - Optional. A **string** representing the oldest point of extraction as an ISO 8601 timestamp. - -- _workersNewest_ - - Optional. A **string** representing the newest point of extraction as an ISO 8601 timestamp. - -- _toDevRev_ - - Optional. An object of type **ToDevRev** containing data to be sent to DevRev. - -- _fromDevRev_ - - Optional. An object of type **FromDevRev** containing data received from DevRev. - -- _snapInVersionId_ - - Optional. A **string** representing the snap-in version ID. - -### `AdapterState` type - -A generic type that combines snap-in-specific state with the SDK's base state. - -#### Usage - -```typescript -type AdapterState = ConnectorState & SdkState; -``` - -The `AdapterState` type extends a snap-in's state type with additional fields from `SdkState`, providing a complete state structure to share with Airdrop platform. - -### `ToDevRev` interface - -Provides additional information within the state that is available only during data synchronization to DevRev (extraction). - -#### Properties - -- _attachmentsMetadata_ - - _artifactIds_: An array of **strings** containing artifact IDs - - _lastProcessed_: A **number** which is the index of the last processed attachment from the array - - _lastProcessedAttachmentsIdsList_: Optional. An array of **ProcessedAttachment** objects for deduplication on the SDK side - -### `FromDevRev` interface - -Provides additional information within the state that is available only during data synchronization from DevRev to external system (loading). - -#### Properties - -- _filesToLoad_ - - An array of **FileToLoad** objects representing files that need to be loaded. - -### `StateInterface` interface - -Defines the configuration structure for initializing state of the worker adapter. - -#### Properties - -- _event_ - - Required. An object of type **AirdropEvent** that is received from the Airdrop platform. - -- _initialState_ - - Required. An object of type **ConnectorState** representing the initial state of the snap-in. - -- _initialDomainMapping_ - - Optional. An object of type **InitialDomainMapping** representing the initial domain mapping configuration. - -- _options_ - - Optional. An object of type **WorkerAdapterOptions** for configuring the worker adapter. - -### `NormalizedItem` interface - -Represents the standardized structure of an item after normalization. - -#### Properties - -- _id_ - - Required. A **string** that uniquely identifies the normalized item. - -- _created_date_ - - Required. A **string** representing the timestamp, formatted as RFC3339, when the item was created. - -- _modified_date_ - - Required. A **string** representing the timestamp, formatted as RFC3339, when the item was last modified. - -- _data_ - - Required. An **object** containing the actual data of the normalized item. - -### `NormalizedAttachment` interface - -Represents the standardized structure of an attachment after normalization in the Airdrop platform. This interface defines the essential properties needed to identify and link attachments to their parent items. - -#### Properties - -- _url_ - - Required. A **string** representing the URL where the attachment can be accessed. - -- _id_ - - Required. A **string** that uniquely identifies the normalized attachment. - -- _file_name_ - - Required. A **string** representing the name of the attachment file. - -- _parent_id_ - - Required. A **string** identifying the parent item this attachment belongs to. - -- _author_id_ - - Optional. A **string** identifying the author or creator of the attachment. - -- _inline_ - - Optional. A **boolean** indicating whether the attachment is inline. - -- _content_type_ - - Optional. A **string** specifying the MIME type of the attachment (e.g. `'application/pdf'`, `'image/png'`). When provided, this takes precedence over the content type from the HTTP response header during streaming. Falls back to the HTTP `Content-Type` header, or `'application/octet-stream'` if neither is available. - -- _grand_parent_id_ - - Optional. A **number** or **string** identifying a higher-level parent entity, if applicable. - -#### Example - -```typescript -const normalizedAttachment: NormalizedAttachment = { - url: 'https://example.com/files/document.pdf', - id: 'att_123456', - file_name: 'document.pdf', - parent_id: 'task_789', - author_id: 'user_456', - inline: false, - content_type: 'application/pdf', - grand_parent_id: 1001, -}; -``` - -### `RepoInterface` interface - -Defines the structure of a repo which is used to store and upload extracted data. This interface provides the basic structure for repositories that handle data extraction and normalization. - -#### Properties - -- _itemType_ - - Required. A **string** that specifies the type of items stored in this repository. - -- _normalize_ - - Optional. A **function** that takes an object and returns either a **NormalizedItem** or **NormalizedAttachment**. This function is responsible for transforming raw data into a standardized format. - -- _overridenOptions_ - - Optional. An object of type **WorkerAdapterOptions** that overrides the default options for this specific repo. - -#### Example - -```typescript -const taskRepo: RepoInterface = { - itemType: 'tasks', - normalize: (rawTask) => ({ - id: rawTask.id, - created_date: rawTask.created_at, - modified_date: rawTask.updated_at, - data: rawTask, - }), -}; -``` - -### `ExternalSyncUnit` interface - -Represents an external sync unit (such as repositories, projects, etc.) that can be extracted. This interface defines the structure for organizing and identifying extractable units of data. - -#### Properties - -- _id_ - - Required. A **string** that uniquely identifies the external sync unit. - -- _name_ - - Required. A **string** representing the name of the external sync unit. - -- _description_ - - Required. A **string** providing a description of the external sync unit. - -- _item_count_ - - Optional. A **number** indicating the total count of items in this external sync unit. - -- _item_type_ - - Optional. A **string** specifying the type of items contained in this external sync unit. - -### `EventContext` interface - -Defines the structure of the event context that is sent to the external connector from Airdrop. - -#### Properties - -- _callback_url_ - - Required. A **string** representing the callback URL. - -- _dev_org_ - - Required. A **string** representing the organization ID. **Deprecated** - use `dev_oid` instead. - -- _dev_oid_ - - Required. A **string** representing the organization ID. - -- _dev_org_id_ - - Required. A **string** representing the organization ID. - -- _dev_user_ - - Required. A **string** representing the user ID. **Deprecated** - use `dev_uid` instead. - -- _dev_user_id_ - - Required. A **string** representing the user ID. **Deprecated** - use `dev_uid` instead. - -- _dev_uid_ - - Required. A **string** representing the user ID. - -- _event_type_adaas_ - - Required. A **string** representing the event type in ADaaS. - -- _external_sync_unit_ - - Required. A **string** representing the external sync unit ID. **Deprecated** - use `external_sync_unit_id` instead. - -- _external_sync_unit_id_ - - Required. A **string** representing the external sync unit ID. - -- _external_sync_unit_name_ - - Required. A **string** representing the external sync unit name. - -- _external_system_ - - Required. A **string** representing the external system. **Deprecated** - use `external_system_id` instead. - -- _external_system_id_ - - Required. A **string** representing the external system ID. - -- _external_system_name_ - - Required. A **string** representing the external system name. - -- _external_system_type_ - - Required. A **string** representing the external system type. - -- _extract_from_ - - Optional. A **string** representing the resolved start timestamp of extraction in ISO 8601 format. Automatically computed by the SDK from `extraction_start_time` and worker state. This is the field developers should read to know when to start extracting from. - -- _extract_to_ - - Optional. A **string** representing the resolved end timestamp of extraction in ISO 8601 format. Automatically computed by the SDK from `extraction_end_time` and worker state. This is the field developers should read to know when to stop extracting at. - -- _extraction_start_time_ - - Optional. An object of type **TimeValue** representing the start time value for extraction as sent by the platform. The SDK resolves this into a concrete ISO 8601 timestamp on `extract_from`. - -- _extraction_end_time_ - - Optional. An object of type **TimeValue** representing the end time value for extraction as sent by the platform. The SDK resolves this into a concrete ISO 8601 timestamp on `extract_to`. - -- _import_slug_ - - Required. A **string** representing the import slug. - -- _initial_sync_scope_ - - Optional. An enum **InitialSyncScope** representing the scope of the initial sync (can be 'full-history' or 'time-scoped'). - -- _mode_ - - Required. A **string** representing the mode (can be 'INITIAL', 'INCREMENTAL', or 'LOADING'). - -- _request_id_ - - Required. A **string** representing the request ID. - -- _request_id_adaas_ - - Required. A **string** representing the ADaaS request ID. - -- _reset_extraction_ - - Optional. A **boolean** signifying the incremental sync should start from the given `extract_from` timestamp if true or from `lastSuccessfulSyncStarted` timestamp if false. **Deprecated** - use `reset_extract_from` instead. - -- _reset_extract_from_ - - Optional. A **boolean** signifying the incremental sync should start from the given `extract_from` timestamp if true or from `lastSuccessfulSyncStarted` timestamp if false. **Deprecated** - use `extraction_start_time`/`extraction_end_time` instead, which are automatically resolved into `extract_from` and `extract_to`. - -- _run_id_ - - Required. A **string** representing the run ID. - -- _sequence_version_ - - Required. A **string** representing the sequence version. - -- _snap_in_slug_ - - Required. A **string** representing the snap-in slug. - -- _snap_in_version_id_ - - Required. A **string** representing the snap-in version ID. - -- _sync_run_ - - Required. A **string** representing the sync run ID. **Deprecated** - use `run_id` instead. - -- _sync_run_id_ - - Required. A **string** representing the sync run ID. **Deprecated** - use `run_id` instead. - -- _sync_tier_ - - Required. A **string** representing the sync tier. - -- _sync_unit_ - - Required. A **string** representing the sync unit ID. - -- _sync_unit_id_ - - Required. A **string** representing the sync unit ID. - -- _uuid_ - - Required. A **string** representing the unique identifier. **Deprecated** - use `request_id_adaas` instead. - -- _worker_data_url_ - - Required. A **string** representing the worker data URL. - -### `AirdropEvent` interface - -Defines the structure of events sent to external extractors from Airdrop platform. This interface encapsulates all necessary information for processing Airdrop events, including authentication, context, and payload data. - -#### Properties - -- _context_ - - Required. An object containing: - - _secrets_: An object containing: - - _service_account_token_: A **string** representing the DevRev authentication token for Airdrop platform - - _snap_in_version_id_: A **string** representing the version ID of the snap-in - - _snap_in_id_: A **string** representing the ID of the snap-in - -- _payload_ - - Required. An object of type **AirdropMessage** containing: - - _connection_data_: An object containing: - - _org_id_: A **string** representing the organization ID - - _org_name_: A **string** representing the organization name - - _key_: A **string** representing the key - - _key_type_: A **string** representing the key type - - _event_context_: An object of type [**EventContext**](#EventContext-interface) - - _event_type_: A value from the **EventType** enum (see `EventType` enum documentation below) - - _event_data_: Optional. An object that may contain: - - _external_sync_units_: Optional array of **ExternalSyncUnit** objects - - _progress_: Optional **number** indicating progress - - _error_: Optional error record - - _delay_: Optional **number** indicating delay - - _reports_: Optional array of loader reports - - _processed_files_: Optional array of **strings** representing processed files - - _stats_file_: Optional **string** representing stats file - -- _execution_metadata_ - - Required. An object containing: - - _devrev_endpoint_: A **string** representing the DevRev endpoint URL - -- _input_data_ - - Required. An object containing input data for snap-ins from '@devrev/typescript-sdk' - -### `EventData` interface - -Defines the structure of event data that is sent from the external extractor to Airdrop. This interface encapsulates various types of data that can be included in events, such as progress updates, errors, and processing results. - -#### Properties - -- _external_sync_units_ - - Optional. An array of **ExternalSyncUnit** objects representing external sync units to be processed. **Deprecated** - external sync units should be pushed to the `AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS` repo instead. - -- _progress_ - - Optional. A **number** indicating the progress of the current operation. **Deprecated** - progress is now calculated on the backend. - -- _error_ - - Optional. An object of type **ErrorRecord** containing error information if an error occurred. - -- _delay_ - - Optional. A **number** specifying a delay duration in seconds. - -- _artifacts_ - - Optional. An array of **Artifact** objects. **Deprecated** - should not be used directly. - -- _reports_ - - Optional. An array of **LoaderReport** objects representing loader reports. - -- _processed_files_ - - Optional. An array of **strings** representing processed file IDs. - -- _stats_file_ - - Optional. A **string** representing the stats file artifact ID. - -### `EventType` enum - -Defines the different types of events that can be sent to the external extractor from ADaaS. The external extractor uses these events to know what to do next in the extraction process. - -#### Extraction Events (preferred) - -- `StartExtractingExternalSyncUnits` = `'START_EXTRACTING_EXTERNAL_SYNC_UNITS'` -- `StartExtractingMetadata` = `'START_EXTRACTING_METADATA'` -- `StartExtractingData` = `'START_EXTRACTING_DATA'` -- `ContinueExtractingData` = `'CONTINUE_EXTRACTING_DATA'` -- `StartDeletingExtractorState` = `'START_DELETING_EXTRACTOR_STATE'` -- `StartExtractingAttachments` = `'START_EXTRACTING_ATTACHMENTS'` -- `ContinueExtractingAttachments` = `'CONTINUE_EXTRACTING_ATTACHMENTS'` -- `StartDeletingExtractorAttachmentsState` = `'START_DELETING_EXTRACTOR_ATTACHMENTS_STATE'` - -#### Extraction Events (deprecated) - -- `ExtractionExternalSyncUnitsStart` **Deprecated** - use `StartExtractingExternalSyncUnits` -- `ExtractionMetadataStart` **Deprecated** - use `StartExtractingMetadata` -- `ExtractionDataStart` **Deprecated** - use `StartExtractingData` -- `ExtractionDataContinue` **Deprecated** - use `ContinueExtractingData` -- `ExtractionDataDelete` **Deprecated** - use `StartDeletingExtractorState` -- `ExtractionAttachmentsStart` **Deprecated** - use `StartExtractingAttachments` -- `ExtractionAttachmentsContinue` **Deprecated** - use `ContinueExtractingAttachments` -- `ExtractionAttachmentsDelete` **Deprecated** - use `StartDeletingExtractorAttachmentsState` - -#### Loading Events - -- `StartLoadingData` = `'START_LOADING_DATA'` -- `ContinueLoadingData` = `'CONTINUE_LOADING_DATA'` -- `StartLoadingAttachments` = `'START_LOADING_ATTACHMENTS'` -- `ContinueLoadingAttachments` = `'CONTINUE_LOADING_ATTACHMENTS'` -- `StartDeletingLoaderState` = `'START_DELETING_LOADER_STATE'` -- `StartDeletingLoaderAttachmentState` = `'START_DELETING_LOADER_ATTACHMENT_STATE'` - -#### Other - -- `UnknownEventType` = `'UNKNOWN_EVENT_TYPE'` - -### `ExtractorEventType` enum - -Defines the different types of events that can be sent from the external extractor to ADaaS. The external extractor uses these events to inform ADaaS about the progress of the extraction process. - -#### Extraction Events (preferred) - -- `ExternalSyncUnitExtractionDone` = `'EXTERNAL_SYNC_UNIT_EXTRACTION_DONE'` -- `ExternalSyncUnitExtractionError` = `'EXTERNAL_SYNC_UNIT_EXTRACTION_ERROR'` -- `MetadataExtractionDone` = `'METADATA_EXTRACTION_DONE'` -- `MetadataExtractionError` = `'METADATA_EXTRACTION_ERROR'` -- `DataExtractionProgress` = `'DATA_EXTRACTION_PROGRESS'` -- `DataExtractionDelayed` = `'DATA_EXTRACTION_DELAYED'` -- `DataExtractionDone` = `'DATA_EXTRACTION_DONE'` -- `DataExtractionError` = `'DATA_EXTRACTION_ERROR'` -- `ExtractorStateDeletionDone` = `'EXTRACTOR_STATE_DELETION_DONE'` -- `ExtractorStateDeletionError` = `'EXTRACTOR_STATE_DELETION_ERROR'` -- `AttachmentExtractionProgress` = `'ATTACHMENT_EXTRACTION_PROGRESS'` -- `AttachmentExtractionDelayed` = `'ATTACHMENT_EXTRACTION_DELAYED'` -- `AttachmentExtractionDone` = `'ATTACHMENT_EXTRACTION_DONE'` -- `AttachmentExtractionError` = `'ATTACHMENT_EXTRACTION_ERROR'` -- `ExtractorAttachmentsStateDeletionDone` = `'EXTRACTOR_ATTACHMENTS_STATE_DELETION_DONE'` -- `ExtractorAttachmentsStateDeletionError` = `'EXTRACTOR_ATTACHMENTS_STATE_DELETION_ERROR'` - -#### Extraction Events (deprecated) - -- `ExtractionExternalSyncUnitsDone` **Deprecated** - use `ExternalSyncUnitExtractionDone` -- `ExtractionExternalSyncUnitsError` **Deprecated** - use `ExternalSyncUnitExtractionError` -- `ExtractionMetadataDone` **Deprecated** - use `MetadataExtractionDone` -- `ExtractionMetadataError` **Deprecated** - use `MetadataExtractionError` -- `ExtractionDataProgress` **Deprecated** - use `DataExtractionProgress` -- `ExtractionDataDelay` **Deprecated** - use `DataExtractionDelayed` -- `ExtractionDataDone` **Deprecated** - use `DataExtractionDone` -- `ExtractionDataError` **Deprecated** - use `DataExtractionError` -- `ExtractionDataDeleteDone` **Deprecated** - use `ExtractorStateDeletionDone` -- `ExtractionDataDeleteError` **Deprecated** - use `ExtractorStateDeletionError` -- `ExtractionAttachmentsProgress` **Deprecated** - use `AttachmentExtractionProgress` -- `ExtractionAttachmentsDelay` **Deprecated** - use `AttachmentExtractionDelayed` -- `ExtractionAttachmentsDone` **Deprecated** - use `AttachmentExtractionDone` -- `ExtractionAttachmentsError` **Deprecated** - use `AttachmentExtractionError` -- `ExtractionAttachmentsDeleteDone` **Deprecated** - use `ExtractorAttachmentsStateDeletionDone` -- `ExtractionAttachmentsDeleteError` **Deprecated** - use `ExtractorAttachmentsStateDeletionError` - -#### Other - -- `UnknownEventType` = `'UNKNOWN_EVENT_TYPE'` - -### `LoaderEventType` enum - -Defines the different types of events that can be sent from the loader to ADaaS. - -#### Data Loading Events - -- `DataLoadingProgress` = `'DATA_LOADING_PROGRESS'` -- `DataLoadingDelayed` = `'DATA_LOADING_DELAYED'` -- `DataLoadingDone` = `'DATA_LOADING_DONE'` -- `DataLoadingError` = `'DATA_LOADING_ERROR'` -- `DataLoadingDelay` **Deprecated** - this was a typo, use `DataLoadingDelayed` instead - -#### Attachment Loading Events - -- `AttachmentLoadingProgress` = `'ATTACHMENT_LOADING_PROGRESS'` -- `AttachmentLoadingDelayed` = `'ATTACHMENT_LOADING_DELAYED'` -- `AttachmentLoadingDone` = `'ATTACHMENT_LOADING_DONE'` -- `AttachmentLoadingError` = `'ATTACHMENT_LOADING_ERROR'` - -#### Attachment Loading Events (deprecated aliases) - -- `AttachmentsLoadingProgress` **Deprecated** - use `AttachmentLoadingProgress` -- `AttachmentsLoadingDelayed` **Deprecated** - use `AttachmentLoadingDelayed` -- `AttachmentsLoadingDone` **Deprecated** - use `AttachmentLoadingDone` -- `AttachmentsLoadingError` **Deprecated** - use `AttachmentLoadingError` - -#### State Deletion Events - -- `LoaderStateDeletionDone` = `'LOADER_STATE_DELETION_DONE'` -- `LoaderStateDeletionError` = `'LOADER_STATE_DELETION_ERROR'` -- `LoaderAttachmentStateDeletionDone` = `'LOADER_ATTACHMENT_STATE_DELETION_DONE'` -- `LoaderAttachmentStateDeletionError` = `'LOADER_ATTACHMENT_STATE_DELETION_ERROR'` - -#### Other - -- `UnknownEventType` = `'UNKNOWN_EVENT_TYPE'` - -### `SyncMode` enum - -Defines the different modes of sync that can be used by the external extractor. It can be either INITIAL, INCREMENTAL or LOADING. - -#### Values - -- `INITIAL` = `'INITIAL'` - Used for the first/initial import -- `INCREMENTAL` = `'INCREMENTAL'` - Used for doing syncs -- `LOADING` = `'LOADING'` - Used for loading data from DevRev to the external system - -### `ExtractionMode` enum **Deprecated** - -Defines the different modes of extraction. Use `SyncMode` instead. - -#### Values - -- `INITIAL` = `'INITIAL'` -- `INCREMENTAL` = `'INCREMENTAL'` - -### `InitialSyncScope` enum - -Defines the different scopes of initial sync that can be used by the external extractor. - -#### Values - -- `FULL_HISTORY` = `'full-history'` -- `TIME_SCOPED` = `'time-scoped'` - -### `TimeUnit` enum - -Defines the supported Go duration units for time window calculations. These correspond directly to Go's `time.ParseDuration` units. - -#### Values - -- `NANOSECONDS` = `'ns'` -- `MICROSECONDS` = `'us'` -- `MICROSECONDS_MU` = `'µs'` -- `MILLISECONDS` = `'ms'` -- `SECONDS` = `'s'` -- `MINUTES` = `'m'` -- `HOURS` = `'h'` - -### `TimeValueType` enum - -Defines the type of a time value used in extraction start/end times. The platform sends these types to indicate how the extraction time should be resolved by the SDK. - -#### Values - -- `WORKERS_OLDEST` = `'workers_oldest'` - Oldest timestamp from worker state -- `WORKERS_OLDEST_MINUS_WINDOW` = `'workers_oldest_minus_window'` - Oldest timestamp from worker state minus a duration window -- `WORKERS_NEWEST` = `'workers_newest'` - Newest timestamp from worker state -- `WORKERS_NEWEST_PLUS_WINDOW` = `'workers_newest_plus_window'` - Newest timestamp from worker state plus a duration window -- `CURRENT_TIME` = `'current_time'` - Current time -- `ABSOLUTE_TIME` = `'absolute_time'` - User-specified absolute timestamp -- `UNBOUNDED` = `'unbounded'` - No bound, extract all available data - -### `TimeValue` interface - -Represents a time value used in extraction start/end times. - -#### Properties - -- _type_ - - Required. A **TimeValueType** enum value which denotes how the value should be resolved. - -- _value_ - - Optional. A **string** whose meaning depends on the type: - - For `ABSOLUTE_TIME`: an ISO 8601 timestamp - - For `*_WINDOW` types: a Go duration string (e.g. `'500ms'`, `'30s'`, `'5m'`, `'2h'`) - - For other types: not used - -### `ExtractionScope` type - -Represents the parsed extraction scope from the platform. Each key is an item type name, and the value indicates whether it should be extracted. - -#### Usage - -```typescript -type ExtractionScope = Record; -``` - -### `ExtractionCommonError` const enum - -Provides predefined error codes for common extraction errors. - -#### Values - -- `EXTERNAL_SYNC_UNIT_DELETED` = `'ERROR_CODE=EXTERNAL_SYNC_UNIT_DELETED'` -- `EXTERNAL_SYNC_UNIT_DEACTIVATED` = `'ERROR_CODE=EXTERNAL_SYNC_UNIT_DEACTIVATED'` -- `USER_DELETED` = `'ERROR_CODE=USER_DELETED'` - -### `AirSyncDefaultItemTypes` enum - -Defines the default item types used by the SDK. - -#### Values - -- `EXTERNAL_DOMAIN_METADATA` = `'external_domain_metadata'` -- `ATTACHMENTS` = `'attachments'` -- `EXTERNAL_SYNC_UNITS` = `'external_sync_units'` - -### `UNBOUNDED_DATE_TIME_VALUE` constant - -Sentinel value representing an unbounded (no limit) extraction timestamp. Used as the resolved value for `TimeValueType.UNBOUNDED`. Its value is `'1970-01-01T00:00:00.000Z'`. - -### `spawn` function - -This function initializes a new worker thread and oversees its lifecycle. It should be invoked when the snap-in receives a message from the Airdrop platform. The worker script provided then handles the event accordingly. - -#### Usage - -```typescript -spawn({ event, initialState, options, baseWorkerPath }); -``` - -#### Parameters - -- _event_ - - Required. An object of type **AirdropEvent** that is received from the Airdrop platform. - -- _initialState_ - - Required. Object of **any** type that represents the initial state of the snap-in. - -- _workerPath_ - - Optional. A **string** that represents the path to the worker file. **Deprecated** - use `baseWorkerPath` instead. - -- _options_ - - Optional. An object of type **WorkerAdapterOptions**, which will be passed to the newly created worker. This worker will then initialize a `WorkerAdapter` by invoking the `processTask` function. The options include: - - `isLocalDevelopment` - - A **boolean** flag. If set to `true`, intermediary files containing extracted data will be stored on the local machine, which is useful during development. The default value is `false`. - - - `timeout` - - A **number** that specifies the timeout duration for the lambda function, in milliseconds. The default is 10 minutes (10 \* 60 \* 1000 milliseconds). - - - `batchSize` - - A **number** that determines the maximum number of items to be processed and saved to an intermediary file before being sent to the Airdrop platform. The default batch size is 2,000. - - - `workerPathOverrides` - - Optional. A partial map of **EventType** to **string** paths, allowing you to override the default worker path for specific event types. - - - `skipConfirmation` - - Optional. A **boolean** flag. If set to `true`, skips artifact upload confirmation. - -- _initialDomainMapping_ - - Optional. An object of type **InitialDomainMapping** representing the initial domain mapping configuration. - -- _baseWorkerPath_ - - Optional. A **string** that represents the base path for the worker files, usually `__dirname`. When provided, the SDK automatically resolves the worker script based on the event type. - -#### Return value - -A **promise** that resolves once the worker has completed processing. - -#### Example - -```typescript -const run = async (events: AirdropEvent[]) => { - for (const event of events) { - await spawn({ - event, - initialState, - baseWorkerPath: __dirname, - }); - } -}; -``` - -### `processTask` function - -The `processTask` function retrieves the current state from the Airdrop platform and initializes a new `WorkerAdapter`. It executes the code specified in the `task` parameter, which contains the worker's functionality. If a timeout occurs, the function handles it by executing the `onTimeout` callback, ensuring the worker exits gracefully. Both functions receive an `adapter` parameter, representing the initialized `WorkerAdapter` object. - -#### Usage - -```typescript -processTask({ task, onTimeout }); -``` - -#### Parameters - -- _task_ - - Required. A **function** that defines the logic associated with the given event type. - -- _onTimeout_ - - Required. A **function** managing the timeout of the lambda invocation, including saving any necessary progress at the time of timeout. - -#### Example - -```typescript -// External sync units extraction -processTask({ - task: async ({ adapter }) => { - const httpClient = new HttpClient(adapter.event); - - const todoLists = await httpClient.getTodoLists(); - - const externalSyncUnits: ExternalSyncUnit[] = todoLists.map((todoList) => - normalizeTodoList(todoList) - ); - - await adapter.emit(ExtractorEventType.ExternalSyncUnitExtractionDone, { - external_sync_units: externalSyncUnits, - }); - }, - onTimeout: async ({ adapter }) => { - await adapter.emit(ExtractorEventType.ExternalSyncUnitExtractionError, { - error: { - message: 'Failed to extract external sync units. Lambda timeout.', - }, - }); - }, -}); -``` - -### `Spawn` class - -`Spawn` class is responsible for spawning a new worker thread and managing the lifecycle of the worker. Provides utilities to emit control events to the platform and exit the worker gracefully. In case of lambda timeout, the class emits a lambda timeout event to the platform. - -#### Usage - -```typescript -new Spawn({ - event, - worker, - options, - resolve, - originalConsole, -}); -``` - -#### Parameters - -- _event_ - - Required. An object of type **AirdropEvent** that is received from the Airdrop platform. - -- _worker_ - - Required. A Node worker of the **Worker** class, created with the createWorker function, which represents an independent JavaScript execution thread. - -- _options_ - - Optional. An object of type **WorkerAdapterOptions**, which defines the options to create a new instance of Spawn class. - -- _resolve_ - - Required. A resolve **function** for the promise inside which the Spawn class was created. - -- _originalConsole_ - - Optional. A **Console** object representing the original console before the SDK logger replaces it. - -#### Example - -```typescript -new Promise((resolve) => { - new Spawn({ - event, - worker, - options, - resolve, - }); -}); -``` - -### `WorkerAdapter` class - -Used to interact with Airdrop platform. Provides utilities to emit events to the Airdrop platform, update the state of the snap-in and upload artifacts (files with data) to the platform. - -### Usage - -```typescript -new WorkerAdapter({ - event, - adapterState, - options, -}); -``` - -#### Parameters - -- _event_ - - Required. An object of type **AirdropEvent** that is received from the Airdrop platform. - -- _adapterState_ - - Required. An object of type **State**, which represents the initial state of the adapter. - -- _options_ - - Optional. An object of type **WorkerAdapterOptions** that specifies additional configuration options for the `WorkerAdapter`. This object is passed via the `spawn` function. - -#### Example - -```typescript -const adapter = new WorkerAdapter({ - event, - adapterState, - options, -}); -``` - -### `WorkerAdapter.state` property - -Getter and setter methods for working with the adapter state. - -### Usage - -```typescript -// get state -const adapterState = adapter.state; - -// set state -adapter.state = newAdapterState; -``` - -#### Example - -```typescript -export const initialState: ExtractorState = { - users: { completed: false }, - tasks: { completed: false }, - attachments: { completed: false }, -}; - -adapter.state = initialState; -``` - -### `WorkerAdapter.extractionScope` property - -Getter for the parsed extraction scope from the platform. Returns an `ExtractionScope` object. - -### Usage - -```typescript -const scope = adapter.extractionScope; -``` - -### `WorkerAdapter.shouldExtract` method - -Returns whether the given item type should be extracted. Defaults to `true` if the scope is empty or the item type is not listed. - -### Usage - -```typescript -adapter.shouldExtract(itemType); -``` - -#### Parameters - -- _itemType_ - - Required. A **string** representing the item type to check. - -#### Return value - -A **boolean** indicating whether the item type should be extracted. - -#### Example - -```typescript -if (adapter.shouldExtract('tasks')) { - // Extract tasks -} -``` - -### `WorkerAdapter.initializeRepos` method - -Initializes a `Repo` object for each item provided. - -### Usage - -```typescript -adapter.initializeRepos(repos); -``` - -#### Parameters - -- _repos_ - - Required. An array of objects of type `RepoInterface`. - -#### Example - -This should typically be called within the function passed as a parameter to the `processTask` function in the data extraction phase. - -```typescript -const repos = [ - { - itemType: 'tasks', - normalize: normalizeTask, - }, -]; - -adapter.initializeRepos(repos); -``` - -### `WorkerAdapter.getRepo` method - -Finds a Repo from the initialized repos. - -### Usage - -```typescript -adapter.getRepo(itemType); -``` - -#### Parameters - -- _itemType_ - - Required. A **string** that represents the itemType property for the searched repo. - -#### Return value - -An object of type **Repo** if the repo is found, otherwise **undefined**. - -#### Example - -This should typically be called within the function passed as a parameter to the `processTask` function. - -```typescript -// Push users to the repository designated for 'users' data. -await adapter.getRepo('users')?.push(users); -``` - -### `WorkerAdapter.emit` method - -Emits an event to the Airdrop platform. - -### Usage - -```typescript -adapter.emit( newEventType, data ): -``` - -#### Parameters - -- _newEventType_ - - Required. The event type to be emitted, of type **ExtractorEventType** or **LoaderEventType**. - -- _data_ - - Optional. An object of type **EventData** which represents the data to be sent with the event. - -#### Return value - -A **promise**, which resolves to undefined after the emit function completes its execution or rejects with an error. - -#### Example - -This should typically be called within the function passed as a parameter to the `processTask` function. - -```typescript -// Emitting successfully finished data extraction. -await adapter.emit(ExtractorEventType.DataExtractionDone); - -// Emitting a delay in attachments extraction phase. -await adapter.emit(ExtractorEventType.AttachmentExtractionDelayed, { - delay: 10, -}); -``` - -### `WorkerAdapter.postState` method - -Saves the current adapter state to the Airdrop platform. - -### Usage - -```typescript -await adapter.postState(); -``` - -#### Return value - -A **promise** that resolves once the state has been posted. - -### `WorkerAdapter.mappers` property - -Provides access to the `Mappers` helper within the worker during loading. Use it to look up, create, or update sync mapper records that link external system items to DevRev items. - -#### Usage - -```typescript -// inside processTask({ task }) -await adapter.mappers.getByTargetId({ - sync_unit: adapter.event.payload.event_context.sync_unit, - target: devrevId, -}); -``` - -### `WorkerAdapter.reports` property - -Getter for the accumulated loader reports. Returns an array of **LoaderReport** objects. - -### `WorkerAdapter.processedFiles` property - -Getter for the list of processed file IDs. Returns an array of **strings**. - -### `WorkerAdapter.loadItemTypes` method - -Loads item types from DevRev to the external system during the loading phase. - -#### Usage - -```typescript -const response = await adapter.loadItemTypes({ itemTypesToLoad }); -``` - -#### Parameters - -- _itemTypesToLoad_ - - Required. An array of **ItemTypeToLoad** objects, each containing an `itemType` string, a `create` function, and an `update` function. - -#### Return value - -A **promise** resolving to a **LoadItemTypesResponse** containing `reports` and `processed_files`. - -### `WorkerAdapter.loadAttachments` method - -Loads attachments from DevRev to the external system during the loading phase. - -#### Usage - -```typescript -const response = await adapter.loadAttachments({ create }); -``` - -#### Parameters - -- _create_ - - Required. A function of type **ExternalSystemLoadingFunction\** that creates the attachment in the external system. - -#### Return value - -A **promise** resolving to a **LoadItemTypesResponse** containing `reports` and `processed_files`. - -### `WorkerAdapter.streamAttachments` method - -Streams attachments to the DevRev platform during the attachment extraction phase. Handles batching, deduplication, and progress tracking. - -#### Usage - -```typescript -await adapter.streamAttachments({ stream, processors, batchSize }); -``` - -#### Parameters - -- _stream_ - - Required. A function of type **ExternalSystemAttachmentStreamingFunction** that opens an HTTP stream for a given attachment. - -- _processors_ - - Optional. An object of type **ExternalSystemAttachmentProcessors** for custom attachment processing with `reducer` and `iterator` functions. - -- _batchSize_ - - Optional. A **number** specifying how many attachments to stream concurrently. Default is `1`, maximum is `50`. - -#### Return value - -A **promise** that resolves to a **StreamAttachmentsReturnType** (may contain `delay` or `error`), or `undefined` on success. - -### `WorkerAdapter.processAttachment` method - -Processes a single attachment: streams it from the external system, uploads it to DevRev, and records the SSOR attachment mapping. - -#### Usage - -```typescript -const result = await adapter.processAttachment(attachment, stream); -``` - -#### Parameters - -- _attachment_ - - Required. A **NormalizedAttachment** object representing the attachment to process. - -- _stream_ - - Required. A function of type **ExternalSystemAttachmentStreamingFunction** that returns the HTTP stream for the attachment. - -#### Return value - -A **promise** resolving to a **ProcessAttachmentReturnType** (may contain `error` or `delay`), or `undefined` on success. - ---- - -### `Mappers` class - -Manages sync mapper records that link external system items to DevRev items during loading. Access it via `adapter.mappers` inside your worker code. - -#### Methods - -- `getByTargetId(params)` - - **params**: `MappersGetByTargetIdParams` - - **returns**: `Promise>` - - Use when you know the DevRev ID and want the corresponding mapping. - -- `getByExternalId(params)` - - **params**: `MappersGetByExternalIdParams` - - **returns**: `Promise>` - - Use when you know an external ID and need the DevRev mapping. - -- `create(params)` - - **params**: `MappersCreateParams` - - **returns**: `Promise>` - - Call after creating an item in the external system to persist the mapping. - -- `update(params)` - - **params**: `MappersUpdateParams` - - **returns**: `Promise>` - - Call after updating an item in the external system to add IDs, targets, or version markers. - -### `SyncMapperRecordStatus` enum - -Status of a sync mapper record indicating its operational state. - -#### Values - -- `OPERATIONAL` = `'operational'` - The mapping is active and operational (default) -- `FILTERED` = `'filtered'` - The mapping was filtered out by user filter settings -- `IGNORED` = `'ignored'` - The external object should be ignored in sync operations - -### `SyncMapperRecordTargetType` enum - -Types of DevRev entities that can be targets in sync mapper records. - -#### Values - -- `ACCESS_CONTROL_ENTRY` = `'access_control_entry'` -- `ACCOUNT` = `'account'` -- `AIRDROP_AUTHORIZATION_POLICY` = `'airdrop_authorization_policy'` -- `AIRDROP_FIELD_AUTHORIZATION_POLICY` = `'airdrop_field_authorization_policy'` -- `AIRDROP_PLATFORM_GROUP` = `'airdrop_platform_group'` -- `ARTICLE` = `'article'` -- `ARTIFACT` = `'artifact'` -- `CHAT` = `'chat'` -- `CONVERSATION` = `'conversation'` -- `CUSTOM_OBJECT` = `'custom_object'` -- `DIRECTORY` = `'directory'` -- `GROUP` = `'group'` -- `INCIDENT` = `'incident'` -- `LINK` = `'link'` -- `MEETING` = `'meeting'` -- `OBJECT_MEMBER` = `'object_member'` -- `PART` = `'part'` -- `REV_ORG` = `'rev_org'` -- `ROLE` = `'role'` -- `ROLE_SET` = `'role_set'` -- `TAG` = `'tag'` -- `TIMELINE_COMMENT` = `'timeline_comment'` -- `USER` = `'user'` -- `WORK` = `'work'` - -### `installInitialDomainMapping` function - -Installs the initial domain mapping for a snap-in. This creates recipe blueprints and installs domain mappings via the DevRev API. - -#### Usage - -```typescript -await installInitialDomainMapping(event, initialDomainMappingJson); -``` - -#### Parameters - -- _event_ - - Required. An object of type **AirdropEvent**. - -- _initialDomainMappingJson_ - - Required. An object of type **InitialDomainMapping** containing: - - `starting_recipe_blueprint`: Optional object with the recipe blueprint configuration - - `additional_mappings`: Optional object with additional mapping configuration - -### `MockServer` class - -A lightweight HTTP mock server for local development and testing of connectors. Allows you to define routes with static responses or custom handlers. - -#### Exported types - -- **RequestInfo** - Information about a request received by the mock server (method, url, body) -- **RetryConfig** - Configuration for retry simulation behavior (failureCount, errorStatus, errorBody, headers, delay) -- **RouteConfig** - Configuration object for setting up a route response (path, method, status, body, headers, retry, delay) - -### `formatAxiosError` function - -Formats an Axios error into a structured object for logging. - -#### Usage - -```typescript -const formatted = formatAxiosError(error); -``` - -#### Parameters - -- _error_ - - Required. An **AxiosError** object. - -#### Return value - -An **object** with structured error information. - -### `serializeAxiosError` function - -Serializes an Axios error into a structured response object. - -#### Usage - -```typescript -const serialized = serializeAxiosError(error); -``` - -#### Parameters - -- _error_ - - Required. An **AxiosError** object. - -#### Return value - -An **AxiosErrorResponse** object with structured error details. diff --git a/eslint.config.mts b/eslint.config.mts index c785f010..65664b08 100644 --- a/eslint.config.mts +++ b/eslint.config.mts @@ -1,10 +1,17 @@ +import { builtinModules } from 'node:module'; + import js from '@eslint/js'; import prettierConfig from 'eslint-config-prettier'; import prettierPlugin from 'eslint-plugin-prettier'; +import simpleImportSort from 'eslint-plugin-simple-import-sort'; import { defineConfig } from 'eslint/config'; import globals from 'globals'; import tseslint from 'typescript-eslint'; +const nodeBuiltins = builtinModules + .filter((m) => !m.startsWith('_')) + .join('|'); + export default defineConfig([ { ignores: ['dist/**', 'test/**', 'src/deprecated/**', 'coverage'] }, @@ -54,10 +61,26 @@ export default defineConfig([ }, plugins: { prettier: prettierPlugin, + 'simple-import-sort': simpleImportSort, }, rules: { 'prettier/prettier': 'error', + // Import order: node builtins, external packages, + // parent-relative paths, same-directory paths. + 'simple-import-sort/imports': [ + 'error', + { + groups: [ + [`^node:`, `^(${nodeBuiltins})(/|$)`], + ['^@?\\w'], + ['^\\.\\.(?!/?$)', '^\\.\\./?$'], + ['^\\./(?=.*/)(?!/?$)', '^\\.(?!/?$)', '^\\./?$'], + ], + }, + ], + 'simple-import-sort/exports': 'error', + // Custom rules 'require-await': 'off', '@typescript-eslint/await-thenable': 'error', diff --git a/jest.config.cjs b/jest.config.cjs index be2cf47a..fab696c9 100644 --- a/jest.config.cjs +++ b/jest.config.cjs @@ -11,7 +11,7 @@ module.exports = { 'src/tests/timeout-handling/.*.ts', // These tests are slow (10-15s per) 'src/tests/dummy-connector/metadata-extraction.test.ts', - 'src/http/axios-client-internal.test.ts', + 'src/http/client.test.ts', 'src/tests/event-data-size-limit/.*.test.ts', ], }, @@ -31,7 +31,7 @@ module.exports = { preset: 'ts-jest', testMatch: [ '/src/tests/dummy-connector/metadata-extraction.test.ts', - '/src/http/axios-client-internal.test.ts', + '/src/http/client.test.ts', '/src/tests/event-data-size-limit/size-limit-1.test.ts', ], }, diff --git a/package-lock.json b/package-lock.json index d22ebabf..9aac02d6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,21 +1,21 @@ { - "name": "@devrev/ts-adaas", - "version": "1.20.0", + "name": "@devrev/airsync-sdk", + "version": "2.0.0-beta.6", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "@devrev/ts-adaas", - "version": "1.20.0", + "name": "@devrev/airsync-sdk", + "version": "2.0.0-beta.6", "license": "ISC", "dependencies": { - "@devrev/typescript-sdk": "^1.1.78", + "@devrev/typescript-sdk": "^1.1.76", "axios": "^1.17.0", "axios-retry": "^4.5.0", "form-data": "^4.0.4", "js-jsonl": "^1.1.1", "ts-node": "^10.9.2", - "yargs": "^17.7.3" + "yargs": "^17.7.2" }, "devDependencies": { "@microsoft/api-extractor": "^7.57.6", @@ -27,6 +27,7 @@ "eslint": "9.32.0", "eslint-config-prettier": "^9.1.2", "eslint-plugin-prettier": "4.0.0", + "eslint-plugin-simple-import-sort": "^13.0.0", "jest": "^29.7.0", "jiti": "^2.6.1", "prettier": "^2.8.3", @@ -574,9 +575,9 @@ } }, "node_modules/@devrev/typescript-sdk": { - "version": "1.1.78", - "resolved": "https://registry.npmjs.org/@devrev/typescript-sdk/-/typescript-sdk-1.1.78.tgz", - "integrity": "sha512-rgku5NFZHp2g31RHdF3BCSriQ62SABmhgMF6EIxw9QnZn7p3T/uckWejGSevPBmobJ49D7kkwqHTcbGFT6sjJg==", + "version": "1.1.76", + "resolved": "https://registry.npmjs.org/@devrev/typescript-sdk/-/typescript-sdk-1.1.76.tgz", + "integrity": "sha512-jnIXIhqhVOkH0VhWNNJPQNB3Eb3MNIlR6Pc0HcXEh48ygoJmkuIYZYeR7A4MG8d/JTIAw6a8cK14NB6JKnLrmA==", "license": "MIT", "dependencies": { "@types/yargs": "^17.0.22", @@ -2952,6 +2953,16 @@ } } }, + "node_modules/eslint-plugin-simple-import-sort": { + "version": "13.0.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-simple-import-sort/-/eslint-plugin-simple-import-sort-13.0.0.tgz", + "integrity": "sha512-McAc+/Nlvcg4byY/CABGH8kqnefWBj8s3JA2okEtz8ixbECQgU46p0HkTUKa4YS7wvgGceimlc34p1nXqbWqtA==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "eslint": ">=5.0.0" + } + }, "node_modules/eslint-scope": { "version": "8.4.0", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.4.0.tgz", @@ -6076,9 +6087,9 @@ "license": "ISC" }, "node_modules/yargs": { - "version": "17.7.3", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.3.tgz", - "integrity": "sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g==", + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", "license": "MIT", "dependencies": { "cliui": "^8.0.1", diff --git a/package.json b/package.json index 2cf17877..dfcf96d0 100644 --- a/package.json +++ b/package.json @@ -1,12 +1,12 @@ { - "name": "@devrev/ts-adaas", - "version": "1.20.0", - "description": "Typescript library containing the ADaaS(AirDrop as a Service) control protocol.", + "name": "@devrev/airsync-sdk", + "version": "2.0.0-beta.6", + "description": "Typescript SDK for building AirSync snap-ins on the DevRev platform.", "type": "commonjs", "main": "./dist/index.js", "typings": "./dist/index.d.ts", "scripts": { - "build": "tsc -p ./tsconfig.json", + "build": "tsc -p ./tsconfig.build.json", "prepare": "npm run build", "start": "ts-node src/index.ts", "lint": "eslint .", @@ -38,6 +38,7 @@ "eslint": "9.32.0", "eslint-config-prettier": "^9.1.2", "eslint-plugin-prettier": "4.0.0", + "eslint-plugin-simple-import-sort": "^13.0.0", "jest": "^29.7.0", "jiti": "^2.6.1", "prettier": "^2.8.3", @@ -46,16 +47,17 @@ "typescript-eslint": "^8.46.1" }, "dependencies": { - "@devrev/typescript-sdk": "^1.1.78", + "@devrev/typescript-sdk": "^1.1.76", "axios": "^1.17.0", "axios-retry": "^4.5.0", "form-data": "^4.0.4", "js-jsonl": "^1.1.1", "ts-node": "^10.9.2", - "yargs": "^17.7.3" + "yargs": "^17.7.2" }, "files": [ "dist/**/*", - "!dist/tests/**/*" + "!dist/tests/**/*", + "MIGRATION.md" ] } diff --git a/src/attachments-streaming/attachments-streaming-pool.interfaces.ts b/src/attachments-streaming/attachments-streaming-pool.interfaces.ts index 9067c4f6..a7c94ef5 100644 --- a/src/attachments-streaming/attachments-streaming-pool.interfaces.ts +++ b/src/attachments-streaming/attachments-streaming-pool.interfaces.ts @@ -1,11 +1,9 @@ -import { - ExternalSystemAttachmentStreamingFunction, - NormalizedAttachment, -} from '../types'; -import { WorkerAdapter } from '../multithreading/worker-adapter/worker-adapter'; +import { ExtractionAdapter } from '../multithreading/adapters/extraction-adapter'; +import { NormalizedAttachment } from '../repo/repo.interfaces'; +import { ExternalSystemAttachmentStreamingFunction } from '../types/extraction'; export interface AttachmentsStreamingPoolParams { - adapter: WorkerAdapter; + adapter: ExtractionAdapter; attachments: NormalizedAttachment[]; batchSize?: number; stream: ExternalSystemAttachmentStreamingFunction; diff --git a/src/attachments-streaming/attachments-streaming-pool.test.ts b/src/attachments-streaming/attachments-streaming-pool.test.ts index fefbe13c..52c7b142 100644 --- a/src/attachments-streaming/attachments-streaming-pool.test.ts +++ b/src/attachments-streaming/attachments-streaming-pool.test.ts @@ -1,10 +1,11 @@ -import { WorkerAdapter } from '../multithreading/worker-adapter/worker-adapter'; +import { ExtractionAdapter } from '../multithreading/adapters/extraction-adapter'; +import { NormalizedAttachment } from '../repo/repo.interfaces'; import { ProcessedAttachmentStatus } from '../state/state.interfaces'; import { ExternalSystemAttachmentStreamingFunction, - NormalizedAttachment, ProcessAttachmentReturnType, -} from '../types'; +} from '../types/extraction'; + import { AttachmentsStreamingPool } from './attachments-streaming-pool'; interface TestState { @@ -14,7 +15,7 @@ interface TestState { /* eslint-disable @typescript-eslint/no-explicit-any */ describe(AttachmentsStreamingPool.name, () => { - let mockAdapter: jest.Mocked>; + let mockAdapter: jest.Mocked>; let mockStream: jest.MockedFunction; let mockAttachments: NormalizedAttachment[]; @@ -23,6 +24,8 @@ describe(AttachmentsStreamingPool.name, () => { mockAdapter = { state: { attachments: { completed: false }, + }, + sdkState: { toDevRev: { attachmentsMetadata: { lastProcessedAttachmentsIdsList: [], @@ -108,7 +111,7 @@ describe(AttachmentsStreamingPool.name, () => { describe(AttachmentsStreamingPool.prototype.streamAll.name, () => { it('should initialize lastProcessedAttachmentsIdsList if it does not exist', async () => { - mockAdapter.state.toDevRev!.attachmentsMetadata.lastProcessedAttachmentsIdsList = + mockAdapter.sdkState.toDevRev!.attachmentsMetadata.lastProcessedAttachmentsIdsList = undefined as any; mockAdapter.processAttachment.mockResolvedValue({}); @@ -126,7 +129,7 @@ describe(AttachmentsStreamingPool.name, () => { await pool.streamAll(); expect( - mockAdapter.state.toDevRev!.attachmentsMetadata + mockAdapter.sdkState.toDevRev!.attachmentsMetadata .lastProcessedAttachmentsIdsList ).toEqual([]); }); @@ -174,11 +177,23 @@ describe(AttachmentsStreamingPool.name, () => { expect(result).toEqual({ delay: 5000 }); }); - it('should resume attachment extraction if it encounters old ids', async () => { - // Test migration from old string[] format to new ProcessedAttachment[] format - // Using 'as any' because we're intentionally testing legacy data format - mockAdapter.state.toDevRev!.attachmentsMetadata.lastProcessedAttachmentsIdsList = - ['attachment-1', 'attachment-2'] as any; + it('should resume attachment extraction, skipping already-processed ids', async () => { + // Resume case: the state already records attachment-1 and attachment-2 as + // processed (in the v2 ProcessedAttachment {id, parent_id} format), so a + // re-run must skip them and only process the remaining attachment-3. + mockAdapter.sdkState.toDevRev!.attachmentsMetadata.lastProcessedAttachmentsIdsList = + [ + { + id: 'attachment-1', + parent_id: 'parent-1', + status: ProcessedAttachmentStatus.Success, + }, + { + id: 'attachment-2', + parent_id: 'parent-2', + status: ProcessedAttachmentStatus.Success, + }, + ]; const pool = new AttachmentsStreamingPool({ adapter: mockAdapter, @@ -188,12 +203,11 @@ describe(AttachmentsStreamingPool.name, () => { const result = await pool.streamAll(); + // attachment-1/2 are skipped (already processed); attachment-3 is appended. expect( - mockAdapter.state.toDevRev?.attachmentsMetadata + mockAdapter.sdkState.toDevRev?.attachmentsMetadata .lastProcessedAttachmentsIdsList ).toEqual([ - { id: 'attachment-1', parent_id: '', status: ProcessedAttachmentStatus.Success }, - { id: 'attachment-2', parent_id: '', status: ProcessedAttachmentStatus.Success }, { id: 'attachment-1', parent_id: 'parent-1', @@ -215,7 +229,7 @@ describe(AttachmentsStreamingPool.name, () => { }); it('should skip attachments already marked as permanently failed', async () => { - mockAdapter.state.toDevRev!.attachmentsMetadata.lastProcessedAttachmentsIdsList = + mockAdapter.sdkState.toDevRev!.attachmentsMetadata.lastProcessedAttachmentsIdsList = [ { id: 'attachment-1', @@ -236,7 +250,7 @@ describe(AttachmentsStreamingPool.name, () => { expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(2); // Only 2 out of 3 }); - it('should mark an attachment as permanently failed on any error', async () => { + it('should mark an attachment as permanently failed on a stream error', async () => { mockAdapter.processAttachment.mockResolvedValueOnce({ error: { message: 'timeout' }, }); @@ -252,7 +266,7 @@ describe(AttachmentsStreamingPool.name, () => { expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(1); expect( - mockAdapter.state.toDevRev!.attachmentsMetadata + mockAdapter.sdkState.toDevRev!.attachmentsMetadata .lastProcessedAttachmentsIdsList ).toEqual([ { @@ -322,7 +336,7 @@ describe(AttachmentsStreamingPool.name, () => { expect(warnSpy).toHaveBeenCalledTimes(3); expect( - mockAdapter.state.toDevRev!.attachmentsMetadata + mockAdapter.sdkState.toDevRev!.attachmentsMetadata .lastProcessedAttachmentsIdsList ).toEqual([]); }); @@ -330,7 +344,7 @@ describe(AttachmentsStreamingPool.name, () => { describe(AttachmentsStreamingPool.prototype.startPoolStreaming.name, () => { it('should skip already processed attachments', async () => { - mockAdapter.state.toDevRev!.attachmentsMetadata.lastProcessedAttachmentsIdsList = + mockAdapter.sdkState.toDevRev!.attachmentsMetadata.lastProcessedAttachmentsIdsList = [ { id: 'attachment-1', @@ -363,7 +377,7 @@ describe(AttachmentsStreamingPool.name, () => { await pool.streamAll(); expect( - mockAdapter.state.toDevRev!.attachmentsMetadata + mockAdapter.sdkState.toDevRev!.attachmentsMetadata .lastProcessedAttachmentsIdsList ).toEqual([ { @@ -405,7 +419,7 @@ describe(AttachmentsStreamingPool.name, () => { error ); expect( - mockAdapter.state.toDevRev!.attachmentsMetadata + mockAdapter.sdkState.toDevRev!.attachmentsMetadata .lastProcessedAttachmentsIdsList ).toEqual([ { @@ -437,7 +451,7 @@ describe(AttachmentsStreamingPool.name, () => { expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(3); expect( - mockAdapter.state.toDevRev!.attachmentsMetadata + mockAdapter.sdkState.toDevRev!.attachmentsMetadata .lastProcessedAttachmentsIdsList ).toEqual([ { @@ -768,7 +782,10 @@ describe(AttachmentsStreamingPool.name, () => { }); mockAdapter.processAttachment.mockImplementation( - async (attachment, stream) => { + async ( + attachment: NormalizedAttachment, + stream: ExternalSystemAttachmentStreamingFunction + ) => { // processAttachment should be called with the user's stream function const result = await stream({ item: attachment, @@ -838,7 +855,7 @@ describe(AttachmentsStreamingPool.name, () => { await pool.streamAll(); expect( - mockAdapter.state.toDevRev!.attachmentsMetadata + mockAdapter.sdkState.toDevRev!.attachmentsMetadata .lastProcessedAttachmentsIdsList ).toEqual([]); expect(mockAdapter.processAttachment).toHaveBeenCalledTimes(1); @@ -865,7 +882,7 @@ describe(AttachmentsStreamingPool.name, () => { await pool.streamAll(); expect( - mockAdapter.state.toDevRev!.attachmentsMetadata + mockAdapter.sdkState.toDevRev!.attachmentsMetadata .lastProcessedAttachmentsIdsList ).toEqual([ { diff --git a/src/attachments-streaming/attachments-streaming-pool.ts b/src/attachments-streaming/attachments-streaming-pool.ts index b75cdd5f..3f98eff6 100644 --- a/src/attachments-streaming/attachments-streaming-pool.ts +++ b/src/attachments-streaming/attachments-streaming-pool.ts @@ -1,18 +1,19 @@ -import { sleep } from '../common/helpers'; -import { WorkerAdapter } from '../multithreading/worker-adapter/worker-adapter'; +import { yieldToEventLoop } from '../common/helpers'; +import { ExtractionAdapter } from '../multithreading/adapters/extraction-adapter'; +import { NormalizedAttachment } from '../repo/repo.interfaces'; import { - ProcessedAttachmentStatus, ProcessedAttachment, + ProcessedAttachmentStatus, } from '../state/state.interfaces'; import { ExternalSystemAttachmentStreamingFunction, - NormalizedAttachment, ProcessAttachmentReturnType, -} from '../types'; +} from '../types/extraction'; + import { AttachmentsStreamingPoolParams } from './attachments-streaming-pool.interfaces'; export class AttachmentsStreamingPool { - private adapter: WorkerAdapter; + private adapter: ExtractionAdapter; private attachments: NormalizedAttachment[]; private batchSize: number; private delay: number | undefined; @@ -39,7 +40,7 @@ export class AttachmentsStreamingPool { status: ProcessedAttachmentStatus ): void { const attachmentsMetadata = - this.adapter.state.toDevRev?.attachmentsMetadata; + this.adapter.sdkState.toDevRev?.attachmentsMetadata; if (!attachmentsMetadata?.lastProcessedAttachmentsIdsList) { return; } @@ -59,47 +60,27 @@ export class AttachmentsStreamingPool { } } + /** + * Backfills `status` on entries recorded before the field existed (those + * only ever held successes); state from >= v1.15.2 needs no shape migration. + */ + private backfillProcessedAttachmentStatus( + attachments: ProcessedAttachment[] + ): ProcessedAttachment[] { + return attachments.map((it) => ({ + ...it, + status: it.status ?? ProcessedAttachmentStatus.Success, + })); + } + private async updateProgress() { this.totalProcessedCount++; if (this.totalProcessedCount % this.PROGRESS_REPORT_INTERVAL === 0) { console.info(`Processed ${this.totalProcessedCount} attachments so far.`); - // Sleep for 100ms to avoid blocking the event loop - await sleep(100); - } - } - - /** - * Migrates processed attachments from older state shapes to the current - * ProcessedAttachment[] format: legacy string[] IDs, and entries recorded before the - * `status` field existed (which only ever held successes). - * - * @param attachments - The attachments list to migrate (string[] or partial ProcessedAttachment[]) - * @returns Migrated array of ProcessedAttachment objects, or empty array if input is invalid - */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any - private migrateProcessedAttachments(attachments: any): ProcessedAttachment[] { - if (!attachments || !Array.isArray(attachments)) { - return []; - } - - // Migrate old string[] format - if (attachments.length > 0 && typeof attachments[0] === 'string') { - return attachments.map((it) => ({ - id: it as string, - parent_id: '', - status: ProcessedAttachmentStatus.Success, - })); + // Let a pending soft-timeout message (WorkerMessageExit) be delivered so + // adapter.isTimeout can flip before the next batch is processed. + await yieldToEventLoop(); } - - // Backfill status on entries recorded before it existed - if (attachments.length > 0 && typeof attachments[0] === 'object') { - return (attachments as ProcessedAttachment[]).map((it) => ({ - ...it, - status: it.status ?? ProcessedAttachmentStatus.Success, - })); - } - - return []; } async streamAll(): Promise { @@ -107,30 +88,27 @@ export class AttachmentsStreamingPool { `Starting download of ${this.attachments.length} attachments, streaming ${this.batchSize} at once.` ); - if (!this.adapter.state.toDevRev) { + if (!this.adapter.sdkState.toDevRev) { const error = new Error('toDevRev state is not initialized'); console.error(error); return { error }; } - // Get the list of attachments processed (successfully or not) in a previous - // (possibly incomplete) batch extraction. If no such list exists, create an empty one. + // Attachments processed (successfully or not) by a previous, possibly incomplete, run. if ( - !this.adapter.state.toDevRev.attachmentsMetadata + !this.adapter.sdkState.toDevRev.attachmentsMetadata .lastProcessedAttachmentsIdsList ) { - this.adapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList = + this.adapter.sdkState.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList = []; } - // Migrate old processed attachments to the current format. - this.adapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList = - this.migrateProcessedAttachments( - this.adapter.state.toDevRev.attachmentsMetadata + this.adapter.sdkState.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList = + this.backfillProcessedAttachmentStatus( + this.adapter.sdkState.toDevRev.attachmentsMetadata .lastProcessedAttachmentsIdsList ); - // Start initial batch of promises up to batchSize limit const initialBatchSize = Math.min(this.batchSize, this.attachments.length); const initialPromises = []; @@ -151,14 +129,11 @@ export class AttachmentsStreamingPool { } async startPoolStreaming() { - // Process attachments until the attachments array is empty while (this.attachments.length > 0) { - // If delay is set, stop streaming if (this.delay) { break; } - // If timeout is set, stop streaming if (this.adapter.isTimeout) { console.log( 'Timeout detected while streaming attachments. Stopping streaming.' @@ -166,20 +141,19 @@ export class AttachmentsStreamingPool { break; } - // Check if we can process next attachment const attachment = this.attachments.shift(); if (!attachment) { - break; // Exit if no more attachments + break; } if ( - this.adapter.state.toDevRev && - this.adapter.state.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList?.some( + this.adapter.sdkState.toDevRev && + this.adapter.sdkState.toDevRev.attachmentsMetadata.lastProcessedAttachmentsIdsList?.some( (it) => it.id == attachment.id && it.parent_id == attachment.parent_id ) ) { - continue; // Skip if the attachment was already processed (succeeded or failed) + continue; // Already processed in a previous run } try { @@ -188,9 +162,9 @@ export class AttachmentsStreamingPool { this.stream ); - // Check if rate limit was hit + // Rate limit hit if (response?.delay) { - this.delay = response.delay; // Set the delay for rate limiting + this.delay = response.delay; return; } diff --git a/src/common/constants.ts b/src/common/constants.ts index e697144c..c4ecb8de 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -1,4 +1,5 @@ import { EventType } from '../types/extraction'; + import { getLibraryVersion } from './helpers'; export const ALLOWED_EXTRACTION_EVENT_TYPES = [ @@ -57,7 +58,7 @@ export const STATEFUL_EVENT_TYPES = [ export const ARTIFACT_BATCH_SIZE = 2000; export const MAX_DEVREV_ARTIFACT_SIZE = 2 * 1024 * 1024 * 1024; // 2GB export const MAX_DEVREV_FILENAME_LENGTH = 256; -export const MAX_DEVREV_FILENAME_EXTENSION_LENGTH = 20; // 20 characters for the file extension +export const MAX_DEVREV_FILENAME_EXTENSION_LENGTH = 20; // Max SQS message size is 250KB, we want to leave some room for the other data in the message export const MAX_EVENT_SIZE_BYTES = 200_000; @@ -67,6 +68,7 @@ export const EVENT_SIZE_THRESHOLD_BYTES = Math.floor( ); export const SSOR_ATTACHMENT: string = 'ssor_attachment'; +export const UNKNOWN_EVENT_TYPE = 'UNKNOWN_EVENT_TYPE'; export enum AirSyncDefaultItemTypes { EXTERNAL_DOMAIN_METADATA = 'external_domain_metadata', @@ -83,10 +85,8 @@ export const MEMORY_LOG_INTERVAL = 30 * 1000; // 30 seconds export const DEFAULT_SLEEP_DELAY_MS = 3 * 60 * 1000; // 3 minutes /** - * Sentinel value representing an unbounded (no limit) extraction timestamp. - * Used as the resolved value for TimeValueType.UNBOUNDED, stored as workersOldest - * when the initial import has no lower time bound. The Unix epoch ensures that - * no real extraction timestamp can be earlier, preventing accidental overwrites - * of the boundary by subsequent syncs (e.g. reconciliation with absolute dates). + * Sentinel for an unbounded extraction timestamp (TimeValueType.UNBOUNDED), + * stored as workersOldest. The Unix epoch guarantees no real timestamp can be + * earlier, so subsequent syncs cannot accidentally overwrite the boundary. */ export const UNBOUNDED_DATE_TIME_VALUE = '1970-01-01T00:00:00.000Z'; diff --git a/src/common/control-protocol.test.ts b/src/common/control-protocol.test.ts deleted file mode 100644 index d9753559..00000000 --- a/src/common/control-protocol.test.ts +++ /dev/null @@ -1,173 +0,0 @@ -import { axiosClient } from '../http/axios-client-internal'; -import { createAxiosResponse } from '../tests/test-helpers'; -import { createMockEvent } from './test-utils'; -import { emit } from './control-protocol'; -import { EventType, ExtractorEventType } from '../types/extraction'; -import { LoaderEventType } from '../types/loading'; - -jest.mock('../http/axios-client-internal'); - -const mockedAxiosClient = jest.mocked(axiosClient); - -describe('control-protocol.emit', () => { - beforeEach(() => { - jest.clearAllMocks(); - mockedAxiosClient.post.mockResolvedValue(createAxiosResponse()); - }); - - it.each([ - { - title: 'extractor events', - inputEventType: EventType.StartExtractingData, - outputEventType: ExtractorEventType.DataExtractionProgress, - }, - { - title: 'loader events', - inputEventType: EventType.StartLoadingData, - outputEventType: LoaderEventType.DataLoadingProgress, - }, - { - title: 'unknown events', - inputEventType: EventType.StartExtractingData, - outputEventType: 'SOME_UNKNOWN_EVENT' as ExtractorEventType, - }, - ])( - 'sets state dates from event context for $title', - async ({ inputEventType, outputEventType }) => { - const event = createMockEvent(undefined, { - payload: { - event_type: inputEventType, - event_context: { - extract_from: '2024-01-01T00:00:00.000Z', - extract_to: '2024-06-01T00:00:00.000Z', - }, - }, - }); - - await emit({ - event, - eventType: outputEventType, - }); - - expect(mockedAxiosClient.post).toHaveBeenCalledTimes(1); - - const [, body, config] = mockedAxiosClient.post.mock.calls[0] as [ - string, - { - event_type: string; - event_context: { - extract_from?: string; - extract_to?: string; - }; - worker_metadata: Record; - }, - { headers: Record } - ]; - - expect(body).toMatchObject({ - event_type: outputEventType, - event_context: expect.objectContaining({ - extract_from: '2024-01-01T00:00:00.000Z', - extract_to: '2024-06-01T00:00:00.000Z', - }), - worker_metadata: expect.objectContaining({ - oldest_state_date: '2024-01-01T00:00:00.000Z', - newest_state_date: '2024-06-01T00:00:00.000Z', - }), - }); - - expect(config).toEqual( - expect.objectContaining({ - headers: expect.objectContaining({ - 'X-DevRev-Client-Version': expect.any(String), - }), - }) - ); - } - ); - - it.each([ - { - title: 'only extract_from is set', - extractFrom: '2024-01-01T00:00:00.000Z', - extractTo: undefined, - }, - { - title: 'only extract_to is set', - extractFrom: undefined, - extractTo: '2024-06-01T00:00:00.000Z', - }, - { - title: 'neither extract_from nor extract_to is set', - extractFrom: undefined, - extractTo: undefined, - }, - ])( - 'handles state-date absence when $title', - async ({ extractFrom, extractTo }) => { - const event = createMockEvent(undefined, { - payload: { - event_type: EventType.StartExtractingData, - event_context: { - extract_from: extractFrom, - extract_to: extractTo, - }, - }, - }); - - await emit({ - event, - eventType: ExtractorEventType.DataExtractionProgress, - }); - - const [, body] = mockedAxiosClient.post.mock.calls[0] as [ - string, - { - worker_metadata: Record; - }, - unknown - ]; - const workerMetadata = body.worker_metadata; - - expect(workerMetadata.oldest_state_date).toBe(extractFrom); - expect(workerMetadata.newest_state_date).toBe(extractTo); - } - ); - - it('overrides caller-provided worker_metadata state dates', async () => { - const event = createMockEvent(undefined, { - payload: { - event_type: EventType.StartExtractingData, - event_context: { - extract_from: '2024-01-01T00:00:00.000Z', - extract_to: '2024-06-01T00:00:00.000Z', - }, - }, - }); - - await emit({ - event, - eventType: ExtractorEventType.DataExtractionProgress, - worker_metadata: { - item_type: 'tasks', - oldest_state_date: 'should-be-overwritten', - newest_state_date: 'should-be-overwritten', - }, - }); - - const [, body] = mockedAxiosClient.post.mock.calls[0] as [ - string, - { - worker_metadata: Record; - }, - unknown - ]; - expect(body.worker_metadata).toEqual( - expect.objectContaining({ - item_type: 'tasks', - oldest_state_date: '2024-01-01T00:00:00.000Z', - newest_state_date: '2024-06-01T00:00:00.000Z', - }) - ); - }); -}); diff --git a/src/common/event-type-translation.test.ts b/src/common/event-type-translation.test.ts deleted file mode 100644 index 85b4825b..00000000 --- a/src/common/event-type-translation.test.ts +++ /dev/null @@ -1,193 +0,0 @@ -import { EventType, ExtractorEventType } from '../types/extraction'; -import { LoaderEventType } from '../types/loading'; -import { - translateExtractorEventType, - translateIncomingEventType, - translateLoaderEventType, - translateOutgoingEventType, -} from './event-type-translation'; - -describe(translateIncomingEventType.name, () => { - it.each([ - [ - EventType.ExtractionExternalSyncUnitsStart, - EventType.StartExtractingExternalSyncUnits, - ], - [EventType.ExtractionMetadataStart, EventType.StartExtractingMetadata], - [EventType.ExtractionDataStart, EventType.StartExtractingData], - [EventType.ExtractionDataContinue, EventType.ContinueExtractingData], - [EventType.ExtractionDataDelete, EventType.StartDeletingExtractorState], - [ - EventType.ExtractionAttachmentsStart, - EventType.StartExtractingAttachments, - ], - [ - EventType.ExtractionAttachmentsContinue, - EventType.ContinueExtractingAttachments, - ], - [ - EventType.ExtractionAttachmentsDelete, - EventType.StartDeletingExtractorAttachmentsState, - ], - ])('maps legacy extraction event %s to %s', (legacy, modern) => { - expect(translateIncomingEventType(legacy)).toBe(modern); - }); - - it.each([ - [EventType.StartExtractingExternalSyncUnits], - [EventType.StartExtractingMetadata], - [EventType.StartExtractingData], - [EventType.ContinueExtractingData], - [EventType.StartDeletingExtractorState], - [EventType.StartExtractingAttachments], - [EventType.ContinueExtractingAttachments], - [EventType.StartDeletingExtractorAttachmentsState], - [EventType.StartLoadingData], - [EventType.ContinueLoadingData], - [EventType.StartLoadingAttachments], - [EventType.ContinueLoadingAttachments], - [EventType.StartDeletingLoaderState], - [EventType.StartDeletingLoaderAttachmentState], - [EventType.UnknownEventType], - ])('is a no-op for already-modern event type %s', (eventType) => { - expect(translateIncomingEventType(eventType)).toBe(eventType); - }); - - it('returns the input verbatim for an unrecognised event type', () => { - const result = translateIncomingEventType('NONSENSE_EVENT' as EventType); - - expect(result).toBe('NONSENSE_EVENT'); - }); -}); - -describe(translateExtractorEventType.name, () => { - it.each([ - [ - ExtractorEventType.ExtractionExternalSyncUnitsDone, - ExtractorEventType.ExternalSyncUnitExtractionDone, - ], - [ - ExtractorEventType.ExtractionExternalSyncUnitsError, - ExtractorEventType.ExternalSyncUnitExtractionError, - ], - [ - ExtractorEventType.ExtractionMetadataDone, - ExtractorEventType.MetadataExtractionDone, - ], - [ - ExtractorEventType.ExtractionMetadataError, - ExtractorEventType.MetadataExtractionError, - ], - [ - ExtractorEventType.ExtractionDataProgress, - ExtractorEventType.DataExtractionProgress, - ], - [ - ExtractorEventType.ExtractionDataDelay, - ExtractorEventType.DataExtractionDelayed, - ], - [ - ExtractorEventType.ExtractionDataDone, - ExtractorEventType.DataExtractionDone, - ], - [ - ExtractorEventType.ExtractionDataError, - ExtractorEventType.DataExtractionError, - ], - [ - ExtractorEventType.ExtractionDataDeleteDone, - ExtractorEventType.ExtractorStateDeletionDone, - ], - [ - ExtractorEventType.ExtractionDataDeleteError, - ExtractorEventType.ExtractorStateDeletionError, - ], - [ - ExtractorEventType.ExtractionAttachmentsProgress, - ExtractorEventType.AttachmentExtractionProgress, - ], - [ - ExtractorEventType.ExtractionAttachmentsDelay, - ExtractorEventType.AttachmentExtractionDelayed, - ], - [ - ExtractorEventType.ExtractionAttachmentsDone, - ExtractorEventType.AttachmentExtractionDone, - ], - [ - ExtractorEventType.ExtractionAttachmentsError, - ExtractorEventType.AttachmentExtractionError, - ], - [ - ExtractorEventType.ExtractionAttachmentsDeleteDone, - ExtractorEventType.ExtractorAttachmentsStateDeletionDone, - ], - [ - ExtractorEventType.ExtractionAttachmentsDeleteError, - ExtractorEventType.ExtractorAttachmentsStateDeletionError, - ], - ])('maps legacy extractor event %s to %s', (legacy, modern) => { - expect(translateExtractorEventType(legacy)).toBe(modern); - }); - - it.each([ - [ExtractorEventType.DataExtractionDone], - [ExtractorEventType.DataExtractionProgress], - [ExtractorEventType.AttachmentExtractionDone], - [ExtractorEventType.MetadataExtractionDone], - [ExtractorEventType.UnknownEventType], - ])('is a no-op for already-modern extractor event %s', (eventType) => { - expect(translateExtractorEventType(eventType)).toBe(eventType); - }); -}); - -describe(translateLoaderEventType.name, () => { - it.each([ - [LoaderEventType.DataLoadingDelay, LoaderEventType.DataLoadingDelayed], - [ - LoaderEventType.AttachmentsLoadingProgress, - LoaderEventType.AttachmentLoadingProgress, - ], - [ - LoaderEventType.AttachmentsLoadingDelayed, - LoaderEventType.AttachmentLoadingDelayed, - ], - [ - LoaderEventType.AttachmentsLoadingDone, - LoaderEventType.AttachmentLoadingDone, - ], - [ - LoaderEventType.AttachmentsLoadingError, - LoaderEventType.AttachmentLoadingError, - ], - ])('maps legacy loader event %s to %s', (legacy, modern) => { - expect(translateLoaderEventType(legacy)).toBe(modern); - }); - - it.each([ - [LoaderEventType.DataLoadingDone], - [LoaderEventType.DataLoadingProgress], - [LoaderEventType.AttachmentLoadingDone], - ])('is a no-op for already-modern loader event %s', (eventType) => { - expect(translateLoaderEventType(eventType)).toBe(eventType); - }); -}); - -describe(translateOutgoingEventType.name, () => { - it('routes extractor events through translateExtractorEventType', () => { - expect( - translateOutgoingEventType(ExtractorEventType.ExtractionDataDone) - ).toBe(ExtractorEventType.DataExtractionDone); - }); - - it('routes loader events through translateLoaderEventType', () => { - expect( - translateOutgoingEventType(LoaderEventType.AttachmentsLoadingDone) - ).toBe(LoaderEventType.AttachmentLoadingDone); - }); - - it('passes through unknown event types unchanged', () => { - const unknown = 'SOME_UNKNOWN_EVENT' as ExtractorEventType; - expect(translateOutgoingEventType(unknown)).toBe(unknown); - }); -}); diff --git a/src/common/event-type-translation.ts b/src/common/event-type-translation.ts deleted file mode 100644 index f1a32abe..00000000 --- a/src/common/event-type-translation.ts +++ /dev/null @@ -1,163 +0,0 @@ -import { EventType, ExtractorEventType } from '../types/extraction'; -import { LoaderEventType } from '../types/loading'; - -/** - * Maps old incoming event type strings to new EventType enum values. - * This ensures backwards compatibility when the platform sends old event types. - * @param eventTypeString The raw event type string from the platform - * @returns The translated EventType enum value - */ -export function translateIncomingEventType(eventTypeString: string): EventType { - // Create a reverse mapping from OLD string values to NEW enum member names - const eventTypeMap: Record = { - // Old extraction event types from platform -> New enum members - [EventType.ExtractionExternalSyncUnitsStart]: - EventType.StartExtractingExternalSyncUnits, - [EventType.ExtractionMetadataStart]: EventType.StartExtractingMetadata, - [EventType.ExtractionDataStart]: EventType.StartExtractingData, - [EventType.ExtractionDataContinue]: EventType.ContinueExtractingData, - [EventType.ExtractionDataDelete]: EventType.StartDeletingExtractorState, - [EventType.ExtractionAttachmentsStart]: - EventType.StartExtractingAttachments, - [EventType.ExtractionAttachmentsContinue]: - EventType.ContinueExtractingAttachments, - [EventType.ExtractionAttachmentsDelete]: - EventType.StartDeletingExtractorAttachmentsState, - - // New extraction event types (already correct, map to new enum members) - [EventType.StartExtractingExternalSyncUnits]: - EventType.StartExtractingExternalSyncUnits, - [EventType.StartExtractingMetadata]: EventType.StartExtractingMetadata, - [EventType.StartExtractingData]: EventType.StartExtractingData, - [EventType.ContinueExtractingData]: EventType.ContinueExtractingData, - [EventType.StartDeletingExtractorState]: - EventType.StartDeletingExtractorState, - [EventType.StartExtractingAttachments]: - EventType.StartExtractingAttachments, - [EventType.ContinueExtractingAttachments]: - EventType.ContinueExtractingAttachments, - [EventType.StartDeletingExtractorAttachmentsState]: - EventType.StartDeletingExtractorAttachmentsState, - - // Loading events - [EventType.StartLoadingData]: EventType.StartLoadingData, - [EventType.ContinueLoadingData]: EventType.ContinueLoadingData, - [EventType.StartLoadingAttachments]: EventType.StartLoadingAttachments, - [EventType.ContinueLoadingAttachments]: - EventType.ContinueLoadingAttachments, - [EventType.StartDeletingLoaderState]: EventType.StartDeletingLoaderState, - [EventType.StartDeletingLoaderAttachmentState]: - EventType.StartDeletingLoaderAttachmentState, - - // Unknown - [EventType.UnknownEventType]: EventType.UnknownEventType, - }; - - const translated = eventTypeMap[eventTypeString]; - if (!translated) { - console.warn( - `Unknown event type received: ${eventTypeString}. This may indicate a new event type or a typo.` - ); - // Return the original string cast as EventType as a fallback - return eventTypeString as EventType; - } - - return translated; -} - -/** - * Translates ExtractorEventType enum values by converting old enum members to new ones. - * Old enum members are deprecated and should be replaced with new ones. - */ -export function translateExtractorEventType( - eventType: ExtractorEventType -): ExtractorEventType { - // Map old enum members to new enum members - const stringValue = eventType as string; - - const mapping: Record = { - // Old string values -> New enum members - [ExtractorEventType.ExtractionExternalSyncUnitsDone]: - ExtractorEventType.ExternalSyncUnitExtractionDone, - [ExtractorEventType.ExtractionExternalSyncUnitsError]: - ExtractorEventType.ExternalSyncUnitExtractionError, - [ExtractorEventType.ExtractionMetadataDone]: - ExtractorEventType.MetadataExtractionDone, - [ExtractorEventType.ExtractionMetadataError]: - ExtractorEventType.MetadataExtractionError, - [ExtractorEventType.ExtractionDataProgress]: - ExtractorEventType.DataExtractionProgress, - [ExtractorEventType.ExtractionDataDelay]: - ExtractorEventType.DataExtractionDelayed, - [ExtractorEventType.ExtractionDataDone]: - ExtractorEventType.DataExtractionDone, - [ExtractorEventType.ExtractionDataError]: - ExtractorEventType.DataExtractionError, - [ExtractorEventType.ExtractionDataDeleteDone]: - ExtractorEventType.ExtractorStateDeletionDone, - [ExtractorEventType.ExtractionDataDeleteError]: - ExtractorEventType.ExtractorStateDeletionError, - [ExtractorEventType.ExtractionAttachmentsProgress]: - ExtractorEventType.AttachmentExtractionProgress, - [ExtractorEventType.ExtractionAttachmentsDelay]: - ExtractorEventType.AttachmentExtractionDelayed, - [ExtractorEventType.ExtractionAttachmentsDone]: - ExtractorEventType.AttachmentExtractionDone, - [ExtractorEventType.ExtractionAttachmentsError]: - ExtractorEventType.AttachmentExtractionError, - [ExtractorEventType.ExtractionAttachmentsDeleteDone]: - ExtractorEventType.ExtractorAttachmentsStateDeletionDone, - [ExtractorEventType.ExtractionAttachmentsDeleteError]: - ExtractorEventType.ExtractorAttachmentsStateDeletionError, - }; - - // If there's a mapping, use it; otherwise return original (already new) - return mapping[stringValue] ?? eventType; -} - -/** - * Translates LoaderEventType enum values by converting old enum members to new ones. - * Old enum members are deprecated and should be replaced with new ones. - */ -export function translateLoaderEventType( - eventType: LoaderEventType -): LoaderEventType { - // Map old enum members to new enum members - const stringValue = eventType as string; - - const mapping: Record = { - // Old string values -> New enum members - [LoaderEventType.DataLoadingDelay]: LoaderEventType.DataLoadingDelayed, - [LoaderEventType.AttachmentsLoadingProgress]: - LoaderEventType.AttachmentLoadingProgress, - [LoaderEventType.AttachmentsLoadingDelayed]: - LoaderEventType.AttachmentLoadingDelayed, - [LoaderEventType.AttachmentsLoadingDone]: - LoaderEventType.AttachmentLoadingDone, - [LoaderEventType.AttachmentsLoadingError]: - LoaderEventType.AttachmentLoadingError, - }; - - // If there's a mapping, use it; otherwise return original (already new) - return mapping[stringValue] ?? eventType; -} - -/** - * Translates any outgoing event type (Extractor or Loader) to ensure new event types are used. - */ -export function translateOutgoingEventType( - eventType: ExtractorEventType | LoaderEventType -): ExtractorEventType | LoaderEventType { - // Check if it's an ExtractorEventType by checking if the value exists in ExtractorEventType - if ( - Object.values(ExtractorEventType).includes(eventType as ExtractorEventType) - ) { - return translateExtractorEventType(eventType as ExtractorEventType); - } - // Otherwise treat as LoaderEventType - if (Object.values(LoaderEventType).includes(eventType as LoaderEventType)) { - return translateLoaderEventType(eventType as LoaderEventType); - } - // If neither, return as-is - return eventType; -} diff --git a/src/common/helpers.ts b/src/common/helpers.ts index e4abeef9..d532d635 100644 --- a/src/common/helpers.ts +++ b/src/common/helpers.ts @@ -2,16 +2,6 @@ import { readFileSync } from 'fs'; import * as path from 'path'; import * as v8 from 'v8'; -import { - MAX_DEVREV_FILENAME_EXTENSION_LENGTH, - MAX_DEVREV_FILENAME_LENGTH, -} from './constants'; -import { MAX_LOG_STRING_LENGTH } from '../logger/logger.constants'; - -/** - * Gets the library version from the package.json file. - * @returns {string} The library version - */ export function getLibraryVersion() { try { const version = JSON.parse( @@ -31,52 +21,18 @@ export function getLibraryVersion() { } } -/** - * Sleeps for a given number of milliseconds. - * @param {number} ms - The number of milliseconds to sleep - * @returns {Promise} A promise that resolves after the given number of milliseconds - */ export async function sleep(ms: number) { - console.log(`Sleeping for ${ms}ms.`); return new Promise((resolve) => setTimeout(resolve, ms)); } /** - * Truncates a filename if it exceeds the maximum allowed length. - * @param {string} filename - The filename to truncate - * @returns {string} The truncated filename + * Yields once to the event loop so pending events (e.g. worker messages) can + * be delivered before continuing. */ -export function truncateFilename(filename: string): string { - // If the filename is already within the limit, return it as is. - if (filename.length <= MAX_DEVREV_FILENAME_LENGTH) { - return filename; - } - - console.warn( - `Filename length exceeds the maximum limit of ${MAX_DEVREV_FILENAME_LENGTH} characters. Truncating filename.` - ); - - const extension = filename.slice(-MAX_DEVREV_FILENAME_EXTENSION_LENGTH); - // Calculate how many characters are available for the name part after accounting for the extension and "..." - const availableNameLength = - MAX_DEVREV_FILENAME_LENGTH - MAX_DEVREV_FILENAME_EXTENSION_LENGTH - 3; // -3 for "..." - - // Truncate the name part and add an ellipsis - const truncatedFilename = filename.slice(0, availableNameLength); - - return `${truncatedFilename}...${extension}`; +export async function yieldToEventLoop(): Promise { + return new Promise((resolve) => setImmediate(resolve)); } -/** - * MemoryInfo is an interface that represents the memory usage information. - * @interface MemoryInfo - * @property {string} rssUsedMB - The RSS used in MB - * @property {string} rssUsedPercent - The RSS used percentage - * @property {string} heapUsedPercent - The heap used percentage - * @property {string} externalMB - The external memory used in MB - * @property {string} arrayBuffersMB - The array buffers memory used in MB - * @property {string} formattedMessage - The formatted message - */ export interface MemoryInfo { rssUsedMB: string; rssUsedPercent: string; // Critical for OOM detection @@ -86,10 +42,6 @@ export interface MemoryInfo { formattedMessage: string; } -/** - * Gets the memory usage information. - * @returns {MemoryInfo} The memory usage information - */ export function getMemoryUsage(): MemoryInfo { try { const memUsage = process.memoryUsage(); @@ -100,15 +52,12 @@ export function getMemoryUsage(): MemoryInfo { const effectiveMemoryLimitMB = heapLimitMB; - // Calculate heap values for consistent format const heapUsedMB = heapStats.used_heap_size / 1024 / 1024; const heapTotalMB = heapStats.heap_size_limit / 1024 / 1024; - // Calculate external and buffer values (critical for detecting stream leaks) const externalMB = memUsage.external / 1024 / 1024; const arrayBuffersMB = memUsage.arrayBuffers / 1024 / 1024; - // Critical percentages for OOM detection const rssUsedPercent = ((rssUsedMB / effectiveMemoryLimitMB) * 100).toFixed(2) + '%'; const heapUsedPercent = @@ -116,7 +65,6 @@ export function getMemoryUsage(): MemoryInfo { 2 ) + '%'; - // Detailed message showing RSS breakdown for leak detection const formattedMessage = `Memory: RSS ${rssUsedMB.toFixed( 2 )}/${effectiveMemoryLimitMB.toFixed( @@ -142,19 +90,3 @@ export function getMemoryUsage(): MemoryInfo { throw err; } } - -/** - * Truncates a message if it exceeds the maximum allowed length. - * Adds a suffix indicating how many characters were omitted. - * - * @param message - The message to truncate - * @returns Truncated message or original if within limits - */ -export function truncateMessage(message: string): string { - if (message.length > MAX_LOG_STRING_LENGTH) { - return `${message.substring(0, MAX_LOG_STRING_LENGTH)}... ${ - message.length - MAX_LOG_STRING_LENGTH - } more characters`; - } - return message; -} diff --git a/src/deprecated/adapter/index.ts b/src/deprecated/adapter/index.ts deleted file mode 100644 index 69547684..00000000 --- a/src/deprecated/adapter/index.ts +++ /dev/null @@ -1,209 +0,0 @@ -import axios from 'axios'; - -import { - AirdropEvent, - EventData, - ExtractorEvent, - ExtractorEventType, -} from '../../types/extraction'; -import { Artifact } from '../../uploader/uploader.interfaces'; - -import { AdapterState } from '../../state/state.interfaces'; - -import { STATELESS_EVENT_TYPES } from '../../common/constants'; -import { getTimeoutExtractorEventType } from '../common/helpers'; -// import { Logger } from '../../logger/logger'; -import { State, createAdapterState } from '../../state/state'; -import { translateIncomingEventType } from '../../common/event-type-translation'; -import { runWithSdkLogContext } from '../../logger/logger.context'; - -/** - * Adapter class is used to interact with Airdrop platform. The class provides - * utilities to - * - emit control events to the platform - * - update the state of the extractor - * - set the last saved state in case of a timeout - * - * @class Adapter - * @constructor - * @deprecated - * @param {AirdropEvent} event - The event object received from the platform - * @param {object=} initialState - The initial state of the adapter - * @param {boolean=} isLocalDevelopment - A flag to indicate if the adapter is being used in local development - */ - -/** - * Creates an adapter instance. - * - * @param {AirdropEvent} event - The event object received from the platform - * @param initialState - * @param {boolean=} isLocalDevelopment - A flag to indicate if the adapter is being used in local development - * @return The adapter instance - */ - -export async function createAdapter( - event: AirdropEvent, - initialState: ConnectorState, - isLocalDevelopment: boolean = false -) { - event.payload.event_type = translateIncomingEventType(event.payload.event_type); - - const newInitialState = structuredClone(initialState); - const adapterState: State = await createAdapterState({ - event, - initialState: newInitialState, - }); - - const a = new Adapter( - event, - adapterState, - isLocalDevelopment - ); - - return a; -} - -export class Adapter { - private adapterState: State; - private _artifacts: Artifact[]; - - private event: AirdropEvent; - private callbackUrl: string; - private devrevToken: string; - private startTime: number; - private heartBeatFn: ReturnType | undefined; - private exit: boolean = false; - private lambdaTimeout: number = 10 * 60 * 1000; // 10 minutes in milliseconds - private heartBeatInterval: number = 30 * 1000; // 30 seconds in milliseconds - - constructor( - event: AirdropEvent, - adapterState: State, - isLocalDevelopment: boolean = false - ) { - if (!isLocalDevelopment) { - // Logger.init(event); - } - - this.adapterState = adapterState; - this._artifacts = []; - - this.event = event; - this.callbackUrl = event.payload.event_context.callback_url; - this.devrevToken = event.context.secrets.service_account_token; - - this.startTime = Date.now(); - - // Run heartbeat every 30 seconds - this.heartBeatFn = setInterval(async () => { - const b = await this.heartbeat(); - if (b) { - this.exitAdapter(); - } - }, this.heartBeatInterval); - } - - get state(): AdapterState { - return this.adapterState.state; - } - - set state(value: AdapterState) { - this.adapterState.state = value; - } - - get artifacts(): Artifact[] { - return this._artifacts; - } - - set artifacts(value: Artifact[]) { - this._artifacts = value; - } - - /** - * Emits an event to the platform. - * - * @param {ExtractorEventType} newEventType - The event type to be emitted - * @param {EventData=} data - The data to be sent with the event - */ - async emit(newEventType: ExtractorEventType, data?: EventData) { - if (this.exit) { - console.warn( - 'Adapter is already in exit state. No more events can be emitted.' - ); - return; - } - - // We want to save the state every time we emit an event, except for the start and delete events - if (!STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) { - runWithSdkLogContext(() => - console.log(`Saving state before emitting event`) - ); - await this.adapterState.postState(this.state); - } - - const newEvent: ExtractorEvent = { - event_type: newEventType, - event_context: this.event.payload.event_context, - event_data: { - ...data, - }, - }; - - try { - await axios.post( - this.callbackUrl, - { ...newEvent }, - { - headers: { - Accept: 'application/json, text/plain, */*', - Authorization: this.devrevToken, - 'Content-Type': 'application/json', - }, - } - ); - - console.log('Successfully emitted event: ' + JSON.stringify(newEvent)); - } catch (error) { - // If this request fails the extraction will be stuck in loop and - // we need to stop it through UI or think about retrying this request - console.log( - 'Failed to emit event: ' + - JSON.stringify(newEvent) + - ', error: ' + - error - ); - } finally { - this.exitAdapter(); - } - } - - /** - * Exit the adapter. This will stop the heartbeat and no - * further events will be emitted. - */ - private exitAdapter() { - this.exit = true; - } - - /** - * Heartbeat function to check if the lambda is about to timeout. - * @returns true if 10 minutes have passed since the start of the lambda. - */ - private async heartbeat(): Promise { - if (this.exit) { - return true; - } - if (Date.now() - this.startTime > this.lambdaTimeout) { - const timeoutEventType = getTimeoutExtractorEventType( - this.event.payload.event_type - ); - if (timeoutEventType !== null) { - const { eventType, isError } = timeoutEventType; - const err = isError ? { message: 'Lambda Timeout' } : undefined; - await this.emit(eventType, { error: err, artifacts: this._artifacts }); - return true; - } - } - return false; - } -} diff --git a/src/deprecated/common/helpers.ts b/src/deprecated/common/helpers.ts deleted file mode 100644 index 41eb9477..00000000 --- a/src/deprecated/common/helpers.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { jsonl } from 'js-jsonl'; - -import { EventType, ExtractorEventType } from '../../types/extraction'; - -export function createFormData( - //eslint-disable-next-line @typescript-eslint/no-explicit-any - preparedArtifact: any, - fetchedObjects: object[] | object -): FormData { - const formData = new FormData(); - for (const item of preparedArtifact.form_data) { - formData.append(item.key, item.value); - } - - const output = jsonl.stringify(fetchedObjects); - formData.append('file', output); - - return formData; -} - -export function getTimeoutExtractorEventType(eventType: EventType): { - eventType: ExtractorEventType; - isError: boolean; -} | null { - switch (eventType) { - case EventType.ExtractionMetadataStart: - return { - eventType: ExtractorEventType.ExtractionMetadataError, - isError: true, - }; - case EventType.ExtractionDataStart: - case EventType.ExtractionDataContinue: - return { - eventType: ExtractorEventType.ExtractionDataProgress, - isError: false, - }; - case EventType.ExtractionAttachmentsStart: - case EventType.ExtractionAttachmentsContinue: - return { - eventType: ExtractorEventType.ExtractionAttachmentsProgress, - isError: false, - }; - case EventType.ExtractionExternalSyncUnitsStart: - return { - eventType: ExtractorEventType.ExtractionExternalSyncUnitsError, - isError: true, - }; - default: - console.log( - 'Event type not recognized in getTimeoutExtractorEventType function: ' + - eventType - ); - return null; - } -} diff --git a/src/deprecated/demo-extractor/external_domain_metadata.json b/src/deprecated/demo-extractor/external_domain_metadata.json deleted file mode 100644 index ea876481..00000000 --- a/src/deprecated/demo-extractor/external_domain_metadata.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "record_types": { - "users": { - "fields": { - "name": { - "is_required": true, - "type": "text", - "name": "Name", - "text": { - "min_length": 1 - } - }, - "email": { - "type": "text", - "name": "Email", - "is_required": true - } - } - }, - "contacts": { - "fields": { - "name": { - "is_required": true, - "type": "text", - "name": "Name", - "text": { - "min_length": 1 - } - }, - "email": { - "type": "text", - "name": "Email", - "is_required": true - } - } - } - } -} diff --git a/src/deprecated/demo-extractor/index.ts b/src/deprecated/demo-extractor/index.ts deleted file mode 100644 index 30e937ff..00000000 --- a/src/deprecated/demo-extractor/index.ts +++ /dev/null @@ -1,235 +0,0 @@ -import { - AirdropEvent, - EventType, - ExternalSyncUnit, - ExtractorEventType, -} from '../../types/extraction'; -import { Adapter } from '../adapter'; -import { Uploader } from '../uploader'; -import externalDomainMetadata from './external_domain_metadata.json'; - -type ConnectorState = object; - -/** - * Demo extractor is a reference implementation of an ADaaS connector to facilitate rapid immersion into ADaaS. - * - * @class DemoExtractor - * @deprecated - **/ -export class DemoExtractor { - private event: AirdropEvent; - private adapter: Adapter; - private uploader: Uploader; - - constructor(event: AirdropEvent, adapter: Adapter) { - this.event = event; - this.adapter = adapter; - this.uploader = new Uploader( - this.event.execution_metadata.devrev_endpoint, - this.event.context.secrets.service_account_token - ); - } - - async run() { - switch (this.event.payload.event_type) { - case EventType.ExtractionExternalSyncUnitsStart: { - const externalSyncUnits: ExternalSyncUnit[] = [ - { - id: 'devrev', - name: 'devrev', - description: 'Demo external sync unit', - }, - ]; - - await this.adapter.emit( - ExtractorEventType.ExtractionExternalSyncUnitsDone, - { - external_sync_units: externalSyncUnits, - } - ); - - break; - } - - case EventType.ExtractionMetadataStart: { - const { artifact, error } = await this.uploader.upload( - 'metadata_1.jsonl', - 'external_domain_metadata', - externalDomainMetadata - ); - - if (error || !artifact) { - await this.adapter.emit(ExtractorEventType.ExtractionMetadataError, { - error, - }); - return; - } - - await this.adapter.emit(ExtractorEventType.ExtractionMetadataDone, { - artifacts: [artifact], - }); - - break; - } - - case EventType.ExtractionDataStart: { - const contacts = [ - { - id: 'contact-1', - created_date: '1999-12-25T01:00:03+01:00', - modified_date: '1999-12-25T01:00:03+01:00', - data: { - email: 'johnsmith@test.com', - name: 'John Smith', - }, - }, - { - id: 'contact-2', - created_date: '1999-12-27T15:31:34+01:00', - modified_date: '2002-04-09T01:55:31+02:00', - data: { - email: 'janesmith@test.com', - name: 'Jane Smith', - }, - }, - ]; - - const { artifact, error } = await this.uploader.upload( - 'contacts_1.json', - 'contacts', - contacts - ); - - if (error || !artifact) { - await this.adapter.emit(ExtractorEventType.ExtractionDataError, { - error, - }); - - return; - } - - await this.adapter.emit(ExtractorEventType.ExtractionDataProgress, { - progress: 50, - artifacts: [artifact], - }); - - break; - } - - case EventType.ExtractionDataContinue: { - const users = [ - { - id: 'user-1', - created_date: '1999-12-25T01:00:03+01:00', - modified_date: '1999-12-25T01:00:03+01:00', - data: { - email: 'johndoe@test.com', - name: 'John Doe', - }, - }, - { - id: 'user-2', - created_date: '1999-12-27T15:31:34+01:00', - modified_date: '2002-04-09T01:55:31+02:00', - data: { - email: 'janedoe@test.com', - name: 'Jane Doe', - }, - }, - ]; - - const { artifact, error } = await this.uploader.upload( - 'users_1.json', - 'users', - users - ); - - if (error || !artifact) { - await this.adapter.emit(ExtractorEventType.ExtractionDataError, { - error, - }); - return; - } - - await this.adapter.emit(ExtractorEventType.ExtractionDataDone, { - progress: 100, - artifacts: [artifact], - }); - - break; - } - - case EventType.ExtractionDataDelete: { - await this.adapter.emit(ExtractorEventType.ExtractionDataDeleteDone); - break; - } - - case EventType.ExtractionAttachmentsStart: { - const attachment1 = ['This is attachment1.txt content']; - const { artifact, error } = await this.uploader.upload( - 'attachment1.txt', - 'attachment', - attachment1 - ); - - if (error || !artifact) { - await this.adapter.emit( - ExtractorEventType.ExtractionAttachmentsError, - { - error, - } - ); - return; - } - - await this.adapter.emit( - ExtractorEventType.ExtractionAttachmentsProgress, - { - artifacts: [artifact], - } - ); - - break; - } - - case EventType.ExtractionAttachmentsContinue: { - const attachment2 = ['This is attachment2.txt content']; - const { artifact, error } = await this.uploader.upload( - 'attachment2.txt', - 'attachment', - attachment2 - ); - - if (error || !artifact) { - await this.adapter.emit( - ExtractorEventType.ExtractionAttachmentsError, - { - error, - } - ); - return; - } - - await this.adapter.emit(ExtractorEventType.ExtractionAttachmentsDone, { - artifacts: [artifact], - }); - - break; - } - - case EventType.ExtractionAttachmentsDelete: { - await this.adapter.emit( - ExtractorEventType.ExtractionAttachmentsDeleteDone - ); - break; - } - - default: { - console.error( - 'Event in DemoExtractor run not recognized: ' + - JSON.stringify(this.event.payload.event_type) - ); - } - } - } -} diff --git a/src/deprecated/http/client.ts b/src/deprecated/http/client.ts deleted file mode 100644 index e4430437..00000000 --- a/src/deprecated/http/client.ts +++ /dev/null @@ -1,149 +0,0 @@ -import axios, { - InternalAxiosRequestConfig, - isAxiosError, - RawAxiosRequestHeaders, -} from 'axios'; -import { - RATE_LIMIT_EXCEEDED, - RATE_LIMIT_EXCEEDED_STATUS_CODE, -} from '../../http/constants'; -import { HTTPResponse } from '../../http/types'; - -export const defaultResponse: HTTPResponse = { - data: { - delay: 0, - nextPage: 1, - records: [], - }, - message: '', - success: false, -}; - -/** - * HTTPClient class to make HTTP requests - * @deprecated - */ -export class HTTPClient { - private retryAfter = 0; - private retryAt = 0; - private axiosInstance = axios.create(); - - constructor() { - // Add request interceptor to check for retryAfter before making a request - this.axiosInstance.interceptors.request.use( - (config: InternalAxiosRequestConfig) => { - // Check if retryAfter is not 0 and return a LIMIT_EXCEEDED error - if (this.retryAfter !== 0) { - // check if the current time is greater than the retryAt time - const currentTime = new Date().getTime(); - if (currentTime < this.retryAt) { - console.error( - 'Rate limit exceeded. Interceptor has retryAfter: ' + - this.retryAfter - ); - // Rate limit exceeded. - return Promise.reject(RATE_LIMIT_EXCEEDED); - } else { - // Reset the retryAfter - this.retryAfter = 0; - } - } - return config; - }, - (error) => { - return Promise.reject(error); - } - ); - } - - /** - * - * Function to make a GET call to the endpoint. - * There is special handling for rate limit exceeded error. - * In case of rate limit exceeded, the function returns success as true and the delay time in seconds - * In case of any other error, the function returns success as false and the error message - */ - async getCall( - endpoint: string, - headers: RawAxiosRequestHeaders, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - params?: any - ): Promise { - // Return the LIMIT_EXCEEDED error if the retryAfter is not 0 - try { - const res = await this.axiosInstance.get(endpoint, { - headers: headers, - params: params, - }); - return { - ...defaultResponse, - data: { - delay: 0, - records: res.data, - }, - success: true, - }; - } catch (error: unknown) { - console.log('Error in getCall: ' + JSON.stringify(error)); - // send error to adapter - if (isAxiosError(error)) { - if (error.response?.status === RATE_LIMIT_EXCEEDED_STATUS_CODE) { - this.retryAfter = error.response.headers['retry-after'] - ? error.response.headers['retry-after'] - : 0; - this.retryAt = new Date().getTime() + this.retryAfter * 1000; - console.warn( - 'Rate limit exceeded. Error code: ' + - error.response.status + - ' RetryAfter: ' + - this.retryAfter + - ' RetryAt: ' + - this.retryAt - ); - return { - data: { - delay: this.retryAfter, - records: [], - }, - message: RATE_LIMIT_EXCEEDED, - success: true, - }; - } - if (error.response) { - return { ...defaultResponse, message: error.response.data }; - } else { - return { ...defaultResponse, message: error.message }; - } - } else { - if (this.retryAfter !== 0) { - console.warn( - 'Rate limit exceeded. Going to return the following response: ' + - JSON.stringify(error) - ); - return { - data: { - delay: this.retryAfter, - records: [], - }, - message: - typeof error === 'string' - ? error - : JSON.stringify(error, Object.getOwnPropertyNames(error)), - success: true, - }; - } - return { - data: { - delay: this.retryAfter, - records: [], - }, - message: - typeof error === 'string' - ? error - : JSON.stringify(error, Object.getOwnPropertyNames(error)), - success: false, - }; - } - } - } -} diff --git a/src/deprecated/uploader/index.ts b/src/deprecated/uploader/index.ts deleted file mode 100644 index eca7203b..00000000 --- a/src/deprecated/uploader/index.ts +++ /dev/null @@ -1,156 +0,0 @@ -import { betaSDK, client } from '@devrev/typescript-sdk'; -import fs, { promises as fsPromises } from 'fs'; -import { axiosClient } from '../../http/axios-client-internal'; -import { Artifact, UploadResponse } from '../../uploader/uploader.interfaces'; -import { createFormData } from '../common/helpers'; - -/** - * Uploader class is used to upload files to the DevRev platform. - * The class provides utilities to - * - prepare artifact - * - upload artifact - * - return the artifact information to the platform - * - * @class Uploader - * @constructor - * @param {string} endpoint - The endpoint of the DevRev platform - * @param {string} token - The token to authenticate with the DevRev platform - * @param {boolean} local - Flag to indicate if the uploader should upload to the file-system. - */ -export class Uploader { - private betaDevrevSdk: betaSDK.Api; - private local: boolean; - constructor(endpoint: string, token: string, local = false) { - this.betaDevrevSdk = client.setupBeta({ - endpoint, - token, - }); - this.local = local; - } - - /** - * - * Uploads the file to the DevRev platform. The file is uploaded to the platform - * and the artifact information is returned. - * - * @param {string} filename - The name of the file to be uploaded - * @param {string} entity - The entity type of the file to be uploaded - * @param {object[] | object} fetchedObjects - The objects to be uploaded - * @param filetype - The type of the file to be uploaded - * @returns {Promise} - The response object containing the artifact information - */ - async upload( - filename: string, - entity: string, - fetchedObjects: object[] | object, - filetype: string = 'application/jsonl+json' - ): Promise { - if (this.local) { - await this.downloadToLocal(filename, fetchedObjects); - } - - const preparedArtifact = await this.prepareArtifact(filename, filetype); - - if (!preparedArtifact) { - return { - artifact: undefined, - error: { message: 'Error while preparing artifact' }, - }; - } - - const uploadedArtifact = await this.uploadToArtifact( - preparedArtifact, - fetchedObjects - ); - - if (!uploadedArtifact) { - return { - artifact: undefined, - error: { message: 'Error while uploading artifact' }, - }; - } - - // If file was successfully uploaded we want to post data about that file when emitting - const itemCount = Array.isArray(fetchedObjects) ? fetchedObjects.length : 1; - const artifact: Artifact = { - id: preparedArtifact.id, - item_type: entity, - item_count: itemCount, - }; - - console.log(`Artifact uploaded successfully: ${artifact.id}`); - - return { artifact, error: undefined }; - } - - private async prepareArtifact( - filename: string, - filetype: string - ): Promise { - try { - const response = await this.betaDevrevSdk.artifactsPrepare({ - file_name: filename, - file_type: filetype, - }); - - return response.data; - } catch (error) { - console.error('Error while preparing artifact: ' + error); - return null; - } - } - - private async uploadToArtifact( - // eslint-disable-next-line @typescript-eslint/no-explicit-any - preparedArtifact: any, - fetchedObjects: object[] | object - // eslint-disable-next-line @typescript-eslint/no-explicit-any - ): Promise { - const formData = createFormData(preparedArtifact, fetchedObjects); - try { - const response = await axiosClient.post(preparedArtifact.url, formData, { - headers: { - 'Content-Type': 'multipart/form-data', - }, - }); - - return response; - } catch (error) { - console.error('Error while uploading artifact: ' + error); - return null; - } - } - - private async downloadToLocal( - filePath: string, - fetchedObjects: object | object[] - ) { - console.log(`Uploading ${filePath} to local file system`); - try { - if (!fs.existsSync('extracted_files')) { - fs.mkdirSync('extracted_files'); - } - - const timestamp = new Date().getTime(); - const fileHandle = await fsPromises.open( - `extracted_files/${timestamp}_${filePath}`, - 'w' - ); - let objArray = []; - if (!Array.isArray(fetchedObjects)) { - objArray.push(fetchedObjects); - } else { - objArray = fetchedObjects; - } - for (const jsonObject of objArray) { - const jsonLine = JSON.stringify(jsonObject) + '\n'; - await fileHandle.write(jsonLine); - } - await fileHandle.close(); - console.log('Data successfully written to', filePath); - } catch (error) { - console.error('Error writing data to file:', error); - return Promise.reject(error); - } - } -} diff --git a/src/http/axios-client.ts b/src/http/axios-client.ts deleted file mode 100644 index 703b1e59..00000000 --- a/src/http/axios-client.ts +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Axios client setup with retry capabilities using axios-retry. - * - * This module exports an Axios client instance (`axiosClient`) that is configured to automatically retry - * failed requests under certain conditions. - * - * Retry Conditions: - * 1. Network errors (where no response is received). - * 2. Idempotent requests (defaults include GET, HEAD, OPTIONS, PUT). - * 3. All 5xx server errors. - * - * Retry Strategy: - * - A maximum of 5 retries are attempted. - * - Exponential backoff delay is applied between retries, increasing with each retry attempt. - * - * Additional Features: - * - When the maximum number of retry attempts is reached, sensitive headers (like authorization) - * are removed from error logs for security reasons. - * - * Exported: - * - `axios`: Original axios instance for additional customizations or direct use. - * - `axiosClient`: Configured axios instance with retry logic. - */ - -import axios, { AxiosError } from 'axios'; -import axiosRetry from 'axios-retry'; -import { runWithUserLogContext } from '../logger/logger.context'; - -const axiosClient = axios.create(); - -axiosRetry(axiosClient, { - retries: 5, - retryDelay: (retryCount, error) => { - // exponential backoff algorithm: 1 * 2 ^ retryCount * 1000ms - const delay = axiosRetry.exponentialDelay(retryCount, error, 1000); - - runWithUserLogContext(() => - console.warn( - `Request to ${error.config?.url} failed with response status code ${ - error.response?.status - }. Method ${ - error.config?.method - }. Retry count: ${retryCount}. Retrying in ${Math.round( - delay / 1000 - )}s.` - ) - ); - - return delay; - }, - retryCondition: (error: AxiosError) => { - return ( - (axiosRetry.isNetworkOrIdempotentRequestError(error) && - error.response?.status !== 429) || - (error.response?.status ?? 0) >= 500 - ); - }, - onMaxRetryTimesExceeded(error: AxiosError) { - delete error.config?.headers?.authorization; - delete error.config?.headers?.Authorization; - delete error.request._header; - runWithUserLogContext(() => - console.warn('Max retry times exceeded. Error', error) - ); - }, -}); - -export { axios, axiosClient }; diff --git a/src/http/axios-client-internal.test.ts b/src/http/client.test.ts similarity index 99% rename from src/http/axios-client-internal.test.ts rename to src/http/client.test.ts index daa9e07f..404742c2 100644 --- a/src/http/axios-client-internal.test.ts +++ b/src/http/client.test.ts @@ -1,5 +1,6 @@ import { mockServer } from '../tests/jest.setup'; -import { axiosClient } from './axios-client-internal'; + +import { axiosClient } from './client'; jest.setTimeout(60000); diff --git a/src/http/axios-client-internal.ts b/src/http/client.ts similarity index 80% rename from src/http/axios-client-internal.ts rename to src/http/client.ts index 6286b62d..4314c453 100644 --- a/src/http/axios-client-internal.ts +++ b/src/http/client.ts @@ -43,12 +43,11 @@ axiosRetry(axiosClient, { error.response?.headers?.['retry-after'] || error.response?.headers?.['Retry-After']; - // 5xx errors if (error.response?.status && error.response.status >= 500) { return true; } - // 429 errors when retry-after header is present + // 429 only when a valid non-negative Retry-After header is present else if ( error.response?.status && error.response.status === 429 && @@ -59,7 +58,7 @@ axiosRetry(axiosClient, { return true; } - // Network errors for idempotent requests if not 429, because 429 is handled above + // Network errors for idempotent requests; 429 already handled above else if ( axiosRetry.isNetworkOrIdempotentRequestError(error) && error.response?.status !== 429 @@ -67,16 +66,11 @@ axiosRetry(axiosClient, { return true; } - // Request timeout errors (ECONNABORTED) — axios-retry explicitly excludes - // ECONNABORTED from isNetworkError, so we handle it here separately. - // Axios only produces ECONNABORTED on client-side timeouts and browser - // cancellations, never on server responses, so no response guard is needed. + // axios-retry excludes ECONNABORTED (client-side timeout only, never a + // server response) from isNetworkError, so handle it here separately. else if (error.code === 'ECONNABORTED') { return true; - } - - // all other errors - else { + } else { return false; } }, diff --git a/src/http/constants.ts b/src/http/constants.ts deleted file mode 100644 index 25cfbf91..00000000 --- a/src/http/constants.ts +++ /dev/null @@ -1,3 +0,0 @@ -export const RATE_LIMIT_EXCEEDED = 'LIMIT_EXCEEDED'; -export const LAMBDA_LIMIT_EXCEEDED = 'LAMBDA_LIMIT_EXCEEDED'; -export const RATE_LIMIT_EXCEEDED_STATUS_CODE = 429; diff --git a/src/http/index.ts b/src/http/index.ts deleted file mode 100644 index 4edc2c55..00000000 --- a/src/http/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './axios-client'; -export * from './types'; diff --git a/src/http/types.ts b/src/http/types.ts deleted file mode 100644 index 3206229a..00000000 --- a/src/http/types.ts +++ /dev/null @@ -1,16 +0,0 @@ -/** - * HTTP Response type - * @deprecated - */ -export type HTTPResponse = { - success: boolean; - message: string; - data: Data; -}; - -interface Data { - records: object[]; // List of records of the entity - delay: number; // Delay in seconds(used for ratelimiting), Time to wait before next call - nextPage?: number; // The next page of the entity to be processed - metadata?: object; // Other information that should be returned -} diff --git a/src/index.ts b/src/index.ts index 26dd8ff5..07f15f00 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,22 +1,166 @@ +// ──────────────────────────────────────────────────────────────────────────── +// Public API barrel for @devrev/airsync-sdk. Single source of the public +// surface: every export is named explicitly (no `export *`), so any public API +// change shows up as a diff here. +// ──────────────────────────────────────────────────────────────────────────── + +// ── Entry points & adapters ── +export { ExtractionAdapter } from './multithreading/adapters/extraction-adapter'; +export { LoadingAdapter } from './multithreading/adapters/loading-adapter'; +export { + processExtractionTask, + processLoadingTask, +} from './multithreading/process-task'; +export { spawn } from './multithreading/spawn/spawn'; + +// ── Worker contract types ── +export type { + ExtractionScope, + ProcessTaskInterface, + SpawnFactoryInterface, + SpawnInterface, + TaskAdapterInterface, + TaskResult, + TaskStatus, + WorkerAdapterInterface, + WorkerAdapterOptions, + WorkerPathOverrides, +} from './types/workers'; + +// ── Constants & enums ── export { AirSyncDefaultItemTypes } from './common/constants'; -export { ExtractionCommonError } from './common/errors'; -export * from './common/install-initial-domain-mapping'; -export * from './deprecated/adapter'; -export * from './deprecated/demo-extractor'; -export * from './deprecated/http/client'; -export * from './deprecated/uploader'; -export * from './http'; -export { formatAxiosError, serializeAxiosError } from './logger/logger'; -export { MockServer } from './mock-server/mock-server'; +export { UNBOUNDED_DATE_TIME_VALUE } from './common/constants'; +export { ExtractionCommonError } from './types/errors'; + +// ── Domain mapping install ── +export { installInitialDomainMapping } from './state/install-initial-domain-mapping'; + +// ── Error formatting ── +export { serializeError } from './logger/logger'; + +// ── Common types ── +export type { ErrorRecord, InitialDomainMapping } from './types/common'; +export { SyncMode } from './types/common'; + +// ── Extraction types ── +export type { + AirSyncEvent, + AirSyncMessage, + ConnectionData, + EventContext, + EventData, + ExternalProcessAttachmentFunction, + ExternalSyncUnit, + ExternalSystemAttachmentIteratorFunction, + ExternalSystemAttachmentReducerFunction, + ExternalSystemAttachmentStreamingFunction, + ExternalSystemAttachmentStreamingParams, + ExternalSystemAttachmentStreamingResponse, + ExtractorEvent, + HttpStreamResponse, + ProcessAttachmentReturnType, + TimeValue, +} from './types/extraction'; +export { + EventType, + ExtractorEventType, + InitialSyncScope, + TimeUnit, + TimeValueType, +} from './types/extraction'; + +// ── Loading types ── +export type { + ExternalSystemAttachment, + ExternalSystemItem, + ExternalSystemItemLoadingParams, + ExternalSystemItemLoadingResponse, + ItemTypeToLoad, +} from './types/loading'; +export { LoaderEventType } from './types/loading'; + +// ── Repo types ── +export type { + Item, + NormalizedAttachment, + NormalizedItem, + RepoInterface, +} from './repo/repo.interfaces'; + +// ── Mappers ── +export { Mappers } from './mappers/mappers'; +export type { + MappersCreateParams, + MappersGetByExternalIdParams, + MappersGetByTargetIdParams, + MappersUpdateParams, +} from './mappers/mappers.interfaces'; +export { + SyncMapperRecordStatus, + SyncMapperRecordTargetType, +} from './mappers/mappers.interfaces'; + +// ── Uploader types ── +export type { + Artifact, + ArtifactsPrepareResponse, + SsorAttachment, + StreamAttachmentsResponse, + StreamResponse, + UploadResponse, +} from './uploader/uploader.interfaces'; + +// ── External domain metadata types ── +export type { + CollectionData, + ConditionalPrivilegeData, + CustomLinkData, + CustomLinkNames, + CustomStage, + CustomState, + EnumData, + EnumValue, + EnumValueKey, + ExternalDomainMetadata, + Field, + FieldCondition, + FieldConditionComparator, + FieldConditionEffect, + FieldConditions, + FieldKey, + FieldPrivilegeData, + FieldReferenceData, + FieldType, + FloatData, + IntData, + PermissionData, + RecordType, + RecordTypeCategory, + RecordTypeCategoryKey, + RecordTypeKey, + RecordTypePrivilegeData, + RecordTypeScope, + ReferenceData, + ReferenceDetail, + ReferenceType, + SchemaVersion, + StageDiagram, + StageKey, + StateKey, + StructData, + StructType, + StructTypeKey, + TargetTypeKeyData, + TextData, + TypedReferenceData, +} from './types/external-domain-metadata'; + +// ── Testing utilities (public test-support surface) ── +export type { DeepPartial } from './testing/mock-event'; +export { createMockEvent } from './testing/mock-event'; +export { MOCK_SERVER_DEFAULT_URL, MockServer } from './testing/mock-server'; export type { RequestInfo, RetryConfig, RouteConfig, -} from './mock-server/mock-server.interfaces'; -export { processTask } from './multithreading/process-task'; -export { spawn } from './multithreading/spawn/spawn'; -export { WorkerAdapter } from './multithreading/worker-adapter/worker-adapter'; -export { createMockEvent, MOCK_SERVER_DEFAULT_URL } from './common/test-utils'; -export type { DeepPartial } from './common/test-utils'; -export * from './types'; -export * from './types/workers'; +} from './testing/mock-server.interfaces'; diff --git a/src/logger/logger.constants.ts b/src/logger/logger.constants.ts deleted file mode 100644 index b183cb43..00000000 --- a/src/logger/logger.constants.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { InspectOptions } from 'node:util'; - -export const MAX_LOG_STRING_LENGTH = 10000; -export const MAX_LOG_DEPTH = 10; -export const MAX_LOG_ARRAY_LENGTH = 100; - -export const INSPECT_OPTIONS: InspectOptions = { - compact: false, - breakLength: Infinity, - depth: MAX_LOG_DEPTH, - maxArrayLength: MAX_LOG_ARRAY_LENGTH, - maxStringLength: MAX_LOG_STRING_LENGTH, -}; diff --git a/src/logger/logger.context.ts b/src/logger/logger.context.ts deleted file mode 100644 index 59378788..00000000 --- a/src/logger/logger.context.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { AsyncLocalStorage } from 'node:async_hooks'; - -/** - * Async-aware context storage for tracking whether code is executing within SDK or user context. - * - * AsyncLocalStorage is used instead of a simple global variable because it automatically - * propagates the context across async boundaries (await, Promise.then, setTimeout, etc.). - * This ensures that even after multiple await calls in user code, the logging context - * remains correct without requiring explicit context passing through function parameters. - * - * The stored boolean value indicates: - * - `true`: Code is executing within SDK internals (logs tagged as is_sdk_log: true) - * - `false`: Code is executing within user-provided handlers (logs tagged as is_sdk_log: false) - */ -const sdkLogContext = new AsyncLocalStorage(); - -/** - * Executes a function within user log context, marking all logs as user-originated. - * - * Use this to wrap user-provided callback functions (e.g., task handlers, event callbacks). - * The context automatically propagates through any async operations within the function, - * ensuring that all console.log calls inside are correctly tagged as user logs. - * - * @template T - The return type of the function - * @param fn - The function to execute within user context (can be sync or async) - * @returns The result of the function execution - * - * @example - * ```typescript - * await runWithUserLogContext(async () => { - * console.log('This is a user log'); // is_sdk_log: false - * await someAsyncOperation(); - * console.log('Still a user log'); // is_sdk_log: false (context preserved) - * }); - * ``` - */ -export function runWithUserLogContext(fn: () => T): T { - return sdkLogContext.run(false, fn); -} - -/** - * Executes a function within SDK log context, marking all logs as SDK-originated. - * - * Use this to wrap SDK internal operations (e.g., emit, postState, adapter methods). - * The context automatically propagates through any async operations within the function, - * ensuring that all console.log calls inside are correctly tagged as SDK logs. - * - * This allows proper nesting: SDK code can call user code via runWithUserLogContext, - * and when control returns to SDK code, logs are correctly attributed. - * - * @template T - The return type of the function - * @param fn - The function to execute within SDK context (can be sync or async) - * @returns The result of the function execution - * - * @example - * ```typescript - * await runWithSdkLogContext(async () => { - * console.log('SDK internal log'); // is_sdk_log: true - * runWithUserLogContext(() => { - * console.log('User handler log'); // is_sdk_log: false - * }); - * console.log('Back to SDK log'); // is_sdk_log: true - * }); - * ``` - */ -export function runWithSdkLogContext(fn: () => T): T { - return sdkLogContext.run(true, fn); -} - -/** - * Retrieves the current SDK log context value. - * - * Returns whether the current execution context is within SDK code (true) or user code (false). - * If no context has been set (e.g., during testing or edge cases), returns the provided default. - * - * @param defaultValue - The value to return if no context is currently set - * @returns `true` if in SDK context, `false` if in user context, or defaultValue if unset - */ -export function getSdkLogContextValue(defaultValue: boolean): boolean { - const storeValue = sdkLogContext.getStore(); - if (typeof storeValue === 'boolean') { - return storeValue; - } - return defaultValue; -} diff --git a/src/logger/logger.interfaces.ts b/src/logger/logger.interfaces.ts index 9c39bde9..f19c208c 100644 --- a/src/logger/logger.interfaces.ts +++ b/src/logger/logger.interfaces.ts @@ -1,9 +1,10 @@ import type { RawAxiosResponseHeaders } from 'axios'; -import type { AirdropEvent, EventContext } from '../types/extraction'; + +import type { AirSyncEvent, EventContext } from '../types/extraction'; import type { WorkerAdapterOptions } from '../types/workers'; export interface LoggerFactoryInterface { - event: AirdropEvent; + event: AirSyncEvent; options?: WorkerAdapterOptions; } @@ -48,5 +49,4 @@ export interface AxiosErrorResponse { export interface LoggerTags extends EventContext { sdk_version: string; - is_sdk_log: boolean; } diff --git a/src/logger/logger.test.ts b/src/logger/logger.test.ts index 71374ec7..d2b3cb8e 100644 --- a/src/logger/logger.test.ts +++ b/src/logger/logger.test.ts @@ -1,20 +1,21 @@ -import { AxiosError } from 'axios'; import { inspect } from 'node:util'; + +import { AxiosError } from 'axios'; + import { LIBRARY_VERSION } from '../common/constants'; +import { createMockEvent } from '../testing/mock-event'; import { mockServer } from '../tests/jest.setup'; -import { createMockEvent } from '../common/test-utils'; -import { AirdropEvent, EventType } from '../types/extraction'; +import { AirSyncEvent, EventType } from '../types/extraction'; import { WorkerAdapterOptions } from '../types/workers'; + import { getPrintableState, + INSPECT_OPTIONS as EXPECTED_INSPECT_OPTIONS, Logger, + MAX_LOG_STRING_LENGTH, serializeAxiosError, serializeError, } from './logger'; -import { - INSPECT_OPTIONS as EXPECTED_INSPECT_OPTIONS, - MAX_LOG_STRING_LENGTH, -} from './logger.constants'; // Mock console methods const mockConsoleInfo = jest.spyOn(console, 'info').mockImplementation(); @@ -32,7 +33,7 @@ jest.mock('node:worker_threads', () => { }); describe(Logger.name, () => { - let mockEvent: AirdropEvent; + let mockEvent: AirSyncEvent; let mockOptions: WorkerAdapterOptions; beforeEach(() => { @@ -61,7 +62,6 @@ describe(Logger.name, () => { expect(tags).toEqual({ ...mockEvent.payload.event_context, sdk_version: LIBRARY_VERSION, - is_sdk_log: true, }); }); @@ -79,7 +79,6 @@ describe(Logger.name, () => { message, ...mockEvent.payload.event_context, sdk_version: LIBRARY_VERSION, - is_sdk_log: true, }) ); }); @@ -99,7 +98,6 @@ describe(Logger.name, () => { message: expectedMessage, ...mockEvent.payload.event_context, sdk_version: LIBRARY_VERSION, - is_sdk_log: true, }) ); }); @@ -120,7 +118,6 @@ describe(Logger.name, () => { message: `${text} ${expectedDataMessage}`, ...mockEvent.payload.event_context, sdk_version: LIBRARY_VERSION, - is_sdk_log: true, }) ); }); @@ -142,7 +139,6 @@ describe(Logger.name, () => { message: `${text1} ${expectedDataMessage} ${text2}`, ...mockEvent.payload.event_context, sdk_version: LIBRARY_VERSION, - is_sdk_log: true, }) ); }); @@ -346,7 +342,6 @@ describe(Logger.name, () => { const logObject = JSON.parse(callArgs); expect(logObject.message).toBe(''); expect(logObject.sdk_version).toBe(LIBRARY_VERSION); - expect(logObject.is_sdk_log).toBe(true); }); it('[edge] should handle null and undefined values in log arguments', () => { diff --git a/src/logger/logger.ts b/src/logger/logger.ts index 6d06ef09..92a44e7d 100644 --- a/src/logger/logger.ts +++ b/src/logger/logger.ts @@ -1,14 +1,12 @@ -import { AxiosError, isAxiosError, RawAxiosResponseHeaders } from 'axios'; - import { Console } from 'node:console'; -import { inspect } from 'node:util'; +import { inspect, InspectOptions } from 'node:util'; import { isMainThread, parentPort } from 'node:worker_threads'; +import { AxiosError, isAxiosError, RawAxiosResponseHeaders } from 'axios'; + import { LIBRARY_VERSION } from '../common/constants'; import { WorkerAdapterOptions, WorkerMessageSubject } from '../types/workers'; -import { INSPECT_OPTIONS } from './logger.constants'; -import { getSdkLogContextValue } from './logger.context'; import { AxiosErrorResponse, LoggerFactoryInterface, @@ -17,11 +15,34 @@ import { PrintableArray, PrintableState, } from './logger.interfaces'; -import { truncateMessage } from '../common/helpers'; + +// ── Log formatting ── + +export const MAX_LOG_STRING_LENGTH = 10000; +const MAX_LOG_DEPTH = 10; +const MAX_LOG_ARRAY_LENGTH = 100; + +export const INSPECT_OPTIONS: InspectOptions = { + compact: false, + breakLength: Infinity, + depth: MAX_LOG_DEPTH, + maxArrayLength: MAX_LOG_ARRAY_LENGTH, + maxStringLength: MAX_LOG_STRING_LENGTH, +}; + +export function truncateMessage(message: string): string { + if (message.length > MAX_LOG_STRING_LENGTH) { + return `${message.substring(0, MAX_LOG_STRING_LENGTH)}... ${ + message.length - MAX_LOG_STRING_LENGTH + } more characters`; + } + return message; +} /** - * Custom logger that extends Node.js Console with context-aware logging. - * Handles local development, main thread, and worker thread logging differently. + * Console replacement that tags every log line with the event context. Worker + * threads forward log lines to the main thread, because the snap-in platform + * only captures logs written through the main thread's console. */ export class Logger extends Console { private originalConsole: Console; @@ -35,16 +56,9 @@ export class Logger extends Console { this.tags = { ...event.payload.event_context, sdk_version: LIBRARY_VERSION, - is_sdk_log: true, }; } - /** - * Converts any value to a string using `util.inspect()` for complex types. - * - * @param value - The value to convert - * @returns String representation of the value - */ private valueToString(value: unknown): string { if (typeof value === 'string') { return value; @@ -52,21 +66,8 @@ export class Logger extends Console { return inspect(value, INSPECT_OPTIONS); } - /** - * Logs a pre-formatted message string to the console. - * In production mode, wraps the message with JSON formatting and event context tags. - * In local development mode, logs the message directly without JSON wrapping. - * This is useful when you need to log already-stringified content. - * - * @param message - The pre-formatted message string to log - * @param level - Log level (info, warn, error) - * @param isSdkLog - Flag indicating if the log originated from the SDK - */ - logFn( - message: string, - level: LogLevel, - isSdkLog: boolean = getSdkLogContextValue(true) - ): void { + /** In production wraps the message in JSON with event context tags; in local development logs as-is. */ + logFn(message: string, level: LogLevel): void { if (this.options?.isLocalDevelopment) { this.originalConsole[level](message); return; @@ -75,32 +76,20 @@ export class Logger extends Console { const logObject = { message, ...this.tags, - is_sdk_log: isSdkLog, }; this.originalConsole[level](JSON.stringify(logObject)); } - /** - * Stringifies and logs arguments to the appropriate destination. - * On main thread, converts arguments to strings and calls logFn. - * In worker threads, forwards stringified arguments to the main thread for processing. - * All arguments are converted to strings using util.inspect and joined with spaces. - * - * @param args - Values to log (will be stringified and truncated if needed) - * @param level - Log level (info, warn, error) - */ private stringifyAndLog(args: unknown[], level: LogLevel): void { let stringifiedArgs = args.map((arg) => this.valueToString(arg)).join(' '); stringifiedArgs = truncateMessage(stringifiedArgs); - const isSdkLog = getSdkLogContextValue(true); - if (isMainThread) { - this.logFn(stringifiedArgs, level, isSdkLog); + this.logFn(stringifiedArgs, level); } else { parentPort?.postMessage({ subject: WorkerMessageSubject.WorkerMessageLog, - payload: { stringifiedArgs, level, isSdkLog }, + payload: { stringifiedArgs, level }, }); } } @@ -121,14 +110,8 @@ export class Logger extends Console { this.stringifyAndLog(args, LogLevel.ERROR); } } -/** - * Converts a state object into a printable format where arrays are summarized. - * Arrays show their length, first item, and last item instead of all elements. - * Objects are recursively processed and primitives are returned as-is. - * - * @param state - State object to convert - * @returns Printable representation with summarized arrays - */ + +/** Summarizes arrays as `{ length, firstItem, lastItem }` instead of listing all elements. */ // eslint-disable-next-line @typescript-eslint/no-explicit-any export function getPrintableState(state: Record): PrintableState { // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -153,21 +136,13 @@ export function getPrintableState(state: Record): PrintableState { return processValue(state) as PrintableState; } -/** - * Serializes an error into a structured format. - * Automatically detects and formats Axios errors with HTTP details. - * Returns other error types as-is. - * - * @param error - Error to serialize - * @returns Serialized error or original if not an Axios error - */ +/** Serializes any error into a loggable string; Axios errors get HTTP details. */ export function serializeError(error: unknown): string { if (isAxiosError(error)) { return JSON.stringify(serializeAxiosError(error)); } if (error instanceof Error) { - // Include error name (e.g. TypeError, RangeError) alongside message - // for easier debugging + // Include non-default error name (e.g. TypeError) for easier debugging return error.name !== 'Error' ? `${error.name}: ${error.message}` : error.message; @@ -199,14 +174,6 @@ export function serializeError(error: unknown): string { return stringified; } -/** - * Serializes an Axios error into a structured format with HTTP request/response details. - * Extracts method, URL, parameters, status code, headers, and data. - * Includes CORS/network failure indicator when no response is available. - * - * @param error - Axios error to serialize - * @returns Structured object with error details - */ export function serializeAxiosError(error: AxiosError): AxiosErrorResponse { const serializedAxiosError: AxiosErrorResponse = { config: { @@ -232,14 +199,3 @@ export function serializeAxiosError(error: AxiosError): AxiosErrorResponse { return serializedAxiosError; } - -/** - * Formats an Axios error to a printable format. - * - * @param error - Axios error to format - * @returns Formatted error object - * @deprecated Use {@link serializeAxiosError} instead - */ -export function formatAxiosError(error: AxiosError): object { - return serializeAxiosError(error); -} diff --git a/src/mappers/mappers.interface.ts b/src/mappers/mappers.interface.ts deleted file mode 100644 index b37272a1..00000000 --- a/src/mappers/mappers.interface.ts +++ /dev/null @@ -1,307 +0,0 @@ -import { AirdropEvent } from '../types'; -import { DonV2 } from '../types/loading'; -import { WorkerAdapterOptions } from '../types/workers'; - -/** - * Configuration interface for creating a Mappers instance. - */ -export interface MappersFactoryInterface { - event: AirdropEvent; - options?: WorkerAdapterOptions; -} - -/** - * Parameters for updating a sync mapper record. - */ -export interface UpdateSyncMapperRecordParams { - /** External system IDs to add */ - external_ids: { - add: string[]; - }; - /** - * Optional map that labels values in `external_ids` with their usage context. - * Use for example when an external system requires different identifiers for different API calls - * (for example, a UUID for one endpoint and a login username for another). - * - * Example: - * external_ids: ["2a1c...-uuid", "john_doe"] - * secondary_ids: { "username": "john_doe" } - * - * Note: Values in `secondary_ids` are not indexed. If you need to look up by a - * secondary value (e.g., username), you must also include that value in `external_ids`. - */ - secondary_ids?: Record; - /** DevRev entity IDs to add */ - targets: { - add: DonV2[]; - }; - status: SyncMapperRecordStatus; - /** - * Optionally populated with the file name of the input file that contains the - * object data. Can be populated on create and on update of the object to help - * in finding the object later if some debugging is needed. - */ - input_files?: { - add: string[]; - }; - /** - * Records external-system changes to prevent update loops. - * When you create/update the object in the external system during loading, - * add that object's modified_date here. Later, when the object is extracted - * and the DevRev Loader evaluates whether to apply it, if the modified_date - * is present in this list the update is skipped (because the change - * originated in DevRev). - */ - external_versions?: { - add: SyncMapperRecordExternalVersion[]; - }; - /** - * Free-form data storage for your use. Store any additional information here. - */ - extra_data?: string; -} - -/** - * Represents a sync mapper record that links external system entities to DevRev entities. - */ -export interface SyncMapperRecord { - id: DonV2; - /** Array of external system IDs that map to the same DevRev object */ - external_ids: string[]; - /** - * Optional map that labels values in `external_ids` with their usage context. - * Use when an external system requires different identifiers for different API calls - * (for example, a UUID for one endpoint and a login username for another). - * - * Example: - * external_ids: ["2a1c...-uuid", "john_doe"] - * secondary_ids: { "username": "john_doe" } - * - * Note: Values in `secondary_ids` are not indexed. If you need to look up by a - * secondary value (e.g., username), you must also include that value in `external_ids`. - */ - secondary_ids?: Record; - /** Array of DevRev entity IDs this mapping points to */ - targets: DonV2[]; - status: SyncMapperRecordStatus; - /** - * Optional file name where the object data was found. - * Useful for debugging - helps locate the source of object data later. - */ - input_files?: string[]; - /** - * Records external-system changes to prevent update loops. - * When the Loader writes to the external system, store the object's - * modified_date here. During the next sync back to DevRev, if the extracted - * object's modified_date exists in this list the update is skipped (avoids - * re-applying a DevRev-originated change). - */ - external_versions?: SyncMapperRecordExternalVersion[]; - /** - * Free-form data storage for your use. Store any additional information here. - * Completely opaque to the platform - use however you need. - */ - extra_data?: string; -} - -/** - * Parameters for retrieving a sync mapper record by DevRev target ID. - */ -export interface MappersGetByTargetIdParams { - /** The sync unit ID that scopes the synchronization context */ - sync_unit: DonV2; - /** The DevRev entity ID to look up */ - target: DonV2; -} - -/** - * Response containing a sync mapper record retrieved by target ID. - */ -export interface MappersGetByTargetIdResponse { - sync_mapper_record: SyncMapperRecord; -} - -/** - * Parameters for creating a new sync mapper record. - */ -export interface MappersCreateParams { - /** The sync unit ID that scopes the synchronization context */ - sync_unit: DonV2; - /** Array of external system identifiers */ - external_ids: string[]; - /** - * Optional map that labels values in `external_ids` with their usage context. - * Use when an external system requires different identifiers for different API calls - * (for example, a UUID for one endpoint and a login username for another). - * - * Example: - * external_ids: ["2a1c...-uuid", "john_doe"] - * secondary_ids: { "username": "john_doe" } - * - * Note: Values in `secondary_ids` are not indexed. If you need to look up by a - * secondary value (e.g., username), you must also include that value in `external_ids`. - */ - secondary_ids?: Record; - /** Array of DevRev entity IDs this mapping points to */ - targets: DonV2[]; - status: SyncMapperRecordStatus; - /** - * Input file names where the object was encountered. - * Used for observability and tracking. - */ - input_files?: string[]; - /** - * External version markers used to avoid update loops. - * After creating or updating the object in the external system, add its - * modified_date here. On subsequent extraction, the Loader skips applying the - * update if the modified_date is present (change originated in DevRev). - */ - external_versions?: SyncMapperRecordExternalVersion[]; - /** - * Opaque data for storing additional client-specific information. - * Fully managed by snapin authors. - */ - extra_data?: string; -} - -/** - * Response containing the newly created sync mapper record. - */ -export interface MappersCreateResponse { - sync_mapper_record: SyncMapperRecord; -} - -/** - * Parameters for updating an existing sync mapper record. - */ -export interface MappersUpdateParams { - /** The ID of the existing sync mapper record to update */ - id: DonV2; - /** The sync unit ID that scopes the synchronization context */ - sync_unit: DonV2; - /** External system IDs to add to the existing mapping */ - external_ids: { - add: string[]; - }; - /** - * Optional map that labels values in `external_ids` with their usage context. - * Use when an external system requires different identifiers for different API calls - * (for example, a UUID for one endpoint and a login username for another). - * - * Example: - * external_ids: ["2a1c...-uuid", "john_doe"] - * secondary_ids: { "username": "john_doe" } - * - * Note: Values in `secondary_ids` are not indexed. If you need to look up by a - * secondary value (e.g., username), you must also include that value in `external_ids`. - */ - secondary_ids?: Record; - /** DevRev entity IDs to add to the existing mapping */ - targets: { - add: DonV2[]; - }; - status: SyncMapperRecordStatus; - /** - * Input file names where the object was encountered. - * Used for observability and tracking. - */ - input_files?: { - add: string[]; - }; - /** - * External version markers used to avoid update loops. - * After creating or updating the object in the external system, add its - * modified_date here. On subsequent extraction, the Loader skips applying the - * update if the modified_date is present (change originated in DevRev). - */ - external_versions?: { - add: SyncMapperRecordExternalVersion[]; - }; - /** - * Opaque data for storing additional client-specific information. - * Fully managed by snapin authors. - */ - extra_data?: string; -} - -/** - * Response containing the updated sync mapper record. - */ -export interface MappersUpdateResponse { - sync_mapper_record: SyncMapperRecord; -} - -/** - * Status of a sync mapper record indicating its operational state. - */ -export enum SyncMapperRecordStatus { - /** The mapping is active and operational (default) */ - OPERATIONAL = 'operational', - /** The mapping was filtered out by user filter settings */ - FILTERED = 'filtered', - /** - * The external object should be ignored in sync operations. - * Use to prevent objects from being created or updated in DevRev. - */ - IGNORED = 'ignored', -} - -/** - * External version tracking to prevent update loops. - * Used to identify changes that originated from your system. - */ -export interface SyncMapperRecordExternalVersion { - /** Sync recipe version at the time the external change was written */ - recipe_version: number; - /** External system modified timestamp (ISO 8601 string) used for loop detection */ - modified_date: string; -} - -/** - * Parameters for retrieving a sync mapper record by external system ID. - */ -export interface MappersGetByExternalIdParams { - /** The sync unit ID that scopes the synchronization context */ - sync_unit: DonV2; - /** The identifier from the external system */ - external_id: string; - /** The type of DevRev entity to look for */ - target_type: SyncMapperRecordTargetType; -} - -/** - * Types of DevRev entities that can be targets in sync mapper records. - */ -export enum SyncMapperRecordTargetType { - ACCESS_CONTROL_ENTRY = 'access_control_entry', - ACCOUNT = 'account', - AIRDROP_AUTHORIZATION_POLICY = 'airdrop_authorization_policy', - AIRDROP_FIELD_AUTHORIZATION_POLICY = 'airdrop_field_authorization_policy', - AIRDROP_PLATFORM_GROUP = 'airdrop_platform_group', - ARTICLE = 'article', - ARTIFACT = 'artifact', - CHAT = 'chat', - CONVERSATION = 'conversation', - CUSTOM_OBJECT = 'custom_object', - DIRECTORY = 'directory', - GROUP = 'group', - INCIDENT = 'incident', - LINK = 'link', - MEETING = 'meeting', - OBJECT_MEMBER = 'object_member', - PART = 'part', - REV_ORG = 'rev_org', - ROLE = 'role', - ROLE_SET = 'role_set', - TAG = 'tag', - TIMELINE_COMMENT = 'timeline_comment', - USER = 'user', - WORK = 'work', -} - -/** - * Response containing a sync mapper record retrieved by external ID. - */ -export interface MappersGetByExternalIdResponse { - sync_mapper_record: SyncMapperRecord; -} diff --git a/src/mappers/mappers.interfaces.ts b/src/mappers/mappers.interfaces.ts new file mode 100644 index 00000000..270de167 --- /dev/null +++ b/src/mappers/mappers.interfaces.ts @@ -0,0 +1,188 @@ +import { AirSyncEvent } from '../types/extraction'; +import { DonV2 } from '../types/loading'; +import { WorkerAdapterOptions } from '../types/workers'; + +export interface MappersFactoryInterface { + event: AirSyncEvent; + options?: WorkerAdapterOptions; +} + +export interface UpdateSyncMapperRecordParams { + external_ids: { + add: string[]; + }; + /** + * Labels values in `external_ids` with their usage context (e.g. a UUID for + * one API call, a login username for another). Not indexed: to look up by a + * secondary value it must also be present in `external_ids`. + */ + secondary_ids?: Record; + targets: { + add: DonV2[]; + }; + status: SyncMapperRecordStatus; + /** Input file name(s) containing the object data; helps later debugging. */ + input_files?: { + add: string[]; + }; + /** + * Prevents update loops: after writing the object to the external system, + * add its modified_date here. The Loader skips extracted updates whose + * modified_date is listed (the change originated in DevRev). + */ + external_versions?: { + add: SyncMapperRecordExternalVersion[]; + }; + /** Free-form storage; opaque to the platform. */ + extra_data?: string; +} + +/** Links external system entities to DevRev entities. */ +export interface SyncMapperRecord { + id: DonV2; + external_ids: string[]; + /** + * Labels values in `external_ids` with their usage context (e.g. a UUID for + * one API call, a login username for another). Not indexed: to look up by a + * secondary value it must also be present in `external_ids`. + */ + secondary_ids?: Record; + targets: DonV2[]; + status: SyncMapperRecordStatus; + /** Input file name(s) containing the object data; helps later debugging. */ + input_files?: string[]; + /** + * Prevents update loops: after writing the object to the external system, + * its modified_date is stored here. The Loader skips extracted updates whose + * modified_date is listed (the change originated in DevRev). + */ + external_versions?: SyncMapperRecordExternalVersion[]; + /** Free-form storage; opaque to the platform. */ + extra_data?: string; +} + +export interface MappersGetByTargetIdParams { + sync_unit: DonV2; + target: DonV2; +} + +export interface MappersGetByTargetIdResponse { + sync_mapper_record: SyncMapperRecord; +} + +export interface MappersCreateParams { + sync_unit: DonV2; + external_ids: string[]; + /** + * Labels values in `external_ids` with their usage context (e.g. a UUID for + * one API call, a login username for another). Not indexed: to look up by a + * secondary value it must also be present in `external_ids`. + */ + secondary_ids?: Record; + targets: DonV2[]; + status: SyncMapperRecordStatus; + /** Input file name(s) containing the object data; helps later debugging. */ + input_files?: string[]; + /** + * Prevents update loops: after writing the object to the external system, + * add its modified_date here. The Loader skips extracted updates whose + * modified_date is listed (the change originated in DevRev). + */ + external_versions?: SyncMapperRecordExternalVersion[]; + /** Free-form storage; opaque to the platform. */ + extra_data?: string; +} + +export interface MappersCreateResponse { + sync_mapper_record: SyncMapperRecord; +} + +export interface MappersUpdateParams { + id: DonV2; + sync_unit: DonV2; + external_ids: { + add: string[]; + }; + /** + * Labels values in `external_ids` with their usage context (e.g. a UUID for + * one API call, a login username for another). Not indexed: to look up by a + * secondary value it must also be present in `external_ids`. + */ + secondary_ids?: Record; + targets: { + add: DonV2[]; + }; + status: SyncMapperRecordStatus; + /** Input file name(s) containing the object data; helps later debugging. */ + input_files?: { + add: string[]; + }; + /** + * Prevents update loops: after writing the object to the external system, + * add its modified_date here. The Loader skips extracted updates whose + * modified_date is listed (the change originated in DevRev). + */ + external_versions?: { + add: SyncMapperRecordExternalVersion[]; + }; + /** Free-form storage; opaque to the platform. */ + extra_data?: string; +} + +export interface MappersUpdateResponse { + sync_mapper_record: SyncMapperRecord; +} + +export enum SyncMapperRecordStatus { + /** The mapping is active and operational (default) */ + OPERATIONAL = 'operational', + /** The mapping was filtered out by user filter settings */ + FILTERED = 'filtered', + /** Ignore the external object in sync; prevents create/update in DevRev. */ + IGNORED = 'ignored', +} + +/** Marks external changes as DevRev-originated to prevent update loops. */ +export interface SyncMapperRecordExternalVersion { + /** Sync recipe version at the time the external change was written */ + recipe_version: number; + /** External system modified timestamp (ISO 8601 string) used for loop detection */ + modified_date: string; +} + +export interface MappersGetByExternalIdParams { + sync_unit: DonV2; + external_id: string; + target_type: SyncMapperRecordTargetType; +} + +export enum SyncMapperRecordTargetType { + ACCESS_CONTROL_ENTRY = 'access_control_entry', + ACCOUNT = 'account', + AIRDROP_AUTHORIZATION_POLICY = 'airdrop_authorization_policy', + AIRDROP_FIELD_AUTHORIZATION_POLICY = 'airdrop_field_authorization_policy', + AIRDROP_PLATFORM_GROUP = 'airdrop_platform_group', + ARTICLE = 'article', + ARTIFACT = 'artifact', + CHAT = 'chat', + CONVERSATION = 'conversation', + CUSTOM_OBJECT = 'custom_object', + DIRECTORY = 'directory', + GROUP = 'group', + INCIDENT = 'incident', + LINK = 'link', + MEETING = 'meeting', + OBJECT_MEMBER = 'object_member', + PART = 'part', + REV_ORG = 'rev_org', + ROLE = 'role', + ROLE_SET = 'role_set', + TAG = 'tag', + TIMELINE_COMMENT = 'timeline_comment', + USER = 'user', + WORK = 'work', +} + +export interface MappersGetByExternalIdResponse { + sync_mapper_record: SyncMapperRecord; +} diff --git a/src/mappers/mappers.test.ts b/src/mappers/mappers.test.ts index 68a8ef23..c38556b6 100644 --- a/src/mappers/mappers.test.ts +++ b/src/mappers/mappers.test.ts @@ -1,6 +1,8 @@ -import { createMockEvent, MOCK_SERVER_DEFAULT_URL } from '../common/test-utils'; -import { axiosClient } from '../http/axios-client-internal'; +import { axiosClient } from '../http/client'; +import { createMockEvent } from '../testing/mock-event'; +import { MOCK_SERVER_DEFAULT_URL } from '../testing/mock-server'; import { EventType } from '../types/extraction'; + import { Mappers } from './mappers'; import { MappersCreateParams, @@ -9,10 +11,10 @@ import { MappersUpdateParams, SyncMapperRecordStatus, SyncMapperRecordTargetType, -} from './mappers.interface'; +} from './mappers.interfaces'; // Mock the axios client -jest.mock('../http/axios-client-internal'); +jest.mock('../http/client'); const mockAxiosClient = axiosClient as jest.Mocked; describe(Mappers.name, () => { diff --git a/src/mappers/mappers.ts b/src/mappers/mappers.ts index c552b02c..0a34cf4f 100644 --- a/src/mappers/mappers.ts +++ b/src/mappers/mappers.ts @@ -1,6 +1,4 @@ -import { AxiosResponse } from 'axios'; - -import { axiosClient } from '../http/axios-client-internal'; +import { axiosClient } from '../http/client'; import { MappersCreateParams, @@ -12,13 +10,9 @@ import { MappersGetByTargetIdResponse, MappersUpdateParams, MappersUpdateResponse, -} from './mappers.interface'; +} from './mappers.interfaces'; -/** - * Manages sync mapper records that link external system items to DevRev items. - * - * Used for tracking relationships between external and DevRev entities during sync operations. - */ +/** Manages sync mapper records that link external system items to DevRev items. */ export class Mappers { private devrevApiEndpoint: string; private devrevApiToken: string; @@ -28,19 +22,11 @@ export class Mappers { this.devrevApiToken = event.context.secrets.service_account_token; } - /** - * Retrieves a sync mapper record by DevRev ID. - * - * Used to find the mapping when you know the DevRev ID and want to find the external system ID. - * - * @param params - Query parameters of type MappersGetByTargetIdParams - * @returns Promise with response data containing the sync mapper record - */ async getByTargetId( params: MappersGetByTargetIdParams - ): Promise> { + ): Promise { const { sync_unit, target } = params; - return axiosClient.get( + const response = await axiosClient.get( `${this.devrevApiEndpoint}/internal/airdrop.sync-mapper-record.get-by-target`, { headers: { @@ -49,21 +35,14 @@ export class Mappers { params: { sync_unit, target }, } ); + return response.data; } - /** - * Retrieves a sync mapper record by external system ID. - * - * Used to find the mapping when you know the external system ID and want to find the DevRev ID. - * - * @param params - Query parameters of type MappersGetByExternalIdParams - * @returns Promise with response data containing the sync mapper record - */ async getByExternalId( params: MappersGetByExternalIdParams - ): Promise> { + ): Promise { const { sync_unit, external_id, target_type } = params; - return axiosClient.get( + const response = await axiosClient.get( `${this.devrevApiEndpoint}/internal/airdrop.sync-mapper-record.get-by-external-id`, { headers: { @@ -72,22 +51,12 @@ export class Mappers { params: { sync_unit, external_id, target_type }, } ); + return response.data; } - /** - * Creates a new sync mapper record to establish a relationship between external system - * entities and DevRev entities. - * - * This is called after importing an item from external system to DevRev to record - * the mapping for future synchronization operations. - * - * @param params - Creation parameters of type MappersCreateParams - * @returns Promise with response data containing the created sync mapper record - */ - async create( - params: MappersCreateParams - ): Promise> { - return axiosClient.post( + /** Called after importing an item to DevRev to record the mapping for future syncs. */ + async create(params: MappersCreateParams): Promise { + const response = await axiosClient.post( `${this.devrevApiEndpoint}/internal/airdrop.sync-mapper-record.create`, params, { @@ -96,21 +65,11 @@ export class Mappers { }, } ); + return response.data; } - /** - * Updates an existing sync mapper record. - * - * Used to modify existing mappings when external system entities change or when - * additional DevRev entities need to be associated. - * - * @param params - Update parameters of type MappersUpdateParams - * @returns Promise with response data containing the updated sync mapper record - */ - async update( - params: MappersUpdateParams - ): Promise> { - return axiosClient.post( + async update(params: MappersUpdateParams): Promise { + const response = await axiosClient.post( `${this.devrevApiEndpoint}/internal/airdrop.sync-mapper-record.update`, params, { @@ -119,5 +78,6 @@ export class Mappers { }, } ); + return response.data; } } diff --git a/src/mock-server/mock-server.interfaces.ts b/src/mock-server/mock-server.interfaces.ts deleted file mode 100644 index 0094407f..00000000 --- a/src/mock-server/mock-server.interfaces.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { IncomingMessage, ServerResponse } from 'http'; - -export const DEFAULT_MOCK_SERVER_PORT = 3001; - -export interface ParsedRequest extends IncomingMessage { - /** Parsed URL path (without query string) */ - path: string; - /** Parsed JSON body (if any) */ - body?: unknown; -} - -export interface MockResponse extends ServerResponse { - /** Set response headers from a record */ - set(headers: Record): MockResponse; - /** Set the HTTP status code */ - status(code: number): MockResponse; - /** Send a JSON response */ - json(data: unknown): void; - /** Send a raw binary (Buffer) response */ - buffer(data: Buffer): void; - /** Send an empty response */ - send(): void; -} - -/** - * Configuration for retry simulation behavior. - */ -export interface RetryConfig { - /** Number of times to return error before succeeding (default: 4) */ - failureCount?: number; - /** 5xx status code to return during failures (default: 500) */ - errorStatus?: number; - /** Optional error response body to send as JSON during failures */ - errorBody?: unknown; - /** Optional headers to send with the error response */ - headers?: Record; - /** Optional delay in milliseconds before sending each failure response */ - delay?: number; -} - -/** - * Configuration object for setting up a route response. - */ -export interface RouteConfig { - /** The path of the route (e.g., '/callback_url', '/worker_data_url.get') */ - path: string; - /** The HTTP method (e.g., 'GET', 'POST', 'PUT', 'DELETE') */ - method: string; - /** The HTTP status code to return (e.g., 200, 401, 500) */ - status: number; - /** Optional response body to send as JSON */ - body?: unknown; - /** Optional raw binary response body, e.g. gzipped JSONL (takes precedence over `body`) */ - bodyBuffer?: Buffer; - /** Optional headers to send with the response */ - headers?: Record; - /** Optional retry configuration for simulating failures before success */ - retry?: RetryConfig; - /** Optional delay in milliseconds before sending the response */ - delay?: number; -} - -/** - * Type for custom route handler functions. - */ -export type RouteHandler = (req: ParsedRequest, res: MockResponse) => unknown; - -/** - * Information about a request received by the mock server. - */ -export interface RequestInfo { - /** The HTTP method (e.g., 'GET', 'POST') */ - method: string; - /** The full URL path of the request */ - url: string; - /** Optional request body (for POST/PUT requests) */ - body?: unknown; -} - -export type RouteHandlers = Map; - -/** - * Type for tracking request counts per route. - */ -export type RequestCounts = Map; diff --git a/src/multithreading/adapters/base-adapter.ts b/src/multithreading/adapters/base-adapter.ts new file mode 100644 index 00000000..408716a8 --- /dev/null +++ b/src/multithreading/adapters/base-adapter.ts @@ -0,0 +1,213 @@ +import { parentPort } from 'node:worker_threads'; + +import { STATELESS_EVENT_TYPES } from '../../common/constants'; +import { serializeError, truncateMessage } from '../../logger/logger'; +import { BaseState } from '../../state/state'; +import { SdkState } from '../../state/state.interfaces'; +import { + AirSyncEvent, + EventData, + ExtractorEventType, + WorkerMetadata, +} from '../../types/extraction'; +import { LoaderEventType } from '../../types/loading'; +import { + TaskResult, + WorkerAdapterOptions, + WorkerMessageEmitted, + WorkerMessageSubject, +} from '../../types/workers'; +import { Uploader } from '../../uploader/uploader'; +import { emit } from '../emit'; +import { getEventTypeForResult } from '../spawn/spawn.helpers'; + +/** + * Shared state/behavior for both sync modes; owns the `emit` control-protocol + * flow as a template method with mode-specific hooks. + */ +export abstract class BaseAdapter { + readonly event: AirSyncEvent; + readonly options?: WorkerAdapterOptions; + hasWorkerEmitted: boolean; + + private _isTimeout: boolean = false; + private resolveTimeoutSignal!: () => void; + readonly timeoutSignal: Promise = new Promise((resolve) => { + this.resolveTimeoutSignal = resolve; + }); + + protected adapterState: BaseState; + protected uploader: Uploader; + + constructor({ + event, + adapterState, + options, + }: { + event: AirSyncEvent; + adapterState: BaseState; + options?: WorkerAdapterOptions; + }) { + this.event = event; + this.options = options; + this.adapterState = adapterState; + this.hasWorkerEmitted = false; + this.uploader = new Uploader({ + event, + options, + }); + } + + get isTimeout(): boolean { + return this._isTimeout; + } + + set isTimeout(value: boolean) { + this._isTimeout = value; + if (value) { + this.resolveTimeoutSignal(); + } + } + + /** Connector-owned state exposed to snap-in code. */ + get state(): ConnectorState { + return this.adapterState.state; + } + + set state(value: ConnectorState) { + this.adapterState.state = value; + } + + /** SDK-internal bookkeeping state; not for connector use. */ + get sdkState(): SdkState { + return this.adapterState.sdkState; + } + + get extractionScope() { + return this.adapterState.extractionScope; + } + + async postState() { + await this.adapterState.postState(); + } + + /** Pre-emit hook, runs before state is persisted. Throwing aborts the emit. */ + protected abstract beforeEmit( + newEventType: ExtractorEventType | LoaderEventType + ): Promise; + + /** Mode-specific extras merged into the emitted event payload. */ + protected abstract buildEmitPayload( + newEventType: ExtractorEventType | LoaderEventType + ): EventData; + + /** + * Mode-specific worker metadata merged into the emitted event. The library + * version and state-date range are added centrally in `emit`, so subclasses + * only contribute their own statistics. + */ + protected buildWorkerMetadata( + // eslint-disable-next-line @typescript-eslint/no-unused-vars + newEventType: ExtractorEventType | LoaderEventType + ): WorkerMetadata | undefined { + return undefined; + } + + /** Post-emit hook, runs after the event has been sent successfully. */ + protected abstract afterEmit( + newEventType: ExtractorEventType | LoaderEventType + ): void; + + /** + * Maps a {@link TaskResult} to the phase-appropriate platform event and emits + * it exactly once. Invoked by the worker driver, not by connectors — + * connectors signal outcomes by returning a `TaskResult`, never by emitting. + */ + async emitFromResult(result: TaskResult): Promise { + const { eventType, illegal } = getEventTypeForResult( + this.event.payload.event_type, + result.status + ); + + const data: EventData = {}; + if (result.status === 'delay') { + data.delay = result.delaySeconds; + } else if (result.status === 'error') { + data.error = result.error; + } else if (illegal) { + data.error = { + message: `Worker returned status '${result.status}' for a non-resumable phase (${this.event.payload.event_type}), which is not allowed. Emitting an error event instead.`, + }; + } + + await this.emit(eventType, data); + } + + protected async emit( + newEventType: ExtractorEventType | LoaderEventType, + data?: EventData + ): Promise { + if (this.hasWorkerEmitted) { + console.warn( + `Trying to emit event with event type: ${newEventType}. Ignoring emit request because it has already been emitted.` + ); + return; + } + + try { + await this.beforeEmit(newEventType); + } catch (error) { + console.error('Error while preparing to emit event', error); + parentPort?.postMessage(WorkerMessageSubject.WorkerMessageExit); + this.hasWorkerEmitted = true; + return; + } + + // Save state on every emit, except for start and delete events + if (!STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) { + console.log( + `Saving state before emitting event with event type: ${newEventType}.` + ); + + try { + await this.adapterState.postState(); + } catch (error) { + console.error('Error while posting state', error); + parentPort?.postMessage(WorkerMessageSubject.WorkerMessageExit); + this.hasWorkerEmitted = true; + return; + } + } + + try { + if (data?.error?.message) { + data.error.message = truncateMessage(data.error.message); + } + + await emit({ + eventType: newEventType, + event: this.event, + data: { + ...data, + ...this.buildEmitPayload(newEventType), + }, + worker_metadata: this.buildWorkerMetadata(newEventType), + }); + + const message: WorkerMessageEmitted = { + subject: WorkerMessageSubject.WorkerMessageEmitted, + payload: { eventType: newEventType }, + }; + this.afterEmit(newEventType); + parentPort?.postMessage(message); + this.hasWorkerEmitted = true; + } catch (error) { + console.error( + `Error while emitting event with event type: ${newEventType}.`, + serializeError(error) + ); + parentPort?.postMessage(WorkerMessageSubject.WorkerMessageExit); + this.hasWorkerEmitted = true; + } + } +} diff --git a/src/multithreading/adapters/extraction-adapter.ts b/src/multithreading/adapters/extraction-adapter.ts new file mode 100644 index 00000000..26a377e9 --- /dev/null +++ b/src/multithreading/adapters/extraction-adapter.ts @@ -0,0 +1,497 @@ +import { AttachmentsStreamingPool } from '../../attachments-streaming/attachments-streaming-pool'; +import { + AirSyncDefaultItemTypes, + EVENT_SIZE_THRESHOLD_BYTES, + SSOR_ATTACHMENT, +} from '../../common/constants'; +import { serializeError } from '../../logger/logger'; +import { Repo } from '../../repo/repo'; +import { + NormalizedAttachment, + RepoInterface, +} from '../../repo/repo.interfaces'; +import { BaseState } from '../../state/state'; +import { + AirSyncEvent, + EventData, + ExternalSystemAttachmentProcessors, + ExternalSystemAttachmentStreamingFunction, + ExtractorEventType, + HttpStreamResponse, + ProcessAttachmentReturnType, + WorkerMetadata, +} from '../../types/extraction'; +import { LoaderEventType } from '../../types/loading'; +import { TaskResult, WorkerAdapterOptions } from '../../types/workers'; +import { Artifact, SsorAttachment } from '../../uploader/uploader.interfaces'; + +import { BaseAdapter } from './base-adapter'; + +/** Adapter passed to extraction tasks (repos, artifacts, attachment streaming). @public */ +export class ExtractionAdapter< + ConnectorState +> extends BaseAdapter { + private _artifacts: Artifact[]; + private repos: Repo[] = []; + private lastExtractedItemType?: string; + private currentEventDataLength: number = 0; + + constructor(params: { + event: AirSyncEvent; + adapterState: BaseState; + options?: WorkerAdapterOptions; + }) { + super(params); + this._artifacts = []; + } + + /** Defaults to true if the scope is empty or the item type is not listed. */ + shouldExtract(itemType: string): boolean { + const scope = this.extractionScope; + if (Object.keys(scope).length === 0) return true; + if (!(itemType in scope)) return true; + return scope[itemType].extract; + } + + initializeRepos(repos: RepoInterface[]) { + this.repos = repos.map((repo) => { + const shouldNormalize = + repo.itemType !== AirSyncDefaultItemTypes.EXTERNAL_DOMAIN_METADATA && + repo.itemType !== SSOR_ATTACHMENT; + + return new Repo({ + event: this.event, + itemType: repo.itemType, + ...(shouldNormalize && { normalize: repo.normalize }), + onUpload: (artifact: Artifact) => { + this.lastExtractedItemType = repo.itemType; + + // Artifact ids are kept in state for later attachment streaming + if (repo.itemType === AirSyncDefaultItemTypes.ATTACHMENTS) { + this.sdkState.toDevRev?.attachmentsMetadata.artifactIds.push( + artifact.id + ); + } + + // Track size of artifact objects that go in the SQS message; flip + // isTimeout once the threshold is exceeded to force an early emit. + this.currentEventDataLength += Buffer.byteLength( + JSON.stringify(artifact), + 'utf8' + ); + + if ( + this.currentEventDataLength > EVENT_SIZE_THRESHOLD_BYTES && + !this.isTimeout + ) { + this.isTimeout = true; + } + }, + options: { + ...this.options, + ...repo.overridenOptions, + }, + }); + }); + } + + getRepo(itemType: string): Repo | undefined { + const repo = this.repos.find((repo) => repo.itemType === itemType); + + if (!repo) { + console.error(`Repo for item type ${itemType} not found.`); + return; + } + + return repo; + } + + get artifacts(): Artifact[] { + return this._artifacts; + } + + set artifacts(artifacts: Artifact[]) { + this._artifacts = this._artifacts + .concat(artifacts) + .filter((value, index, self) => self.indexOf(value) === index); + } + + protected async beforeEmit( + newEventType: ExtractorEventType | LoaderEventType + ): Promise { + console.log( + `Uploading all repos before emitting event with event type: ${newEventType}.` + ); + await this.uploadAllRepos(); + + // When the full extraction cycle completes, commit the extraction window + // boundaries so subsequent incremental syncs can resume from them. + if (newEventType === ExtractorEventType.AttachmentExtractionDone) { + const sdkState = this.sdkState; + + sdkState.pendingWorkersOldest = ''; + sdkState.pendingWorkersNewest = ''; + + // Expand boundaries: workersOldest keeps the earliest timestamp seen, + // workersNewest the latest. + const extractionStart = this.event.payload.event_context.extract_from; + const extractionEnd = this.event.payload.event_context.extract_to; + + if ( + extractionStart && + (!sdkState.workersOldest || extractionStart < sdkState.workersOldest) + ) { + console.log( + `Updating workersOldest from '${sdkState.workersOldest}' to '${extractionStart}'.` + ); + sdkState.workersOldest = extractionStart; + } + + if ( + extractionEnd && + (!sdkState.workersNewest || extractionEnd > sdkState.workersNewest) + ) { + console.log( + `Updating workersNewest from '${sdkState.workersNewest}' to '${extractionEnd}'.` + ); + sdkState.workersNewest = extractionEnd; + } + } + } + + protected buildEmitPayload( + newEventType: ExtractorEventType | LoaderEventType + ): EventData { + const isExtractionEvent = Object.values(ExtractorEventType).includes( + newEventType as ExtractorEventType + ); + return isExtractionEvent ? { artifacts: this.artifacts } : {}; + } + + /** + * Attaches the last-extracted item type and its created/modified date ranges + * to data/attachment done/progress events, mirroring the metadata v1 sent. + */ + protected override buildWorkerMetadata( + newEventType: ExtractorEventType | LoaderEventType + ): WorkerMetadata | undefined { + if ( + newEventType !== ExtractorEventType.DataExtractionDone && + newEventType !== ExtractorEventType.DataExtractionProgress && + newEventType !== ExtractorEventType.AttachmentExtractionDone && + newEventType !== ExtractorEventType.AttachmentExtractionProgress + ) { + return undefined; + } + + const repo = this.lastExtractedItemType + ? this.repos.find((r) => r.itemType === this.lastExtractedItemType) + : undefined; + + if (!repo) { + return undefined; + } + + return { + item_type: repo.itemType, + newest_created_date: this.toRfc3339Timestamp( + repo.dateRanges.creationDate.newest + ), + oldest_created_date: this.toRfc3339Timestamp( + repo.dateRanges.creationDate.oldest + ), + newest_modified_date: this.toRfc3339Timestamp( + repo.dateRanges.modifiedDate.newest + ), + oldest_modified_date: this.toRfc3339Timestamp( + repo.dateRanges.modifiedDate.oldest + ), + }; + } + + private toRfc3339Timestamp(ms?: number): string | undefined { + if (ms === undefined || !Number.isFinite(ms)) { + return undefined; + } + + return new Date(ms).toISOString(); + } + + protected afterEmit(): void { + this.artifacts = []; + } + + async uploadAllRepos(): Promise { + for (const repo of this.repos) { + const error = await repo.upload(); + this.artifacts.push(...repo.uploadedArtifacts); + if (error) { + throw error; + } + } + } + + async processAttachment( + attachment: NormalizedAttachment, + stream: ExternalSystemAttachmentStreamingFunction + ): Promise { + const { httpStream, delay, error } = await stream({ + item: attachment, + event: this.event, + }); + + if (error) { + return { error }; + } else if (delay) { + return { delay }; + } + + if (httpStream) { + const fileType = + attachment.content_type || + httpStream.headers['content-type']?.toString() || + 'application/octet-stream'; + const contentLength = httpStream.headers['content-length']?.toString(); + const fileSize = contentLength ? parseInt(contentLength) : undefined; + + const { error: artifactUrlError, response: artifactUrlResponse } = + await this.uploader.getArtifactUploadUrl( + attachment.file_name, + fileType, + fileSize + ); + + if (artifactUrlError) { + this.destroyHttpStream(httpStream); + return { + error: { + message: `Error while preparing artifact for attachment ID ${ + attachment.id + }. Skipping attachment. ${serializeError(artifactUrlError)}`, + fileSize: fileSize, + }, + }; + } + + const { error: uploadedArtifactError } = + await this.uploader.streamArtifact(artifactUrlResponse!, httpStream); + + if (uploadedArtifactError) { + this.destroyHttpStream(httpStream); + return { + error: { + message: + `Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment. ` + + serializeError(uploadedArtifactError), + fileSize: fileSize, + }, + }; + } + + const { error: confirmArtifactUploadError } = + await this.uploader.confirmArtifactUpload( + artifactUrlResponse!.artifact_id + ); + if (confirmArtifactUploadError) { + return { + error: { + message: + `Error while confirming upload for attachment ID ${attachment.id}. ` + + serializeError(confirmArtifactUploadError), + fileSize: fileSize, + }, + }; + } + + const ssorAttachment: SsorAttachment = { + id: { + devrev: artifactUrlResponse!.artifact_id, + external: attachment.id, + }, + parent_id: { + external: attachment.parent_id, + }, + }; + + if (attachment.author_id) { + ssorAttachment.actor_id = { + external: attachment.author_id, + }; + } + + // Set inline flag only if it is explicitly set on the attachment. + if (attachment.inline === true) { + ssorAttachment.inline = true; + } else if (attachment.inline === false) { + ssorAttachment.inline = false; + } + + if (this.isTimeout) { + this.destroyHttpStream(httpStream); + return; + } + + await this.getRepo('ssor_attachment')?.push([ssorAttachment]); + return; + } + return { + error: { + message: `Error while opening attachment stream. Skipping attachment.`, + }, + }; + } + + /** Destroys a stream to prevent memory leaks. */ + private destroyHttpStream(httpStream: HttpStreamResponse): void { + try { + if (httpStream && httpStream.data) { + if (typeof httpStream.data.destroy === 'function') { + httpStream.data.destroy(); + } else if (typeof httpStream.data.close === 'function') { + httpStream.data.close(); + } + } + } catch (error) { + console.warn('Error while destroying HTTP stream:', error); + } + } + + async streamAttachments({ + stream, + processors, + batchSize = 1, + }: { + stream: ExternalSystemAttachmentStreamingFunction; + processors?: ExternalSystemAttachmentProcessors< + ConnectorState, + NormalizedAttachment[], + NewBatch + >; + batchSize?: number; + }): Promise { + if (batchSize <= 0) { + console.warn( + `The specified batch size (${batchSize}) is invalid. Using 1 instead.` + ); + batchSize = 1; + } + + if (batchSize > 50) { + console.warn( + `The specified batch size (${batchSize}) is too large. Using 50 instead.` + ); + batchSize = 50; + } + + const repos = [ + { + itemType: 'ssor_attachment', + }, + ]; + this.initializeRepos(repos); + + const attachmentsMetadata = this.sdkState.toDevRev?.attachmentsMetadata; + + if (!attachmentsMetadata?.artifactIds?.length) { + console.log(`No attachments metadata artifact IDs found in state.`); + return { status: 'success' }; + } else { + console.log( + `Found ${attachmentsMetadata.artifactIds.length} attachments metadata artifact IDs in state.` + ); + } + + while (attachmentsMetadata.artifactIds.length > 0) { + const attachmentsMetadataArtifactId = attachmentsMetadata.artifactIds[0]; + + console.log( + `Started processing attachments for attachments metadata artifact ID: ${attachmentsMetadataArtifactId}.` + ); + + const { attachments, error } = + await this.uploader.getAttachmentsFromArtifactId({ + artifact: attachmentsMetadataArtifactId, + }); + + if (error) { + console.error( + `Failed to get attachments for artifact ID: ${attachmentsMetadataArtifactId}.` + ); + return { status: 'error', error }; + } + + if (!attachments || attachments.length === 0) { + console.warn( + `No attachments found for artifact ID: ${attachmentsMetadataArtifactId}.` + ); + attachmentsMetadata.artifactIds.shift(); + attachmentsMetadata.lastProcessed = 0; + continue; + } + + console.log( + `Found ${attachments.length} attachments for artifact ID: ${attachmentsMetadataArtifactId}.` + ); + + let response; + + if (processors) { + console.log(`Using custom processors for attachments.`); + + const reducer = processors.reducer; + const iterator = processors.iterator; + + const reducedAttachments = reducer({ + attachments, + adapter: this, + batchSize, + }); + + response = await iterator({ + reducedAttachments, + adapter: this, + stream, + }); + } else { + console.log( + `Using attachments streaming pool for attachments streaming.` + ); + + const attachmentsPool = new AttachmentsStreamingPool({ + adapter: this, + attachments, + batchSize, + stream, + }); + + response = await attachmentsPool.streamAll(); + } + + if (response?.error) { + return { status: 'error', error: response.error }; + } + + if (response?.delay) { + return { status: 'delay', delaySeconds: response.delay }; + } + + if (this.isTimeout) { + console.log( + `Timeout detected after processing attachments for artifact ID: ${attachmentsMetadataArtifactId}. Returning progress to allow continuation.` + ); + return { status: 'progress' }; + } + + console.log( + `Finished processing all attachments for artifact ID: ${attachmentsMetadataArtifactId}.` + ); + attachmentsMetadata.artifactIds.shift(); + attachmentsMetadata.lastProcessed = 0; + if (attachmentsMetadata.lastProcessedAttachmentsIdsList) { + attachmentsMetadata.lastProcessedAttachmentsIdsList.length = 0; + } + } + + return { status: 'success' }; + } +} diff --git a/src/multithreading/worker-adapter/worker-adapter.helpers.ts b/src/multithreading/adapters/loading-adapter.helpers.ts similarity index 77% rename from src/multithreading/worker-adapter/worker-adapter.helpers.ts rename to src/multithreading/adapters/loading-adapter.helpers.ts index efcf2fde..60e7dafa 100644 --- a/src/multithreading/worker-adapter/worker-adapter.helpers.ts +++ b/src/multithreading/adapters/loading-adapter.helpers.ts @@ -5,12 +5,7 @@ import { StatsFileObject, } from '../../types/loading'; -/** - * Gets the files to load for the loader. - * @param {string[]} supportedItemTypes - The supported item types - * @param {StatsFileObject[]} statsFile - The stats file - * @returns {FileToLoad[]} The files to load - */ +/** Filters the stats file to supported item types, ordered by their position in supportedItemTypes. */ export function getFilesToLoad({ supportedItemTypes, statsFile, @@ -49,12 +44,7 @@ export function getFilesToLoad({ return filesToLoad; } -/** - * Adds a report to the loader report. - * @param {LoaderReport[]} loaderReports - The loader reports - * @param {LoaderReport} report - The report to add - * @returns {LoaderReport[]} The updated loader reports - */ +/** Merges a report into the loader reports, summing counts per item type. */ export function addReportToLoaderReport({ loaderReports, report, @@ -90,11 +80,3 @@ export function addReportToLoaderReport({ return loaderReports; } - -export function toRfc3339Timestamp(ms?: number): string | undefined { - if (ms === undefined || !Number.isFinite(ms)) { - return undefined; - } - - return new Date(ms).toISOString(); -} diff --git a/src/multithreading/adapters/loading-adapter.ts b/src/multithreading/adapters/loading-adapter.ts new file mode 100644 index 00000000..90869824 --- /dev/null +++ b/src/multithreading/adapters/loading-adapter.ts @@ -0,0 +1,572 @@ +import axios from 'axios'; + +import { serializeError } from '../../logger/logger'; +import { Mappers } from '../../mappers/mappers'; +import { SyncMapperRecordStatus } from '../../mappers/mappers.interfaces'; +import { BaseState } from '../../state/state'; +import { + AirSyncEvent, + EventData, + EventType, + ExtractorEventType, +} from '../../types/extraction'; +import { + ActionType, + ExternalSystemAttachment, + ExternalSystemItem, + ExternalSystemLoadingFunction, + FileToLoad, + ItemTypesToLoadParams, + ItemTypeToLoad, + LoaderEventType, + LoaderReport, + LoadItemResponse, + StatsFileObject, +} from '../../types/loading'; +import { TaskResult, WorkerAdapterOptions } from '../../types/workers'; + +import { BaseAdapter } from './base-adapter'; +import { + addReportToLoaderReport, + getFilesToLoad, +} from './loading-adapter.helpers'; + +/** Adapter passed to loading tasks (item/attachment loading, mappers, loader reports). @public */ +export class LoadingAdapter< + ConnectorState +> extends BaseAdapter { + private loaderReports: LoaderReport[]; + private _processedFiles: string[]; + private _mappers: Mappers; + + constructor(params: { + event: AirSyncEvent; + adapterState: BaseState; + options?: WorkerAdapterOptions; + }) { + super(params); + this.loaderReports = []; + this._processedFiles = []; + this._mappers = new Mappers({ + event: params.event, + options: params.options, + }); + } + + get reports(): LoaderReport[] { + return this.loaderReports; + } + + get processedFiles(): string[] { + return this._processedFiles; + } + + get mappers(): Mappers { + return this._mappers; + } + + protected async beforeEmit(): Promise { + // Loading has no pre-emit work. + } + + protected buildEmitPayload( + newEventType: ExtractorEventType | LoaderEventType + ): EventData { + const isLoaderEvent = Object.values(LoaderEventType).includes( + newEventType as LoaderEventType + ); + return isLoaderEvent + ? { + reports: this.reports, + processed_files: this.processedFiles, + } + : {}; + } + + protected afterEmit(): void { + // Loading keeps its accumulated reports/processed files across emits. + } + + async loadItemTypes({ + itemTypesToLoad, + }: ItemTypesToLoadParams): Promise { + if (this.event.payload.event_type === EventType.StartLoadingData) { + const itemTypes = itemTypesToLoad.map( + (itemTypeToLoad) => itemTypeToLoad.itemType + ); + + if (!itemTypes.length) { + console.warn('No item types to load, returning.'); + return { status: 'success' }; + } + + const filesToLoad = await this.getLoaderBatches({ + supportedItemTypes: itemTypes, + }); + this.sdkState.fromDevRev = { + filesToLoad, + }; + } + + if ( + !this.sdkState.fromDevRev || + !this.sdkState.fromDevRev.filesToLoad.length + ) { + console.warn('No files to load, returning.'); + return { status: 'success' }; + } + + console.log( + 'Files to load in state', + this.sdkState.fromDevRev?.filesToLoad + ); + + try { + for (const fileToLoad of this.sdkState.fromDevRev.filesToLoad) { + const itemTypeToLoad = itemTypesToLoad.find( + (itemTypeToLoad: ItemTypeToLoad) => + itemTypeToLoad.itemType === fileToLoad.itemType + ); + + if (!itemTypeToLoad) { + console.error( + `Item type to load not found for item type: ${fileToLoad.itemType}.` + ); + + return { + status: 'error', + error: { + message: `Item type to load not found for item type: ${fileToLoad.itemType}.`, + }, + }; + } + + if (!fileToLoad.completed) { + const { response, error: transformerFileError } = + await this.uploader.getJsonObjectByArtifactId({ + artifactId: fileToLoad.id, + isGzipped: true, + }); + + if (transformerFileError) { + console.error( + `Transformer file not found for artifact ID: ${fileToLoad.id}.` + ); + return { + status: 'error', + error: { + message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`, + }, + }; + } + + const transformerFile = response as ExternalSystemItem[]; + + for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) { + if (this.isTimeout) { + console.log( + 'Timeout detected during data loading. Returning progress to allow continuation.' + ); + return { status: 'progress' }; + } + + const { report, rateLimit } = await this.loadItem({ + item: transformerFile[i], + itemTypeToLoad, + }); + + if (rateLimit?.delay) { + return { status: 'delay', delaySeconds: rateLimit.delay }; + } + + if (report) { + addReportToLoaderReport({ + loaderReports: this.loaderReports, + report, + }); + fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1; + } + } + + fileToLoad.completed = true; + this._processedFiles.push(fileToLoad.id); + } + } + } catch (error) { + console.error('Error during data loading.', serializeError(error)); + return { + status: 'error', + error: { + message: `Error during data loading. ${serializeError(error)}`, + }, + }; + } + + return { status: 'success' }; + } + + async getLoaderBatches({ + supportedItemTypes, + }: { + supportedItemTypes: string[]; + }) { + const statsFileArtifactId = this.event.payload.event_data?.stats_file; + + if (statsFileArtifactId) { + const { response, error: statsFileError } = + await this.uploader.getJsonObjectByArtifactId({ + artifactId: statsFileArtifactId, + }); + + const statsFile = response as StatsFileObject[]; + + if (statsFileError || statsFile.length === 0) { + return [] as FileToLoad[]; + } + + const filesToLoad = getFilesToLoad({ + supportedItemTypes, + statsFile, + }); + + return filesToLoad; + } + + return [] as FileToLoad[]; + } + + async loadAttachments({ + create, + }: { + create: ExternalSystemLoadingFunction; + }): Promise { + if (this.event.payload.event_type === EventType.StartLoadingAttachments) { + this.sdkState.fromDevRev = { + filesToLoad: await this.getLoaderBatches({ + supportedItemTypes: ['attachment'], + }), + }; + } + + if ( + !this.sdkState.fromDevRev || + this.sdkState.fromDevRev?.filesToLoad.length === 0 + ) { + console.log('No files to load, returning.'); + return { status: 'success' }; + } + + const filesToLoad = this.sdkState.fromDevRev?.filesToLoad; + + try { + for (const fileToLoad of filesToLoad) { + if (!fileToLoad.completed) { + const { response, error: transformerFileError } = + await this.uploader.getJsonObjectByArtifactId({ + artifactId: fileToLoad.id, + isGzipped: true, + }); + + const transformerFile = response as ExternalSystemAttachment[]; + + if (transformerFileError) { + console.error( + `Transformer file not found for artifact ID: ${fileToLoad.id}.` + ); + return { + status: 'error', + error: { + message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`, + }, + }; + } + + for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) { + if (this.isTimeout) { + console.log( + 'Timeout detected during attachment loading. Returning progress to allow continuation.' + ); + return { status: 'progress' }; + } + + const { report, rateLimit } = await this.loadAttachment({ + item: transformerFile[i], + create, + }); + + if (rateLimit?.delay) { + return { status: 'delay', delaySeconds: rateLimit.delay }; + } + + if (report) { + addReportToLoaderReport({ + loaderReports: this.loaderReports, + report, + }); + fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1; + } + } + + fileToLoad.completed = true; + this._processedFiles.push(fileToLoad.id); + } + } + } catch (error) { + console.error('Error during attachment loading.', serializeError(error)); + return { + status: 'error', + error: { + message: `Error during attachment loading. ${serializeError(error)}`, + }, + }; + } + + return { status: 'success' }; + } + + async loadItem({ + item, + itemTypeToLoad, + }: { + item: ExternalSystemItem; + itemTypeToLoad: ItemTypeToLoad; + }): Promise { + const devrevId = item.id.devrev; + + try { + const syncMapperRecord = await this._mappers.getByTargetId({ + sync_unit: this.event.payload.event_context.sync_unit, + target: devrevId, + }); + + if (!syncMapperRecord) { + console.warn('Failed to get sync mapper record from response.'); + return { + error: { + message: 'Failed to get sync mapper record from response.', + }, + }; + } + + const { id, modifiedDate, delay, error } = await itemTypeToLoad.update({ + item, + mappers: this._mappers, + event: this.event, + }); + + if (id) { + try { + const syncMapperRecordUpdate = await this._mappers.update({ + id: syncMapperRecord.sync_mapper_record.id, + sync_unit: this.event.payload.event_context.sync_unit, + status: SyncMapperRecordStatus.OPERATIONAL, + ...(modifiedDate && { + external_versions: { + add: [ + { + modified_date: modifiedDate, + recipe_version: 0, + }, + ], + }, + }), + external_ids: { + add: [id], + }, + targets: { + add: [devrevId], + }, + }); + + console.log( + 'Successfully updated sync mapper record.', + syncMapperRecordUpdate + ); + } catch (error) { + console.warn( + 'Failed to update sync mapper record.', + serializeError(error) + ); + return { + error: { + message: + 'Failed to update sync mapper record' + serializeError(error), + }, + }; + } + + return { + report: { + item_type: itemTypeToLoad.itemType, + [ActionType.UPDATED]: 1, + }, + }; + } else if (delay) { + console.log( + `Rate limited while updating item in external system, delaying for ${delay} seconds.` + ); + + return { + rateLimit: { + delay, + }, + }; + } else { + console.warn('Failed to update item in external system', error); + return { + report: { + item_type: itemTypeToLoad.itemType, + [ActionType.FAILED]: 1, + }, + }; + } + + // TODO: Update mapper (optional) + } catch (error) { + if (axios.isAxiosError(error)) { + if (error.response?.status === 404) { + // Create item in external system if mapper record not found + const { id, modifiedDate, delay, error } = + await itemTypeToLoad.create({ + item, + mappers: this._mappers, + event: this.event, + }); + + if (id) { + try { + const syncMapperRecordCreate = await this._mappers.create({ + sync_unit: this.event.payload.event_context.sync_unit, + status: SyncMapperRecordStatus.OPERATIONAL, + external_ids: [id], + targets: [devrevId], + ...(modifiedDate && { + external_versions: [ + { + modified_date: modifiedDate, + recipe_version: 0, + }, + ], + }), + }); + + console.log( + 'Successfully created sync mapper record.', + syncMapperRecordCreate + ); + + return { + report: { + item_type: itemTypeToLoad.itemType, + [ActionType.CREATED]: 1, + }, + }; + } catch (error) { + console.warn( + 'Failed to create sync mapper record.', + serializeError(error) + ); + return { + error: { + message: + 'Failed to create sync mapper record. ' + + serializeError(error), + }, + }; + } + } else if (delay) { + return { + rateLimit: { + delay, + }, + }; + } else { + console.warn( + 'Failed to create item in external system.', + serializeError(error) + ); + return { + report: { + item_type: itemTypeToLoad.itemType, + [ActionType.FAILED]: 1, + }, + }; + } + } else { + console.warn( + 'Failed to get sync mapper record.', + serializeError(error) + ); + return { + error: { + message: error.message, + }, + }; + } + } + + console.warn('Failed to get sync mapper record.', serializeError(error)); + return { + error: { + message: 'Failed to get sync mapper record. ' + serializeError(error), + }, + }; + } + } + + async loadAttachment({ + item, + create, + }: { + item: ExternalSystemAttachment; + create: ExternalSystemLoadingFunction; + }): Promise { + const { id, delay, error } = await create({ + item, + mappers: this._mappers, + event: this.event, + }); + + if (delay) { + return { + rateLimit: { + delay, + }, + }; + } else if (id) { + try { + const syncMapperRecordCreate = await this._mappers.create({ + sync_unit: this.event.payload.event_context.sync_unit, + external_ids: [id], + targets: [item.reference_id], + status: SyncMapperRecordStatus.OPERATIONAL, + }); + + console.log( + 'Successfully created sync mapper record.', + syncMapperRecordCreate + ); + } catch (error) { + console.warn( + 'Failed to create sync mapper record.', + serializeError(error) + ); + } + + return { + report: { + item_type: 'attachments', + [ActionType.CREATED]: 1, + }, + }; + } else { + console.warn('Failed to create attachment in external system', error); + return { + report: { + item_type: 'attachments', + [ActionType.FAILED]: 1, + }, + }; + } + } +} diff --git a/src/multithreading/create-worker.test.ts b/src/multithreading/create-worker.test.ts index 0c9eb07e..2dde0e06 100644 --- a/src/multithreading/create-worker.test.ts +++ b/src/multithreading/create-worker.test.ts @@ -1,8 +1,9 @@ import { isMainThread, Worker } from 'worker_threads'; +import { createMockEvent } from '../testing/mock-event'; import { mockServer } from '../tests/jest.setup'; -import { createMockEvent } from '../common/test-utils'; import { EventType } from '../types/extraction'; + import { createWorker } from './create-worker'; describe(createWorker.name, () => { @@ -10,7 +11,7 @@ describe(createWorker.name, () => { // Arrange const workerPath = __dirname + '../tests/dummy-worker.ts'; const event = createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.ExtractionExternalSyncUnitsStart }, + payload: { event_type: EventType.StartExtractingExternalSyncUnits }, }); // Act @@ -38,7 +39,7 @@ describe(createWorker.name, () => { (isMainThread as any) = false; const workerPath = __dirname + '../tests/dummy-worker.ts'; const event = createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.ExtractionExternalSyncUnitsStart }, + payload: { event_type: EventType.StartExtractingExternalSyncUnits }, }); // Act & Assert @@ -58,7 +59,7 @@ describe(createWorker.name, () => { // Arrange const workerPath = __dirname + '../tests/dummy-worker.ts'; const event = createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.ExtractionExternalSyncUnitsStart }, + payload: { event_type: EventType.StartExtractingExternalSyncUnits }, }); if (isMainThread) { @@ -85,7 +86,7 @@ describe(createWorker.name, () => { }, }; const event = createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.ExtractionDataStart }, + payload: { event_type: EventType.StartExtractingData }, }); if (isMainThread) { @@ -106,7 +107,7 @@ describe(createWorker.name, () => { // Arrange const workerPath = __dirname + '../tests/dummy-worker.ts'; const event = createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.ExtractionMetadataStart }, + payload: { event_type: EventType.StartExtractingMetadata }, }); if (isMainThread) { diff --git a/src/common/control-protocol.ts b/src/multithreading/emit.ts similarity index 75% rename from src/common/control-protocol.ts rename to src/multithreading/emit.ts index 0649b4b2..9c1c81cd 100644 --- a/src/common/control-protocol.ts +++ b/src/multithreading/emit.ts @@ -1,7 +1,9 @@ import { AxiosResponse } from 'axios'; -import { axiosClient } from '../http/axios-client-internal'; + +import { LIBRARY_VERSION } from '../common/constants'; +import { axiosClient } from '../http/client'; import { - AirdropEvent, + AirSyncEvent, EventData, ExtractorEvent, ExtractorEventType, @@ -9,11 +11,9 @@ import { WorkerMetadata, } from '../types/extraction'; import { LoaderEventType } from '../types/loading'; -import { LIBRARY_VERSION } from './constants'; -import { translateOutgoingEventType } from './event-type-translation'; export interface EmitInterface { - event: AirdropEvent; + event: AirSyncEvent; eventType: ExtractorEventType | LoaderEventType; data?: EventData; worker_metadata?: WorkerMetadata; @@ -25,12 +25,8 @@ export const emit = async ({ data, worker_metadata, }: EmitInterface): Promise => { - // Translate outgoing event type to ensure we always send new event types - // TODO: Remove when the old types are completely phased out - const translatedEventType = translateOutgoingEventType(eventType); - const newEvent: ExtractorEvent | LoaderEvent = { - event_type: translatedEventType, + event_type: eventType, event_context: event.payload.event_context, event_data: { ...data, diff --git a/src/multithreading/process-task.test.ts b/src/multithreading/process-task.test.ts index 015c5612..17d96103 100644 --- a/src/multithreading/process-task.test.ts +++ b/src/multithreading/process-task.test.ts @@ -3,9 +3,6 @@ import { WorkerEvent, WorkerMessageSubject } from '../types/workers'; // These tests cover logic that is NOT exercised by the end-to-end integration // tests under src/tests/timeout-handling/: -// - translation of legacy wire event types into the new enum (mutates event in place) -// - the hasWorkerEmitted guard that prevents onTimeout from firing after a -// successful emit (integration tests only exercise the positive case) // - the error branch that posts WorkerMessageFailed and exits(1) // - the WorkerMessage handler's guard that only flips isTimeout on // WorkerMessageExit (integration tests can't cleanly target non-Exit messages) @@ -34,10 +31,6 @@ jest.mock('node:worker_threads', () => ({ }, })); -jest.mock('../common/event-type-translation', () => ({ - translateIncomingEventType: jest.fn((t: string) => t), -})); - jest.mock('../logger/logger', () => ({ Logger: jest.fn().mockImplementation(() => ({ log: jest.fn(), @@ -49,27 +42,22 @@ jest.mock('../logger/logger', () => ({ serializeError: jest.fn((e: unknown) => String(e)), })); -jest.mock('../logger/logger.context', () => ({ - runWithSdkLogContext: jest.fn((fn: () => unknown) => fn()), - runWithUserLogContext: jest.fn((fn: () => unknown) => fn()), -})); - -jest.mock('../state/state', () => ({ - createAdapterState: jest.fn(), +jest.mock('../state/extraction-state', () => ({ + createExtractionState: jest.fn(), })); -jest.mock('./worker-adapter/worker-adapter', () => ({ - WorkerAdapter: jest.fn().mockImplementation(() => ({ +jest.mock('./adapters/extraction-adapter', () => ({ + ExtractionAdapter: jest.fn().mockImplementation(() => ({ isTimeout: false, - hasWorkerEmitted: false, + emitFromResult: jest.fn().mockResolvedValue(undefined), })), })); -import { processTask } from './process-task'; -import { translateIncomingEventType } from '../common/event-type-translation'; -import { createAdapterState } from '../state/state'; -import { WorkerAdapter } from './worker-adapter/worker-adapter'; -import { createMockEvent } from '../common/test-utils'; +import { createExtractionState } from '../state/extraction-state'; +import { createMockEvent } from '../testing/mock-event'; + +import { ExtractionAdapter } from './adapters/extraction-adapter'; +import { processExtractionTask } from './process-task'; function setWorkerData(data: Record) { (global as Record).__workerData__ = data; @@ -84,7 +72,7 @@ function makeEvent(eventType = EventType.StartExtractingData) { // Flush the microtask queue enough to let the async IIFE inside processTask run. const flush = async () => new Promise((r) => setTimeout(r, 0)); -describe(processTask.name, () => { +describe(processExtractionTask.name, () => { let processExitSpy: jest.SpyInstance; beforeEach(() => { @@ -95,97 +83,106 @@ describe(processTask.name, () => { .spyOn(process, 'exit') .mockImplementation((() => {}) as () => never); - (createAdapterState as jest.Mock).mockResolvedValue({}); + (createExtractionState as jest.Mock).mockResolvedValue({}); }); afterEach(() => { processExitSpy.mockRestore(); }); - it('should translate incoming event type before passing to task', async () => { + it('should NOT flip adapter.isTimeout when a non-Exit WorkerMessage arrives', async () => { // Arrange - const event = makeEvent(EventType.StartExtractingData); + const event = makeEvent(); setWorkerData({ event, initialState: {}, options: {} }); - (translateIncomingEventType as jest.Mock).mockReturnValue( - EventType.StartExtractingMetadata - ); - const task = jest.fn().mockResolvedValue(undefined); - const onTimeout = jest.fn().mockResolvedValue(undefined); + const mockAdapter = { + isTimeout: false, + emitFromResult: jest.fn().mockResolvedValue(undefined), + }; + (ExtractionAdapter as jest.Mock).mockImplementation(() => mockAdapter); + const task = jest.fn().mockResolvedValue({ status: 'success' }); + const onTimeout = jest.fn().mockResolvedValue({ status: 'progress' }); // Act - processTask({ task, onTimeout }); + processExtractionTask({ task, onTimeout }); await flush(); - // Assert - expect(translateIncomingEventType).toHaveBeenCalledWith( - EventType.StartExtractingData + // Grab the handler registered for WorkerMessage events and invoke it with + // subjects that must NOT flip isTimeout (log messages, unknown subjects). + const messageHandlerCall = mockParentPortOn.mock.calls.find( + ([eventName]) => eventName === WorkerEvent.WorkerMessage ); - // The event is mutated in place — downstream code (including task) sees the - // translated type, not the original wire type. - expect(event.payload.event_type).toBe(EventType.StartExtractingMetadata); + expect(messageHandlerCall).toBeDefined(); + const handler = messageHandlerCall![1] as (m: unknown) => void; + + handler({ subject: WorkerMessageSubject.WorkerMessageLog }); + handler({ subject: 'NONSENSE_SUBJECT' }); + + // Assert + expect(mockAdapter.isTimeout).toBe(false); }); - it('should NOT call onTimeout when the worker already emitted before timeout check', async () => { + it('should emit a progress result on timeout when onTimeout is omitted in a resumable phase', async () => { // Arrange - const event = makeEvent(); + const event = makeEvent(EventType.StartExtractingData); setWorkerData({ event, initialState: {}, options: {} }); - // Both flags true: a timeout arrived but the worker had already emitted — - // onTimeout must be skipped. This is the guard the integration suite cannot - // target cleanly because it requires a precise race between emit and timeout. - const mockAdapter = { isTimeout: true, hasWorkerEmitted: true }; - (WorkerAdapter as jest.Mock).mockImplementation(() => mockAdapter); - const task = jest.fn().mockResolvedValue(undefined); - const onTimeout = jest.fn().mockResolvedValue(undefined); + const mockAdapter = { + event, + isTimeout: true, + emitFromResult: jest.fn().mockResolvedValue(undefined), + }; + (ExtractionAdapter as jest.Mock).mockImplementation(() => mockAdapter); + const task = jest.fn().mockResolvedValue({ status: 'success' }); // Act - processTask({ task, onTimeout }); + processExtractionTask({ task }); await flush(); // Assert - expect(onTimeout).not.toHaveBeenCalled(); - expect(processExitSpy).toHaveBeenCalledWith(0); + expect(mockAdapter.emitFromResult).toHaveBeenCalledWith({ + status: 'progress', + }); }); - it('should NOT flip adapter.isTimeout when a non-Exit WorkerMessage arrives', async () => { + it('should emit a timeout error result when onTimeout is omitted in a non-resumable phase', async () => { // Arrange - const event = makeEvent(); + const event = makeEvent(EventType.StartExtractingMetadata); setWorkerData({ event, initialState: {}, options: {} }); - const mockAdapter = { isTimeout: false, hasWorkerEmitted: false }; - (WorkerAdapter as jest.Mock).mockImplementation(() => mockAdapter); - const task = jest.fn().mockResolvedValue(undefined); - const onTimeout = jest.fn().mockResolvedValue(undefined); + const mockAdapter = { + event, + isTimeout: true, + emitFromResult: jest.fn().mockResolvedValue(undefined), + }; + (ExtractionAdapter as jest.Mock).mockImplementation(() => mockAdapter); + const task = jest.fn().mockResolvedValue({ status: 'success' }); // Act - processTask({ task, onTimeout }); + processExtractionTask({ task }); await flush(); - // Grab the handler registered for WorkerMessage events and invoke it with - // subjects that must NOT flip isTimeout (log messages, unknown subjects). - const messageHandlerCall = mockParentPortOn.mock.calls.find( - ([eventName]) => eventName === WorkerEvent.WorkerMessage - ); - expect(messageHandlerCall).toBeDefined(); - const handler = messageHandlerCall![1] as (m: unknown) => void; - - handler({ subject: WorkerMessageSubject.WorkerMessageLog }); - handler({ subject: 'NONSENSE_SUBJECT' }); - // Assert - expect(mockAdapter.isTimeout).toBe(false); + expect(mockAdapter.emitFromResult).toHaveBeenCalledWith({ + status: 'error', + error: { + message: expect.stringContaining('non-resumable phase'), + }, + }); }); it('should post WorkerMessageFailed with the error message and exit(1) when task throws', async () => { // Arrange const event = makeEvent(); setWorkerData({ event, initialState: {}, options: {} }); - const mockAdapter = { isTimeout: false, hasWorkerEmitted: false }; - (WorkerAdapter as jest.Mock).mockImplementation(() => mockAdapter); + const mockAdapter = { + isTimeout: false, + emitFromResult: jest.fn().mockResolvedValue(undefined), + }; + (ExtractionAdapter as jest.Mock).mockImplementation(() => mockAdapter); const taskError = new Error('task boom'); const task = jest.fn().mockRejectedValue(taskError); - const onTimeout = jest.fn().mockResolvedValue(undefined); + const onTimeout = jest.fn().mockResolvedValue({ status: 'progress' }); // Act - processTask({ task, onTimeout }); + processExtractionTask({ task, onTimeout }); await flush(); // Assert diff --git a/src/multithreading/process-task.ts b/src/multithreading/process-task.ts index 4f3e75a3..68565e5e 100644 --- a/src/multithreading/process-task.ts +++ b/src/multithreading/process-task.ts @@ -1,81 +1,144 @@ import { isMainThread, parentPort, workerData } from 'node:worker_threads'; -import { translateIncomingEventType } from '../common/event-type-translation'; + import { Logger, serializeError } from '../logger/logger'; -import { - runWithSdkLogContext, - runWithUserLogContext, -} from '../logger/logger.context'; -import { createAdapterState } from '../state/state'; +import { createExtractionState } from '../state/extraction-state'; +import { createLoadingState } from '../state/loading-state'; import { ProcessTaskInterface, + TaskResult, WorkerEvent, WorkerMessageSubject, } from '../types/workers'; -import { WorkerAdapter } from './worker-adapter/worker-adapter'; -export function processTask({ +import { BaseAdapter } from './adapters/base-adapter'; +import { ExtractionAdapter } from './adapters/extraction-adapter'; +import { LoadingAdapter } from './adapters/loading-adapter'; +import { getEventTypeForResult } from './spawn/spawn.helpers'; + +/** + * Shared worker-thread driver: builds the adapter, runs the task, maps the + * returned {@link TaskResult} to a platform event and emits it exactly once. + * On soft timeout the timeout outcome always wins: the `onTimeout` result (or + * a phase-appropriate default) is emitted and the task's return value is + * ignored. + */ +async function runWorkerTask>( + buildAdapter: () => Promise, + { task, onTimeout }: ProcessTaskInterface +): Promise { + try { + const adapter = await buildAdapter(); + + parentPort?.on(WorkerEvent.WorkerMessage, (message) => { + if (message.subject !== WorkerMessageSubject.WorkerMessageExit) { + return; + } + console.log('Timeout received. Waiting for the task to finish.'); + adapter.isTimeout = true; + }); + + let result: TaskResult = await task({ adapter }); + + if (adapter.isTimeout) { + if (onTimeout) { + result = await onTimeout({ adapter }); + } else { + // Non-resumable phases can't hand off with `progress`; report a timeout error. + const eventType = adapter.event.payload.event_type; + const { illegal } = getEventTypeForResult(eventType, 'progress'); + result = illegal + ? { + status: 'error', + error: { + message: `Worker timed out during a non-resumable phase (${eventType}), which cannot be continued.`, + }, + } + : { status: 'progress' }; + } + } + + await adapter.emitFromResult(result); + + process.exit(0); + } catch (error) { + const errorMessage = `Error while processing task. ${serializeError( + error + )}`; + console.error(errorMessage); + parentPort?.postMessage({ + subject: WorkerMessageSubject.WorkerMessageFailed, + payload: { message: errorMessage }, + }); + process.exit(1); + } +} + +/** Entry point for an extraction worker. @public */ +export function processExtractionTask({ task, onTimeout, -}: ProcessTaskInterface) { +}: ProcessTaskInterface>) { if (isMainThread) { return; } - void (async () => { - await runWithSdkLogContext(async () => { - try { - const event = workerData.event; - - // TODO: Remove when the old types are completely phased out - event.payload.event_type = translateIncomingEventType( - event.payload.event_type - ); - - const initialState = workerData.initialState as ConnectorState; - const initialDomainMapping = workerData.initialDomainMapping; - const options = workerData.options; - // eslint-disable-next-line no-global-assign - console = new Logger({ event, options }); - - const adapterState = await createAdapterState({ - event, - initialState, - initialDomainMapping, - options, - }); - - const adapter = new WorkerAdapter({ - event, - adapterState, - options, - }); - - parentPort?.on(WorkerEvent.WorkerMessage, (message) => { - if (message.subject !== WorkerMessageSubject.WorkerMessageExit) { - return; - } - console.log('Timeout received. Waiting for the task to finish.'); - adapter.isTimeout = true; - }); - - await runWithUserLogContext(async () => task({ adapter })); - if (adapter.isTimeout && !adapter.hasWorkerEmitted) { - await runWithUserLogContext(async () => onTimeout({ adapter })); - } - process.exit(0); - } catch (error) { - runWithUserLogContext(() => { - const errorMessage = `Error while processing task. ${serializeError( - error - )}`; - console.error(errorMessage); - parentPort?.postMessage({ - subject: WorkerMessageSubject.WorkerMessageFailed, - payload: { message: errorMessage }, - }); - process.exit(1); - }); - } - }); - })(); + void runWorkerTask>( + async () => { + const event = workerData.event; + const initialState = workerData.initialState as ConnectorState; + const initialDomainMapping = workerData.initialDomainMapping; + const options = workerData.options; + // eslint-disable-next-line no-global-assign + console = new Logger({ event, options }); + + const adapterState = await createExtractionState({ + event, + initialState, + initialDomainMapping, + options, + }); + + return new ExtractionAdapter({ + event, + adapterState, + options, + }); + }, + { task, onTimeout } + ); +} + +/** Entry point for a loading worker. @public */ +export function processLoadingTask({ + task, + onTimeout, +}: ProcessTaskInterface>) { + if (isMainThread) { + return; + } + + void runWorkerTask>( + async () => { + const event = workerData.event; + const initialState = workerData.initialState as ConnectorState; + const initialDomainMapping = workerData.initialDomainMapping; + const options = workerData.options; + // eslint-disable-next-line no-global-assign + console = new Logger({ event, options }); + + const adapterState = await createLoadingState({ + event, + initialState, + initialDomainMapping, + options, + }); + + return new LoadingAdapter({ + event, + adapterState, + options, + }); + }, + { task, onTimeout } + ); } diff --git a/src/multithreading/spawn/spawn.helpers.test.ts b/src/multithreading/spawn/spawn.helpers.test.ts index 6645ac80..668fdb10 100644 --- a/src/multithreading/spawn/spawn.helpers.test.ts +++ b/src/multithreading/spawn/spawn.helpers.test.ts @@ -1,3 +1,4 @@ +import { UNKNOWN_EVENT_TYPE } from '../../common/constants'; import { EventType, ExtractorEventType } from '../../types/extraction'; import { LoaderEventType } from '../../types/loading'; @@ -19,9 +20,9 @@ describe(getTimeoutErrorEventType.name, () => { expect(result.eventType).toBe(ExtractorEventType.MetadataExtractionError); }); - it('should return MetadataExtractionError for deprecated ExtractionMetadataStart', () => { + it('should return MetadataExtractionError for StartExtractingMetadata (renamed from ExtractionMetadataStart)', () => { // Arrange - const eventType = EventType.ExtractionMetadataStart; + const eventType = EventType.StartExtractingMetadata; // Act const result = getTimeoutErrorEventType(eventType); @@ -54,9 +55,9 @@ describe(getTimeoutErrorEventType.name, () => { expect(result.eventType).toBe(ExtractorEventType.DataExtractionError); }); - it('should return DataExtractionError for deprecated ExtractionDataStart', () => { + it('should return DataExtractionError for StartExtractingData (renamed from ExtractionDataStart)', () => { // Arrange - const eventType = EventType.ExtractionDataStart; + const eventType = EventType.StartExtractingData; // Act const result = getTimeoutErrorEventType(eventType); @@ -65,9 +66,9 @@ describe(getTimeoutErrorEventType.name, () => { expect(result.eventType).toBe(ExtractorEventType.DataExtractionError); }); - it('should return DataExtractionError for deprecated ExtractionDataContinue', () => { + it('should return DataExtractionError for ContinueExtractingData (renamed from ExtractionDataContinue)', () => { // Arrange - const eventType = EventType.ExtractionDataContinue; + const eventType = EventType.ContinueExtractingData; // Act const result = getTimeoutErrorEventType(eventType); @@ -91,9 +92,9 @@ describe(getTimeoutErrorEventType.name, () => { ); }); - it('should return ExtractorStateDeletionError for deprecated ExtractionDataDelete', () => { + it('should return ExtractorStateDeletionError for StartDeletingExtractorState (renamed from ExtractionDataDelete)', () => { // Arrange - const eventType = EventType.ExtractionDataDelete; + const eventType = EventType.StartDeletingExtractorState; // Act const result = getTimeoutErrorEventType(eventType); @@ -132,9 +133,9 @@ describe(getTimeoutErrorEventType.name, () => { ); }); - it('should return AttachmentExtractionError for deprecated ExtractionAttachmentsStart', () => { + it('should return AttachmentExtractionError for StartExtractingAttachments (renamed from ExtractionAttachmentsStart)', () => { // Arrange - const eventType = EventType.ExtractionAttachmentsStart; + const eventType = EventType.StartExtractingAttachments; // Act const result = getTimeoutErrorEventType(eventType); @@ -145,9 +146,9 @@ describe(getTimeoutErrorEventType.name, () => { ); }); - it('should return AttachmentExtractionError for deprecated ExtractionAttachmentsContinue', () => { + it('should return AttachmentExtractionError for ContinueExtractingAttachments (renamed from ExtractionAttachmentsContinue)', () => { // Arrange - const eventType = EventType.ExtractionAttachmentsContinue; + const eventType = EventType.ContinueExtractingAttachments; // Act const result = getTimeoutErrorEventType(eventType); @@ -173,9 +174,9 @@ describe(getTimeoutErrorEventType.name, () => { ); }); - it('should return ExtractorAttachmentsStateDeletionError for deprecated ExtractionAttachmentsDelete', () => { + it('should return ExtractorAttachmentsStateDeletionError for StartDeletingExtractorAttachmentsState (renamed from ExtractionAttachmentsDelete)', () => { // Arrange - const eventType = EventType.ExtractionAttachmentsDelete; + const eventType = EventType.StartDeletingExtractorAttachmentsState; // Act const result = getTimeoutErrorEventType(eventType); @@ -201,9 +202,9 @@ describe(getTimeoutErrorEventType.name, () => { ); }); - it('should return ExternalSyncUnitExtractionError for deprecated ExtractionExternalSyncUnitsStart', () => { + it('should return ExternalSyncUnitExtractionError for StartExtractingExternalSyncUnits (renamed from ExtractionExternalSyncUnitsStart)', () => { // Arrange - const eventType = EventType.ExtractionExternalSyncUnitsStart; + const eventType = EventType.StartExtractingExternalSyncUnits; // Act const result = getTimeoutErrorEventType(eventType); @@ -292,9 +293,9 @@ describe(getTimeoutErrorEventType.name, () => { }); describe('unknown event types', () => { - it('[edge] should return UnknownEventType and log error for unrecognized event type', () => { + it('[edge] should return UNKNOWN_EVENT_TYPE and log error for unrecognized event type', () => { // Arrange - const eventType = EventType.UnknownEventType; + const eventType = 'TOTALLY_UNKNOWN' as EventType; const consoleErrorSpy = jest .spyOn(console, 'error') .mockImplementation(() => {}); @@ -303,7 +304,7 @@ describe(getTimeoutErrorEventType.name, () => { const result = getTimeoutErrorEventType(eventType); // Assert - expect(result.eventType).toBe(LoaderEventType.UnknownEventType); + expect(result.eventType).toBe(UNKNOWN_EVENT_TYPE); expect(consoleErrorSpy).toHaveBeenCalledWith( 'Event type not recognized in getTimeoutErrorEventType function: ' + eventType @@ -375,7 +376,7 @@ describe(getNoScriptEventType.name, () => { }); describe('unknown event types', () => { - it('[edge] should return UnknownEventType and log error for unrecognized event type', () => { + it('[edge] should return UNKNOWN_EVENT_TYPE and log error for unrecognized event type', () => { // Arrange const eventType = EventType.StartExtractingData; const consoleErrorSpy = jest @@ -386,7 +387,7 @@ describe(getNoScriptEventType.name, () => { const result = getNoScriptEventType(eventType); // Assert - expect(result.eventType).toBe(LoaderEventType.UnknownEventType); + expect(result.eventType).toBe(UNKNOWN_EVENT_TYPE); expect(consoleErrorSpy).toHaveBeenCalledWith( 'Event type not recognized in getNoScriptEventType function: ' + eventType @@ -396,7 +397,7 @@ describe(getNoScriptEventType.name, () => { consoleErrorSpy.mockRestore(); }); - it('[edge] should return UnknownEventType for StartLoadingData', () => { + it('[edge] should return UNKNOWN_EVENT_TYPE for StartLoadingData', () => { // Arrange const eventType = EventType.StartLoadingData; const consoleErrorSpy = jest @@ -407,7 +408,7 @@ describe(getNoScriptEventType.name, () => { const result = getNoScriptEventType(eventType); // Assert - expect(result.eventType).toBe(LoaderEventType.UnknownEventType); + expect(result.eventType).toBe(UNKNOWN_EVENT_TYPE); expect(consoleErrorSpy).toHaveBeenCalled(); // Cleanup diff --git a/src/multithreading/spawn/spawn.helpers.ts b/src/multithreading/spawn/spawn.helpers.ts index abbe9908..71d08644 100644 --- a/src/multithreading/spawn/spawn.helpers.ts +++ b/src/multithreading/spawn/spawn.helpers.ts @@ -1,82 +1,210 @@ +import { UNKNOWN_EVENT_TYPE } from '../../common/constants'; import { EventType, ExtractorEventType } from '../../types/extraction'; import { LoaderEventType } from '../../types/loading'; +import { TaskStatus } from '../../types/workers'; /** - * Gets the event type for the timeout error. - * @param {EventType} eventType - The event type to get the timeout error event type for - * @returns {ExtractorEventType | LoaderEventType} The event type for the timeout error + * Resolves the outgoing event type to emit for an incoming event type and a + * {@link TaskResult} status. Resumable phases honor every status + * (success/progress/delay/error -> *_DONE/*_PROGRESS/*_DELAYED/*_ERROR); + * non-resumable phases only have done/error, so `progress`/`delay` there is + * illegal and maps to the phase's error event. */ +export function getEventTypeForResult( + eventType: EventType, + status: TaskStatus +): { + eventType: ExtractorEventType | LoaderEventType; + illegal: boolean; +} { + const phase = EVENT_PHASE_MAP[eventType]; + + if (!phase) { + console.error( + 'Event type not recognized in getEventTypeForResult function: ' + + eventType + ); + return { + eventType: UNKNOWN_EVENT_TYPE as ExtractorEventType | LoaderEventType, + illegal: true, + }; + } + + // Non-resumable phases only define done/error events. + if (!phase.resumable) { + if (status === 'success') { + return { eventType: phase.done, illegal: false }; + } + // progress/delay are illegal here; collapse them (and error) to the error event. + return { eventType: phase.error, illegal: status !== 'error' }; + } + + switch (status) { + case 'success': + return { eventType: phase.done, illegal: false }; + case 'progress': + return { eventType: phase.progress!, illegal: false }; + case 'delay': + return { eventType: phase.delayed!, illegal: false }; + case 'error': + return { eventType: phase.error, illegal: false }; + } +} + +/** Per-phase outgoing event types, keyed by the incoming {@link EventType}. */ +const EVENT_PHASE_MAP: Partial< + Record< + EventType, + { + resumable: boolean; + done: ExtractorEventType | LoaderEventType; + error: ExtractorEventType | LoaderEventType; + progress?: ExtractorEventType | LoaderEventType; + delayed?: ExtractorEventType | LoaderEventType; + } + > +> = { + [EventType.StartExtractingExternalSyncUnits]: { + resumable: false, + done: ExtractorEventType.ExternalSyncUnitExtractionDone, + error: ExtractorEventType.ExternalSyncUnitExtractionError, + }, + [EventType.StartExtractingMetadata]: { + resumable: false, + done: ExtractorEventType.MetadataExtractionDone, + error: ExtractorEventType.MetadataExtractionError, + }, + [EventType.StartExtractingData]: { + resumable: true, + done: ExtractorEventType.DataExtractionDone, + error: ExtractorEventType.DataExtractionError, + progress: ExtractorEventType.DataExtractionProgress, + delayed: ExtractorEventType.DataExtractionDelayed, + }, + [EventType.ContinueExtractingData]: { + resumable: true, + done: ExtractorEventType.DataExtractionDone, + error: ExtractorEventType.DataExtractionError, + progress: ExtractorEventType.DataExtractionProgress, + delayed: ExtractorEventType.DataExtractionDelayed, + }, + [EventType.StartDeletingExtractorState]: { + resumable: false, + done: ExtractorEventType.ExtractorStateDeletionDone, + error: ExtractorEventType.ExtractorStateDeletionError, + }, + [EventType.StartExtractingAttachments]: { + resumable: true, + done: ExtractorEventType.AttachmentExtractionDone, + error: ExtractorEventType.AttachmentExtractionError, + progress: ExtractorEventType.AttachmentExtractionProgress, + delayed: ExtractorEventType.AttachmentExtractionDelayed, + }, + [EventType.ContinueExtractingAttachments]: { + resumable: true, + done: ExtractorEventType.AttachmentExtractionDone, + error: ExtractorEventType.AttachmentExtractionError, + progress: ExtractorEventType.AttachmentExtractionProgress, + delayed: ExtractorEventType.AttachmentExtractionDelayed, + }, + [EventType.StartDeletingExtractorAttachmentsState]: { + resumable: false, + done: ExtractorEventType.ExtractorAttachmentsStateDeletionDone, + error: ExtractorEventType.ExtractorAttachmentsStateDeletionError, + }, + [EventType.StartLoadingData]: { + resumable: true, + done: LoaderEventType.DataLoadingDone, + error: LoaderEventType.DataLoadingError, + progress: LoaderEventType.DataLoadingProgress, + delayed: LoaderEventType.DataLoadingDelayed, + }, + [EventType.ContinueLoadingData]: { + resumable: true, + done: LoaderEventType.DataLoadingDone, + error: LoaderEventType.DataLoadingError, + progress: LoaderEventType.DataLoadingProgress, + delayed: LoaderEventType.DataLoadingDelayed, + }, + [EventType.StartLoadingAttachments]: { + resumable: true, + done: LoaderEventType.AttachmentLoadingDone, + error: LoaderEventType.AttachmentLoadingError, + progress: LoaderEventType.AttachmentLoadingProgress, + delayed: LoaderEventType.AttachmentLoadingDelayed, + }, + [EventType.ContinueLoadingAttachments]: { + resumable: true, + done: LoaderEventType.AttachmentLoadingDone, + error: LoaderEventType.AttachmentLoadingError, + progress: LoaderEventType.AttachmentLoadingProgress, + delayed: LoaderEventType.AttachmentLoadingDelayed, + }, + [EventType.StartDeletingLoaderState]: { + resumable: false, + done: LoaderEventType.LoaderStateDeletionDone, + error: LoaderEventType.LoaderStateDeletionError, + }, + [EventType.StartDeletingLoaderAttachmentState]: { + resumable: false, + done: LoaderEventType.LoaderAttachmentStateDeletionDone, + error: LoaderEventType.LoaderAttachmentStateDeletionError, + }, +}; + export function getTimeoutErrorEventType(eventType: EventType): { eventType: ExtractorEventType | LoaderEventType; } { switch (eventType) { - // Metadata extraction (handles both old and new enum members) case EventType.StartExtractingMetadata: - case EventType.ExtractionMetadataStart: return { eventType: ExtractorEventType.MetadataExtractionError, }; - // Data extraction (handles both old and new enum members) case EventType.StartExtractingData: case EventType.ContinueExtractingData: - case EventType.ExtractionDataStart: - case EventType.ExtractionDataContinue: return { eventType: ExtractorEventType.DataExtractionError, }; - // Data deletion (handles both old and new enum members) case EventType.StartDeletingExtractorState: - case EventType.ExtractionDataDelete: return { eventType: ExtractorEventType.ExtractorStateDeletionError, }; - // Attachments extraction (handles both old and new enum members) case EventType.StartExtractingAttachments: case EventType.ContinueExtractingAttachments: - case EventType.ExtractionAttachmentsStart: - case EventType.ExtractionAttachmentsContinue: return { eventType: ExtractorEventType.AttachmentExtractionError, }; - // Attachments deletion (handles both old and new enum members) case EventType.StartDeletingExtractorAttachmentsState: - case EventType.ExtractionAttachmentsDelete: return { eventType: ExtractorEventType.ExtractorAttachmentsStateDeletionError, }; - // External sync units (handles both old and new enum members) case EventType.StartExtractingExternalSyncUnits: - case EventType.ExtractionExternalSyncUnitsStart: return { eventType: ExtractorEventType.ExternalSyncUnitExtractionError, }; - // Loading data case EventType.StartLoadingData: case EventType.ContinueLoadingData: return { eventType: LoaderEventType.DataLoadingError, }; - // Deleting loader state case EventType.StartDeletingLoaderState: return { eventType: LoaderEventType.LoaderStateDeletionError, }; - // Loading attachments case EventType.StartLoadingAttachments: case EventType.ContinueLoadingAttachments: return { eventType: LoaderEventType.AttachmentLoadingError, }; - // Deleting loader attachment state case EventType.StartDeletingLoaderAttachmentState: return { eventType: LoaderEventType.LoaderAttachmentStateDeletionError, @@ -88,16 +216,12 @@ export function getTimeoutErrorEventType(eventType: EventType): { eventType ); return { - eventType: LoaderEventType.UnknownEventType, + eventType: UNKNOWN_EVENT_TYPE as ExtractorEventType | LoaderEventType, }; } } -/** - * Gets the event type for the no script error. - * @param {EventType} eventType - The event type to get the no script error event type for - * @returns {ExtractorEventType | LoaderEventType} The event type for the no script error - */ +/** Event type to emit when no worker script exists for the incoming event. */ export function getNoScriptEventType(eventType: EventType) { switch (eventType) { case EventType.StartDeletingExtractorState: @@ -122,7 +246,7 @@ export function getNoScriptEventType(eventType: EventType) { eventType ); return { - eventType: LoaderEventType.UnknownEventType, + eventType: UNKNOWN_EVENT_TYPE as ExtractorEventType | LoaderEventType, }; } } diff --git a/src/multithreading/spawn/spawn.test.ts b/src/multithreading/spawn/spawn.test.ts index 9445c20f..f0e2ad11 100644 --- a/src/multithreading/spawn/spawn.test.ts +++ b/src/multithreading/spawn/spawn.test.ts @@ -1,8 +1,9 @@ import { EventEmitter } from 'events'; + import { DEFAULT_LAMBDA_TIMEOUT } from '../../common/constants'; +import { createMockEvent } from '../../testing/mock-event'; import { EventType, ExtractorEventType } from '../../types/extraction'; import { WorkerEvent, WorkerMessageSubject } from '../../types/workers'; -import { createMockEvent } from '../../common/test-utils'; // --------------------------------------------------------------------------- // Mocks @@ -12,7 +13,7 @@ jest.mock('../create-worker', () => ({ createWorker: jest.fn(), })); -jest.mock('../../common/control-protocol', () => ({ +jest.mock('../emit', () => ({ emit: jest.fn().mockResolvedValue({}), })); @@ -38,17 +39,16 @@ jest.mock('../../common/helpers', () => ({ arrayBuffersMB: '5.00', }), sleep: jest.fn(), - truncateFilename: jest.fn((f: string) => f), - truncateMessage: jest.fn((m: string) => m), })); // --------------------------------------------------------------------------- // Imports after mocks // --------------------------------------------------------------------------- -import { spawn, Spawn } from './spawn'; -import { createWorker } from '../create-worker'; -import { emit } from '../../common/control-protocol'; import { getMemoryUsage } from '../../common/helpers'; +import { createWorker } from '../create-worker'; +import { emit } from '../emit'; + +import { Spawn, spawn } from './spawn'; // --------------------------------------------------------------------------- // Factory for a fake worker (EventEmitter with postMessage + terminate) @@ -113,7 +113,7 @@ describe('spawn() factory', () => { it('should emit a no-script event and NOT spawn a worker for an unknown event type', async () => { // Arrange const event = createMockEvent('http://localhost:0', { - payload: { event_type: EventType.UnknownEventType }, + payload: { event_type: 'TOTALLY_UNKNOWN' as EventType }, }); // Act @@ -137,7 +137,16 @@ describe('spawn() factory', () => { // Act & Assert await expect( - spawn({ event, initialState: {}, workerPath: '/fake/path.js' }) + spawn({ + event, + initialState: {}, + baseWorkerPath: '', + options: { + workerPathOverrides: { + [event.payload.event_type]: '/fake/path.js', + }, + }, + }) ).rejects.toThrow('worker boom'); }); }); diff --git a/src/multithreading/spawn/spawn.ts b/src/multithreading/spawn/spawn.ts index 0350d34d..1c18a5ef 100644 --- a/src/multithreading/spawn/spawn.ts +++ b/src/multithreading/spawn/spawn.ts @@ -1,11 +1,15 @@ import yargs from 'yargs'; import { hideBin } from 'yargs/helpers'; -import { emit } from '../../common/control-protocol'; -import { translateIncomingEventType } from '../../common/event-type-translation'; +import { + DEFAULT_LAMBDA_TIMEOUT, + HARD_TIMEOUT_MULTIPLIER, + MEMORY_LOG_INTERVAL, +} from '../../common/constants'; import { getMemoryUsage } from '../../common/helpers'; import { Logger, serializeError } from '../../logger/logger'; -import { AirdropEvent, EventType } from '../../types/extraction'; +import { LogLevel } from '../../logger/logger.interfaces'; +import { AirSyncEvent, EventType } from '../../types/extraction'; import { GetWorkerPathInterface, SpawnFactoryInterface, @@ -13,17 +17,12 @@ import { WorkerEvent, WorkerMessageSubject, } from '../../types/workers'; - -import { - DEFAULT_LAMBDA_TIMEOUT, - HARD_TIMEOUT_MULTIPLIER, - MEMORY_LOG_INTERVAL, -} from '../../common/constants'; -import { LogLevel } from '../../logger/logger.interfaces'; import { createWorker } from '../create-worker'; +import { emit } from '../emit'; + import { - getTimeoutErrorEventType, getNoScriptEventType, + getTimeoutErrorEventType, } from './spawn.helpers'; function getWorkerPath({ @@ -60,34 +59,17 @@ function getWorkerPath({ } /** - * Creates a new instance of Spawn class. - * Spawn class is responsible for spawning a new worker thread and managing the lifecycle of the worker. - * The class provides utilities to emit control events to the platform and exit the worker gracefully. - * In case of lambda timeout, the class emits a lambda timeout event to the platform. - * @param {SpawnFactoryInterface} options - The options to create a new instance of Spawn class - * @param {AirdropEvent} options.event - The event object received from the platform - * @param {object} options.initialState - The initial state of the adapter - * @param {string} [options.workerPath] Remove getWorkerPath function and use baseWorkerPath: __dirname instead of workerPath - * @param {string} [options.baseWorkerPath] - The base path for the worker files, usually `__dirname` - * @returns {Promise} - A new instance of Spawn class + * Spawns a worker thread for the event and manages its lifecycle (control + * events, graceful exit, lambda timeout handling). Resolves when the worker + * run is fully finished. */ export async function spawn({ event, initialState, - workerPath, initialDomainMapping, options, baseWorkerPath, }: SpawnFactoryInterface): Promise { - // Translate incoming event type for backwards compatibility. This allows the - // SDK to accept both old and new event type formats. Then update the event with the translated event type. - const originalEventType = event.payload.event_type; - const translatedEventType = translateIncomingEventType( - event.payload.event_type as string - ); - event.payload.event_type = translatedEventType; - - // Read the command line arguments to check if the local flag is passed. const argv = await yargs(hideBin(process.argv)).argv; if (argv._.includes('local') || argv.local) { options = { @@ -100,26 +82,19 @@ export async function spawn({ // eslint-disable-next-line no-global-assign console = new Logger({ event, options }); - if (translatedEventType !== originalEventType) { - console.log( - `Event type translated from ${originalEventType} to ${translatedEventType}.` - ); - } if (options?.isLocalDevelopment) { console.log('Snap-in is running in local development mode.'); } let script = null; - if (workerPath != null) { - script = workerPath; - } else if ( + if ( baseWorkerPath != null && options?.workerPathOverrides != null && - options.workerPathOverrides[translatedEventType as EventType] != null + options.workerPathOverrides[event.payload.event_type as EventType] != null ) { script = baseWorkerPath + - options.workerPathOverrides[translatedEventType as EventType]; + options.workerPathOverrides[event.payload.event_type as EventType]; } else { script = getWorkerPath({ event, @@ -127,7 +102,6 @@ export async function spawn({ }); } - // If a script is found for the event type, spawn a new worker. if (script) { try { const worker = await createWorker({ @@ -169,7 +143,7 @@ export async function spawn({ } export class Spawn { - private event: AirdropEvent; + private event: AirSyncEvent; private alreadyEmitted: boolean; private softTimeoutSent: boolean; private defaultLambdaTimeout: number = DEFAULT_LAMBDA_TIMEOUT; @@ -198,7 +172,7 @@ export class Spawn { : this.defaultLambdaTimeout; this.resolve = resolve; - // If soft timeout is reached, send a message to the worker to gracefully exit. + // Soft timeout: ask the worker to gracefully exit. this.softTimeoutTimer = setTimeout( () => void (async () => { @@ -218,7 +192,7 @@ export class Spawn { this.lambdaTimeout ); - // If hard timeout is reached, that means the worker did not exit in time. Terminate the worker. + // Hard timeout: the worker did not exit in time, terminate it. this.hardTimeoutTimer = setTimeout( () => void (async () => { @@ -235,13 +209,10 @@ export class Spawn { this.lambdaTimeout * HARD_TIMEOUT_MULTIPLIER ); - // If worker exits with process.exit(code), clear the timeouts and exit from - // main thread. When a soft timeout was sent, we use setImmediate to defer - // processing so that any pending WorkerMessage events (e.g. - // WorkerMessageEmitted from onTimeout) already queued in the event loop are - // handled first, preventing a race condition where exitFromMainThread sees - // alreadyEmitted=false and emits an error even though the worker - // successfully emitted an event. + // After a soft timeout, defer exit handling via setImmediate so pending + // WorkerMessage events (e.g. WorkerMessageEmitted from onTimeout) are + // handled first; otherwise exitFromMainThread could see + // alreadyEmitted=false and emit a spurious error. worker.on(WorkerEvent.WorkerExit, (code: number) => { const handler = async () => { console.info('Worker exited with exit code: ' + code + '.'); @@ -257,29 +228,25 @@ export class Spawn { }); worker.on(WorkerEvent.WorkerMessage, (message) => { - // Since logs from the worker thread are handled differently in snap-in - // platform, we need to catch the log messages from worker thread and log - // them in main thread. + // The snap-in platform handles worker-thread logs differently, so worker + // log messages are re-logged in the main thread. if (message?.subject === WorkerMessageSubject.WorkerMessageLog) { const stringifiedArgs = message.payload?.stringifiedArgs; const level = message.payload?.level as LogLevel; - const isSdkLog = message.payload?.isSdkLog ?? true; - this.logger.logFn(stringifiedArgs, level, isSdkLog); + this.logger.logFn(stringifiedArgs, level); } - // If worker sends a message that it has emitted an event, then set alreadyEmitted to true. if (message?.subject === WorkerMessageSubject.WorkerMessageEmitted) { - console.info('Worker has emitted message to ADaaS.'); + console.info('Worker has emitted message to AirSync.'); this.alreadyEmitted = true; } - // If worker sends a failure message before exiting, capture it for use in the error event. + // Capture the worker's failure reason for use in the error event. if (message?.subject === WorkerMessageSubject.WorkerMessageFailed) { this.workerFailedMessage = message.payload?.message; } }); - // Log memory usage every 30 seconds this.memoryMonitoringInterval = setInterval(() => { try { const memoryInfo = getMemoryUsage(); @@ -287,7 +254,7 @@ export class Spawn { console.info(memoryInfo.formattedMessage); } } catch (error) { - // If memory monitoring fails, log the warning and clear the interval to prevent further issues + // Stop monitoring on failure to prevent repeated crashes console.warn( 'Memory monitoring failed, stopping logging of memory usage interval', error diff --git a/src/multithreading/worker-adapter/worker-adapter.artifacts.test.ts b/src/multithreading/worker-adapter/worker-adapter.artifacts.test.ts deleted file mode 100644 index f3bef85e..00000000 --- a/src/multithreading/worker-adapter/worker-adapter.artifacts.test.ts +++ /dev/null @@ -1,224 +0,0 @@ -import { AirSyncDefaultItemTypes } from '../../common/constants'; -import { State } from '../../state/state'; -import { mockServer } from '../../tests/jest.setup'; -import { createItems } from '../../tests/test-helpers'; -import { createMockEvent } from '../../common/test-utils'; -import { Artifact, EventType } from '../../types'; -import { ExternalSyncUnit } from '../../types/extraction'; -import { WorkerAdapter } from './worker-adapter'; - -// 1. Create a mock function for the method you want to override. -const mockUpload = (itemType: string, objects: object[]) => { - return { - error: null, - artifact: { - id: `artifact-${itemType}-${Math.random().toString(36).substring(2, 15)}`, - item_type: itemType, - item_count: objects.length, - }, - }; -}; - -// 2. Mock the entire 'uploader' module. -// The factory function () => { ... } returns the mock implementation. -jest.mock('../../uploader/uploader', () => { - return { - // The mocked Uploader class - Uploader: jest.fn().mockImplementation(() => { - // The constructor of the mocked Uploader returns an object - // with the methods you want to control. - return { - upload: mockUpload, - }; - }), - }; -}); - -function checkArtifactOrder( - artifacts: Artifact[], - expectedOrder: { itemType: string }[] -): boolean { - let outerIndex = 0; - for (const artifact of artifacts) { - try { - // Always increase outer index. If items are out of order, the array will overflow and exception will be thrown - while (artifact.item_type != expectedOrder[outerIndex].itemType) { - outerIndex++; - } - } catch (e) { - console.error('Error finding artifact type in repos:', e); - return false; - } - } - return true; -} - -describe('Artifact ordering when artifacts overflow batch sizes in repositories', () => { - interface TestState { - attachments: { completed: boolean }; - } - let testAdapter: WorkerAdapter; - - beforeEach(() => { - // Create a fresh adapter instance for this test to avoid mocking conflicts - const mockEvent = createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.StartExtractingData }, - }); - const mockAdapterState = new State({ - event: mockEvent, - initialState: { attachments: { completed: false } }, - }); - - testAdapter = new WorkerAdapter({ - event: mockEvent, - adapterState: mockAdapterState, - options: { - batchSize: 50, - }, - }); - }); - - it('should maintain artifact ordering when repo ItemTypeA has items below batch size and repo ItemTypeB has items above batch size', async () => { - const repos = [{ itemType: 'ItemTypeA' }, { itemType: 'ItemTypeB' }]; - - // Initialize repos - testAdapter.initializeRepos(repos); - - await testAdapter.getRepo('ItemTypeA')?.push(createItems(5)); - await testAdapter.getRepo('ItemTypeB')?.push(createItems(105)); - - await testAdapter.uploadAllRepos(); - - const artifacts = testAdapter.artifacts; - expect(artifacts.length).toBe(4); - - expect(checkArtifactOrder(artifacts, repos)).toBe(true); - }); - - it('should work with more than 2 repos', async () => { - const repos = [ - { itemType: 'ItemTypeA' }, - { itemType: 'ItemTypeB' }, - { itemType: 'ItemTypeC' }, - { itemType: 'ItemTypeD' }, - ]; - - // Initialize repos - testAdapter.initializeRepos(repos); - - await testAdapter.getRepo('ItemTypeA')?.push(createItems(101)); - await testAdapter.getRepo('ItemTypeB')?.push(createItems(102)); - await testAdapter.getRepo('ItemTypeC')?.push(createItems(103)); - await testAdapter.getRepo('ItemTypeD')?.push(createItems(104)); - - await testAdapter.uploadAllRepos(); - - const artifacts = testAdapter.artifacts; - expect(artifacts.length).toBe(12); - - expect(checkArtifactOrder(artifacts, repos)).toBe(true); - }); - - it('should maintain order with multiple pushes and uploads', async () => { - const repos = [{ itemType: 'ItemTypeA' }, { itemType: 'ItemTypeB' }]; - - // Initialize repos - testAdapter.initializeRepos(repos); - - await testAdapter.getRepo('ItemTypeA')?.push(createItems(101)); - await testAdapter.getRepo('ItemTypeB')?.push(createItems(102)); - await testAdapter.getRepo('ItemTypeA')?.push(createItems(101)); - await testAdapter.getRepo('ItemTypeB')?.push(createItems(102)); - await testAdapter.getRepo('ItemTypeA')?.upload(); - await testAdapter.getRepo('ItemTypeB')?.upload(); - await testAdapter.getRepo('ItemTypeA')?.push(createItems(101)); - await testAdapter.getRepo('ItemTypeB')?.push(createItems(102)); - await testAdapter.getRepo('ItemTypeA')?.push(createItems(101)); - await testAdapter.getRepo('ItemTypeB')?.push(createItems(102)); - - await testAdapter.uploadAllRepos(); - - const artifacts = testAdapter.artifacts; - expect(artifacts.length).toBe(20); - - expect(checkArtifactOrder(artifacts, repos)).toBe(true); - }); - - it('should not count artifacts if 0 items are pushed to the repo', async () => { - const repos = [{ itemType: 'ItemTypeA' }]; - - // Initialize repos - testAdapter.initializeRepos(repos); - - await testAdapter.getRepo('ItemTypeA')?.push([]); - - await testAdapter.uploadAllRepos(); - - const artifacts = testAdapter.artifacts; - expect(artifacts.length).toBe(0); - - expect(checkArtifactOrder(artifacts, repos)).toBe(true); - }); -}); - -describe('External sync units splitting into artifacts', () => { - let testAdapter: WorkerAdapter>; - - beforeEach(() => { - const mockEvent = createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.StartExtractingExternalSyncUnits }, - }); - const mockAdapterState = new State>({ - event: mockEvent, - initialState: {}, - }); - - testAdapter = new WorkerAdapter({ - event: mockEvent, - adapterState: mockAdapterState, - }); - }); - - it('should split 125k external sync units into 5 artifacts', async () => { - const BATCH_SIZE = 25_000; - const TOTAL_UNITS = 125_000; - - const externalSyncUnits: ExternalSyncUnit[] = Array.from( - { length: TOTAL_UNITS }, - (_, i) => ({ - id: String(i), - name: `Unit ${i}`, - description: `Description ${i}`, - }) - ); - - testAdapter.initializeRepos([ - { - itemType: AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS, - overridenOptions: { - batchSize: BATCH_SIZE, - skipConfirmation: true, - }, - }, - ]); - - const repo = testAdapter.getRepo( - AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS - ); - const chunkSize = 10_000; - for (let i = 0; i < TOTAL_UNITS; i += chunkSize) { - await repo?.push(externalSyncUnits.slice(i, i + chunkSize)); - } - - await testAdapter.uploadAllRepos(); - - expect(testAdapter.artifacts.length).toBe( - TOTAL_UNITS / BATCH_SIZE // 5 - ); - expect( - testAdapter.artifacts.every( - (a) => a.item_type === AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS - ) - ).toBe(true); - }); -}); diff --git a/src/multithreading/worker-adapter/worker-adapter.emit.test.ts b/src/multithreading/worker-adapter/worker-adapter.emit.test.ts deleted file mode 100644 index 2da61f73..00000000 --- a/src/multithreading/worker-adapter/worker-adapter.emit.test.ts +++ /dev/null @@ -1,711 +0,0 @@ -import { UNBOUNDED_DATE_TIME_VALUE } from '../../common/constants'; -import { State } from '../../state/state'; -import { mockServer } from '../../tests/jest.setup'; -import { createMockEvent } from '../../common/test-utils'; -import { - AdapterState, - AirdropEvent, - Artifact, - EventType, - ExtractorEventType, - LoaderEventType, -} from '../../types'; -import { ActionType, LoaderReport } from '../../types/loading'; -import { WorkerAdapter } from './worker-adapter'; - -/* eslint-disable @typescript-eslint/no-require-imports */ - -jest.mock('../../common/control-protocol', () => ({ - emit: jest.fn().mockResolvedValue({}), -})); - -jest.mock('../../mappers/mappers'); -jest.mock('../../uploader/uploader'); -jest.mock('../../repo/repo'); -jest.mock('node:worker_threads', () => ({ - parentPort: { postMessage: jest.fn() }, -})); -jest.mock('../../attachments-streaming/attachments-streaming-pool', () => ({ - AttachmentsStreamingPool: jest.fn().mockImplementation(() => ({ - streamAll: jest.fn().mockResolvedValue(undefined), - })), -})); - -interface TestState { - attachments: { completed: boolean }; -} - -function makeAdapter(eventType: EventType = EventType.StartExtractingData): { - adapter: WorkerAdapter; - event: AirdropEvent; - adapterState: State; -} { - const event = createMockEvent(mockServer.baseUrl, { - payload: { event_type: eventType }, - }); - const initialState: AdapterState = { - attachments: { completed: false }, - lastSyncStarted: '', - lastSuccessfulSyncStarted: '', - snapInVersionId: '', - toDevRev: { - attachmentsMetadata: { - artifactIds: [], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }, - }; - const adapterState = new State({ event, initialState }); - const adapter = new WorkerAdapter({ event, adapterState }); - return { adapter, event, adapterState }; -} - -const iso = (ms: number) => new Date(ms).toISOString(); - -describe(`${WorkerAdapter.name}.emit`, () => { - let adapter: WorkerAdapter; - let mockPostMessage: jest.Mock; - - beforeEach(() => { - jest.clearAllMocks(); - ({ adapter } = makeAdapter()); - - const workerThreads = require('node:worker_threads'); - mockPostMessage = jest.fn(); - if (workerThreads.parentPort) { - jest - .spyOn(workerThreads.parentPort, 'postMessage') - .mockImplementation(mockPostMessage); - } else { - workerThreads.parentPort = { postMessage: mockPostMessage }; - } - }); - - afterEach(() => { - jest.restoreAllMocks(); - }); - - it('should emit only one event when multiple events of same type are sent', async () => { - // Arrange - adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined); - adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined); - - // Act - await adapter.emit(ExtractorEventType.MetadataExtractionError, { - reports: [], - processed_files: [], - }); - await adapter.emit(ExtractorEventType.MetadataExtractionError, { - reports: [], - processed_files: [], - }); - - // Assert - expect(mockPostMessage).toHaveBeenCalledTimes(1); - }); - - it('should emit only once even when a different event type follows', async () => { - // Arrange - adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined); - adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined); - - // Act - await adapter.emit(ExtractorEventType.MetadataExtractionError, { - reports: [], - processed_files: [], - }); - await adapter.emit(ExtractorEventType.DataExtractionError, { - reports: [], - processed_files: [], - }); - await adapter.emit(ExtractorEventType.AttachmentExtractionError, { - reports: [], - processed_files: [], - }); - - // Assert - expect(mockPostMessage).toHaveBeenCalledTimes(1); - }); - - it('should correctly emit one event even if postState errors', async () => { - // Arrange - adapter['adapterState'].postState = jest - .fn() - .mockRejectedValue(new Error('postState error')); - adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined); - - // Act - await adapter.emit(ExtractorEventType.MetadataExtractionError, { - reports: [], - processed_files: [], - }); - - // Assert - expect(mockPostMessage).toHaveBeenCalledTimes(1); - }); - - it('should correctly emit one event even if uploadAllRepos errors', async () => { - // Arrange - adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined); - adapter.uploadAllRepos = jest - .fn() - .mockRejectedValue(new Error('uploadAllRepos error')); - - // Act - await adapter.emit(ExtractorEventType.MetadataExtractionError, { - reports: [], - processed_files: [], - }); - - // Assert - expect(mockPostMessage).toHaveBeenCalledTimes(1); - }); - - it('should include artifacts in data for extraction events', async () => { - // Arrange - const { emit: mockEmit } = require('../../common/control-protocol'); - adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined); - adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined); - adapter['_artifacts'] = [ - { id: 'art-1', item_count: 10, item_type: 'issues' }, - ] as Artifact[]; - - // Act - await adapter.emit(ExtractorEventType.DataExtractionDone); - - // Assert - expect(mockEmit).toHaveBeenCalledWith( - expect.objectContaining({ - data: expect.objectContaining({ - artifacts: expect.arrayContaining([ - expect.objectContaining({ id: 'art-1' }), - ]), - }), - }) - ); - const callData = mockEmit.mock.calls[0][0].data; - expect(callData).not.toHaveProperty('reports'); - expect(callData).not.toHaveProperty('processed_files'); - }); - - it('should include reports and processed_files in data for loader events', async () => { - // Arrange - const { emit: mockEmit } = require('../../common/control-protocol'); - adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined); - adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined); - adapter['loaderReports'] = [ - { item_type: 'tasks', [ActionType.CREATED]: 5 }, - ] as LoaderReport[]; - adapter['_processedFiles'] = ['file-1', 'file-2']; - - // Act - await adapter.emit(LoaderEventType.DataLoadingDone); - - // Assert - expect(mockEmit).toHaveBeenCalledWith( - expect.objectContaining({ - data: expect.objectContaining({ - reports: expect.arrayContaining([ - expect.objectContaining({ item_type: 'tasks' }), - ]), - processed_files: ['file-1', 'file-2'], - }), - }) - ); - const callData = mockEmit.mock.calls[0][0].data; - expect(callData).not.toHaveProperty('artifacts'); - }); - - it('should not include artifacts, reports, or processed_files for unknown event types', async () => { - // Arrange - const { emit: mockEmit } = require('../../common/control-protocol'); - adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined); - adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined); - adapter['_artifacts'] = [ - { id: 'art-1', item_count: 10, item_type: 'issues' }, - ] as Artifact[]; - adapter['loaderReports'] = [ - { item_type: 'tasks', [ActionType.CREATED]: 5 }, - ] as LoaderReport[]; - adapter['_processedFiles'] = ['file-1']; - - // Act - await adapter.emit('SOME_UNKNOWN_EVENT' as ExtractorEventType); - - // Assert - const callData = mockEmit.mock.calls[0][0].data; - expect(callData).not.toHaveProperty('artifacts'); - expect(callData).not.toHaveProperty('reports'); - expect(callData).not.toHaveProperty('processed_files'); - }); - - it('should include artifacts for all ExtractorEventType values', async () => { - // Arrange - const { emit: mockEmit } = require('../../common/control-protocol'); - const extractorEvents = [ - ExtractorEventType.DataExtractionDone, - ExtractorEventType.DataExtractionProgress, - ExtractorEventType.DataExtractionError, - ExtractorEventType.AttachmentExtractionDone, - ExtractorEventType.AttachmentExtractionProgress, - ]; - - for (const eventType of extractorEvents) { - jest.clearAllMocks(); - adapter.hasWorkerEmitted = false; - adapter['adapterState'].postState = jest - .fn() - .mockResolvedValue(undefined); - adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined); - - // Act - await adapter.emit(eventType); - - // Assert - const callData = mockEmit.mock.calls[0]?.[0]?.data; - expect(callData).toHaveProperty('artifacts'); - expect(callData).not.toHaveProperty('reports'); - } - }); - - it('should include reports and processed_files for all LoaderEventType values', async () => { - // Arrange - const { emit: mockEmit } = require('../../common/control-protocol'); - const loaderEvents = [ - LoaderEventType.DataLoadingDone, - LoaderEventType.DataLoadingProgress, - LoaderEventType.DataLoadingError, - LoaderEventType.AttachmentLoadingDone, - LoaderEventType.AttachmentLoadingProgress, - ]; - - for (const eventType of loaderEvents) { - jest.clearAllMocks(); - adapter.hasWorkerEmitted = false; - adapter['adapterState'].postState = jest - .fn() - .mockResolvedValue(undefined); - adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined); - - // Act - await adapter.emit(eventType); - - // Assert - const callData = mockEmit.mock.calls[0]?.[0]?.data; - expect(callData).toHaveProperty('reports'); - expect(callData).toHaveProperty('processed_files'); - expect(callData).not.toHaveProperty('artifacts'); - } - }); - - it('should truncate a long error message, preserving the original prefix', async () => { - // Arrange - adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined); - adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined); - const longMessage = 'E'.repeat(20_000); - - // Act - await adapter.emit(ExtractorEventType.DataExtractionError, { - error: { message: longMessage }, - }); - - // Assert - const { emit: mockEmit } = require('../../common/control-protocol'); - const emittedMessage = mockEmit.mock.calls[0][0].data?.error - ?.message as string; - expect(emittedMessage.length).toBeLessThan(longMessage.length); - expect(emittedMessage.startsWith('E'.repeat(100))).toBe(true); - }); -}); - -describe(`${WorkerAdapter.name}.emit — worker_metadata`, () => { - let adapter: WorkerAdapter; - - beforeEach(() => { - jest.clearAllMocks(); - ({ adapter } = makeAdapter()); - adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined); - adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined); - }); - - function setupRepos( - adapterInstance: WorkerAdapter, - lastExtractedItemType: string - ) { - adapterInstance['repos'] = [ - { - itemType: 'issues', - dateRanges: { - creationDate: { oldest: 100, newest: 200 }, - modifiedDate: { oldest: 150, newest: 250 }, - }, - }, - { - itemType: 'tasks', - dateRanges: { - creationDate: { oldest: 300, newest: 400 }, - modifiedDate: { oldest: 350, newest: 450 }, - }, - }, - ] as never; - adapterInstance['lastExtractedItemType'] = lastExtractedItemType; - } - - it('should emit flat worker_metadata for the latest extracted item type only', async () => { - setupRepos(adapter, 'tasks'); - - await adapter.emit(ExtractorEventType.DataExtractionProgress); - - const { emit: mockEmit } = require('../../common/control-protocol'); - expect(mockEmit.mock.calls[0][0].worker_metadata).toEqual({ - item_type: 'tasks', - oldest_created_date: iso(300), - newest_created_date: iso(400), - oldest_modified_date: iso(350), - newest_modified_date: iso(450), - }); - }); - - it('should omit unset RFC3339 bounds from worker_metadata', async () => { - adapter['repos'] = [ - { - itemType: 'tasks', - dateRanges: { - creationDate: {}, - modifiedDate: {}, - }, - }, - ] as never; - adapter['lastExtractedItemType'] = 'tasks'; - - await adapter.emit(ExtractorEventType.DataExtractionProgress); - - const { emit: mockEmit } = require('../../common/control-protocol'); - expect(mockEmit.mock.calls[0][0].worker_metadata).toEqual({ - item_type: 'tasks', - }); - }); - - it('should include Unix-epoch bounds rather than treating them as unset', async () => { - adapter['repos'] = [ - { - itemType: 'tasks', - dateRanges: { - creationDate: { oldest: 0, newest: 0 }, - modifiedDate: {}, - }, - }, - ] as never; - adapter['lastExtractedItemType'] = 'tasks'; - - await adapter.emit(ExtractorEventType.DataExtractionProgress); - - const { emit: mockEmit } = require('../../common/control-protocol'); - expect(mockEmit.mock.calls[0][0].worker_metadata).toEqual({ - item_type: 'tasks', - oldest_created_date: iso(0), - newest_created_date: iso(0), - }); - }); - - it('should send empty worker_metadata when no item type has been extracted', async () => { - await adapter.emit(ExtractorEventType.DataExtractionProgress); - - const { emit: mockEmit } = require('../../common/control-protocol'); - expect(mockEmit.mock.calls[0][0].worker_metadata).toEqual({}); - }); - - it('should send empty worker_metadata for loader events', async () => { - setupRepos(adapter, 'tasks'); - - await adapter.emit(LoaderEventType.DataLoadingProgress); - - const { emit: mockEmit } = require('../../common/control-protocol'); - expect(mockEmit.mock.calls[0][0].worker_metadata).toEqual({}); - }); - - it('should send empty worker_metadata for non-progress extraction events', async () => { - setupRepos(adapter, 'tasks'); - - await adapter.emit(ExtractorEventType.DataExtractionError); - - const { emit: mockEmit } = require('../../common/control-protocol'); - expect(mockEmit.mock.calls[0][0].worker_metadata).toEqual({}); - }); -}); - -describe(`${WorkerAdapter.name}.emit — ExternalSyncUnitExtractionDone legacy path`, () => { - it('should upload ESUs via a repo and strip external_sync_units from the emitted payload', async () => { - // Arrange - const { adapter } = makeAdapter(EventType.StartExtractingExternalSyncUnits); - adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined); - adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined); - const pushMock = jest.fn().mockResolvedValue(undefined); - jest.spyOn(adapter, 'initializeRepos'); - jest.spyOn(adapter, 'getRepo').mockReturnValue({ push: pushMock } as never); - const esus = [{ id: 'esu-1' }, { id: 'esu-2' }] as never; - - // Act - await adapter.emit(ExtractorEventType.ExternalSyncUnitExtractionDone, { - external_sync_units: esus, - }); - - // Assert - expect(pushMock).toHaveBeenCalledWith(esus); - // external_sync_units must NOT appear in the payload sent to the platform - // (it would be too large for SQS — that is the entire reason this path exists). - const { emit: mockEmit } = require('../../common/control-protocol'); - const emittedData = mockEmit.mock.calls[0][0].data as Record< - string, - unknown - >; - expect(emittedData).not.toHaveProperty('external_sync_units'); - }); -}); - -describe('WorkerAdapter — workersOldest / workersNewest boundary updates', () => { - let adapter: WorkerAdapter; - let mockPostMessage: jest.Mock; - - beforeEach(() => { - jest.clearAllMocks(); - ({ adapter } = makeAdapter()); - - const workerThreads = require('node:worker_threads'); - mockPostMessage = jest.fn(); - if (workerThreads.parentPort) { - jest - .spyOn(workerThreads.parentPort, 'postMessage') - .mockImplementation(mockPostMessage); - } else { - workerThreads.parentPort = { postMessage: mockPostMessage }; - } - - adapter['adapterState'].postState = jest.fn().mockResolvedValue(undefined); - adapter.uploadAllRepos = jest.fn().mockResolvedValue(undefined); - }); - - afterEach(() => { - jest.restoreAllMocks(); - }); - - async function emitDone( - adapterInstance: WorkerAdapter, - extractionStart: string | undefined, - extractionEnd: string | undefined - ) { - adapterInstance.event.payload.event_context.extract_from = extractionStart; - adapterInstance.event.payload.event_context.extract_to = extractionEnd; - // Reset the emit guard so we can emit multiple times within one test. - adapterInstance['hasWorkerEmitted'] = false; - - await adapterInstance.emit(ExtractorEventType.AttachmentExtractionDone, { - reports: [], - processed_files: [], - }); - } - - describe('initial import with UNBOUNDED start', () => { - it('should set workersOldest to UNBOUNDED_DATE_TIME_VALUE and workersNewest to extraction end', async () => { - await emitDone( - adapter, - UNBOUNDED_DATE_TIME_VALUE, - '2025-06-01T00:00:00.000Z' - ); - - expect(adapter.state.workersOldest).toBe(UNBOUNDED_DATE_TIME_VALUE); - expect(adapter.state.workersNewest).toBe('2025-06-01T00:00:00.000Z'); - }); - }); - - describe('reconciliation after UNBOUNDED initial import', () => { - it('should NOT overwrite workersOldest when reconciliation start is later than sentinel', async () => { - await emitDone( - adapter, - UNBOUNDED_DATE_TIME_VALUE, - '2025-06-01T00:00:00.000Z' - ); - - await emitDone( - adapter, - '2025-01-01T00:00:00.000Z', - '2025-03-01T00:00:00.000Z' - ); - - expect(adapter.state.workersOldest).toBe(UNBOUNDED_DATE_TIME_VALUE); - expect(adapter.state.workersNewest).toBe('2025-06-01T00:00:00.000Z'); - }); - - it('should NOT overwrite workersOldest even when reconciliation start is very early', async () => { - await emitDone( - adapter, - UNBOUNDED_DATE_TIME_VALUE, - '2025-06-01T00:00:00.000Z' - ); - - await emitDone( - adapter, - '1980-01-01T00:00:00.000Z', - '1990-01-01T00:00:00.000Z' - ); - - expect(adapter.state.workersOldest).toBe(UNBOUNDED_DATE_TIME_VALUE); - expect(adapter.state.workersNewest).toBe('2025-06-01T00:00:00.000Z'); - }); - }); - - describe('forward sync after UNBOUNDED initial import', () => { - it('should expand workersNewest forward while preserving workersOldest', async () => { - await emitDone( - adapter, - UNBOUNDED_DATE_TIME_VALUE, - '2025-06-01T00:00:00.000Z' - ); - - await emitDone( - adapter, - '2025-06-01T00:00:00.000Z', - '2025-07-01T00:00:00.000Z' - ); - - expect(adapter.state.workersOldest).toBe(UNBOUNDED_DATE_TIME_VALUE); - expect(adapter.state.workersNewest).toBe('2025-07-01T00:00:00.000Z'); - }); - }); - - describe('reconciliation with end beyond current newest', () => { - it('should expand workersNewest when reconciliation end is later', async () => { - await emitDone( - adapter, - UNBOUNDED_DATE_TIME_VALUE, - '2025-06-01T00:00:00.000Z' - ); - - await emitDone( - adapter, - '2024-01-01T00:00:00.000Z', - '2025-08-01T00:00:00.000Z' - ); - - expect(adapter.state.workersOldest).toBe(UNBOUNDED_DATE_TIME_VALUE); - expect(adapter.state.workersNewest).toBe('2025-08-01T00:00:00.000Z'); - }); - }); - - describe('first sync with absolute dates (no UNBOUNDED)', () => { - it('should set both boundaries from the extraction range', async () => { - await emitDone( - adapter, - '2025-01-01T00:00:00.000Z', - '2025-03-01T00:00:00.000Z' - ); - - expect(adapter.state.workersOldest).toBe('2025-01-01T00:00:00.000Z'); - expect(adapter.state.workersNewest).toBe('2025-03-01T00:00:00.000Z'); - }); - }); - - describe('reconciliation after absolute initial sync', () => { - it('should expand workersOldest backward when reconciliation start is earlier', async () => { - await emitDone( - adapter, - '2025-01-01T00:00:00.000Z', - '2025-03-01T00:00:00.000Z' - ); - - await emitDone( - adapter, - '2024-06-01T00:00:00.000Z', - '2025-02-01T00:00:00.000Z' - ); - - expect(adapter.state.workersOldest).toBe('2024-06-01T00:00:00.000Z'); - expect(adapter.state.workersNewest).toBe('2025-03-01T00:00:00.000Z'); - }); - - it('should NOT change boundaries when reconciliation is within existing range', async () => { - await emitDone( - adapter, - '2025-01-01T00:00:00.000Z', - '2025-03-01T00:00:00.000Z' - ); - - await emitDone( - adapter, - '2025-01-15T00:00:00.000Z', - '2025-02-15T00:00:00.000Z' - ); - - expect(adapter.state.workersOldest).toBe('2025-01-01T00:00:00.000Z'); - expect(adapter.state.workersNewest).toBe('2025-03-01T00:00:00.000Z'); - }); - - it('should expand both boundaries when reconciliation exceeds both', async () => { - await emitDone( - adapter, - '2025-01-01T00:00:00.000Z', - '2025-03-01T00:00:00.000Z' - ); - - await emitDone( - adapter, - '2024-06-01T00:00:00.000Z', - '2025-09-01T00:00:00.000Z' - ); - - expect(adapter.state.workersOldest).toBe('2024-06-01T00:00:00.000Z'); - expect(adapter.state.workersNewest).toBe('2025-09-01T00:00:00.000Z'); - }); - }); - - describe('multiple forward syncs', () => { - it('should progressively expand workersNewest while preserving workersOldest', async () => { - await emitDone( - adapter, - UNBOUNDED_DATE_TIME_VALUE, - '2025-06-01T00:00:00.000Z' - ); - - await emitDone( - adapter, - '2025-06-01T00:00:00.000Z', - '2025-07-01T00:00:00.000Z' - ); - expect(adapter.state.workersNewest).toBe('2025-07-01T00:00:00.000Z'); - - await emitDone( - adapter, - '2025-07-01T00:00:00.000Z', - '2025-08-01T00:00:00.000Z' - ); - expect(adapter.state.workersNewest).toBe('2025-08-01T00:00:00.000Z'); - - expect(adapter.state.workersOldest).toBe(UNBOUNDED_DATE_TIME_VALUE); - }); - }); - - describe('non-AttachmentExtractionDone events should NOT update boundaries', () => { - it.each([ - ExtractorEventType.DataExtractionDone, - ExtractorEventType.DataExtractionProgress, - ExtractorEventType.MetadataExtractionError, - ExtractorEventType.AttachmentExtractionError, - ])('should not update boundaries on %s', async (eventType) => { - adapter.state.workersOldest = '2025-01-01T00:00:00.000Z'; - adapter.state.workersNewest = '2025-03-01T00:00:00.000Z'; - adapter.event.payload.event_context.extract_from = - '2024-01-01T00:00:00.000Z'; - adapter.event.payload.event_context.extract_to = - '2025-12-01T00:00:00.000Z'; - - await adapter.emit(eventType, { - reports: [], - processed_files: [], - }); - - expect(adapter.state.workersOldest).toBe('2025-01-01T00:00:00.000Z'); - expect(adapter.state.workersNewest).toBe('2025-03-01T00:00:00.000Z'); - }); - }); -}); diff --git a/src/multithreading/worker-adapter/worker-adapter.extraction.test.ts b/src/multithreading/worker-adapter/worker-adapter.extraction.test.ts deleted file mode 100644 index e695f8e8..00000000 --- a/src/multithreading/worker-adapter/worker-adapter.extraction.test.ts +++ /dev/null @@ -1,1034 +0,0 @@ -import { AttachmentsStreamingPool } from '../../attachments-streaming/attachments-streaming-pool'; -import { State } from '../../state/state'; -import { mockServer } from '../../tests/jest.setup'; -import { createMockEvent } from '../../common/test-utils'; -import { - AdapterState, - AirdropEvent, - Artifact, - EventType, - ExtractorEventType, -} from '../../types'; -import { WorkerAdapter } from './worker-adapter'; - -/* eslint-disable @typescript-eslint/no-require-imports */ - -jest.mock('../../common/control-protocol', () => ({ - emit: jest.fn().mockResolvedValue({}), -})); - -jest.mock('../../mappers/mappers'); -jest.mock('../../uploader/uploader'); -jest.mock('../../repo/repo'); -jest.mock('node:worker_threads', () => ({ - parentPort: { postMessage: jest.fn() }, -})); -jest.mock('../../attachments-streaming/attachments-streaming-pool', () => ({ - AttachmentsStreamingPool: jest.fn().mockImplementation(() => ({ - streamAll: jest.fn().mockResolvedValue(undefined), - })), -})); - -interface TestState { - attachments: { completed: boolean }; -} - -function makeAdapter(eventType: EventType = EventType.StartExtractingData): { - adapter: WorkerAdapter; - event: AirdropEvent; - adapterState: State; -} { - const event = createMockEvent(mockServer.baseUrl, { - payload: { event_type: eventType }, - }); - const initialState: AdapterState = { - attachments: { completed: false }, - lastSyncStarted: '', - lastSuccessfulSyncStarted: '', - snapInVersionId: '', - toDevRev: { - attachmentsMetadata: { - artifactIds: [], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }, - }; - const adapterState = new State({ event, initialState }); - const adapter = new WorkerAdapter({ event, adapterState }); - return { adapter, event, adapterState }; -} - -describe(`${WorkerAdapter.name}.streamAttachments`, () => { - let adapter: WorkerAdapter; - - beforeEach(() => { - jest.clearAllMocks(); - ({ adapter } = makeAdapter()); - }); - - it('should process all artifact batches successfully', async () => { - // Arrange - const mockStream = jest.fn(); - - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: ['artifact1', 'artifact2'], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }; - - adapter['uploader'].getAttachmentsFromArtifactId = jest - .fn() - .mockResolvedValueOnce({ - attachments: [ - { - url: 'http://example.com/file1.pdf', - id: 'attachment1', - file_name: 'file1.pdf', - parent_id: 'parent1', - }, - { - url: 'http://example.com/file2.pdf', - id: 'attachment2', - file_name: 'file2.pdf', - parent_id: 'parent2', - }, - ], - }) - .mockResolvedValueOnce({ - attachments: [ - { - url: 'http://example.com/file3.pdf', - id: 'attachment3', - file_name: 'file3.pdf', - parent_id: 'parent3', - }, - ], - }); - - adapter.initializeRepos = jest.fn(); - - // Act - const result = await adapter.streamAttachments({ - stream: mockStream, - }); - - // Assert - expect(adapter.initializeRepos).toHaveBeenCalledWith([ - { itemType: 'ssor_attachment' }, - ]); - expect(adapter.initializeRepos).toHaveBeenCalledTimes(1); - expect( - adapter['uploader'].getAttachmentsFromArtifactId - ).toHaveBeenCalledTimes(2); - - expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([]); - expect(adapter.state.toDevRev.attachmentsMetadata.lastProcessed).toBe(0); - expect(result).toBeUndefined(); - }); - - it('[edge] should handle invalid batch size by using 1 instead', async () => { - // Arrange - const mockStream = jest.fn(); - - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: ['artifact1'], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }; - - adapter['uploader'].getAttachmentsFromArtifactId = jest - .fn() - .mockResolvedValue({ - attachments: [ - { - url: 'http://example.com/file1.pdf', - id: 'attachment1', - file_name: 'file1.pdf', - parent_id: 'parent1', - }, - ], - }); - - adapter.initializeRepos = jest.fn(); - - // Act - const result = await adapter.streamAttachments({ - stream: mockStream, - batchSize: 0, - }); - - // Assert - expect(result).toBeUndefined(); - }); - - it('[edge] should cap batch size to 50 when batchSize is greater than 50', async () => { - // Arrange - const mockStream = jest.fn(); - - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: ['artifact1'], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }; - - adapter['uploader'].getAttachmentsFromArtifactId = jest - .fn() - .mockResolvedValue({ - attachments: [ - { - url: 'http://example.com/file1.pdf', - id: 'attachment1', - file_name: 'file1.pdf', - parent_id: 'parent1', - }, - ], - }); - - adapter.initializeRepos = jest.fn(); - - // Act - const result = await adapter.streamAttachments({ - stream: mockStream, - batchSize: 100, - }); - - // Assert - expect(result).toBeUndefined(); - }); - - it('[edge] should handle empty attachments metadata artifact IDs', async () => { - // Arrange - const mockStream = jest.fn(); - - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: [], - lastProcessed: 0, - }, - }; - - // Act - const result = await adapter.streamAttachments({ - stream: mockStream, - }); - - // Assert - expect(result).toBeUndefined(); - }); - - it('[edge] should handle errors when getting attachments', async () => { - // Arrange - const mockStream = jest.fn(); - - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: ['artifact1'], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }; - - const mockError = new Error('Failed to get attachments'); - adapter['uploader'].getAttachmentsFromArtifactId = jest - .fn() - .mockResolvedValue({ - error: mockError, - }); - - adapter.initializeRepos = jest.fn(); - - // Act - const result = await adapter.streamAttachments({ - stream: mockStream, - }); - - // Assert - expect(result).toEqual({ - error: mockError, - }); - }); - - it('[edge] should handle empty attachments array from artifact', async () => { - // Arrange - const mockStream = jest.fn(); - - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: ['artifact1'], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }; - - adapter['uploader'].getAttachmentsFromArtifactId = jest - .fn() - .mockResolvedValue({ - attachments: [], - }); - - adapter.initializeRepos = jest.fn(); - - // Act - const result = await adapter.streamAttachments({ - stream: mockStream, - }); - - // Assert - expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([]); - expect(result).toBeUndefined(); - }); - - it('should use custom processors when provided', async () => { - // Arrange - const mockStream = jest.fn(); - const mockReducer = jest.fn().mockReturnValue(['custom-reduced']); - const mockIterator = jest.fn().mockResolvedValue({}); - - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: ['artifact1'], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }; - - adapter['uploader'].getAttachmentsFromArtifactId = jest - .fn() - .mockResolvedValue({ - attachments: [{ id: 'attachment1' }], - }); - - adapter.initializeRepos = jest.fn(); - - // Act - const result = await adapter.streamAttachments({ - stream: mockStream, - processors: { - reducer: mockReducer, - iterator: mockIterator, - }, - }); - - // Assert - expect(mockReducer).toHaveBeenCalledWith({ - attachments: [{ id: 'attachment1' }], - adapter: adapter, - batchSize: 1, - }); - expect(mockIterator).toHaveBeenCalledWith({ - reducedAttachments: ['custom-reduced'], - adapter: adapter, - stream: mockStream, - }); - expect(result).toBeUndefined(); - }); - - it('should handle rate limiting from iterator', async () => { - // Arrange - const mockStream = jest.fn(); - - (AttachmentsStreamingPool as jest.Mock).mockImplementationOnce(() => ({ - streamAll: jest.fn().mockResolvedValue({ delay: 30 }), - })); - - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: ['artifact1'], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }; - - adapter['uploader'].getAttachmentsFromArtifactId = jest - .fn() - .mockResolvedValue({ - attachments: [{ id: 'attachment1' }], - }); - - adapter.initializeRepos = jest.fn(); - - // Act - const result = await adapter.streamAttachments({ - stream: mockStream, - }); - - // Assert - expect(result).toEqual({ delay: 30 }); - expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([ - 'artifact1', - ]); - }); - - it('should handle error from iterator', async () => { - // Arrange - const mockStream = jest.fn(); - - (AttachmentsStreamingPool as jest.Mock).mockImplementationOnce(() => ({ - streamAll: jest.fn().mockResolvedValue({ - error: 'Mock error', - }), - })); - - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: ['artifact1'], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }; - - adapter['uploader'].getAttachmentsFromArtifactId = jest - .fn() - .mockResolvedValue({ - attachments: [{ id: 'attachment1' }], - }); - - adapter.initializeRepos = jest.fn(); - - // Act - const result = await adapter.streamAttachments({ - stream: mockStream, - }); - - // Assert - expect(result).toEqual({ error: 'Mock error' }); - expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([ - 'artifact1', - ]); - }); - - it('should emit progress event and exit process on timeout, preserving state for resumption', async () => { - // Arrange - const mockStream = jest.fn(); - - const exitSpy = jest - .spyOn(process, 'exit') - .mockImplementation(() => undefined as never); - - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: ['artifact1', 'artifact2', 'artifact3'], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }; - - adapter['uploader'].getAttachmentsFromArtifactId = jest - .fn() - .mockResolvedValue({ - attachments: [ - { - url: 'http://example.com/file1.pdf', - id: 'attachment1', - file_name: 'file1.pdf', - parent_id: 'parent1', - }, - ], - }); - - (AttachmentsStreamingPool as jest.Mock).mockImplementationOnce(() => ({ - streamAll: jest.fn().mockImplementation(() => { - adapter.isTimeout = true; - return {}; - }), - })); - - adapter.initializeRepos = jest.fn(); - - const emitSpy = jest.spyOn(adapter, 'emit').mockResolvedValue(); - - // Act - await adapter.streamAttachments({ - stream: mockStream, - }); - - // Assert - expect(emitSpy).toHaveBeenCalledWith( - ExtractorEventType.AttachmentExtractionProgress - ); - expect(exitSpy).toHaveBeenCalledWith(0); - expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([ - 'artifact1', - 'artifact2', - 'artifact3', - ]); - expect( - adapter['uploader'].getAttachmentsFromArtifactId - ).toHaveBeenCalledTimes(1); - - exitSpy.mockRestore(); - }); - - it('should stop after the timeout flips between batches and preserve unprocessed artifacts for resumption', async () => { - // Arrange: three artifacts. The first batch's streamAll completes - // successfully; the second sets isTimeout=true mid-run. The third batch - // must never be reached. - const mockStream = jest.fn(); - const exitSpy = jest - .spyOn(process, 'exit') - .mockImplementation(() => undefined as never); - - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: ['artifact1', 'artifact2', 'artifact3'], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }; - - adapter['uploader'].getAttachmentsFromArtifactId = jest - .fn() - .mockResolvedValue({ - attachments: [ - { - url: 'http://example.com/file.pdf', - id: 'attachment-x', - file_name: 'file.pdf', - parent_id: 'parent-x', - }, - ], - }); - - // First call: clean streamAll. Second call: flip isTimeout AFTER streaming. - (AttachmentsStreamingPool as jest.Mock) - .mockImplementationOnce(() => ({ - streamAll: jest.fn().mockResolvedValue({}), - })) - .mockImplementationOnce(() => ({ - streamAll: jest.fn().mockImplementation(() => { - adapter.isTimeout = true; - return {}; - }), - })); - - adapter.initializeRepos = jest.fn(); - const emitSpy = jest.spyOn(adapter, 'emit').mockResolvedValue(); - - // Act - await adapter.streamAttachments({ stream: mockStream }); - - // Assert - // - Fetched attachments for the first two artifacts only; the third never ran - expect( - adapter['uploader'].getAttachmentsFromArtifactId - ).toHaveBeenCalledTimes(2); - // - Progress emitted and process.exit(0) called once the timeout was detected - expect(emitSpy).toHaveBeenCalledWith( - ExtractorEventType.AttachmentExtractionProgress - ); - expect(exitSpy).toHaveBeenCalledWith(0); - // - Artifact 1 was shifted out cleanly; artifact 2 remains (timeout caught - // before its shift) along with the untouched artifact 3 - expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toEqual([ - 'artifact2', - 'artifact3', - ]); - - exitSpy.mockRestore(); - }); - - it('should reset lastProcessed and attachment IDs list after processing all artifacts', async () => { - // Arrange - const mockStream = jest.fn(); - adapter.state.toDevRev = { - attachmentsMetadata: { - artifactIds: ['artifact1'], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }; - adapter['uploader'].getAttachmentsFromArtifactId = jest - .fn() - .mockResolvedValueOnce({ - attachments: [ - { - url: 'http://example.com/file1.pdf', - id: 'attachment1', - file_name: 'file1.pdf', - parent_id: 'parent1', - }, - { - url: 'http://example.com/file2.pdf', - id: 'attachment2', - file_name: 'file2.pdf', - parent_id: 'parent2', - }, - { - url: 'http://example.com/file3.pdf', - id: 'attachment3', - file_name: 'file3.pdf', - parent_id: 'parent3', - }, - ], - }); - - adapter.processAttachment = jest.fn().mockResolvedValue(null); - - // Act - await adapter.streamAttachments({ - stream: mockStream, - }); - - // Assert - expect(adapter.state.toDevRev.attachmentsMetadata.artifactIds).toHaveLength( - 0 - ); - expect(adapter.state.toDevRev.attachmentsMetadata.lastProcessed).toBe(0); - }); -}); - -describe(`${WorkerAdapter.name}.processAttachment`, () => { - let adapter: WorkerAdapter; - - beforeEach(() => { - jest.clearAllMocks(); - ({ adapter } = makeAdapter(EventType.StartExtractingAttachments)); - }); - - afterEach(() => { - jest.restoreAllMocks(); - }); - - const createMockHttpStream = (headers: Record = {}) => ({ - headers, - data: { destroy: jest.fn() }, - }); - - const makeAttachment = (overrides = {}) => ({ - id: 'att-1', - url: 'https://example.com/file.pdf', - file_name: 'file.pdf', - parent_id: 'parent-1', - content_type: 'application/pdf', - ...overrides, - }); - - function setupUploaderHappyPath() { - adapter['uploader'].getArtifactUploadUrl = jest.fn().mockResolvedValue({ - response: { - artifact_id: 'art_1', - upload_url: 'https://upload', - form_data: [], - }, - }); - adapter['uploader'].streamArtifact = jest - .fn() - .mockResolvedValue({ response: {} }); - adapter['uploader'].confirmArtifactUpload = jest - .fn() - .mockResolvedValue({ response: {} }); - - const pushMock = jest.fn().mockResolvedValue(undefined); - adapter.getRepo = jest.fn().mockReturnValue({ push: pushMock }); - return pushMock; - } - - // ---- content-type resolution ---- - it('should use attachment.content_type when provided, ignoring HTTP header', async () => { - // Arrange - setupUploaderHappyPath(); - const mockStream = jest.fn().mockResolvedValue({ - httpStream: createMockHttpStream({ - 'content-type': 'text/plain', - 'content-length': '100', - }), - }); - - // Act - await adapter.processAttachment( - makeAttachment({ content_type: 'application/pdf' }) as never, - mockStream - ); - - // Assert - expect(adapter['uploader'].getArtifactUploadUrl).toHaveBeenCalledWith( - 'file.pdf', - 'application/pdf', - 100 - ); - }); - - it('should use HTTP header content-type when attachment.content_type is not set', async () => { - // Arrange - setupUploaderHappyPath(); - const mockStream = jest.fn().mockResolvedValue({ - httpStream: createMockHttpStream({ - 'content-type': 'image/jpeg', - 'content-length': '200', - }), - }); - - const attachment = { - id: 'att-2', - url: 'https://example.com/photo.jpg', - file_name: 'photo.jpg', - parent_id: 'parent-2', - }; - - // Act - await adapter.processAttachment(attachment as never, mockStream); - - // Assert - expect(adapter['uploader'].getArtifactUploadUrl).toHaveBeenCalledWith( - 'photo.jpg', - 'image/jpeg', - 200 - ); - }); - - it('should fall back to application/octet-stream when neither content_type nor HTTP header is set', async () => { - // Arrange - setupUploaderHappyPath(); - const mockStream = jest.fn().mockResolvedValue({ - httpStream: createMockHttpStream({}), - }); - - const attachment = { - id: 'att-3', - url: 'https://example.com/file.bin', - file_name: 'file.bin', - parent_id: 'parent-3', - }; - - // Act - await adapter.processAttachment(attachment as never, mockStream); - - // Assert - expect(adapter['uploader'].getArtifactUploadUrl).toHaveBeenCalledWith( - 'file.bin', - 'application/octet-stream', - undefined - ); - }); - - // ---- error paths ---- - it('should return the stream error message when the stream function returns an error', async () => { - // Arrange - const stream = jest - .fn() - .mockResolvedValue({ error: new Error('stream failed') }); - - // Act - const result = await adapter.processAttachment( - makeAttachment() as never, - stream - ); - - // Assert - expect(result?.error?.message).toBe('stream failed'); - }); - - it('should propagate a rate-limit delay from the stream function', async () => { - // Arrange - const stream = jest.fn().mockResolvedValue({ delay: 5 }); - - // Act - const result = await adapter.processAttachment( - makeAttachment() as never, - stream - ); - - // Assert - expect(result?.delay).toBe(5); - }); - - it('should return an error containing the attachment ID when getArtifactUploadUrl fails', async () => { - // Arrange - const stream = jest - .fn() - .mockResolvedValue({ httpStream: createMockHttpStream() }); - adapter['uploader'].getArtifactUploadUrl = jest - .fn() - .mockResolvedValue({ error: new Error('upload url failed') }); - - // Act - const result = await adapter.processAttachment( - makeAttachment() as never, - stream - ); - - // Assert - expect(result?.error?.message).toContain('att-1'); - expect(result?.error?.message).toContain('preparing artifact'); - }); - - it('should return an error when streamArtifact fails', async () => { - // Arrange - const stream = jest - .fn() - .mockResolvedValue({ httpStream: createMockHttpStream() }); - adapter['uploader'].getArtifactUploadUrl = jest.fn().mockResolvedValue({ - response: { - artifact_id: 'art-1', - upload_url: 'https://upload', - form_data: [], - }, - }); - adapter['uploader'].streamArtifact = jest - .fn() - .mockResolvedValue({ error: new Error('stream failed') }); - - // Act - const result = await adapter.processAttachment( - makeAttachment() as never, - stream - ); - - // Assert - expect(result?.error?.message).toContain('streaming to artifact'); - }); - - it('should return an error when confirmArtifactUpload fails', async () => { - // Arrange - const stream = jest - .fn() - .mockResolvedValue({ httpStream: createMockHttpStream() }); - adapter['uploader'].getArtifactUploadUrl = jest.fn().mockResolvedValue({ - response: { - artifact_id: 'art-1', - upload_url: 'https://upload', - form_data: [], - }, - }); - adapter['uploader'].streamArtifact = jest - .fn() - .mockResolvedValue({ response: {} }); - adapter['uploader'].confirmArtifactUpload = jest - .fn() - .mockResolvedValue({ error: new Error('confirm failed') }); - - // Act - const result = await adapter.processAttachment( - makeAttachment() as never, - stream - ); - - // Assert - expect(result?.error?.message).toContain('confirming upload'); - }); - - it.each([ - { inline: true, expected: true }, - { inline: false, expected: false }, - ])( - 'should set inline=$expected on the ssorAttachment when attachment.inline=$inline', - async ({ inline, expected }) => { - // Arrange - const pushMock = setupUploaderHappyPath(); - const stream = jest - .fn() - .mockResolvedValue({ httpStream: createMockHttpStream() }); - - // Act - await adapter.processAttachment( - makeAttachment({ inline }) as never, - stream - ); - - // Assert - const ssorItem = pushMock.mock.calls[0][0][0] as Record; - expect(ssorItem.inline).toBe(expected); - } - ); - - it('should return a descriptive error when the stream function returns no httpStream', async () => { - // Arrange - const stream = jest.fn().mockResolvedValue({ httpStream: null }); - - // Act - const result = await adapter.processAttachment( - makeAttachment() as never, - stream - ); - - // Assert - expect(result?.error?.message).toContain( - 'Error while opening attachment stream' - ); - }); -}); - -describe(`${WorkerAdapter.name}.initializeRepos — event size threshold`, () => { - it('should set isTimeout=true once the cumulative artifact payload exceeds EVENT_SIZE_THRESHOLD_BYTES', () => { - // Arrange - const { adapter } = makeAdapter(); - - let capturedOnUpload: ((artifact: Artifact) => void) | undefined; - const { Repo } = require('../../repo/repo'); - (Repo as jest.Mock).mockImplementationOnce( - (opts: { onUpload: (a: Artifact) => void }) => { - capturedOnUpload = opts.onUpload; - return { itemType: 'issues', upload: jest.fn(), uploadedArtifacts: [] }; - } - ); - - // Act - adapter.initializeRepos([{ itemType: 'issues' }]); - expect(capturedOnUpload).toBeDefined(); - capturedOnUpload!({ - id: 'artifact-x', - item_count: 1, - item_type: 'x'.repeat(200_000), - }); - - // Assert - expect(adapter.isTimeout).toBe(true); - }); -}); - -describe(`${WorkerAdapter.name}.getRepo`, () => { - it('should return undefined when the requested repo was never initialised', () => { - // Arrange - const { adapter } = makeAdapter(); - - // Act - const result = adapter.getRepo('non-existent-type'); - - // Assert - expect(result).toBeUndefined(); - }); -}); - -describe(`${WorkerAdapter.name}.destroyHttpStream`, () => { - let adapter: WorkerAdapter; - - beforeEach(() => { - ({ adapter } = makeAdapter()); - }); - - afterEach(() => { - jest.restoreAllMocks(); - }); - - it.each([ - { - label: 'calls destroy() when available', - data: { destroy: jest.fn(), close: jest.fn() }, - expectDestroy: true, - expectClose: false, - }, - { - label: 'calls close() when destroy is not present', - data: { close: jest.fn() }, - expectDestroy: false, - expectClose: true, - }, - { - label: 'does not throw when neither method is present', - data: {}, - expectDestroy: false, - expectClose: false, - }, - { - label: 'does not throw when data is null', - data: null, - expectDestroy: false, - expectClose: false, - }, - ])('$label', ({ data, expectDestroy, expectClose }) => { - // Arrange - const httpStream = { data } as never; - - // Act & Assert - expect(() => adapter['destroyHttpStream'](httpStream)).not.toThrow(); - - if (expectDestroy) { - expect((data as { destroy: jest.Mock }).destroy).toHaveBeenCalled(); - } - if (expectClose) { - expect((data as { close: jest.Mock }).close).toHaveBeenCalled(); - } - }); - - it('should not re-throw when destroy() itself throws', () => { - // Arrange - const httpStream = { - data: { - destroy: () => { - throw new Error('stream error'); - }, - }, - }; - - // Act & Assert - expect(() => - adapter['destroyHttpStream'](httpStream as never) - ).not.toThrow(); - }); -}); - -describe(`${WorkerAdapter.name} — extractionScope`, () => { - it('should return empty object by default', () => { - const { adapter } = makeAdapter(); - expect(adapter.extractionScope).toEqual({}); - }); - - it('should return extraction scope from adapter state', () => { - const { adapter, adapterState } = makeAdapter(); - const extractionScope = { - tasks: { extract: true }, - users: { extract: false }, - }; - - ( - adapterState as unknown as { - _extractionScope: Record; - } - )._extractionScope = extractionScope; - - expect(adapter.extractionScope).toEqual(extractionScope); - }); -}); - -describe(`${WorkerAdapter.name} — shouldExtract`, () => { - it('should return true when extraction scope is empty', () => { - const { adapter } = makeAdapter(); - expect(adapter.shouldExtract('tasks')).toBe(true); - expect(adapter.shouldExtract('users')).toBe(true); - }); - - it('should return true when item type is not in scope', () => { - const { adapter, adapterState } = makeAdapter(); - ( - adapterState as unknown as { - _extractionScope: Record; - } - )._extractionScope = { - tasks: { extract: true }, - }; - expect(adapter.shouldExtract('users')).toBe(true); - }); - - it('should return true when item type has extract: true', () => { - const { adapter, adapterState } = makeAdapter(); - ( - adapterState as unknown as { - _extractionScope: Record; - } - )._extractionScope = { - tasks: { extract: true }, - }; - expect(adapter.shouldExtract('tasks')).toBe(true); - }); - - it('should return false when item type has extract: false', () => { - const { adapter, adapterState } = makeAdapter(); - ( - adapterState as unknown as { - _extractionScope: Record; - } - )._extractionScope = { - tasks: { extract: false }, - users: { extract: true }, - }; - expect(adapter.shouldExtract('tasks')).toBe(false); - expect(adapter.shouldExtract('users')).toBe(true); - }); -}); diff --git a/src/multithreading/worker-adapter/worker-adapter.helpers.test.ts b/src/multithreading/worker-adapter/worker-adapter.helpers.test.ts deleted file mode 100644 index 74e8ffd8..00000000 --- a/src/multithreading/worker-adapter/worker-adapter.helpers.test.ts +++ /dev/null @@ -1,635 +0,0 @@ -import { - ActionType, - ItemTypeToLoad, - LoaderReport, - StatsFileObject, -} from '../../types/loading'; - -import { - addReportToLoaderReport, - getFilesToLoad, - toRfc3339Timestamp, -} from './worker-adapter.helpers'; - -describe(getFilesToLoad.name, () => { - let statsFile: StatsFileObject[]; - - beforeEach(() => { - statsFile = [ - { - id: 'test-artifact-1', - file_name: 'test_file_1.json.gz', - item_type: 'issues', - count: '79', - }, - { - id: 'test-artifact-2', - file_name: 'test_file_2.json.gz', - item_type: 'comments', - count: '1079', - }, - { - id: 'test-artifact-3', - file_name: 'test_file_3.json.gz', - item_type: 'issues', - count: '1921', - }, - { - id: 'test-artifact-4', - file_name: 'test_file_4.json.gz', - item_type: 'comments', - count: '921', - }, - { - id: 'test-artifact-5', - file_name: 'test_file_5.json.gz', - item_type: 'attachments', - count: '50', - }, - { - id: 'test-artifact-6', - file_name: 'test_file_6.json.gz', - item_type: 'unknown', - count: '50', - }, - { - id: 'test-artifact-7', - file_name: 'test_file_7.json.gz', - item_type: 'issues', - count: '32', - }, - ]; - }); - - it('should filter files by supported item types and order them correctly', () => { - // Arrange - const itemTypesToLoad: ItemTypeToLoad[] = [ - { itemType: 'attachments', create: jest.fn(), update: jest.fn() }, - { itemType: 'issues', create: jest.fn(), update: jest.fn() }, - ]; - const expectedResult = [ - { - id: 'test-artifact-5', - itemType: 'attachments', - count: 50, - file_name: 'test_file_5.json.gz', - completed: false, - lineToProcess: 0, - }, - { - id: 'test-artifact-1', - itemType: 'issues', - count: 79, - file_name: 'test_file_1.json.gz', - completed: false, - lineToProcess: 0, - }, - { - id: 'test-artifact-3', - itemType: 'issues', - count: 1921, - file_name: 'test_file_3.json.gz', - completed: false, - lineToProcess: 0, - }, - { - id: 'test-artifact-7', - itemType: 'issues', - count: 32, - file_name: 'test_file_7.json.gz', - completed: false, - lineToProcess: 0, - }, - ]; - - // Act - const result = getFilesToLoad({ - supportedItemTypes: itemTypesToLoad.map((it) => it.itemType), - statsFile, - }); - - // Assert - expect(result).toEqual(expectedResult); - }); - - it('should ignore files with unrecognized item types in statsFile', () => { - // Arrange - const itemTypesToLoad: ItemTypeToLoad[] = [ - { itemType: 'issues', create: jest.fn(), update: jest.fn() }, - { itemType: 'unrecognized', create: jest.fn(), update: jest.fn() }, - ]; - const expectedResult = [ - { - id: 'test-artifact-1', - itemType: 'issues', - count: 79, - file_name: 'test_file_1.json.gz', - completed: false, - lineToProcess: 0, - }, - { - id: 'test-artifact-3', - itemType: 'issues', - count: 1921, - file_name: 'test_file_3.json.gz', - completed: false, - lineToProcess: 0, - }, - { - id: 'test-artifact-7', - itemType: 'issues', - count: 32, - file_name: 'test_file_7.json.gz', - completed: false, - lineToProcess: 0, - }, - ]; - - // Act - const result = getFilesToLoad({ - supportedItemTypes: itemTypesToLoad.map((it) => it.itemType), - statsFile, - }); - - // Assert - expect(result).toEqual(expectedResult); - }); - - it('should parse count string to number', () => { - // Arrange - const singleItemStatsFile: StatsFileObject[] = [ - { - id: 'test-artifact-single', - file_name: 'test_file_single.json.gz', - item_type: 'issues', - count: '12345', - }, - ]; - const supportedItemTypes = ['issues']; - - // Act - const result = getFilesToLoad({ - supportedItemTypes, - statsFile: singleItemStatsFile, - }); - - // Assert - expect(result[0].count).toBe(12345); - expect(typeof result[0].count).toBe('number'); - }); - - it('should initialize completed as false and lineToProcess as 0', () => { - // Arrange - const singleItemStatsFile: StatsFileObject[] = [ - { - id: 'test-artifact-init', - file_name: 'test_file_init.json.gz', - item_type: 'issues', - count: '100', - }, - ]; - const supportedItemTypes = ['issues']; - - // Act - const result = getFilesToLoad({ - supportedItemTypes, - statsFile: singleItemStatsFile, - }); - - // Assert - expect(result[0].completed).toBe(false); - expect(result[0].lineToProcess).toBe(0); - }); - - it('[edge] should return an empty array when statsFile is empty', () => { - // Arrange - const emptyStatsFile: StatsFileObject[] = []; - const itemTypesToLoad: ItemTypeToLoad[] = [ - { itemType: 'issues', create: jest.fn(), update: jest.fn() }, - ]; - - // Act - const result = getFilesToLoad({ - supportedItemTypes: itemTypesToLoad.map((it) => it.itemType), - statsFile: emptyStatsFile, - }); - - // Assert - expect(result).toEqual([]); - }); - - it('[edge] should return an empty array when supportedItemTypes is empty', () => { - // Arrange - const supportedItemTypes: string[] = []; - - // Act - const result = getFilesToLoad({ - supportedItemTypes, - statsFile, - }); - - // Assert - expect(result).toEqual([]); - }); - - it('[edge] should return an empty array when statsFile has no matching items', () => { - // Arrange - const itemTypesToLoad: ItemTypeToLoad[] = [ - { itemType: 'users', create: jest.fn(), update: jest.fn() }, - ]; - - // Act - const result = getFilesToLoad({ - supportedItemTypes: itemTypesToLoad.map((it) => it.itemType), - statsFile, - }); - - // Assert - expect(result).toEqual([]); - }); - - it('[edge] should return an empty array when both statsFile and supportedItemTypes are empty', () => { - // Arrange - const emptyStatsFile: StatsFileObject[] = []; - const supportedItemTypes: string[] = []; - - // Act - const result = getFilesToLoad({ - supportedItemTypes, - statsFile: emptyStatsFile, - }); - - // Assert - expect(result).toEqual([]); - }); -}); - -describe(addReportToLoaderReport.name, () => { - it('should add a new report when no existing report for the item type', () => { - // Arrange - const loaderReports: LoaderReport[] = []; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.CREATED]: 10, - [ActionType.UPDATED]: 5, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0]).toEqual({ - item_type: 'issues', - [ActionType.CREATED]: 10, - [ActionType.UPDATED]: 5, - }); - }); - - it('should merge created counts when report for item type already exists', () => { - // Arrange - const loaderReports: LoaderReport[] = [ - { - item_type: 'issues', - [ActionType.CREATED]: 10, - }, - ]; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.CREATED]: 5, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0][ActionType.CREATED]).toBe(15); - }); - - it('should merge updated counts when report for item type already exists', () => { - // Arrange - const loaderReports: LoaderReport[] = [ - { - item_type: 'issues', - [ActionType.UPDATED]: 20, - }, - ]; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.UPDATED]: 8, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0][ActionType.UPDATED]).toBe(28); - }); - - it('should merge failed counts when report for item type already exists', () => { - // Arrange - const loaderReports: LoaderReport[] = [ - { - item_type: 'issues', - [ActionType.FAILED]: 3, - }, - ]; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.FAILED]: 2, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0][ActionType.FAILED]).toBe(5); - }); - - it('should merge all action types when report for item type already exists', () => { - // Arrange - const loaderReports: LoaderReport[] = [ - { - item_type: 'issues', - [ActionType.CREATED]: 10, - [ActionType.UPDATED]: 20, - [ActionType.FAILED]: 3, - }, - ]; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.CREATED]: 5, - [ActionType.UPDATED]: 8, - [ActionType.FAILED]: 2, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0]).toEqual({ - item_type: 'issues', - [ActionType.CREATED]: 15, - [ActionType.UPDATED]: 28, - [ActionType.FAILED]: 5, - }); - }); - - it('should add reports for different item types separately', () => { - // Arrange - const loaderReports: LoaderReport[] = [ - { - item_type: 'issues', - [ActionType.CREATED]: 10, - }, - ]; - const report: LoaderReport = { - item_type: 'comments', - [ActionType.CREATED]: 50, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(2); - expect(result[0]).toEqual({ - item_type: 'issues', - [ActionType.CREATED]: 10, - }); - expect(result[1]).toEqual({ - item_type: 'comments', - [ActionType.CREATED]: 50, - }); - }); - - it('should preserve existing count when new report has undefined for an action type', () => { - // Arrange - const loaderReports: LoaderReport[] = [ - { - item_type: 'issues', - [ActionType.CREATED]: 10, - [ActionType.UPDATED]: 5, - }, - ]; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.CREATED]: 3, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0][ActionType.CREATED]).toBe(13); - expect(result[0][ActionType.UPDATED]).toBe(5); - }); - - it('should use new report count when existing report has undefined for an action type', () => { - // Arrange - const loaderReports: LoaderReport[] = [ - { - item_type: 'issues', - [ActionType.CREATED]: 10, - }, - ]; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.CREATED]: 3, - [ActionType.UPDATED]: 7, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0][ActionType.CREATED]).toBe(13); - expect(result[0][ActionType.UPDATED]).toBe(7); - }); - - it('should mutate and return the same loaderReports array', () => { - // Arrange - const loaderReports: LoaderReport[] = []; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.CREATED]: 10, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toBe(loaderReports); - }); - - it('[edge] should handle report with only item_type and no action counts', () => { - // Arrange - const loaderReports: LoaderReport[] = []; - const report: LoaderReport = { - item_type: 'issues', - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0]).toEqual({ item_type: 'issues' }); - }); - - it('[edge] should handle merging when both reports have zero counts', () => { - // Arrange - const loaderReports: LoaderReport[] = [ - { - item_type: 'issues', - [ActionType.CREATED]: 0, - }, - ]; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.CREATED]: 0, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0][ActionType.CREATED]).toBe(0); - }); - - it('[edge] should handle empty loaderReports array', () => { - // Arrange - const loaderReports: LoaderReport[] = []; - const report: LoaderReport = { - item_type: 'attachments', - [ActionType.CREATED]: 25, - [ActionType.FAILED]: 1, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0]).toEqual({ - item_type: 'attachments', - [ActionType.CREATED]: 25, - [ActionType.FAILED]: 1, - }); - }); - - it('[edge] should preserve existing created count when new report has undefined created', () => { - // Arrange - const loaderReports: LoaderReport[] = [ - { - item_type: 'issues', - [ActionType.CREATED]: 10, - }, - ]; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.UPDATED]: 5, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0][ActionType.CREATED]).toBe(10); - expect(result[0][ActionType.UPDATED]).toBe(5); - }); - - it('[edge] should preserve existing updated count when new report has undefined updated', () => { - // Arrange - const loaderReports: LoaderReport[] = [ - { - item_type: 'issues', - [ActionType.UPDATED]: 15, - }, - ]; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.CREATED]: 3, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0][ActionType.UPDATED]).toBe(15); - expect(result[0][ActionType.CREATED]).toBe(3); - }); - - it('[edge] should preserve existing failed count when new report has undefined failed', () => { - // Arrange - const loaderReports: LoaderReport[] = [ - { - item_type: 'issues', - [ActionType.FAILED]: 7, - }, - ]; - const report: LoaderReport = { - item_type: 'issues', - [ActionType.CREATED]: 2, - }; - - // Act - const result = addReportToLoaderReport({ loaderReports, report }); - - // Assert - expect(result).toHaveLength(1); - expect(result[0][ActionType.FAILED]).toBe(7); - expect(result[0][ActionType.CREATED]).toBe(2); - }); -}); - -describe(toRfc3339Timestamp.name, () => { - it('should convert milliseconds to an RFC3339 string', () => { - const ms = new Date('2024-06-01T00:00:00.000Z').getTime(); - - const result = toRfc3339Timestamp(ms); - - expect(result).toBe('2024-06-01T00:00:00.000Z'); - }); - - it('[edge] should convert the Unix epoch (0) instead of treating it as unset', () => { - const result = toRfc3339Timestamp(0); - - expect(result).toBe('1970-01-01T00:00:00.000Z'); - }); - - it('[edge] should return undefined when ms is undefined', () => { - const result = toRfc3339Timestamp(undefined); - - expect(result).toBeUndefined(); - }); - - it('[edge] should return undefined for NaN', () => { - const result = toRfc3339Timestamp(NaN); - - expect(result).toBeUndefined(); - }); - - it('[edge] should return undefined for Infinity', () => { - const result = toRfc3339Timestamp(Infinity); - - expect(result).toBeUndefined(); - }); - - it('[edge] should convert negative milliseconds (pre-1970 dates)', () => { - const ms = new Date('1969-12-31T00:00:00.000Z').getTime(); - - const result = toRfc3339Timestamp(ms); - - expect(result).toBe('1969-12-31T00:00:00.000Z'); - }); -}); diff --git a/src/multithreading/worker-adapter/worker-adapter.interfaces.ts b/src/multithreading/worker-adapter/worker-adapter.interfaces.ts deleted file mode 100644 index 0b357f39..00000000 --- a/src/multithreading/worker-adapter/worker-adapter.interfaces.ts +++ /dev/null @@ -1,16 +0,0 @@ -/** - * ProgressData represents the progress information sent with extraction events. - * Includes statistics about the last extracted item type and calculated time ranges. - */ -export interface ProgressData { - // Last extracted item type statistics - item_type?: string; - oldest_created_date?: string; - newest_created_date?: string; - oldest_modified_date?: string; - newest_modified_date?: string; - - // Calculated time ranges in absolute times - oldest_state_date?: string; - newest_state_date?: string; -} diff --git a/src/multithreading/worker-adapter/worker-adapter.loading.test.ts b/src/multithreading/worker-adapter/worker-adapter.loading.test.ts deleted file mode 100644 index da447da0..00000000 --- a/src/multithreading/worker-adapter/worker-adapter.loading.test.ts +++ /dev/null @@ -1,730 +0,0 @@ -import { State } from '../../state/state'; -import { mockServer } from '../../tests/jest.setup'; -import { createMockEvent } from '../../common/test-utils'; -import { - AdapterState, - AirdropEvent, - EventType, - LoaderEventType, -} from '../../types'; -import { - ActionType, - ExternalSystemAttachment, - ExternalSystemItem, -} from '../../types/loading'; -import { WorkerAdapter } from './worker-adapter'; - -jest.mock('../../common/control-protocol', () => ({ - emit: jest.fn().mockResolvedValue({}), -})); - -jest.mock('../../mappers/mappers'); -jest.mock('../../uploader/uploader'); -jest.mock('../../repo/repo'); -jest.mock('node:worker_threads', () => ({ - parentPort: { postMessage: jest.fn() }, -})); -jest.mock('../../attachments-streaming/attachments-streaming-pool', () => ({ - AttachmentsStreamingPool: jest.fn().mockImplementation(() => ({ - streamAll: jest.fn().mockResolvedValue(undefined), - })), -})); - -interface TestState { - attachments: { completed: boolean }; -} - -function makeAdapter(eventType: EventType): { - adapter: WorkerAdapter; - event: AirdropEvent; - adapterState: State; -} { - const event = createMockEvent(mockServer.baseUrl, { - payload: { event_type: eventType }, - }); - const initialState: AdapterState = { - attachments: { completed: false }, - lastSyncStarted: '', - lastSuccessfulSyncStarted: '', - snapInVersionId: '', - toDevRev: { - attachmentsMetadata: { - artifactIds: [], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], - }, - }, - }; - const adapterState = new State({ event, initialState }); - const adapter = new WorkerAdapter({ event, adapterState }); - return { adapter, event, adapterState }; -} - -function makeLoaderItem(devrevId = 'dev-1'): ExternalSystemItem { - return { - id: { devrev: devrevId, external: 'ext-1' }, - created_date: '', - modified_date: '', - data: {}, - }; -} - -function setupLoaderFile( - adapter: WorkerAdapter, - items: ExternalSystemItem[], - itemType = 'tasks' -) { - adapter['adapterState'].state.fromDevRev = { - filesToLoad: [ - { - id: 'artifact-1', - file_name: 'file.json', - itemType, - count: items.length, - lineToProcess: 0, - completed: false, - }, - ], - }; - adapter['uploader'].getJsonObjectByArtifactId = jest - .fn() - .mockResolvedValue({ response: items }); -} - -describe(`${WorkerAdapter.name}.loadItemTypes — timeout and unexpected errors`, () => { - let adapter: WorkerAdapter; - let exitSpy: jest.SpyInstance; - let emitSpy: jest.SpyInstance; - - beforeEach(() => { - jest.clearAllMocks(); - ({ adapter } = makeAdapter(EventType.ContinueLoadingData)); - exitSpy = jest - .spyOn(process, 'exit') - .mockImplementation(() => undefined as never); - emitSpy = jest.spyOn(adapter, 'emit').mockResolvedValue(); - }); - - afterEach(() => { - exitSpy.mockRestore(); - jest.restoreAllMocks(); - }); - - it('should emit DataLoadingProgress and exit on timeout', async () => { - // Arrange - const items = [makeLoaderItem('dev-1'), makeLoaderItem('dev-2')]; - setupLoaderFile(adapter, items); - adapter.isTimeout = true; - const itemTypesToLoad = [ - { itemType: 'tasks', create: jest.fn(), update: jest.fn() }, - ]; - - // Act - await adapter.loadItemTypes({ itemTypesToLoad }); - - // Assert - expect(emitSpy).toHaveBeenCalledWith(LoaderEventType.DataLoadingProgress); - expect(exitSpy).toHaveBeenCalledWith(0); - }); - - it('should emit DataLoadingProgress mid-loop when timeout arrives between items', async () => { - // Arrange - const items = [ - makeLoaderItem('dev-1'), - makeLoaderItem('dev-2'), - makeLoaderItem('dev-3'), - ]; - setupLoaderFile(adapter, items); - exitSpy.mockRestore(); - exitSpy = jest.spyOn(process, 'exit').mockImplementation((() => { - throw new Error('process.exit'); - }) as never); - let loadItemCallCount = 0; - // eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/require-await - jest.spyOn(adapter as any, 'loadItem').mockImplementation(async () => { - loadItemCallCount++; - if (loadItemCallCount === 1) { - adapter.isTimeout = true; - } - return { report: { item_type: 'tasks', updated: 1 } }; - }); - const itemTypesToLoad = [ - { itemType: 'tasks', create: jest.fn(), update: jest.fn() }, - ]; - - // Act & Assert - await expect(adapter.loadItemTypes({ itemTypesToLoad })).rejects.toThrow( - 'process.exit' - ); - expect(loadItemCallCount).toBe(1); - expect(emitSpy).toHaveBeenCalledWith(LoaderEventType.DataLoadingProgress); - }); - - it('should emit DataLoadingError and exit(1) on unexpected error', async () => { - // Arrange - adapter['adapterState'].state.fromDevRev = { - filesToLoad: [ - { - id: 'artifact-1', - file_name: 'file1.json', - itemType: 'tasks', - count: 1, - lineToProcess: 0, - completed: false, - }, - ], - }; - adapter['uploader'].getJsonObjectByArtifactId = jest - .fn() - .mockRejectedValue(new Error('Unexpected network failure')); - const itemTypesToLoad = [ - { itemType: 'tasks', create: jest.fn(), update: jest.fn() }, - ]; - - // Act - await adapter.loadItemTypes({ itemTypesToLoad }); - - // Assert - expect(emitSpy).toHaveBeenCalledWith( - LoaderEventType.DataLoadingError, - expect.objectContaining({ - error: expect.objectContaining({ - message: expect.stringContaining('Error during data loading'), - }), - }) - ); - expect(exitSpy).toHaveBeenCalledWith(1); - }); -}); - -describe(`${WorkerAdapter.name}.loadItemTypes — loadItem branch coverage via public API`, () => { - let adapter: WorkerAdapter; - let emitSpy: jest.SpyInstance; - let exitSpy: jest.SpyInstance; - - const itemTypesToLoad = [ - { itemType: 'tasks', create: jest.fn(), update: jest.fn() }, - ]; - - beforeEach(() => { - jest.clearAllMocks(); - ({ adapter } = makeAdapter(EventType.ContinueLoadingData)); - emitSpy = jest.spyOn(adapter, 'emit').mockResolvedValue(); - exitSpy = jest - .spyOn(process, 'exit') - .mockImplementation(() => undefined as never); - - itemTypesToLoad[0].create = jest.fn(); - itemTypesToLoad[0].update = jest.fn(); - }); - - afterEach(() => { - exitSpy.mockRestore(); - jest.restoreAllMocks(); - }); - - it('should accumulate an UPDATED report when the connector updates the item and the mapper sync succeeds', async () => { - // Arrange - setupLoaderFile(adapter, [makeLoaderItem('dev-1')]); - adapter['_mappers'].getByTargetId = jest.fn().mockResolvedValue({ - data: { sync_mapper_record: { id: 'smr-1' } }, - }); - adapter['_mappers'].update = jest.fn().mockResolvedValue({ data: {} }); - itemTypesToLoad[0].update = jest - .fn() - .mockResolvedValue({ id: 'ext-updated-1' }); - - // Act - const { reports } = await adapter.loadItemTypes({ itemTypesToLoad }); - - // Assert - expect(reports).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - item_type: 'tasks', - [ActionType.UPDATED]: 1, - }), - ]) - ); - expect(emitSpy).not.toHaveBeenCalled(); - }); - - it('should fall back to create and accumulate a CREATED report when the mapper record does not exist (404)', async () => { - // Arrange - setupLoaderFile(adapter, [makeLoaderItem('dev-2')]); - const axiosError = { isAxiosError: true, response: { status: 404 } }; - adapter['_mappers'].getByTargetId = jest.fn().mockRejectedValue(axiosError); - adapter['_mappers'].create = jest.fn().mockResolvedValue({ data: {} }); - itemTypesToLoad[0].create = jest - .fn() - .mockResolvedValue({ id: 'new-ext-id' }); - - // Act - const { reports } = await adapter.loadItemTypes({ itemTypesToLoad }); - - // Assert - expect(reports).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - item_type: 'tasks', - [ActionType.CREATED]: 1, - }), - ]) - ); - expect(emitSpy).not.toHaveBeenCalled(); - }); - - it('should emit DataLoadingDelayed and stop processing when the connector signals a rate-limit delay', async () => { - // Arrange - setupLoaderFile(adapter, [makeLoaderItem('dev-3')]); - adapter['_mappers'].getByTargetId = jest.fn().mockResolvedValue({ - data: { sync_mapper_record: { id: 'smr-1' } }, - }); - itemTypesToLoad[0].update = jest.fn().mockResolvedValue({ delay: 15 }); - - // Act - await adapter.loadItemTypes({ itemTypesToLoad }); - - // Assert - expect(emitSpy).toHaveBeenCalledWith( - LoaderEventType.DataLoadingDelayed, - expect.objectContaining({ delay: 15 }) - ); - }); - - it('should count the item as FAILED when the update succeeds but the mapper sync throws', async () => { - // Arrange - setupLoaderFile(adapter, [makeLoaderItem('dev-4')]); - adapter['_mappers'].getByTargetId = jest.fn().mockResolvedValue({ - data: { sync_mapper_record: { id: 'smr-1' } }, - }); - adapter['_mappers'].update = jest - .fn() - .mockRejectedValue(new Error('mapper down')); - itemTypesToLoad[0].update = jest.fn().mockResolvedValue({ id: 'ext-id' }); - - // Act - const { reports } = await adapter.loadItemTypes({ itemTypesToLoad }); - - // Assert - expect(emitSpy).not.toHaveBeenCalled(); - expect(reports).toBeDefined(); - }); - - it('should not emit for a non-404 Axios error from the mapper (recorded as item-level error)', async () => { - // Arrange - setupLoaderFile(adapter, [makeLoaderItem('dev-5')]); - const axiosError = { - isAxiosError: true, - message: 'internal server error', - response: { status: 500 }, - }; - adapter['_mappers'].getByTargetId = jest.fn().mockRejectedValue(axiosError); - - // Act - await adapter.loadItemTypes({ itemTypesToLoad }); - - // Assert - expect(emitSpy).not.toHaveBeenCalled(); - }); - - it('should handle a null sync_mapper_record gracefully and continue loading', async () => { - // Arrange - setupLoaderFile(adapter, [makeLoaderItem('dev-6')]); - adapter['_mappers'].getByTargetId = jest - .fn() - .mockResolvedValue({ data: null }); - - // Act - const { reports } = await adapter.loadItemTypes({ itemTypesToLoad }); - - // Assert - expect(emitSpy).not.toHaveBeenCalled(); - expect(reports).toBeDefined(); - }); -}); - -describe(`${WorkerAdapter.name}.loadItemTypes — additional branches`, () => { - let adapter: WorkerAdapter; - let emitSpy: jest.SpyInstance; - - beforeEach(() => { - jest.clearAllMocks(); - ({ adapter } = makeAdapter(EventType.ContinueLoadingData)); - emitSpy = jest.spyOn(adapter, 'emit').mockResolvedValue(); - }); - - afterEach(() => { - jest.restoreAllMocks(); - }); - - it('should return immediately with empty reports when filesToLoad is empty', async () => { - // Arrange - adapter['adapterState'].state.fromDevRev = { filesToLoad: [] }; - - // Act - const result = await adapter.loadItemTypes({ - itemTypesToLoad: [ - { itemType: 'tasks', create: jest.fn(), update: jest.fn() }, - ], - }); - - // Assert - expect(result.reports).toEqual([]); - expect(emitSpy).not.toHaveBeenCalled(); - }); - - it('should emit DataLoadingError when a file references an item type not in itemTypesToLoad', async () => { - // Arrange - adapter['adapterState'].state.fromDevRev = { - filesToLoad: [ - { - id: 'art-1', - file_name: 'file.json', - itemType: 'unknown-type', - count: 1, - lineToProcess: 0, - completed: false, - }, - ], - }; - adapter['uploader'].getJsonObjectByArtifactId = jest - .fn() - .mockResolvedValue({ response: [makeLoaderItem()] }); - - // Act - await adapter.loadItemTypes({ - itemTypesToLoad: [ - { itemType: 'tasks', create: jest.fn(), update: jest.fn() }, - ], - }); - - // Assert - expect(emitSpy).toHaveBeenCalledWith( - LoaderEventType.DataLoadingError, - expect.objectContaining({ - error: expect.objectContaining({ - message: expect.stringContaining('unknown-type'), - }), - }) - ); - }); -}); - -describe(`${WorkerAdapter.name}.loadAttachments — timeout, transformer errors, unexpected errors`, () => { - let adapter: WorkerAdapter; - let exitSpy: jest.SpyInstance; - let emitSpy: jest.SpyInstance; - - function setupFilesToLoad( - a: WorkerAdapter, - items: ExternalSystemAttachment[] - ) { - a['adapterState'].state.fromDevRev = { - filesToLoad: [ - { - id: 'artifact-1', - file_name: 'attachments.json', - itemType: 'attachment', - count: items.length, - lineToProcess: 0, - completed: false, - }, - ], - }; - - a['uploader'].getJsonObjectByArtifactId = jest - .fn() - .mockResolvedValue({ response: items }); - } - - beforeEach(() => { - jest.clearAllMocks(); - ({ adapter } = makeAdapter(EventType.ContinueLoadingAttachments)); - exitSpy = jest - .spyOn(process, 'exit') - .mockImplementation(() => undefined as never); - emitSpy = jest.spyOn(adapter, 'emit').mockResolvedValue(); - }); - - afterEach(() => { - exitSpy.mockRestore(); - jest.restoreAllMocks(); - }); - - it('should emit AttachmentLoadingProgress and exit on timeout', async () => { - // Arrange - const items = [ - { - reference_id: 'ref-1', - parent_type: 'task', - parent_reference_id: 'parent-1', - file_name: 'file.pdf', - file_type: 'application/pdf', - file_size: 100, - url: 'https://example.com/file.pdf', - valid_until: '', - created_by_id: 'user-1', - created_date: '', - modified_by_id: 'user-1', - modified_date: '', - }, - ] as ExternalSystemAttachment[]; - setupFilesToLoad(adapter, items); - adapter.isTimeout = true; - - // Act - await adapter.loadAttachments({ - create: jest.fn(), - }); - - // Assert - expect(emitSpy).toHaveBeenCalledWith( - LoaderEventType.AttachmentLoadingProgress - ); - expect(exitSpy).toHaveBeenCalledWith(0); - }); - - it('should emit AttachmentLoadingError on transformer file error', async () => { - // Arrange - adapter['adapterState'].state.fromDevRev = { - filesToLoad: [ - { - id: 'bad-artifact', - file_name: 'attachments.json', - itemType: 'attachment', - count: 1, - lineToProcess: 0, - completed: false, - }, - ], - }; - adapter['uploader'].getJsonObjectByArtifactId = jest - .fn() - .mockResolvedValue({ - response: null, - error: new Error('Artifact not found'), - }); - - // Act - await adapter.loadAttachments({ - create: jest.fn(), - }); - - // Assert - expect(emitSpy).toHaveBeenCalledWith( - LoaderEventType.AttachmentLoadingError, - expect.objectContaining({ - error: expect.objectContaining({ - message: expect.stringContaining('Transformer file not found'), - }), - }) - ); - }); - - it('should emit AttachmentLoadingError and exit(1) on unexpected error', async () => { - // Arrange - const items = [ - { - reference_id: 'ref-1', - parent_type: 'task', - parent_reference_id: 'parent-1', - file_name: 'file.pdf', - file_type: 'application/pdf', - file_size: 100, - url: 'https://example.com/file.pdf', - valid_until: '', - created_by_id: 'user-1', - created_date: '', - modified_by_id: 'user-1', - modified_date: '', - }, - ] as ExternalSystemAttachment[]; - setupFilesToLoad(adapter, items); - const mockCreate = jest - .fn() - .mockRejectedValue(new Error('Unexpected API failure')); - - // Act - await adapter.loadAttachments({ create: mockCreate }); - - // Assert - expect(emitSpy).toHaveBeenCalledWith( - LoaderEventType.AttachmentLoadingError, - expect.objectContaining({ - error: expect.objectContaining({ - message: expect.stringContaining('Error during attachment loading'), - }), - }) - ); - expect(exitSpy).toHaveBeenCalledWith(1); - }); -}); - -describe(`${WorkerAdapter.name}.loadAttachments — additional branches`, () => { - let adapter: WorkerAdapter; - let emitSpy: jest.SpyInstance; - - beforeEach(() => { - jest.clearAllMocks(); - ({ adapter } = makeAdapter(EventType.ContinueLoadingAttachments)); - emitSpy = jest.spyOn(adapter, 'emit').mockResolvedValue(); - }); - - afterEach(() => { - jest.restoreAllMocks(); - }); - - it('should return immediately with empty reports when fromDevRev is not set', async () => { - // Arrange - adapter['adapterState'].state.fromDevRev = undefined; - - // Act - const result = await adapter.loadAttachments({ create: jest.fn() }); - - // Assert - expect(result.reports).toEqual([]); - expect(emitSpy).not.toHaveBeenCalled(); - }); - - it('should emit AttachmentLoadingDelayed and stop the loop when the connector signals a rate-limit delay', async () => { - // Arrange - adapter['adapterState'].state.fromDevRev = { - filesToLoad: [ - { - id: 'art-1', - file_name: 'attachments.json', - itemType: 'attachment', - count: 1, - lineToProcess: 0, - completed: false, - }, - ], - }; - adapter['uploader'].getJsonObjectByArtifactId = jest - .fn() - .mockResolvedValue({ - response: [ - { - reference_id: 'ref-1', - parent_type: 'task', - parent_reference_id: 'parent-1', - file_name: 'file.pdf', - file_type: 'application/pdf', - file_size: 100, - url: 'https://example.com/file.pdf', - valid_until: '', - created_by_id: 'user-1', - created_date: '', - modified_by_id: 'user-1', - modified_date: '', - }, - ], - }); - jest - // eslint-disable-next-line @typescript-eslint/no-explicit-any - .spyOn(adapter as any, 'loadAttachment') - .mockResolvedValue({ rateLimit: { delay: 20 } } as never); - - // Act - await adapter.loadAttachments({ create: jest.fn() }); - - // Assert - expect(emitSpy).toHaveBeenCalledWith( - LoaderEventType.AttachmentLoadingDelayed, - expect.objectContaining({ delay: 20 }) - ); - }); -}); - -describe(`${WorkerAdapter.name}.loadAttachment`, () => { - let adapter: WorkerAdapter; - - beforeEach(() => { - jest.clearAllMocks(); - ({ adapter } = makeAdapter(EventType.ContinueLoadingAttachments)); - }); - - afterEach(() => { - jest.restoreAllMocks(); - }); - - function makeAttachment(): ExternalSystemAttachment { - return { - reference_id: 'ref-1', - parent_type: 'task', - parent_reference_id: 'parent-1', - file_name: 'file.pdf', - file_type: 'application/pdf', - file_size: 100, - url: 'https://example.com/file.pdf', - valid_until: '', - created_by_id: 'user-1', - created_date: '', - modified_by_id: 'user-1', - modified_date: '', - } as ExternalSystemAttachment; - } - - it('should return a CREATED report when create succeeds', async () => { - // Arrange - adapter['_mappers'].create = jest.fn().mockResolvedValue({ data: {} }); - const create = jest.fn().mockResolvedValue({ id: 'att-ext-1' }); - - // Act - const result = await adapter['loadAttachment']({ - item: makeAttachment(), - create, - }); - - // Assert - expect(result.report?.item_type).toBe('attachments'); - expect(result.report?.[ActionType.CREATED]).toBe(1); - }); - - it('should still return CREATED even when mapper create fails — attachment loading is resilient', async () => { - // Arrange - adapter['_mappers'].create = jest - .fn() - .mockRejectedValue(new Error('mapper failed')); - const create = jest.fn().mockResolvedValue({ id: 'att-ext-1' }); - - // Act - const result = await adapter['loadAttachment']({ - item: makeAttachment(), - create, - }); - - // Assert - expect(result.report?.[ActionType.CREATED]).toBe(1); - }); - - it('should propagate rate-limit delay when the connector signals one', async () => { - // Arrange - const create = jest.fn().mockResolvedValue({ delay: 30 }); - - // Act - const result = await adapter['loadAttachment']({ - item: makeAttachment(), - create, - }); - - // Assert - expect(result.rateLimit?.delay).toBe(30); - }); - - it('should return a FAILED report when create returns neither id nor delay', async () => { - // Arrange - const create = jest.fn().mockResolvedValue({ id: null, delay: null }); - - // Act - const result = await adapter['loadAttachment']({ - item: makeAttachment(), - create, - }); - - // Assert - expect(result.report?.item_type).toBe('attachments'); - expect(result.report?.[ActionType.FAILED]).toBe(1); - }); -}); diff --git a/src/multithreading/worker-adapter/worker-adapter.serialization.test.ts b/src/multithreading/worker-adapter/worker-adapter.serialization.test.ts deleted file mode 100644 index bf488fd7..00000000 --- a/src/multithreading/worker-adapter/worker-adapter.serialization.test.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { jsonl } from 'js-jsonl'; - -// Pin the serialization contract for items that reach the uploader. -// -// The SDK uploads items via Uploader.upload(), which calls jsonl.stringify() on -// the input. That means user `create`/`update` callbacks and normalizers can -// silently produce inputs that fail or lose information at the wire boundary: -// -// - Circular references throw "Converting circular structure to JSON" -// - BigInt values throw "Do not know how to serialize a BigInt" -// - Date objects are converted to ISO strings (information loss: no Date on -// the other side, just a string) -// - undefined fields are dropped (null fields are preserved) -// -// These tests exist to catch regressions in the serialization layer (e.g., -// silently switching to a different serializer that masks BigInt or mangles -// Dates) before they reach production. - -describe('serialization boundary for items uploaded via jsonl', () => { - it('throws when an item contains a circular reference', () => { - // Arrange - const item: Record = { id: 'a' }; - item.self = item; - - // Act & Assert - expect(() => jsonl.stringify([item])).toThrow(/circular/i); - }); - - it('throws when an item contains a BigInt field', () => { - // Arrange - const item = { id: 'a', counter: BigInt(1) }; - - // Act & Assert - expect(() => jsonl.stringify([item])).toThrow(/BigInt/i); - }); - - it('serializes Date instances to ISO strings (information loss — consumer receives a string)', () => { - // Arrange - const item = { - id: 'a', - created: new Date('2025-01-01T00:00:00.000Z'), - }; - - // Act - const output = jsonl.stringify([item]); - const parsed = JSON.parse(output) as Record; - - // Assert - expect(parsed.created).toBe('2025-01-01T00:00:00.000Z'); - expect(typeof parsed.created).toBe('string'); - }); - - it('drops undefined fields but preserves null fields', () => { - // Arrange - const item = { - id: 'a', - present: null, - missing: undefined, - }; - - // Act - const output = jsonl.stringify([item]); - const parsed = JSON.parse(output) as Record; - - // Assert - expect(parsed).toEqual({ id: 'a', present: null }); - expect(parsed).not.toHaveProperty('missing'); - }); - - it('emits one newline-terminated line per item (jsonl format)', () => { - // Arrange - const items = [{ id: 'a' }, { id: 'b' }, { id: 'c' }]; - - // Act - const output = jsonl.stringify(items); - const lines = output.split('\n').filter((l) => l.length > 0); - - // Assert - expect(lines).toHaveLength(3); - expect(JSON.parse(lines[0])).toEqual({ id: 'a' }); - expect(JSON.parse(lines[2])).toEqual({ id: 'c' }); - }); -}); diff --git a/src/multithreading/worker-adapter/worker-adapter.ts b/src/multithreading/worker-adapter/worker-adapter.ts deleted file mode 100644 index aa5cee74..00000000 --- a/src/multithreading/worker-adapter/worker-adapter.ts +++ /dev/null @@ -1,1296 +0,0 @@ -import axios, { AxiosResponse } from 'axios'; -import { parentPort } from 'node:worker_threads'; -import { AttachmentsStreamingPool } from '../../attachments-streaming/attachments-streaming-pool'; -import { - AirSyncDefaultItemTypes, - EVENT_SIZE_THRESHOLD_BYTES, - SSOR_ATTACHMENT, - STATELESS_EVENT_TYPES, -} from '../../common/constants'; -import { emit } from '../../common/control-protocol'; -import { - addReportToLoaderReport, - getFilesToLoad, - toRfc3339Timestamp, -} from './worker-adapter.helpers'; -import { ProgressData } from './worker-adapter.interfaces'; -import { serializeError } from '../../logger/logger'; -import { - runWithSdkLogContext, - runWithUserLogContext, -} from '../../logger/logger.context'; -import { Mappers } from '../../mappers/mappers'; -import { SyncMapperRecordStatus } from '../../mappers/mappers.interface'; -import { Repo } from '../../repo/repo'; -import { - NormalizedAttachment, - RepoInterface, -} from '../../repo/repo.interfaces'; -import { State } from '../../state/state'; -import { AdapterState } from '../../state/state.interfaces'; -import { - AirdropEvent, - EventData, - EventType, - ExternalSystemAttachmentProcessors, - ExternalSystemAttachmentStreamingFunction, - ExtractorEventType, - ProcessAttachmentReturnType, - StreamAttachmentsReturnType, -} from '../../types/extraction'; -import { - ActionType, - ExternalSystemAttachment, - ExternalSystemItem, - ExternalSystemLoadingFunction, - FileToLoad, - ItemTypesToLoadParams, - ItemTypeToLoad, - LoaderEventType, - LoaderReport, - LoadItemResponse, - LoadItemTypesResponse, - StatsFileObject, -} from '../../types/loading'; -import { - WorkerAdapterInterface, - WorkerAdapterOptions, - WorkerMessageEmitted, - WorkerMessageSubject, -} from '../../types/workers'; -import { Uploader } from '../../uploader/uploader'; -import { Artifact, SsorAttachment } from '../../uploader/uploader.interfaces'; -import { translateOutgoingEventType } from '../../common/event-type-translation'; -import { truncateMessage } from '../../common/helpers'; - -export function createWorkerAdapter({ - event, - adapterState, - options, -}: WorkerAdapterInterface): WorkerAdapter { - return new WorkerAdapter({ - event, - adapterState, - options, - }); -} - -/** - * WorkerAdapter class is used to interact with Airdrop platform. It is passed to the snap-in - * as parameter in processTask and onTimeout functions. The class provides - * utilities to emit control events to the platform, update the state of the connector, - * and upload artifacts to the platform. - * @class WorkerAdapter - * @constructor - * @param options - The options to create a new instance of WorkerAdapter class - * @param event - The event object received from the platform - * @param initialState - The initial state of the adapter - * @param isLocalDevelopment - A flag to indicate if the adapter is being used in local development - * @param workerPath - The path to the worker file - * - * @public - */ -export class WorkerAdapter { - readonly event: AirdropEvent; - readonly options?: WorkerAdapterOptions; - hasWorkerEmitted: boolean; - - private _isTimeout: boolean = false; - private resolveTimeoutSignal!: () => void; - readonly timeoutSignal: Promise = new Promise((resolve) => { - this.resolveTimeoutSignal = resolve; - }); - - private adapterState: State; - private _artifacts: Artifact[]; - private repos: Repo[] = []; - private lastExtractedItemType?: string; - private currentEventDataLength: number = 0; - - // Loader - private loaderReports: LoaderReport[]; - private _processedFiles: string[]; - private _mappers: Mappers; - private uploader: Uploader; - - constructor({ - event, - adapterState, - options, - }: WorkerAdapterInterface) { - this.event = event; - this.options = options; - this.adapterState = adapterState; - this._artifacts = []; - this.hasWorkerEmitted = false; - - // Loader - this.loaderReports = []; - this._processedFiles = []; - this._mappers = new Mappers({ - event, - options, - }); - this.uploader = new Uploader({ - event, - options, - }); - } - - get isTimeout(): boolean { - return this._isTimeout; - } - - set isTimeout(value: boolean) { - this._isTimeout = value; - if (value) { - this.resolveTimeoutSignal(); - } - } - - get state(): AdapterState { - return this.adapterState.state; - } - - set state(value: AdapterState) { - this.adapterState.state = value; - } - - get reports(): LoaderReport[] { - return this.loaderReports; - } - - get processedFiles(): string[] { - return this._processedFiles; - } - - get mappers(): Mappers { - return this._mappers; - } - - get extractionScope() { - return this.adapterState.extractionScope; - } - - /** - * Returns whether the given item type should be extracted. - * Defaults to true if the scope is empty or the item type is not listed. - */ - shouldExtract(itemType: string): boolean { - const scope = this.extractionScope; - if (Object.keys(scope).length === 0) return true; - if (!(itemType in scope)) return true; - return scope[itemType].extract; - } - - initializeRepos(repos: RepoInterface[]) { - this.repos = repos.map((repo) => { - const shouldNormalize = - repo.itemType !== AirSyncDefaultItemTypes.EXTERNAL_DOMAIN_METADATA && - repo.itemType !== SSOR_ATTACHMENT; - - return new Repo({ - event: this.event, - itemType: repo.itemType, - ...(shouldNormalize && { normalize: repo.normalize }), - onUpload: (artifact: Artifact) => { - this.lastExtractedItemType = repo.itemType; - - // We need to store artifacts ids in state for later use when streaming attachments - if (repo.itemType === AirSyncDefaultItemTypes.ATTACHMENTS) { - this.state.toDevRev?.attachmentsMetadata.artifactIds.push( - artifact.id - ); - } - - // Calculate size of the entire artifact object that goes in the SQS message - this.currentEventDataLength += Buffer.byteLength( - JSON.stringify(artifact), - 'utf8' - ); - - if ( - this.currentEventDataLength > EVENT_SIZE_THRESHOLD_BYTES && - !this.isTimeout - ) { - this.isTimeout = true; - } - }, - options: { - ...this.options, - ...repo.overridenOptions, - }, - }); - }); - } - - getRepo(itemType: string): Repo | undefined { - return runWithSdkLogContext(() => { - const repo = this.repos.find((repo) => repo.itemType === itemType); - - if (!repo) { - console.error(`Repo for item type ${itemType} not found.`); - return; - } - - return repo; - }); - } - - async postState() { - return runWithSdkLogContext(async () => { - await this.adapterState.postState(); - }); - } - - get artifacts(): Artifact[] { - return this._artifacts; - } - - set artifacts(artifacts: Artifact[]) { - this._artifacts = this._artifacts - .concat(artifacts) - .filter((value, index, self) => self.indexOf(value) === index); - } - - /** - * Emits an event to the platform. - * - * @param newEventType - The event type to be emitted - * @param data - The data to be sent with the event - */ - async emit( - newEventType: ExtractorEventType | LoaderEventType, - data?: EventData - ): Promise { - return runWithSdkLogContext(async () => { - newEventType = translateOutgoingEventType(newEventType); - - if (this.hasWorkerEmitted) { - console.warn( - `Trying to emit event with event type: ${newEventType}. Ignoring emit request because it has already been emitted.` - ); - return; - } - - // If the event is ExternalSyncUnitExtractionDone, upload external sync units via a Repo before emitting - // TODO: Remove in v2.0.0 - if ( - newEventType === ExtractorEventType.ExternalSyncUnitExtractionDone && - data?.external_sync_units && - data.external_sync_units.length > 0 - ) { - console.log( - `Uploading ${data.external_sync_units.length} external sync units via repo before emitting event.` - ); - - this.initializeRepos([ - { - itemType: AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS, - overridenOptions: { - batchSize: 25000, - skipConfirmation: true, - }, - }, - ]); - - await this.getRepo(AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS)?.push( - data.external_sync_units - ); - - // Remove inline external_sync_units from data to avoid SQS size issues - delete data.external_sync_units; - } - - // Upload all repos before emitting the event - console.log( - `Uploading all repos before emitting event with event type: ${newEventType}.` - ); - - try { - await this.uploadAllRepos(); - } catch (error) { - console.error('Error while uploading repos', error); - parentPort?.postMessage(WorkerMessageSubject.WorkerMessageExit); - this.hasWorkerEmitted = true; - return; - } - - // If the extraction is done, we want to save the timestamp of the last successful sync - if (newEventType === ExtractorEventType.AttachmentExtractionDone) { - console.log( - `Overwriting lastSuccessfulSyncStarted with lastSyncStarted (${this.state.lastSyncStarted}).` - ); - - this.state.lastSuccessfulSyncStarted = this.state.lastSyncStarted; - this.state.lastSyncStarted = ''; - - // Clear pending extraction boundaries now that the cycle is complete - this.state.pendingWorkersOldest = ''; - this.state.pendingWorkersNewest = ''; - - // Update workersOldest and workersNewest boundaries from resolved extraction timestamps. - // Expand boundaries: workersOldest gets the earliest timestamp, workersNewest gets the latest. - const extractionStart = this.event.payload.event_context.extract_from; - const extractionEnd = this.event.payload.event_context.extract_to; - - if ( - extractionStart && - (!this.state.workersOldest || - extractionStart < this.state.workersOldest) - ) { - console.log( - `Updating workersOldest from '${this.state.workersOldest}' to '${extractionStart}'.` - ); - this.state.workersOldest = extractionStart; - } - - if ( - extractionEnd && - (!this.state.workersNewest || - extractionEnd > this.state.workersNewest) - ) { - console.log( - `Updating workersNewest from '${this.state.workersNewest}' to '${extractionEnd}'.` - ); - this.state.workersNewest = extractionEnd; - } - } - - // We want to save the state every time we emit an event, except for the start and delete events - if (!STATELESS_EVENT_TYPES.includes(this.event.payload.event_type)) { - console.log( - `Saving state before emitting event with event type: ${newEventType}.` - ); - - try { - await this.adapterState.postState(this.state); - } catch (error) { - console.error('Error while posting state', error); - parentPort?.postMessage(WorkerMessageSubject.WorkerMessageExit); - this.hasWorkerEmitted = true; - return; - } - } - - try { - // Always prune error messages to make them shorter before emit - if (data?.error?.message) { - data.error.message = truncateMessage(data.error.message); - } - - const isExtractionEvent = Object.values(ExtractorEventType).includes( - newEventType as ExtractorEventType - ); - const isLoaderEvent = Object.values(LoaderEventType).includes( - newEventType as LoaderEventType - ); - - const progressData: ProgressData = {}; - - if ( - isExtractionEvent && - (newEventType == ExtractorEventType.DataExtractionDone || - newEventType == ExtractorEventType.DataExtractionProgress || - newEventType == ExtractorEventType.AttachmentExtractionDone || - newEventType == ExtractorEventType.AttachmentExtractionProgress) - ) { - const repo = this.lastExtractedItemType - ? this.repos.find((r) => r.itemType === this.lastExtractedItemType) - : undefined; - if (repo) { - progressData.item_type = repo.itemType; - progressData.newest_created_date = toRfc3339Timestamp( - repo.dateRanges.creationDate.newest - ); - progressData.oldest_created_date = toRfc3339Timestamp( - repo.dateRanges.creationDate.oldest - ); - progressData.newest_modified_date = toRfc3339Timestamp( - repo.dateRanges.modifiedDate.newest - ); - progressData.oldest_modified_date = toRfc3339Timestamp( - repo.dateRanges.modifiedDate.oldest - ); - } - } - - await emit({ - eventType: newEventType, - event: this.event, - data: { - ...data, - ...(isExtractionEvent ? { artifacts: this.artifacts } : {}), - ...(isLoaderEvent - ? { reports: this.reports, processed_files: this.processedFiles } - : {}), - }, - worker_metadata: { ...progressData }, - }); - - const message: WorkerMessageEmitted = { - subject: WorkerMessageSubject.WorkerMessageEmitted, - payload: { eventType: newEventType }, - }; - this.artifacts = []; - parentPort?.postMessage(message); - this.hasWorkerEmitted = true; - } catch (error) { - console.error( - `Error while emitting event with event type: ${newEventType}.`, - serializeError(error) - ); - parentPort?.postMessage(WorkerMessageSubject.WorkerMessageExit); - this.hasWorkerEmitted = true; - } - }); - } - - async uploadAllRepos(): Promise { - for (const repo of this.repos) { - const error = await repo.upload(); - this.artifacts.push(...repo.uploadedArtifacts); - if (error) { - throw error; - } - } - } - - async loadItemTypes({ - itemTypesToLoad, - }: ItemTypesToLoadParams): Promise { - return runWithSdkLogContext(async () => { - if (this.event.payload.event_type === EventType.StartLoadingData) { - const itemTypes = itemTypesToLoad.map( - (itemTypeToLoad) => itemTypeToLoad.itemType - ); - - if (!itemTypes.length) { - console.warn('No item types to load, returning.'); - return { - reports: this.reports, - processed_files: this.processedFiles, - }; - } - - const filesToLoad = await this.getLoaderBatches({ - supportedItemTypes: itemTypes, - }); - this.adapterState.state.fromDevRev = { - filesToLoad, - }; - } - - if ( - !this.adapterState.state.fromDevRev || - !this.adapterState.state.fromDevRev.filesToLoad.length - ) { - console.warn('No files to load, returning.'); - return { - reports: this.reports, - processed_files: this.processedFiles, - }; - } - - console.log( - 'Files to load in state', - this.adapterState.state.fromDevRev?.filesToLoad - ); - - try { - outerloop: for (const fileToLoad of this.adapterState.state.fromDevRev - .filesToLoad) { - const itemTypeToLoad = itemTypesToLoad.find( - (itemTypeToLoad: ItemTypeToLoad) => - itemTypeToLoad.itemType === fileToLoad.itemType - ); - - if (!itemTypeToLoad) { - console.error( - `Item type to load not found for item type: ${fileToLoad.itemType}.` - ); - - await this.emit(LoaderEventType.DataLoadingError, { - error: { - message: `Item type to load not found for item type: ${fileToLoad.itemType}.`, - }, - }); - - break; - } - - if (!fileToLoad.completed) { - const { response, error: transformerFileError } = - await this.uploader.getJsonObjectByArtifactId({ - artifactId: fileToLoad.id, - isGzipped: true, - }); - - if (transformerFileError) { - console.error( - `Transformer file not found for artifact ID: ${fileToLoad.id}.` - ); - await this.emit(LoaderEventType.DataLoadingError, { - error: { - message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`, - }, - }); - break outerloop; - } - - const transformerFile = response as ExternalSystemItem[]; - - for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) { - if (this.isTimeout) { - console.log( - 'Timeout detected during data loading. Emitting progress to allow continuation.' - ); - await this.emit(LoaderEventType.DataLoadingProgress); - process.exit(0); - } - - const { report, rateLimit } = await this.loadItem({ - item: transformerFile[i], - itemTypeToLoad, - }); - - if (rateLimit?.delay) { - await this.emit(LoaderEventType.DataLoadingDelayed, { - delay: rateLimit.delay, - reports: this.reports, - processed_files: this.processedFiles, - }); - - break outerloop; - } - - if (report) { - addReportToLoaderReport({ - loaderReports: this.loaderReports, - report, - }); - fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1; - } - } - - fileToLoad.completed = true; - this._processedFiles.push(fileToLoad.id); - } - } - } catch (error) { - console.error('Error during data loading.', serializeError(error)); - await this.emit(LoaderEventType.DataLoadingError, { - error: { - message: `Error during data loading. ${serializeError(error)}`, - }, - }); - process.exit(1); - } - - return { - reports: this.reports, - processed_files: this.processedFiles, - }; - }); - } - - async getLoaderBatches({ - supportedItemTypes, - }: { - supportedItemTypes: string[]; - }) { - return runWithSdkLogContext(async () => { - const statsFileArtifactId = this.event.payload.event_data?.stats_file; - - if (statsFileArtifactId) { - const { response, error: statsFileError } = - await this.uploader.getJsonObjectByArtifactId({ - artifactId: statsFileArtifactId, - }); - - const statsFile = response as StatsFileObject[]; - - if (statsFileError || statsFile.length === 0) { - return [] as FileToLoad[]; - } - - const filesToLoad = getFilesToLoad({ - supportedItemTypes, - statsFile, - }); - - return filesToLoad; - } - - return [] as FileToLoad[]; - }); - } - - async loadAttachments({ - create, - }: { - create: ExternalSystemLoadingFunction; - }): Promise { - return runWithSdkLogContext(async () => { - if (this.event.payload.event_type === EventType.StartLoadingAttachments) { - this.adapterState.state.fromDevRev = { - filesToLoad: await this.getLoaderBatches({ - supportedItemTypes: ['attachment'], - }), - }; - } - - if ( - !this.adapterState.state.fromDevRev || - this.adapterState.state.fromDevRev?.filesToLoad.length === 0 - ) { - console.log('No files to load, returning.'); - return { - reports: this.reports, - processed_files: this.processedFiles, - }; - } - - const filesToLoad = this.adapterState.state.fromDevRev?.filesToLoad; - - try { - outerloop: for (const fileToLoad of filesToLoad) { - if (!fileToLoad.completed) { - const { response, error: transformerFileError } = - await this.uploader.getJsonObjectByArtifactId({ - artifactId: fileToLoad.id, - isGzipped: true, - }); - - const transformerFile = response as ExternalSystemAttachment[]; - - if (transformerFileError) { - console.error( - `Transformer file not found for artifact ID: ${fileToLoad.id}.` - ); - await this.emit(LoaderEventType.AttachmentLoadingError, { - error: { - message: `Transformer file not found for artifact ID: ${fileToLoad.id}.`, - }, - }); - break outerloop; - } - - for (let i = fileToLoad.lineToProcess; i < fileToLoad.count; i++) { - if (this.isTimeout) { - console.log( - 'Timeout detected during attachment loading. Emitting progress to allow continuation.' - ); - await this.emit(LoaderEventType.AttachmentLoadingProgress); - process.exit(0); - } - - const { report, rateLimit } = await this.loadAttachment({ - item: transformerFile[i], - create, - }); - - if (rateLimit?.delay) { - await this.emit(LoaderEventType.AttachmentLoadingDelayed, { - delay: rateLimit.delay, - reports: this.reports, - processed_files: this.processedFiles, - }); - - break outerloop; - } - - if (report) { - addReportToLoaderReport({ - loaderReports: this.loaderReports, - report, - }); - fileToLoad.lineToProcess = fileToLoad.lineToProcess + 1; - } - } - - fileToLoad.completed = true; - this._processedFiles.push(fileToLoad.id); - } - } - } catch (error) { - console.error( - 'Error during attachment loading.', - serializeError(error) - ); - await this.emit(LoaderEventType.AttachmentLoadingError, { - error: { - message: `Error during attachment loading. ${serializeError( - error - )}`, - }, - }); - process.exit(1); - } - - return { - reports: this.reports, - processed_files: this.processedFiles, - }; - }); - } - - async loadItem({ - item, - itemTypeToLoad, - }: { - item: ExternalSystemItem; - itemTypeToLoad: ItemTypeToLoad; - }): Promise { - return runWithSdkLogContext(async () => { - const devrevId = item.id.devrev; - - try { - const syncMapperRecordResponse = await this._mappers.getByTargetId({ - sync_unit: this.event.payload.event_context.sync_unit, - target: devrevId, - }); - - const syncMapperRecord = syncMapperRecordResponse.data; - if (!syncMapperRecord) { - console.warn('Failed to get sync mapper record from response.'); - return { - error: { - message: 'Failed to get sync mapper record from response.', - }, - }; - } - - // Update item in external system - const { id, modifiedDate, delay, error } = await runWithUserLogContext( - async () => { - return await itemTypeToLoad.update({ - item, - mappers: this._mappers, - event: this.event, - }); - } - ); - - if (id) { - try { - const syncMapperRecordUpdateResponse = await this._mappers.update({ - id: syncMapperRecord.sync_mapper_record.id, - sync_unit: this.event.payload.event_context.sync_unit, - status: SyncMapperRecordStatus.OPERATIONAL, - ...(modifiedDate && { - external_versions: { - add: [ - { - modified_date: modifiedDate, - recipe_version: 0, - }, - ], - }, - }), - external_ids: { - add: [id], - }, - targets: { - add: [devrevId], - }, - }); - - console.log( - 'Successfully updated sync mapper record.', - syncMapperRecordUpdateResponse.data - ); - } catch (error) { - console.warn( - 'Failed to update sync mapper record.', - serializeError(error) - ); - return { - error: { - message: - 'Failed to update sync mapper record' + serializeError(error), - }, - }; - } - - return { - report: { - item_type: itemTypeToLoad.itemType, - [ActionType.UPDATED]: 1, - }, - }; - } else if (delay) { - console.log( - `Rate limited while updating item in external system, delaying for ${delay} seconds.` - ); - - return { - rateLimit: { - delay, - }, - }; - } else { - console.warn('Failed to update item in external system', error); - return { - report: { - item_type: itemTypeToLoad.itemType, - [ActionType.FAILED]: 1, - }, - }; - } - - // TODO: Update mapper (optional) - } catch (error) { - if (axios.isAxiosError(error)) { - if (error.response?.status === 404) { - // Create item in external system if mapper record not found - const { id, modifiedDate, delay, error } = - await runWithUserLogContext(async () => { - return await itemTypeToLoad.create({ - item, - mappers: this._mappers, - event: this.event, - }); - }); - - if (id) { - // Create mapper - try { - const syncMapperRecordCreateResponse = - await this._mappers.create({ - sync_unit: this.event.payload.event_context.sync_unit, - status: SyncMapperRecordStatus.OPERATIONAL, - external_ids: [id], - targets: [devrevId], - ...(modifiedDate && { - external_versions: [ - { - modified_date: modifiedDate, - recipe_version: 0, - }, - ], - }), - }); - - console.log( - 'Successfully created sync mapper record.', - syncMapperRecordCreateResponse.data - ); - - return { - report: { - item_type: itemTypeToLoad.itemType, - [ActionType.CREATED]: 1, - }, - }; - } catch (error) { - console.warn( - 'Failed to create sync mapper record.', - serializeError(error) - ); - return { - error: { - message: - 'Failed to create sync mapper record. ' + - serializeError(error), - }, - }; - } - } else if (delay) { - return { - rateLimit: { - delay, - }, - }; - } else { - console.warn( - 'Failed to create item in external system.', - serializeError(error) - ); - return { - report: { - item_type: itemTypeToLoad.itemType, - [ActionType.FAILED]: 1, - }, - }; - } - } else { - console.warn( - 'Failed to get sync mapper record.', - serializeError(error) - ); - return { - error: { - message: error.message, - }, - }; - } - } - - console.warn( - 'Failed to get sync mapper record.', - serializeError(error) - ); - return { - error: { - message: - 'Failed to get sync mapper record. ' + serializeError(error), - }, - }; - } - }); - } - - async processAttachment( - attachment: NormalizedAttachment, - stream: ExternalSystemAttachmentStreamingFunction - ): Promise { - return runWithSdkLogContext(async () => { - const { httpStream, delay, error } = await runWithUserLogContext( - async () => - stream({ - item: attachment, - event: this.event, - }) - ); - - if (error) { - return { error }; - } else if (delay) { - return { delay }; - } - - if (httpStream) { - const fileType = - attachment.content_type || - httpStream.headers['content-type']?.toString() || - 'application/octet-stream'; - const contentLength = httpStream.headers['content-length']?.toString(); - const fileSize = contentLength ? parseInt(contentLength) : undefined; - - // Get upload URL - const { error: artifactUrlError, response: artifactUrlResponse } = - await this.uploader.getArtifactUploadUrl( - attachment.file_name, - fileType, - fileSize - ); - - if (artifactUrlError) { - this.destroyHttpStream(httpStream); - return { - error: { - message: `Error while preparing artifact for attachment ID ${ - attachment.id - }. Skipping attachment. ${serializeError(artifactUrlError)}`, - fileSize: fileSize, - }, - }; - } - - // Stream attachment - const { error: uploadedArtifactError } = - await this.uploader.streamArtifact(artifactUrlResponse!, httpStream); - - if (uploadedArtifactError) { - this.destroyHttpStream(httpStream); - return { - error: { - message: - `Error while streaming to artifact for attachment ID ${attachment.id}. Skipping attachment. ` + - serializeError(uploadedArtifactError), - fileSize: fileSize, - }, - }; - } - - // Confirm attachment upload - const { error: confirmArtifactUploadError } = - await this.uploader.confirmArtifactUpload( - artifactUrlResponse!.artifact_id - ); - if (confirmArtifactUploadError) { - return { - error: { - message: - `Error while confirming upload for attachment ID ${attachment.id}. ` + - serializeError(confirmArtifactUploadError), - fileSize: fileSize, - }, - }; - } - - const ssorAttachment: SsorAttachment = { - id: { - devrev: artifactUrlResponse!.artifact_id, - external: attachment.id, - }, - parent_id: { - external: attachment.parent_id, - }, - }; - - if (attachment.author_id) { - ssorAttachment.actor_id = { - external: attachment.author_id, - }; - } - - // This will set inline flag in ssor_attachment only if it is explicity - // set in the attachment object. - if (attachment.inline === true) { - ssorAttachment.inline = true; - } else if (attachment.inline === false) { - ssorAttachment.inline = false; - } - - if (this.isTimeout) { - this.destroyHttpStream(httpStream); - return; - } - - await this.getRepo('ssor_attachment')?.push([ssorAttachment]); - return; - } - return { - error: { - message: `Error while opening attachment stream. Skipping attachment.`, - }, - }; - }); - } - - /** - * Destroys a stream to prevent memory leaks. - * @param httpStream - The axios response stream to destroy - */ - private destroyHttpStream(httpStream: AxiosResponse): void { - try { - if (httpStream && httpStream.data) { - if (typeof httpStream.data.destroy === 'function') { - httpStream.data.destroy(); - } else if (typeof httpStream.data.close === 'function') { - httpStream.data.close(); - } - } - } catch (error) { - console.warn('Error while destroying HTTP stream:', error); - } - } - - async loadAttachment({ - item, - create, - }: { - item: ExternalSystemAttachment; - create: ExternalSystemLoadingFunction; - }): Promise { - return runWithSdkLogContext(async () => { - // Create item - const { id, delay, error } = await runWithUserLogContext(async () => - create({ - item, - mappers: this._mappers, - event: this.event, - }) - ); - - if (delay) { - return { - rateLimit: { - delay, - }, - }; - } else if (id) { - try { - const syncMapperRecordCreateResponse = await this._mappers.create({ - sync_unit: this.event.payload.event_context.sync_unit, - external_ids: [id], - targets: [item.reference_id], - status: SyncMapperRecordStatus.OPERATIONAL, - }); - - console.log( - 'Successfully created sync mapper record.', - syncMapperRecordCreateResponse.data - ); - } catch (error) { - console.warn( - 'Failed to create sync mapper record.', - serializeError(error) - ); - } - - return { - report: { - item_type: 'attachments', - [ActionType.CREATED]: 1, - }, - }; - } else { - console.warn('Failed to create attachment in external system', error); - return { - report: { - item_type: 'attachments', - [ActionType.FAILED]: 1, - }, - }; - } - }); - } - - /** - * Streams the attachments to the DevRev platform. - * The attachments are streamed to the platform and the artifact information is returned. - * @param params - The parameters to stream the attachments - * @returns The response object containing the ssorAttachment artifact information - * or error information if there was an error - */ - async streamAttachments({ - stream, - processors, - batchSize = 1, // By default, we want to stream one attachment at a time - }: { - stream: ExternalSystemAttachmentStreamingFunction; - processors?: ExternalSystemAttachmentProcessors< - ConnectorState, - NormalizedAttachment[], - NewBatch - >; - batchSize?: number; - }): Promise { - return runWithSdkLogContext(async () => { - if (batchSize <= 0) { - console.warn( - `The specified batch size (${batchSize}) is invalid. Using 1 instead.` - ); - batchSize = 1; - } - - if (batchSize > 50) { - console.warn( - `The specified batch size (${batchSize}) is too large. Using 50 instead.` - ); - batchSize = 50; - } - - const repos = [ - { - itemType: 'ssor_attachment', - }, - ]; - this.initializeRepos(repos); - - const attachmentsMetadata = this.state.toDevRev?.attachmentsMetadata; - - // If there are no attachments metadata artifact IDs in state, finish here - if (!attachmentsMetadata?.artifactIds?.length) { - console.log(`No attachments metadata artifact IDs found in state.`); - return; - } else { - console.log( - `Found ${attachmentsMetadata.artifactIds.length} attachments metadata artifact IDs in state.` - ); - } - - // Loop through the attachments metadata artifact IDs - while (attachmentsMetadata.artifactIds.length > 0) { - const attachmentsMetadataArtifactId = - attachmentsMetadata.artifactIds[0]; - - console.log( - `Started processing attachments for attachments metadata artifact ID: ${attachmentsMetadataArtifactId}.` - ); - - const { attachments, error } = - await this.uploader.getAttachmentsFromArtifactId({ - artifact: attachmentsMetadataArtifactId, - }); - - if (error) { - console.error( - `Failed to get attachments for artifact ID: ${attachmentsMetadataArtifactId}.` - ); - return { error }; - } - - if (!attachments || attachments.length === 0) { - console.warn( - `No attachments found for artifact ID: ${attachmentsMetadataArtifactId}.` - ); - // Remove empty artifact and reset lastProcessed - attachmentsMetadata.artifactIds.shift(); - attachmentsMetadata.lastProcessed = 0; - continue; - } - - console.log( - `Found ${attachments.length} attachments for artifact ID: ${attachmentsMetadataArtifactId}.` - ); - - let response; - - if (processors) { - console.log(`Using custom processors for attachments.`); - - const reducer = processors.reducer; - const iterator = processors.iterator; - - const reducedAttachments = runWithUserLogContext(() => - reducer({ - attachments, - adapter: this, - batchSize, - }) - ); - - response = await runWithUserLogContext(async () => { - return await iterator({ - reducedAttachments, - adapter: this, - stream, - }); - }); - } else { - console.log( - `Using attachments streaming pool for attachments streaming.` - ); - - const attachmentsPool = new AttachmentsStreamingPool({ - adapter: this, - attachments, - batchSize, - stream, - }); - - response = await attachmentsPool.streamAll(); - } - - if (response?.delay || response?.error) { - return response; - } - - // On timeout, emit progress and exit to allow continuation. - if (this.isTimeout) { - console.log( - `Timeout detected after processing attachments for artifact ID: ${attachmentsMetadataArtifactId}. Emitting progress to allow continuation.` - ); - await this.emit(ExtractorEventType.AttachmentExtractionProgress); - process.exit(0); - return; - } - - console.log( - `Finished processing all attachments for artifact ID: ${attachmentsMetadataArtifactId}.` - ); - attachmentsMetadata.artifactIds.shift(); - attachmentsMetadata.lastProcessed = 0; - if (attachmentsMetadata.lastProcessedAttachmentsIdsList) { - attachmentsMetadata.lastProcessedAttachmentsIdsList.length = 0; - } - } - - return; - }); - } -} diff --git a/src/multithreading/worker.js b/src/multithreading/worker.js index f19e44c6..d5abf69e 100644 --- a/src/multithreading/worker.js +++ b/src/multithreading/worker.js @@ -3,10 +3,7 @@ const { workerData } = require('node:worker_threads'); require('ts-node').register(); const { Logger } = require('../logger/logger'); -const { runWithUserLogContext } = require('../logger/logger.context'); console = new Logger({ event: workerData.event, options: workerData.options }); -runWithUserLogContext(() => { - require(workerData.workerPath); -}); +require(workerData.workerPath); diff --git a/src/repo/repo.interfaces.ts b/src/repo/repo.interfaces.ts index a3747811..1ccd3168 100644 --- a/src/repo/repo.interfaces.ts +++ b/src/repo/repo.interfaces.ts @@ -1,31 +1,22 @@ -import { Artifact } from '../uploader/uploader.interfaces'; - -import { AirdropEvent } from '../types/extraction'; +import { AirSyncEvent } from '../types/extraction'; import { WorkerAdapterOptions } from '../types/workers'; +import { Artifact } from '../uploader/uploader.interfaces'; -/** - * RepoInterface is an interface that defines the structure of a repo which is used to store and upload extracted data. - */ +/** Stores and uploads extracted data. */ export interface RepoInterface { itemType: string; normalize?: (record: object) => NormalizedItem | NormalizedAttachment; overridenOptions?: WorkerAdapterOptions; } -/** - * RepoFactoryInterface is an interface that defines the structure of a repo factory which is used to create a repo. - */ export interface RepoFactoryInterface { - event: AirdropEvent; + event: AirSyncEvent; itemType: string; normalize?: (record: object) => NormalizedItem | NormalizedAttachment; onUpload: (artifact: Artifact) => void; options?: WorkerAdapterOptions; } -/** - * NormalizedItem is an interface of item after normalization. - */ export interface NormalizedItem { id: string; created_date: string; @@ -33,9 +24,6 @@ export interface NormalizedItem { data: object; } -/** - * NormalizedAttachment is an interface of attachment after normalization. - */ export interface NormalizedAttachment { url: string; id: string; @@ -47,13 +35,9 @@ export interface NormalizedAttachment { created_date?: string; modified_date?: string; - // This should be a string, but it was a number in the past. Due to backwards - // compatibility we are keeping it also as a number. + // number kept only for backwards compatibility; should be a string grand_parent_id?: number | string; } -/** - * Item is an interface that defines the structure of an item. - */ // eslint-disable-next-line @typescript-eslint/no-explicit-any export type Item = Record; diff --git a/src/repo/repo.test.ts b/src/repo/repo.test.ts index cfebef42..7282a343 100644 --- a/src/repo/repo.test.ts +++ b/src/repo/repo.test.ts @@ -1,9 +1,9 @@ import { AirSyncDefaultItemTypes, SSOR_ATTACHMENT } from '../common/constants'; -import { createItems, normalizeItem } from '../tests/test-helpers'; +import { createMockEvent } from '../testing/mock-event'; import { mockServer } from '../tests/jest.setup'; -import { createMockEvent } from '../common/test-utils'; -import { EventType } from '../types'; -import { NormalizedAttachment, NormalizedItem } from './repo.interfaces'; +import { createItems, normalizeItem } from '../tests/test-helpers'; +import { EventType } from '../types/extraction'; + import { Repo } from './repo'; jest.mock('../tests/test-helpers', () => ({ @@ -11,31 +11,6 @@ jest.mock('../tests/test-helpers', () => ({ normalizeItem: jest.fn(), })); -const mockUploadFn = jest.fn().mockResolvedValue({ - error: null, - artifact: { id: 'art-1', item_type: 'test', item_count: 0 }, -}); - -jest.mock('../uploader/uploader', () => ({ - Uploader: jest.fn().mockImplementation(() => ({ - upload: mockUploadFn, - })), -})); - -function itemWithDate(id: string, created_date: string): NormalizedItem { - return { id, created_date, modified_date: created_date, data: {} }; -} - -function itemWithDates( - id: string, - created_date: string, - modified_date: string -): NormalizedItem { - return { id, created_date, modified_date, data: {} }; -} - -const ts = (iso: string) => new Date(iso).getTime(); - describe(Repo.name, () => { let repo: Repo; let normalize: jest.Mock; @@ -44,7 +19,7 @@ describe(Repo.name, () => { normalize = jest.fn(); repo = new Repo({ event: createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.ExtractionDataStart }, + payload: { event_type: EventType.StartExtractingData }, }), itemType: 'test_item_type', normalize, @@ -73,7 +48,7 @@ describe(Repo.name, () => { // Arrange repo = new Repo({ event: createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.ExtractionDataStart }, + payload: { event_type: EventType.StartExtractingData }, }), itemType: 'test_item_type', onUpload: jest.fn(), @@ -100,7 +75,7 @@ describe(Repo.name, () => { // Arrange repo = new Repo({ event: createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.ExtractionDataStart }, + payload: { event_type: EventType.StartExtractingData }, }), itemType: AirSyncDefaultItemTypes.EXTERNAL_DOMAIN_METADATA, normalize, @@ -120,7 +95,7 @@ describe(Repo.name, () => { // Arrange repo = new Repo({ event: createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.ExtractionDataStart }, + payload: { event_type: EventType.StartExtractingData }, }), itemType: SSOR_ATTACHMENT, normalize, @@ -186,7 +161,7 @@ describe(Repo.name, () => { beforeEach(() => { repo = new Repo({ event: createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.ExtractionDataStart }, + payload: { event_type: EventType.StartExtractingData }, }), itemType: 'test_item_type', normalize, @@ -254,243 +229,4 @@ describe(Repo.name, () => { expect(repo.getItems().length).toBe(5); }); }); - - describe('dateRanges', () => { - beforeEach(() => { - mockUploadFn.mockResolvedValue({ - error: null, - artifact: { id: 'art-1', item_type: 'test', item_count: 0 }, - }); - }); - - it('should track min and max created_date from a single upload batch', async () => { - await repo.upload([ - itemWithDate('1', '2023-06-15T12:00:00.000Z'), - itemWithDate('2', '2020-01-01T00:00:00.000Z'), - itemWithDate('3', '2021-03-01T00:00:00.000Z'), - ]); - - expect(repo.dateRanges.creationDate.oldest).toBe( - ts('2020-01-01T00:00:00.000Z') - ); - expect(repo.dateRanges.creationDate.newest).toBe( - ts('2023-06-15T12:00:00.000Z') - ); - expect(repo.dateRanges.modifiedDate.oldest).toBe( - ts('2020-01-01T00:00:00.000Z') - ); - expect(repo.dateRanges.modifiedDate.newest).toBe( - ts('2023-06-15T12:00:00.000Z') - ); - }); - - it('should skip items without created_date', async () => { - const attachmentWithoutDate: NormalizedAttachment = { - id: 'att-1', - url: 'https://example.com/file', - file_name: 'file.txt', - parent_id: 'parent-1', - }; - - await repo.upload([ - itemWithDate('1', '2022-06-01T00:00:00.000Z'), - { id: '2', created_date: null, modified_date: '', data: {} }, - { id: '3', modified_date: '', data: {} }, - attachmentWithoutDate, - ]); - - expect(repo.dateRanges.creationDate.oldest).toBe( - ts('2022-06-01T00:00:00.000Z') - ); - expect(repo.dateRanges.creationDate.newest).toBe( - ts('2022-06-01T00:00:00.000Z') - ); - }); - - it('should leave date ranges unset when no items have created_date', async () => { - await repo.upload([ - { id: '1', modified_date: '', data: {} }, - { - id: 'att-1', - url: 'https://example.com/file', - file_name: 'file.txt', - parent_id: 'parent-1', - }, - ]); - - expect(repo.dateRanges).toEqual({ - creationDate: {}, - modifiedDate: {}, - }); - }); - - it('should not update timestamps or call uploader on empty upload', async () => { - await repo.upload([]); - - expect(repo.dateRanges).toEqual({ - creationDate: {}, - modifiedDate: {}, - }); - expect(mockUploadFn).not.toHaveBeenCalled(); - }); - - it('should track an item dated exactly at the Unix epoch instead of treating it as unset', async () => { - await repo.upload([itemWithDate('1', '1970-01-01T00:00:00.000Z')]); - - expect(repo.dateRanges.creationDate).toEqual({ oldest: 0, newest: 0 }); - - await repo.upload([itemWithDate('2', '2020-01-01T00:00:00.000Z')]); - - expect(repo.dateRanges.creationDate).toEqual({ - oldest: 0, - newest: ts('2020-01-01T00:00:00.000Z'), - }); - }); - - it('should accumulate min and max across multiple upload batches via push', async () => { - repo = new Repo({ - event: createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.ExtractionDataStart }, - }), - itemType: 'test_item_type', - onUpload: jest.fn(), - options: { batchSize: 3 }, - }); - - const dates = [ - '2020-01-01T00:00:00.000Z', - '2021-01-01T00:00:00.000Z', - '2022-01-01T00:00:00.000Z', - '2023-01-01T00:00:00.000Z', - '2024-06-01T00:00:00.000Z', - '2024-12-01T00:00:00.000Z', - '2024-12-31T00:00:00.000Z', - ]; - const items = dates.map((created_date, index) => - itemWithDate(String(index), created_date) - ); - - await repo.push(items); - - expect(repo.getItems()).toHaveLength(1); - expect(repo.dateRanges.creationDate.oldest).toBe( - ts('2020-01-01T00:00:00.000Z') - ); - expect(repo.dateRanges.creationDate.newest).toBe( - ts('2024-12-01T00:00:00.000Z') - ); - - await repo.push([ - itemWithDate('7', '2019-01-01T00:00:00.000Z'), - itemWithDate('8', '2025-01-01T00:00:00.000Z'), - ]); - - expect(repo.getItems()).toHaveLength(0); - expect(repo.dateRanges.creationDate.oldest).toBe( - ts('2019-01-01T00:00:00.000Z') - ); - expect(repo.dateRanges.creationDate.newest).toBe( - ts('2025-01-01T00:00:00.000Z') - ); - }); - - it('should extend min and max when subsequent batches have wider date range', async () => { - await repo.upload([ - itemWithDate('1', '2022-06-01T00:00:00.000Z'), - itemWithDate('2', '2023-06-01T00:00:00.000Z'), - ]); - - expect(repo.dateRanges.creationDate.oldest).toBe( - ts('2022-06-01T00:00:00.000Z') - ); - expect(repo.dateRanges.creationDate.newest).toBe( - ts('2023-06-01T00:00:00.000Z') - ); - - await repo.upload([ - itemWithDate('3', '2020-01-01T00:00:00.000Z'), - itemWithDate('4', '2024-01-01T00:00:00.000Z'), - ]); - - expect(repo.dateRanges.creationDate.oldest).toBe( - ts('2020-01-01T00:00:00.000Z') - ); - expect(repo.dateRanges.creationDate.newest).toBe( - ts('2024-01-01T00:00:00.000Z') - ); - }); - - it('should update timestamps even when upload fails', async () => { - mockUploadFn.mockResolvedValueOnce({ - error: new Error('fail'), - artifact: null, - }); - - await repo.upload([itemWithDate('1', '2022-01-01T00:00:00.000Z')]); - - expect(repo.dateRanges.creationDate.oldest).toBe( - ts('2022-01-01T00:00:00.000Z') - ); - expect(repo.dateRanges.creationDate.newest).toBe( - ts('2022-01-01T00:00:00.000Z') - ); - }); - - it('should ignore invalid created_date and modified_date values', async () => { - await repo.upload([ - { - id: '1', - created_date: 'not-a-date', - modified_date: 'still-not-a-date', - data: {}, - }, - itemWithDates( - '2', - '2022-01-01T00:00:00.000Z', - '2023-01-01T00:00:00.000Z' - ), - ]); - - expect(repo.dateRanges.creationDate.oldest).toBe( - ts('2022-01-01T00:00:00.000Z') - ); - expect(repo.dateRanges.creationDate.newest).toBe( - ts('2022-01-01T00:00:00.000Z') - ); - expect(repo.dateRanges.modifiedDate.oldest).toBe( - ts('2023-01-01T00:00:00.000Z') - ); - expect(repo.dateRanges.modifiedDate.newest).toBe( - ts('2023-01-01T00:00:00.000Z') - ); - }); - - it('should track modified_date independently from created_date', async () => { - await repo.upload([ - itemWithDates( - '1', - '2020-01-01T00:00:00.000Z', - '2023-01-01T00:00:00.000Z' - ), - itemWithDates( - '2', - '2024-01-01T00:00:00.000Z', - '2021-06-01T00:00:00.000Z' - ), - ]); - - expect(repo.dateRanges.creationDate.oldest).toBe( - ts('2020-01-01T00:00:00.000Z') - ); - expect(repo.dateRanges.creationDate.newest).toBe( - ts('2024-01-01T00:00:00.000Z') - ); - expect(repo.dateRanges.modifiedDate.oldest).toBe( - ts('2021-06-01T00:00:00.000Z') - ); - expect(repo.dateRanges.modifiedDate.newest).toBe( - ts('2023-01-01T00:00:00.000Z') - ); - }); - }); }); diff --git a/src/repo/repo.ts b/src/repo/repo.ts index 2ae1ee85..33dcf9f6 100644 --- a/src/repo/repo.ts +++ b/src/repo/repo.ts @@ -5,17 +5,16 @@ import { } from '../common/constants'; import { Item } from '../repo/repo.interfaces'; import { ErrorRecord } from '../types/common'; +import { WorkerAdapterOptions } from '../types/workers'; import { Uploader } from '../uploader/uploader'; import { Artifact } from '../uploader/uploader.interfaces'; -import { WorkerAdapterOptions } from '../types/workers'; -import { runWithUserLogContext } from '../logger/logger.context'; +import { toValidTimestamp, updateRange } from './repo.helpers'; import { NormalizedAttachment, NormalizedItem, RepoFactoryInterface, } from './repo.interfaces'; -import { updateRange, toValidTimestamp } from './repo.helpers'; export class Repo { readonly itemType: string; @@ -60,14 +59,14 @@ export class Repo { if (itemsToUpload.length > 0) { for (const item of itemsToUpload) { - const createdDate = item?.created_date; + const createdDate = (item as NormalizedItem)?.created_date; if (createdDate != null) { const createdMs = toValidTimestamp(createdDate); if (createdMs !== undefined) { updateRange(this.dateRanges.creationDate, createdMs); } } - const modifiedDate = item?.modified_date; + const modifiedDate = (item as NormalizedItem)?.modified_date; if (modifiedDate != null && modifiedDate !== '') { const modifiedMs = toValidTimestamp(modifiedDate); if (modifiedMs !== undefined) { @@ -94,7 +93,7 @@ export class Repo { this.uploadedArtifacts.push(artifact); - // Clear the uploaded items from the main items array if no batch was specified + // An explicit batch was already spliced out of this.items by the caller if (!batch) { this.items = []; } @@ -117,30 +116,23 @@ export class Repo { return true; } - // Normalize items if needed if ( this.normalize && this.itemType != AirSyncDefaultItemTypes.EXTERNAL_DOMAIN_METADATA && this.itemType != SSOR_ATTACHMENT ) { - recordsToPush = runWithUserLogContext(() => - items.map((item: Item) => this.normalize!(item)) - ); + recordsToPush = items.map((item: Item) => this.normalize!(item)); } else { recordsToPush = items; } - // Add the new records to the items array this.items.push(...recordsToPush); - // Upload in batches while the number of items exceeds the batch size const batchSize = this.options?.batchSize || ARTIFACT_BATCH_SIZE; while (this.items.length >= batchSize) { - // Slice out a batch of batchSize items to upload const batch = this.items.splice(0, batchSize); try { - // Upload the batch await this.upload(batch); } catch (error) { console.error('Error while uploading batch', error); diff --git a/src/state/base-state.ts b/src/state/base-state.ts new file mode 100644 index 00000000..87e1afcf --- /dev/null +++ b/src/state/base-state.ts @@ -0,0 +1,302 @@ +import { parentPort } from 'node:worker_threads'; + +import axios from 'axios'; + +import { axiosClient } from '../http/client'; +import { getPrintableState, serializeError } from '../logger/logger'; +import { InitialDomainMapping } from '../types/common'; +import { AirSyncEvent } from '../types/extraction'; +import { WorkerMessageSubject } from '../types/workers'; +import { ExtractionScope } from '../types/workers'; + +import { installInitialDomainMapping } from './install-initial-domain-mapping'; +import { + AdapterStateEnvelope, + SdkState, + StateInterface, + V1_SDK_STATE_KEYS, +} from './state.interfaces'; + +/** + * State lifecycle shared by all sync modes: connector/SDK state separation, + * fetch/init/post, the v1->v2 migration shim, and the snap-in-version-gated + * initial domain mapping install. Subclasses seed the SDK-owned state. + */ +export abstract class BaseState { + protected _connectorState: ConnectorState; + protected _sdkState: SdkState; + protected _extractionScope: ExtractionScope = {}; + protected readonly initialSdkState: SdkState; + protected readonly event: AirSyncEvent; + private workerUrl: string; + private devrevToken: string; + private syncUnitId: string; + private requestId: string; + + constructor( + { event, initialState }: StateInterface, + initialSdkState: SdkState + ) { + this.event = event; + this.initialSdkState = initialSdkState; + this._connectorState = initialState; + this._sdkState = { ...initialSdkState }; + this.workerUrl = event.payload.event_context.worker_data_url; + this.devrevToken = event.context.secrets.service_account_token; + this.syncUnitId = event.payload.event_context.sync_unit_id; + this.requestId = event.payload.event_context.request_id_adaas; + } + + /** Connector-owned state; what `adapter.state` exposes to snap-in code. */ + get state(): ConnectorState { + return this._connectorState; + } + + set state(value: ConnectorState) { + this._connectorState = value; + } + + /** SDK-internal bookkeeping state. Never exposed to connector code. */ + get sdkState(): SdkState { + return this._sdkState; + } + + set sdkState(value: SdkState) { + this._sdkState = value; + } + + get extractionScope(): ExtractionScope { + return this._extractionScope; + } + + /** + * Installs the initial domain mapping when the snap-in version in state + * differs from the event context. Shared by all modes so a loading run still + * installs the mapping if extraction has not done so. + */ + async installInitialDomainMappingIfNeeded( + initialDomainMapping?: InitialDomainMapping + ): Promise { + const snapInVersionId = this.event.context.snap_in_version_id; + const hasSnapInVersionInState = 'snapInVersionId' in this.sdkState; + const shouldUpdateIDM = + !hasSnapInVersionInState || + this.sdkState.snapInVersionId !== snapInVersionId; + + if (!shouldUpdateIDM) { + console.log( + `Snap-in version in state matches the version in event context "${snapInVersionId}". Skipping initial domain mapping installation.` + ); + return; + } + + try { + console.log( + `Snap-in version in state "${this.sdkState.snapInVersionId}" does not match the version in event context "${snapInVersionId}". Installing initial domain mapping.` + ); + + if (initialDomainMapping) { + await installInitialDomainMapping(this.event, initialDomainMapping); + this.sdkState.snapInVersionId = snapInVersionId; + } else { + throw new Error( + 'No initial domain mapping was passed to spawn function. Skipping initial domain mapping installation.' + ); + } + } catch (error) { + const errorMessage = `Error while installing initial domain mapping. ${serializeError( + error + )}`; + console.error(errorMessage); + parentPort?.postMessage({ + subject: WorkerMessageSubject.WorkerMessageFailed, + payload: { message: errorMessage }, + }); + process.exit(1); + } + } + + /** + * Fetches state from API, or creates the initial state on 404. Reads both + * the v2 envelope and a legacy flat v1 blob (migrated on read); always + * persists the v2 envelope going forward. + */ + async init(initialState: ConnectorState): Promise { + try { + const { state: stringifiedState, objects } = await this.fetchState(); + if (!stringifiedState) { + throw new Error('No state found in response.'); + } + + let parsed: unknown; + try { + parsed = JSON.parse(stringifiedState); + } catch (error) { + throw new Error(`Failed to parse state. ${error}`); + } + + const { connectorState, sdkState } = this.normalizeFetchedState(parsed); + this.state = connectorState; + this.sdkState = sdkState; + + console.log('State fetched successfully. Current state', { + connectorState: getPrintableState( + this.state as Record + ), + sdkState: getPrintableState(this.sdkState), + }); + + if (objects) { + try { + this._extractionScope = JSON.parse(objects); + } catch (error) { + console.warn(`Failed to parse extractionScope. ${error}`); + } + } + } catch (error) { + if (axios.isAxiosError(error) && error.response?.status === 404) { + console.log('State not found. Initializing state with initial state.'); + this.state = initialState; + this.sdkState = { ...this.initialSdkState }; + await this.postState(); + } else { + const errorMessage = `Failed to init state. ${serializeError(error)}`; + console.error(errorMessage); + parentPort?.postMessage({ + subject: WorkerMessageSubject.WorkerMessageFailed, + payload: { message: errorMessage }, + }); + process.exit(1); + } + } + } + + /** + * Normalizes fetched state into the `{ connectorState, sdkState }` envelope. + * A flat v1 blob is split by `V1_SDK_STATE_KEYS`; an envelope with only one + * side present fails loud. + */ + private normalizeFetchedState(parsed: unknown): { + connectorState: ConnectorState; + sdkState: SdkState; + } { + if (parsed === null || typeof parsed !== 'object') { + throw new Error('Fetched state is not a JSON object.'); + } + + const record = parsed as Record; + const hasConnector = 'connectorState' in record; + const hasSdk = 'sdkState' in record; + + if (hasConnector || hasSdk) { + if (!hasConnector || !hasSdk) { + throw new Error( + 'Malformed state envelope: expected both "connectorState" and "sdkState".' + ); + } + return { + connectorState: record.connectorState as ConnectorState, + sdkState: { ...this.initialSdkState, ...(record.sdkState as SdkState) }, + }; + } + + // Legacy flat v1 blob: split known SDK keys out of the connector state. + const connectorState: Record = {}; + const sdkState: Record = {}; + for (const [key, value] of Object.entries(record)) { + if (V1_SDK_STATE_KEYS.has(key)) { + sdkState[key] = value; + } else { + connectorState[key] = value; + } + } + + return { + connectorState: connectorState as ConnectorState, + sdkState: { ...this.initialSdkState, ...(sdkState as SdkState) }, + }; + } + + /** Posts the v2 `{ connectorState, sdkState }` envelope to the API. */ + async postState(state?: ConnectorState) { + const url = this.workerUrl + '.update'; + this.state = state || this.state; + + const envelope: AdapterStateEnvelope = { + connectorState: this.state, + sdkState: this.sdkState, + }; + + let stringifiedState: string; + try { + stringifiedState = JSON.stringify(envelope); + } catch (error) { + const errorMessage = `Failed to stringify state. ${serializeError( + error + )}`; + console.error(errorMessage); + parentPort?.postMessage({ + subject: WorkerMessageSubject.WorkerMessageFailed, + payload: { message: errorMessage }, + }); + process.exit(1); + } + + try { + await axiosClient.post( + url, + { + state: stringifiedState, + }, + { + headers: { + Authorization: this.devrevToken, + }, + params: { + sync_unit: this.syncUnitId, + request_id: this.requestId, + }, + } + ); + + console.log('State updated successfully to', { + connectorState: getPrintableState( + this.state as Record + ), + sdkState: getPrintableState(this.sdkState), + }); + } catch (error) { + const errorMessage = `Failed to update the state. ${serializeError( + error + )}`; + console.error(errorMessage); + parentPort?.postMessage({ + subject: WorkerMessageSubject.WorkerMessageFailed, + payload: { message: errorMessage }, + }); + process.exit(1); + } + } + + async fetchState(): Promise<{ state: string; objects?: string }> { + console.log( + `Fetching state with sync unit id ${this.syncUnitId} and request id ${this.requestId}.` + ); + + const url = this.workerUrl + '.get'; + const response = await axiosClient.get(url, { + headers: { + Authorization: this.devrevToken, + }, + params: { + sync_unit: this.syncUnitId, + request_id: this.requestId, + }, + }); + + return { + state: response.data?.state, + objects: response.data?.objects, + }; + } +} diff --git a/src/state/extraction-state.ts b/src/state/extraction-state.ts new file mode 100644 index 00000000..821ef142 --- /dev/null +++ b/src/state/extraction-state.ts @@ -0,0 +1,125 @@ +import { parentPort } from 'node:worker_threads'; + +import { STATELESS_EVENT_TYPES } from '../common/constants'; +import { serializeError } from '../logger/logger'; +import { EventType } from '../types/extraction'; +import { WorkerMessageSubject } from '../types/workers'; + +import { BaseState } from './base-state'; +import { extractionSdkState, StateInterface } from './state.interfaces'; +import { resolveTimeValue } from './time-value-resolver'; + +/** State for extraction workers: seeds extraction SDK state and adds extraction-window resolution. */ +export class ExtractionState extends BaseState { + constructor(params: StateInterface) { + super(params, extractionSdkState); + } + + /** + * Resolves the extraction window onto the event context. StartExtractingMetadata + * resolves fresh from the event's TimeValues and overwrites the pending boundaries; + * all other events reuse those cached boundaries. Validates extract_from < extract_to. + */ + resolveExtractionWindow(): void { + const sdkState = this.sdkState; + + const eventContext = this.event.payload.event_context; + + if (this.event.payload.event_type === EventType.StartExtractingMetadata) { + const timeFields = [ + { + source: 'extraction_start_time', + target: 'extract_from', + pending: 'pendingWorkersOldest', + }, + { + source: 'extraction_end_time', + target: 'extract_to', + pending: 'pendingWorkersNewest', + }, + ] as const; + + for (const { source, target, pending } of timeFields) { + const timeValue = eventContext[source]; + if (timeValue && timeValue.type) { + try { + const resolved = resolveTimeValue(timeValue, sdkState); + eventContext[target] = resolved; + sdkState[pending] = resolved; + console.log( + `Resolved ${target} to ${resolved}. Stored in ${pending}.` + ); + } catch (error) { + const errorMessage = `Failed to resolve ${source}: ${serializeError( + error + )}`; + console.error(errorMessage); + parentPort?.postMessage({ + subject: WorkerMessageSubject.WorkerMessageFailed, + payload: { message: errorMessage }, + }); + process.exit(1); + } + } + } + } else { + if (sdkState.pendingWorkersOldest) { + eventContext.extract_from = sdkState.pendingWorkersOldest; + console.log( + `Reusing pendingWorkersOldest as extract_from: ${sdkState.pendingWorkersOldest}.` + ); + } else { + console.log( + 'pendingWorkersOldest is not set in state. extract_from will not be populated for this invocation.' + ); + } + if (sdkState.pendingWorkersNewest) { + eventContext.extract_to = sdkState.pendingWorkersNewest; + console.log( + `Reusing pendingWorkersNewest as extract_to: ${sdkState.pendingWorkersNewest}.` + ); + } else { + console.log( + 'pendingWorkersNewest is not set in state. extract_to will not be populated for this invocation.' + ); + } + } + + if (eventContext.extract_from && eventContext.extract_to) { + if (eventContext.extract_from >= eventContext.extract_to) { + const errorMessage = `Invalid extraction window: extract_from (${eventContext.extract_from}) must be older than extract_to (${eventContext.extract_to}). This indicates an error in the platform.`; + console.error(errorMessage); + parentPort?.postMessage({ + subject: WorkerMessageSubject.WorkerMessageFailed, + payload: { message: errorMessage }, + }); + process.exit(1); + } + } + } +} + +export async function createExtractionState({ + event, + initialState, + initialDomainMapping, + options, +}: StateInterface): Promise> { + // Clone so the caller's initialState is never mutated. + const deepCloneInitialState: ConnectorState = structuredClone(initialState); + + const state = new ExtractionState({ + event, + initialState: deepCloneInitialState, + initialDomainMapping, + options, + }); + + if (!STATELESS_EVENT_TYPES.includes(event.payload.event_type)) { + await state.init(deepCloneInitialState); + await state.installInitialDomainMappingIfNeeded(initialDomainMapping); + state.resolveExtractionWindow(); + } + + return state; +} diff --git a/src/common/install-initial-domain-mapping.test.ts b/src/state/install-initial-domain-mapping.test.ts similarity index 95% rename from src/common/install-initial-domain-mapping.test.ts rename to src/state/install-initial-domain-mapping.test.ts index ec56b02c..ce0a9411 100644 --- a/src/common/install-initial-domain-mapping.test.ts +++ b/src/state/install-initial-domain-mapping.test.ts @@ -1,9 +1,11 @@ import axios from 'axios'; -import { axiosClient } from '../http/axios-client-internal'; + +import { axiosClient } from '../http/client'; +import { createMockEvent } from '../testing/mock-event'; import { mockServer } from '../tests/jest.setup'; -import { createMockEvent } from './test-utils'; -import { InitialDomainMapping } from '../types'; +import { InitialDomainMapping } from '../types/common'; import { EventType } from '../types/extraction'; + import { installInitialDomainMapping } from './install-initial-domain-mapping'; // Mock dependencies @@ -11,8 +13,8 @@ jest.mock('axios', () => ({ ...jest.requireActual('axios'), isAxiosError: jest.fn(), })); -jest.mock('../http/axios-client-internal', () => { - const originalModule = jest.requireActual('../http/axios-client-internal'); +jest.mock('../http/client', () => { + const originalModule = jest.requireActual('../http/client'); return { ...originalModule, axiosClient: { @@ -29,7 +31,7 @@ const mockIsAxiosError = axios.isAxiosError as unknown as jest.Mock; describe(installInitialDomainMapping.name, () => { // Create mock objects const mockEvent = createMockEvent(mockServer.baseUrl, { - payload: { event_type: EventType.ExtractionDataStart }, + payload: { event_type: EventType.StartExtractingData }, }); const mockInitialDomainMapping: InitialDomainMapping = { diff --git a/src/common/install-initial-domain-mapping.ts b/src/state/install-initial-domain-mapping.ts similarity index 91% rename from src/common/install-initial-domain-mapping.ts rename to src/state/install-initial-domain-mapping.ts index f65026ea..6de36440 100644 --- a/src/common/install-initial-domain-mapping.ts +++ b/src/state/install-initial-domain-mapping.ts @@ -1,11 +1,10 @@ -import { axiosClient } from '../http/axios-client-internal'; -import { AirdropEvent } from '../types/extraction'; - +import { axiosClient } from '../http/client'; import { serializeError } from '../logger/logger'; import { InitialDomainMapping } from '../types/common'; +import { AirSyncEvent } from '../types/extraction'; export async function installInitialDomainMapping( - event: AirdropEvent, + event: AirSyncEvent, initialDomainMappingJson: InitialDomainMapping ): Promise { const devrevEndpoint = event.execution_metadata.devrev_endpoint; @@ -17,7 +16,6 @@ export async function installInitialDomainMapping( return; } - // Get snap-in details const snapInResponse = await axiosClient.get( devrevEndpoint + '/internal/snap-ins.get', { @@ -41,7 +39,7 @@ export async function installInitialDomainMapping( const startingRecipeBlueprint = initialDomainMappingJson?.starting_recipe_blueprint; - // Try to create a recipe blueprint + // Recipe blueprint creation is best-effort; install proceeds without it. let recipeBlueprintId; if ( startingRecipeBlueprint && @@ -73,7 +71,6 @@ export async function installInitialDomainMapping( } } - // Install the initial domain mappings const additionalMappings = initialDomainMappingJson.additional_mappings || {}; const initialDomainMappingInstallResponse = await axiosClient.post( `${devrevEndpoint}/internal/airdrop.recipe.initial-domain-mappings.install`, diff --git a/src/state/loading-state.ts b/src/state/loading-state.ts new file mode 100644 index 00000000..34516116 --- /dev/null +++ b/src/state/loading-state.ts @@ -0,0 +1,35 @@ +import { STATELESS_EVENT_TYPES } from '../common/constants'; + +import { BaseState } from './base-state'; +import { loadingSdkState, StateInterface } from './state.interfaces'; + +/** State for loading workers: seeds loading SDK state; no extraction-window resolution. */ +export class LoadingState extends BaseState { + constructor(params: StateInterface) { + super(params, loadingSdkState); + } +} + +export async function createLoadingState({ + event, + initialState, + initialDomainMapping, + options, +}: StateInterface): Promise> { + // Clone so the caller's initialState is never mutated. + const deepCloneInitialState: ConnectorState = structuredClone(initialState); + + const state = new LoadingState({ + event, + initialState: deepCloneInitialState, + initialDomainMapping, + options, + }); + + if (!STATELESS_EVENT_TYPES.includes(event.payload.event_type)) { + await state.init(deepCloneInitialState); + await state.installInitialDomainMappingIfNeeded(initialDomainMapping); + } + + return state; +} diff --git a/src/state/state.extract-window.test.ts b/src/state/state.extract-window.test.ts index 74d76452..d624f49f 100644 --- a/src/state/state.extract-window.test.ts +++ b/src/state/state.extract-window.test.ts @@ -1,7 +1,8 @@ +import { createMockEvent } from '../testing/mock-event'; import { mockServer } from '../tests/jest.setup'; -import { createMockEvent } from '../common/test-utils'; import { EventType, TimeValueType } from '../types/extraction'; -import { State, createAdapterState } from './state'; + +import { createExtractionState, ExtractionState } from './extraction-state'; describe('State — extraction window validation', () => { let fetchStateSpy: jest.SpyInstance; @@ -11,7 +12,7 @@ describe('State — extraction window validation', () => { jest.clearAllMocks(); jest.restoreAllMocks(); - fetchStateSpy = jest.spyOn(State.prototype, 'fetchState'); + fetchStateSpy = jest.spyOn(ExtractionState.prototype, 'fetchState'); processExitSpy = jest.spyOn(process, 'exit').mockImplementation(() => { throw new Error('process.exit called'); }); @@ -43,7 +44,7 @@ describe('State — extraction window validation', () => { // Act & Assert await expect( - createAdapterState({ + createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -77,7 +78,7 @@ describe('State — extraction window validation', () => { // Act & Assert await expect( - createAdapterState({ + createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -110,7 +111,7 @@ describe('State — extraction window validation', () => { fetchStateSpy.mockResolvedValue({ state: stringifiedState }); // Act - await createAdapterState({ + await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -140,7 +141,7 @@ describe('State — extraction window validation', () => { fetchStateSpy.mockResolvedValue({ state: stringifiedState }); // Act - await createAdapterState({ + await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -173,7 +174,7 @@ describe('State — extraction window validation', () => { fetchStateSpy.mockResolvedValue({ state: stringifiedState }); // Act - await createAdapterState({ + await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, diff --git a/src/state/state.interfaces.ts b/src/state/state.interfaces.ts index 21eef31a..a7fdfc9d 100644 --- a/src/state/state.interfaces.ts +++ b/src/state/state.interfaces.ts @@ -1,39 +1,26 @@ import { InitialDomainMapping } from '../types/common'; -import { AirdropEvent } from '../types/extraction'; +import { AirSyncEvent } from '../types/extraction'; import { FileToLoad } from '../types/loading'; import { WorkerAdapterOptions } from '../types/workers'; export interface SdkState { - /** - * @deprecated Use extract_from and extract_to from the event context instead, - * which are automatically resolved by the SDK from extraction_start_time and extraction_end_time. - */ - lastSyncStarted?: string; - /** - * @deprecated Use extract_from and extract_to from the event context instead, - * which are automatically resolved by the SDK from extraction_start_time and extraction_end_time. - */ - lastSuccessfulSyncStarted?: string; - /** The pending (not yet committed) oldest extraction boundary (ISO 8601 timestamp). - * Set on StartExtractingMetadata, reused across subsequent phases, cleared on AttachmentExtractionDone. */ + // Pending (uncommitted) extraction boundaries (ISO 8601): set on + // StartExtractingMetadata, reused across phases, cleared on AttachmentExtractionDone. pendingWorkersOldest?: string; - /** The pending (not yet committed) newest extraction boundary (ISO 8601 timestamp). - * Set on StartExtractingMetadata, reused across subsequent phases, cleared on AttachmentExtractionDone. */ pendingWorkersNewest?: string; - /** The oldest point of extraction (ISO 8601 timestamp). */ + // Committed extraction boundaries (ISO 8601). workersOldest?: string; - /** The newest point of extraction (ISO 8601 timestamp). */ workersNewest?: string; toDevRev?: ToDevRev; fromDevRev?: FromDevRev; snapInVersionId?: string; } -/** - * AdapterState is an interface that defines the structure of the adapter state that is used by the external extractor. - * It extends the connector state with additional fields: lastSyncStarted, lastSuccessfulSyncStarted, snapInVersionId and attachmentsMetadata. - */ -export type AdapterState = ConnectorState & SdkState; +/** v2 on-disk state shape: SDK bookkeeping kept disjoint from connector keys. */ +export interface AdapterStateEnvelope { + connectorState: ConnectorState; + sdkState: SdkState; +} export interface ToDevRev { attachmentsMetadata: { @@ -43,16 +30,12 @@ export interface ToDevRev { }; } -/** Outcome of sending an attachment for processing. */ export enum ProcessedAttachmentStatus { Success = 'success', Failed = 'failed', } -/** - * Attachment structure, that stores both attachment id and its parent_id for deduplication - * on the SDK side, along with whether it succeeded or failed. - */ +/** id + parent_id identify an attachment for SDK-side deduplication. */ export interface ProcessedAttachment { id: string; parent_id: string; @@ -64,15 +47,13 @@ export interface FromDevRev { } export interface StateInterface { - event: AirdropEvent; + event: AirSyncEvent; initialState: ConnectorState; initialDomainMapping?: InitialDomainMapping; options?: WorkerAdapterOptions; } export const extractionSdkState = { - lastSyncStarted: '', - lastSuccessfulSyncStarted: '', pendingWorkersOldest: '', pendingWorkersNewest: '', workersOldest: '', @@ -93,3 +74,15 @@ export const loadingSdkState = { filesToLoad: [], }, }; + +/** + * SDK-owned top-level state keys, used to split a flat v1 blob during migration. + * `lastSyncStarted` / `lastSuccessfulSyncStarted` are no longer on `SdkState` but + * stay listed so v1 blobs carrying them don't leak them into connector state. + */ +export const V1_SDK_STATE_KEYS: ReadonlySet = new Set([ + ...Object.keys(extractionSdkState), + ...Object.keys(loadingSdkState), + 'lastSyncStarted', + 'lastSuccessfulSyncStarted', +]); diff --git a/src/state/state.pending-boundaries.test.ts b/src/state/state.pending-boundaries.test.ts index 78c00858..842a011e 100644 --- a/src/state/state.pending-boundaries.test.ts +++ b/src/state/state.pending-boundaries.test.ts @@ -1,7 +1,8 @@ +import { createMockEvent } from '../testing/mock-event'; import { mockServer } from '../tests/jest.setup'; -import { createMockEvent } from '../common/test-utils'; import { EventType, TimeValueType } from '../types/extraction'; -import { State, createAdapterState } from './state'; + +import { createExtractionState, ExtractionState } from './extraction-state'; /* eslint-disable @typescript-eslint/no-require-imports */ @@ -16,10 +17,10 @@ describe('State — pending extraction boundaries', () => { jest.clearAllMocks(); jest.restoreAllMocks(); - postStateSpy = jest.spyOn(State.prototype, 'postState'); - fetchStateSpy = jest.spyOn(State.prototype, 'fetchState'); + postStateSpy = jest.spyOn(ExtractionState.prototype, 'postState'); + fetchStateSpy = jest.spyOn(ExtractionState.prototype, 'fetchState'); installInitialDomainMappingSpy = jest.spyOn( - require('../common/install-initial-domain-mapping'), + require('./install-initial-domain-mapping'), 'installInitialDomainMapping' ); jest.spyOn(process, 'exit').mockImplementation(() => { @@ -61,15 +62,17 @@ describe('State — pending extraction boundaries', () => { postStateSpy.mockResolvedValue({ success: true }); // Act - const state = await createAdapterState({ + const state = await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, }); // Assert - expect(state.state.pendingWorkersOldest).toBe('1970-01-01T00:00:00.000Z'); - expect(state.state.pendingWorkersNewest).toBe(FIXED_NOW); + expect(state.sdkState.pendingWorkersOldest).toBe( + '1970-01-01T00:00:00.000Z' + ); + expect(state.sdkState.pendingWorkersNewest).toBe(FIXED_NOW); expect(event.payload.event_context.extract_from).toBe( '1970-01-01T00:00:00.000Z' ); @@ -106,16 +109,18 @@ describe('State — pending extraction boundaries', () => { fetchStateSpy.mockResolvedValue({ state: stringifiedState }); // Act - const state = await createAdapterState({ + const state = await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, }); // Assert: pending values are overwritten with fresh resolution, not stale values - expect(state.state.pendingWorkersOldest).toBe('1970-01-01T00:00:00.000Z'); - expect(state.state.pendingWorkersNewest).toBe(FIXED_NOW); - expect(state.state.pendingWorkersNewest).not.toBe(staleNewest); + expect(state.sdkState.pendingWorkersOldest).toBe( + '1970-01-01T00:00:00.000Z' + ); + expect(state.sdkState.pendingWorkersNewest).toBe(FIXED_NOW); + expect(state.sdkState.pendingWorkersNewest).not.toBe(staleNewest); }); it('should reuse pending values from state on ContinueExtractingData instead of re-resolving', async () => { @@ -149,7 +154,7 @@ describe('State — pending extraction boundaries', () => { fetchStateSpy.mockResolvedValue({ state: stringifiedState }); // Act - const state = await createAdapterState({ + const state = await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -159,8 +164,8 @@ describe('State — pending extraction boundaries', () => { expect(event.payload.event_context.extract_from).toBe(pendingOldest); expect(event.payload.event_context.extract_to).toBe(pendingNewest); // Pending values in state remain unchanged - expect(state.state.pendingWorkersOldest).toBe(pendingOldest); - expect(state.state.pendingWorkersNewest).toBe(pendingNewest); + expect(state.sdkState.pendingWorkersOldest).toBe(pendingOldest); + expect(state.sdkState.pendingWorkersNewest).toBe(pendingNewest); }); it('should not set extract_from/extract_to on ContinueExtractingData if no pending values exist', async () => { @@ -178,7 +183,7 @@ describe('State — pending extraction boundaries', () => { fetchStateSpy.mockResolvedValue({ state: stringifiedState }); // Act - await createAdapterState({ + await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -209,7 +214,7 @@ describe('State — pending extraction boundaries', () => { fetchStateSpy.mockResolvedValue({ state: stringifiedState }); // Act - await createAdapterState({ + await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, diff --git a/src/state/state.post-state.test.ts b/src/state/state.post-state.test.ts index 83ff39e0..cbd4bfff 100644 --- a/src/state/state.post-state.test.ts +++ b/src/state/state.post-state.test.ts @@ -1,11 +1,12 @@ +import { createMockEvent } from '../testing/mock-event'; import { mockServer } from '../tests/jest.setup'; -import { createMockEvent } from '../common/test-utils'; import { EventType } from '../types/extraction'; -import { State, createAdapterState } from './state'; + +import { createExtractionState, ExtractionState } from './extraction-state'; /* eslint-disable @typescript-eslint/no-require-imports */ -describe('State.postState', () => { +describe('ExtractionState.postState', () => { let postStateSpy: jest.SpyInstance; let fetchStateSpy: jest.SpyInstance; let processExitSpy: jest.SpyInstance; @@ -14,8 +15,8 @@ describe('State.postState', () => { jest.clearAllMocks(); jest.restoreAllMocks(); - postStateSpy = jest.spyOn(State.prototype, 'postState'); - fetchStateSpy = jest.spyOn(State.prototype, 'fetchState'); + postStateSpy = jest.spyOn(ExtractionState.prototype, 'postState'); + fetchStateSpy = jest.spyOn(ExtractionState.prototype, 'fetchState'); processExitSpy = jest.spyOn(process, 'exit').mockImplementation(() => { throw new Error('process.exit called'); }); @@ -37,7 +38,7 @@ describe('State.postState', () => { postStateSpy.mockRestore(); - const adapterState = await createAdapterState({ + const adapterState = await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -51,11 +52,14 @@ describe('State.postState', () => { expect(requests).toHaveLength(1); const body = requests[0].body as { state: string }; - // Body must contain the stringified state, preserving the original fields + // Body must contain the stringified v2 envelope, with the posted state + // preserved under connectorState. expect(typeof body.state).toBe('string'); - const parsed = JSON.parse(body.state) as Record; - expect(parsed.foo).toBe('bar'); - expect(parsed.snapInVersionId).toBe('1.0.0'); + const parsed = JSON.parse(body.state) as { + connectorState: Record; + }; + expect(parsed.connectorState.foo).toBe('bar'); + expect(parsed.connectorState.snapInVersionId).toBe('1.0.0'); }); it('should exit(1) when postState HTTP request fails', async () => { @@ -70,14 +74,14 @@ describe('State.postState', () => { postStateSpy.mockRestore(); - const adapterState = await createAdapterState({ + const adapterState = await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, }); // Mock axiosClient.post directly to bypass the retry backoff - const axiosClientModule = require('../http/axios-client-internal'); + const axiosClientModule = require('../http/client'); const axiosPostSpy = jest .spyOn(axiosClientModule.axiosClient, 'post') .mockRejectedValue(new Error('network error')); diff --git a/src/state/state.test.ts b/src/state/state.test.ts index e8a2e5ce..082b6498 100644 --- a/src/state/state.test.ts +++ b/src/state/state.test.ts @@ -2,15 +2,16 @@ import { STATEFUL_EVENT_TYPES, STATELESS_EVENT_TYPES, } from '../common/constants'; +import { createMockEvent } from '../testing/mock-event'; import { mockServer } from '../tests/jest.setup'; -import { createMockEvent } from '../common/test-utils'; import { EventType } from '../types/extraction'; -import { State, createAdapterState } from './state'; + +import { createExtractionState, ExtractionState } from './extraction-state'; import { extractionSdkState } from './state.interfaces'; /* eslint-disable @typescript-eslint/no-require-imports */ -describe(State.name, () => { +describe(ExtractionState.name, () => { let initSpy: jest.SpyInstance; let postStateSpy: jest.SpyInstance; let fetchStateSpy: jest.SpyInstance; @@ -21,11 +22,11 @@ describe(State.name, () => { jest.clearAllMocks(); jest.restoreAllMocks(); - initSpy = jest.spyOn(State.prototype, 'init'); - postStateSpy = jest.spyOn(State.prototype, 'postState'); - fetchStateSpy = jest.spyOn(State.prototype, 'fetchState'); + initSpy = jest.spyOn(ExtractionState.prototype, 'init'); + postStateSpy = jest.spyOn(ExtractionState.prototype, 'postState'); + fetchStateSpy = jest.spyOn(ExtractionState.prototype, 'fetchState'); installInitialDomainMappingSpy = jest.spyOn( - require('../common/install-initial-domain-mapping'), + require('./install-initial-domain-mapping'), 'installInitialDomainMapping' ); processExitSpy = jest.spyOn(process, 'exit').mockImplementation(() => { @@ -42,7 +43,7 @@ describe(State.name, () => { }); // Act - await createAdapterState({ + await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -70,7 +71,7 @@ describe(State.name, () => { // Act & Assert await expect( - createAdapterState({ + createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -91,7 +92,7 @@ describe(State.name, () => { // Act & Assert await expect( - createAdapterState({ + createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -112,7 +113,7 @@ describe(State.name, () => { // Act & Assert await expect( - createAdapterState({ + createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -151,59 +152,20 @@ describe(State.name, () => { }); // Act - await createAdapterState({ + const result = await createExtractionState({ event, initialState, initialDomainMapping: {}, }); - const expectedState = { - ...initialState, - ...extractionSdkState, - }; - expect(postStateSpy).toHaveBeenCalledWith(expectedState); + // Assert: on 404 the SDK persists the full adapter state — the initial + // connector state and the seeded SDK state — via postState. + expect(postStateSpy).toHaveBeenCalled(); + expect(result.state).toEqual(initialState); + expect(result.sdkState).toEqual(extractionSdkState); } ); - it(EventType.StartExtractingData, async () => { - // Arrange - const initialState = { - test: 'test', - }; - const event = createMockEvent(mockServer.baseUrl, { - context: { - snap_in_version_id: '', - }, - payload: { event_type: EventType.StartExtractingData }, - }); - fetchStateSpy.mockRejectedValue({ - isAxiosError: true, - response: { status: 404 }, - }); - installInitialDomainMappingSpy.mockResolvedValue({ - success: true, - }); - postStateSpy.mockResolvedValue({ - success: true, - }); - - // Act - await createAdapterState({ - event, - initialState, - initialDomainMapping: {}, - }); - - // Assert - // Verify that post state is called with object that contains - // lastSyncStarted which is not empty string - expect(postStateSpy).toHaveBeenCalledWith( - expect.objectContaining({ - lastSyncStarted: expect.not.stringMatching(/^$/), - }) - ); - }); - it.each(STATEFUL_EVENT_TYPES)( 'should exit the process if initialDomainMapping is not provided for event type %s', async (eventType) => { @@ -220,7 +182,7 @@ describe(State.name, () => { // Act & Assert await expect( - createAdapterState({ + createExtractionState({ event, initialState: {}, initialDomainMapping: undefined, @@ -248,7 +210,7 @@ describe(State.name, () => { fetchStateSpy.mockResolvedValue({ state: stringifiedState }); // Act & Assert - await createAdapterState({ + await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -280,7 +242,7 @@ describe(State.name, () => { }); // Act - await createAdapterState({ + await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -308,7 +270,7 @@ describe(State.name, () => { }); // Act - const result = await createAdapterState({ + const result = await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -337,7 +299,7 @@ describe(State.name, () => { postStateSpy.mockResolvedValue({ success: true }); // Act - const result = await createAdapterState({ + const result = await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -354,7 +316,7 @@ describe(State.name, () => { }); // Act - const result = await createAdapterState({ + const result = await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -376,7 +338,7 @@ describe(State.name, () => { }); // Act - const result = await createAdapterState({ + const result = await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, diff --git a/src/state/state.time-value-resolution.test.ts b/src/state/state.time-value-resolution.test.ts index a2265a58..6720dc03 100644 --- a/src/state/state.time-value-resolution.test.ts +++ b/src/state/state.time-value-resolution.test.ts @@ -1,7 +1,9 @@ +import { createMockEvent } from '../testing/mock-event'; import { mockServer } from '../tests/jest.setup'; -import { createMockEvent } from '../common/test-utils'; import { EventType, TimeValue, TimeValueType } from '../types/extraction'; -import { State, createAdapterState } from './state'; + +import { BaseState } from './base-state'; +import { createExtractionState } from './extraction-state'; describe('State — TimeValue resolution', () => { let fetchStateSpy: jest.SpyInstance; @@ -11,7 +13,7 @@ describe('State — TimeValue resolution', () => { jest.clearAllMocks(); jest.restoreAllMocks(); - fetchStateSpy = jest.spyOn(State.prototype, 'fetchState'); + fetchStateSpy = jest.spyOn(BaseState.prototype, 'fetchState'); processExitSpy = jest.spyOn(process, 'exit').mockImplementation(() => { throw new Error('process.exit called'); }); @@ -19,13 +21,14 @@ describe('State — TimeValue resolution', () => { describe('Enhanced Control Protocol - TimeValue resolution failures', () => { it('should exit the process if extraction_start_time resolution fails', async () => { - // Arrange: WORKERS_NEWEST type but state has no workersNewest + // Arrange: ABSOLUTE_TIME with an invalid ISO timestamp (still rejected in v2) const event = createMockEvent(mockServer.baseUrl, { payload: { event_type: EventType.StartExtractingMetadata, event_context: { extraction_start_time: { - type: TimeValueType.WORKERS_NEWEST, + type: TimeValueType.ABSOLUTE_TIME, + value: 'not-a-date', }, }, }, @@ -40,7 +43,7 @@ describe('State — TimeValue resolution', () => { // Act & Assert await expect( - createAdapterState({ + createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -50,7 +53,7 @@ describe('State — TimeValue resolution', () => { }); it('should exit the process if extraction_end_time resolution fails', async () => { - // Arrange: WORKERS_NEWEST type but state has no workersNewest + // Arrange: ABSOLUTE_TIME end_time with an invalid ISO timestamp (still rejected in v2) const event = createMockEvent(mockServer.baseUrl, { payload: { event_type: EventType.StartExtractingMetadata, @@ -59,7 +62,8 @@ describe('State — TimeValue resolution', () => { type: TimeValueType.UNBOUNDED, }, extraction_end_time: { - type: TimeValueType.WORKERS_NEWEST, + type: TimeValueType.ABSOLUTE_TIME, + value: 'not-a-date', }, }, }, @@ -74,7 +78,7 @@ describe('State — TimeValue resolution', () => { // Act & Assert await expect( - createAdapterState({ + createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -109,7 +113,7 @@ describe('State — TimeValue resolution', () => { fetchStateSpy.mockResolvedValue({ state: stringifiedState }); // Act - const state = await createAdapterState({ + const state = await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -121,7 +125,9 @@ describe('State — TimeValue resolution', () => { expect(event.payload.event_context.extract_to).toBe( '2025-06-01T00:00:00.000Z' ); - expect(state.state.pendingWorkersNewest).toBe('2025-06-01T00:00:00.000Z'); + expect(state.sdkState.pendingWorkersNewest).toBe( + '2025-06-01T00:00:00.000Z' + ); }); it('should skip resolution when extraction_end_time has no type', async () => { @@ -148,7 +154,7 @@ describe('State — TimeValue resolution', () => { fetchStateSpy.mockResolvedValue({ state: stringifiedState }); // Act - await createAdapterState({ + await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, @@ -187,7 +193,7 @@ describe('State — TimeValue resolution', () => { fetchStateSpy.mockResolvedValue({ state: stringifiedState }); // Act - await createAdapterState({ + await createExtractionState({ event, initialState: {}, initialDomainMapping: {}, diff --git a/src/state/state.ts b/src/state/state.ts index 6f13224c..919ae6b9 100644 --- a/src/state/state.ts +++ b/src/state/state.ts @@ -1,342 +1,3 @@ -import axios from 'axios'; -import { parentPort } from 'node:worker_threads'; - -import { STATELESS_EVENT_TYPES } from '../common/constants'; -import { installInitialDomainMapping } from '../common/install-initial-domain-mapping'; -import { resolveTimeValue } from '../common/time-value-resolver'; -import { axiosClient } from '../http/axios-client-internal'; -import { getPrintableState, serializeError } from '../logger/logger'; -import { SyncMode } from '../types/common'; -import { EventType } from '../types/extraction'; -import { WorkerMessageSubject } from '../types/workers'; - -import { - AdapterState, - extractionSdkState, - loadingSdkState, - SdkState, - StateInterface, -} from './state.interfaces'; -import { ExtractionScope } from '../types/workers'; - -export async function createAdapterState({ - event, - initialState, - initialDomainMapping, - options, -}: StateInterface): Promise> { - // Deep clone the initial state to avoid mutating the original state - const deepCloneInitialState: ConnectorState = structuredClone(initialState); - - const as = new State({ - event, - initialState: deepCloneInitialState, - initialDomainMapping, - options, - }); - - if (!STATELESS_EVENT_TYPES.includes(event.payload.event_type)) { - await as.init(deepCloneInitialState); - - // Check if IDM needs to be updated - const snapInVersionId = event.context.snap_in_version_id; - const hasSnapInVersionInState = 'snapInVersionId' in as.state; - const shouldUpdateIDM = - !hasSnapInVersionInState || as.state.snapInVersionId !== snapInVersionId; - - if (!shouldUpdateIDM) { - console.log( - `Snap-in version in state matches the version in event context "${snapInVersionId}". Skipping initial domain mapping installation.` - ); - } else { - try { - console.log( - `Snap-in version in state "${as.state.snapInVersionId}" does not match the version in event context "${snapInVersionId}". Installing initial domain mapping.` - ); - - if (initialDomainMapping) { - await installInitialDomainMapping(event, initialDomainMapping); - as.state.snapInVersionId = snapInVersionId; - } else { - throw new Error( - 'No initial domain mapping was passed to spawn function. Skipping initial domain mapping installation.' - ); - } - } catch (error) { - const errorMessage = `Error while installing initial domain mapping. ${serializeError( - error - )}`; - console.error(errorMessage); - parentPort?.postMessage({ - subject: WorkerMessageSubject.WorkerMessageFailed, - payload: { message: errorMessage }, - }); - process.exit(1); - } - } - - // Set lastSyncStarted if the event type is StartExtractingData - if ( - event.payload.event_type === EventType.StartExtractingData && - !as.state.lastSyncStarted - ) { - as.state.lastSyncStarted = new Date().toISOString(); - console.log(`Setting lastSyncStarted to ${as.state.lastSyncStarted}.`); - } - - // Resolve extraction timestamps from TimeValue objects, or reuse pending values from a prior invocation. - // On StartExtractingMetadata: resolve fresh from TimeValue objects and store in pending state (always overwrite). - // On all other events: reuse the pending values cached during StartExtractingMetadata. - const eventContext = event.payload.event_context; - - if (event.payload.event_type === EventType.StartExtractingMetadata) { - const timeFields = [ - { - source: 'extraction_start_time', - target: 'extract_from', - pending: 'pendingWorkersOldest', - }, - { - source: 'extraction_end_time', - target: 'extract_to', - pending: 'pendingWorkersNewest', - }, - ] as const; - - for (const { source, target, pending } of timeFields) { - const timeValue = eventContext[source]; - if (timeValue && timeValue.type) { - try { - const resolved = resolveTimeValue(timeValue, as.state); - eventContext[target] = resolved; - as.state[pending] = resolved; - console.log( - `Resolved ${target} to ${resolved}. Stored in ${pending}.` - ); - } catch (error) { - const errorMessage = `Failed to resolve ${source}: ${serializeError( - error - )}`; - console.error(errorMessage); - parentPort?.postMessage({ - subject: WorkerMessageSubject.WorkerMessageFailed, - payload: { message: errorMessage }, - }); - process.exit(1); - } - } - } - } else { - // Non-StartExtractingMetadata events: reuse pending values from state - if (as.state.pendingWorkersOldest) { - eventContext.extract_from = as.state.pendingWorkersOldest; - console.log( - `Reusing pendingWorkersOldest as extract_from: ${as.state.pendingWorkersOldest}.` - ); - } else { - console.log( - 'pendingWorkersOldest is not set in state. extract_from will not be populated for this invocation.' - ); - } - if (as.state.pendingWorkersNewest) { - eventContext.extract_to = as.state.pendingWorkersNewest; - console.log( - `Reusing pendingWorkersNewest as extract_to: ${as.state.pendingWorkersNewest}.` - ); - } else { - console.log( - 'pendingWorkersNewest is not set in state. extract_to will not be populated for this invocation.' - ); - } - } - - // Validate that extract_from is before extract_to - if (eventContext.extract_from && eventContext.extract_to) { - if (eventContext.extract_from >= eventContext.extract_to) { - const errorMessage = `Invalid extraction window: extract_from (${eventContext.extract_from}) must be older than extract_to (${eventContext.extract_to}). This indicates an error in the platform.`; - console.error(errorMessage); - parentPort?.postMessage({ - subject: WorkerMessageSubject.WorkerMessageFailed, - payload: { message: errorMessage }, - }); - process.exit(1); - } - } - } - - return as; -} - -export class State { - private _state: AdapterState; - private _extractionScope: ExtractionScope = {}; - private initialSdkState: SdkState; - private workerUrl: string; - private devrevToken: string; - private syncUnitId: string; - private requestId: string; - - constructor({ event, initialState }: StateInterface) { - this.initialSdkState = - event.payload.event_context.mode === SyncMode.LOADING - ? loadingSdkState - : extractionSdkState; - this._state = { - ...initialState, - ...this.initialSdkState, - } as AdapterState; - this.workerUrl = event.payload.event_context.worker_data_url; - this.devrevToken = event.context.secrets.service_account_token; - this.syncUnitId = event.payload.event_context.sync_unit_id; - this.requestId = event.payload.event_context.request_id_adaas; - } - - get state(): AdapterState { - return this._state; - } - - set state(value: AdapterState) { - this._state = value; - } - - get extractionScope(): ExtractionScope { - return this._extractionScope; - } - - /** - * Initializes the state for this adapter instance by fetching from API - * or creating an initial state if none exists (404). - * @param initialState The initial connector state provided by the spawn function - */ - async init(initialState: ConnectorState): Promise { - try { - const { state: stringifiedState, objects } = await this.fetchState(); - if (!stringifiedState) { - throw new Error('No state found in response.'); - } - - let parsedState: AdapterState; - try { - parsedState = JSON.parse(stringifiedState); - } catch (error) { - throw new Error(`Failed to parse state. ${error}`); - } - - this.state = parsedState; - console.log( - 'State fetched successfully. Current state', - getPrintableState(this.state) - ); - - if (objects) { - try { - this._extractionScope = JSON.parse(objects); - } catch (error) { - console.warn(`Failed to parse extractionScope. ${error}`); - } - } - } catch (error) { - if (axios.isAxiosError(error) && error.response?.status === 404) { - console.log('State not found. Initializing state with initial state.'); - const initialAdapterState: AdapterState = { - ...initialState, - ...this.initialSdkState, - }; - - this.state = initialAdapterState; - await this.postState(initialAdapterState); - } else { - const errorMessage = `Failed to init state. ${serializeError(error)}`; - console.error(errorMessage); - parentPort?.postMessage({ - subject: WorkerMessageSubject.WorkerMessageFailed, - payload: { message: errorMessage }, - }); - process.exit(1); - } - } - } - - /** - * Updates the state of the adapter by posting to API. - * @param {object} state - The state to be updated - */ - async postState(state?: AdapterState) { - const url = this.workerUrl + '.update'; - this.state = state || this.state; - - let stringifiedState: string; - try { - stringifiedState = JSON.stringify(this.state); - } catch (error) { - const errorMessage = `Failed to stringify state. ${serializeError( - error - )}`; - console.error(errorMessage); - parentPort?.postMessage({ - subject: WorkerMessageSubject.WorkerMessageFailed, - payload: { message: errorMessage }, - }); - process.exit(1); - } - - try { - await axiosClient.post( - url, - { - state: stringifiedState, - }, - { - headers: { - Authorization: this.devrevToken, - }, - params: { - sync_unit: this.syncUnitId, - request_id: this.requestId, - }, - } - ); - - console.log( - 'State updated successfully to', - getPrintableState(this.state) - ); - } catch (error) { - const errorMessage = `Failed to update the state. ${serializeError( - error - )}`; - console.error(errorMessage); - parentPort?.postMessage({ - subject: WorkerMessageSubject.WorkerMessageFailed, - payload: { message: errorMessage }, - }); - process.exit(1); - } - } - - /** - * Fetches the state of the adapter from API. - * @return The raw state data from API - */ - async fetchState(): Promise<{ state: string; objects?: string }> { - console.log( - `Fetching state with sync unit id ${this.syncUnitId} and request id ${this.requestId}.` - ); - - const url = this.workerUrl + '.get'; - const response = await axiosClient.get(url, { - headers: { - Authorization: this.devrevToken, - }, - params: { - sync_unit: this.syncUnitId, - request_id: this.requestId, - }, - }); - - return { - state: response.data?.state, - objects: response.data?.objects, - }; - } -} +export { BaseState } from './base-state'; +export { createExtractionState, ExtractionState } from './extraction-state'; +export { createLoadingState, LoadingState } from './loading-state'; diff --git a/src/common/time-value-resolver.test.ts b/src/state/time-value-resolver.test.ts similarity index 94% rename from src/common/time-value-resolver.test.ts rename to src/state/time-value-resolver.test.ts index f84a3dab..b52a2ae6 100644 --- a/src/common/time-value-resolver.test.ts +++ b/src/state/time-value-resolver.test.ts @@ -1,9 +1,10 @@ -import { TimeValueType } from '../types/extraction'; +import { UNBOUNDED_DATE_TIME_VALUE } from '../common/constants'; import { SdkState } from '../state/state.interfaces'; -import { UNBOUNDED_DATE_TIME_VALUE } from './constants'; +import { TimeValueType } from '../types/extraction'; + import { - parseDuration, applyDuration, + parseDuration, resolveTimeValue, } from './time-value-resolver'; @@ -154,8 +155,6 @@ describe('time-value-resolver', () => { describe('resolveTimeValue', () => { const baseState: SdkState = { - lastSyncStarted: '', - lastSuccessfulSyncStarted: '', workersOldest: '2024-01-01T00:00:00.000Z', workersNewest: '2024-06-01T00:00:00.000Z', }; @@ -255,13 +254,12 @@ describe('time-value-resolver', () => { expect(result).toBe('2024-06-01T00:00:00.000Z'); }); - it('should throw if workersNewest is not set', () => { - expect(() => - resolveTimeValue( - { type: TimeValueType.WORKERS_NEWEST }, - { workersNewest: '' } - ) - ).toThrow('workersNewest is not set in state'); + it('should return UNBOUNDED_DATE_TIME_VALUE if workersNewest is not set', () => { + const result = resolveTimeValue( + { type: TimeValueType.WORKERS_NEWEST }, + { workersNewest: '' } + ); + expect(result).toBe(UNBOUNDED_DATE_TIME_VALUE); }); }); @@ -332,16 +330,15 @@ describe('time-value-resolver', () => { expect(result).toBe('2024-06-01T00:30:00.000Z'); }); - it('should throw if workersNewest is not set', () => { - expect(() => - resolveTimeValue( - { - type: TimeValueType.WORKERS_NEWEST_PLUS_WINDOW, - value: '2h', - }, - { workersNewest: '' } - ) - ).toThrow('workersNewest is not set in state'); + it('should return UNBOUNDED_DATE_TIME_VALUE if workersNewest is not set', () => { + const result = resolveTimeValue( + { + type: TimeValueType.WORKERS_NEWEST_PLUS_WINDOW, + value: '30m', + }, + { workersNewest: '' } + ); + expect(result).toBe(UNBOUNDED_DATE_TIME_VALUE); }); it('should throw if value (duration) is missing', () => { @@ -367,8 +364,6 @@ describe('time-value-resolver', () => { const FIXED_NOW = '2026-02-26T15:30:00.000Z'; const scenarioState: SdkState = { - lastSyncStarted: '', - lastSuccessfulSyncStarted: '', workersOldest: '2024-01-01T00:00:00.000Z', workersNewest: '2024-06-01T00:00:00.000Z', }; diff --git a/src/common/time-value-resolver.ts b/src/state/time-value-resolver.ts similarity index 59% rename from src/common/time-value-resolver.ts rename to src/state/time-value-resolver.ts index ce2494d8..ce28338c 100644 --- a/src/common/time-value-resolver.ts +++ b/src/state/time-value-resolver.ts @@ -1,19 +1,8 @@ -import { TimeUnit, TimeValue, TimeValueType } from '../types/extraction'; +import { UNBOUNDED_DATE_TIME_VALUE } from '../common/constants'; import { SdkState } from '../state/state.interfaces'; -import { UNBOUNDED_DATE_TIME_VALUE } from './constants'; +import { TimeUnit, TimeValue, TimeValueType } from '../types/extraction'; -/** - * Parses a shorthand duration string into its numeric value and unit. - * Supported units: - * - 'ns' for nanoseconds - * - 'us' or 'µs' for microseconds - * - 'ms' for milliseconds - * - 's' for seconds - * - 'm' for minutes - * - 'h' for hours - * - * @throws Error if the format is invalid - */ +/** Parses a shorthand duration (e.g. '100ns', '500ms', '5m', '2h'); units are the `TimeUnit` values. */ export function parseDuration(shorthand: string): { value: number; unit: TimeUnit; @@ -33,14 +22,7 @@ export function parseDuration(shorthand: string): { }; } -/** - * Applies a shorthand duration to a base ISO 8601 timestamp. - * - * @param baseTimestamp - ISO 8601 timestamp to apply duration to - * @param duration - Shorthand duration string (e.g. '100ns', '500ms', '30s', '5m', '2h') - * @param operation - Whether to 'add' or 'subtract' the duration - * @returns ISO 8601 timestamp with the duration applied - */ +/** Adds/subtracts a shorthand duration to/from an ISO 8601 timestamp. */ export function applyDuration( baseTimestamp: string, duration: string, @@ -77,22 +59,10 @@ export function applyDuration( } /** - * Resolves a TimeValue into a concrete ISO 8601 timestamp string. - * - * Resolution rules: - * - ABSOLUTE: Returns the value directly (must be an ISO 8601 timestamp) - * - NOW: Returns the current time as ISO 8601 - * - UNBOUNDED: Returns UNBOUNDED_DATE_TIME_VALUE ('1970-01-01T00:00:00.000Z') - * - WORKERS_OLDEST: Returns workersOldest from state, or throws if not set - * - WORKERS_NEWEST: Returns workersNewest from state, or throws if not set - * - WORKERS_OLDEST_MINUS_WINDOW: Subtracts duration from workersOldest, or throws if not set - * - WORKERS_NEWEST_PLUS_WINDOW: Adds duration to workersNewest, or throws if not set - * - * @param timeValue - The TimeValue to resolve - * @param state - The current SDK state containing workersOldest and workersNewest - * @returns Resolved ISO 8601 timestamp string - * @throws Error if required TimeValue.value is missing for ABSOLUTE or *_WINDOW types - * @throws Error if workersOldest/workersNewest is not set in state for WORKERS_* types + * Resolves a TimeValue into a concrete ISO 8601 timestamp. WORKERS_* types + * read the boundaries from state and fall back to UNBOUNDED when unset + * (backwards compatibility with old state); *_WINDOW types apply the duration + * in `timeValue.value` to the boundary. */ export function resolveTimeValue( timeValue: TimeValue, @@ -126,7 +96,6 @@ export function resolveTimeValue( case TimeValueType.WORKERS_OLDEST: { if (!state.workersOldest) { - // To support backwards-compatibility for the old state return UNBOUNDED_DATE_TIME_VALUE; } return state.workersOldest; @@ -134,14 +103,7 @@ export function resolveTimeValue( case TimeValueType.WORKERS_NEWEST: { if (!state.workersNewest) { - // To support backwards-compatibility for the old state - if (state.lastSuccessfulSyncStarted) { - return state.lastSuccessfulSyncStarted; - } - - throw new Error( - 'Field workersNewest is not set in state. Cannot resolve TimeValue of type WORKERS_NEWEST without a prior extraction boundary.' - ); + return UNBOUNDED_DATE_TIME_VALUE; } return state.workersNewest; } @@ -153,7 +115,6 @@ export function resolveTimeValue( ); } if (!state.workersOldest) { - // To support backwards-compatibility for the old state return UNBOUNDED_DATE_TIME_VALUE; } return applyDuration(state.workersOldest, timeValue.value, 'subtract'); @@ -166,14 +127,7 @@ export function resolveTimeValue( ); } if (!state.workersNewest) { - // To support backwards-compatibility for the old state - if (state.lastSuccessfulSyncStarted) { - return state.lastSuccessfulSyncStarted; - } - - throw new Error( - 'Field workersNewest is not set in state. Cannot resolve TimeValue of type WORKERS_NEWEST_PLUS_WINDOW without a prior extraction boundary.' - ); + return UNBOUNDED_DATE_TIME_VALUE; } return applyDuration(state.workersNewest, timeValue.value, 'add'); } diff --git a/src/mock-server/README.md b/src/testing/README.md similarity index 100% rename from src/mock-server/README.md rename to src/testing/README.md diff --git a/src/common/test-utils.ts b/src/testing/mock-event.ts similarity index 80% rename from src/common/test-utils.ts rename to src/testing/mock-event.ts index 1584a345..1e002e03 100644 --- a/src/common/test-utils.ts +++ b/src/testing/mock-event.ts @@ -1,18 +1,12 @@ -import { AirdropEvent, EventType } from '../types/extraction'; +import { AirSyncEvent, EventType } from '../types/extraction'; -export const MOCK_SERVER_DEFAULT_URL = 'http://localhost:0'; +import { MOCK_SERVER_DEFAULT_URL } from './mock-server'; -/** - * Recursively makes all properties of T optional. - */ export type DeepPartial = { [P in keyof T]?: T[P] extends object ? DeepPartial : T[P]; }; -/** - * Deep merges source into target. Arrays and primitives from source replace - * those in target; plain objects are merged recursively. - */ +/** Arrays and primitives from source replace target's; plain objects merge recursively. */ function deepMerge>( target: T, source: DeepPartial @@ -44,25 +38,24 @@ function deepMerge>( } /** - * Creates a mock AirdropEvent for testing. - * - * @param mockServerUrl - Base URL for the mock server. Defaults to {@link MOCK_SERVER_DEFAULT_URL}. - * The `callback_url`, `worker_data_url`, and `devrev_endpoint` fields are - * derived from this value unless explicitly overridden. - * @param overrides - Deep partial of AirdropEvent. Any provided fields are - * deep-merged on top of the defaults. + * Creates a mock AirSyncEvent for testing. `callback_url`, `worker_data_url`, + * and `devrev_endpoint` derive from `mockServerUrl` unless explicitly overridden. */ export function createMockEvent( mockServerUrl: string = MOCK_SERVER_DEFAULT_URL, - overrides: DeepPartial = {} -): AirdropEvent { - const base: AirdropEvent = { + overrides: DeepPartial = {} +): AirSyncEvent { + const base: AirSyncEvent = { context: { secrets: { service_account_token: 'test_token', }, snap_in_version_id: 'test_snap_in_version_id', snap_in_id: 'test_snap_in_id', + user_id: 'test_user_id', + dev_oid: 'test_dev_oid', + source_id: 'test_source_id', + service_account_id: 'test_service_account_id', }, payload: { connection_data: { @@ -118,7 +111,7 @@ export function createMockEvent( const merged = deepMerge( base as unknown as Record, overrides as DeepPartial> - ) as unknown as AirdropEvent; + ) as unknown as AirSyncEvent; // Ensure mock server URLs always win over overrides, unless the caller // explicitly provided them. diff --git a/src/testing/mock-server.interfaces.ts b/src/testing/mock-server.interfaces.ts new file mode 100644 index 00000000..ebc3c8fe --- /dev/null +++ b/src/testing/mock-server.interfaces.ts @@ -0,0 +1,56 @@ +import { IncomingMessage, ServerResponse } from 'http'; + +export const DEFAULT_MOCK_SERVER_PORT = 3001; + +export interface ParsedRequest extends IncomingMessage { + /** URL path without query string */ + path: string; + /** Parsed JSON body (if any) */ + body?: unknown; +} + +export interface MockResponse extends ServerResponse { + set(headers: Record): MockResponse; + status(code: number): MockResponse; + json(data: unknown): void; + buffer(data: Buffer): void; + send(): void; +} + +/** Simulates failures before succeeding. */ +export interface RetryConfig { + /** Failures before success (default: 4) */ + failureCount?: number; + /** Status code during failures (default: 500) */ + errorStatus?: number; + errorBody?: unknown; + headers?: Record; + /** Delay in ms before each failure response */ + delay?: number; +} + +export interface RouteConfig { + path: string; + method: string; + status: number; + body?: unknown; + /** Raw binary body, e.g. gzipped JSONL (takes precedence over `body`) */ + bodyBuffer?: Buffer; + headers?: Record; + retry?: RetryConfig; + /** Delay in ms before sending the response */ + delay?: number; +} + +export type RouteHandler = (req: ParsedRequest, res: MockResponse) => unknown; + +export interface RequestInfo { + method: string; + url: string; + body?: unknown; +} + +export type RouteHandlers = Map; + +/** Request counts per route. */ +export type RequestCounts = Map; diff --git a/src/mock-server/mock-server.ts b/src/testing/mock-server.ts similarity index 89% rename from src/mock-server/mock-server.ts rename to src/testing/mock-server.ts index 2ad6ee72..528a7a69 100644 --- a/src/mock-server/mock-server.ts +++ b/src/testing/mock-server.ts @@ -10,11 +10,12 @@ import { RouteHandlers, } from './mock-server.interfaces'; +// Port 0 lets the OS assign a free port at listen time; tests read the +// resolved URL from `mockServer.baseUrl`. +export const MOCK_SERVER_DEFAULT_URL = 'http://localhost:0'; + const MAX_BODY_SIZE = 10 * 1024 * 1024; // 10mb -/** - * Parses the JSON body from an incoming request. - */ async function parseJsonBody(req: IncomingMessage): Promise { return new Promise((resolve, reject) => { const chunks: Buffer[] = []; @@ -47,9 +48,6 @@ async function parseJsonBody(req: IncomingMessage): Promise { }); } -/** - * Wraps a ServerResponse with helper methods (status, json, set, send). - */ function wrapResponse(res: ServerResponse): MockResponse { const mock = res as MockResponse; let statusCode = 200; @@ -85,9 +83,8 @@ function wrapResponse(res: ServerResponse): MockResponse { } /** - * MockServer used in tests to mock internal AirSync endpoints. - * This is a simple mock server that listens on a port and responds to requests. - * Supports per-test route configuration to simulate different response scenarios. + * Mocks internal AirSync endpoints in tests, with per-test route + * configuration to simulate different response scenarios. */ export class MockServer { private server: Server | null = null; @@ -175,10 +172,7 @@ export class MockServer { } } - /** - * Default route handler for the mock server. Returns { success: true } for - * routes that are not explicitly set. - */ + /** Falls back to { success: true } for routes not explicitly set. */ private defaultRouteHandler(req: ParsedRequest, res: MockResponse): void { if (req.method === 'GET' && req.path === '/worker_data_url.get') { res.status(200).json({ @@ -212,9 +206,6 @@ export class MockServer { return `${method.toUpperCase()}:${path}`; } - /** - * Configures a route to return a specific status code and optional response body. - */ public setRoute(config: RouteConfig): void { const { path, method, status, body, bodyBuffer, retry, headers, delay } = config; @@ -293,19 +284,13 @@ export class MockServer { }); } - /** - * Resets all custom route handlers, restoring all default handlers. - * Also clears request tracking data. - */ + /** Restores default handlers and clears request tracking data. */ public resetRoutes(): void { this.routeHandlers.clear(); this.requestCounts.clear(); this.requests = []; } - /** - * Returns the most recent request or undefined if no requests exist. - */ public getLastRequest(): RequestInfo | undefined { if (this.requests.length === 0) { return undefined; @@ -313,16 +298,10 @@ export class MockServer { return this.requests[this.requests.length - 1]; } - /** - * Gets the number of requests made to a specific endpoint. - */ public getRequestCount(method: string, path: string): number { return this.getRequests(method, path).length; } - /** - * Gets all requests made to a specific endpoint. - */ public getRequests(method: string, path: string): RequestInfo[] { const pathWithoutQuery = path.split('?')[0]; return this.requests.filter( diff --git a/src/tests/backwards-compatibility/backwards-compatibility.test.ts b/src/tests/backwards-compatibility/backwards-compatibility.test.ts index 6bf50bf9..2ce66f4f 100644 --- a/src/tests/backwards-compatibility/backwards-compatibility.test.ts +++ b/src/tests/backwards-compatibility/backwards-compatibility.test.ts @@ -27,201 +27,6 @@ import { updateCurrentApiJson, } from './helpers'; -export const normalizeTypeText = (text: string): string => - text.replace(/\s/g, ''); - -/** - * Parse a normalized object type string (e.g. `{delay?:number;error?:{message:string};}`) - * into a map of property name -> { optional: boolean, type: string }. - * - * This handles nested braces so that e.g. `error?:{message:string;fileSize?:number}` - * is treated as a single property rather than being split at the inner semicolons. - */ -export const parseObjectProperties = ( - obj: string -): Map | null => { - const trimmed = obj.replace(/^\{/, '').replace(/\}$/, ''); - if (!trimmed) return new Map(); - - const properties = new Map(); - - // Walk character-by-character, tracking brace depth, to split on - // top-level semicolons only. - let depth = 0; - let current = ''; - for (const ch of trimmed) { - if (ch === '{') depth++; - if (ch === '}') depth--; - if (ch === ';' && depth === 0) { - if (current.length > 0) { - const colonIdx = current.indexOf(':'); - if (colonIdx === -1) return null; // not a valid property - let name = current.slice(0, colonIdx); - const type = current.slice(colonIdx + 1); - const optional = name.endsWith('?'); - if (optional) name = name.slice(0, -1); - properties.set(name, { optional, type }); - } - current = ''; - } else { - current += ch; - } - } - // Handle trailing segment (when the text doesn't end with ';') - if (current.length > 0) { - const colonIdx = current.indexOf(':'); - if (colonIdx !== -1) { - let name = current.slice(0, colonIdx); - const type = current.slice(colonIdx + 1); - const optional = name.endsWith('?'); - if (optional) name = name.slice(0, -1); - properties.set(name, { optional, type }); - } - } - - return properties; -}; - -/** - * Check whether `newMember` is a backwards-compatible evolution of - * `currentMember`. Both strings are normalised (whitespace stripped) - * object type literals. - * - * Compatible means: - * 1. Every property in `currentMember` still exists in `newMember`. - * 2. Existing properties that were in `currentMember` are structurally - * compatible in `newMember` (recursively, for nested object types). - * 3. Properties that were optional stay optional (not promoted to required). - * 4. Any *new* properties in `newMember` must be optional. - */ -export const isObjectTypeBackwardsCompatible = ( - currentMember: string, - newMember: string -): boolean => { - if (!currentMember.startsWith('{') || !newMember.startsWith('{')) { - return false; - } - - const currentProps = parseObjectProperties(currentMember); - const newProps = parseObjectProperties(newMember); - - if (!currentProps || !newProps) return false; - - // 1. Every current property must exist in new - for (const [name, currentProp] of currentProps) { - const newProp = newProps.get(name); - if (!newProp) return false; - - // 3. Optional properties must stay optional - if (currentProp.optional && !newProp.optional) return false; - - // 2. Property types must be compatible - // If both types are object types, recurse - if (currentProp.type.startsWith('{') && newProp.type.startsWith('{')) { - if (!isObjectTypeBackwardsCompatible(currentProp.type, newProp.type)) { - return false; - } - } else if (currentProp.type !== newProp.type) { - return false; - } - } - - // 4. New properties must be optional - for (const [name, newProp] of newProps) { - if (!currentProps.has(name) && !newProp.optional) { - return false; - } - } - - return true; -}; - -/** - * Splits a normalized intersection type (e.g. `Foo&{bar?:string}`) into its - * top-level `&`-separated members, respecting brace depth so that nested - * object types aren't split on their own semicolons/ampersands. - */ -const getIntersectionMembers = (text: string): string[] => { - const members: string[] = []; - let depth = 0; - let current = ''; - for (const ch of text) { - if (ch === '{') depth++; - if (ch === '}') depth--; - if (ch === '&' && depth === 0) { - members.push(current); - current = ''; - } else { - current += ch; - } - } - members.push(current); - return members; -}; - -/** - * Check whether a `newType` is a backwards-compatible evolution of - * `currentType`. Both are raw (non-normalized) type-excerpt strings. - * - * Handles two shapes of widening in addition to an exact match: - * - Inline object types gaining new optional properties (delegates to - * isObjectTypeBackwardsCompatible). - * - Intersection types gaining a new `& { ...optional fields }` member, e.g. - * `ErrorRecord` -> `ErrorRecord & { statusCode?: number }`. Every member of - * the current intersection must still be present (verbatim, or as a - * backwards-compatible object widening) in the new intersection, and any - * brand-new intersection members must themselves reduce to all-optional - * properties (so they don't add a new requirement). - */ -export const isTypeBackwardsCompatible = ( - currentType: string, - newType: string -): boolean => { - const currentText = normalizeTypeText(currentType); - const newText = normalizeTypeText(newType); - - if (currentText === newText) { - return true; - } - - if (currentText.startsWith('{') && newText.startsWith('{')) { - return isObjectTypeBackwardsCompatible(currentText, newText); - } - - if (currentText.includes('&') || newText.includes('&')) { - const currentMembers = getIntersectionMembers(currentText); - const newMembers = getIntersectionMembers(newText); - - // Every current member must still be satisfied by some new member. - const allCurrentMembersSatisfied = currentMembers.every( - (currentMember) => - newMembers.includes(currentMember) || - newMembers.some( - (newMember) => - currentMember.startsWith('{') && - newMember.startsWith('{') && - isObjectTypeBackwardsCompatible(currentMember, newMember) - ) - ); - if (!allCurrentMembersSatisfied) { - return false; - } - - // Any brand-new intersection member must only add optional properties, - // so it doesn't impose a new requirement on existing callers/values. - const newMembersAreOptionalOnly = newMembers.every((newMember) => { - if (currentMembers.includes(newMember)) return true; - if (!newMember.startsWith('{')) return false; - const props = parseObjectProperties(newMember); - return !!props && [...props.values()].every((p) => p.optional); - }); - - return newMembersAreOptionalOnly; - } - - return false; -}; - export function checkFunctionCompatibility( newFunction: ApiFunction | ApiConstructor | ApiMethodSignature, currentFunction: ApiFunction | ApiConstructor | ApiMethodSignature @@ -252,9 +57,7 @@ export function checkFunctionCompatibility( const newParam = newFunctionParamNames[i]; const currentParam = currentFunctionParamNames[i]; - // If both are destructured parameters (contain '{') if (newParam.includes('{') && currentParam.includes('{')) { - // Extract field names from destructured parameters const extractFields = (param: string) => param .replace(/[{}\s]/g, '') @@ -264,7 +67,7 @@ export function checkFunctionCompatibility( const newFields = extractFields(newParam); const currentFields = extractFields(currentParam); - // Check that all current fields are present in new fields + // All current fields must still be present in new fields const missingFields = currentFields.filter( (field) => !newFields.includes(field) ); @@ -286,8 +89,7 @@ export function checkFunctionCompatibility( expect(newFunctionParamTypes).toEqual(currentFunctionParameterTypes); }); - // Check return type compatibility - // This check fails if it's a constructor, as those don't have a return type + // Return type check — skipped for constructors, which have no return type if ( currentFunction instanceof ApiFunction && newFunction instanceof ApiFunction @@ -330,7 +132,6 @@ export function checkFunctionCompatibility( const newParam = newFunction.parameters[i]; const currentParam = currentFunction.parameters[i]; - // If current parameter was optional, new parameter should also be optional if (currentParam.isOptional && !newParam.isOptional) { throw new Error( `Parameter ${newParam.name} became required but was optional` @@ -395,7 +196,6 @@ describe('Backwards Compatibility', () => { (f: ApiFunction) => f.name === newFunction.name ); - // Skip if function doesn't exist in current API if (!currentFunction) { continue; } @@ -421,7 +221,6 @@ describe('Backwards Compatibility', () => { (c: ApiClass) => c.name === newClass.name ); - // Skip if class doesn't exist in current API if (!currentClass) { continue; } @@ -449,32 +248,26 @@ describe('Backwards Compatibility', () => { (p) => p.name === currentProperty.name ); if (newProperty && currentProperty.isOptional) { - // If the current property was optional, the new property should also be optional expect(newProperty.isOptional).toBe(true); } }); } }); - // Check property compatibility const oldProperties = currentClassProperties; const newProperties = newClassProperties; for (const newProperty of newProperties) { const currentProperty = oldProperties.find( (p: ApiProperty) => p.name === newProperty.name ); - // If the property is new, there's no need to check for compatibility if (!currentProperty) { continue; } - it(`Class ${newClass.name} property ${newProperty.name} should have a backwards-compatible type with the current property`, () => { - expect( - isTypeBackwardsCompatible( - currentProperty.propertyTypeExcerpt.text, - newProperty.propertyTypeExcerpt.text - ) - ).toBe(true); + it(`Class ${newClass.name} property ${newProperty.name} should have the same type as the current property`, () => { + expect(newProperty.propertyTypeExcerpt.text).toEqual( + currentProperty.propertyTypeExcerpt.text + ); }); it(`Class ${newClass.name} property ${newProperty.name} should have the same optionality as the current property`, () => { @@ -482,12 +275,11 @@ describe('Backwards Compatibility', () => { }); } - // Check constructor signature compatibility (same rules as functions) + // Constructor signatures follow the same compatibility rules as functions const currentMethod = getConstructor(currentClass.members); const newMethod = getConstructor(newClass.members); checkFunctionCompatibility(newMethod, currentMethod); - // Check method count const newClassMethods = getFunctions(newClass.members); const currentClassMethods = getFunctions(currentClass.members); @@ -500,13 +292,11 @@ describe('Backwards Compatibility', () => { } }); - // Check method compatibility (same rules as functions) - // Make sure to allow optional parameters to be added to the end + // Methods follow the same rules as functions (optional params may be appended) for (const newMethod of newClassMethods) { const currentMethod = currentClassMethods.find( (m: ApiFunction) => m.name === newMethod.name ); - // If the method is new, there's no need to check for compatibility if (!currentMethod) { continue; } @@ -558,40 +348,30 @@ describe('Backwards Compatibility', () => { ); }); - // Check property compatibility const oldProperties = currentInterfaceProperties; const newProperties = newInterfaceProperties; for (const newProperty of newProperties) { const currentProperty = oldProperties.find( (p: ApiPropertySignature) => p.name === newProperty.name ); - // If the property is new, there's no need to check for compatibility if (!currentProperty) { continue; } - it(`Interface ${newInterface.name} property ${newProperty.name} should have a backwards-compatible type with the current property`, () => { - expect( - isTypeBackwardsCompatible( - currentProperty.propertyTypeExcerpt.text, - newProperty.propertyTypeExcerpt.text - ) - ).toBe(true); + it(`Interface ${newInterface.name} property ${newProperty.name} should have the same type as the current property`, () => { + expect(newProperty.propertyTypeExcerpt.text).toEqual( + currentProperty.propertyTypeExcerpt.text + ); }); it(`Interface ${newInterface.name} property ${newProperty.name} should have not been made required if it was optional`, () => { - // If the new property is required, it must have been required before. - // Otherwise we break backwards-compatibility. + // optional -> required is a breaking change; required -> optional is fine expect( - // If it was required before, it can be either now. - !currentProperty.isOptional || - // If it was optional before, it can only be optional now. - newProperty.isOptional + !currentProperty.isOptional || newProperty.isOptional ).toEqual(true); }); } - // Check method count const newInterfaceMethods = getMethodSignatures(newInterface.members); const currentInterfaceMethods = getMethodSignatures( currentInterface.members @@ -603,13 +383,11 @@ describe('Backwards Compatibility', () => { ); }); - // Check method compatibility (same rules as functions) - // Make sure to allow optional parameters to be added to the end + // Methods follow the same rules as functions (optional params may be appended) for (const newMethod of newInterfaceMethods) { const currentMethod = currentInterfaceMethods.find( (m: ApiMethodSignature) => m.name === newMethod.name ); - // If the method is new, there's no need to check for compatibility if (!currentMethod) { continue; } @@ -632,13 +410,11 @@ describe('Backwards Compatibility', () => { newEnums = getEnums(newApiMembers); currentEnums = getEnums(currentApiMembers); - // Verify no enum values were removed for (const newEnum of newEnums) { const currentEnum = currentEnums.find( (e: ApiEnum) => e.name === newEnum.name ); - // If it's a new enum, there's no need to check for compatibility if (!currentEnum) { continue; } @@ -657,7 +433,6 @@ describe('Backwards Compatibility', () => { (v: ApiEnumMember) => v.name === currentEnumValue.name ); - // If it's a new enum value, there's no need to check for compatibility if (!newEnumValue) { continue; } @@ -669,7 +444,6 @@ describe('Backwards Compatibility', () => { } }); - // Verify numeric enum values haven't changed (if numeric enum) describe('should verify numeric enum values have not changed', () => { const { newApiMembers, currentApiMembers } = loadApiData(); newEnums = getEnums(newApiMembers); @@ -680,15 +454,13 @@ describe('Backwards Compatibility', () => { (e: ApiEnum) => e.name === newEnum.name ); - // If it's a new enum, there's no need to check for compatibility if (!currentEnum) { continue; } - // Helper function to determine if an enum is numeric based on its members' initializer values + // An enum is numeric if every member has a numeric initializer const isNumericEnum = (enumMembers: ApiEnumMember[]): boolean => { return enumMembers.every((member: ApiEnumMember) => { - // Check if the member has an initializer and if it's a numeric value const initializerText = member.excerptTokens ?.find( (token) => @@ -722,7 +494,6 @@ describe('Backwards Compatibility', () => { (v: ApiEnumMember) => v.name === currentEnumValue.name ); - // If it's not defined, an existing value is missing from the new enum it(`Enum ${newEnum.name} should contain enum value: ${currentEnumValue.name}`, () => { expect(newEnumValue).toBeDefined(); }); @@ -749,7 +520,6 @@ describe('Backwards Compatibility', () => { (e: ApiEnum) => e.name === newEnum.name ); - // If it's a new enum, there's no need to check for compatibility if (!currentEnum) { continue; } @@ -774,7 +544,6 @@ describe('Backwards Compatibility', () => { const newTypes = getTypes(newApiMembers); const currentTypes = getTypes(currentApiMembers); - // Verify type aliases weren't removed describe("should verify type aliases weren't removed", () => { for (const newType of newTypes) { const currentType = currentTypes.find( @@ -789,14 +558,119 @@ describe('Backwards Compatibility', () => { } }); - // Verify that the type alias is the same as the current type alias describe('should verify type aliases are the same as the current type aliases', () => { + const normalizeTypeText = (text: string): string => + text.replace(/\s/g, ''); const getUnionMembers = (text: string): string[] => normalizeTypeText(text) .split('|') .map((member) => member.trim()) .filter(Boolean); + /** + * Parse a normalized object type string into a map of property name -> + * { optional, type }. Tracks brace depth so nested object types aren't + * split at their inner semicolons. + */ + const parseObjectProperties = ( + obj: string + ): Map | null => { + const trimmed = obj.replace(/^\{/, '').replace(/\}$/, ''); + if (!trimmed) return new Map(); + + const properties = new Map< + string, + { optional: boolean; type: string } + >(); + + let depth = 0; + let current = ''; + for (const ch of trimmed) { + if (ch === '{') depth++; + if (ch === '}') depth--; + if (ch === ';' && depth === 0) { + if (current.length > 0) { + const colonIdx = current.indexOf(':'); + if (colonIdx === -1) return null; // not a valid property + let name = current.slice(0, colonIdx); + const type = current.slice(colonIdx + 1); + const optional = name.endsWith('?'); + if (optional) name = name.slice(0, -1); + properties.set(name, { optional, type }); + } + current = ''; + } else { + current += ch; + } + } + // Handle trailing segment (when the text doesn't end with ';') + if (current.length > 0) { + const colonIdx = current.indexOf(':'); + if (colonIdx !== -1) { + let name = current.slice(0, colonIdx); + const type = current.slice(colonIdx + 1); + const optional = name.endsWith('?'); + if (optional) name = name.slice(0, -1); + properties.set(name, { optional, type }); + } + } + + return properties; + }; + + /** + * Compatible means: (1) every current property still exists, (2) types + * stay compatible (recursively for nested objects), (3) optional stays + * optional, (4) any new property is optional. Inputs are normalised + * object type literals. + */ + const isObjectTypeBackwardsCompatible = ( + currentMember: string, + newMember: string + ): boolean => { + if (!currentMember.startsWith('{') || !newMember.startsWith('{')) { + return false; + } + + const currentProps = parseObjectProperties(currentMember); + const newProps = parseObjectProperties(newMember); + + if (!currentProps || !newProps) return false; + + // 1. Every current property must exist in new + for (const [name, currentProp] of currentProps) { + const newProp = newProps.get(name); + if (!newProp) return false; + + // 3. Optional properties must stay optional + if (currentProp.optional && !newProp.optional) return false; + + // 2. Property types must be compatible + // If both types are object types, recurse + if ( + currentProp.type.startsWith('{') && + newProp.type.startsWith('{') + ) { + if ( + !isObjectTypeBackwardsCompatible(currentProp.type, newProp.type) + ) { + return false; + } + } else if (currentProp.type !== newProp.type) { + return false; + } + } + + // 4. New properties must be optional + for (const [name, newProp] of newProps) { + if (!currentProps.has(name) && !newProp.optional) { + return false; + } + } + + return true; + }; + for (const newType of newTypes) { const currentType = currentTypes.find( (t: ApiTypeAlias) => t.name === newType.name @@ -819,7 +693,6 @@ describe('Backwards Compatibility', () => { expect(!!currentUnionMembers.length).toBe(true); for (const member of currentUnionMembers) { - // Exact match: member is unchanged if (newUnionMembersSet.has(member)) { continue; } diff --git a/src/tests/backwards-compatibility/helpers.ts b/src/tests/backwards-compatibility/helpers.ts index dd0cee08..e0deee83 100644 --- a/src/tests/backwards-compatibility/helpers.ts +++ b/src/tests/backwards-compatibility/helpers.ts @@ -1,3 +1,7 @@ +import { execSync } from 'node:child_process'; +import * as fs from 'fs'; +import * as path from 'path'; + import { Extractor, ExtractorConfig, @@ -17,10 +21,6 @@ import { ApiTypeAlias, } from '@microsoft/api-extractor-model'; -import * as fs from 'fs'; -import { execSync } from 'node:child_process'; -import * as path from 'path'; - export const newApiMdPath = path.join(__dirname, 'temp', 'ts-adaas.md'); export const currentApiMdPath = path.join(__dirname, 'ts-adaas.md'); export const newApiJsonPath = path.join(__dirname, 'temp', 'ts-adaas.api.json'); @@ -28,15 +28,13 @@ export const currentApiJsonPath = path.join(__dirname, 'latest.json'); /* eslint-disable @typescript-eslint/no-explicit-any */ -// Generate API report before all tests run export function generateApiReport(): void { - // Before running the api extractor, make sure that the code compiles using `tsc` command + // Make sure the code compiles before running the api extractor const tscCommand = 'npm run build'; try { execSync(tscCommand) as any; } catch (error: any) { - // Jest has a nice feature: if any of the setup scripts throw an error, the test run will fail - // This Error is rethrown to get more information from the error. + // Rethrow with stdout so Jest fails the run with the tsc output throw new Error( `Failed to compile code using tsc command:\n${error.stdout.toString()}` ); @@ -47,7 +45,6 @@ export function generateApiReport(): void { 'api-extractor.json' ); - // Ensure the temp and report directories exist const tempDir = path.join(__dirname, 'temp'); const reportDir = path.join(__dirname, 'report'); @@ -81,7 +78,6 @@ export function generateApiReport(): void { } } -// Helper function to load API data export const loadApiData = (): { newApiMembers: readonly ApiItem[]; currentApiMembers: readonly ApiItem[]; @@ -107,7 +103,6 @@ export const loadApiData = (): { return { newApiMembers, currentApiMembers }; }; -// Helper functions for getting different kinds of items from the API members export const getFunctions = (members: readonly ApiItem[]): ApiFunction[] => { return members.filter( (m: ApiItem) => m instanceof ApiFunction && m.kind === 'Function' @@ -174,7 +169,6 @@ export const updateCurrentApiJson = () => { if (fs.existsSync(newApiMdPath) && fs.existsSync(newApiJsonPath)) { fs.copyFileSync(newApiMdPath, currentApiMdPath); - // Copy new API JSON into latest.json after all tests pass const latestJsonPath = path.join(__dirname, 'latest.json'); fs.copyFileSync(newApiJsonPath, latestJsonPath); diff --git a/src/tests/backwards-compatibility/jest.setup.ts b/src/tests/backwards-compatibility/jest.setup.ts index eec04285..c204eeaf 100644 --- a/src/tests/backwards-compatibility/jest.setup.ts +++ b/src/tests/backwards-compatibility/jest.setup.ts @@ -1,4 +1,3 @@ import { generateApiReport } from './helpers'; -// Run the API report generation generateApiReport(); diff --git a/src/tests/control-protocol-reconciliation.test.ts b/src/tests/control-protocol-reconciliation.test.ts index 26752f2f..d0ed087c 100644 --- a/src/tests/control-protocol-reconciliation.test.ts +++ b/src/tests/control-protocol-reconciliation.test.ts @@ -1,5 +1,5 @@ +import { createMockEvent } from '../testing/mock-event'; import { mockServer } from '../tests/jest.setup'; -import { createMockEvent } from '../common/test-utils'; import { EventType, TimeValueType } from '../types/extraction'; describe('Enhanced Control Protocol', () => { diff --git a/src/tests/dummy-connector/data-extraction.test.ts b/src/tests/dummy-connector/data-extraction.test.ts index 82b812a1..2d1e3b26 100644 --- a/src/tests/dummy-connector/data-extraction.test.ts +++ b/src/tests/dummy-connector/data-extraction.test.ts @@ -1,15 +1,15 @@ +import { createMockEvent } from '../../testing/mock-event'; import { - AirdropEvent, + AirSyncEvent, EventType, ExtractorEventType, } from '../../types/extraction'; import { mockServer } from '../jest.setup'; -import { createMockEvent } from '../../common/test-utils'; import run from './extraction'; describe('Dummy Connector - Data Extraction', () => { - let event: AirdropEvent; + let event: AirSyncEvent; beforeEach(() => { event = createMockEvent(mockServer.baseUrl, { payload: { event_type: EventType.StartExtractingData }, diff --git a/src/tests/dummy-connector/data-extraction.ts b/src/tests/dummy-connector/data-extraction.ts index 8f1f7a6f..57311cbe 100644 --- a/src/tests/dummy-connector/data-extraction.ts +++ b/src/tests/dummy-connector/data-extraction.ts @@ -1,7 +1,6 @@ import { - ExtractorEventType, NormalizedItem, - processTask, + processExtractionTask, RepoInterface, } from '../../index'; @@ -25,7 +24,7 @@ const repos: RepoInterface[] = [ }, ]; -processTask({ +processExtractionTask({ task: async ({ adapter }) => { adapter.initializeRepos(repos); @@ -44,9 +43,10 @@ processTask({ } await adapter.getRepo('tasks')?.push(tasks); - await adapter.emit(ExtractorEventType.DataExtractionDone); + return { status: 'success' }; }, - onTimeout: async ({ adapter }) => { - await adapter.emit(ExtractorEventType.DataExtractionProgress); + // eslint-disable-next-line @typescript-eslint/require-await + onTimeout: async () => { + return { status: 'progress' }; }, }); diff --git a/src/tests/dummy-connector/external-sync-units-extraction.test.ts b/src/tests/dummy-connector/external-sync-units-extraction.test.ts index d0f87c50..259924de 100644 --- a/src/tests/dummy-connector/external-sync-units-extraction.test.ts +++ b/src/tests/dummy-connector/external-sync-units-extraction.test.ts @@ -1,6 +1,6 @@ +import { createMockEvent } from '../../testing/mock-event'; import { EventType, ExtractorEventType } from '../../types/extraction'; import { mockServer } from '../jest.setup'; -import { createMockEvent } from '../../common/test-utils'; import run from './extraction'; diff --git a/src/tests/dummy-connector/external-sync-units-extraction.ts b/src/tests/dummy-connector/external-sync-units-extraction.ts index 0cb6d2f6..c091f1a1 100644 --- a/src/tests/dummy-connector/external-sync-units-extraction.ts +++ b/src/tests/dummy-connector/external-sync-units-extraction.ts @@ -1,6 +1,10 @@ -import { ExternalSyncUnit, ExtractorEventType, processTask } from '../../index'; +import { + AirSyncDefaultItemTypes, + ExternalSyncUnit, + processExtractionTask, +} from '../../index'; -processTask({ +processExtractionTask({ task: async ({ adapter }) => { const dummyExternalSyncUnits: ExternalSyncUnit[] = [ { @@ -12,15 +16,26 @@ processTask({ }, ]; - await adapter.emit(ExtractorEventType.ExternalSyncUnitExtractionDone, { - external_sync_units: dummyExternalSyncUnits, - }); + adapter.initializeRepos([ + { + itemType: AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS, + overridenOptions: { batchSize: 25000, skipConfirmation: true }, + }, + ]); + + await adapter + .getRepo(AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS) + ?.push(dummyExternalSyncUnits); + + return { status: 'success' }; }, - onTimeout: async ({ adapter }) => { - await adapter.emit(ExtractorEventType.ExternalSyncUnitExtractionError, { + // eslint-disable-next-line @typescript-eslint/require-await + onTimeout: async () => { + return { + status: 'error', error: { message: 'Failed to extract external sync units. Lambda timeout.', }, - }); + }; }, }); diff --git a/src/tests/dummy-connector/extraction.ts b/src/tests/dummy-connector/extraction.ts index 4889fb6a..5ff952b8 100644 --- a/src/tests/dummy-connector/extraction.ts +++ b/src/tests/dummy-connector/extraction.ts @@ -1,4 +1,4 @@ -import { AirdropEvent, spawn } from '../../index'; +import { AirSyncEvent, spawn } from '../../index'; interface ExtractorState { [key: string]: unknown; @@ -7,15 +7,18 @@ interface ExtractorState { const initialState = {}; const initialDomainMapping = {}; -const run = async (events: AirdropEvent[], workerPath: string) => { +const run = async (events: AirSyncEvent[], workerPath: string) => { for (const event of events) { await spawn({ event, initialState, - workerPath, initialDomainMapping, + baseWorkerPath: '', options: { isLocalDevelopment: true, + workerPathOverrides: workerPath + ? { [event.payload.event_type]: workerPath } + : undefined, }, }); } diff --git a/src/tests/dummy-connector/metadata-extraction.test.ts b/src/tests/dummy-connector/metadata-extraction.test.ts index 18ac043a..51c1812c 100644 --- a/src/tests/dummy-connector/metadata-extraction.test.ts +++ b/src/tests/dummy-connector/metadata-extraction.test.ts @@ -1,17 +1,17 @@ +import { createMockEvent } from '../../testing/mock-event'; import { - AirdropEvent, + AirSyncEvent, EventType, ExtractorEventType, } from '../../types/extraction'; import { mockServer } from '../jest.setup'; -import { createMockEvent } from '../../common/test-utils'; import run from './extraction'; jest.setTimeout(60000); describe('Dummy Connector - Metadata Extraction', () => { - let event: AirdropEvent; + let event: AirSyncEvent; beforeEach(() => { event = createMockEvent(mockServer.baseUrl, { payload: { event_type: EventType.StartExtractingMetadata }, diff --git a/src/tests/dummy-connector/metadata-extraction.ts b/src/tests/dummy-connector/metadata-extraction.ts index 3427a559..89b383cf 100644 --- a/src/tests/dummy-connector/metadata-extraction.ts +++ b/src/tests/dummy-connector/metadata-extraction.ts @@ -1,4 +1,4 @@ -import { ExtractorEventType, processTask } from '../../index'; +import { processExtractionTask } from '../../index'; const repos = [ { @@ -6,7 +6,7 @@ const repos = [ }, ]; -processTask({ +processExtractionTask({ task: async ({ adapter }) => { adapter.initializeRepos(repos); @@ -16,11 +16,13 @@ processTask({ .getRepo('external_domain_metadata') ?.push([externalDomainMetadata]); - await adapter.emit(ExtractorEventType.MetadataExtractionDone); + return { status: 'success' }; }, - onTimeout: async ({ adapter }) => { - await adapter.emit(ExtractorEventType.MetadataExtractionError, { + // eslint-disable-next-line @typescript-eslint/require-await + onTimeout: async () => { + return { + status: 'error', error: { message: 'Failed to extract metadata. Lambda timeout.' }, - }); + }; }, }); diff --git a/src/tests/event-data-size-limit/extraction.ts b/src/tests/event-data-size-limit/extraction.ts index 9fa4a366..80066b92 100644 --- a/src/tests/event-data-size-limit/extraction.ts +++ b/src/tests/event-data-size-limit/extraction.ts @@ -1,4 +1,4 @@ -import { AirdropEvent, spawn } from '../../index'; +import { AirSyncEvent, spawn } from '../../index'; interface ExtractorState { [key: string]: unknown; @@ -8,21 +8,22 @@ const initialState = {}; const initialDomainMapping = {}; /** - * Run function for attachment size limit tests. - * Uses batch size of 1 to create many artifacts. - * With 3000 items and batch size 1, we get 3000 artifacts. - * Each artifact metadata is ~55 bytes, so 3000 * 55 = 165KB > 160KB threshold. + * Run function for size limit tests. Batch size 1 makes each item an artifact + * (~55 bytes of metadata), so 3000 items (~165KB) exceed the 160KB threshold. */ -const run = async (events: AirdropEvent[], workerPath: string) => { +const run = async (events: AirSyncEvent[], workerPath: string) => { for (const event of events) { await spawn({ event, initialState, - workerPath, initialDomainMapping, + baseWorkerPath: '', options: { - batchSize: 1, // Batch size of 1 to generate many artifacts + batchSize: 1, isLocalDevelopment: true, + workerPathOverrides: workerPath + ? { [event.payload.event_type]: workerPath } + : undefined, }, }); } diff --git a/src/tests/event-data-size-limit/size-limit-1.test.ts b/src/tests/event-data-size-limit/size-limit-1.test.ts index 72850e8e..f62c30db 100644 --- a/src/tests/event-data-size-limit/size-limit-1.test.ts +++ b/src/tests/event-data-size-limit/size-limit-1.test.ts @@ -1,10 +1,11 @@ +import { createMockEvent } from '../../testing/mock-event'; import { EventType, ExtractorEvent, ExtractorEventType, } from '../../types/extraction'; import { mockServer } from '../jest.setup'; -import { createMockEvent } from '../../common/test-utils'; + import run from './extraction'; // Increase timeout for this test since we're doing many uploads diff --git a/src/tests/event-data-size-limit/size-limit-1.ts b/src/tests/event-data-size-limit/size-limit-1.ts index aff16624..0c69e0d4 100644 --- a/src/tests/event-data-size-limit/size-limit-1.ts +++ b/src/tests/event-data-size-limit/size-limit-1.ts @@ -1,16 +1,13 @@ -import { ExtractorEventType, processTask } from '../../index'; +import { processExtractionTask } from '../../index'; /** - * Test worker that generates items to trigger the SQS size limit. - * - * The size limit is 160KB (80% of 200KB max). - * With batch size 1, each item creates 1 artifact. - * Each artifact metadata is ~55 bytes (id, item_type, item_count). - * We need ~2857 artifacts to reach 160KB, so generating 3000 items. + * Test worker that triggers the SQS size limit (160KB = 80% of 200KB max). + * Batch size 1 makes each item an artifact; artifact metadata is ~55 bytes, + * so 3000 items (~165KB) exceed the threshold (~2857 needed). */ -processTask({ +processExtractionTask({ task: async ({ adapter }) => { - // Using external_domain_metadata itemType which doesn't require normalize + // external_domain_metadata itemType doesn't require normalize adapter.initializeRepos([ { itemType: 'external_domain_metadata', @@ -20,15 +17,12 @@ processTask({ const repo = adapter.getRepo('external_domain_metadata'); if (!repo) { console.error('Repo not found after init'); - await adapter.emit(ExtractorEventType.DataExtractionError, { + return { + status: 'error', error: { message: 'Repo not found after init!' }, - }); - return; + }; } - // Generate 3000 items with batch size 1 = 3000 artifacts - // Each artifact metadata is ~55 bytes (id, item_type, item_count) - // 3000 * 55 = 165KB > 160KB threshold for (let i = 0; i < 3000; i++) { await repo.push([ { @@ -41,15 +35,16 @@ processTask({ ]); if (adapter.isTimeout) { - return; + return { status: 'progress' }; } } console.log('Size limit was NOT triggered, emitting done'); - await adapter.emit(ExtractorEventType.DataExtractionDone); + return { status: 'success' }; }, - onTimeout: async ({ adapter }) => { + // eslint-disable-next-line @typescript-eslint/require-await + onTimeout: async () => { console.log('onTimeout called - emitting progress'); - await adapter.emit(ExtractorEventType.DataExtractionProgress); + return { status: 'progress' }; }, }); diff --git a/src/tests/external-domain-metadata/external-domain-metadata.test.ts b/src/tests/external-domain-metadata/external-domain-metadata.test.ts index edcb92d2..d4733c14 100644 --- a/src/tests/external-domain-metadata/external-domain-metadata.test.ts +++ b/src/tests/external-domain-metadata/external-domain-metadata.test.ts @@ -3,12 +3,14 @@ // against the JSON schema at runtime using Ajv, catching any drift between the two. import Ajv, { ValidateFunction } from 'ajv'; -import * as schema from './external_domain_metadata_schema.json'; + import { ExternalDomainMetadata, Field, } from '../../types/external-domain-metadata'; +import * as schema from './external_domain_metadata_schema.json'; + const ajv = new Ajv({ allErrors: true }); const validate: ValidateFunction = ajv.compile(schema); diff --git a/src/tests/extract-from-collision.test.ts b/src/tests/extract-from-collision.test.ts index a5e8dbb5..fa80b78e 100644 --- a/src/tests/extract-from-collision.test.ts +++ b/src/tests/extract-from-collision.test.ts @@ -1,5 +1,5 @@ +import { createMockEvent } from '../testing/mock-event'; import { mockServer } from '../tests/jest.setup'; -import { createMockEvent } from '../common/test-utils'; import { EventContext, EventType, TimeValueType } from '../types/extraction'; /** diff --git a/src/tests/jest.setup.ts b/src/tests/jest.setup.ts index 87c86273..8887f19b 100644 --- a/src/tests/jest.setup.ts +++ b/src/tests/jest.setup.ts @@ -1,4 +1,4 @@ -import { MockServer } from '../mock-server/mock-server'; +import { MockServer } from '../testing/mock-server'; // Use port 0 for dynamic port allocation, enabling parallel test execution export const mockServer = new MockServer(0); diff --git a/src/tests/spawn-worker/delete-event-type.test.ts b/src/tests/spawn-worker/delete-event-type.test.ts index b2b2e445..c6c53481 100644 --- a/src/tests/spawn-worker/delete-event-type.test.ts +++ b/src/tests/spawn-worker/delete-event-type.test.ts @@ -1,6 +1,6 @@ +import { createMockEvent } from '../../testing/mock-event'; import { EventType, ExtractorEventType } from '../../types/extraction'; import { mockServer } from '../jest.setup'; -import { createMockEvent } from '../../common/test-utils'; import run from './extraction'; diff --git a/src/tests/spawn-worker/extraction.ts b/src/tests/spawn-worker/extraction.ts index ed0023b7..8965667a 100644 --- a/src/tests/spawn-worker/extraction.ts +++ b/src/tests/spawn-worker/extraction.ts @@ -1,4 +1,4 @@ -import { AirdropEvent, spawn } from '../../index'; +import { AirSyncEvent, spawn } from '../../index'; interface ExtractorState { [key: string]: unknown; @@ -7,15 +7,18 @@ interface ExtractorState { const initialState = {}; const initialDomainMapping = {}; -const run = async (events: AirdropEvent[], workerPath?: string) => { +const run = async (events: AirSyncEvent[], workerPath?: string) => { for (const event of events) { await spawn({ event, initialState, - workerPath, initialDomainMapping, + baseWorkerPath: '', options: { isLocalDevelopment: true, + workerPathOverrides: workerPath + ? { [event.payload.event_type]: workerPath } + : undefined, }, }); } diff --git a/src/tests/spawn-worker/some-cleanup-worker.ts b/src/tests/spawn-worker/some-cleanup-worker.ts index 268c5f07..52934d85 100644 --- a/src/tests/spawn-worker/some-cleanup-worker.ts +++ b/src/tests/spawn-worker/some-cleanup-worker.ts @@ -1,15 +1,18 @@ -import { ExtractorEventType, processTask } from '../../index'; +import { processExtractionTask } from '../../index'; -processTask({ - task: async ({ adapter }) => { +processExtractionTask({ + // eslint-disable-next-line @typescript-eslint/require-await + task: async () => { console.log('Some cleanup logic executed.'); - await adapter.emit(ExtractorEventType.ExtractorStateDeletionDone); + return { status: 'success' }; }, - onTimeout: async ({ adapter }) => { - await adapter.emit(ExtractorEventType.ExtractorStateDeletionError, { + // eslint-disable-next-line @typescript-eslint/require-await + onTimeout: async () => { + return { + status: 'error', error: { message: 'Failed to execute cleanup logic. Lambda timeout.', }, - }); + }; }, }); diff --git a/src/tests/spawn-worker/unknown-event-type.test.ts b/src/tests/spawn-worker/unknown-event-type.test.ts index 00ca5f0a..26e5335d 100644 --- a/src/tests/spawn-worker/unknown-event-type.test.ts +++ b/src/tests/spawn-worker/unknown-event-type.test.ts @@ -1,6 +1,7 @@ -import { EventType, ExtractorEventType } from '../../types/extraction'; +import { UNKNOWN_EVENT_TYPE } from '../../common/constants'; +import { createMockEvent } from '../../testing/mock-event'; +import { EventType } from '../../types/extraction'; import { mockServer } from '../jest.setup'; -import { createMockEvent } from '../../common/test-utils'; import run from './extraction'; @@ -18,7 +19,7 @@ describe('Unknown event type', () => { expect(lastRequest?.url).toContain('/callback_url'); expect(lastRequest?.method).toBe('POST'); expect((lastRequest?.body as { event_type: string }).event_type).toBe( - ExtractorEventType.UnknownEventType + UNKNOWN_EVENT_TYPE ); }); }); diff --git a/src/tests/spawn-worker/unknown-event-type.ts b/src/tests/spawn-worker/unknown-event-type.ts index 05284e4e..02c8e7cc 100644 --- a/src/tests/spawn-worker/unknown-event-type.ts +++ b/src/tests/spawn-worker/unknown-event-type.ts @@ -1,12 +1,14 @@ -import { processTask } from '../../index'; +import { processExtractionTask } from '../../index'; -processTask({ - // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/require-await - task: async ({ adapter }) => { +processExtractionTask({ + // eslint-disable-next-line @typescript-eslint/require-await + task: async () => { console.log('task should not be called.'); + return { status: 'success' }; }, - // eslint-disable-next-line @typescript-eslint/no-unused-vars, @typescript-eslint/require-await - onTimeout: async ({ adapter }) => { + // eslint-disable-next-line @typescript-eslint/require-await + onTimeout: async () => { console.log('onTimeout should not be called.'); + return { status: 'success' }; }, }); diff --git a/src/tests/test-helpers.interfaces.ts b/src/tests/test-helpers.interfaces.ts index 085d7c3a..ee1c4fb9 100644 --- a/src/tests/test-helpers.interfaces.ts +++ b/src/tests/test-helpers.interfaces.ts @@ -1,26 +1,16 @@ -import { DeepPartial } from '../common/test-utils'; -import { AirdropEvent } from '../types/extraction'; +import { DeepPartial } from '../testing/mock-event'; +import { AirSyncEvent } from '../types/extraction'; -/** - * Internal variant of the createMockEvent overrides — a deep partial of - * {@link AirdropEvent}. The shared test wrapper injects defaults automatically. - */ -export type CreateMockEventOverrides = DeepPartial; +/** Overrides for createMockEvent; the shared test wrapper injects defaults. */ +export type CreateMockEventOverrides = DeepPartial; -/** - * Options for creating a file stream response. - */ export interface CreateFileStreamOptions { - /** File content as Buffer or string (default: 'test file content') */ content?: Buffer | string; - /** Override content-length header (auto-calculated from content if not provided) */ + /** Overrides the content-length header (defaults to actual content length) */ contentLength?: number; - /** Set to false to omit content-length header (for testing missing header scenarios) */ + /** Set to false to omit the content-length header entirely */ includeContentLength?: boolean; - /** Optional filename for metadata */ filename?: string; - /** Optional MIME type (default: 'application/octet-stream') */ mimeType?: string; - /** Optional custom destroy function for testing stream cleanup */ destroyFn?: () => void; } diff --git a/src/tests/test-helpers.ts b/src/tests/test-helpers.ts index 74ff77ad..f0ee940e 100644 --- a/src/tests/test-helpers.ts +++ b/src/tests/test-helpers.ts @@ -1,11 +1,14 @@ -import { AxiosResponse } from 'axios'; import { Readable } from 'stream'; + +import { AxiosResponse } from 'axios'; + import { Item, NormalizedAttachment, NormalizedItem, } from '../repo/repo.interfaces'; import { ArtifactToUpload } from '../uploader/uploader.interfaces'; + import { CreateFileStreamOptions } from './test-helpers.interfaces'; export function createItem(id: number): Item { @@ -46,10 +49,6 @@ export function createAttachments(count: number): NormalizedAttachment[] { return Array.from({ length: count }, (_, index) => createAttachment(index)); } -/** - * Creates a mock artifact object for testing upload flows. - * Use the `overrides` parameter to customize specific fields for your test case. - */ export function createArtifact( overrides: Partial = {} ): ArtifactToUpload { @@ -61,10 +60,6 @@ export function createArtifact( }; } -/** - * Creates a mock Axios success response for testing HTTP calls. - * Use the `overrides` parameter to customize response properties. - */ export function createAxiosResponse( overrides: Partial = {} ): AxiosResponse { @@ -78,10 +73,6 @@ export function createAxiosResponse( } as AxiosResponse; } -/** - * Creates a mock download URL response matching the DevRev API format. - * Used when testing artifact download flows. - */ export function createDownloadUrlResponse( downloadUrl = 'https://s3.example.com/download' ) { @@ -90,18 +81,10 @@ export function createDownloadUrlResponse( }; } -/** - * Creates a mock file buffer for testing file upload/download operations. - * Use the `content` parameter to customize the file content. - */ export function createFileBuffer(content = 'test file content'): Buffer { return Buffer.from(content); } -/** - * Creates an AxiosResponse-like object with a Readable stream for testing file streaming operations. - * Useful for testing upload/download flows that work with streamed file data. - */ export function createFileStream( options: CreateFileStreamOptions = {} ): AxiosResponse { @@ -146,13 +129,8 @@ export function createFileStream( } /** - * Calls a private method on an instance. - * Use with a type parameter to get the specific method signature. - * - * @example - * type MyClassPrivate = { privateMethod: (x: number) => string }; - * const fn = callPrivateMethod()(instance, 'privateMethod'); - * const result = fn(42); + * Calls a private method on an instance. Curried so the private-method map is + * supplied as a type parameter: callPrivateMethod()(instance, 'method'). */ export function callPrivateMethod() { return ( @@ -164,14 +142,6 @@ export function callPrivateMethod() { }; } -/** - * Spies on a private method of an instance. - * - * @example - * type MyClassPrivate = { privateMethod: (x: number) => string }; - * const spy = spyOnPrivateMethod(instance, 'privateMethod'); - * spy.mockResolvedValueOnce('mocked'); - */ export function spyOnPrivateMethod( instance: object, methodName: keyof TPrivateMethods diff --git a/src/tests/timeout-handling/attachments-extraction.ts b/src/tests/timeout-handling/attachments-extraction.ts index e17cbba0..edd6966e 100644 --- a/src/tests/timeout-handling/attachments-extraction.ts +++ b/src/tests/timeout-handling/attachments-extraction.ts @@ -1,4 +1,4 @@ -import { AirdropEvent, spawn } from '../../index'; +import { AirSyncEvent, spawn } from '../../index'; interface ExtractorState { [key: string]: unknown; @@ -7,17 +7,20 @@ interface ExtractorState { const initialState = {}; const initialDomainMapping = {}; -const run = async (events: AirdropEvent[], workerPath: string) => { +const run = async (events: AirSyncEvent[], workerPath: string) => { for (const event of events) { await spawn({ event, initialState, - workerPath, initialDomainMapping, + baseWorkerPath: '', options: { batchSize: 1000, timeout: 4 * 1000, // 4s soft timeout -> ~5.2s hard timeout isLocalDevelopment: true, + workerPathOverrides: workerPath + ? { [event.payload.event_type]: workerPath } + : undefined, }, }); } diff --git a/src/tests/timeout-handling/attachments-timeout-hung-stream.ts b/src/tests/timeout-handling/attachments-timeout-hung-stream.ts index 4567e619..89e1d3ad 100644 --- a/src/tests/timeout-handling/attachments-timeout-hung-stream.ts +++ b/src/tests/timeout-handling/attachments-timeout-hung-stream.ts @@ -1,17 +1,18 @@ -import { AxiosResponse } from 'axios'; import { Readable } from 'stream'; -import { ExtractorEventType, processTask } from '../../index'; +import { AxiosResponse } from 'axios'; + +import { processExtractionTask } from '../../index'; import { - ExternalSystemAttachmentStreamingResponse, ExternalSystemAttachmentStreamingParams, + ExternalSystemAttachmentStreamingResponse, } from '../../types/extraction'; // Repro for logs2.csv: one attachment's stream() hangs forever, keeping a pool // worker (and streamAll) pending past the soft timeout. -processTask({ +processExtractionTask({ task: async ({ adapter }) => { - await adapter.streamAttachments({ + return adapter.streamAttachments({ stream: async ({ item, }: ExternalSystemAttachmentStreamingParams): Promise => { @@ -29,10 +30,9 @@ processTask({ }, batchSize: 10, }); - - await adapter.emit(ExtractorEventType.AttachmentExtractionDone); }, - onTimeout: async ({ adapter }) => { - await adapter.emit(ExtractorEventType.AttachmentExtractionProgress); + // eslint-disable-next-line @typescript-eslint/require-await + onTimeout: async () => { + return { status: 'progress' }; }, }); diff --git a/src/tests/timeout-handling/attachments-timeout-retry-storm.ts b/src/tests/timeout-handling/attachments-timeout-retry-storm.ts index e375fd54..0a68cbb9 100644 --- a/src/tests/timeout-handling/attachments-timeout-retry-storm.ts +++ b/src/tests/timeout-handling/attachments-timeout-retry-storm.ts @@ -1,15 +1,16 @@ -import { AxiosResponse } from 'axios'; import { Readable } from 'stream'; -import { ExtractorEventType, processTask } from '../../index'; +import { AxiosResponse } from 'axios'; + +import { processExtractionTask } from '../../index'; import { ExternalSystemAttachmentStreamingResponse } from '../../types/extraction'; // Repro for logs1.csv: stream() succeeds but the upload 5xxs, so axios-retry // backs off (2s, 4s, ...) and a pool worker is stuck mid-retry across the soft // timeout (it only re-checks the flag between attachments, never mid-retry). -processTask({ +processExtractionTask({ task: async ({ adapter }) => { - await adapter.streamAttachments({ + return adapter.streamAttachments({ stream: async (): Promise => { await Promise.resolve(); const body = Buffer.from('hello world'); @@ -26,10 +27,9 @@ processTask({ }, batchSize: 10, }); - - await adapter.emit(ExtractorEventType.AttachmentExtractionDone); }, - onTimeout: async ({ adapter }) => { - await adapter.emit(ExtractorEventType.AttachmentExtractionProgress); + // eslint-disable-next-line @typescript-eslint/require-await + onTimeout: async () => { + return { status: 'progress' }; }, }); diff --git a/src/tests/timeout-handling/attachments-timeout.test.ts b/src/tests/timeout-handling/attachments-timeout.test.ts index 93ff1ef4..04e400b3 100644 --- a/src/tests/timeout-handling/attachments-timeout.test.ts +++ b/src/tests/timeout-handling/attachments-timeout.test.ts @@ -1,15 +1,16 @@ import zlib from 'zlib'; + import { jsonl } from 'js-jsonl'; +import { NormalizedAttachment } from '../../repo/repo.interfaces'; +import { createMockEvent } from '../../testing/mock-event'; import { - AirdropEvent, + AirSyncEvent, EventType, ExtractorEvent, ExtractorEventType, } from '../../types/extraction'; -import { NormalizedAttachment } from '../../repo/repo.interfaces'; import { mockServer } from '../jest.setup'; -import { createMockEvent } from '../../common/test-utils'; import run from './attachments-extraction'; @@ -50,18 +51,19 @@ function seedAttachmentsState( attachments: NormalizedAttachment[] ): void { const state = { - lastSyncStarted: '', - lastSuccessfulSyncStarted: '', - pendingWorkersOldest: '', - pendingWorkersNewest: '', - workersOldest: '', - workersNewest: '', - snapInVersionId: 'test_snap_in_version_id', - toDevRev: { - attachmentsMetadata: { - artifactIds: [METADATA_ARTIFACT_ID], - lastProcessed: 0, - lastProcessedAttachmentsIdsList: [], + connectorState: {}, + sdkState: { + pendingWorkersOldest: '', + pendingWorkersNewest: '', + workersOldest: '', + workersNewest: '', + snapInVersionId: 'test_snap_in_version_id', + toDevRev: { + attachmentsMetadata: { + artifactIds: [METADATA_ARTIFACT_ID], + lastProcessed: 0, + lastProcessedAttachmentsIdsList: [], + }, }, }, }; @@ -91,7 +93,7 @@ function seedAttachmentsState( } describe('Attachments streaming soft timeout', () => { - let event: AirdropEvent; + let event: AirSyncEvent; beforeEach(() => { event = createMockEvent(mockServer.baseUrl, { diff --git a/src/tests/timeout-handling/extraction.ts b/src/tests/timeout-handling/extraction.ts index f24232aa..bc9d00d6 100644 --- a/src/tests/timeout-handling/extraction.ts +++ b/src/tests/timeout-handling/extraction.ts @@ -1,4 +1,4 @@ -import { AirdropEvent, spawn } from '../../index'; +import { AirSyncEvent, spawn } from '../../index'; interface ExtractorState { [key: string]: unknown; @@ -7,17 +7,20 @@ interface ExtractorState { const initialState = {}; const initialDomainMapping = {}; -const run = async (events: AirdropEvent[], workerPath: string) => { +const run = async (events: AirSyncEvent[], workerPath: string) => { for (const event of events) { await spawn({ event, initialState, - workerPath, initialDomainMapping, + baseWorkerPath: '', options: { batchSize: 1000, timeout: 5 * 1000, // 5 seconds isLocalDevelopment: true, + workerPathOverrides: workerPath + ? { [event.payload.event_type]: workerPath } + : undefined, }, }); } diff --git a/src/tests/timeout-handling/no-timeout.test.ts b/src/tests/timeout-handling/no-timeout.test.ts index 7b9df7dc..de455b09 100644 --- a/src/tests/timeout-handling/no-timeout.test.ts +++ b/src/tests/timeout-handling/no-timeout.test.ts @@ -1,16 +1,15 @@ +import { createMockEvent } from '../../testing/mock-event'; import { - AirdropEvent, + AirSyncEvent, EventType, ExtractorEventType, } from '../../types/extraction'; - import { mockServer } from '../jest.setup'; -import { createMockEvent } from '../../common/test-utils'; import run from './extraction'; describe('No timeout', () => { - let event: AirdropEvent; + let event: AirSyncEvent; beforeEach(() => { event = createMockEvent(mockServer.baseUrl, { payload: { event_type: EventType.StartExtractingData }, diff --git a/src/tests/timeout-handling/no-timeout.ts b/src/tests/timeout-handling/no-timeout.ts index 8eb47e39..2a87475b 100644 --- a/src/tests/timeout-handling/no-timeout.ts +++ b/src/tests/timeout-handling/no-timeout.ts @@ -1,14 +1,16 @@ -import { ExtractorEventType, processTask } from '../../index'; +import { processExtractionTask } from '../../index'; -processTask({ - task: async ({ adapter }) => { +processExtractionTask({ + // eslint-disable-next-line @typescript-eslint/require-await + task: async () => { for (let i = 0; i < 10; i++) { console.log('no-timeout iteration', i); } - await adapter.emit(ExtractorEventType.DataExtractionDone); + return { status: 'success' }; }, - onTimeout: async ({ adapter }) => { - await adapter.emit(ExtractorEventType.DataExtractionProgress); + // eslint-disable-next-line @typescript-eslint/require-await + onTimeout: async () => { + return { status: 'progress' }; }, }); diff --git a/src/tests/timeout-handling/timeout-blocked.test.ts b/src/tests/timeout-handling/timeout-blocked.test.ts index ec66e516..2d8d3431 100644 --- a/src/tests/timeout-handling/timeout-blocked.test.ts +++ b/src/tests/timeout-handling/timeout-blocked.test.ts @@ -1,17 +1,17 @@ +import { createMockEvent } from '../../testing/mock-event'; import { - AirdropEvent, + AirSyncEvent, EventType, ExtractorEventType, } from '../../types/extraction'; import { mockServer } from '../jest.setup'; -import { createMockEvent } from '../../common/test-utils'; import run from './extraction'; jest.setTimeout(10000); describe('Timeout blocked', () => { - let event: AirdropEvent; + let event: AirSyncEvent; beforeEach(() => { event = createMockEvent(mockServer.baseUrl, { payload: { event_type: EventType.StartExtractingData }, diff --git a/src/tests/timeout-handling/timeout-blocked.ts b/src/tests/timeout-handling/timeout-blocked.ts index f7c4ee6d..aa3ef4ed 100644 --- a/src/tests/timeout-handling/timeout-blocked.ts +++ b/src/tests/timeout-handling/timeout-blocked.ts @@ -1,7 +1,8 @@ -import { ExtractorEventType, processTask } from '../../index'; +import { processExtractionTask } from '../../index'; -processTask({ - task: async ({ adapter }) => { +processExtractionTask({ + // eslint-disable-next-line @typescript-eslint/require-await + task: async () => { // Simple CPU-intensive nested loops that block the event loop let result = 0; for (let i = 0; i < 100000; i++) { @@ -10,15 +11,15 @@ processTask({ result = Math.abs(result) % 1000000; } - // Log every 10000 iterations to show progress if (i % 10000 === 0) { console.log(`timeout-blocked iteration ${i}`); } } - await adapter.emit(ExtractorEventType.DataExtractionDone); + return { status: 'success' }; }, - onTimeout: async ({ adapter }) => { - await adapter.emit(ExtractorEventType.DataExtractionProgress); + // eslint-disable-next-line @typescript-eslint/require-await + onTimeout: async () => { + return { status: 'progress' }; }, }); diff --git a/src/tests/timeout-handling/timeout-graceful.test.ts b/src/tests/timeout-handling/timeout-graceful.test.ts index f37a5062..1df3e768 100644 --- a/src/tests/timeout-handling/timeout-graceful.test.ts +++ b/src/tests/timeout-handling/timeout-graceful.test.ts @@ -1,17 +1,17 @@ +import { createMockEvent } from '../../testing/mock-event'; import { - AirdropEvent, + AirSyncEvent, EventType, ExtractorEventType, } from '../../types/extraction'; import { mockServer } from '../jest.setup'; -import { createMockEvent } from '../../common/test-utils'; import run from './extraction'; jest.setTimeout(10000); describe('Timeout graceful', () => { - let event: AirdropEvent; + let event: AirSyncEvent; beforeEach(() => { event = createMockEvent(mockServer.baseUrl, { payload: { event_type: EventType.StartExtractingData }, diff --git a/src/tests/timeout-handling/timeout-graceful.ts b/src/tests/timeout-handling/timeout-graceful.ts index 56ccb396..5293aaef 100644 --- a/src/tests/timeout-handling/timeout-graceful.ts +++ b/src/tests/timeout-handling/timeout-graceful.ts @@ -1,21 +1,22 @@ import { sleep } from '../../common/helpers'; -import { ExtractorEventType, processTask } from '../../index'; +import { processExtractionTask } from '../../index'; -processTask({ +processExtractionTask({ task: async ({ adapter }) => { // Use async delays that allow the event loop to process timeout messages for (let i = 0; i < 10; i++) { if (adapter.isTimeout) { - return; + return { status: 'progress' }; } console.log('timeout-graceful iteration', i); await sleep(1000); } - await adapter.emit(ExtractorEventType.DataExtractionDone); + return { status: 'success' }; }, - onTimeout: async ({ adapter }) => { - await adapter.emit(ExtractorEventType.DataExtractionProgress); + // eslint-disable-next-line @typescript-eslint/require-await + onTimeout: async () => { + return { status: 'progress' }; }, }); diff --git a/src/tests/timeout-handling/timeout-unblocked.test.ts b/src/tests/timeout-handling/timeout-unblocked.test.ts index 8f4fe8b5..6a86cf2f 100644 --- a/src/tests/timeout-handling/timeout-unblocked.test.ts +++ b/src/tests/timeout-handling/timeout-unblocked.test.ts @@ -1,17 +1,17 @@ +import { createMockEvent } from '../../testing/mock-event'; import { - AirdropEvent, + AirSyncEvent, EventType, ExtractorEventType, } from '../../types/extraction'; import { mockServer } from '../jest.setup'; -import { createMockEvent } from '../../common/test-utils'; import run from './extraction'; jest.setTimeout(10000); // 10 seconds describe('Timeout unblocked', () => { - let event: AirdropEvent; + let event: AirSyncEvent; beforeEach(() => { event = createMockEvent(mockServer.baseUrl, { payload: { event_type: EventType.StartExtractingData }, diff --git a/src/tests/timeout-handling/timeout-unblocked.ts b/src/tests/timeout-handling/timeout-unblocked.ts index 0ac268c1..af717677 100644 --- a/src/tests/timeout-handling/timeout-unblocked.ts +++ b/src/tests/timeout-handling/timeout-unblocked.ts @@ -1,31 +1,31 @@ import { sleep } from '../../common/helpers'; -import { ExtractorEventType, processTask } from '../../index'; +import { processExtractionTask } from '../../index'; -processTask({ +processExtractionTask({ task: async ({ adapter }) => { - // CPU-intensive nested loops that yield control after logging - // This allows the event loop to process timeout messages + // CPU-intensive loops that periodically yield so the event loop can + // process timeout messages let result = 0; for (let i = 0; i < 100000; i++) { for (let j = 0; j < 10000; j++) { if (adapter.isTimeout) { - return; + return { status: 'progress' }; } result += Math.sqrt(i * j) * Math.sin(i + j); result = Math.abs(result) % 1000000; } - // Log every 1000 iterations and yield control to event loop if (i % 1000 === 0) { console.log(`timeout-unblocked iteration ${i}`); await sleep(0); } } - await adapter.emit(ExtractorEventType.DataExtractionDone); + return { status: 'success' }; }, - onTimeout: async ({ adapter }) => { - await adapter.emit(ExtractorEventType.DataExtractionProgress); + // eslint-disable-next-line @typescript-eslint/require-await + onTimeout: async () => { + return { status: 'progress' }; }, }); diff --git a/src/types/common.ts b/src/types/common.ts index f0428be9..72df0e6e 100644 --- a/src/types/common.ts +++ b/src/types/common.ts @@ -1,53 +1,13 @@ -import { Artifact } from '../uploader/uploader.interfaces'; - -/** - * ErrorLevel is an enum that represents the level of an error. - * @deprecated - */ -export enum ErrorLevel { - Warning = 'WARNING', - Error = 'ERROR', - Info = 'INFO', -} - -/** - * ErrorRecord is an interface that defines the structure of an error record. - */ export interface ErrorRecord { message: string; } -/** - * LogRecord is an interface that defines the structure of a log record. - * @deprecated - */ -export interface LogRecord { - level: ErrorLevel; - message: string; -} - -/** - * AdapterUpdateParams is an interface that defines the structure of the parameters that can be passed to the update adapter. - * @deprecated - */ -export interface AdapterUpdateParams { - artifact?: Artifact; -} - -/** - * InitialDomainMapping is an interface that defines the structure of the initial domain mapping. - */ export interface InitialDomainMapping { starting_recipe_blueprint?: object; additional_mappings?: object; } -/** - * SyncMode is an enum that defines the different modes of sync that can be used by the external extractor. - * It can be either INITIAL, INCREMENTAL or LOADING. INITIAL mode is used for - * the first/initial import, while INCREMENTAL mode is used for doing syncs. LOADING mode is used for - * loading data from DevRev to the external system. - */ +/** INITIAL = first import, INCREMENTAL = subsequent syncs, LOADING = DevRev -> external system. */ export enum SyncMode { INITIAL = 'INITIAL', INCREMENTAL = 'INCREMENTAL', diff --git a/src/common/errors.ts b/src/types/errors.ts similarity index 100% rename from src/common/errors.ts rename to src/types/errors.ts diff --git a/src/types/external-domain-metadata.ts b/src/types/external-domain-metadata.ts index 53d39ea9..c2fe2cd8 100644 --- a/src/types/external-domain-metadata.ts +++ b/src/types/external-domain-metadata.ts @@ -1,32 +1,19 @@ -/** - * Schema version for the external domain metadata format. - */ export type SchemaVersion = 'v0.2.0'; -/** Key identifying a record type in the record_types map, refers_to maps, or type_keys arrays. */ export type RecordTypeKey = string; -/** Key identifying a field within a record type or struct type fields map. */ export type FieldKey = string; -/** Key identifying an enum value in EnumValue.key or stage diagram stages map keys. */ export type EnumValueKey = string; -/** Key identifying a struct type in the struct_types map. */ export type StructTypeKey = string; -/** Key identifying a record type category in the record_type_categories map. */ export type RecordTypeCategoryKey = string; -/** Key identifying a state in the stage diagram states map. */ export type StateKey = string; -/** Key identifying a stage in the stage diagram stages map. */ export type StageKey = string; -/** - * Field type discriminator. - */ export type FieldType = | 'bool' | 'int' @@ -45,192 +32,120 @@ export type FieldType = | 'conditional_privilege' | 'participation'; -/** - * Reference type indicating parent-child relationship. - */ export type ReferenceType = 'child' | 'parent'; -/** - * Comparator for field conditions. - */ export type FieldConditionComparator = 'eq' | 'ne'; -/** - * Effect applied when a field condition is met. - */ export type FieldConditionEffect = 'require' | 'show'; -/** - * Scope of a record type. - */ export type RecordTypeScope = | 'metadata_is_system_scoped' | 'data_is_system_scoped'; -/** - * Collection constraints for fields that are collections of values. - */ export interface CollectionData { min_length?: number; max_length?: number; } -/** - * Integer field constraints. - */ export interface IntData { min?: number; max?: number; } -/** - * Float field constraints. - */ export interface FloatData { min?: number; max?: number; } -/** - * Text field constraints. - */ export interface TextData { min_length?: number; max_length?: number; } -/** - * Enum value definition. - */ export interface EnumValue { - /** The enum value that actually occurs in the json data */ + /** The enum value as it actually occurs in the json data. */ key: EnumValueKey; - /** The human readable name of the enum value */ name?: string; description?: string; - /** Deprecated enum values may still occur in the data, but should not be used in new data */ + /** Deprecated values may still occur in the data, but should not be used in new data. */ is_deprecated?: boolean; } -/** - * Enum field data containing possible values. - */ export interface EnumData { values: EnumValue[]; } -/** - * Details about how a reference targets another record type. - */ export interface ReferenceDetail { /** The field in the target record type by which it is referenced. Assumed to be the primary key if not set. */ by_field?: FieldKey; } -/** - * Reference field data specifying target record types. - */ export interface ReferenceData { - /** The record types that this reference can refer to */ refers_to: Record; - /** The parent reference refers to a record that has special ownership over the child */ + /** A 'parent' reference refers to a record that has special ownership over the child. */ reference_type?: ReferenceType; } -/** - * Typed reference field data specifying target record types. - */ export interface TypedReferenceData { - /** The record types that this reference can refer to */ refers_to: Record; - /** The parent reference refers to a record that has special ownership over the child */ + /** A 'parent' reference refers to a record that has special ownership over the child. */ reference_type?: ReferenceType; } -/** - * Struct field data referencing a struct type. - */ export interface StructData { key?: StructTypeKey; } -/** - * Participation field data specifying target record types. - */ export interface ParticipationData { - /** The record types that this participation reference can refer to */ refers_to: Record; } -/** - * Permission data associating a reference with a role. - */ export interface PermissionData { member_id?: ReferenceData; role?: EnumData; } -/** - * Conditional privilege data for authorization. - */ export interface ConditionalPrivilegeData { - /** The possible record types or record type categories that can be targeted in conditional privilege. */ + /** Record types or record type categories that can be targeted. */ type_keys: RecordTypeKey[]; } -/** - * Field privilege data for authorization. - */ export interface FieldPrivilegeData { - /** The possible record types or record type categories that can be targeted in field privilege. */ + /** Record types or record type categories that can be targeted. */ type_keys: RecordTypeKey[]; } -/** - * Record type privilege data for authorization. - */ export interface RecordTypePrivilegeData { - /** The possible record types or record type categories that can be targeted in record type privilege. */ + /** Record types or record type categories that can be targeted. */ type_keys: RecordTypeKey[]; } -/** - * Target type key data for authorization policy. - */ export interface TargetTypeKeyData { - /** The possible record types or record type categories that can be targeted in authorization policy. */ + /** Record types or record type categories that can be targeted. */ type_keys: RecordTypeKey[]; } -/** - * Field reference data (currently empty, reserved for future use). - */ +/** Currently empty, reserved for future use. */ export interface FieldReferenceData { [key: string]: never; } -/** - * Field definition with type discriminator and type-specific data. - */ +/** Field definition; `type` selects which type-specific data property applies. */ export interface Field { - /** The type of the field */ type: FieldType; - /** The human readable name of the field */ name?: string; description?: string; - /** Required fields are required in the domain model of the external system. */ + /** Required in the domain model of the external system. */ is_required?: boolean; - /** Read only fields can't be set (when creating or updating a record), but are filled in by some process in the system. */ + /** Can't be set on create/update; filled in by some process in the system. */ is_read_only?: boolean | null; - /** Fields that are write only should only be written to. */ is_write_only?: boolean | null; - /** Indexed fields can be used for searching, sorting or filtering. */ + /** Can be used for searching, sorting or filtering. */ is_indexed?: boolean | null; - /** Indicates that the field can be used to uniquely lookup a record. */ + /** Can be used to uniquely look up a record. */ is_identifier?: boolean | null; - /** Default value for the field */ default_value?: boolean | number | string; - /** If collection is set, the field is a 'collection' of the given type. */ + /** If set, the field is a collection of the given type. */ collection?: CollectionData; // Type-specific data @@ -250,150 +165,94 @@ export interface Field { participation?: ParticipationData; } -/** - * Field condition definition. - */ +/** When the controlling field's value matches `value` (per `comparator`), `effect` is applied to `affected_fields`. */ export interface FieldCondition { - /** The value of the controlling field that will be compared against to see if the condition is met. */ value: unknown; - /** The comparator that will be used to compare the controlling field's value against the Value. */ comparator: FieldConditionComparator; - /** The fields that will be affected by the condition being met. */ affected_fields: FieldKey[]; - /** The effect that will be applied to the affected fields if the condition is met. */ effect: FieldConditionEffect; } -/** - * Array of field conditions. - */ export type FieldConditions = FieldCondition[]; -/** - * Custom link names for forward and backward directions. - */ export interface CustomLinkNames { - /** The forward name of the link */ forward_name: string; - /** The backward name of the link */ backward_name: string; } -/** - * Custom link data for defining link types. - */ export interface CustomLinkData { /** The field that defines the link types in the system. */ link_type_field: FieldKey; link_direction_names: Record | null; } -/** - * Custom stage definition in a stage diagram. - */ export interface CustomStage { - /** The state this stage belongs to. Must match the ones defined in the diagram 'states' field or be one of the default options: 'open', 'in_progress', 'closed'. */ + /** Must match a key in the diagram's 'states' map or be a default: 'open', 'in_progress', 'closed'. */ state?: StateKey; - /** A list of stage names that this stage can transition to. */ + /** Stage names this stage can transition to. */ transitions_to?: StageKey[]; } -/** - * Custom state definition in a stage diagram. - */ export interface CustomState { - /** The human readable name of the custom state. */ name: string; - /** Denotes that this state is an end state. */ is_end_state?: boolean; - /** The sort order of the state. */ + /** Sort order. */ ordinal?: number; } -/** - * Stage diagram definition for record type workflow. - */ export interface StageDiagram { /** The field that represents the stage in the external system. */ controlling_field: FieldKey; - /** A map of the stages that should be created. Keys must match the enum values in the controlling field. */ + /** Stage keys must match the enum values in the controlling field. */ stages: Record; - /** The stage that the parent record type starts in when it is created. */ + /** The stage the parent record type starts in when created. */ starting_stage?: StageKey; - /** A map of the states/status categories that should be created. */ states?: Record; - /** Denotes that this diagram has no explicit transitions and should be created as an 'all-to-all' diagram. */ + /** No explicit transitions; create as an 'all-to-all' diagram. */ all_transitions_allowed?: boolean; } -/** - * Record type category definition. - */ export interface RecordTypeCategory { - /** The human readable name of the record type category */ name?: string; - /** Indicates whether a record can move between the record types of this category while preserving its identity */ + /** Whether a record can move between record types of this category while preserving its identity. */ are_record_type_conversions_possible?: boolean; } -/** - * Attachments configuration for a record type. - */ export interface Attachments { - /** Whether attachments can be extracted */ is_extractable?: boolean; - /** Whether attachments can be loaded: that is, whether the connector supports creating it in the system */ + /** Whether the connector supports creating attachments in the external system. */ is_loadable?: boolean; } -/** - * Record type definition. - */ export interface RecordType { - /** The fields of the record type */ fields: Record; - /** The human readable name of the record type */ name?: string; description?: string; category?: RecordTypeCategoryKey; - /** Whether the record type can be loaded (connector supports creating it in the system) */ + /** Whether the connector supports creating this record type in the external system. */ is_loadable?: boolean; - /** Whether the record type sends the complete system state in every sync */ + /** Whether the record type sends the complete system state in every sync. */ is_snapshot?: boolean; - /** Denotes that the record type has no ID field, primarily used for authorization policies */ + /** No ID field; primarily used for authorization policies. */ no_identifier?: boolean; - /** Indicates the scope of this record type */ scope?: RecordTypeScope; - /** Field conditions for this record type */ conditions?: Record; - /** Stage diagram for workflow */ stage_diagram?: StageDiagram; - /** Link naming data for custom links */ link_naming_data?: CustomLinkData; - /** Attachments configuration */ attachments?: Attachments; } -/** - * Struct type definition for reusable field structures. - */ +/** Reusable field structure. */ export interface StructType { - /** The fields of the struct type */ fields: Record; - /** The human readable name of the struct type */ name?: string; } -/** - * External domain metadata describing the logical structure of an external system. - */ +/** Describes the logical structure of an external system. */ export interface ExternalDomainMetadata { - /** The record types in the domain */ record_types: Record; - /** Record type categories */ record_type_categories?: Record; - /** Struct types for reusable field structures */ struct_types?: Record; - /** The schema version of the metadata format itself. */ + /** Version of the metadata format itself. */ schema_version?: SchemaVersion; } diff --git a/src/types/extraction.test.ts b/src/types/extraction.test.ts index 1dd9cb9e..1dcf1433 100644 --- a/src/types/extraction.test.ts +++ b/src/types/extraction.test.ts @@ -1,5 +1,6 @@ -import { createMockEvent } from '../common/test-utils'; +import { createMockEvent } from '../testing/mock-event'; import { mockServer } from '../tests/jest.setup'; + import { EventType, InitialSyncScope, TimeValueType } from './extraction'; describe('ExtractionTypes', () => { diff --git a/src/types/extraction.ts b/src/types/extraction.ts index 64136e90..983d8eaf 100644 --- a/src/types/extraction.ts +++ b/src/types/extraction.ts @@ -1,64 +1,15 @@ import { InputData } from '@devrev/typescript-sdk/dist/snap-ins'; +import { ExtractionAdapter } from '../multithreading/adapters/extraction-adapter'; +import { NormalizedAttachment } from '../repo/repo.interfaces'; import { Artifact } from '../uploader/uploader.interfaces'; import { ErrorRecord } from './common'; -import { AxiosResponse } from 'axios'; -import { NormalizedAttachment } from '../repo/repo.interfaces'; -import { WorkerAdapter } from '../multithreading/worker-adapter/worker-adapter'; import { DonV2, LoaderReport, RateLimited } from './loading'; -/** - * EventType is an enum that defines the different types of events that can be sent to the external extractor from ADaaS. - * The external extractor can use these events to know what to do next in the extraction process. - */ +/** Events sent from AirSync to the connector. */ export enum EventType { - // Extraction - Old member names with OLD values (deprecated, kept for backwards compatibility) - /** - * @deprecated Use StartExtractingExternalSyncUnits instead - */ - ExtractionExternalSyncUnitsStart = 'EXTRACTION_EXTERNAL_SYNC_UNITS_START', - /** - * @deprecated Use StartExtractingMetadata instead - */ - ExtractionMetadataStart = 'EXTRACTION_METADATA_START', - /** - * @deprecated Use StartExtractingData instead - */ - ExtractionDataStart = 'EXTRACTION_DATA_START', - /** - * @deprecated Use ContinueExtractingData instead - */ - ExtractionDataContinue = 'EXTRACTION_DATA_CONTINUE', - /** - * @deprecated Use StartDeletingExtractorState instead - */ - ExtractionDataDelete = 'EXTRACTION_DATA_DELETE', - /** - * @deprecated Use StartExtractingAttachments instead - */ - ExtractionAttachmentsStart = 'EXTRACTION_ATTACHMENTS_START', - /** - * @deprecated Use ContinueExtractingAttachments instead - */ - ExtractionAttachmentsContinue = 'EXTRACTION_ATTACHMENTS_CONTINUE', - /** - * @deprecated Use StartDeletingExtractorAttachmentsState instead - */ - ExtractionAttachmentsDelete = 'EXTRACTION_ATTACHMENTS_DELETE', - - // Loading - StartLoadingData = 'START_LOADING_DATA', - ContinueLoadingData = 'CONTINUE_LOADING_DATA', - StartLoadingAttachments = 'START_LOADING_ATTACHMENTS', - ContinueLoadingAttachments = 'CONTINUE_LOADING_ATTACHMENTS', - StartDeletingLoaderState = 'START_DELETING_LOADER_STATE', - StartDeletingLoaderAttachmentState = 'START_DELETING_LOADER_ATTACHMENT_STATE', - - // Unknown - UnknownEventType = 'UNKNOWN_EVENT_TYPE', - - // Extraction - New member names with NEW values (preferred) + // Extraction StartExtractingExternalSyncUnits = 'START_EXTRACTING_EXTERNAL_SYNC_UNITS', StartExtractingMetadata = 'START_EXTRACTING_METADATA', StartExtractingData = 'START_EXTRACTING_DATA', @@ -67,83 +18,19 @@ export enum EventType { StartExtractingAttachments = 'START_EXTRACTING_ATTACHMENTS', ContinueExtractingAttachments = 'CONTINUE_EXTRACTING_ATTACHMENTS', StartDeletingExtractorAttachmentsState = 'START_DELETING_EXTRACTOR_ATTACHMENTS_STATE', + + // Loading + StartLoadingData = 'START_LOADING_DATA', + ContinueLoadingData = 'CONTINUE_LOADING_DATA', + StartLoadingAttachments = 'START_LOADING_ATTACHMENTS', + ContinueLoadingAttachments = 'CONTINUE_LOADING_ATTACHMENTS', + StartDeletingLoaderState = 'START_DELETING_LOADER_STATE', + StartDeletingLoaderAttachmentState = 'START_DELETING_LOADER_ATTACHMENT_STATE', } -/** - * ExtractorEventType is an enum that defines the different types of events that can be sent from the external extractor to ADaaS. - * The external extractor can use these events to inform ADaaS about the progress of the extraction process. - */ +/** Events sent from the connector to AirSync. */ export enum ExtractorEventType { - // Extraction - Old member names with OLD values (deprecated, kept for backwards compatibility) - /** - * @deprecated Use ExternalSyncUnitExtractionDone instead - */ - ExtractionExternalSyncUnitsDone = 'EXTRACTION_EXTERNAL_SYNC_UNITS_DONE', - /** - * @deprecated Use ExternalSyncUnitExtractionError instead - */ - ExtractionExternalSyncUnitsError = 'EXTRACTION_EXTERNAL_SYNC_UNITS_ERROR', - /** - * @deprecated Use MetadataExtractionDone instead - */ - ExtractionMetadataDone = 'EXTRACTION_METADATA_DONE', - /** - * @deprecated Use MetadataExtractionError instead - */ - ExtractionMetadataError = 'EXTRACTION_METADATA_ERROR', - /** - * @deprecated Use DataExtractionProgress instead - */ - ExtractionDataProgress = 'EXTRACTION_DATA_PROGRESS', - /** - * @deprecated Use DataExtractionDelayed instead - */ - ExtractionDataDelay = 'EXTRACTION_DATA_DELAY', - /** - * @deprecated Use DataExtractionDone instead - */ - ExtractionDataDone = 'EXTRACTION_DATA_DONE', - /** - * @deprecated Use DataExtractionError instead - */ - ExtractionDataError = 'EXTRACTION_DATA_ERROR', - /** - * @deprecated Use ExtractorStateDeletionDone instead - */ - ExtractionDataDeleteDone = 'EXTRACTION_DATA_DELETE_DONE', - /** - * @deprecated Use ExtractorStateDeletionError instead - */ - ExtractionDataDeleteError = 'EXTRACTION_DATA_DELETE_ERROR', - /** - * @deprecated Use AttachmentExtractionProgress instead - */ - ExtractionAttachmentsProgress = 'EXTRACTION_ATTACHMENTS_PROGRESS', - /** - * @deprecated Use AttachmentExtractionDelayed instead - */ - ExtractionAttachmentsDelay = 'EXTRACTION_ATTACHMENTS_DELAY', - /** - * @deprecated Use AttachmentExtractionDone instead - */ - ExtractionAttachmentsDone = 'EXTRACTION_ATTACHMENTS_DONE', - /** - * @deprecated Use AttachmentExtractionError instead - */ - ExtractionAttachmentsError = 'EXTRACTION_ATTACHMENTS_ERROR', - /** - * @deprecated Use ExtractorAttachmentsStateDeletionDone instead - */ - ExtractionAttachmentsDeleteDone = 'EXTRACTION_ATTACHMENTS_DELETE_DONE', - /** - * @deprecated Use ExtractorAttachmentsStateDeletionError instead - */ - ExtractionAttachmentsDeleteError = 'EXTRACTION_ATTACHMENTS_DELETE_ERROR', - - // Unknown - UnknownEventType = 'UNKNOWN_EVENT_TYPE', - - // Extraction - New member names with NEW values (preferred) + // Extraction ExternalSyncUnitExtractionDone = 'EXTERNAL_SYNC_UNIT_EXTRACTION_DONE', ExternalSyncUnitExtractionError = 'EXTERNAL_SYNC_UNIT_EXTRACTION_ERROR', MetadataExtractionDone = 'METADATA_EXTRACTION_DONE', @@ -162,20 +49,7 @@ export enum ExtractorEventType { ExtractorAttachmentsStateDeletionError = 'EXTRACTOR_ATTACHMENTS_STATE_DELETION_ERROR', } -/** - * @deprecated - * ExtractionMode is an enum that defines the different modes of extraction that can be used by the external extractor. - * It can be either INITIAL or INCREMENTAL. INITIAL mode is used for the first/initial import, while INCREMENTAL mode is used for doing syncs. - */ -export enum ExtractionMode { - INITIAL = 'INITIAL', - INCREMENTAL = 'INCREMENTAL', -} - -/** - * ExternalSyncUnit is an interface that defines the structure of an external sync unit (repos, projects, ...) that can be extracted. - * It must contain an ID, a name, and a description. It can also contain the number of items in the external sync unit. - */ +/** An extractable unit in the external system (repo, project, ...). */ export interface ExternalSyncUnit { id: string; name: string; @@ -184,202 +58,99 @@ export interface ExternalSyncUnit { item_type?: string; } -/** - * InitialSyncScope is an enum that defines the different scopes of initial sync that can be used by the external extractor. - */ export enum InitialSyncScope { FULL_HISTORY = 'full-history', TIME_SCOPED = 'time-scoped', } -/** - * TimeUnit is an enum that defines the supported Go duration units for time window calculations. - * These correspond directly to Go's time.ParseDuration units. - */ +/** Duration units for time windows; matches Go's time.ParseDuration units. */ export enum TimeUnit { - /** Nanoseconds */ NANOSECONDS = 'ns', - /** Microseconds (ASCII alias) */ MICROSECONDS = 'us', - /** Microseconds (Unicode alias) */ MICROSECONDS_MU = 'µs', - /** Milliseconds */ MILLISECONDS = 'ms', - /** Seconds */ SECONDS = 's', - /** Minutes */ MINUTES = 'm', - /** Hours */ HOURS = 'h', } /** - * TimeValueType is an enum that defines the type of a time value used in extraction start/end times. - * The platform sends these types to indicate how the extraction time should be resolved by the SDK. + * How the SDK resolves an extraction start/end time sent by the platform. + * WORKERS_* variants resolve against worker state timestamps; UNBOUNDED means no bound. */ export enum TimeValueType { - /** Oldest timestamp from worker state */ WORKERS_OLDEST = 'workers_oldest', - /** Oldest timestamp from worker state minus a duration window */ WORKERS_OLDEST_MINUS_WINDOW = 'workers_oldest_minus_window', - /** Newest timestamp from worker state */ WORKERS_NEWEST = 'workers_newest', - /** Newest timestamp from worker state plus a duration window */ WORKERS_NEWEST_PLUS_WINDOW = 'workers_newest_plus_window', - /** Current time */ CURRENT_TIME = 'current_time', - /** User-specified absolute timestamp */ ABSOLUTE_TIME = 'absolute_time', - /** No bound - extract all available data */ UNBOUNDED = 'unbounded', } /** - * TimeValue is an interface that represents a time value used in extraction start/end times. - * It contains a type (which denotes how the value should be resolved) and an optional value. - * - For ABSOLUTE: value is an ISO 8601 timestamp - * - For *_WINDOW types: value is a Go duration string (e.g. '500ms', '30s', '5m', '2h') - * - For other types: value is not used + * Extraction start/end time value. `value` is an ISO 8601 timestamp for ABSOLUTE_TIME, + * a Go duration string (e.g. '30s', '2h') for *_WINDOW types, and unused otherwise. */ export interface TimeValue { type: TimeValueType; value?: string; } -/** - * EventContextIn is an interface that defines the structure of the input event context that is sent to the external extractor from ADaaS. - * @deprecated - */ -export interface EventContextIn { - callback_url: string; - dev_org: string; - dev_org_id: string; - dev_user: string; - dev_user_id: string; - external_sync_unit: string; - external_sync_unit_id: string; - external_sync_unit_name: string; - external_system: string; - external_system_type: string; - import_slug: string; - mode: string; - request_id: string; - snap_in_slug: string; - sync_run: string; - sync_run_id: string; - sync_tier: string; - sync_unit: DonV2; - sync_unit_id: string; - uuid: string; - worker_data_url: string; -} - -/** - * EventContextOut is an interface that defines the structure of the output event context that is sent from the external extractor to ADaaS. - * @deprecated - */ -export interface EventContextOut { - uuid: string; - sync_run: string; - sync_unit?: string; -} - -/** - * EventContext is an interface that defines the structure of the event context that is sent to the external connector from Airdrop. - */ export interface EventContext { callback_url: string; - /** - * @deprecated dev_org is deprecated and should not be used. Use dev_oid instead. - */ + /** @deprecated Use dev_oid instead. */ dev_org: string; dev_oid: string; dev_org_id: string; - /** - * @deprecated dev_user is deprecated and should not be used. Use dev_uid instead. - */ + /** @deprecated Use dev_uid instead. */ dev_user: string; - /** - * @deprecated dev_user_id is deprecated and should not be used. Use dev_uid instead. - */ + /** @deprecated Use dev_uid instead. */ dev_user_id: string; dev_uid: string; event_type_adaas: string; - /** - * @deprecated external_sync_unit is deprecated and should not be used. Use external_sync_unit_id instead. - */ + /** @deprecated Use external_sync_unit_id instead. */ external_sync_unit: string; external_sync_unit_id: string; external_sync_unit_name: string; - /** - * @deprecated external_system is deprecated and should not be used. Use external_system_id instead. - */ + /** @deprecated Use external_system_id instead. */ external_system: string; external_system_id: string; external_system_name: string; external_system_type: string; - /** - * Resolved start timestamp of extraction (ISO 8601 format). - * Automatically computed by the SDK from extraction_start_time and worker state. - * This is the field developers should read to know when to start extracting from. - */ + /** Start of extraction (ISO 8601), resolved by the SDK from extraction_start_time and worker state. */ extract_from?: string; import_slug: string; initial_sync_scope?: InitialSyncScope; mode: string; request_id: string; request_id_adaas: string; - /** - * @deprecated reset_extraction is deprecated and should not be used. - */ + /** @deprecated */ reset_extraction?: boolean; - /** - * @deprecated reset_extract_from is deprecated. Use extraction_start_time/extraction_end_time instead, - * which are automatically resolved into extract_from and extract_to. - */ + /** @deprecated Use extraction_start_time/extraction_end_time (resolved into extract_from/extract_to). */ reset_extract_from?: boolean; run_id: string; sequence_version: string; snap_in_slug: string; snap_in_version_id: string; - /** - * @deprecated sync_run is deprecated and should not be used. Use run_id instead. - */ + /** @deprecated Use run_id instead. */ sync_run: string; - /** - * @deprecated sync_run_id is deprecated and should not be used. Use run_id instead. - */ + /** @deprecated Use run_id instead. */ sync_run_id: string; sync_tier: string; sync_unit: DonV2; sync_unit_id: string; - /** - * @deprecated uuid is deprecated and should not be used. Use request_id_adaas instead. - */ + /** @deprecated Use request_id_adaas instead. */ uuid: string; worker_data_url: string; - /** - * Start time value for extraction, as sent by the platform. - * The SDK resolves this into a concrete ISO 8601 timestamp on extract_from. - */ + /** Platform-sent start time; the SDK resolves it into extract_from. */ extraction_start_time?: TimeValue; - /** - * End time value for extraction, as sent by the platform. - * The SDK resolves this into a concrete ISO 8601 timestamp on extract_to. - */ + /** Platform-sent end time; the SDK resolves it into extract_to. */ extraction_end_time?: TimeValue; - /** - * Resolved end timestamp of extraction (ISO 8601 format). - * Automatically computed by the SDK from extraction_end_time and worker state. - * This is the field developers should read to know when to stop extracting at. - */ + /** End of extraction (ISO 8601), resolved by the SDK from extraction_end_time and worker state. */ extract_to?: string; } -/** - * ConnectionData is an interface that defines the structure of the connection data that is sent to the external extractor from ADaaS. - * It contains the organization ID, organization name, key, and key type. - */ export interface ConnectionData { org_id: string; org_name: string; @@ -387,24 +158,13 @@ export interface ConnectionData { key_type: string; } -/** - * EventData is an interface that defines the structure of the event data that is sent from the external extractor to ADaaS. - */ export interface EventData { - /** - * @deprecated This field is deprecated and should not be used. External sync units should be pushed to the AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS repo. - * - */ - external_sync_units?: ExternalSyncUnit[]; - /** - * @deprecated This field is deprecated and should not be used. Progress is - * now calculated on the backend. - */ - progress?: number; error?: ErrorRecord; delay?: number; /** - * @deprecated This field is deprecated and should not be used. + * Artifacts produced by the worker's repos, attached to the emitted event by + * the SDK. Includes external sync units, which are pushed to the + * AirSyncDefaultItemTypes.EXTERNAL_SYNC_UNITS repo and uploaded as artifacts. */ artifacts?: Artifact[]; @@ -414,9 +174,6 @@ export interface EventData { stats_file?: string; } -/** - * WorkerMetadata is an interface that defines the structure of the worker metadata that is sent from the external extractor to ADaaS. - */ export interface WorkerMetadata { adaas_library_version?: string; @@ -427,61 +184,43 @@ export interface WorkerMetadata { oldest_modified_date?: string; newest_modified_date?: string; - // Calculated time ranges in absolute times - // Times present in `extract_from` and `extract_to` given to the connector. + // Absolute times from the `extract_from`/`extract_to` given to the connector. oldest_state_date?: string; newest_state_date?: string; } -/** - * DomainObject is an interface that defines the structure of a domain object that can be extracted. - * It must contain a name, a next chunk ID, the pages, the last modified date, whether it is done, and the count. - * @deprecated - */ -export interface DomainObjectState { - name: string; - nextChunkId: number; - pages?: { - pages: number[]; - }; - lastModified: string; - isDone: boolean; - count: number; -} - -/** - * AirdropEvent is an interface that defines the structure of the event that is sent to the external extractor from ADaaS. - * It contains the context, payload, execution metadata, and input data as common snap-ins. - */ -export interface AirdropEvent { +/** Event sent from AirSync to the connector. */ +export interface AirSyncEvent { context: { secrets: { service_account_token: string; }; snap_in_version_id: string; snap_in_id: string; + /** DevRev identity of the user who triggered the sync. */ + user_id: string; + /** DevRev org id (don:identity:.../devo/...). */ + dev_oid: string; + /** External source identity, when the platform provides one. */ + source_id: string; + /** DevRev service-account identity used for the sync. */ + service_account_id: string; }; - payload: AirdropMessage; + payload: AirSyncMessage; execution_metadata: { devrev_endpoint: string; }; input_data: InputData; } -/** - * AirdropMessage is an interface that defines the structure of the payload/message that is sent to the external extractor from ADaaS. - */ -export interface AirdropMessage { +export interface AirSyncMessage { connection_data: ConnectionData; event_context: EventContext; event_type: EventType; event_data?: EventData; } -/** - * ExtractorEvent is an interface that defines the structure of the event that is sent from the external extractor to ADaaS. - * It contains the event type, event context, extractor state, and event data. - */ +/** Event sent from the connector to AirSync. */ export interface ExtractorEvent { event_type: string; event_context: EventContext; @@ -489,9 +228,6 @@ export interface ExtractorEvent { worker_metadata?: WorkerMetadata; } -/** - * LoaderEvent - */ export interface LoaderEvent { event_type: string; event_context: EventContext; @@ -506,15 +242,19 @@ export type ExternalSystemAttachmentStreamingFunction = ({ export interface ExternalSystemAttachmentStreamingParams { item: NormalizedAttachment; - event: AirdropEvent; + event: AirSyncEvent; +} + +export interface HttpStreamResponse { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + data: any; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + headers: Record; } export interface ExternalSystemAttachmentStreamingResponse { - httpStream?: AxiosResponse; - error?: ErrorRecord & { - /** The HTTP status code of the failed request, if the error originated from one. */ - statusCode?: number; - }; + httpStream?: HttpStreamResponse; + error?: ErrorRecord; delay?: number; } @@ -527,10 +267,7 @@ export interface StreamAttachmentsResponse { export type ProcessAttachmentReturnType = | { delay?: number; - error?: { - message: string; - fileSize?: number; - }; + error?: { message: string; fileSize?: number }; } | undefined; @@ -551,7 +288,7 @@ export type ExternalSystemAttachmentReducerFunction< batchSize, }: { attachments: Batch; - adapter: WorkerAdapter; + adapter: ExtractionAdapter; batchSize?: number; }) => NewBatch; @@ -570,7 +307,7 @@ export type ExternalSystemAttachmentIteratorFunction = stream, }: { reducedAttachments: NewBatch; - adapter: WorkerAdapter; + adapter: ExtractionAdapter; stream: ExternalSystemAttachmentStreamingFunction; }) => Promise; diff --git a/src/types/index.ts b/src/types/index.ts deleted file mode 100644 index f49ef309..00000000 --- a/src/types/index.ts +++ /dev/null @@ -1,126 +0,0 @@ -// Common -export { - AdapterUpdateParams, - ErrorLevel, - ErrorRecord, - InitialDomainMapping, - LogRecord, - SyncMode, -} from './common'; - -// Extraction -export { - AirdropEvent, - AirdropMessage, - ConnectionData, - DomainObjectState, - EventContextIn, - EventContextOut, - EventContext, - EventData, - EventType, - ExternalProcessAttachmentFunction, - ExternalSyncUnit, - ExternalSystemAttachmentIteratorFunction, - ExternalSystemAttachmentReducerFunction, - ExternalSystemAttachmentStreamingFunction, - ExternalSystemAttachmentStreamingParams, - ExternalSystemAttachmentStreamingResponse, - ExtractionMode, - ExtractorEvent, - ExtractorEventType, - InitialSyncScope, - ProcessAttachmentReturnType, - TimeUnit, - TimeValue, - TimeValueType, -} from './extraction'; - -// Loading -export { - ExternalSystemAttachment, - ExternalSystemItem, - ExternalSystemItemLoadingParams, - ExternalSystemItemLoadingResponse, - LoaderEventType, -} from './loading'; - -// Repo -export { - NormalizedAttachment, - NormalizedItem, - RepoInterface, -} from '../repo/repo.interfaces'; - -// State -export { AdapterState } from '../state/state.interfaces'; - -export { UNBOUNDED_DATE_TIME_VALUE } from '../common/constants'; - -// Uploader -export { - Artifact, - ArtifactsPrepareResponse, - SsorAttachment, - StreamAttachmentsResponse, - StreamResponse, - UploadResponse, -} from '../uploader/uploader.interfaces'; - -// Mappers -export type { - MappersCreateParams, - MappersGetByExternalIdParams, - MappersGetByTargetIdParams, - MappersUpdateParams, -} from '../mappers/mappers.interface'; - -export { - SyncMapperRecordStatus, - SyncMapperRecordTargetType, -} from '../mappers/mappers.interface'; - -// External Domain Metadata -export type { - CollectionData, - ConditionalPrivilegeData, - CustomLinkData, - CustomLinkNames, - CustomStage, - CustomState, - EnumData, - EnumValue, - EnumValueKey, - ExternalDomainMetadata, - Field, - FieldCondition, - FieldConditionComparator, - FieldConditionEffect, - FieldConditions, - FieldKey, - FieldPrivilegeData, - FieldReferenceData, - FieldType, - FloatData, - IntData, - PermissionData, - RecordType, - RecordTypeCategory, - RecordTypeCategoryKey, - RecordTypeKey, - RecordTypePrivilegeData, - RecordTypeScope, - ReferenceData, - ReferenceDetail, - ReferenceType, - SchemaVersion, - StageKey, - StageDiagram, - StateKey, - StructData, - StructTypeKey, - StructType, - TargetTypeKeyData, - TextData, - TypedReferenceData, -} from './external-domain-metadata'; diff --git a/src/types/loading.ts b/src/types/loading.ts index b08fb1cb..fcbde520 100644 --- a/src/types/loading.ts +++ b/src/types/loading.ts @@ -1,6 +1,7 @@ import { Mappers } from '../mappers/mappers'; + import { ErrorRecord } from './common'; -import { AirdropEvent } from './extraction'; +import { AirSyncEvent } from './extraction'; export interface StatsFileObject { id: string; @@ -60,7 +61,7 @@ export interface ExternalSystemItem { export interface ExternalSystemItemLoadingParams { item: Type; mappers: Mappers; - event: AirdropEvent; + event: AirSyncEvent; } export interface ExternalSystemItemLoadingResponse { @@ -135,13 +136,9 @@ export type SyncMapperRecord = { input_file?: string; }; -/* eslint-disable @typescript-eslint/no-duplicate-enum-values */ export enum LoaderEventType { DataLoadingProgress = 'DATA_LOADING_PROGRESS', - /** - * @deprecated This was a typo. Use DataLoadingDelayed for the corrected spelling - */ - DataLoadingDelay = 'DATA_LOADING_DELAYED', + DataLoadingDelayed = 'DATA_LOADING_DELAYED', DataLoadingDone = 'DATA_LOADING_DONE', DataLoadingError = 'DATA_LOADING_ERROR', @@ -155,24 +152,4 @@ export enum LoaderEventType { LoaderAttachmentStateDeletionDone = 'LOADER_ATTACHMENT_STATE_DELETION_DONE', LoaderAttachmentStateDeletionError = 'LOADER_ATTACHMENT_STATE_DELETION_ERROR', - - UnknownEventType = 'UNKNOWN_EVENT_TYPE', - DataLoadingDelayed = 'DATA_LOADING_DELAYED', - - /** - * @deprecated Use AttachmentsLoadingProgress instead (note: singular changed to plural) - */ - AttachmentsLoadingProgress = 'ATTACHMENT_LOADING_PROGRESS', - /** - * @deprecated Use AttachmentsLoadingDelayed instead (note: singular changed to plural) - */ - AttachmentsLoadingDelayed = 'ATTACHMENT_LOADING_DELAYED', - /** - * @deprecated Use AttachmentsLoadingDone instead (note: singular changed to plural) - */ - AttachmentsLoadingDone = 'ATTACHMENT_LOADING_DONE', - /** - * @deprecated Use AttachmentsLoadingError instead (note: singular changed to plural) - */ - AttachmentsLoadingError = 'ATTACHMENT_LOADING_ERROR', } diff --git a/src/types/workers.ts b/src/types/workers.ts index 4123c291..23921888 100644 --- a/src/types/workers.ts +++ b/src/types/workers.ts @@ -1,117 +1,81 @@ import { Worker } from 'worker_threads'; import type { LogLevel } from '../logger/logger.interfaces'; -import { State } from '../state/state'; -import { WorkerAdapter } from '../multithreading/worker-adapter/worker-adapter'; - -import { AirdropEvent, EventType, ExtractorEventType } from './extraction'; +import { BaseState } from '../state/state'; +import { ErrorRecord, InitialDomainMapping } from './common'; +import { AirSyncEvent, EventType, ExtractorEventType } from './extraction'; import { LoaderEventType } from './loading'; -import { InitialDomainMapping } from './common'; - -/** - * WorkerAdapterInterface is an interface for WorkerAdapter class. - * @interface WorkerAdapterInterface - * @constructor - * @param {AirdropEvent} event - The event object received from the platform - * @param {object=} initialState - The initial state of the adapter - * @param {WorkerAdapterInterface} options - The options to create a new instance of WorkerAdapter class - */ export interface WorkerAdapterInterface { - event: AirdropEvent; - adapterState: State; + event: AirSyncEvent; + adapterState: BaseState; options?: WorkerAdapterOptions; } -/** - * ExtractionScope represents the parsed extraction scope from the platform. - * Each key is an item type name, and the value indicates whether it should be extracted. - */ +/** Parsed extraction scope from the platform, keyed by item type name. */ export type ExtractionScope = Record; -/** - * WorkerAdapterOptions represents the options for WorkerAdapter class. - * @interface WorkerAdapterOptions - * @constructor - * @param {boolean=} isLocalDevelopment - A flag to indicate if the adapter is being used in local development - * @param {number=} timeout - The timeout for the worker thread - * @param {number=} batchSize - Maximum number of extracted items in a batch - * @param {Record=} workerPathOverrides - A map of event types to custom worker paths to override default worker paths - */ export interface WorkerAdapterOptions { isLocalDevelopment?: boolean; + /** Worker thread timeout. */ timeout?: number; + /** Maximum number of extracted items in a batch. */ batchSize?: number; workerPathOverrides?: WorkerPathOverrides; skipConfirmation?: boolean; } -/** - * SpawnInterface is an interface for Spawn class. - * @interface SpawnInterface - * @constructor - * @param {AirdropEvent} event - The event object received from the platform - * @param {Worker} worker - The worker thread - */ export interface SpawnInterface { - event: AirdropEvent; + event: AirSyncEvent; worker: Worker; options?: WorkerAdapterOptions; resolve: (value: void | PromiseLike) => void; originalConsole?: Console; } -/** - * SpawnFactoryInterface is an interface for Spawn class factory. - * Spawn class is responsible for spawning a new worker thread and managing the lifecycle of the worker. - * The class provides utilities to emit control events to the platform and exit the worker gracefully. - * In case of lambda timeout, the class emits a lambda timeout event to the platform. - * @interface SpawnFactoryInterface - * @constructor - * @param {AirdropEvent} event - The event object received from the platform - * @param {object=} initialState - The initial state of the adapter - * @param {string} workerPath - The path to the worker file - * @param {string} initialDomainMapping - The initial domain mapping - * @param {WorkerAdapterOptions} options - The options to create a new instance of Spawn class - * @param {string=} baseWorkerPath - The base path for the worker files, usually `__dirname` - */ export interface SpawnFactoryInterface { - event: AirdropEvent; + event: AirSyncEvent; initialState: ConnectorState; - - /** @deprecated Remove getWorkerPath function and use baseWorkerPath: __dirname instead of workerPath */ - workerPath?: string; options?: WorkerAdapterOptions; initialDomainMapping?: InitialDomainMapping; + /** Base path for the worker files, usually `__dirname`. */ baseWorkerPath?: string; } /** - * TaskAdapterInterface is an interface for TaskAdapter class. - * @interface TaskAdapterInterface - * @constructor - * @param {WorkerAdapter} adapter - The adapter object + * Returned by a worker's `task`/`onTimeout` callback; the SDK (never the connector) + * maps it to the phase-appropriate platform event and emits it exactly once. + * One invocation = one worker = one emitted event; continuation happens in a + * fresh platform-driven invocation. + * + * Status -> emitted event: 'success' -> *_DONE; 'error' -> *_ERROR; + * 'progress' -> *_PROGRESS and 'delay' -> *_DELAYED in resumable phases + * (data/attachment extraction and loading), but *_ERROR in non-resumable + * phases (external sync units, metadata), where they are illegal. */ -export interface TaskAdapterInterface { - adapter: WorkerAdapter; +export type TaskResult = + | { status: 'success' } + | { status: 'progress' } + | { status: 'delay'; delaySeconds: number } + | { status: 'error'; error: ErrorRecord }; + +export type TaskStatus = TaskResult['status']; + +/** Parameter shape passed to a worker's task and onTimeout callbacks. */ +export interface TaskAdapterInterface { + adapter: Adapter; } /** - * ProcessTaskInterface is an interface for ProcessTask class. - * @interface ProcessTaskInterface - * @constructor - * @param {function} task - The task to be executed, returns exit code - * @param {function} onTimeout - The task to be executed on timeout, returns exit code + * If `onTimeout` is omitted, the SDK emits a phase-appropriate default on + * timeout: progress for resumable phases, error for ESU/metadata. */ -export interface ProcessTaskInterface { - task: (params: TaskAdapterInterface) => Promise; - onTimeout: (params: TaskAdapterInterface) => Promise; +export interface ProcessTaskInterface { + task: (params: TaskAdapterInterface) => Promise; + onTimeout?: (params: TaskAdapterInterface) => Promise; } -/** - * WorkerEvent represents the standard worker events. - */ export enum WorkerEvent { WorkerMessage = 'message', WorkerOnline = 'online', @@ -119,9 +83,6 @@ export enum WorkerEvent { WorkerExit = 'exit', } -/** - * WorkerMessageSubject represents the handled worker message subjects. - */ export enum WorkerMessageSubject { WorkerMessageEmitted = 'emit', WorkerMessageExit = 'exit', @@ -129,9 +90,6 @@ export enum WorkerMessageSubject { WorkerMessageFailed = 'failed', } -/** - * WorkerMessageEmitted interface represents the structure of the emitted worker message. - */ export interface WorkerMessageEmitted { subject: WorkerMessageSubject.WorkerMessageEmitted; payload: { @@ -139,64 +97,42 @@ export interface WorkerMessageEmitted { }; } -/** - * WorkerMessageExit interface represents the structure of the exit worker message. - */ export interface WorkerMessageExit { subject: WorkerMessageSubject.WorkerMessageExit; } -/** - * WorkerMessageLog interface represents the structure of the worker log message. - */ export interface WorkerMessageLog { subject: WorkerMessageSubject.WorkerMessageLog; payload: { stringifiedArgs: string; level: LogLevel; - isSdkLog?: boolean; }; } -/** - * WorkerMessageFailed interface represents the structure of the worker failed message. - * Sent from the worker thread before calling process.exit(1) to convey the specific - * error reason to the main thread. - */ +/** Sent from the worker thread before process.exit(1) to convey the error reason to the main thread. */ export interface WorkerMessageFailed { subject: WorkerMessageSubject.WorkerMessageFailed; payload: { message: string }; } -/** - * WorkerMessage represents the structure of the worker message. - */ export type WorkerMessage = | WorkerMessageEmitted | WorkerMessageExit | WorkerMessageLog | WorkerMessageFailed; -/** - * WorkerData represents the structure of the worker data object. - */ export interface WorkerData { - event: AirdropEvent; + event: AirSyncEvent; initialState: ConnectorState; workerPath: string; initialDomainMapping?: InitialDomainMapping; options?: WorkerAdapterOptions; } -/** - * GetWorkerPathInterface is an interface for getting the worker path. - */ export interface GetWorkerPathInterface { - event: AirdropEvent; + event: AirSyncEvent; workerBasePath?: string | null; } -/** - * WorkerPathOverrides represents a mapping of event types to custom worker paths. - */ +/** Maps event types to custom worker paths, overriding the defaults. */ export type WorkerPathOverrides = Partial>; diff --git a/src/uploader/uploader.helpers.test.ts b/src/uploader/uploader.helpers.test.ts index 44fecb73..7d74d63c 100644 --- a/src/uploader/uploader.helpers.test.ts +++ b/src/uploader/uploader.helpers.test.ts @@ -1,12 +1,15 @@ -import fs, { promises as fsPromises } from 'fs'; +import fs from 'fs'; +import { promises as fsPromises } from 'fs'; import type { FileHandle } from 'fs/promises'; -import { jsonl } from 'js-jsonl'; import zlib from 'zlib'; +import { jsonl } from 'js-jsonl'; + import { MAX_DEVREV_FILENAME_EXTENSION_LENGTH, MAX_DEVREV_FILENAME_LENGTH, } from '../common/constants'; + import { compressGzip, computeArtifactDateRanges, @@ -229,6 +232,107 @@ describe('uploader.helpers', () => { }); }); + describe(truncateFilename.name, () => { + it('should return filename unchanged when within the limit', () => { + // Arrange + const filename = 'short-filename.txt'; + + // Act + const result = truncateFilename(filename); + + // Assert + expect(result).toBe(filename); + }); + + it('should return filename unchanged when exactly at the limit', () => { + // Arrange + const filename = 'a'.repeat(MAX_DEVREV_FILENAME_LENGTH); + + // Act + const result = truncateFilename(filename); + + // Assert + expect(result).toBe(filename); + expect(result.length).toBe(MAX_DEVREV_FILENAME_LENGTH); + }); + + it('should truncate filename and preserve extension when exceeding the limit', () => { + // Arrange + const longName = 'a'.repeat(300); + const extension = '.txt'; + const filename = longName + extension; + + // Act + const result = truncateFilename(filename); + + // Assert + expect(result.length).toBe(MAX_DEVREV_FILENAME_LENGTH); + expect(result).toContain('...'); + expect(result.endsWith(extension)).toBe(true); + }); + + it('should preserve the last MAX_DEVREV_FILENAME_EXTENSION_LENGTH characters as extension', () => { + // Arrange + const longName = 'document-'.repeat(50); + const extension = '.verylongextension'; + const filename = longName + extension; + + // Act + const result = truncateFilename(filename); + + // Assert + expect(result.length).toBe(MAX_DEVREV_FILENAME_LENGTH); + const expectedExtension = filename.slice( + -MAX_DEVREV_FILENAME_EXTENSION_LENGTH + ); + expect(result.endsWith(expectedExtension)).toBe(true); + }); + + it('should correctly format the truncated filename with ellipsis', () => { + // Arrange + const filename = 'x'.repeat(300) + '.pdf'; + + // Act + const result = truncateFilename(filename); + + // Assert + const availableNameLength = + MAX_DEVREV_FILENAME_LENGTH - MAX_DEVREV_FILENAME_EXTENSION_LENGTH - 3; + const expectedPrefix = 'x'.repeat(availableNameLength); + const expectedExtension = filename.slice( + -MAX_DEVREV_FILENAME_EXTENSION_LENGTH + ); + expect(result).toBe(`${expectedPrefix}...${expectedExtension}`); + }); + + it('[edge] should handle filename with no extension', () => { + // Arrange + const filename = 'a'.repeat(300); + + // Act + const result = truncateFilename(filename); + + // Assert + expect(result.length).toBe(MAX_DEVREV_FILENAME_LENGTH); + expect(result).toContain('...'); + // Last 20 chars are preserved as "extension" + expect( + result.endsWith('a'.repeat(MAX_DEVREV_FILENAME_EXTENSION_LENGTH)) + ).toBe(true); + }); + + it('[edge] should handle filename that is just one character over the limit', () => { + // Arrange + const filename = 'a'.repeat(MAX_DEVREV_FILENAME_LENGTH + 1); + + // Act + const result = truncateFilename(filename); + + // Assert + expect(result.length).toBe(MAX_DEVREV_FILENAME_LENGTH); + }); + }); + describe(computeArtifactDateRanges.name, () => { it('should compute min and max across multiple items', () => { // Arrange @@ -251,18 +355,10 @@ describe('uploader.helpers', () => { const result = computeArtifactDateRanges(items); // Assert - expect(result.oldest_created_date).toBe( - '2020-01-01T00:00:00.000Z' - ); - expect(result.newest_created_date).toBe( - '2022-03-15T12:00:00.000Z' - ); - expect(result.oldest_modified_date).toBe( - '2020-12-31T23:59:59.000Z' - ); - expect(result.newest_modified_date).toBe( - '2021-06-01T00:00:00.000Z' - ); + expect(result.oldest_created_date).toBe('2020-01-01T00:00:00.000Z'); + expect(result.newest_created_date).toBe('2022-03-15T12:00:00.000Z'); + expect(result.oldest_modified_date).toBe('2020-12-31T23:59:59.000Z'); + expect(result.newest_modified_date).toBe('2021-06-01T00:00:00.000Z'); }); it('should return zeros when no items have date fields', () => { @@ -298,18 +394,10 @@ describe('uploader.helpers', () => { const result = computeArtifactDateRanges(items); // Assert - expect(result.oldest_created_date).toBe( - '2021-01-01T00:00:00.000Z' - ); - expect(result.newest_created_date).toBe( - '2021-01-01T00:00:00.000Z' - ); - expect(result.oldest_modified_date).toBe( - '2023-01-01T00:00:00.000Z' - ); - expect(result.newest_modified_date).toBe( - '2023-01-01T00:00:00.000Z' - ); + expect(result.oldest_created_date).toBe('2021-01-01T00:00:00.000Z'); + expect(result.newest_created_date).toBe('2021-01-01T00:00:00.000Z'); + expect(result.oldest_modified_date).toBe('2023-01-01T00:00:00.000Z'); + expect(result.newest_modified_date).toBe('2023-01-01T00:00:00.000Z'); }); it('should handle single object input', () => { @@ -367,113 +455,8 @@ describe('uploader.helpers', () => { const result = computeArtifactDateRanges(items); // Assert - expect(result.oldest_created_date).toBe( - '2024-01-01T00:00:00.000Z' - ); - expect(result.newest_created_date).toBe( - '2024-01-01T00:00:00.000Z' - ); - }); - }); - - describe(truncateFilename.name, () => { - it('should return filename unchanged when within the limit', () => { - // Arrange - const filename = 'short-filename.txt'; - - // Act - const result = truncateFilename(filename); - - // Assert - expect(result).toBe(filename); - }); - - it('should return filename unchanged when exactly at the limit', () => { - // Arrange - const filename = 'a'.repeat(MAX_DEVREV_FILENAME_LENGTH); - - // Act - const result = truncateFilename(filename); - - // Assert - expect(result).toBe(filename); - expect(result.length).toBe(MAX_DEVREV_FILENAME_LENGTH); - }); - - it('should truncate filename and preserve extension when exceeding the limit', () => { - // Arrange - const longName = 'a'.repeat(300); - const extension = '.txt'; - const filename = longName + extension; - - // Act - const result = truncateFilename(filename); - - // Assert - expect(result.length).toBe(MAX_DEVREV_FILENAME_LENGTH); - expect(result).toContain('...'); - expect(result.endsWith(extension)).toBe(true); - }); - - it('should preserve the last MAX_DEVREV_FILENAME_EXTENSION_LENGTH characters as extension', () => { - // Arrange - const longName = 'document-'.repeat(50); - const extension = '.verylongextension'; - const filename = longName + extension; - - // Act - const result = truncateFilename(filename); - - // Assert - expect(result.length).toBe(MAX_DEVREV_FILENAME_LENGTH); - const expectedExtension = filename.slice( - -MAX_DEVREV_FILENAME_EXTENSION_LENGTH - ); - expect(result.endsWith(expectedExtension)).toBe(true); - }); - - it('should correctly format the truncated filename with ellipsis', () => { - // Arrange - const filename = 'x'.repeat(300) + '.pdf'; - - // Act - const result = truncateFilename(filename); - - // Assert - const availableNameLength = - MAX_DEVREV_FILENAME_LENGTH - MAX_DEVREV_FILENAME_EXTENSION_LENGTH - 3; - const expectedPrefix = 'x'.repeat(availableNameLength); - const expectedExtension = filename.slice( - -MAX_DEVREV_FILENAME_EXTENSION_LENGTH - ); - expect(result).toBe(`${expectedPrefix}...${expectedExtension}`); - }); - - it('[edge] should handle filename with no extension', () => { - // Arrange - const filename = 'a'.repeat(300); - - // Act - const result = truncateFilename(filename); - - // Assert - expect(result.length).toBe(MAX_DEVREV_FILENAME_LENGTH); - expect(result).toContain('...'); - // Last 20 chars are preserved as "extension" - expect( - result.endsWith('a'.repeat(MAX_DEVREV_FILENAME_EXTENSION_LENGTH)) - ).toBe(true); - }); - - it('[edge] should handle filename that is just one character over the limit', () => { - // Arrange - const filename = 'a'.repeat(MAX_DEVREV_FILENAME_LENGTH + 1); - - // Act - const result = truncateFilename(filename); - - // Assert - expect(result.length).toBe(MAX_DEVREV_FILENAME_LENGTH); + expect(result.oldest_created_date).toBe('2024-01-01T00:00:00.000Z'); + expect(result.newest_created_date).toBe('2024-01-01T00:00:00.000Z'); }); }); }); diff --git a/src/uploader/uploader.helpers.ts b/src/uploader/uploader.helpers.ts index 60e39750..7f55bcb0 100644 --- a/src/uploader/uploader.helpers.ts +++ b/src/uploader/uploader.helpers.ts @@ -1,22 +1,21 @@ import fs, { promises as fsPromises } from 'fs'; -import { jsonl } from 'js-jsonl'; import zlib from 'zlib'; +import { jsonl } from 'js-jsonl'; + import { MAX_DEVREV_FILENAME_EXTENSION_LENGTH, MAX_DEVREV_FILENAME_LENGTH, } from '../common/constants'; import { NormalizedItem } from '../repo/repo.interfaces'; + import { ArtifactDateField, ArtifactDateRanges, UploaderResult, } from './uploader.interfaces'; -/** - * Computes oldest/newest created and modified timestamps (RFC3339) across uploaded items. - * @param fetchedObjects - Single object or array of objects (e.g. NormalizedItem[]) - */ +/** Computes oldest/newest created and modified timestamps (RFC3339) across uploaded items. */ export function computeArtifactDateRanges( fetchedObjects: object[] | object ): ArtifactDateRanges { @@ -93,11 +92,6 @@ export function computeArtifactDateRanges( return result; } -/** - * Compresses a JSONL string using gzip compression. - * @param {string} jsonlObject - The JSONL string to compress - * @returns {Buffer | void} The compressed buffer or undefined on error - */ export function compressGzip(jsonlObject: string): UploaderResult { try { return { response: zlib.gzipSync(jsonlObject) }; @@ -106,11 +100,6 @@ export function compressGzip(jsonlObject: string): UploaderResult { } } -/** - * Decompresses a gzipped buffer to a JSONL string. - * @param {Buffer} gzippedJsonlObject - The gzipped buffer to decompress - * @returns {string | void} The decompressed JSONL string or undefined on error - */ export function decompressGzip( gzippedJsonlObject: Buffer ): UploaderResult { @@ -122,11 +111,6 @@ export function decompressGzip( } } -/** - * Parses a JSONL string into an array of objects. - * @param {string} jsonlObject - The JSONL string to parse - * @returns {object[] | null} The parsed array of objects or null on error - */ export function parseJsonl(jsonlObject: string): UploaderResult { try { return { response: jsonl.parse(jsonlObject) }; @@ -135,12 +119,7 @@ export function parseJsonl(jsonlObject: string): UploaderResult { } } -/** - * Downloads fetched objects to the local file system (for local development). - * @param {string} itemType - The type of items being downloaded - * @param {object | object[]} fetchedObjects - The objects to write to file - * @returns {Promise} Resolves when the file is written or rejects on error - */ +/** Writes fetched objects to the local file system (local development only). */ export async function downloadToLocal( itemType: string, fetchedObjects: object | object[] @@ -174,13 +153,7 @@ export async function downloadToLocal( } } -/** - * Truncates a filename if it exceeds the maximum allowed length. - * @param {string} filename - The filename to truncate - * @returns {string} The truncated filename - */ export function truncateFilename(filename: string): string { - // If the filename is already within the limit, return it as is. if (filename.length <= MAX_DEVREV_FILENAME_LENGTH) { return filename; } @@ -190,11 +163,9 @@ export function truncateFilename(filename: string): string { ); const extension = filename.slice(-MAX_DEVREV_FILENAME_EXTENSION_LENGTH); - // Calculate how many characters are available for the name part after accounting for the extension and "..." const availableNameLength = MAX_DEVREV_FILENAME_LENGTH - MAX_DEVREV_FILENAME_EXTENSION_LENGTH - 3; // -3 for "..." - // Truncate the name part and add an ellipsis const truncatedFilename = filename.slice(0, availableNameLength); return `${truncatedFilename}...${extension}`; diff --git a/src/uploader/uploader.interfaces.ts b/src/uploader/uploader.interfaces.ts index 6650b911..95a31c84 100644 --- a/src/uploader/uploader.interfaces.ts +++ b/src/uploader/uploader.interfaces.ts @@ -1,24 +1,17 @@ import { ErrorRecord } from '../types/common'; -import { AirdropEvent } from '../types/extraction'; +import { AirSyncEvent } from '../types/extraction'; import { ExternalSystemItem, StatsFileObject } from '../types/loading'; import { WorkerAdapterOptions } from '../types/workers'; export interface UploaderFactoryInterface { - event: AirdropEvent; + event: AirSyncEvent; options?: WorkerAdapterOptions; } -/** - * Generic result type for uploader operations that can either succeed with a response or fail with an error. - * @template T The type of the successful response data - */ export type UploaderResult = | { response: T; error?: never } | { response?: never; error: unknown }; -/** - * Artifact is an interface that defines the structure of an artifact. Artifact is a file that is generated by the extractor and uploaded to ADaaS. - */ export enum ArtifactDateField { OldestCreatedDate = 'oldest_created_date', NewestCreatedDate = 'newest_created_date', @@ -26,6 +19,7 @@ export enum ArtifactDateField { NewestModifiedDate = 'newest_modified_date', } +/** A file generated by the extractor and uploaded to AirSync. */ export interface Artifact { id: string; item_type: string; @@ -44,9 +38,7 @@ export type ArtifactDateRanges = Pick< | ArtifactDateField.NewestModifiedDate >; -/** - * ArtifactsPrepareResponse is an interface that defines the structure of the response from the prepare artifacts endpoint. - */ +/** Response from the prepare artifacts endpoint. */ export interface ArtifactsPrepareResponse { url: string; id: string; @@ -56,9 +48,7 @@ export interface ArtifactsPrepareResponse { }[]; } -/** - * ArtifactToUpload is an interface that defines the structure of the response from the get upload url endpoint. - */ +/** Response from the get upload url endpoint. */ export interface ArtifactToUpload { upload_url: string; artifact_id: string; @@ -68,33 +58,22 @@ export interface ArtifactToUpload { }[]; } -/** - * UploadResponse is an interface that defines the structure of the response from upload through Uploader. - */ export interface UploadResponse { artifact?: Artifact; error?: ErrorRecord; } -/** - * StreamAttachmentsResponse is an interface that defines the structure of the response from the stream attachments through Uploader. - */ export interface StreamAttachmentsResponse { ssorAttachments?: SsorAttachment[]; error?: ErrorRecord; } -/** - * StreamResponse is an interface that defines the structure of the response from the stream of single attachment through Uploader. - */ +/** Result of streaming a single attachment through Uploader. */ export interface StreamResponse { ssorAttachment?: SsorAttachment; error?: ErrorRecord; } -/** - * SsorAttachment is an interface that defines the structure of the SSOR attachment. - */ export interface SsorAttachment { id: { devrev: string; diff --git a/src/uploader/uploader.test.ts b/src/uploader/uploader.test.ts index 19dc4e9f..69ee211c 100644 --- a/src/uploader/uploader.test.ts +++ b/src/uploader/uploader.test.ts @@ -1,10 +1,11 @@ +import zlib from 'zlib'; + import { AxiosResponse } from 'axios'; import FormData from 'form-data'; import { jsonl } from 'js-jsonl'; -import zlib from 'zlib'; -import { createMockEvent } from '../common/test-utils'; -import { axiosClient } from '../http/axios-client-internal'; +import { axiosClient } from '../http/client'; +import { createMockEvent } from '../testing/mock-event'; import { mockServer } from '../tests/jest.setup'; import { callPrivateMethod, @@ -20,7 +21,7 @@ import { Uploader } from './uploader'; import { compressGzip, downloadToLocal } from './uploader.helpers'; import { ArtifactToUpload, UploaderResult } from './uploader.interfaces'; -jest.mock('../http/axios-client-internal'); +jest.mock('../http/client'); jest.mock('./uploader.helpers', () => ({ ...jest.requireActual('./uploader.helpers'), downloadToLocal: jest.fn(), @@ -31,10 +32,7 @@ const mockedAxiosClient = jest.mocked(axiosClient); const mockedDownloadToLocal = jest.mocked(downloadToLocal); const mockedCompressGzip = jest.mocked(compressGzip); -/** - * Type definition for private Uploader methods that need testing. - * This provides type safety when testing private methods. - */ +/** Type safety for private Uploader methods accessed in tests. */ type UploaderPrivateMethods = { destroyStream: (fileStream: AxiosResponse) => void; getArtifactDownloadUrl: ( @@ -86,75 +84,6 @@ describe(Uploader.name, () => { expect(result.error).toBeUndefined(); }); - it('should compute oldest/newest created and modified dates from normalized items', async () => { - // Arrange - const itemType = 'tasks'; - const fetchedObjects = [ - { - id: '1', - created_date: '2020-06-15T10:00:00.000Z', - modified_date: '2021-01-20T10:00:00.000Z', - data: { name: 'Task 1' }, - }, - { - id: '2', - created_date: '2019-03-01T08:00:00.000Z', - modified_date: '2022-11-30T18:00:00.000Z', - data: { name: 'Task 2' }, - }, - ]; - - mockedAxiosClient.get.mockResolvedValueOnce( - mockArtifactUploadUrlResponse - ); - mockedAxiosClient.post.mockResolvedValue(createAxiosResponse()); - - // Act - const result = await uploader.upload(itemType, fetchedObjects); - - // Assert - expect(result.artifact?.oldest_created_date).toBe( - '2019-03-01T08:00:00.000Z' - ); - expect(result.artifact?.newest_created_date).toBe( - '2020-06-15T10:00:00.000Z' - ); - expect(result.artifact?.oldest_modified_date).toBe( - '2021-01-20T10:00:00.000Z' - ); - expect(result.artifact?.newest_modified_date).toBe( - '2022-11-30T18:00:00.000Z' - ); - expect(result.error).toBeUndefined(); - }); - - it('should compute date ranges for single object upload', async () => { - // Arrange - const itemType = 'metadata'; - const fetchedObject = { - id: '1', - created_date: '2018-12-25T00:00:00.000Z', - modified_date: '2018-12-26T00:00:00.000Z', - data: { key: 'value' }, - }; - - mockedAxiosClient.get.mockResolvedValueOnce( - mockArtifactUploadUrlResponse - ); - mockedAxiosClient.post.mockResolvedValue(createAxiosResponse()); - - // Act - const result = await uploader.upload(itemType, fetchedObject); - - // Assert - const createdTs = '2018-12-25T00:00:00.000Z'; - const modifiedTs = '2018-12-26T00:00:00.000Z'; - expect(result.artifact?.oldest_created_date).toBe(createdTs); - expect(result.artifact?.newest_created_date).toBe(createdTs); - expect(result.artifact?.oldest_modified_date).toBe(modifiedTs); - expect(result.artifact?.newest_modified_date).toBe(modifiedTs); - }); - it('should report item_count as 1 when uploading single object', async () => { // Arrange const itemType = 'metadata'; diff --git a/src/uploader/uploader.ts b/src/uploader/uploader.ts index b5468109..84b6288a 100644 --- a/src/uploader/uploader.ts +++ b/src/uploader/uploader.ts @@ -1,11 +1,12 @@ import { AxiosResponse } from 'axios'; import FormData from 'form-data'; import { jsonl } from 'js-jsonl'; -import { axiosClient } from '../http/axios-client-internal'; import { MAX_DEVREV_ARTIFACT_SIZE } from '../common/constants'; -import { NormalizedAttachment } from '../repo/repo.interfaces'; +import { axiosClient } from '../http/client'; import { serializeError } from '../logger/logger'; +import { NormalizedAttachment } from '../repo/repo.interfaces'; +import { HttpStreamResponse } from '../types/extraction'; import { compressGzip, @@ -18,9 +19,9 @@ import { import { Artifact, ArtifactToUpload, - UploadResponse, UploaderFactoryInterface, UploaderResult, + UploadResponse, } from './uploader.interfaces'; export class Uploader { @@ -42,12 +43,7 @@ export class Uploader { }; } - /** - * Uploads the fetched objects to the DevRev platform. Fetched objects are compressed to a gzipped jsonl object and uploaded to the platform. - * @param {string} itemType - The type of the item to be uploaded - * @param {object[] | object} fetchedObjects - The objects to be uploaded - * @returns {Promise} - The response object containing the artifact information or error information if there was an error - */ + /** Compresses fetched objects to gzipped JSONL and uploads them as an artifact. */ async upload( itemType: string, fetchedObjects: object[] | object @@ -55,7 +51,6 @@ export class Uploader { if (this.isLocalDevelopment) { await downloadToLocal(itemType, fetchedObjects); } - // Compress the fetched objects to a gzipped jsonl object const { response: file, error: fileError } = compressGzip( jsonl.stringify(fetchedObjects) ); @@ -72,7 +67,6 @@ export class Uploader { const filename = itemType + '.jsonl.gz'; const fileType = 'application/x-gzip'; - // Get upload url const { error: preparedArtifactError, response: preparedArtifact } = await this.getArtifactUploadUrl(filename, fileType); if (preparedArtifactError) { @@ -85,7 +79,6 @@ export class Uploader { }; } - // Upload prepared artifact to the given url const { error: uploadItemError } = await this.uploadArtifact( preparedArtifact!, file! @@ -100,10 +93,9 @@ export class Uploader { }; } - // Skip confirmation for External Sync Units, as this confirmation attachments - // uploads to the sync, which we haven't created yet when extracting External Sync Units. + // Skip confirmation for External Sync Units: confirmation attaches the upload + // to the sync, which doesn't exist yet when extracting External Sync Units. if (!this.skipConfirmation) { - // Confirm upload const { error: confirmArtifactUploadError } = await this.confirmArtifactUpload(preparedArtifact!.artifact_id); if (confirmArtifactUploadError) { @@ -111,7 +103,7 @@ export class Uploader { error: { message: 'Error while confirming artifact upload. ' + - serializeError(confirmArtifactUploadError), + JSON.stringify(confirmArtifactUploadError), }, }; } @@ -119,7 +111,6 @@ export class Uploader { const artifactDateRanges = computeArtifactDateRanges(fetchedObjects); - // Return the artifact information to the platform const artifact: Artifact = { id: preparedArtifact!.artifact_id, item_type: itemType, @@ -130,13 +121,6 @@ export class Uploader { return { artifact }; } - /** - * Gets the upload URL for an artifact from the DevRev API. - * @param {string} filename - The name of the file to upload - * @param {string} fileType - The MIME type of the file - * @param {number} [fileSize] - Optional file size in bytes - * @returns {Promise} The artifact upload information or undefined on error - */ async getArtifactUploadUrl( filename: string, fileType: string, @@ -168,12 +152,7 @@ export class Uploader { } } - /** - * Uploads an artifact file to the provided upload URL using multipart form data. - * @param {ArtifactToUpload} artifact - The artifact upload information containing upload URL and form data - * @param {Buffer} file - The file buffer to upload - * @returns {Promise} The axios response or undefined on error - */ + /** Uploads the file buffer to the artifact's upload URL as multipart form data. */ async uploadArtifact( artifact: ArtifactToUpload, file: Buffer @@ -196,15 +175,10 @@ export class Uploader { } } - /** - * Streams an artifact file from an axios response to the upload URL. - * @param {ArtifactToUpload} artifact - The artifact upload information containing upload URL and form data - * @param {AxiosResponse} fileStream - The axios response stream containing the file data - * @returns {Promise} The axios response or undefined on error - */ + /** Streams a file from an HTTP response directly to the artifact's upload URL. */ async streamArtifact( artifact: ArtifactToUpload, - fileStream: AxiosResponse + fileStream: HttpStreamResponse ): Promise> { const formData = new FormData(); for (const field in artifact.form_data) { @@ -226,13 +200,12 @@ export class Uploader { } : {}), }, - // Prevents buffering of the response in the memory + // Prevents buffering of the response in memory maxRedirects: 0, - // Allow 2xx and 3xx (redirects) to be considered successful, 4xx and 5xx will throw errors and be caught in the catch block + // 2xx and 3xx are success; 4xx/5xx throw into the catch block validateStatus: (status) => status >= 200 && status < 400, - // The fallback Content-Length above is a guess, not the real size, so the - // upload will hit the same ECONNABORTED timeout on every attempt. Retrying - // it just multiplies the delay for no chance of success. + // The fallback Content-Length is a guess, so the upload hits the same + // ECONNABORTED timeout on every attempt — retrying only multiplies the delay. ...(!hasContentLength ? { 'axios-retry': { retries: 0 } } : {}), }); this.destroyStream(fileStream); @@ -243,11 +216,6 @@ export class Uploader { } } - /** - * Confirms that an artifact upload has been completed successfully. - * @param {string} artifactId - The ID of the artifact to confirm - * @returns {Promise} The axios response or undefined on error - */ async confirmArtifactUpload(artifactId: string): Promise<{ response?: AxiosResponse; error?: unknown; @@ -267,25 +235,26 @@ export class Uploader { } ); - // If response exists and the status is 2xx, return the response if (response?.status >= 200 && response?.status < 300) { return { response }; } else { - return { error: response }; + return { + error: { + message: + 'Error while confirming artifact upload. ' + + serializeError(response), + }, + }; } } catch (error) { - return { error }; + return { error: { message: serializeError(error) } }; } } - /** - * Destroys a stream to prevent resource leaks. - * @param {any} fileStream - The axios response stream to destroy - */ - private destroyStream(fileStream: AxiosResponse): void { + private destroyStream(fileStream: HttpStreamResponse): void { try { if (fileStream && fileStream.data) { - // For axios response streams, the data property contains the actual stream + // For axios response streams, `data` holds the actual stream if (typeof fileStream.data.destroy === 'function') { fileStream.data.destroy(); } else if (typeof fileStream.data.close === 'function') { @@ -297,12 +266,7 @@ export class Uploader { } } - /** - * Retrieves attachment metadata from an artifact by downloading and parsing it. - * @param {object} param0 - Configuration object - * @param {string} param0.artifact - The artifact ID to download attachments from - * @returns {Promise<{attachments?: NormalizedAttachment[], error?: {message: string}}>} The attachments array or error object - */ + /** Downloads an artifact and parses it into attachment metadata. */ async getAttachmentsFromArtifactId({ artifact, }: { @@ -311,7 +275,6 @@ export class Uploader { attachments?: NormalizedAttachment[]; error?: { message: string }; }> { - // Get the URL of the attachments metadata artifact const { response: artifactUrl, error: artifactUrlError } = await this.getArtifactDownloadUrl(artifact); @@ -325,7 +288,6 @@ export class Uploader { }; } - // Download artifact from the URL const { response: gzippedJsonlObject, error: gzippedJsonlObjectError } = await this.downloadArtifact(artifactUrl!); if (gzippedJsonlObjectError) { @@ -338,7 +300,6 @@ export class Uploader { }; } - // Decompress the gzipped jsonl object const { response: jsonlObject, error: jsonlObjectError } = decompressGzip( gzippedJsonlObject! ); @@ -352,7 +313,6 @@ export class Uploader { }; } - // Parse the jsonl object to get the attachment metadata const { response: jsonObject, error: jsonObjectError } = parseJsonl( jsonlObject! ); @@ -369,11 +329,6 @@ export class Uploader { return { attachments: jsonObject! as NormalizedAttachment[] }; } - /** - * Gets the download URL for an artifact from the DevRev API. - * @param {string} artifactId - The ID of the artifact to download - * @returns {Promise} The download URL or undefined on error - */ private async getArtifactDownloadUrl( artifactId: string ): Promise> { @@ -396,11 +351,6 @@ export class Uploader { } } - /** - * Downloads an artifact file from the given URL. - * @param {string} artifactUrl - The URL to download the artifact from - * @returns {Promise} The artifact file buffer or undefined on error - */ private async downloadArtifact( artifactUrl: string ): Promise> { @@ -415,13 +365,7 @@ export class Uploader { } } - /** - * Retrieves and parses JSON objects from an artifact by artifact ID. - * @param {object} param0 - Configuration object - * @param {string} param0.artifactId - The artifact ID to download and parse - * @param {boolean} [param0.isGzipped=false] - Whether the artifact is gzipped - * @returns {Promise} The parsed JSON objects or undefined on error - */ + /** Downloads an artifact (optionally gzipped) and parses it as JSONL. */ async getJsonObjectByArtifactId({ artifactId, isGzipped = false, diff --git a/tsconfig.build.json b/tsconfig.build.json new file mode 100644 index 00000000..81df2d6a --- /dev/null +++ b/tsconfig.build.json @@ -0,0 +1,4 @@ +{ + "extends": "./tsconfig.json", + "exclude": ["node_modules", "dist", "**/*.test.ts", "src/tests"] +}