|
| 1 | +/** Real PostgreSQL retention removes only finished events that nothing reads again. */ |
| 2 | + |
| 3 | +import { outboxEvent } from '@sim/db/schema' |
| 4 | +import { readTestDatabaseUrl } from '@sim/db/testing/test-infrastructure' |
| 5 | +import { withUtcTimestamps } from '@sim/db/timestamps' |
| 6 | +import { generateId } from '@sim/utils/id' |
| 7 | +import { sql } from 'drizzle-orm' |
| 8 | +import { drizzle, type PostgresJsDatabase } from 'drizzle-orm/postgres-js' |
| 9 | +import postgres from 'postgres' |
| 10 | +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest' |
| 11 | + |
| 12 | +const database = vi.hoisted(() => ({ current: undefined as PostgresJsDatabase | undefined })) |
| 13 | + |
| 14 | +vi.mock('@sim/db', () => ({ |
| 15 | + get db() { |
| 16 | + if (!database.current) throw new Error('Outbox PostgreSQL test database is not initialized') |
| 17 | + return database.current |
| 18 | + }, |
| 19 | +})) |
| 20 | + |
| 21 | +import { |
| 22 | + COMPLETED_OUTBOX_RETENTION_MS, |
| 23 | + OUTBOX_PRUNE_BATCH_SIZE, |
| 24 | + pruneCompletedOutboxEvents, |
| 25 | +} from '@/lib/core/outbox/retention' |
| 26 | + |
| 27 | +const RECOVER = 'knowledge.document.processing.recover' |
| 28 | +const STORAGE_CLEANUP = 'knowledge.document.storage.cleanup' |
| 29 | +const ADMIN_OPERATION = 'admin.organization-member-operation' |
| 30 | +const OCR_CHECKPOINT_EXPIRY = 'knowledge.document.ocr-checkpoint.expire' |
| 31 | + |
| 32 | +describe('completed outbox retention in PostgreSQL', () => { |
| 33 | + const schemaName = `outbox_retention_${generateId().replaceAll('-', '')}` |
| 34 | + const connection = postgres( |
| 35 | + readTestDatabaseUrl(), |
| 36 | + withUtcTimestamps({ |
| 37 | + max: 2, |
| 38 | + prepare: false, |
| 39 | + fetch_types: false, |
| 40 | + connection: { search_path: schemaName }, |
| 41 | + onnotice: () => {}, |
| 42 | + }) |
| 43 | + ) |
| 44 | + const expired = () => new Date(Date.now() - COMPLETED_OUTBOX_RETENTION_MS - 60_000) |
| 45 | + |
| 46 | + beforeAll(async () => { |
| 47 | + await connection`CREATE SCHEMA ${connection(schemaName)}` |
| 48 | + await connection`CREATE TABLE outbox_event (LIKE public.outbox_event INCLUDING ALL)` |
| 49 | + database.current = drizzle(connection) |
| 50 | + }) |
| 51 | + |
| 52 | + afterEach(async () => { |
| 53 | + await connection`TRUNCATE outbox_event` |
| 54 | + }) |
| 55 | + |
| 56 | + afterAll(async () => { |
| 57 | + try { |
| 58 | + await connection`DROP SCHEMA ${connection(schemaName)} CASCADE` |
| 59 | + } finally { |
| 60 | + await connection.end() |
| 61 | + database.current = undefined |
| 62 | + } |
| 63 | + }) |
| 64 | + |
| 65 | + async function seed(eventType: string, status: string, createdAt: Date) { |
| 66 | + const id = generateId() |
| 67 | + await database.current!.insert(outboxEvent).values({ |
| 68 | + id, |
| 69 | + eventType, |
| 70 | + payload: {}, |
| 71 | + status, |
| 72 | + createdAt, |
| 73 | + availableAt: createdAt, |
| 74 | + }) |
| 75 | + return id |
| 76 | + } |
| 77 | + |
| 78 | + async function remainingIds() { |
| 79 | + const rows = await database.current!.select({ id: outboxEvent.id }).from(outboxEvent) |
| 80 | + return new Set(rows.map((row) => row.id)) |
| 81 | + } |
| 82 | + |
| 83 | + it('deletes only expired completed events of prunable types', async () => { |
| 84 | + const pruned = [ |
| 85 | + await seed(RECOVER, 'completed', expired()), |
| 86 | + await seed(STORAGE_CLEANUP, 'completed', expired()), |
| 87 | + ] |
| 88 | + const kept = [ |
| 89 | + await seed(RECOVER, 'completed', new Date(Date.now() - 60_000)), |
| 90 | + await seed(STORAGE_CLEANUP, 'pending', expired()), |
| 91 | + await seed(RECOVER, 'processing', expired()), |
| 92 | + await seed(STORAGE_CLEANUP, 'dead_letter', expired()), |
| 93 | + await seed(ADMIN_OPERATION, 'completed', expired()), |
| 94 | + await seed(OCR_CHECKPOINT_EXPIRY, 'completed', expired()), |
| 95 | + ] |
| 96 | + |
| 97 | + expect(await pruneCompletedOutboxEvents()).toBe(pruned.length) |
| 98 | + expect(await remainingIds()).toEqual(new Set(kept)) |
| 99 | + }) |
| 100 | + |
| 101 | + it('deletes at most one oldest batch per type per run, and the next run continues', async () => { |
| 102 | + const createdAt = expired().toISOString() |
| 103 | + for (const [prefix, eventType] of [ |
| 104 | + ['recover', RECOVER], |
| 105 | + ['storage', STORAGE_CLEANUP], |
| 106 | + ]) { |
| 107 | + await database.current!.execute(sql` |
| 108 | + INSERT INTO outbox_event (id, event_type, payload, status, available_at, created_at) |
| 109 | + SELECT ${prefix} || ':' || n, ${eventType}, '{}'::json, 'completed', |
| 110 | + ${createdAt}::timestamp, ${createdAt}::timestamp - n * interval '1 millisecond' |
| 111 | + FROM generate_series(0, ${OUTBOX_PRUNE_BATCH_SIZE}::integer) AS n |
| 112 | + `) |
| 113 | + } |
| 114 | + const pending = await seed(RECOVER, 'pending', expired()) |
| 115 | + |
| 116 | + expect(await pruneCompletedOutboxEvents()).toBe(2 * OUTBOX_PRUNE_BATCH_SIZE) |
| 117 | + expect(await remainingIds()).toEqual(new Set(['recover:0', 'storage:0', pending])) |
| 118 | + expect(await pruneCompletedOutboxEvents()).toBe(2) |
| 119 | + expect(await remainingIds()).toEqual(new Set([pending])) |
| 120 | + }) |
| 121 | +}) |
0 commit comments