feat: add ItemsSyncStep and VehiclesSyncStep for ISSUE-198

backend/src/data-source.ts

@@ -29,6 +29,13 @@ import { OauthClient } from './modules/oauth-clients/oauth-client.entity';
 import { BigBangBaselineMigration1748000000000 } from './migrations/1748000000000-BigBangBaselineMigration';
 import { CatalogEtlSchemaMigration1748000000001 } from './migrations/1748000000001-CatalogEtlSchemaMigration';
 import { AddNoStepsEtlStatus1748000000002 } from './migrations/1748000000002-AddNoStepsEtlStatus';
+import { AddDiscordAuthToUsers1779608598950 } from './migrations/1779608598950-1748100000000-AddDiscordAuthToUsers';
+import { AddPasswordExpiryToUsers1779642418093 } from './migrations/1779642418093-AddPasswordExpiryToUsers';
+import { AlterJumpPointsForSyntheticRows1779664556916 } from './migrations/1779664556916-AlterJumpPointsForSyntheticRows';
+import { FixCategoriesSectionTypeExpressionIndex1779700000000 } from './migrations/1779700000000-FixCategoriesSectionTypeExpressionIndex';
+import { AddStepNameToEtlRun1779710000000 } from './migrations/1779710000000-AddStepNameToEtlRun';
+import { AddUniqueUuidToStationItem1780010901444 } from './migrations/1780010901444-AddUniqueUuidToStationItem';
+import { MakeItemFksDeferrable1780020000000 } from './migrations/1780020000000-MakeItemFksDeferrable';
 
 export const AppDataSource = new DataSource({
   type: 'postgres',
@@ -67,6 +74,13 @@ export const AppDataSource = new DataSource({
     BigBangBaselineMigration1748000000000,
     CatalogEtlSchemaMigration1748000000001,
     AddNoStepsEtlStatus1748000000002,
+    AddDiscordAuthToUsers1779608598950,
+    AddPasswordExpiryToUsers1779642418093,
+    AlterJumpPointsForSyntheticRows1779664556916,
+    FixCategoriesSectionTypeExpressionIndex1779700000000,
+    AddStepNameToEtlRun1779710000000,
+    AddUniqueUuidToStationItem1780010901444,
+    MakeItemFksDeferrable1780020000000,
   ],
   synchronize: false,
   extra: { parseInt8: true },

backend/src/migrations/1780010901444-AddUniqueUuidToStationItem.ts

@@ -0,0 +1,41 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class AddUniqueUuidToStationItem1780010901444
+  implements MigrationInterface
+{
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    // Remove duplicate uuid rows before adding the constraint, keeping the
+    // most recently synced row for each uuid (NULL uuids are never deduplicated
+    // by a UNIQUE constraint so they are left as-is).
+    await queryRunner.query(`
+      DELETE FROM station_item
+      WHERE id IN (
+        SELECT id FROM (
+          SELECT id,
+                 ROW_NUMBER() OVER (PARTITION BY uuid ORDER BY synced_at DESC, id DESC) AS rn
+          FROM station_item
+          WHERE uuid IS NOT NULL
+        ) ranked
+        WHERE rn > 1
+      )
+    `);
+
+    await queryRunner.query(
+      `CREATE UNIQUE INDEX "uq_station_item_uuid" ON "station_item" ("uuid") WHERE "uuid" IS NOT NULL`,
+    );
+
+    // Drop the baseline non-unique idx_items_uuid — it is fully superseded by
+    // the partial unique index above, which also serves as a lookup index.
+    // Keeping both would leave a non-unique index that contradicts the new
+    // uniqueness guarantee and could cause the reconciliation SELECT to return
+    // an arbitrary row if a duplicate somehow slipped through.
+    await queryRunner.query(`DROP INDEX IF EXISTS "idx_items_uuid"`);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`DROP INDEX IF EXISTS "uq_station_item_uuid"`);
+    await queryRunner.query(
+      `CREATE INDEX "idx_items_uuid" ON "station_item" ("uuid")`,
+    );
+  }
+}

backend/src/migrations/1780020000000-MakeItemFksDeferrable.ts

@@ -0,0 +1,89 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class MakeItemFksDeferrable1780020000000 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    // Drop the auto-named inline FK from station_item_attribute.item_uex_id
+    // and recreate it as DEFERRABLE INITIALLY DEFERRED so the UUID-based
+    // uex_id reconciliation in ItemsSyncStep can re-key dependents and update
+    // the referenced uex_id within a single transaction without violating the
+    // FK mid-statement.
+    await queryRunner.query(`
+      DO $$
+      DECLARE
+        con_name TEXT;
+      BEGIN
+        SELECT conname INTO con_name
+        FROM pg_constraint
+        WHERE conrelid = 'station_item_attribute'::regclass
+          AND confrelid = 'station_item'::regclass
+          AND contype = 'f'
+        LIMIT 1;
+
+        IF con_name IS NOT NULL THEN
+          EXECUTE format('ALTER TABLE station_item_attribute DROP CONSTRAINT %I', con_name);
+        END IF;
+      END $$;
+    `);
+
+    await queryRunner.query(`
+      ALTER TABLE station_item_attribute
+        ADD CONSTRAINT fk_item_attr_item_uex_id
+        FOREIGN KEY (item_uex_id)
+        REFERENCES station_item (uex_id)
+        ON DELETE CASCADE
+        DEFERRABLE INITIALLY DEFERRED
+    `);
+
+    // Drop and recreate the self-referential FK on station_item.parent_uex_id.
+    await queryRunner.query(`
+      DO $$
+      DECLARE
+        con_name TEXT;
+      BEGIN
+        SELECT conname INTO con_name
+        FROM pg_constraint
+        WHERE conrelid = 'station_item'::regclass
+          AND confrelid = 'station_item'::regclass
+          AND contype = 'f'
+        LIMIT 1;
+
+        IF con_name IS NOT NULL THEN
+          EXECUTE format('ALTER TABLE station_item DROP CONSTRAINT %I', con_name);
+        END IF;
+      END $$;
+    `);
+
+    await queryRunner.query(`
+      ALTER TABLE station_item
+        ADD CONSTRAINT fk_item_parent_uex_id
+        FOREIGN KEY (parent_uex_id)
+        REFERENCES station_item (uex_id)
+        ON DELETE SET NULL
+        DEFERRABLE INITIALLY DEFERRED
+    `);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE station_item_attribute DROP CONSTRAINT IF EXISTS fk_item_attr_item_uex_id`,
+    );
+    await queryRunner.query(`
+      ALTER TABLE station_item_attribute
+        ADD CONSTRAINT fk_item_attr_item_uex_id
+        FOREIGN KEY (item_uex_id)
+        REFERENCES station_item (uex_id)
+        ON DELETE CASCADE
+    `);
+
+    await queryRunner.query(
+      `ALTER TABLE station_item DROP CONSTRAINT IF EXISTS fk_item_parent_uex_id`,
+    );
+    await queryRunner.query(`
+      ALTER TABLE station_item
+        ADD CONSTRAINT fk_item_parent_uex_id
+        FOREIGN KEY (parent_uex_id)
+        REFERENCES station_item (uex_id)
+        ON DELETE SET NULL
+    `);
+  }
+}

backend/src/modules/catalog-etl/catalog-etl.module.ts

@@ -21,6 +21,7 @@ import { CategoriesSyncStep } from './steps/categories-sync.step';
 import { TerminalsSyncStep } from './steps/terminals-sync.step';
 import { TerminalDistancesSyncStep } from './steps/terminal-distances-sync.step';
 import { VehiclesSyncStep } from './steps/vehicles-sync.step';
+import { ItemsSyncStep } from './steps/items-sync.step';
 import { CommoditiesSyncStep } from './steps/commodities-sync.step';
 import { CatalogEtlScheduler } from './schedulers/catalog-etl.scheduler';
 import { UexSyncModule } from '../uex-sync/uex-sync.module';
@@ -48,6 +49,7 @@ import { UexSyncModule } from '../uex-sync/uex-sync.module';
     TerminalsSyncStep,
     TerminalDistancesSyncStep,
     VehiclesSyncStep,
+    ItemsSyncStep,
     CommoditiesSyncStep,
   ],
   exports: [CatalogEtlService],

backend/src/modules/catalog-etl/catalog-etl.service.spec.ts

@@ -24,6 +24,7 @@ import { CategoriesSyncStep } from './steps/categories-sync.step';
 import { TerminalsSyncStep } from './steps/terminals-sync.step';
 import { TerminalDistancesSyncStep } from './steps/terminal-distances-sync.step';
 import { VehiclesSyncStep } from './steps/vehicles-sync.step';
+import { ItemsSyncStep } from './steps/items-sync.step';
 import { CommoditiesSyncStep } from './steps/commodities-sync.step';
 
 function buildMockRun(overrides: Partial<EtlRun> = {}): EtlRun {
@@ -159,6 +160,10 @@ describe('CatalogEtlService', () => {
           provide: VehiclesSyncStep,
           useValue: { name: 'vehicles-sync', execute: jest.fn() },
         },
+        {
+          provide: ItemsSyncStep,
+          useValue: { name: 'items-sync', execute: jest.fn() },
+        },
         {
           provide: CommoditiesSyncStep,
           useValue: { name: 'commodities-sync', execute: jest.fn() },

backend/src/modules/catalog-etl/catalog-etl.service.ts

@@ -22,6 +22,7 @@ import { CategoriesSyncStep } from './steps/categories-sync.step';
 import { TerminalsSyncStep } from './steps/terminals-sync.step';
 import { TerminalDistancesSyncStep } from './steps/terminal-distances-sync.step';
 import { VehiclesSyncStep } from './steps/vehicles-sync.step';
+import { ItemsSyncStep } from './steps/items-sync.step';
 import { CommoditiesSyncStep } from './steps/commodities-sync.step';
 
 @Injectable()
@@ -53,6 +54,7 @@ export class CatalogEtlService {
     private readonly terminalsSyncStep: TerminalsSyncStep,
     private readonly terminalDistancesSyncStep: TerminalDistancesSyncStep,
     private readonly vehiclesSyncStep: VehiclesSyncStep,
+    private readonly itemsSyncStep: ItemsSyncStep,
     private readonly commoditiesSyncStep: CommoditiesSyncStep,
   ) {
     this.ETL_STEPS = [
@@ -72,6 +74,7 @@ export class CatalogEtlService {
       terminalsSyncStep,
       terminalDistancesSyncStep,
       vehiclesSyncStep,
+      itemsSyncStep,
       commoditiesSyncStep,
     ];
   }