feat(storage): 增加文件内容校验与存储完整性巡检

.github/workflows/ci.yml

@@ -11,6 +11,20 @@ on:
 jobs:
   verify:
     runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:17-alpine
+        env:
+          POSTGRES_DB: icedr_ci
+          POSTGRES_PASSWORD: icedr_ci_password
+          POSTGRES_USER: icedr_ci
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U icedr_ci -d icedr_ci"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
 
     steps:
       - name: Checkout
@@ -30,6 +44,11 @@ jobs:
       - name: Install dependencies
         run: pnpm install --frozen-lockfile
 
+      - name: Verify PostgreSQL migrations
+        run: pnpm --filter backend exec prisma migrate deploy --schema ../database/schema.prisma --config ../prisma.config.ts
+        env:
+          DATABASE_URL: postgresql://icedr_ci:icedr_ci_password@127.0.0.1:5432/icedr_ci
+
       - name: Install Playwright browser
         run: pnpm --filter frontend exec playwright install --with-deps chromium
 
@@ -39,8 +58,38 @@ jobs:
       - name: Build
         run: pnpm build
 
+      - name: Start MinIO service
+        run: |
+          docker run --detach --rm --name icedr-ci-minio \
+            --publish 9000:9000 \
+            --env MINIO_ROOT_USER=icedr-ci-minio \
+            --env MINIO_ROOT_PASSWORD=icedr-ci-minio-secret \
+            quay.io/minio/minio:RELEASE.2025-09-07T16-13-09Z server /data
+          for attempt in $(seq 1 30); do
+            if curl --fail --silent http://127.0.0.1:9000/minio/health/live >/dev/null; then
+              exit 0
+            fi
+            if ! docker inspect icedr-ci-minio >/dev/null 2>&1; then
+              docker logs icedr-ci-minio || true
+              exit 1
+            fi
+            sleep 1
+          done
+          docker logs icedr-ci-minio || true
+          exit 1
+
       - name: Test
         run: pnpm test
+        env:
+          STORAGE_INTEGRITY_MINIO_ACCESS_KEY: icedr-ci-minio
+          STORAGE_INTEGRITY_MINIO_ENDPOINT: http://127.0.0.1:9000
+          STORAGE_INTEGRITY_MINIO_REGION: us-east-1
+          STORAGE_INTEGRITY_MINIO_SECRET_KEY: icedr-ci-minio-secret
+          STORAGE_INTEGRITY_MINIO_TEST: "1"
+
+      - name: Stop MinIO service
+        if: always()
+        run: docker rm --force icedr-ci-minio || true
 
       - name: E2E smoke
         run: pnpm test:e2e

.gitignore

@@ -32,5 +32,11 @@ skills*
 data/
 designpics/
 test*/
+!frontend/tests/
+frontend/tests/*
+!frontend/tests/e2e/
+frontend/tests/e2e/*
+!frontend/tests/e2e/admin-oauth-visual.spec.ts
+!frontend/tests/e2e/storage-integrity.spec.ts
 frontend/visual-artifacts/
 private-docs/

backend/src/app.module.ts

@@ -22,6 +22,7 @@ import { StorageModule } from './modules/storage/storage.module';
 import { TransfersModule } from './modules/downloads/transfers/transfers.module';
 import { WorkerModule } from './modules/downloads/worker/worker.module';
 import { WorkspacesModule } from './modules/admin/workspaces/workspaces.module';
+import { StorageIntegrityModule } from './modules/admin/storage-integrity/storage-integrity.module';
 
 @Module({
   imports: [
@@ -45,6 +46,7 @@ import { WorkspacesModule } from './modules/admin/workspaces/workspaces.module';
     SharesModule,
     AuditModule,
     StorageModule,
+    StorageIntegrityModule,
     TransfersModule,
     QueueModule,
     WorkerModule,

backend/src/database/file-node-parent-integrity.spec.ts

@@ -0,0 +1,182 @@
+import { PrismaBetterSqlite3 } from '@prisma/adapter-better-sqlite3';
+import { randomUUID } from 'node:crypto';
+import { existsSync, readFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, resolve } from 'node:path';
+import { PrismaClient } from '../generated/prisma-sqlite/client';
+import { PrismaService } from './prisma.service';
+
+function sqliteClient(databasePath: string) {
+  return new PrismaClient({
+    adapter: new PrismaBetterSqlite3(
+      { url: databasePath },
+      { timestampFormat: 'iso8601' },
+    ),
+  });
+}
+
+function sqliteService(client: PrismaClient) {
+  const service = Object.create(PrismaService.prototype) as PrismaService;
+  Object.assign(service as unknown as Record<string, unknown>, {
+    activeClient: client,
+    activeSource: { provider: 'sqlite' },
+  });
+  return service;
+}
+
+async function ensureParentIntegrity(service: PrismaService) {
+  await (
+    service as unknown as {
+      ensureSqliteFileNodeParentIntegrity: () => Promise<void>;
+    }
+  ).ensureSqliteFileNodeParentIntegrity();
+}
+
+describe('FileNode parent integrity', () => {
+  let client: PrismaClient;
+  let databasePath: string;
+
+  beforeEach(async () => {
+    databasePath = join(
+      tmpdir(),
+      `icedr-file-node-parent-integrity-${randomUUID()}.sqlite`,
+    );
+    client = sqliteClient(databasePath);
+    await client.$connect();
+    await client.$executeRawUnsafe(`
+      CREATE TABLE "file_nodes" (
+        "id" TEXT NOT NULL PRIMARY KEY,
+        "parent_node_id" TEXT,
+        "name" TEXT NOT NULL
+      )
+    `);
+    await client.$executeRawUnsafe(`
+      CREATE TABLE "file_versions" (
+        "id" TEXT NOT NULL PRIMARY KEY,
+        "node_id" TEXT NOT NULL,
+        CONSTRAINT "file_versions_node_id_fkey"
+          FOREIGN KEY ("node_id") REFERENCES "file_nodes" ("id")
+          ON DELETE CASCADE ON UPDATE CASCADE
+      )
+    `);
+  });
+
+  afterEach(async () => {
+    await client.$disconnect();
+    for (const suffix of ['', '-shm', '-wal']) {
+      const path = `${databasePath}${suffix}`;
+      if (existsSync(path)) rmSync(path, { force: true });
+    }
+  });
+
+  it('guards inserts and parent changes in an existing SQLite database', async () => {
+    await ensureParentIntegrity(sqliteService(client));
+    await client.$executeRawUnsafe(
+      'INSERT INTO "file_nodes" ("id", "parent_node_id", "name") VALUES (?, ?, ?)',
+      'parent',
+      null,
+      'Parent',
+    );
+    await client.$executeRawUnsafe(
+      'INSERT INTO "file_nodes" ("id", "parent_node_id", "name") VALUES (?, ?, ?)',
+      'child',
+      'parent',
+      'Child',
+    );
+
+    await expect(
+      client.$executeRawUnsafe(
+        'INSERT INTO "file_nodes" ("id", "parent_node_id", "name") VALUES (?, ?, ?)',
+        'orphan',
+        'missing-parent',
+        'Orphan',
+      ),
+    ).rejects.toThrow(/missing node/i);
+    await expect(
+      client.$executeRawUnsafe(
+        'UPDATE "file_nodes" SET "parent_node_id" = ? WHERE "id" = ?',
+        'missing-parent',
+        'child',
+      ),
+    ).rejects.toThrow(/missing node/i);
+  });
+
+  it('refuses startup when an existing SQLite database contains an orphan', async () => {
+    await client.$executeRawUnsafe(
+      'INSERT INTO "file_nodes" ("id", "parent_node_id", "name") VALUES (?, ?, ?)',
+      'orphan',
+      'missing-parent',
+      'Orphan',
+    );
+
+    await expect(ensureParentIntegrity(sqliteService(client))).rejects.toThrow(
+      'orphan -> missing-parent',
+    );
+  });
+
+  it('cascades deletion through every descendant in an existing SQLite database', async () => {
+    await ensureParentIntegrity(sqliteService(client));
+    for (const [id, parentNodeId] of [
+      ['parent', null],
+      ['child', 'parent'],
+      ['grandchild', 'child'],
+    ] as const) {
+      await client.$executeRawUnsafe(
+        'INSERT INTO "file_nodes" ("id", "parent_node_id", "name") VALUES (?, ?, ?)',
+        id,
+        parentNodeId,
+        id,
+      );
+    }
+    await client.$executeRawUnsafe(
+      'INSERT INTO "file_versions" ("id", "node_id") VALUES (?, ?), (?, ?)',
+      'child-version',
+      'child',
+      'grandchild-version',
+      'grandchild',
+    );
+
+    await client.$executeRawUnsafe(
+      'DELETE FROM "file_nodes" WHERE "id" = ?',
+      'parent',
+    );
+
+    await expect(
+      client.$queryRawUnsafe('SELECT "id" FROM "file_nodes" ORDER BY "id"'),
+    ).resolves.toEqual([]);
+    await expect(
+      client.$queryRawUnsafe('SELECT "id" FROM "file_versions" ORDER BY "id"'),
+    ).resolves.toEqual([]);
+  });
+
+  it('declares the same cascading parent relationship for PostgreSQL', () => {
+    const workspaceRoot = resolve(__dirname, '..', '..', '..');
+    const schema = readFileSync(
+      join(workspaceRoot, 'database', 'schema.prisma'),
+      'utf8',
+    );
+    const migration = readFileSync(
+      join(
+        workspaceRoot,
+        'database',
+        'migrations',
+        '20260813124000_file_node_parent_integrity',
+        'migration.sql',
+      ),
+      'utf8',
+    );
+
+    expect(schema).toContain(
+      '@relation("FileNodeHierarchy", fields: [parentNodeId], references: [id], onDelete: Cascade, onUpdate: Cascade)',
+    );
+    expect(migration).toContain('CONSTRAINT "file_nodes_parent_node_id_fkey"');
+    expect(migration).toContain(
+      'FOREIGN KEY ("parent_node_id") REFERENCES "file_nodes"("id")',
+    );
+    expect(migration).toContain('ON DELETE CASCADE ON UPDATE CASCADE');
+    expect(migration).toContain('NOT VALID');
+    expect(migration).toContain(
+      'VALIDATE CONSTRAINT "file_nodes_parent_node_id_fkey"',
+    );
+  });
+});

backend/src/database/prisma.service.spec.ts

@@ -188,6 +188,76 @@ describe('PrismaService', () => {
     expect(client.$executeRawUnsafe).toHaveBeenCalledWith(
       'ALTER TABLE "file_nodes" ADD COLUMN "name_key" TEXT NOT NULL DEFAULT \'\'',
     );
+    expect(client.$executeRawUnsafe).toHaveBeenCalledWith(
+      'ALTER TABLE "file_nodes" ADD COLUMN "checksum_algorithm" TEXT',
+    );
+    expect(client.$executeRawUnsafe).toHaveBeenCalledWith(
+      'ALTER TABLE "file_nodes" ADD COLUMN "checksum_value" TEXT',
+    );
+    expect(client.$executeRawUnsafe).toHaveBeenCalledWith(
+      'ALTER TABLE "file_nodes" ADD COLUMN "integrity_status" TEXT NOT NULL DEFAULT \'unknown\'',
+    );
+    expect(client.$executeRawUnsafe).toHaveBeenCalledWith(
+      'ALTER TABLE "file_nodes" ADD COLUMN "last_verified_at" TEXT',
+    );
+    expect(client.$executeRawUnsafe).toHaveBeenCalledWith(
+      'ALTER TABLE "file_nodes" ADD COLUMN "verification_failure_code" TEXT',
+    );
+    expect(client.$executeRawUnsafe).toHaveBeenCalledWith(
+      'ALTER TABLE "file_nodes" ADD COLUMN "integrity_acknowledged_at" TEXT',
+    );
+    expect(client.$executeRawUnsafe).toHaveBeenCalledWith(
+      'ALTER TABLE "file_nodes" ADD COLUMN "integrity_acknowledged_by" TEXT',
+    );
+    expect(client.$executeRawUnsafe).toHaveBeenCalledWith(
+      'ALTER TABLE "file_versions" ADD COLUMN "checksum_algorithm" TEXT',
+    );
+    expect(client.$executeRawUnsafe).toHaveBeenCalledWith(
+      'ALTER TABLE "file_versions" ADD COLUMN "checksum_value" TEXT',
+    );
+    expect(client.$executeRawUnsafe).toHaveBeenCalledWith(
+      'ALTER TABLE "file_versions" ADD COLUMN "integrity_status" TEXT NOT NULL DEFAULT \'unknown\'',
+    );
+    expect(client.$executeRawUnsafe).toHaveBeenCalledWith(
+      'ALTER TABLE "file_versions" ADD COLUMN "last_verified_at" TEXT',
+    );
+    expect(client.$executeRawUnsafe).toHaveBeenCalledWith(
+      'ALTER TABLE "file_versions" ADD COLUMN "verification_failure_code" TEXT',
+    );
+    expect(client.$executeRawUnsafe).toHaveBeenCalledWith(
+      'ALTER TABLE "file_versions" ADD COLUMN "integrity_acknowledged_at" TEXT',
+    );
+    expect(client.$executeRawUnsafe).toHaveBeenCalledWith(
+      'ALTER TABLE "file_versions" ADD COLUMN "integrity_acknowledged_by" TEXT',
+    );
+    expect(client.$executeRawUnsafe).toHaveBeenCalledWith(
+      expect.stringContaining(
+        'CREATE TABLE IF NOT EXISTS "blob_integrity_tasks"',
+      ),
+    );
+    expect(client.$executeRawUnsafe).toHaveBeenCalledWith(
+      expect.stringContaining(
+        'CREATE TABLE IF NOT EXISTS "blob_integrity_results"',
+      ),
+    );
+    expect(client.$executeRawUnsafe).toHaveBeenCalledWith(
+      'ALTER TABLE "blob_integrity_results" ADD COLUMN "acknowledged_at" TEXT',
+    );
+    expect(client.$executeRawUnsafe).toHaveBeenCalledWith(
+      'ALTER TABLE "blob_integrity_results" ADD COLUMN "acknowledged_by" TEXT',
+    );
+    expect(client.$executeRawUnsafe).toHaveBeenCalledWith(
+      'CREATE UNIQUE INDEX IF NOT EXISTS "blob_integrity_tasks_lease_key_key" ON "blob_integrity_tasks"("lease_key")',
+    );
+    expect(client.$executeRawUnsafe).toHaveBeenCalledWith(
+      'CREATE INDEX IF NOT EXISTS "blob_integrity_tasks_status_created_at_idx" ON "blob_integrity_tasks"("status", "created_at")',
+    );
+    expect(client.$executeRawUnsafe).toHaveBeenCalledWith(
+      'CREATE INDEX IF NOT EXISTS "blob_integrity_tasks_status_lease_expires_at_idx" ON "blob_integrity_tasks"("status", "lease_expires_at")',
+    );
+    expect(client.$executeRawUnsafe).toHaveBeenCalledWith(
+      'CREATE UNIQUE INDEX IF NOT EXISTS "blob_integrity_results_task_target_key" ON "blob_integrity_results"("task_id", "target_key")',
+    );
     expect(client.$executeRawUnsafe).toHaveBeenCalledWith(
       'CREATE INDEX IF NOT EXISTS "file_nodes_workspace_id_space_scope_idx" ON "file_nodes"("workspace_id", "space_scope")',
     );
@@ -227,7 +297,24 @@ describe('PrismaService', () => {
             'directory_key',
             'owner_scope_key',
             'name_key',
+            'checksum_algorithm',
+            'checksum_value',
+            'integrity_status',
+            'last_verified_at',
+            'verification_failure_code',
+            'integrity_acknowledged_at',
+            'integrity_acknowledged_by',
+          ],
+          file_versions: [
+            'checksum_algorithm',
+            'checksum_value',
+            'integrity_status',
+            'last_verified_at',
+            'verification_failure_code',
+            'integrity_acknowledged_at',
+            'integrity_acknowledged_by',
           ],
+          blob_integrity_results: ['acknowledged_at', 'acknowledged_by'],
           upload_sessions: [
             'space_scope',
             'requested_file_name',
@@ -266,6 +353,11 @@ describe('PrismaService', () => {
         statement.startsWith('ALTER TABLE "upload_sessions"'),
       ),
     ).toBe(false);
+    expect(
+      statements.some((statement) =>
+        statement.startsWith('ALTER TABLE "file_versions"'),
+      ),
+    ).toBe(false);
     expect(client.$executeRawUnsafe).toHaveBeenCalledWith(
       'CREATE INDEX IF NOT EXISTS "file_nodes_workspace_id_space_scope_idx" ON "file_nodes"("workspace_id", "space_scope")',
     );