From 93db41f59206ea5d96ae71491bb516dfc3432c6d Mon Sep 17 00:00:00 2001 From: AJtheManager Date: Mon, 28 Sep 2026 10:45:50 +0100 Subject: [PATCH] Account Export Stale File Cleanup Scheduling --- backend/.env.example | 14 ++ .../account-export-cleanup.scheduler.ts | 89 ++++++++++++ backend/src/account/account.module.ts | 3 +- backend/src/account/account.service.ts | 137 +++++++++++++++++- backend/src/config/env.validation.ts | 16 ++ 5 files changed, 253 insertions(+), 6 deletions(-) create mode 100644 backend/src/account/account-export-cleanup.scheduler.ts diff --git a/backend/.env.example b/backend/.env.example index f5752eb35..e7d584129 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -22,6 +22,20 @@ RECONCILE_WINDOW=200 LEADERBOARD_SNAPSHOT_CRON=0 * * * * LEADERBOARD_SNAPSHOT_RETENTION_DAYS=30 +# Account Data Export Configuration +EXPORT_DIR=./exports +# Hours a ready export stays downloadable before its file is cleaned up +EXPORT_TTL_HOURS=48 +# Stale export file cleanup schedule +EXPORT_CLEANUP_ENABLED=true +EXPORT_CLEANUP_CRON=0 * * * * +# Hours a failed export job row is kept before removal +EXPORT_FAILED_RETENTION_HOURS=24 +# Minutes a job may stay in "processing" before it is treated as stuck and failed +EXPORT_STUCK_PROCESSING_MINUTES=60 +# Minimum age (minutes) of an export file with no job row before it is deleted +EXPORT_ORPHAN_GRACE_MINUTES=60 + # Idempotency Key Configuration IDEMPOTENCY_KEY_TTL_HOURS=24 diff --git a/backend/src/account/account-export-cleanup.scheduler.ts b/backend/src/account/account-export-cleanup.scheduler.ts new file mode 100644 index 000000000..a6f25b7a6 --- /dev/null +++ b/backend/src/account/account-export-cleanup.scheduler.ts @@ -0,0 +1,89 @@ +import { + Injectable, + Logger, + OnModuleDestroy, + OnModuleInit, +} from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { SchedulerRegistry } from '@nestjs/schedule'; +import { CronJob } from 'cron'; +import { AccountService } from './account.service'; + +export const EXPORT_CLEANUP_JOB_NAME = 'account-export-stale-file-cleanup'; + +/** + * Schedules periodic removal of stale account export files and job rows on + * the cadence given by EXPORT_CLEANUP_CRON. Runs never overlap: a tick that + * fires while the previous run is still in progress is skipped. + */ +@Injectable() +export class AccountExportCleanupScheduler + implements OnModuleInit, OnModuleDestroy +{ + private readonly logger = new Logger(AccountExportCleanupScheduler.name); + private running = false; + + constructor( + private readonly accountService: AccountService, + private readonly schedulerRegistry: SchedulerRegistry, + private readonly configService: ConfigService, + ) {} + + onModuleInit(): void { + const enabled = + String( + this.configService.get('EXPORT_CLEANUP_ENABLED', 'true'), + ).toLowerCase() !== 'false'; + if (!enabled) { + this.logger.log('Account export cleanup is disabled'); + return; + } + + const cronExpression = this.configService.get( + 'EXPORT_CLEANUP_CRON', + '0 * * * *', + ); + + const job = new CronJob(cronExpression, () => { + void this.handleCleanup(); + }); + + this.schedulerRegistry.addCronJob(EXPORT_CLEANUP_JOB_NAME, job); + job.start(); + + this.logger.log( + `Account export cleanup scheduled with cron "${cronExpression}"`, + ); + } + + onModuleDestroy(): void { + if (this.schedulerRegistry.doesExist('cron', EXPORT_CLEANUP_JOB_NAME)) { + this.schedulerRegistry.deleteCronJob(EXPORT_CLEANUP_JOB_NAME); + } + } + + async handleCleanup(): Promise { + if (this.running) { + this.logger.warn( + 'Previous account export cleanup still running; skipping this tick', + ); + return; + } + + this.running = true; + try { + const { expired, failed, stuck, orphans } = + await this.accountService.cleanupExports(); + if (expired + failed + stuck + orphans > 0) { + this.logger.log( + `Account export cleanup: expired=${expired} failed=${failed} ` + + `stuck=${stuck} orphans=${orphans}`, + ); + } + } catch (err) { + this.logger.error('Account export cleanup failed', err); + } finally { + this.running = false; + } + } +} diff --git a/backend/src/account/account.module.ts b/backend/src/account/account.module.ts index 6acd77913..28c28c8b3 100644 --- a/backend/src/account/account.module.ts +++ b/backend/src/account/account.module.ts @@ -1,5 +1,6 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; +import { AccountExportCleanupScheduler } from './account-export-cleanup.scheduler'; import { AccountController } from './account.controller'; import { AccountService } from './account.service'; import { DataExportJob } from './entities/data-export-job.entity'; @@ -7,6 +8,6 @@ import { DataExportJob } from './entities/data-export-job.entity'; @Module({ imports: [TypeOrmModule.forFeature([DataExportJob])], controllers: [AccountController], - providers: [AccountService], + providers: [AccountService, AccountExportCleanupScheduler], }) export class AccountModule {} diff --git a/backend/src/account/account.service.ts b/backend/src/account/account.service.ts index e45382073..927fcc150 100644 --- a/backend/src/account/account.service.ts +++ b/backend/src/account/account.service.ts @@ -7,11 +7,21 @@ import { import { ConfigService } from '@nestjs/config'; import { InjectDataSource, InjectRepository } from '@nestjs/typeorm'; import { Cron, CronExpression } from '@nestjs/schedule'; -import { DataSource, LessThan, Repository } from 'typeorm'; +import { DataSource, In, LessThan, Repository } from 'typeorm'; import * as fs from 'fs/promises'; import * as path from 'path'; import { DataExportJob } from './entities/data-export-job.entity'; +const UUID_PATTERN = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +export interface ExportCleanupResult { + expired: number; + failed: number; + stuck: number; + orphans: number; +} + @Injectable() export class AccountService { constructor( @@ -219,14 +229,131 @@ export class AccountService { } } - @Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT) - async cleanupExports(): Promise { + /** + * Removes stale export artifacts. Invoked on a configurable schedule by + * {@link AccountExportCleanupScheduler}. + * + * - `ready` jobs past `expires_at`: file unlinked, row deleted + * - `failed` jobs older than EXPORT_FAILED_RETENTION_HOURS: file unlinked, row deleted + * - `processing` jobs older than EXPORT_STUCK_PROCESSING_MINUTES: partial file + * unlinked, job marked `failed` (deleted on a later run by the rule above) + * - files in EXPORT_DIR with no matching job row and older than + * EXPORT_ORPHAN_GRACE_MINUTES: unlinked + */ + async cleanupExports(): Promise { + const now = Date.now(); + const result: ExportCleanupResult = { + expired: 0, + failed: 0, + stuck: 0, + orphans: 0, + }; + const expired = await this.jobRepo.find({ - where: { status: 'ready', expires_at: LessThan(new Date()) }, + where: { status: 'ready', expires_at: LessThan(new Date(now)) }, }); for (const job of expired) { - if (job.file_path) await fs.unlink(job.file_path).catch(() => {}); + await this.removeJobFile(job); + await this.jobRepo.delete(job.id); + result.expired++; + } + + const failedRetentionHours = Number( + this.configService.get('EXPORT_FAILED_RETENTION_HOURS', 24), + ); + const failed = await this.jobRepo.find({ + where: { + status: 'failed', + updated_at: LessThan(new Date(now - failedRetentionHours * 3_600_000)), + }, + }); + for (const job of failed) { + await this.removeJobFile(job); await this.jobRepo.delete(job.id); + result.failed++; + } + + const stuckMinutes = Number( + this.configService.get('EXPORT_STUCK_PROCESSING_MINUTES', 60), + ); + const stuck = await this.jobRepo.find({ + where: { + status: 'processing', + updated_at: LessThan(new Date(now - stuckMinutes * 60_000)), + }, + }); + for (const job of stuck) { + await this.removeJobFile(job); + await this.jobRepo.update(job.id, { + status: 'failed', + file_path: null, + expires_at: null, + }); + result.stuck++; + } + + result.orphans = await this.removeOrphanFiles(now); + + return result; + } + + private exportDir(): string { + return this.configService.get('EXPORT_DIR', './exports'); + } + + private async removeJobFile(job: DataExportJob): Promise { + const filePath = + job.file_path ?? path.join(this.exportDir(), `${job.id}.json`); + await fs.unlink(filePath).catch(() => {}); + } + + private async removeOrphanFiles(now: number): Promise { + const dir = this.exportDir(); + let entries: string[]; + try { + entries = await fs.readdir(dir); + } catch { + return 0; + } + + const graceMinutes = Number( + this.configService.get('EXPORT_ORPHAN_GRACE_MINUTES', 60), + ); + const cutoff = now - graceMinutes * 60_000; + + // Only consider files named like export jobs (`.json`) so unrelated + // files that happen to live in EXPORT_DIR are never touched. + const candidates: { jobId: string; filePath: string }[] = []; + for (const name of entries) { + const jobId = path.basename(name, '.json'); + if (!name.endsWith('.json') || !UUID_PATTERN.test(jobId)) continue; + const filePath = path.join(dir, name); + try { + const stat = await fs.stat(filePath); + if (!stat.isFile() || stat.mtimeMs > cutoff) continue; + } catch { + continue; + } + candidates.push({ jobId, filePath }); + } + if (candidates.length === 0) return 0; + + const knownJobs = await this.jobRepo.find({ + where: { id: In(candidates.map((c) => c.jobId)) }, + select: ['id'], + }); + const known = new Set(knownJobs.map((j) => j.id)); + + let removed = 0; + for (const c of candidates) { + if (known.has(c.jobId)) continue; + try { + await fs.unlink(c.filePath); + removed++; + } catch { + // already gone or not removable; skip + } } + return removed; } } diff --git a/backend/src/config/env.validation.ts b/backend/src/config/env.validation.ts index bb221a57c..a2c4da172 100644 --- a/backend/src/config/env.validation.ts +++ b/backend/src/config/env.validation.ts @@ -72,6 +72,22 @@ class EnvironmentVariables { @IsNumber() EXPORT_TTL_HOURS: number = 48; + @IsOptional() + @IsString() + EXPORT_CLEANUP_ENABLED?: string; + + @IsString() + EXPORT_CLEANUP_CRON: string = '0 * * * *'; + + @IsNumber() + EXPORT_FAILED_RETENTION_HOURS: number = 24; + + @IsNumber() + EXPORT_STUCK_PROCESSING_MINUTES: number = 60; + + @IsNumber() + EXPORT_ORPHAN_GRACE_MINUTES: number = 60; + @IsString() LEADERBOARD_SNAPSHOT_CRON: string = '0 * * * *';