From 8e5dda0015c5e73afa3e64147e89268c86fe28dd Mon Sep 17 00:00:00 2001 From: prismn Date: Wed, 23 Sep 2026 16:58:09 +0100 Subject: [PATCH] feat(users,database): optimize CSV import with email pre-fetching and add ExportJob archive cleanup --- src/database/cleanup.service.ts | 36 ++++++++++++++++++++++++++ src/users/user-import.service.ts | 44 ++++++++++---------------------- 2 files changed, 50 insertions(+), 30 deletions(-) diff --git a/src/database/cleanup.service.ts b/src/database/cleanup.service.ts index 94093343..f3cc507c 100644 --- a/src/database/cleanup.service.ts +++ b/src/database/cleanup.service.ts @@ -12,6 +12,7 @@ import { Injectable, Logger } from '@nestjs/common'; import { Cron, CronExpression } from '@nestjs/schedule'; import { PrismaService } from './prisma.service'; +import { promises as fs } from 'fs'; /** Default retention periods (in days) for each record type. */ const DEFAULT_RETENTION = { @@ -85,6 +86,7 @@ export class CleanupService { results.push(await this.cleanOldLoginHistory(now)); results.push(await this.cleanOldSearchAnalytics(now)); results.push(await this.cleanOldSearchHistory(now)); + results.push(await this.cleanExportJobs(now)); const summary: CleanupSummary = { ranAt: now.toISOString(), @@ -314,4 +316,38 @@ export class CleanupService { ); return { entity: 'SearchHistory', deleted, durationMs: Date.now() - start }; } + + private async cleanExportJobs(now: Date): Promise { + const start = Date.now(); + const retentionHours = parseInt( + process.env.CLEANUP_EXPORT_JOB_RETENTION_HOURS ?? '24', + 10, + ); + const cutoff = new Date(now.getTime() - retentionHours * 60 * 60 * 1000); + let deleted = 0; + + const oldJobs = await this.prisma.exportJob.findMany({ + where: { createdAt: { lt: cutoff } }, + select: { id: true, fileUrl: true }, + }); + + for (const job of oldJobs) { + if (job.fileUrl) { + try { + await fs.unlink(job.fileUrl); + await fs.unlink(`${job.fileUrl}.json`).catch(() => {}); + } catch {} + } + } + + if (oldJobs.length > 0) { + const res = await this.prisma.exportJob.deleteMany({ + where: { id: { in: oldJobs.map((j) => j.id) } }, + }); + deleted = res.count; + } + + this.logger.log(`cleanExportJobs: removed ${deleted} record(s) (retention: ${retentionHours}h)`); + return { entity: 'ExportJob', deleted, durationMs: Date.now() - start }; + } } diff --git a/src/users/user-import.service.ts b/src/users/user-import.service.ts index 44964448..bab64424 100644 --- a/src/users/user-import.service.ts +++ b/src/users/user-import.service.ts @@ -7,6 +7,7 @@ import { validatePassword } from '../auth/password.utils'; import { ActivityLogService } from './activity-log.service'; import { UserRole } from '../types/prisma.types'; import { Prisma } from '@prisma/client'; +import { randomBytes } from 'crypto'; interface UserImportRecord { email: string; @@ -61,6 +62,14 @@ export class UserImportService { throw new BadRequestException('CSV file is empty'); } + // Pre-fetch existing emails in a single query to eliminate N+1 roundtrips + const emailList = records.map((r) => r.email).filter(Boolean); + const existingUsers = await this.prisma.user.findMany({ + where: { email: { in: emailList } }, + select: { email: true }, + }); + const existingEmailSet = new Set(existingUsers.map((u) => u.email.toLowerCase())); + const usersToCreate: Prisma.UserCreateInput[] = []; for (let i = 0; i < records.length; i++) { @@ -79,15 +88,11 @@ export class UserImportService { throw new Error(`Invalid email format: ${email}`); } - // #1199 – password policy: validate against the same PASSWORD_* config - // used by registration, collecting every violation for this row. const passwordErrors = validatePassword(password, this.configService); if (passwordErrors.length > 0) { throw new Error(`Password does not meet policy: ${passwordErrors.join('; ')}`); } - // #1198 – role whitelist: reject privileged/unknown roles per row instead - // of silently coercing them. let normalizedRole: UserRole = UserRole.USER; if (role) { normalizedRole = role.toUpperCase() as UserRole; @@ -100,39 +105,20 @@ export class UserImportService { } } - // Check for existing user in database - const existingUser = await this.prisma.user.findUnique({ - where: { email }, - }); - if (existingUser) { + // Check pre-fetched existing users + if (existingEmailSet.has(email.toLowerCase())) { throw new Error('User with this email already exists'); } // Check for duplicate in current CSV - if (usersToCreate.some((u) => u.email === email)) { + if (usersToCreate.some((u) => u.email.toLowerCase() === email.toLowerCase())) { throw new Error('Duplicate email in CSV'); } const hashedPassword = await hashPassword(password); - // Generate unique referral code - let referralCode: string; - let isUnique = false; - let attempts = 0; - - // Basic unique code generation - do { - referralCode = Math.random().toString(36).substring(2, 8).toUpperCase(); - const existingCode = await this.prisma.user.findUnique({ where: { referralCode } }); - if (!existingCode) { - isUnique = true; - } - attempts++; - } while (!isUnique && attempts < 10); - - if (!isUnique) { - throw new Error('Could not generate a unique referral code'); - } + // Generate unpredictable referral code using crypto.randomBytes + const referralCode = `REF-${randomBytes(3).toString('hex').toUpperCase()}`; usersToCreate.push({ email, @@ -171,7 +157,6 @@ export class UserImportService { } } - // #1198 – audit the actor + import summary at the row level. await this.recordImportAudit(actorUser, report); return report; @@ -202,7 +187,6 @@ export class UserImportService { description: `CSV user import completed (${summary})`, }); } catch (error) { - // Audit must never fail the whole import. this.logger.error('Failed to write user-import audit entry', error as Error); } }