From 5ee8927942e61a6215a1f0a79a1b505312a43de4 Mon Sep 17 00:00:00 2001 From: Madjo Diapena Date: Wed, 24 Jun 2026 00:38:38 -0400 Subject: [PATCH 01/11] fix: avoid concurrent query warning on pg clients (#370) * fix: avoid concurrent query warning on pg clients Set search_path during pool.connect so each checked-out client is initialized before use, preventing deprecated concurrent client.query calls triggered by the connect event hook. --- backend/src/common/database/prisma.service.ts | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/backend/src/common/database/prisma.service.ts b/backend/src/common/database/prisma.service.ts index bd7bb840..8d8b776b 100644 --- a/backend/src/common/database/prisma.service.ts +++ b/backend/src/common/database/prisma.service.ts @@ -22,11 +22,7 @@ class PrismaService const pool = new Pool({ connectionString: databaseConfig.url, }) - // Set search_path on each new connection (compatible with Openshift Crunchy DB that - // block startup parameters like `-c search_path=` in the options field) - pool.on('connect', (client) => { - client.query(`SET search_path TO ${databaseConfig.schema}`) - }) + PrismaService.wrapPoolConnectWithSearchPath(pool) const adapter = new PrismaPg(pool) super({ adapter, @@ -60,6 +56,21 @@ class PrismaService return this.pool } + /** + * Ensure every checked-out client has search_path set before first use. + * We wrap pool.connect instead of using pool "connect" event to avoid issuing + * a concurrent query on a client that may already be executing work. + */ + private static wrapPoolConnectWithSearchPath(pool: Pool): void { + const originalConnect = pool.connect.bind(pool) + + pool.connect = async (...args: unknown[]) => { + const client = await originalConnect(...args) + await client.query(`SET search_path TO ${databaseConfig.schema}`) + return client + } + } + async onModuleDestroy() { await this.$disconnect() } From 12a9b60fd00de1f125ad346d13a3bbefa3e7b893 Mon Sep 17 00:00:00 2001 From: saifrazabc Date: Thu, 25 Jun 2026 06:25:47 +0530 Subject: [PATCH 02/11] fix: Weekly File Date displays one day earlier due to timezone conversion (#375) * fix weekly file processing date * reused the date transformation function for transforming dates in the weekly file tab --------- Co-authored-by: Raza Co-authored-by: plakkara --- frontend/src/App.tsx | 67 ++-------------- .../components/WeeklyFileProcessingTab.tsx | 25 +----- frontend/src/utils/date-format.ts | 76 +++++++++++++++++++ 3 files changed, 86 insertions(+), 82 deletions(-) create mode 100644 frontend/src/utils/date-format.ts diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 6bebddf4..8dd21dd4 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -87,6 +87,12 @@ import { type LastSuccessfulRuns, } from './service/contacts-service' import type { AppEnvironment } from './types/runtime-config' +import { + formatDateTimeYMD, + formatDateTimeYMDHMS, + formatDateYMD, + parseFormattedDate, +} from './utils/date-format' import { buildPlacementDisplayValues } from './utils/mock-placement' // Environment-based toolbar background colors @@ -239,67 +245,6 @@ const DATE_FORMAT: Intl.DateTimeFormatOptions = { year: 'numeric', month: 'short const HOLD_REASON_PREVIEW_LENGTH = 150 const HOLD_REASON_COLUMN_WIDTH = 240 -const toYMD = (date: Date, timeZone: string): string => { - const parts = new Intl.DateTimeFormat('en-US', { ...DATE_FORMAT, timeZone }).formatToParts(date) - const get = (type: string) => parts.find((p) => p.type === type)?.value || '' - return `${get('year')}-${get('month')}-${get('day')}` -} - -const formatDateYMD = (dateString: string): string => { - return toYMD(new Date(dateString + 'T00:00:00Z'), 'UTC') -} - -const formatDateTimeYMD = (dateString: string): string => { - return toYMD(new Date(dateString), 'America/Vancouver') -} - -const formatDateTimeYMDHMS = (dateString: string): string => { - const date = new Date(dateString) - const parts = new Intl.DateTimeFormat('en-US', { - ...DATE_FORMAT, - timeZone: 'America/Vancouver', - hour: '2-digit', - minute: '2-digit', - second: '2-digit', - hour12: false, - }).formatToParts(date) - const get = (type: string) => parts.find((p) => p.type === type)?.value || '' - return `${get('year')}-${get('month')}-${get('day')} ${get('hour')}:${get('minute')}:${get('second')}` -} - -// Parse formatted date string (YYYY-MMM-DD or YYYY-MMM-DD HH:MM:SS) back to Date for sorting -const parseFormattedDate = (dateStr: string): Date | null => { - if (!dateStr) return null - const months: Record = { - Jan: 0, - Feb: 1, - Mar: 2, - Apr: 3, - May: 4, - Jun: 5, - Jul: 6, - Aug: 7, - Sep: 8, - Oct: 9, - Nov: 10, - Dec: 11, - } - // Handle both "YYYY-MMM-DD" and "YYYY-MMM-DD HH:MM:SS" formats - const match = dateStr.match(/^(\d{4})-(\w{3})-(\d{2})(?:\s+(\d{2}):(\d{2}):(\d{2}))?$/) - if (!match) return null - const [, year, month, day, hour = '0', minute = '0', second = '0'] = match - const monthNum = months[month] - if (monthNum === undefined) return null - return new Date( - parseInt(year), - monthNum, - parseInt(day), - parseInt(hour), - parseInt(minute), - parseInt(second), - ) -} - // System comments are prepended with newest first, one entry per line. // Batch Requests should display only the latest entry. const latestSystemComment = (comments: string | null | undefined): string => { diff --git a/frontend/src/components/WeeklyFileProcessingTab.tsx b/frontend/src/components/WeeklyFileProcessingTab.tsx index c411352b..fdfd443e 100644 --- a/frontend/src/components/WeeklyFileProcessingTab.tsx +++ b/frontend/src/components/WeeklyFileProcessingTab.tsx @@ -36,6 +36,7 @@ import { type WeeklyFileRecord, type WeeklyFileSummary, } from '../service/weekly-files-service' +import { formatDateTimeYMDHMS, formatDateYMD } from '../utils/date-format' const SUMMARY_PAGE_SIZE = 10 const DETAILS_PAGE_SIZE = 10 @@ -115,28 +116,10 @@ type SortConfig = { direction: SortDirection } | null -const formatDateDisplay = (value: string | null): string => { - if (!value) return '' - const parsed = new Date(value) - if (Number.isNaN(parsed.getTime())) return value +const formatDateDisplay = (value: string | null): string => (value ? formatDateYMD(value) : '') - const month = parsed.toLocaleString('en-US', { month: 'short' }) - const day = String(parsed.getDate()).padStart(2, '0') - return `${parsed.getFullYear()}-${month}-${day}` -} - -const formatDateTimeDisplay = (value: string | null): string => { - if (!value) return '' - const parsed = new Date(value) - if (Number.isNaN(parsed.getTime())) return value - - const month = parsed.toLocaleString('en-US', { month: 'short' }) - const day = String(parsed.getDate()).padStart(2, '0') - const hours = String(parsed.getHours()).padStart(2, '0') - const minutes = String(parsed.getMinutes()).padStart(2, '0') - const seconds = String(parsed.getSeconds()).padStart(2, '0') - return `${parsed.getFullYear()}-${month}-${day} ${hours}:${minutes}:${seconds}` -} +const formatDateTimeDisplay = (value: string | null): string => + value ? formatDateTimeYMDHMS(value) : '' const valueOrBlank = (value: string | null | undefined): string => value ?? '' diff --git a/frontend/src/utils/date-format.ts b/frontend/src/utils/date-format.ts new file mode 100644 index 00000000..4e41ffe9 --- /dev/null +++ b/frontend/src/utils/date-format.ts @@ -0,0 +1,76 @@ +const DATE_FORMAT: Intl.DateTimeFormatOptions = { + year: 'numeric', + month: 'short', + day: '2-digit', +} + +const toYMD = (date: Date, timeZone: string): string => { + const parts = new Intl.DateTimeFormat('en-US', { ...DATE_FORMAT, timeZone }).formatToParts(date) + const get = (type: string) => parts.find((p) => p.type === type)?.value || '' + return `${get('year')}-${get('month')}-${get('day')}` +} + +export const formatDateYMD = (dateString: string): string => { + const date = new Date(`${dateString}T00:00:00Z`) + if (Number.isNaN(date.getTime())) return dateString + return toYMD(date, 'UTC') +} + +export const formatDateTimeYMD = (dateString: string): string => { + const date = new Date(dateString) + if (Number.isNaN(date.getTime())) return dateString + return toYMD(date, 'America/Vancouver') +} + +export const formatDateTimeYMDHMS = (dateString: string): string => { + const date = new Date(dateString) + if (Number.isNaN(date.getTime())) return dateString + + const parts = new Intl.DateTimeFormat('en-US', { + ...DATE_FORMAT, + timeZone: 'America/Vancouver', + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + hour12: false, + }).formatToParts(date) + + const get = (type: string) => parts.find((p) => p.type === type)?.value || '' + return `${get('year')}-${get('month')}-${get('day')} ${get('hour')}:${get('minute')}:${get('second')}` +} + +// Parse formatted date string (YYYY-MMM-DD or YYYY-MMM-DD HH:MM:SS) back to Date for sorting. +export const parseFormattedDate = (dateStr: string): Date | null => { + if (!dateStr) return null + + const months: Record = { + Jan: 0, + Feb: 1, + Mar: 2, + Apr: 3, + May: 4, + Jun: 5, + Jul: 6, + Aug: 7, + Sep: 8, + Oct: 9, + Nov: 10, + Dec: 11, + } + + const match = dateStr.match(/^(\d{4})-(\w{3})-(\d{2})(?:\s+(\d{2}):(\d{2}):(\d{2}))?$/) + if (!match) return null + + const [, year, month, day, hour = '0', minute = '0', second = '0'] = match + const monthNum = months[month] + if (monthNum === undefined) return null + + return new Date( + parseInt(year), + monthNum, + parseInt(day), + parseInt(hour), + parseInt(minute), + parseInt(second), + ) +} From 86de28711041e47974551a680f55fd94ad0b25bc Mon Sep 17 00:00:00 2001 From: plakkara-bc Date: Wed, 24 Jun 2026 21:33:53 -0400 Subject: [PATCH 03/11] fix: created apis for search and filter of the weekly file - details table (#383) * added api filter calls for the details table in weekly file processing tab * added api filter calls for the details table in weekly file processing tab * added api filter calls for the details table in weekly file processing tab * added api filter calls for the details table in weekly file processing tab * added api filter calls for the details table in weekly file processing tab * added api filter calls for the details table in weekly file processing tab * added api filter calls for the details table in weekly file processing tab * removed search boxes from weekly file and details tables * removed column filter list for a few columns in details table * removed column filter list for a few columns in details table * removed column filter list for a few columns in details table * removed column filter list for a few columns in details table * removed column filter list for a few columns in details table * removed column filter list for a few columns in details table * disabled filters on certai columns * disabled filters on certai columns * disabled filters on certai columns * disabled filters on certai columns * created columns for filtering * updated the cra status filter list * created migrations for the 3 new columns * fixed failing migrations * removed transformation from migrations * removed transformation from migrations * added migrations for one time backfill * fix: align CRA status filter with in-progress display format Normalize stored hyphenated statuses to "IN PROGRESS" for filtering and update the details dropdown label to match the table display. * fix: whitelist CRA status filters from WEEKLY_FILE.STATUS Derive display labels and filter-to-stored mapping from the canonical status constants so filtering matches persisted values without SQL normalization. * fix lint * fix: use stored filter values and Prisma where for weekly file records Send database values from filter dropdowns while keeping display labels in the UI, and replace raw SQL string building with a typed Prisma where builder. * fixed db error --------- Co-authored-by: Madjo Diapena --- backend/prisma/schema.prisma | 6 + .../weekly-file-record-filters.spec.ts | 33 ++ .../weekly-file-record-filters.ts | 104 ++++ .../weekly-files/weekly-file.mapper.spec.ts | 22 + .../api/weekly-files/weekly-file.mapper.ts | 25 +- .../weekly-files/weekly-files.controller.ts | 73 ++- .../api/weekly-files/weekly-files.service.ts | 27 +- backend/src/common/database/prisma.service.ts | 25 +- .../cra/inbound/wkl-file-record.service.ts | 32 ++ .../components/WeeklyFileProcessingTab.tsx | 474 ++++++++---------- frontend/src/service/weekly-files-service.ts | 53 +- .../sql/V22__add_wkl_filter_columns.sql | 14 + .../sql/V23__backfill_wkl_filter_columns.sql | 31 ++ 13 files changed, 652 insertions(+), 267 deletions(-) create mode 100644 backend/src/api/weekly-files/weekly-file-record-filters.spec.ts create mode 100644 backend/src/api/weekly-files/weekly-file-record-filters.ts create mode 100644 migrations/sql/V22__add_wkl_filter_columns.sql create mode 100644 migrations/sql/V23__backfill_wkl_filter_columns.sql diff --git a/backend/prisma/schema.prisma b/backend/prisma/schema.prisma index 7c76f216..ccbaa319 100644 --- a/backend/prisma/schema.prisma +++ b/backend/prisma/schema.prisma @@ -185,6 +185,9 @@ model WklFileRecord { recordIndex Int @map("record_index") weeklyFileDate DateTime? @map("weekly_file_date") @db.Date recordData Json @map("record_data") @db.JsonB + transactionType String? @map("transaction_type") + craStatus String? @map("cra_status") + transactionSource String? @map("transaction_source") matchStatus String @map("match_status") contactId Int? @map("contact_id") batchDetailId Int? @map("batch_detail_id") @@ -200,6 +203,9 @@ model WklFileRecord { @@index([transferFileId]) @@index([matchStatus]) @@index([contactId]) + @@index([transactionType]) + @@index([craStatus]) + @@index([transactionSource]) @@map("wkl_file_records") @@schema("csa") } diff --git a/backend/src/api/weekly-files/weekly-file-record-filters.spec.ts b/backend/src/api/weekly-files/weekly-file-record-filters.spec.ts new file mode 100644 index 00000000..7e8ed481 --- /dev/null +++ b/backend/src/api/weekly-files/weekly-file-record-filters.spec.ts @@ -0,0 +1,33 @@ +import { CRA_DATA_HANDLING_CONSTANT } from 'src/cra/cra.constant' +import { describe, expect, it } from 'vitest' +import { + buildTransactionSourceWhere, + resolveCsaMatchFoundStatuses, +} from './weekly-file-record-filters' + +const { WKL_MATCH_STATUS } = CRA_DATA_HANDLING_CONSTANT + +describe('weekly-file-record-filters', () => { + it('maps CSA Match Found filter values to match_status groups', () => { + expect(resolveCsaMatchFoundStatuses(['Yes'])).toEqual([WKL_MATCH_STATUS.MATCHED]) + expect(resolveCsaMatchFoundStatuses(['No'])).toEqual([ + WKL_MATCH_STATUS.UNMATCHED, + WKL_MATCH_STATUS.ASSOCIATED, + ]) + expect(resolveCsaMatchFoundStatuses(['Yes', 'No'])).toEqual([ + WKL_MATCH_STATUS.MATCHED, + WKL_MATCH_STATUS.UNMATCHED, + WKL_MATCH_STATUS.ASSOCIATED, + ]) + }) + + it('maps transaction source search terms to stored column predicates', () => { + expect(buildTransactionSourceWhere('elec')).toEqual({ + OR: [{ transactionSource: 'E' }], + }) + expect(buildTransactionSourceWhere('oth')).toEqual({ + OR: [{ OR: [{ transactionSource: '' }, { transactionSource: null }] }], + }) + expect(buildTransactionSourceWhere('ab')).toBeNull() + }) +}) diff --git a/backend/src/api/weekly-files/weekly-file-record-filters.ts b/backend/src/api/weekly-files/weekly-file-record-filters.ts new file mode 100644 index 00000000..10a24703 --- /dev/null +++ b/backend/src/api/weekly-files/weekly-file-record-filters.ts @@ -0,0 +1,104 @@ +import { Prisma } from '@prisma/client' +import { CRA_DATA_HANDLING_CONSTANT } from 'src/cra/cra.constant' +import type { PrismaService } from 'src/common/database/prisma.service' +import { filterAllowedCraStatuses, filterAllowedTransactionTypes } from './weekly-file.mapper' +import type { WeeklyFileRecordFilters } from './weekly-files.service' + +const { WKL_MATCH_STATUS } = CRA_DATA_HANDLING_CONSTANT + +export function resolveCsaMatchFoundStatuses(values: string[]): string[] { + const matchStatuses: string[] = [] + for (const val of values) { + if (val === 'Yes') matchStatuses.push(WKL_MATCH_STATUS.MATCHED) + if (val === 'No') { + matchStatuses.push(WKL_MATCH_STATUS.UNMATCHED, WKL_MATCH_STATUS.ASSOCIATED) + } + } + return [...new Set(matchStatuses)] +} + +/** Maps a search term to stored transaction_source values (display: electronic / other). */ +export function buildTransactionSourceWhere( + term: string | undefined, +): Prisma.WklFileRecordWhereInput | null { + const normalized = term?.trim().toLowerCase() ?? '' + if (normalized.length < 3) return null + + const orConditions: Prisma.WklFileRecordWhereInput[] = [] + if ('electronic'.includes(normalized)) { + orConditions.push({ transactionSource: 'E' }) + } + if ('other'.includes(normalized)) { + orConditions.push({ OR: [{ transactionSource: '' }, { transactionSource: null }] }) + } + if (!orConditions.length) { + orConditions.push({ transactionSource: { contains: normalized, mode: 'insensitive' } }) + } + return { OR: orConditions } +} + +async function findBatchDetailIdsByBatchNumberSubstring( + prisma: PrismaService, + term: string, +): Promise { + const pattern = `%${term}%` + const rows = await prisma.$queryRaw<{ id: number }[]>( + Prisma.sql` + SELECT cbd.id + FROM csa.contact_batch_details cbd + INNER JOIN csa.batches b ON b.id = cbd.batch_id + WHERE LOWER(CAST(b.batch_number AS TEXT)) LIKE ${pattern} + `, + ) + return rows.map((row) => row.id) +} + +export async function buildWklRecordWhereInput( + prisma: PrismaService, + transferFileId: number, + filters?: WeeklyFileRecordFilters, +): Promise { + const andConditions: Prisma.WklFileRecordWhereInput[] = [] + + if (filters?.csaMatchFound?.length) { + const matchStatuses = resolveCsaMatchFoundStatuses(filters.csaMatchFound) + if (matchStatuses.length) { + andConditions.push({ matchStatus: { in: matchStatuses } }) + } + } + + const transactionTypes = filterAllowedTransactionTypes(filters?.transactionType ?? []) + if (transactionTypes.length) { + andConditions.push({ transactionType: { in: transactionTypes } }) + } + + const craStatuses = filterAllowedCraStatuses(filters?.craStatus ?? []) + if (craStatuses.length) { + andConditions.push({ craStatus: { in: craStatuses } }) + } + + if (filters?.matchedBy?.trim()) { + const term = filters.matchedBy.trim() + if (term.length >= 3) { + andConditions.push({ matchedBy: { contains: term, mode: 'insensitive' } }) + } + } + + const transactionSourceWhere = buildTransactionSourceWhere(filters?.transactionSource) + if (transactionSourceWhere) { + andConditions.push(transactionSourceWhere) + } + + if (filters?.batchNumber?.trim()) { + const term = filters.batchNumber.trim().toLowerCase() + if (term.length >= 1) { + const batchDetailIds = await findBatchDetailIdsByBatchNumberSubstring(prisma, term) + andConditions.push({ batchDetailId: { in: batchDetailIds } }) + } + } + + return { + transferFileId, + ...(andConditions.length ? { AND: andConditions } : {}), + } +} diff --git a/backend/src/api/weekly-files/weekly-file.mapper.spec.ts b/backend/src/api/weekly-files/weekly-file.mapper.spec.ts index a11be863..76bcece9 100644 --- a/backend/src/api/weekly-files/weekly-file.mapper.spec.ts +++ b/backend/src/api/weekly-files/weekly-file.mapper.spec.ts @@ -2,6 +2,9 @@ import { CRA_DATA_HANDLING_CONSTANT } from 'src/cra/cra.constant' import { describe, expect, it } from 'vitest' import { aggregateWeeklyFileCounts, + filterAllowedCraStatuses, + filterAllowedTransactionTypes, + toCraStatusDisplayLabel, toCsaMatchFound, toWeeklyFileRecordDto, } from './weekly-file.mapper' @@ -130,4 +133,23 @@ describe('weekly-file.mapper', () => { expect(dto.gender).toBe('Unknown') expect(dto.birthCountry).toBe('Outside Canada') }) + + it('derives CRA status display labels from stored file values', () => { + expect(toCraStatusDisplayLabel('in-progress')).toBe('IN PROGRESS') + expect(toCraStatusDisplayLabel('completed')).toBe('COMPLETED') + }) + + it('whitelists stored transaction type filter values', () => { + expect(filterAllowedTransactionTypes(['A', 'c'])).toEqual(['A', 'C']) + expect(filterAllowedTransactionTypes(['Application', 'INVALID'])).toEqual([]) + }) + + it('whitelists stored CRA status filter values', () => { + expect(filterAllowedCraStatuses(['in-progress'])).toEqual(['in-progress']) + expect(filterAllowedCraStatuses(['completed', 'IN-PROGRESS'])).toEqual([ + 'completed', + 'in-progress', + ]) + expect(filterAllowedCraStatuses(['INVALID', 'completed'])).toEqual(['completed']) + }) }) diff --git a/backend/src/api/weekly-files/weekly-file.mapper.ts b/backend/src/api/weekly-files/weekly-file.mapper.ts index e026777a..4b5d718e 100644 --- a/backend/src/api/weekly-files/weekly-file.mapper.ts +++ b/backend/src/api/weekly-files/weekly-file.mapper.ts @@ -4,7 +4,9 @@ import type { DetailRecord04 } from 'src/cra/inbound/inbound-weekly.interface' import type { WeeklyFileRecordDto, WeeklyFileSummaryDto } from './dto/weekly-file.dto' const { WKL_MATCH_STATUS, WEEKLY_FILE } = CRA_DATA_HANDLING_CONSTANT -const { RECEIVE_MODE } = WEEKLY_FILE +const { RECEIVE_MODE, STATUS: WKL_STATUS } = WEEKLY_FILE + +const WKL_STATUS_STORED_VALUES = Object.values(WKL_STATUS) const TRANSACTION_TYPE_LABELS: Record = { A: 'Application', @@ -28,6 +30,25 @@ const BIRTH_COUNTRY_LABELS: Record = { EX: 'Outside Canada', } +/** Display label for a stored CRA status (e.g. "in-progress" → "IN PROGRESS"). */ +export function toCraStatusDisplayLabel(stored: string): string { + return stored.trim().toUpperCase().replace(/-/g, ' ') +} + +const TRANSACTION_TYPE_CODES = Object.keys(TRANSACTION_TYPE_LABELS) + +/** Whitelist filter params to values stored in transaction_type (A, C, U). */ +export function filterAllowedTransactionTypes(values: string[]): string[] { + const allowed = new Set(TRANSACTION_TYPE_CODES) + return [...new Set(values.map((v) => v.trim().toUpperCase()).filter((v) => allowed.has(v)))] +} + +/** Whitelist filter params to values stored in cra_status. */ +export function filterAllowedCraStatuses(values: string[]): string[] { + const allowed = new Set(WKL_STATUS_STORED_VALUES) + return [...new Set(values.map((v) => v.trim().toLowerCase()).filter((v) => allowed.has(v)))] +} + export interface WeeklyFileCounts { totalCount: number eCount: number @@ -167,7 +188,7 @@ function formatWklDateString(value: string | undefined): string | null { function formatCraStatus(status: string | undefined): string { if (!status?.trim()) return '' - return status.trim().toUpperCase().replace(/-/g, ' ') + return toCraStatusDisplayLabel(status) } function formatTransactionType(value: string | undefined): string { diff --git a/backend/src/api/weekly-files/weekly-files.controller.ts b/backend/src/api/weekly-files/weekly-files.controller.ts index 1bda1640..270a5762 100644 --- a/backend/src/api/weekly-files/weekly-files.controller.ts +++ b/backend/src/api/weekly-files/weekly-files.controller.ts @@ -47,14 +47,85 @@ export class WeeklyFilesController { @Get(':id/records') @ApiQuery({ name: 'page', required: false, type: Number }) @ApiQuery({ name: 'limit', required: false, type: Number }) + @ApiQuery({ + name: 'csaMatchFound', + required: false, + type: String, + description: 'Comma-separated filter values for CSA Match Found: "Yes" and/or "No"', + }) + @ApiQuery({ + name: 'transactionType', + required: false, + type: String, + description: 'Comma-separated stored transaction type codes: "A", "C", "U"', + }) + @ApiQuery({ + name: 'craStatus', + required: false, + type: String, + description: + 'Comma-separated stored CRA status values: "completed", "abandoned", "in-progress", "updated"', + }) + @ApiQuery({ + name: 'matchedBy', + required: false, + type: String, + description: 'Text filter for Matched By (minimum 3 characters)', + }) + @ApiQuery({ + name: 'batchNumber', + required: false, + type: String, + description: 'Text filter for Batch Req ID / batch number (minimum 3 characters)', + }) + @ApiQuery({ + name: 'transactionSource', + required: false, + type: String, + description: 'Text filter for Transaction Source (minimum 3 characters)', + }) @ApiResponse({ status: 200, description: 'Paginated detail records for a weekly file' }) @ApiResponse({ status: 404, description: 'Weekly file not found' }) findRecords( @Param('id', ParseIntPipe) id: number, @Query('page') page?: string, @Query('limit') limit?: string, + @Query('csaMatchFound') csaMatchFound?: string, + @Query('transactionType') transactionType?: string, + @Query('craStatus') craStatus?: string, + @Query('matchedBy') matchedBy?: string, + @Query('batchNumber') batchNumber?: string, + @Query('transactionSource') transactionSource?: string, ): Promise> { - return this.weeklyFilesService.findRecords(id, this.parsePage(page), this.parseLimit(limit)) + const filters = { + csaMatchFound: csaMatchFound + ? csaMatchFound + .split(',') + .map((v) => v.trim()) + .filter(Boolean) + : undefined, + transactionType: transactionType + ? transactionType + .split(',') + .map((v) => v.trim()) + .filter(Boolean) + : undefined, + craStatus: craStatus + ? craStatus + .split(',') + .map((v) => v.trim()) + .filter(Boolean) + : undefined, + matchedBy: matchedBy?.trim() || undefined, + batchNumber: batchNumber?.trim() || undefined, + transactionSource: transactionSource?.trim() || undefined, + } + return this.weeklyFilesService.findRecords( + id, + this.parsePage(page), + this.parseLimit(limit), + filters, + ) } @Post(':id/records/:recordId/associate') diff --git a/backend/src/api/weekly-files/weekly-files.service.ts b/backend/src/api/weekly-files/weekly-files.service.ts index 663ed1cc..d51e064d 100644 --- a/backend/src/api/weekly-files/weekly-files.service.ts +++ b/backend/src/api/weekly-files/weekly-files.service.ts @@ -15,6 +15,7 @@ import { toWeeklyFileRecordDto, toWeeklyFileSummaryDto, } from './weekly-file.mapper' +import { buildWklRecordWhereInput } from './weekly-file-record-filters' import { assertCanAssociate, assertCanDissociate, @@ -23,6 +24,18 @@ import { const { FILE_DIRECTION, FILE_TYPE, WKL_MATCH_STATUS } = CRA_DATA_HANDLING_CONSTANT +export interface WeeklyFileRecordFilters { + /** Semantic filter: "Yes" or "No" (maps to match_status groups). */ + csaMatchFound?: string[] + /** Stored transaction_type codes: A, C, U. */ + transactionType?: string[] + /** Stored cra_status values: completed, in-progress, abandoned, updated. */ + craStatus?: string[] + matchedBy?: string + batchNumber?: string + transactionSource?: string +} + const weeklyFileWhere = { fileType: FILE_TYPE.WKL, direction: FILE_DIRECTION.INBOUND, @@ -141,25 +154,29 @@ export class WeeklyFilesService { id: number, page = 1, limit = 10, + filters?: WeeklyFileRecordFilters, ): Promise> { await this.assertWeeklyFileExists(id) const safePage = page >= 1 ? page : 1 const safeLimit = limit >= 1 ? Math.min(limit, 200) : 10 + const offset = (safePage - 1) * safeLimit - const [total, records] = await Promise.all([ - this.prisma.wklFileRecord.count({ where: { transferFileId: id } }), + const where = await buildWklRecordWhereInput(this.prisma, id, filters) + + const [total, recordsWithRelations] = await Promise.all([ + this.prisma.wklFileRecord.count({ where }), this.prisma.wklFileRecord.findMany({ - where: { transferFileId: id }, + where, orderBy: { recordIndex: 'asc' }, - skip: (safePage - 1) * safeLimit, + skip: offset, take: safeLimit, include: wklRecordDtoInclude, }), ]) return { - data: records.map(toWeeklyFileRecordDto), + data: recordsWithRelations.map(toWeeklyFileRecordDto), page: safePage, limit: safeLimit, total, diff --git a/backend/src/common/database/prisma.service.ts b/backend/src/common/database/prisma.service.ts index 8d8b776b..6c4b78d8 100644 --- a/backend/src/common/database/prisma.service.ts +++ b/backend/src/common/database/prisma.service.ts @@ -60,14 +60,29 @@ class PrismaService * Ensure every checked-out client has search_path set before first use. * We wrap pool.connect instead of using pool "connect" event to avoid issuing * a concurrent query on a client that may already be executing work. + * + * pool.connect has two call signatures: + * Promise form: pool.connect() → Promise + * Callback form: pool.connect(cb) → void + * + * The callback form returns undefined, so awaiting it yields undefined. + * We must detect which form is used and handle each accordingly. + * PrismaPg always uses the Promise form, so search_path is always set for + * Prisma-managed connections. */ private static wrapPoolConnectWithSearchPath(pool: Pool): void { - const originalConnect = pool.connect.bind(pool) + const originalConnect = pool.connect.bind(pool) as (...args: unknown[]) => unknown - pool.connect = async (...args: unknown[]) => { - const client = await originalConnect(...args) - await client.query(`SET search_path TO ${databaseConfig.schema}`) - return client + ;(pool as any).connect = function (...args: unknown[]) { + if (args.length > 0 && typeof args[0] === 'function') { + // Callback form: delegate unchanged so the caller's callback fires normally. + return originalConnect(...args) + } + // Promise form: intercept to set search_path before returning the client. + return (originalConnect() as Promise).then(async (client) => { + await client.query(`SET search_path TO ${databaseConfig.schema}`) + return client + }) } } diff --git a/backend/src/cra/inbound/wkl-file-record.service.ts b/backend/src/cra/inbound/wkl-file-record.service.ts index 25248121..280a5025 100644 --- a/backend/src/cra/inbound/wkl-file-record.service.ts +++ b/backend/src/cra/inbound/wkl-file-record.service.ts @@ -21,6 +21,27 @@ export interface PersistWklRecordParams { export class WklFileRecordService { constructor(private readonly prisma: PrismaService) {} + private extractTransactionType(recordData: DetailRecord04): string | null { + if (recordData.transactionType === undefined || recordData.transactionType === null) { + return null + } + return recordData.transactionType.trim() + } + + private extractCraStatus(recordData: DetailRecord04): string | null { + if (recordData.status === undefined || recordData.status === null) { + return null + } + return recordData.status.trim() + } + + private extractTransactionSource(recordData: DetailRecord04): string | null { + if (recordData.receiveMode === undefined || recordData.receiveMode === null) { + return null + } + return recordData.receiveMode.trim() + } + async persistRecord(params: PersistWklRecordParams): Promise { const { transferFileId, @@ -34,6 +55,11 @@ export class WklFileRecordService { processedAt, } = params + // Persist raw file values; display/query transformations happen in the API layer. + const transactionType = this.extractTransactionType(recordData) + const craStatus = this.extractCraStatus(recordData) + const transactionSource = this.extractTransactionSource(recordData) + await this.prisma.wklFileRecord.upsert({ where: { wkl_file_record_unique: { @@ -46,6 +72,9 @@ export class WklFileRecordService { recordIndex, weeklyFileDate, recordData: recordData as unknown as Prisma.InputJsonValue, + transactionType, + craStatus, + transactionSource, matchStatus, contactId, batchDetailId, @@ -55,6 +84,9 @@ export class WklFileRecordService { update: { weeklyFileDate, recordData: recordData as unknown as Prisma.InputJsonValue, + transactionType, + craStatus, + transactionSource, matchStatus, contactId, batchDetailId, diff --git a/frontend/src/components/WeeklyFileProcessingTab.tsx b/frontend/src/components/WeeklyFileProcessingTab.tsx index fdfd443e..41cdd38a 100644 --- a/frontend/src/components/WeeklyFileProcessingTab.tsx +++ b/frontend/src/components/WeeklyFileProcessingTab.tsx @@ -1,6 +1,5 @@ import ArrowDownwardIcon from '@mui/icons-material/ArrowDownward' import ArrowUpwardIcon from '@mui/icons-material/ArrowUpward' -import CloseIcon from '@mui/icons-material/Close' import FilterAltOffIcon from '@mui/icons-material/FilterAltOff' import FilterListIcon from '@mui/icons-material/FilterList' import { @@ -33,6 +32,9 @@ import { getWeeklyFileRecords, getWeeklyFiles, reprocessWeeklyFileRecord, + WEEKLY_FILE_CRA_STATUS_FILTER_OPTIONS, + WEEKLY_FILE_CSA_MATCH_FOUND_FILTER_OPTIONS, + WEEKLY_FILE_TRANSACTION_TYPE_FILTER_OPTIONS, type WeeklyFileRecord, type WeeklyFileSummary, } from '../service/weekly-files-service' @@ -42,6 +44,7 @@ const SUMMARY_PAGE_SIZE = 10 const DETAILS_PAGE_SIZE = 10 const SEARCH_PAGE_SIZE = 10 const CHILD_SEARCH_MIN_LENGTH = 3 +const DETAILS_TEXT_FILTER_MIN_LENGTH = 3 const MANUAL_REVIEW_WARNING = 'This weekly response record is not matched to a CSA master contact. Search and select a child record below to associate manually.' const ASSOCIATED_RECORD_INFO = 'Contact associated, click Confirm to reprocess this record.' @@ -55,6 +58,7 @@ type WeeklyDetailsColumn = | 'transactionSource' | 'craStatus' | 'matchedBy' +type DetailsTextFilterColumn = 'matchedBy' | 'batchNumber' | 'transactionSource' type ChildSearchColumn = | 'din' | 'firstName' @@ -79,6 +83,11 @@ const WEEKLY_DETAILS_COLUMNS: WeeklyDetailsColumn[] = [ 'transactionSource', 'craStatus', ] +const DETAILS_TEXT_FILTER_COLUMNS: ReadonlySet = new Set([ + 'matchedBy', + 'batchNumber', + 'transactionSource', +]) const CHILD_SEARCH_COLUMNS: ChildSearchColumn[] = [ 'din', 'firstName', @@ -123,6 +132,8 @@ const formatDateTimeDisplay = (value: string | null): string => const valueOrBlank = (value: string | null | undefined): string => value ?? '' +type DetailsFilterOption = { value: string; label: string } + const compareStrings = (left: string, right: string): number => left.localeCompare(right, undefined, { numeric: true, sensitivity: 'base' }) @@ -211,14 +222,6 @@ export default function WeeklyFileProcessingTab() { const [savingAssociation, setSavingAssociation] = useState(false) const [reprocessing, setReprocessing] = useState(false) - const [weeklyReportSearchTerm, setWeeklyReportSearchTerm] = useState('') - const [weeklyReportColumnFilters, setWeeklyReportColumnFilters] = useState< - Record - >({ - weeklyFileDate: [], - csaProcessingDate: [], - }) - const [weeklyReportFilterSearchTerm, setWeeklyReportFilterSearchTerm] = useState('') const [weeklyReportSortConfig, setWeeklyReportSortConfig] = useState>(null) const [weeklyReportSortAnchor, setWeeklyReportSortAnchor] = useState<{ @@ -228,15 +231,7 @@ export default function WeeklyFileProcessingTab() { element: null, column: 'weeklyFileDate', }) - const [weeklyReportFilterAnchor, setWeeklyReportFilterAnchor] = useState<{ - element: HTMLElement | null - column: WeeklyReportColumn - }>({ - element: null, - column: 'weeklyFileDate', - }) - const [detailsSearchTerm, setDetailsSearchTerm] = useState('') const [detailsColumnFilters, setDetailsColumnFilters] = useState< Record >({ @@ -247,6 +242,45 @@ export default function WeeklyFileProcessingTab() { craStatus: [], matchedBy: [], }) + const [detailsTextColumnFilters, setDetailsTextColumnFilters] = useState< + Record + >({ + matchedBy: '', + batchNumber: '', + transactionSource: '', + }) + const getDetailsTextFilterMinLength = useCallback( + (column: DetailsTextFilterColumn): number => + column === 'batchNumber' ? 1 : DETAILS_TEXT_FILTER_MIN_LENGTH, + [], + ) + + const getBackendTextFilterValue = useCallback( + (column: DetailsTextFilterColumn, value: string): string | undefined => { + const trimmed = value.trim() + if (trimmed.length >= getDetailsTextFilterMinLength(column)) { + return trimmed + } + return undefined + }, + [getDetailsTextFilterMinLength], + ) + const detailsBackendTextFilters = useMemo( + () => ({ + matchedBy: getBackendTextFilterValue('matchedBy', detailsTextColumnFilters.matchedBy), + batchNumber: getBackendTextFilterValue('batchNumber', detailsTextColumnFilters.batchNumber), + transactionSource: getBackendTextFilterValue( + 'transactionSource', + detailsTextColumnFilters.transactionSource, + ), + }), + [ + getBackendTextFilterValue, + detailsTextColumnFilters.matchedBy, + detailsTextColumnFilters.batchNumber, + detailsTextColumnFilters.transactionSource, + ], + ) const [detailsFilterSearchTerm, setDetailsFilterSearchTerm] = useState('') const [detailsSortConfig, setDetailsSortConfig] = useState>(null) const [detailsSortAnchor, setDetailsSortAnchor] = useState<{ @@ -385,6 +419,14 @@ export default function WeeklyFileProcessingTab() { recordsPage, DETAILS_PAGE_SIZE, abortController.signal, + { + csaMatchFound: detailsColumnFilters.csaMatchFound, + transactionType: detailsColumnFilters.transactionType, + craStatus: detailsColumnFilters.craStatus, + matchedBy: detailsBackendTextFilters.matchedBy, + batchNumber: detailsBackendTextFilters.batchNumber, + transactionSource: detailsBackendTextFilters.transactionSource, + }, ) setRecords(response.data) setRecordsTotalPages(Math.max(response.totalPages, 1)) @@ -408,7 +450,16 @@ export default function WeeklyFileProcessingTab() { return () => { abortController.abort() } - }, [selectedFileId, recordsPage]) + }, [ + selectedFileId, + recordsPage, + detailsColumnFilters.csaMatchFound, + detailsColumnFilters.transactionType, + detailsColumnFilters.craStatus, + detailsBackendTextFilters.matchedBy, + detailsBackendTextFilters.batchNumber, + detailsBackendTextFilters.transactionSource, + ]) useEffect(() => { childSearchRequestIdRef.current += 1 @@ -444,6 +495,10 @@ export default function WeeklyFileProcessingTab() { !selectedRecord?.processedAt && hasAssociatedPendingRecords + const toggleSelectedRecord = (recordId: number) => { + setSelectedRecordId((prev) => (prev === recordId ? null : recordId)) + } + const getWeeklyReportFieldValue = ( file: WeeklyFileSummary, column: WeeklyReportColumn, @@ -520,19 +575,30 @@ export default function WeeklyFileProcessingTab() { weeklyFiles, WEEKLY_REPORT_COLUMNS, getWeeklyReportFieldValue, - weeklyReportSearchTerm, - weeklyReportColumnFilters, + '', + { weeklyFileDate: [], csaProcessingDate: [] }, weeklyReportSortConfig, ) - }, [weeklyFiles, weeklyReportSearchTerm, weeklyReportColumnFilters, weeklyReportSortConfig]) + }, [weeklyFiles, weeklyReportSortConfig]) const filteredRecords = useMemo(() => { + // csaMatchFound, transactionType, and craStatus are filtered server-side; omit them from + // the client-side pass so they don't double-filter the already-narrowed page of records. + const clientColumnFilters: Record = { + ...detailsColumnFilters, + csaMatchFound: [], + batchNumber: [], + transactionType: [], + transactionSource: [], + craStatus: [], + matchedBy: [], + } const recordsAfterSearchFilterSort = filterAndSortRows( records, WEEKLY_DETAILS_COLUMNS, getDetailsFieldValue, - detailsSearchTerm, - detailsColumnFilters, + '', + clientColumnFilters, detailsSortConfig, ) @@ -545,14 +611,7 @@ export default function WeeklyFileProcessingTab() { } return recordsAfterSearchFilterSort.filter((record) => record.id === selectedRecordId) - }, [ - detailsSearchTerm, - records, - detailsColumnFilters, - detailsSortConfig, - detailsShowSelectedOnly, - selectedRecordId, - ]) + }, [records, detailsColumnFilters, detailsSortConfig, detailsShowSelectedOnly, selectedRecordId]) const filteredSearchedChildren = useMemo(() => { return filterAndSortRows( @@ -581,34 +640,6 @@ export default function WeeklyFileProcessingTab() { handleWeeklyReportSortClose() } - const handleWeeklyReportFilterClick = ( - event: React.MouseEvent, - column: WeeklyReportColumn, - ) => { - setWeeklyReportFilterAnchor({ element: event.currentTarget, column }) - setWeeklyReportFilterSearchTerm('') - } - - const handleWeeklyReportFilterClose = () => { - setWeeklyReportFilterAnchor({ ...weeklyReportFilterAnchor, element: null }) - setWeeklyReportFilterSearchTerm('') - } - - const handleWeeklyReportFilterChange = (column: WeeklyReportColumn, value: string) => { - setWeeklyReportColumnFilters((prev) => toggleColumnFilterValue(prev, column, value)) - } - - const clearWeeklyReportColumnFilter = (column: WeeklyReportColumn) => { - setWeeklyReportColumnFilters((prev) => ({ ...prev, [column]: [] })) - setWeeklyReportFilterSearchTerm('') - } - - const getWeeklyReportUniqueValues = (column: WeeklyReportColumn): string[] => { - return Array.from(new Set(weeklyFiles.map((file) => getWeeklyReportFieldValue(file, column)))) - .filter((value) => value !== '') - .sort((a, b) => compareStrings(a, b)) - } - const handleDetailsSortClick = ( event: React.MouseEvent, column: WeeklyDetailsColumn, @@ -630,7 +661,11 @@ export default function WeeklyFileProcessingTab() { column: WeeklyDetailsColumn, ) => { setDetailsFilterAnchor({ element: event.currentTarget, column }) - setDetailsFilterSearchTerm('') + if (DETAILS_TEXT_FILTER_COLUMNS.has(column as DetailsTextFilterColumn)) { + setDetailsFilterSearchTerm(detailsTextColumnFilters[column as DetailsTextFilterColumn]) + } else { + setDetailsFilterSearchTerm('') + } } const handleDetailsFilterClose = () => { @@ -642,19 +677,69 @@ export default function WeeklyFileProcessingTab() { setDetailsSelectionFilterAnchor(null) } + const SERVER_SIDE_FILTER_COLUMNS: ReadonlySet = new Set([ + 'csaMatchFound', + 'transactionType', + 'craStatus', + ]) + + const isDetailsTextFilterColumn = ( + column: WeeklyDetailsColumn, + ): column is DetailsTextFilterColumn => { + return DETAILS_TEXT_FILTER_COLUMNS.has(column as DetailsTextFilterColumn) + } + + const isDetailsFilterActive = (column: WeeklyDetailsColumn): boolean => { + if (isDetailsTextFilterColumn(column)) { + return (getBackendTextFilterValue(column, detailsTextColumnFilters[column]) ?? '').length > 0 + } + return detailsColumnFilters[column].length > 0 + } + const handleDetailsFilterChange = (column: WeeklyDetailsColumn, value: string) => { - setDetailsColumnFilters((prev) => toggleColumnFilterValue(prev, column, value)) + if (isDetailsTextFilterColumn(column)) { + setDetailsTextColumnFilters((prev) => ({ ...prev, [column]: value })) + const trimmed = value.trim() + if (trimmed.length === 0 || trimmed.length >= getDetailsTextFilterMinLength(column)) { + setRecordsPage(1) + } + return + } + setDetailsColumnFilters((prev) => + toggleColumnFilterValue(prev, column, value), + ) + if (SERVER_SIDE_FILTER_COLUMNS.has(column)) { + setRecordsPage(1) + } } const clearDetailsColumnFilter = (column: WeeklyDetailsColumn) => { - setDetailsColumnFilters((prev) => ({ ...prev, [column]: [] })) + if (isDetailsTextFilterColumn(column)) { + setDetailsTextColumnFilters((prev) => ({ ...prev, [column]: '' })) + } else { + setDetailsColumnFilters((prev) => ({ ...prev, [column]: [] })) + if (SERVER_SIDE_FILTER_COLUMNS.has(column)) { + setRecordsPage(1) + } + } setDetailsFilterSearchTerm('') } - const getDetailsUniqueValues = (column: WeeklyDetailsColumn): string[] => { + // Hardcoded option sets for server-side filtered columns (value = API param, label = table display). + const SERVER_SIDE_COLUMN_OPTIONS: Partial> = { + csaMatchFound: [...WEEKLY_FILE_CSA_MATCH_FOUND_FILTER_OPTIONS], + transactionType: [...WEEKLY_FILE_TRANSACTION_TYPE_FILTER_OPTIONS], + craStatus: [...WEEKLY_FILE_CRA_STATUS_FILTER_OPTIONS], + } + + const getDetailsFilterOptions = (column: WeeklyDetailsColumn): DetailsFilterOption[] => { + if (SERVER_SIDE_COLUMN_OPTIONS[column]) { + return SERVER_SIDE_COLUMN_OPTIONS[column]! + } return Array.from(new Set(records.map((record) => getDetailsFieldValue(record, column)))) .filter((value) => value !== '') .sort((a, b) => compareStrings(a, b)) + .map((value) => ({ value, label: value })) } const handleChildSearchSortClick = ( @@ -773,7 +858,20 @@ export default function WeeklyFileProcessingTab() { const refreshSelectedFileRecords = async () => { if (!selectedFileId) return - const response = await getWeeklyFileRecords(selectedFileId, recordsPage, DETAILS_PAGE_SIZE) + const response = await getWeeklyFileRecords( + selectedFileId, + recordsPage, + DETAILS_PAGE_SIZE, + undefined, + { + csaMatchFound: detailsColumnFilters.csaMatchFound, + transactionType: detailsColumnFilters.transactionType, + craStatus: detailsColumnFilters.craStatus, + matchedBy: detailsBackendTextFilters.matchedBy, + batchNumber: detailsBackendTextFilters.batchNumber, + transactionSource: detailsBackendTextFilters.transactionSource, + }, + ) setRecords(response.data) setRecordsTotalPages(Math.max(response.totalPages, 1)) } @@ -867,46 +965,14 @@ export default function WeeklyFileProcessingTab() { Weekly Report - setWeeklyReportSearchTerm(e.target.value)} - InputProps={{ - startAdornment: ( - - - 🔍 - - - ), - endAdornment: weeklyReportSearchTerm && ( - - setWeeklyReportSearchTerm('')} edge="end"> - - - - ), - }} - sx={{ width: '300px' }} - /> - + - setDetailsSearchTerm(e.target.value)} - InputProps={{ - startAdornment: ( - - - 🔍 - - - ), - endAdornment: detailsSearchTerm && ( - - setDetailsSearchTerm('')} edge="end"> - - - - ), - }} - sx={{ width: '300px' }} - /> - - setWeeklyReportFilterSearchTerm(e.target.value)} - InputProps={{ - startAdornment: ( - - - 🔍 - - - ), - }} - sx={{ mb: 1 }} - /> - - {getWeeklyReportUniqueValues(weeklyReportFilterAnchor.column) - .filter((value) => - value.toLowerCase().includes(weeklyReportFilterSearchTerm.toLowerCase()), - ) - .map((value) => ( - - - handleWeeklyReportFilterChange(weeklyReportFilterAnchor.column, value) - } - /> - {value} - - ))} - - - - setDetailsFilterSearchTerm(e.target.value)} + onChange={(e) => { + const value = e.target.value + setDetailsFilterSearchTerm(value) + if (isDetailsTextFilterColumn(detailsFilterAnchor.column)) { + handleDetailsFilterChange(detailsFilterAnchor.column, value) + } + }} InputProps={{ startAdornment: ( @@ -1795,24 +1758,29 @@ export default function WeeklyFileProcessingTab() { }} sx={{ mb: 1 }} /> - - {getDetailsUniqueValues(detailsFilterAnchor.column) - .filter((value) => - value.toLowerCase().includes(detailsFilterSearchTerm.toLowerCase()), - ) - .map((value) => ( - - handleDetailsFilterChange(detailsFilterAnchor.column, value)} - /> - {value} - - ))} - + {!isDetailsTextFilterColumn(detailsFilterAnchor.column) && ( + + {getDetailsFilterOptions(detailsFilterAnchor.column) + .filter((option) => + option.label.toLowerCase().includes(detailsFilterSearchTerm.toLowerCase()), + ) + .map((option) => ( + + + handleDetailsFilterChange(detailsFilterAnchor.column, option.value) + } + /> + {option.label} + + ))} + + )} diff --git a/frontend/src/service/weekly-files-service.ts b/frontend/src/service/weekly-files-service.ts index 82c94034..c2e45446 100644 --- a/frontend/src/service/weekly-files-service.ts +++ b/frontend/src/service/weekly-files-service.ts @@ -66,14 +66,65 @@ export const getWeeklyFiles = async ( return response.data } +export interface WeeklyFileRecordFilters { + /** Semantic filter: "Yes" or "No" (maps to match_status groups). */ + csaMatchFound?: string[] + /** Stored transaction_type codes: A, C, U. */ + transactionType?: string[] + /** Stored cra_status values: completed, in-progress, abandoned, updated. */ + craStatus?: string[] + matchedBy?: string + batchNumber?: string + transactionSource?: string +} + +/** Filter dropdown options: value is sent to the API; label matches table display. */ +export const WEEKLY_FILE_TRANSACTION_TYPE_FILTER_OPTIONS = [ + { value: 'A', label: 'Application' }, + { value: 'C', label: 'Cancellation' }, + { value: 'U', label: 'CRA Update' }, +] as const + +export const WEEKLY_FILE_CRA_STATUS_FILTER_OPTIONS = [ + { value: 'completed', label: 'COMPLETED' }, + { value: 'abandoned', label: 'ABANDONED' }, + { value: 'in-progress', label: 'IN PROGRESS' }, + { value: 'updated', label: 'UPDATED' }, +] as const + +export const WEEKLY_FILE_CSA_MATCH_FOUND_FILTER_OPTIONS = [ + { value: 'Yes', label: 'Yes' }, + { value: 'No', label: 'No' }, +] as const + export const getWeeklyFileRecords = async ( fileId: number, page: number = 1, limit: number = 10, signal?: AbortSignal, + filters?: WeeklyFileRecordFilters, ): Promise> => { + const params: Record = { page, limit } + if (filters?.csaMatchFound?.length) { + params.csaMatchFound = filters.csaMatchFound.join(',') + } + if (filters?.transactionType?.length) { + params.transactionType = filters.transactionType.join(',') + } + if (filters?.craStatus?.length) { + params.craStatus = filters.craStatus.join(',') + } + if (filters?.matchedBy?.trim()) { + params.matchedBy = filters.matchedBy.trim() + } + if (filters?.batchNumber?.trim()) { + params.batchNumber = filters.batchNumber.trim() + } + if (filters?.transactionSource?.trim()) { + params.transactionSource = filters.transactionSource.trim() + } const response = await APIService.getAxiosInstance().get(`/weekly-files/${fileId}/records`, { - params: { page, limit }, + params, signal, }) return response.data diff --git a/migrations/sql/V22__add_wkl_filter_columns.sql b/migrations/sql/V22__add_wkl_filter_columns.sql new file mode 100644 index 00000000..4a4805cb --- /dev/null +++ b/migrations/sql/V22__add_wkl_filter_columns.sql @@ -0,0 +1,14 @@ +-- Add denormalized filter columns for weekly file details filtering. +-- Data is populated directly from record_data JSON during record creation/update. +-- Keep batch req id filtering as-is (join on batch_detail_id -> batches.batch_number). + +ALTER TABLE csa.wkl_file_records +ADD COLUMN IF NOT EXISTS transaction_type TEXT, +ADD COLUMN IF NOT EXISTS cra_status TEXT, +ADD COLUMN IF NOT EXISTS transaction_source TEXT; + +CREATE INDEX IF NOT EXISTS idx_wkl_file_records_transaction_type ON csa.wkl_file_records (transaction_type); + +CREATE INDEX IF NOT EXISTS idx_wkl_file_records_cra_status ON csa.wkl_file_records (cra_status); + +CREATE INDEX IF NOT EXISTS idx_wkl_file_records_transaction_source ON csa.wkl_file_records (transaction_source); diff --git a/migrations/sql/V23__backfill_wkl_filter_columns.sql b/migrations/sql/V23__backfill_wkl_filter_columns.sql new file mode 100644 index 00000000..2aafe5db --- /dev/null +++ b/migrations/sql/V23__backfill_wkl_filter_columns.sql @@ -0,0 +1,31 @@ +-- Backfill raw file values for denormalized WKL filter columns. +-- +-- These columns intentionally store the original values from record_data. +-- Any display normalization is handled in the backend API layer. + +UPDATE csa.wkl_file_records +SET + transaction_type = CASE + WHEN record_data ? 'transactionType' THEN TRIM( + COALESCE( + record_data ->> 'transactionType', + '' + ) + ) + ELSE NULL + END, + cra_status = CASE + WHEN record_data ? 'status' THEN TRIM( + COALESCE(record_data ->> 'status', '') + ) + ELSE NULL + END, + transaction_source = CASE + WHEN record_data ? 'receiveMode' THEN TRIM( + COALESCE( + record_data ->> 'receiveMode', + '' + ) + ) + ELSE NULL + END; From 55507f7d700a258aff6daeace157b97ad0728179 Mon Sep 17 00:00:00 2001 From: Madjo Diapena Date: Wed, 24 Jun 2026 21:38:07 -0400 Subject: [PATCH 04/11] fix: include Terminated status in OOC agreement lines search spec (#386) Terminated Out of Care agreement lines were excluded from ICM sync because the SearchSpec only matched Active and Inactive statuses. --- backend/src/sync/icm/agreement-lines.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/sync/icm/agreement-lines.ts b/backend/src/sync/icm/agreement-lines.ts index 037edc2f..0742de4a 100644 --- a/backend/src/sync/icm/agreement-lines.ts +++ b/backend/src/sync/icm/agreement-lines.ts @@ -3,7 +3,7 @@ import { IcmApiRecord } from './data-source/icm-data-source' /** OOC agreement line SearchSpec for flat /AgreementLines/AgreementLine reads. */ export const OOC_AGREEMENT_LINES_SEARCH_SPEC = - "([Agreement Status] = 'Active' OR [Agreement Status] = 'Inactive') AND [Agreement Type] = 'Out of Care'" + "([Agreement Status] = 'Active' OR [Agreement Status] = 'Inactive' OR [Agreement Status] = 'Terminated') AND [Agreement Type] = 'Out of Care'" /** Fields required for stg_icm_agreement_line (join bridge). */ export const OOC_AGREEMENT_LINES_FIELDS = 'Id,Updated,ICM Person ID,Agreement Id' From 0fda1289717b37e6b68e1ee482efe69984349ada Mon Sep 17 00:00:00 2001 From: Madjo Diapena Date: Wed, 24 Jun 2026 21:40:45 -0400 Subject: [PATCH 05/11] fix: add source_agreement for agreement details display (#384) * feat: add source_agreement for agreement details display Introduce source_agreement so OOC contacts show ICM/MIS in Agreement Details while placement source stays scoped to primary placement. Includes migration backfill from source_placement, eligibility extract, API/frontend wiring, and tests for OOC and MIS agreement fallback paths. * fix lint * updated the version of migrations --------- Co-authored-by: plakkara --- backend/prisma/schema.prisma | 1 + backend/prisma/seed.ts | 1 + backend/src/api/contacts/dto/contact.dto.ts | 5 +- .../eligibility/eligibility.service.spec.ts | 161 ++++++++++++++++++ .../sync/eligibility/eligibility.service.ts | 5 + frontend/src/App.tsx | 5 +- frontend/src/service/contacts-service.ts | 1 + .../utils/__tests__/mock-placement.test.ts | 16 ++ frontend/src/utils/mock-placement.ts | 5 +- .../V24__add_source_agreement_to_contacts.sql | 9 + 10 files changed, 205 insertions(+), 4 deletions(-) create mode 100644 migrations/sql/V24__add_source_agreement_to_contacts.sql diff --git a/backend/prisma/schema.prisma b/backend/prisma/schema.prisma index ccbaa319..451e2d09 100644 --- a/backend/prisma/schema.prisma +++ b/backend/prisma/schema.prisma @@ -71,6 +71,7 @@ model Contact { agreementEndDate DateTime? @map("agreement_end_date") @db.Date terminationDate DateTime? @map("termination_date") @db.Date mcfdContract String? @map("mcfd_contract") + sourceAgreement String? @map("source_agreement") orderNumber String? @map("order_number") orderType String? @map("order_type") orderStatus String? @map("order_status") diff --git a/backend/prisma/seed.ts b/backend/prisma/seed.ts index a8262986..9cc1f02c 100644 --- a/backend/prisma/seed.ts +++ b/backend/prisma/seed.ts @@ -207,6 +207,7 @@ function generateContact(csaStatus: string) { agreementEndDate: agreementEnd ?? null, terminationDate: terminationDate ?? null, mcfdContract: faker.string.alphanumeric(10).toUpperCase(), + sourceAgreement: faker.helpers.arrayElement(SOURCES), orderNumber: faker.string.alphanumeric(8).toUpperCase(), orderType: faker.helpers.arrayElement(ORDER_TYPES), diff --git a/backend/src/api/contacts/dto/contact.dto.ts b/backend/src/api/contacts/dto/contact.dto.ts index 6fef8756..942202b1 100644 --- a/backend/src/api/contacts/dto/contact.dto.ts +++ b/backend/src/api/contacts/dto/contact.dto.ts @@ -141,7 +141,7 @@ export class ContactDto { @ApiPropertyOptional({ description: 'Whether placement was interrupted' }) interruptedPlacement?: string - @ApiPropertyOptional({ description: 'Source of placement' }) + @ApiPropertyOptional({ description: 'Source of the primary placement (ICM or MIS)' }) sourcePlacement?: string @ApiPropertyOptional({ description: 'Service provider name' }) @@ -171,6 +171,9 @@ export class ContactDto { @ApiPropertyOptional({ description: 'MCFD contract identifier' }) mcfdContract?: string + @ApiPropertyOptional({ description: 'Source of the primary agreement or contract (ICM or MIS)' }) + sourceAgreement?: string + @ApiPropertyOptional({ description: 'Order number' }) orderNumber?: string diff --git a/backend/src/sync/eligibility/eligibility.service.spec.ts b/backend/src/sync/eligibility/eligibility.service.spec.ts index 9712bf7a..d645340e 100644 --- a/backend/src/sync/eligibility/eligibility.service.spec.ts +++ b/backend/src/sync/eligibility/eligibility.service.spec.ts @@ -548,6 +548,167 @@ describe('EligibilityService', () => { expect(mockPrisma.$executeRawUnsafe).toHaveBeenCalledTimes(1) // batchUpsertRows only }) + const SOURCE_PLACEMENT_COLUMN_INDEX = 39 + const SOURCE_AGREEMENT_COLUMN_INDEX = 49 + + function upsertedColumnValues(index: number): unknown[] { + expect(mockPrisma.$executeRawUnsafe).toHaveBeenCalled() + return mockPrisma.$executeRawUnsafe.mock.calls[0][index + 1] as unknown[] + } + + function makeOocIcmContact(overrides: Record = {}) { + return makeEligibleContact({ + misLegalAuthCode: 'OPC', + icmPlacements: [], + icmOrders: [], + icmAgreements: [ + { + rowId: 'AGR-OOC', + agreementType: 'Out of Care', + agreementStatus: 'Active', + agreementStartDate: '2024-01-01', + agreementEndDate: null, + terminationDate: null, + mcfdContract: 'C-OOC', + serviceProviderName: 'OOC Provider', + providerId: 'PROV-OOC', + }, + ], + csaStatus: null, + existingContactId: null, + ...overrides, + }) + } + + function makeOocMisFallbackContact(overrides: Record = {}) { + return makeEligibleContact({ + misLegalAuthCode: 'OPC', + icmPlacements: [], + icmOrders: [], + icmAgreements: [], + misPlacements: [ + { + type: 'PL', + status: 'ACTIVE', + startDate: '2024-01-01', + endDate: null, + contractNumber: 'CON-MIS', + placementNumber: 'MIS-PL-1', + serviceType: '54', + serviceProviderName: 'MIS Provider', + providerId: 'RE-1', + placeOfServiceName: 'Home', + }, + ], + misContracts: [ + { + contractNumber: 'CON-MIS', + providerId: 'RE-1', + type: 'Out of Care', + status: 'Active', + startDate: '2024-01-01', + endDate: null, + terminationDate: null, + serviceProviderName: 'MIS Provider', + }, + ], + misPayments: [], + csaStatus: null, + existingContactId: null, + ...overrides, + }) + } + + it('sets source_agreement from primaryAgreement when OOC has no placement', async () => { + mockPrisma.$queryRawUnsafe.mockResolvedValueOnce([makeOocIcmContact()]) + + await service.run(null) + + expect(upsertedColumnValues(SOURCE_PLACEMENT_COLUMN_INDEX)).toEqual([null]) + expect(upsertedColumnValues(SOURCE_AGREEMENT_COLUMN_INDEX)).toEqual(['ICM']) + }) + + it('sets source_agreement from MIS contract when OOC falls back to MIS', async () => { + mockPrisma.$queryRawUnsafe.mockResolvedValueOnce([makeOocMisFallbackContact()]) + + await service.run(null) + + expect(upsertedColumnValues(SOURCE_PLACEMENT_COLUMN_INDEX)).toEqual([null]) + expect(upsertedColumnValues(SOURCE_AGREEMENT_COLUMN_INDEX)).toEqual(['MIS']) + }) + + it('leaves placement and agreement source null when OOC has no agreement', async () => { + mockPrisma.$queryRawUnsafe.mockResolvedValueOnce([ + makeEligibleContact({ + misLegalAuthCode: 'OPT', + icmPlacements: [], + icmOrders: [], + icmAgreements: [], + csaStatus: null, + existingContactId: null, + }), + ]) + + await service.run(null) + + expect(upsertedColumnValues(SOURCE_PLACEMENT_COLUMN_INDEX)).toEqual([null]) + expect(upsertedColumnValues(SOURCE_AGREEMENT_COLUMN_INDEX)).toEqual([null]) + }) + + it('sets placement and agreement source from placement for non-OOC contacts', async () => { + mockPrisma.$queryRawUnsafe.mockResolvedValueOnce([makeEligibleContact()]) + + await service.run(null) + + expect(upsertedColumnValues(SOURCE_PLACEMENT_COLUMN_INDEX)).toEqual(['ICM']) + expect(upsertedColumnValues(SOURCE_AGREEMENT_COLUMN_INDEX)).toEqual(['ICM']) + }) + + it('sets source_agreement from MIS contract when ICM placement falls back to MIS agreement', async () => { + mockPrisma.$queryRawUnsafe.mockResolvedValueOnce([ + makeEligibleContact({ + icmPlacements: [ + { + type: 'Placement', + status: 'Active', + startDate: '2024-01-01', + endDate: null, + contractNumber: 'CON-1', + agreementRowId: 'AGR-MISSING', + paidUnpaid: 'Paid', + placementNumber: 'PL-1', + serviceType: 'Foster Care', + serviceProviderName: 'Provider A', + providerId: 'RE-1', + placeOfServiceName: 'Home A', + interruptedPlacementId: null, + }, + ], + icmAgreements: [], + icmOrders: [], + misContracts: [ + { + contractNumber: 'con-1', + providerId: 're-1', + type: 'MIS-CONTRACT', + status: 'Active', + startDate: '2024-01-01', + endDate: null, + terminationDate: null, + serviceProviderName: 'MIS Provider', + }, + ], + misPlacements: [], + misPayments: [], + }), + ]) + + await service.run(null) + + expect(upsertedColumnValues(SOURCE_PLACEMENT_COLUMN_INDEX)).toEqual(['ICM']) + expect(upsertedColumnValues(SOURCE_AGREEMENT_COLUMN_INDEX)).toEqual(['MIS']) + }) + it('should skip new contacts who are already over 18', async () => { mockPrisma.$queryRawUnsafe.mockResolvedValueOnce([makeOver18Contact()]) diff --git a/backend/src/sync/eligibility/eligibility.service.ts b/backend/src/sync/eligibility/eligibility.service.ts index 8f3418ef..cdc7ca0b 100644 --- a/backend/src/sync/eligibility/eligibility.service.ts +++ b/backend/src/sync/eligibility/eligibility.service.ts @@ -506,6 +506,11 @@ const CONTACT_COLUMNS: ContactColumnDef[] = [ extract: (row) => row.primaryAgreement?.mcfdContract ?? row.primaryPlacement?.contractNumber ?? null, }, + { + dbColumn: 'source_agreement', + pgType: 'text', + extract: (row) => row.primaryAgreement?.source ?? row.primaryPlacement?.source ?? null, + }, { dbColumn: 'order_number', pgType: 'text', diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 8dd21dd4..3fabd456 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -2867,6 +2867,7 @@ function App() { agreementEndDate: contact.agreementEndDate ? formatDateYMD(contact.agreementEndDate) : '', terminationDate: contact.terminationDate ? formatDateYMD(contact.terminationDate) : '', mcfdContract: contact.mcfdContract || '', + sourceAgreement: contact.sourceAgreement || '', product: contact.product || '', isOver18: contact.isOver18 || false, cgwrks3: contact.holdBy || '', @@ -5984,7 +5985,7 @@ function App() { wordBreak: 'break-word', }} > - {childData.sourcePlacement ? ( + {childData.sourceAgreement ? ( - {childData.sourcePlacement} + {childData.sourceAgreement} ) : ( '-' diff --git a/frontend/src/service/contacts-service.ts b/frontend/src/service/contacts-service.ts index a868932e..0df236a8 100644 --- a/frontend/src/service/contacts-service.ts +++ b/frontend/src/service/contacts-service.ts @@ -61,6 +61,7 @@ export interface Contact { agreementEndDate?: string terminationDate?: string mcfdContract?: string + sourceAgreement?: string // Order fields orderNumber?: string orderType?: string diff --git a/frontend/src/utils/__tests__/mock-placement.test.ts b/frontend/src/utils/__tests__/mock-placement.test.ts index a0fd1d58..99454052 100644 --- a/frontend/src/utils/__tests__/mock-placement.test.ts +++ b/frontend/src/utils/__tests__/mock-placement.test.ts @@ -105,4 +105,20 @@ describe('mock placement helpers', () => { }) expect(formatDateYMD).toHaveBeenCalledTimes(2) }) + + test('hides placement source when placement location is blank', () => { + const result = buildPlacementDisplayValues( + { + placementLocation: '', + locationType: '', + locationSubType: '', + placementStatus: '', + sourcePlacement: 'MIS', + placeOfServiceName: '', + }, + vi.fn(), + ) + + expect(result.sourcePlacement).toBe('') + }) }) diff --git a/frontend/src/utils/mock-placement.ts b/frontend/src/utils/mock-placement.ts index 0029a206..7c71263b 100644 --- a/frontend/src/utils/mock-placement.ts +++ b/frontend/src/utils/mock-placement.ts @@ -38,11 +38,14 @@ export const isMockSection54Placement = (contact: MockPlacementMatchInput): bool normalizeMatchValue(contact.locationSubType) === '54' && normalizeMatchValue(contact.placementStatus) === 'ACTIVE' +const hasPlacementDetails = (contact: PlacementDisplayInput): boolean => + Boolean(contact.placementLocation?.trim()) + export const buildPlacementDisplayValues = ( contact: PlacementDisplayInput, formatDateYMD: (dateString: string) => string, ): PlacementDisplayValues => { - const hidePlacementDetails = isMockSection54Placement(contact) + const hidePlacementDetails = isMockSection54Placement(contact) || !hasPlacementDetails(contact) return { placementLocation: hidePlacementDetails ? '' : contact.placementLocation || '', diff --git a/migrations/sql/V24__add_source_agreement_to_contacts.sql b/migrations/sql/V24__add_source_agreement_to_contacts.sql new file mode 100644 index 00000000..8a014b6f --- /dev/null +++ b/migrations/sql/V24__add_source_agreement_to_contacts.sql @@ -0,0 +1,9 @@ +ALTER TABLE csa.contacts +ADD COLUMN IF NOT EXISTS source_agreement TEXT; + +COMMENT ON COLUMN csa.contacts.source_agreement IS 'ICM or MIS source for the primary agreement/contract displayed in CSA details'; + +UPDATE csa.contacts +SET source_agreement = source_placement +WHERE source_placement IS NOT NULL + AND source_agreement IS NULL; From 22d9c0cc06187694da3ae5c96e6be867b2e5d794 Mon Sep 17 00:00:00 2001 From: Madjo Diapena Date: Wed, 24 Jun 2026 22:10:55 -0400 Subject: [PATCH 06/11] fix: scope WKL in-progress batch lookup by batch date (#382) * fix: scope WKL in-progress batch lookup by batch date Prevent WKL associated-record processing from reusing in-progress batch details from a different batch date when preferExistingInProgressDetail is set. * fix lint * fix: use CSA processing date for WKL CRA batch lookup on poll Poll processing was finding or creating CRA batches from the WKL header date instead of CSA Processing Date, causing records to land on the wrong batch when those dates differ. Align the poll path with manual reprocess via shared csaProcessingBatchDate and findOrCreate by Pacific processing date, while keeping in-progress detail reuse across any batch. --------- Co-authored-by: plakkara-bc --- .../api/weekly-files/weekly-files.service.ts | 11 +- backend/src/common/utils.spec.ts | 20 +++ backend/src/common/utils.ts | 8 ++ .../poll-cra-response.handler.spec.ts | 38 ++++- .../cra/handlers/poll-cra-response.handler.ts | 13 +- ...ssociated-record-processor.service.spec.ts | 136 ++++++++++++++++++ ...wkl-associated-record-processor.service.ts | 11 +- 7 files changed, 216 insertions(+), 21 deletions(-) create mode 100644 backend/src/cra/inbound/wkl-associated-record-processor.service.spec.ts diff --git a/backend/src/api/weekly-files/weekly-files.service.ts b/backend/src/api/weekly-files/weekly-files.service.ts index d51e064d..495b2e1c 100644 --- a/backend/src/api/weekly-files/weekly-files.service.ts +++ b/backend/src/api/weekly-files/weekly-files.service.ts @@ -1,7 +1,7 @@ import { BadRequestException, Injectable, Logger, NotFoundException } from '@nestjs/common' -import { DateTime } from 'luxon' import { PaginatedResponse } from 'src/api/common/dto/paginated-response.dto' import { PrismaService } from 'src/common/database/prisma.service' +import { csaProcessingBatchDate } from 'src/common/utils' import { CRA_DATA_HANDLING_CONSTANT } from 'src/cra/cra.constant' import type { DetailRecord04, HeaderRecord } from 'src/cra/inbound/inbound-weekly.interface' import { RecordTypeCode, TranCode } from 'src/cra/inbound/inbound-weekly.interface' @@ -478,12 +478,3 @@ function buildWklHeader(weeklyFileDate: Date | null): HeaderRecord { filler2: '', } } - -const PACIFIC_ZONE = 'America/Vancouver' - -function csaProcessingBatchDate(deliveredAt: Date | null): Date { - const isoDate = DateTime.fromJSDate(deliveredAt ?? new Date()) - .setZone(PACIFIC_ZONE) - .toISODate()! - return DateTime.fromISO(isoDate, { zone: PACIFIC_ZONE }).toJSDate() -} diff --git a/backend/src/common/utils.spec.ts b/backend/src/common/utils.spec.ts index 314f67f7..a03cba06 100644 --- a/backend/src/common/utils.spec.ts +++ b/backend/src/common/utils.spec.ts @@ -13,6 +13,7 @@ import { parseDateAsPacific, parseISODatePacific, pacificToday, + csaProcessingBatchDate, parseWklDate, } from './utils' @@ -396,6 +397,25 @@ describe('parseWklDate', () => { }) }) +describe('csaProcessingBatchDate', () => { + it('returns Pacific calendar date for a processing timestamp in PDT', () => { + const processedAt = new Date('2026-06-17T06:30:00.000Z') // Jun 16 11:30pm PT + expect(csaProcessingBatchDate(processedAt).toISOString()).toBe('2026-06-16T07:00:00.000Z') + }) + + it('returns Pacific calendar date when processing crosses into the next Pacific day', () => { + const processedAt = new Date('2026-06-17T07:30:00.000Z') // Jun 17 12:30am PT + expect(csaProcessingBatchDate(processedAt).toISOString()).toBe('2026-06-17T07:00:00.000Z') + }) + + it('falls back to now when processedAt is null', () => { + vi.useFakeTimers() + vi.setSystemTime(new Date('2026-06-17T18:30:00.000Z')) + expect(csaProcessingBatchDate(null).toISOString()).toBe('2026-06-17T07:00:00.000Z') + vi.useRealTimers() + }) +}) + describe('pacificToday', () => { it('returns Pacific midnight in PDT', () => { const nowSpy = vi diff --git a/backend/src/common/utils.ts b/backend/src/common/utils.ts index f366768f..8a299a22 100644 --- a/backend/src/common/utils.ts +++ b/backend/src/common/utils.ts @@ -159,6 +159,14 @@ export function pacificToday(): Date { return DateTime.fromISO(isoDate, { zone: PACIFIC_ZONE }).toJSDate() } +/** Pacific calendar date for when CSA processed a WKL file (maps to transfer_file.delivered_at). */ +export function csaProcessingBatchDate(processedAt: Date | null | undefined): Date { + const isoDate = DateTime.fromJSDate(processedAt ?? new Date()) + .setZone(PACIFIC_ZONE) + .toISODate()! + return DateTime.fromISO(isoDate, { zone: PACIFIC_ZONE }).toJSDate() +} + export function pacificTodayISO(): string { return DateTime.now().setZone(PACIFIC_ZONE).toISODate()! } diff --git a/backend/src/cra/handlers/poll-cra-response.handler.spec.ts b/backend/src/cra/handlers/poll-cra-response.handler.spec.ts index f9b83964..38a2a0a7 100644 --- a/backend/src/cra/handlers/poll-cra-response.handler.spec.ts +++ b/backend/src/cra/handlers/poll-cra-response.handler.spec.ts @@ -1477,7 +1477,7 @@ describe('PollCraResponseHandler', () => { } beforeEach(() => { - mockBatchesService.createWklBatchForUnmatchedRecords = vi + mockBatchesService.findOrCreateWklBatchForUnmatchedRecords = vi .fn() .mockResolvedValue(unmatchedBatch) mockBatchesService.createBatchDetailsForWklUnmatchedRecords = vi @@ -1505,7 +1505,10 @@ describe('PollCraResponseHandler', () => { const result = await handler.execute(mockContext) - expect(mockBatchesService.createWklBatchForUnmatchedRecords).toHaveBeenCalledTimes(1) + expect(mockBatchesService.findOrCreateWklBatchForUnmatchedRecords).toHaveBeenCalledTimes(1) + expect(mockBatchesService.findOrCreateWklBatchForUnmatchedRecords).toHaveBeenCalledWith( + expect.any(Date), + ) expect(mockBatchesService.createBatchDetailsForWklUnmatchedRecords).toHaveBeenCalledWith( 500, 99, @@ -1564,6 +1567,32 @@ describe('PollCraResponseHandler', () => { expect(result.metadata.records_wkl_unmatched_approved).toBe(0) }) + it('uses CSA processing date (not weekly file header date) for CRA batch lookup', async () => { + vi.useFakeTimers() + vi.setSystemTime(new Date('2026-06-17T07:30:00.000Z')) + + setupWeeklyFile() + mockInboundWeeklyResponseService.parseWeeklyResponseFile.mockReturnValue({ + header: { tranCode: '6136', recordTypeCode: '00', processDate: '20260616' }, + details: [ + makeWklDetail({ + transactionType: 'C' as const, + careEndDate: '20250601', + status: WKL_STATUS.COMPLETED, + }), + ], + trailer: { tranCode: '6138', recordTypeCode: '00', recordCount: 3 }, + }) + + await handler.execute(mockContext) + + expect(mockBatchesService.findOrCreateWklBatchForUnmatchedRecords).toHaveBeenCalledWith( + new Date('2026-06-17T07:00:00.000Z'), + ) + + vi.useRealTimers() + }) + it('reuses the same unmatched batch across multiple unmatched records in one file', async () => { setupWeeklyFile() setupWeeklyParseFile([ @@ -1581,7 +1610,10 @@ describe('PollCraResponseHandler', () => { await handler.execute(mockContext) - expect(mockBatchesService.createWklBatchForUnmatchedRecords).toHaveBeenCalledTimes(1) + expect(mockBatchesService.findOrCreateWklBatchForUnmatchedRecords).toHaveBeenCalledTimes(1) + expect(mockBatchesService.findOrCreateWklBatchForUnmatchedRecords).toHaveBeenCalledWith( + expect.any(Date), + ) expect(mockBatchesService.createBatchDetailsForWklUnmatchedRecords).toHaveBeenCalledTimes(2) }) diff --git a/backend/src/cra/handlers/poll-cra-response.handler.ts b/backend/src/cra/handlers/poll-cra-response.handler.ts index e18ead34..778b9983 100644 --- a/backend/src/cra/handlers/poll-cra-response.handler.ts +++ b/backend/src/cra/handlers/poll-cra-response.handler.ts @@ -7,7 +7,7 @@ import { ContactsService } from 'src/api/contacts/contacts.service' import { PrismaService } from 'src/common/database/prisma.service' import { BATCH_DETAIL_EVENT, CSA_EVENT } from 'src/common/state-machine/constants' -import { parseWklDate } from 'src/common/utils' +import { csaProcessingBatchDate, parseWklDate } from 'src/common/utils' import { BaseJob } from 'src/jobs/base-job' import { JobType } from 'src/jobs/enums/job-type.enum' import { JobResult } from 'src/jobs/interfaces/job-result.interface' @@ -39,6 +39,8 @@ interface WklRecordContext { transferFileId: number recordIndex: number weeklyFileDate: Date | null + /** Pacific calendar date when this WKL file is being processed (CSA Processing Date). */ + csaProcessingDate: Date } @Injectable() @@ -267,17 +269,21 @@ export class PollCraResponseHandler extends BaseJob { (recordCount !== undefined ? `, Total Records in File = ${recordCount}` : ''), ) + const processedAt = new Date() + if (isWeekly) { this.unmatchedWklBatchId = null await this.weeklyContactMatcher.loadCandidates() const weeklyHeader = header as HeaderRecord const weeklyFileDate = parseWklDate(weeklyHeader.processDate) ?? null + const csaProcessingDate = csaProcessingBatchDate(processedAt) const weeklyDetails = details as DetailRecord04[] for (let i = 0; i < weeklyDetails.length; i++) { await this.processWeeklyDetail(weeklyDetails[i], weeklyHeader, { transferFileId: responseFile.id, recordIndex: i, weeklyFileDate, + csaProcessingDate, }) } } else { @@ -290,7 +296,7 @@ export class PollCraResponseHandler extends BaseJob { where: { id: responseFile.id }, data: { isDetailsProcessed: true, - deliveredAt: new Date(), + deliveredAt: processedAt, referenceNumbers: isWeekly ? [] : (details as CraResDetail[]).map((detail) => detail.referenceNum), @@ -413,6 +419,7 @@ export class PollCraResponseHandler extends BaseJob { contacts.id, contacts.caseNumber, header, + ctx.csaProcessingDate, ) if (contactMatch) { await this.persistWklRecord(ctx, detail, { @@ -530,6 +537,7 @@ export class PollCraResponseHandler extends BaseJob { contactId: number, caseNumber: string, header: HeaderRecord, + batchDate: Date, ): Promise<{ contactId: number; batchDetailId: number } | null> { const unmatchedWklBatchId = { value: this.unmatchedWklBatchId } const counters = { approved: 0, refused: 0, skipped: 0 } @@ -542,6 +550,7 @@ export class PollCraResponseHandler extends BaseJob { processedBatchIds: this.processedBatchIds, header, origin: 'PollCraResponseHandler.processUnmatchedWeeklyDetail', + batchDate, }, counters, ) diff --git a/backend/src/cra/inbound/wkl-associated-record-processor.service.spec.ts b/backend/src/cra/inbound/wkl-associated-record-processor.service.spec.ts new file mode 100644 index 00000000..7c917db4 --- /dev/null +++ b/backend/src/cra/inbound/wkl-associated-record-processor.service.spec.ts @@ -0,0 +1,136 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { CRA_DATA_HANDLING_CONSTANT } from '../cra.constant' +import { WklAssociatedRecordProcessorService } from './wkl-associated-record-processor.service' + +const { WEEKLY_FILE } = CRA_DATA_HANDLING_CONSTANT +const { STATUS: WKL_STATUS } = WEEKLY_FILE + +const detail = { + transactionType: 'A', + status: WKL_STATUS.COMPLETED, + childGivenName: 'Jane', + childSurName: 'Doe', + childInitial: '', + childDin: '123456789', + childSex: 'F', + childBirthDate: '20200101', + childBirthCity: 'Vancouver', + childBirthProv: 'BC', + childBirthCountry: 'CA', + receiveMode: 'E', +} + +describe('WklAssociatedRecordProcessorService', () => { + let service: WklAssociatedRecordProcessorService + let mockBatchesService: { + findInProgressBatchDetailForContact: ReturnType + findOrCreateWklBatchForUnmatchedRecords: ReturnType + createWklBatchForUnmatchedRecords: ReturnType + createBatchDetailsForWklUnmatchedRecords: ReturnType + updateBatchDetailStatus: ReturnType + } + let mockContactsService: { forceUpdateCsaStatus: ReturnType } + let mockWeeklyContactMatcher: { buildWklMatchingSnapshot: ReturnType } + + beforeEach(() => { + mockBatchesService = { + findInProgressBatchDetailForContact: vi.fn().mockResolvedValue(null), + findOrCreateWklBatchForUnmatchedRecords: vi.fn().mockResolvedValue({ id: 700 }), + createWklBatchForUnmatchedRecords: vi.fn().mockResolvedValue({ id: 701 }), + createBatchDetailsForWklUnmatchedRecords: vi.fn().mockResolvedValue({ + id: 800, + contactId: 99, + batchId: 700, + }), + updateBatchDetailStatus: vi.fn().mockResolvedValue(undefined), + } + mockContactsService = { + forceUpdateCsaStatus: vi.fn().mockResolvedValue(undefined), + } + mockWeeklyContactMatcher = { + buildWklMatchingSnapshot: vi.fn().mockReturnValue({ childGivenName: 'Jane' }), + } + + service = new WklAssociatedRecordProcessorService( + mockBatchesService as any, + mockContactsService as any, + mockWeeklyContactMatcher as any, + ) + }) + + it('reuses in-progress batch detail from any batch when preferExistingInProgressDetail is set', async () => { + mockBatchesService.findInProgressBatchDetailForContact.mockResolvedValue({ + id: 50, + contactId: 99, + batchId: 35, + transactionType: 'application', + }) + const counters = { approved: 0, refused: 0, skipped: 0 } + + await service.processAssociatedRecord( + detail as any, + 99, + '1-99', + { + unmatchedWklBatchId: { value: null }, + processedBatchIds: new Set(), + header: { processDate: '20260622' } as any, + origin: 'test', + preferExistingInProgressDetail: true, + batchDate: new Date('2026-06-17T07:00:00.000Z'), + }, + counters, + ) + + expect(mockBatchesService.findInProgressBatchDetailForContact).toHaveBeenCalledWith(99) + expect(mockBatchesService.findOrCreateWklBatchForUnmatchedRecords).not.toHaveBeenCalled() + expect(counters.approved).toBe(1) + }) + + it('finds or creates CRA batch by CSA processing date when batchDate is provided', async () => { + const batchDate = new Date('2026-06-17T07:00:00.000Z') + const counters = { approved: 0, refused: 0, skipped: 0 } + + await service.processAssociatedRecord( + detail as any, + 99, + '1-99', + { + unmatchedWklBatchId: { value: null }, + processedBatchIds: new Set(), + header: { processDate: '20260616' } as any, + origin: 'test', + batchDate, + }, + counters, + ) + + expect(mockBatchesService.findInProgressBatchDetailForContact).not.toHaveBeenCalled() + expect(mockBatchesService.findOrCreateWklBatchForUnmatchedRecords).toHaveBeenCalledWith( + batchDate, + ) + expect(mockBatchesService.createWklBatchForUnmatchedRecords).not.toHaveBeenCalled() + expect(counters.approved).toBe(1) + }) + + it('falls back to header-based batch creation when batchDate is omitted', async () => { + const header = { processDate: '20260616' } as any + const counters = { approved: 0, refused: 0, skipped: 0 } + + await service.processAssociatedRecord( + detail as any, + 99, + '1-99', + { + unmatchedWklBatchId: { value: null }, + processedBatchIds: new Set(), + header, + origin: 'test', + }, + counters, + ) + + expect(mockBatchesService.createWklBatchForUnmatchedRecords).toHaveBeenCalledWith(header) + expect(mockBatchesService.findOrCreateWklBatchForUnmatchedRecords).not.toHaveBeenCalled() + }) +}) diff --git a/backend/src/cra/inbound/wkl-associated-record-processor.service.ts b/backend/src/cra/inbound/wkl-associated-record-processor.service.ts index 3f23c2ff..8e58f975 100644 --- a/backend/src/cra/inbound/wkl-associated-record-processor.service.ts +++ b/backend/src/cra/inbound/wkl-associated-record-processor.service.ts @@ -15,9 +15,9 @@ export interface WklUnmatchedProcessContext { processedBatchIds: Set header: HeaderRecord origin: string - /** CSA processing date (Pacific calendar date) for manual confirm batch lookup */ + /** CSA processing date (Pacific calendar date) for CRA batch find/create */ batchDate?: Date - /** When true, reuse an existing in-progress batch detail or find/create batch by batchDate */ + /** When true, reuse an existing in-progress batch detail for the contact (any batch) */ preferExistingInProgressDetail?: boolean } @@ -84,10 +84,9 @@ export class WklAssociatedRecordProcessorService { if (!batchDetail) { if (!ctx.unmatchedWklBatchId.value) { - const batch = - ctx.preferExistingInProgressDetail && ctx.batchDate - ? await this.batchesService.findOrCreateWklBatchForUnmatchedRecords(ctx.batchDate) - : await this.batchesService.createWklBatchForUnmatchedRecords(ctx.header) + const batch = ctx.batchDate + ? await this.batchesService.findOrCreateWklBatchForUnmatchedRecords(ctx.batchDate) + : await this.batchesService.createWklBatchForUnmatchedRecords(ctx.header) ctx.unmatchedWklBatchId.value = batch.id ctx.processedBatchIds.add(batch.id) } From fdb3da4835565f75d3884e8dc4b54ddb71bb06c2 Mon Sep 17 00:00:00 2001 From: Madjo Diapena Date: Wed, 24 Jun 2026 22:34:43 -0400 Subject: [PATCH 07/11] fix: improve CRA send batch failure handling and UI feedback (#387) * fix: improve CRA send batch failure handling and UI feedback Record job failures on batch system comments and system_error status, invoke onFailure when send setup fails, show batch numbers in send messages, and hide the running banner when the job fails or completes. * fix lint * fix: refresh batch UI after CRA send job failures Reload batch requests and details when a send job fails so updated system_error status and system comments appear without a manual refresh. --- .../machines/batch-status.machine.spec.ts | 17 ++- .../machines/batch-status.machine.ts | 1 + .../state-machine.service.spec.ts | 10 +- .../handlers/send-cra-file.handler.spec.ts | 106 +++++++++++++++++- .../src/cra/handlers/send-cra-file.handler.ts | 63 ++++++++--- backend/src/jobs/job-runner.service.spec.ts | 1 + backend/src/jobs/job-runner.service.ts | 6 + frontend/src/App.tsx | 98 +++++++++++----- 8 files changed, 243 insertions(+), 59 deletions(-) diff --git a/backend/src/common/state-machine/machines/batch-status.machine.spec.ts b/backend/src/common/state-machine/machines/batch-status.machine.spec.ts index 12a5ae09..d311be7c 100644 --- a/backend/src/common/state-machine/machines/batch-status.machine.spec.ts +++ b/backend/src/common/state-machine/machines/batch-status.machine.spec.ts @@ -45,9 +45,14 @@ describe('batchStatusMachine', () => { expect(nextState).toBe(BATCH_STATUS.IN_PROGRESS) }) - it('should return current state for invalid transition', () => { + it('should transition from pending to system_error on SEND_FAILED', () => { const nextState = getNextBatchState(BATCH_STATUS.PENDING, BATCH_EVENT.SEND_FAILED) - expect(nextState).toBe(BATCH_STATUS.PENDING) + expect(nextState).toBe(BATCH_STATUS.SYSTEM_ERROR) + }) + + it('should return current state for invalid transition', () => { + const nextState = getNextBatchState(BATCH_STATUS.PROCESSED, BATCH_EVENT.SEND_FAILED) + expect(nextState).toBe(BATCH_STATUS.PROCESSED) }) it('should not transition from processed (terminal)', () => { @@ -66,8 +71,12 @@ describe('batchStatusMachine', () => { expect(canTransitionBatch(BATCH_STATUS.PENDING, BATCH_EVENT.SEND_TO_CRA)).toBe(true) }) - it('should return false for invalid transition', () => { - expect(canTransitionBatch(BATCH_STATUS.PENDING, BATCH_EVENT.SEND_FAILED)).toBe(false) + it('should return false for invalid transition from processed', () => { + expect(canTransitionBatch(BATCH_STATUS.PROCESSED, BATCH_EVENT.SEND_FAILED)).toBe(false) + }) + + it('should return true for pending to SEND_FAILED', () => { + expect(canTransitionBatch(BATCH_STATUS.PENDING, BATCH_EVENT.SEND_FAILED)).toBe(true) }) it('should return false for terminal processed state', () => { diff --git a/backend/src/common/state-machine/machines/batch-status.machine.ts b/backend/src/common/state-machine/machines/batch-status.machine.ts index 69bda966..bfe0314d 100644 --- a/backend/src/common/state-machine/machines/batch-status.machine.ts +++ b/backend/src/common/state-machine/machines/batch-status.machine.ts @@ -11,6 +11,7 @@ import { export const BATCH_TRANSITIONS: TransitionMap = { [BATCH_STATUS.PENDING]: { [BATCH_EVENT.SEND_TO_CRA]: BATCH_STATUS.IN_PROGRESS, + [BATCH_EVENT.SEND_FAILED]: BATCH_STATUS.SYSTEM_ERROR, }, [BATCH_STATUS.IN_PROGRESS]: { [BATCH_EVENT.SEND_FAILED]: BATCH_STATUS.SYSTEM_ERROR, diff --git a/backend/src/common/state-machine/state-machine.service.spec.ts b/backend/src/common/state-machine/state-machine.service.spec.ts index 324d3b47..cffe570e 100644 --- a/backend/src/common/state-machine/state-machine.service.spec.ts +++ b/backend/src/common/state-machine/state-machine.service.spec.ts @@ -290,9 +290,17 @@ describe('StateMachineService', () => { expect(result.to).toBe(BATCH_STATUS.IN_PROGRESS) }) - it('should return error for invalid transition', () => { + it('should transition from pending to system_error on send failure', () => { const result = service.transitionBatch(BATCH_STATUS.PENDING, BATCH_EVENT.SEND_FAILED) + expect(result.success).toBe(true) + expect(result.from).toBe(BATCH_STATUS.PENDING) + expect(result.to).toBe(BATCH_STATUS.SYSTEM_ERROR) + }) + + it('should return error for invalid transition', () => { + const result = service.transitionBatch(BATCH_STATUS.PROCESSED, BATCH_EVENT.SEND_FAILED) + expect(result.success).toBe(false) expect(result.reason).toBe('Invalid transition') }) diff --git a/backend/src/cra/handlers/send-cra-file.handler.spec.ts b/backend/src/cra/handlers/send-cra-file.handler.spec.ts index 392e6d3f..908ca9f6 100644 --- a/backend/src/cra/handlers/send-cra-file.handler.spec.ts +++ b/backend/src/cra/handlers/send-cra-file.handler.spec.ts @@ -1,3 +1,4 @@ +import { AppLogger } from 'src/common/logger/app-logger' import { BATCH_EVENT, CSA_EVENT } from 'src/common/state-machine/constants' import { JobTrigger } from 'src/jobs/enums/job-trigger.enum' import { JobType } from 'src/jobs/enums/job-type.enum' @@ -82,6 +83,7 @@ describe('SendCraFileHandler', () => { mockPrisma = { batch: { findFirst: vi.fn(), + findUnique: vi.fn(), update: vi.fn(), }, contactBatchDetail: { @@ -449,11 +451,12 @@ describe('SendCraFileHandler', () => { }) describe('onFailure', () => { - it('should transition batch to system_error (SEND_FAILED)', async () => { + it('should transition batch to system_error (SEND_FAILED) with batch system comments', async () => { const batch = makeBatch({ id: 10 }) const detail = makeBatchDetail({ id: 100, contactId: 1, batchId: 10 }) mockPrisma.batch.findFirst.mockResolvedValue(batch) + mockPrisma.batch.findUnique.mockResolvedValue({ systemComments: null }) mockPrisma.contactBatchDetail.findMany.mockResolvedValue([detail]) await handler.onStart(mockContext) @@ -464,14 +467,107 @@ describe('SendCraFileHandler', () => { BATCH_EVENT.SEND_FAILED, { additionalData: { systemComments: expect.stringContaining('Connection refused') } }, ) - expect(mockPrisma.contactBatchDetail.update).toHaveBeenCalledWith({ - where: { id: 100 }, + expect(mockPrisma.contactBatchDetail.update).not.toHaveBeenCalled() + }) + + it('should warn and persist batch failure state when status transition fails', async () => { + const batch = makeBatch({ id: 10 }) + const detail = makeBatchDetail({ id: 100, contactId: 1, batchId: 10 }) + const warnSpy = vi.spyOn(AppLogger.prototype, 'warn').mockImplementation(() => undefined) + + mockPrisma.batch.findFirst.mockResolvedValue(batch) + mockPrisma.batch.findUnique.mockResolvedValue({ + systemComments: 'existing comment', + status: 'pending', + }) + mockPrisma.contactBatchDetail.findMany.mockResolvedValue([detail]) + mockBatchesService.updateBatchStatus + .mockResolvedValueOnce({ success: true, from: 'pending', to: 'in_progress' }) + .mockResolvedValueOnce({ + success: false, + reason: 'Invalid transition', + }) + + await handler.onStart(mockContext) + await handler.onFailure(mockContext, new Error('Connection refused')) + + expect(warnSpy).toHaveBeenCalledWith( + 'Batch 10: SEND_FAILED transition failed (Invalid transition); persisted systemComments and status via direct update', + ) + expect(mockPrisma.batch.update).toHaveBeenCalledWith({ + where: { id: 10 }, data: { systemComments: expect.stringContaining('Connection refused'), - lastUpdatedAt: expect.any(Date), - lastUpdatedBy: 'SYSTEM', + status: 'system_error', }, }) + expect(mockPrisma.contactBatchDetail.update).not.toHaveBeenCalled() + + warnSpy.mockRestore() + }) + + it('should warn with Issue 1 message when only systemComments can be persisted', async () => { + const batch = makeBatch({ id: 10, status: 'processed' }) + const warnSpy = vi.spyOn(AppLogger.prototype, 'warn').mockImplementation(() => undefined) + + mockPrisma.batch.findUnique.mockResolvedValue({ + systemComments: null, + status: 'processed', + }) + mockBatchesService.updateBatchStatus.mockResolvedValue({ + success: false, + reason: 'Invalid transition', + }) + ;(handler as any).batch = batch + + await handler.onFailure(mockContext, new Error('Connection refused')) + + expect(warnSpy).toHaveBeenCalledWith( + 'Batch 10: SEND_FAILED transition failed (Invalid transition); persisted systemComments only', + ) + expect(mockPrisma.batch.update).toHaveBeenCalledWith({ + where: { id: 10 }, + data: { systemComments: expect.stringContaining('Connection refused') }, + }) + + warnSpy.mockRestore() + }) + + it('should mark pending batch as system_error when send transition never completed', async () => { + const batch = makeBatch({ id: 10, status: 'pending' }) + const detail = makeBatchDetail({ id: 100, contactId: 1, batchId: 10 }) + + mockPrisma.batch.findFirst.mockResolvedValue(batch) + mockPrisma.batch.findUnique.mockResolvedValue({ systemComments: null, status: 'pending' }) + mockPrisma.contactBatchDetail.findMany.mockResolvedValue([detail]) + mockBatchesService.updateBatchStatus + .mockResolvedValueOnce({ success: true, from: 'pending', to: 'in_progress' }) + .mockResolvedValueOnce({ success: true, from: 'pending', to: 'system_error' }) + + await handler.onStart(mockContext) + await handler.onFailure(mockContext, new Error('Connection refused')) + + expect(mockBatchesService.updateBatchStatus).toHaveBeenLastCalledWith( + 10, + BATCH_EVENT.SEND_FAILED, + { additionalData: { systemComments: expect.stringContaining('Connection refused') } }, + ) + }) + + it('should throw when batch cannot transition to in_progress', async () => { + const batch = makeBatch({ id: 10 }) + const detail = makeBatchDetail({ id: 100, contactId: 1, batchId: 10 }) + + mockPrisma.batch.findFirst.mockResolvedValue(batch) + mockPrisma.contactBatchDetail.findMany.mockResolvedValue([detail]) + mockBatchesService.updateBatchStatus.mockResolvedValue({ + success: false, + reason: 'Invalid transition', + }) + + await expect(handler.onStart(mockContext)).rejects.toThrow( + 'Failed to transition batch 10 to in_progress: Invalid transition', + ) }) it('should not transition if no batch was found', async () => { diff --git a/backend/src/cra/handlers/send-cra-file.handler.ts b/backend/src/cra/handlers/send-cra-file.handler.ts index a3e728cb..4d30d9b1 100644 --- a/backend/src/cra/handlers/send-cra-file.handler.ts +++ b/backend/src/cra/handlers/send-cra-file.handler.ts @@ -72,7 +72,15 @@ export class SendCraFileHandler extends BaseJob { await this.ensureBatchDetailsReady() - await this.batchesService.updateBatchStatus(this.batch.id, BATCH_EVENT.SEND_TO_CRA) + const sendTransition = await this.batchesService.updateBatchStatus( + this.batch.id, + BATCH_EVENT.SEND_TO_CRA, + ) + if (!sendTransition.success) { + throw new Error( + `Failed to transition batch ${this.batch.id} to in_progress: ${sendTransition.reason}`, + ) + } for (const detail of this.batchDetails) { if (detail.status !== BATCH_DETAIL_STATUS.IN_PROGRESS) { @@ -199,25 +207,44 @@ export class SendCraFileHandler extends BaseJob { async onFailure(context: JobContext, error: Error): Promise { await super.onFailure(context, error) - if (this.batch) { - this.logger.error(`File transfer failed for batch ${this.batch.id}`, error) - const errorMessage = error.message || 'File transfer failed' - await this.batchesService.updateBatchStatus(this.batch.id, BATCH_EVENT.SEND_FAILED, { - additionalData: { - systemComments: appendSystemComment(errorMessage, this.batch.systemComments), - }, - }) + if (!this.batch) return - for (const detail of this.batchDetails) { - await this.prisma.contactBatchDetail.update({ - where: { id: detail.id }, - data: { - systemComments: appendSystemComment(errorMessage, detail.systemComments), - lastUpdatedAt: new Date(), - lastUpdatedBy: 'SYSTEM', - }, - }) + this.logger.error(`File transfer failed for batch ${this.batch.id}`, error) + const errorMessage = error.message || 'File transfer failed' + const batchRecord = await this.prisma.batch.findUnique({ + where: { id: this.batch.id }, + select: { systemComments: true, status: true }, + }) + const systemComments = appendSystemComment(errorMessage, batchRecord?.systemComments ?? null) + + const result = await this.batchesService.updateBatchStatus( + this.batch.id, + BATCH_EVENT.SEND_FAILED, + { + additionalData: { systemComments }, + }, + ) + + if (!result.success) { + const recoverableStatuses: string[] = [BATCH_STATUS.PENDING, BATCH_STATUS.IN_PROGRESS] + const data: { systemComments: string | null; status?: string } = { systemComments } + if (batchRecord?.status && recoverableStatuses.includes(batchRecord.status)) { + data.status = BATCH_STATUS.SYSTEM_ERROR + } + + if (data.status) { + this.logger.warn( + `Batch ${this.batch.id}: SEND_FAILED transition failed (${result.reason}); persisted systemComments and status via direct update`, + ) + } else { + this.logger.warn( + `Batch ${this.batch.id}: SEND_FAILED transition failed (${result.reason}); persisted systemComments only`, + ) } + await this.prisma.batch.update({ + where: { id: this.batch.id }, + data, + }) } } } diff --git a/backend/src/jobs/job-runner.service.spec.ts b/backend/src/jobs/job-runner.service.spec.ts index 739e7178..c32dec6c 100644 --- a/backend/src/jobs/job-runner.service.spec.ts +++ b/backend/src/jobs/job-runner.service.spec.ts @@ -156,6 +156,7 @@ describe('JobRunner', () => { expect(result.success).toBe(false) expect(result.message).toContain('onStart failed') expect(jobsService.markFailed).toHaveBeenCalled() + expect(mockHandler.onFailure).toHaveBeenCalled() // execute should never be called if onStart fails expect(mockHandler.execute).not.toHaveBeenCalled() }) diff --git a/backend/src/jobs/job-runner.service.ts b/backend/src/jobs/job-runner.service.ts index cf84911d..08051b61 100644 --- a/backend/src/jobs/job-runner.service.ts +++ b/backend/src/jobs/job-runner.service.ts @@ -83,6 +83,12 @@ export class JobRunner { const err = error instanceof Error ? error : new Error(String(error)) this.logger.error(`Job ${jobId} onStart hook failed: ${err.message}`, err.stack) await this.safeMarkFailed(jobId, err) + try { + await handler.onFailure?.(context, err) + } catch (hookError) { + const hookErr = hookError instanceof Error ? hookError : new Error(String(hookError)) + this.logger.error(`Job ${jobId} onFailure hook threw: ${hookErr.message}`, hookErr.stack) + } return { success: false, message: `onStart failed: ${err.message}` } } diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 3fabd456..66868b59 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -571,10 +571,31 @@ function App() { const [batches, setBatches] = useState([]) const [loadingBatches, setLoadingBatches] = useState(false) + const getBatchNumberLabel = ( + batchId: number | null | undefined, + batchList: Batch[] = batches, + ): string | null => { + if (batchId == null) return null + const batch = batchList.find((entry) => entry.id === batchId) + return batch != null ? String(batch.batchNumber) : null + } + // Batch details state const [batchDetails, setBatchDetails] = useState([]) const [loadingBatchDetails, setLoadingBatchDetails] = useState(false) + const refreshBatchRequestsAfterSendCra = async (): Promise => { + const updatedBatches = await getAllBatches() + setBatches(updatedBatches) + + if (selectedBatch) { + const updatedDetails = await getBatchContacts(selectedBatch) + setBatchDetails(updatedDetails) + } + + return updatedBatches + } + useEffect(() => { if (!isAuthenticated) return @@ -618,20 +639,21 @@ function App() { contacts_count?: number contactsCount?: number } | null + const updatedBatches = await getAllBatches() + setBatches(updatedBatches) const batchId = metadata?.batch_id ?? metadata?.batchId + const batchNumber = getBatchNumberLabel(batchId, updatedBatches) setSendCraFileJobState('success') setSnackbar({ open: true, - message: batchId - ? `Send CRA file job completed for batch ${batchId}.` + message: batchNumber + ? `Send CRA file job completed for batch ${batchNumber}.` : 'Send CRA file job completed successfully.', severity: 'success', }) - - const updatedBatches = await getAllBatches() - setBatches(updatedBatches) } else { + await refreshBatchRequestsAfterSendCra() setSendCraFileJobState('failed') setSnackbar({ open: true, @@ -644,6 +666,9 @@ function App() { setRunningSendCraBatchId(null) } catch (err) { console.error('Failed to check for running SEND_CRA_FILE job:', err) + setIsRunningSendCraFile(false) + setRunningSendCraBatchId(null) + setSendCraFileJobState('idle') } } @@ -691,20 +716,21 @@ function App() { contacts_count?: number contactsCount?: number } | null + const updatedBatches = await getAllBatches() + setBatches(updatedBatches) const batchId = metadata?.batch_id ?? metadata?.batchId + const batchNumber = getBatchNumberLabel(batchId, updatedBatches) setSendCraFileJobState('success') setSnackbar({ open: true, - message: batchId - ? `Send CRA file job completed for batch ${batchId}. Please try your action again.` + message: batchNumber + ? `Send CRA file job completed for batch ${batchNumber}. Please try your action again.` : 'Send CRA file job completed successfully. Please try your action again.', severity: 'success', }) - - const updatedBatches = await getAllBatches() - setBatches(updatedBatches) } else { + await refreshBatchRequestsAfterSendCra() setSendCraFileJobState('failed') setSnackbar({ open: true, @@ -720,6 +746,9 @@ function App() { return false // No job running, can proceed } catch (err) { console.error('Failed to check for running Send CRA file job:', err) + setIsRunningSendCraFile(false) + setRunningSendCraBatchId(null) + setSendCraFileJobState('idle') return false } } @@ -732,16 +761,6 @@ function App() { setConfirmSendCraDialogOpen(true) } - const refreshBatchRequestsAfterSendCra = async () => { - const updatedBatches = await getAllBatches() - setBatches(updatedBatches) - - if (selectedBatch) { - const updatedDetails = await getBatchContacts(selectedBatch) - setBatchDetails(updatedDetails) - } - } - const handleConfirmSendCra = async () => { setConfirmSendCraDialogOpen(false) @@ -798,23 +817,29 @@ function App() { const recordCount = metadata?.record_count ?? metadata?.recordCount ?? 0 const contactsCount = metadata?.contacts_count ?? metadata?.contactsCount ?? 0 + const updatedBatches = await refreshBatchRequestsAfterSendCra() + const batchNumber = getBatchNumberLabel(batchId, updatedBatches) + setSendCraFileJobState('success') setSnackbar({ open: true, message: recordCount > 0 || contactsCount > 0 - ? `Send CRA file complete for batch ${batchId}: ${recordCount || contactsCount} record${(recordCount || contactsCount) === 1 ? '' : 's'} sent.` - : `Send CRA file complete for batch ${batchId}.`, + ? batchNumber + ? `Send CRA file complete for batch ${batchNumber}: ${recordCount || contactsCount} record${(recordCount || contactsCount) === 1 ? '' : 's'} sent.` + : `Send CRA file complete: ${recordCount || contactsCount} record${(recordCount || contactsCount) === 1 ? '' : 's'} sent.` + : batchNumber + ? `Send CRA file complete for batch ${batchNumber}.` + : 'Send CRA file complete.', severity: 'success', }) - - await refreshBatchRequestsAfterSendCra() } else { setSendCraFileJobState('failed') throw new Error(job.error || 'Send CRA file job failed') } } catch (error: any) { console.error('Send CRA file error:', error) + await refreshBatchRequestsAfterSendCra() const errorMessage = error?.response?.data?.message || error?.message || 'Failed to send CRA file' setSendCraFileJobState('failed') @@ -2108,9 +2133,12 @@ function App() { pendingBatch?.id === runningSendCraBatchId if (isPendingBatchLocked) { + const runningBatchNumber = getBatchNumberLabel(runningSendCraBatchId) setSnackbar({ open: true, - message: `Batch ${runningSendCraBatchId} is currently being sent to CRA. Please wait for it to complete before adding records.`, + message: runningBatchNumber + ? `Batch ${runningBatchNumber} is currently being sent to CRA. Please wait for it to complete before adding records.` + : 'The batch is currently being sent to CRA. Please wait for it to complete before adding records.', severity: 'info', }) return @@ -2372,9 +2400,12 @@ function App() { selectedBatchHistory?.batch.id === runningSendCraBatchId if (isSelectedHistoryBatchLocked) { + const runningBatchNumber = getBatchNumberLabel(runningSendCraBatchId) setSnackbar({ open: true, - message: `Batch ${runningSendCraBatchId} is currently being sent to CRA. Please wait for it to complete before removing records.`, + message: runningBatchNumber + ? `Batch ${runningBatchNumber} is currently being sent to CRA. Please wait for it to complete before removing records.` + : 'The batch is currently being sent to CRA. Please wait for it to complete before removing records.', severity: 'info', }) return @@ -2476,9 +2507,12 @@ function App() { selectedBatchRecord?.id === runningSendCraBatchId if (isSelectedBatchLocked) { + const runningBatchNumber = getBatchNumberLabel(runningSendCraBatchId) setSnackbar({ open: true, - message: `Batch ${runningSendCraBatchId} is currently being sent to CRA. Please wait for it to complete before removing records.`, + message: runningBatchNumber + ? `Batch ${runningBatchNumber} is currently being sent to CRA. Please wait for it to complete before removing records.` + : 'The batch is currently being sent to CRA. Please wait for it to complete before removing records.', severity: 'info', }) return @@ -2993,6 +3027,8 @@ function App() { [isRunningSendCraFile, runningSendCraBatchId, selectedBatchHistoryData], ) + const runningSendCraBatchNumber = getBatchNumberLabel(runningSendCraBatchId) + const canRemoveFromBatchDetails = useMemo(() => { if (selectedBatchDetails.length === 0) return false @@ -7070,12 +7106,12 @@ function App() { - {/* Send CRA file running banner */} - {isRunningSendCraFile && ( + {/* Send CRA file running banner — hidden once the job fails or finishes */} + {isRunningSendCraFile && sendCraFileJobState === 'running' && ( - {runningSendCraBatchId - ? `Send CRA file job is running for batch ${runningSendCraBatchId}. Changes to that batch are temporarily disabled until the job completes.` + {runningSendCraBatchNumber + ? `Send CRA file job is running for batch ${runningSendCraBatchNumber}. Changes to that batch are temporarily disabled until the job completes.` : 'Send CRA file job is running. Changes to the batch being sent are temporarily disabled until the job completes.'} From 64a12453fbd41fc0fc77146a719dc2ed73d4378c Mon Sep 17 00:00:00 2001 From: Madjo Diapena Date: Wed, 24 Jun 2026 23:59:37 -0400 Subject: [PATCH 08/11] fix: set search_path via pg pool onConnect for Prisma raw queries pool.query (used by $executeRawUnsafe) acquires clients through connect(callback), which skipped search_path after the recent pool refactor and caused staging upserts to fail with missing relations. --- .../common/database/prisma.service.spec.ts | 11 ++++++ backend/src/common/database/prisma.service.ts | 37 +++---------------- 2 files changed, 17 insertions(+), 31 deletions(-) diff --git a/backend/src/common/database/prisma.service.spec.ts b/backend/src/common/database/prisma.service.spec.ts index b1450468..4e0d3a36 100644 --- a/backend/src/common/database/prisma.service.spec.ts +++ b/backend/src/common/database/prisma.service.spec.ts @@ -1,5 +1,6 @@ import type { TestingModule } from '@nestjs/testing' import { Test } from '@nestjs/testing' +import { databaseConfig } from 'src/config/database.config' import { PrismaService } from './prisma.service' describe('PrismaService', () => { @@ -20,4 +21,14 @@ describe('PrismaService', () => { it('should be defined', () => { expect(service).toBeDefined() }) + + it('configures pool onConnect to set search_path for all acquisition paths', async () => { + const query = vi.fn().mockResolvedValue({ rows: [], fields: [] }) + const onConnect = service.getPool().options.onConnect + + expect(onConnect).toBeDefined() + await onConnect!({ query } as never) + + expect(query).toHaveBeenCalledWith(`SET search_path TO ${databaseConfig.schema}`) + }) }) diff --git a/backend/src/common/database/prisma.service.ts b/backend/src/common/database/prisma.service.ts index 6c4b78d8..d874d7f5 100644 --- a/backend/src/common/database/prisma.service.ts +++ b/backend/src/common/database/prisma.service.ts @@ -21,8 +21,13 @@ class PrismaService } const pool = new Pool({ connectionString: databaseConfig.url, + // pg-pool awaits onConnect before handing out new clients. pool.query() + // (used by Prisma $executeRawUnsafe) acquires clients via connect(callback), + // so wrapping only the promise form of connect leaves search_path unset. + onConnect: async (client) => { + await client.query(`SET search_path TO ${databaseConfig.schema}`) + }, }) - PrismaService.wrapPoolConnectWithSearchPath(pool) const adapter = new PrismaPg(pool) super({ adapter, @@ -56,36 +61,6 @@ class PrismaService return this.pool } - /** - * Ensure every checked-out client has search_path set before first use. - * We wrap pool.connect instead of using pool "connect" event to avoid issuing - * a concurrent query on a client that may already be executing work. - * - * pool.connect has two call signatures: - * Promise form: pool.connect() → Promise - * Callback form: pool.connect(cb) → void - * - * The callback form returns undefined, so awaiting it yields undefined. - * We must detect which form is used and handle each accordingly. - * PrismaPg always uses the Promise form, so search_path is always set for - * Prisma-managed connections. - */ - private static wrapPoolConnectWithSearchPath(pool: Pool): void { - const originalConnect = pool.connect.bind(pool) as (...args: unknown[]) => unknown - - ;(pool as any).connect = function (...args: unknown[]) { - if (args.length > 0 && typeof args[0] === 'function') { - // Callback form: delegate unchanged so the caller's callback fires normally. - return originalConnect(...args) - } - // Promise form: intercept to set search_path before returning the client. - return (originalConnect() as Promise).then(async (client) => { - await client.query(`SET search_path TO ${databaseConfig.schema}`) - return client - }) - } - } - async onModuleDestroy() { await this.$disconnect() } From bcc2c9077ff3e9406c3edf340dfd20e5d2a6b15b Mon Sep 17 00:00:00 2001 From: Madjo Diapena Date: Thu, 25 Jun 2026 00:25:47 -0400 Subject: [PATCH 09/11] chore: merge test to dev (#379) - Pg Connection Fix - Cra enable CI --- .github/workflows/pr-validate.yml | 2 +- .../common/database/prisma.service.spec.ts | 11 ++++++ backend/src/common/database/prisma.service.ts | 37 +++---------------- 3 files changed, 18 insertions(+), 32 deletions(-) diff --git a/.github/workflows/pr-validate.yml b/.github/workflows/pr-validate.yml index 86fc70df..0f88023f 100644 --- a/.github/workflows/pr-validate.yml +++ b/.github/workflows/pr-validate.yml @@ -3,7 +3,7 @@ name: PR Validate on: pull_request: types: [edited, opened, synchronize, reopened, ready_for_review] - branches: [dev, test, pre-prod, main] + branches: [dev, test, pre-prod, main, release/cra-file-transfer-enabled] paths: - 'backend/**' - 'frontend/**' diff --git a/backend/src/common/database/prisma.service.spec.ts b/backend/src/common/database/prisma.service.spec.ts index b1450468..4e0d3a36 100644 --- a/backend/src/common/database/prisma.service.spec.ts +++ b/backend/src/common/database/prisma.service.spec.ts @@ -1,5 +1,6 @@ import type { TestingModule } from '@nestjs/testing' import { Test } from '@nestjs/testing' +import { databaseConfig } from 'src/config/database.config' import { PrismaService } from './prisma.service' describe('PrismaService', () => { @@ -20,4 +21,14 @@ describe('PrismaService', () => { it('should be defined', () => { expect(service).toBeDefined() }) + + it('configures pool onConnect to set search_path for all acquisition paths', async () => { + const query = vi.fn().mockResolvedValue({ rows: [], fields: [] }) + const onConnect = service.getPool().options.onConnect + + expect(onConnect).toBeDefined() + await onConnect!({ query } as never) + + expect(query).toHaveBeenCalledWith(`SET search_path TO ${databaseConfig.schema}`) + }) }) diff --git a/backend/src/common/database/prisma.service.ts b/backend/src/common/database/prisma.service.ts index 6c4b78d8..d874d7f5 100644 --- a/backend/src/common/database/prisma.service.ts +++ b/backend/src/common/database/prisma.service.ts @@ -21,8 +21,13 @@ class PrismaService } const pool = new Pool({ connectionString: databaseConfig.url, + // pg-pool awaits onConnect before handing out new clients. pool.query() + // (used by Prisma $executeRawUnsafe) acquires clients via connect(callback), + // so wrapping only the promise form of connect leaves search_path unset. + onConnect: async (client) => { + await client.query(`SET search_path TO ${databaseConfig.schema}`) + }, }) - PrismaService.wrapPoolConnectWithSearchPath(pool) const adapter = new PrismaPg(pool) super({ adapter, @@ -56,36 +61,6 @@ class PrismaService return this.pool } - /** - * Ensure every checked-out client has search_path set before first use. - * We wrap pool.connect instead of using pool "connect" event to avoid issuing - * a concurrent query on a client that may already be executing work. - * - * pool.connect has two call signatures: - * Promise form: pool.connect() → Promise - * Callback form: pool.connect(cb) → void - * - * The callback form returns undefined, so awaiting it yields undefined. - * We must detect which form is used and handle each accordingly. - * PrismaPg always uses the Promise form, so search_path is always set for - * Prisma-managed connections. - */ - private static wrapPoolConnectWithSearchPath(pool: Pool): void { - const originalConnect = pool.connect.bind(pool) as (...args: unknown[]) => unknown - - ;(pool as any).connect = function (...args: unknown[]) { - if (args.length > 0 && typeof args[0] === 'function') { - // Callback form: delegate unchanged so the caller's callback fires normally. - return originalConnect(...args) - } - // Promise form: intercept to set search_path before returning the client. - return (originalConnect() as Promise).then(async (client) => { - await client.query(`SET search_path TO ${databaseConfig.schema}`) - return client - }) - } - } - async onModuleDestroy() { await this.$disconnect() } From 3561782956206beda0a98197e086f7aa2dae75ff Mon Sep 17 00:00:00 2001 From: plakkara-bc Date: Thu, 25 Jun 2026 15:27:08 -0400 Subject: [PATCH 10/11] fix: fixed the cra status filter and sorting in the weekly file processing tab (#391) * updated mapping for cra status filter * Created backend api calls for sort * Created backend api calls for sort * enabled sort and filter on transformed values * fixed failing health checks * fixed failing health checks * updated mapping and normalization function * updated column filter labels * revert: restore health.controller.ts to dev --- .../api/weekly-files/dto/weekly-file.dto.ts | 2 +- .../weekly-files/weekly-file.mapper.spec.ts | 8 +- .../api/weekly-files/weekly-file.mapper.ts | 36 ++- .../weekly-files/weekly-files.controller.ts | 16 +- .../weekly-files/weekly-files.service.spec.ts | 249 +++++++++++++++++- .../api/weekly-files/weekly-files.service.ts | 229 ++++++++++++++-- .../components/WeeklyFileProcessingTab.tsx | 19 +- frontend/src/service/weekly-files-service.ts | 31 ++- 8 files changed, 548 insertions(+), 42 deletions(-) diff --git a/backend/src/api/weekly-files/dto/weekly-file.dto.ts b/backend/src/api/weekly-files/dto/weekly-file.dto.ts index 33d9bb57..36243015 100644 --- a/backend/src/api/weekly-files/dto/weekly-file.dto.ts +++ b/backend/src/api/weekly-files/dto/weekly-file.dto.ts @@ -46,7 +46,7 @@ export class WeeklyFileRecordDto { matchStatus: string @ApiProperty({ - description: 'Mapped display value: Application, Cancellation, CRA Update', + description: 'Mapped display value: Application, Cancellation, Update', }) transactionType: string diff --git a/backend/src/api/weekly-files/weekly-file.mapper.spec.ts b/backend/src/api/weekly-files/weekly-file.mapper.spec.ts index 76bcece9..90fd782f 100644 --- a/backend/src/api/weekly-files/weekly-file.mapper.spec.ts +++ b/backend/src/api/weekly-files/weekly-file.mapper.spec.ts @@ -128,7 +128,7 @@ describe('weekly-file.mapper', () => { batchDetail: null, }) - expect(dto.transactionType).toBe('CRA Update') + expect(dto.transactionType).toBe('Update') expect(dto.transactionSource).toBe('Other') expect(dto.gender).toBe('Unknown') expect(dto.birthCountry).toBe('Outside Canada') @@ -137,6 +137,7 @@ describe('weekly-file.mapper', () => { it('derives CRA status display labels from stored file values', () => { expect(toCraStatusDisplayLabel('in-progress')).toBe('IN PROGRESS') expect(toCraStatusDisplayLabel('completed')).toBe('COMPLETED') + expect(toCraStatusDisplayLabel(' ;in_progress*** ')).toBe('IN PROGRESS') }) it('whitelists stored transaction type filter values', () => { @@ -150,6 +151,11 @@ describe('weekly-file.mapper', () => { 'completed', 'in-progress', ]) + expect(filterAllowedCraStatuses([';COMPLETED', '__in progress__', '@@updated!!'])).toEqual([ + 'completed', + 'in-progress', + 'updated', + ]) expect(filterAllowedCraStatuses(['INVALID', 'completed'])).toEqual(['completed']) }) }) diff --git a/backend/src/api/weekly-files/weekly-file.mapper.ts b/backend/src/api/weekly-files/weekly-file.mapper.ts index 4b5d718e..e8eb3289 100644 --- a/backend/src/api/weekly-files/weekly-file.mapper.ts +++ b/backend/src/api/weekly-files/weekly-file.mapper.ts @@ -11,7 +11,7 @@ const WKL_STATUS_STORED_VALUES = Object.values(WKL_STATUS) const TRANSACTION_TYPE_LABELS: Record = { A: 'Application', C: 'Cancellation', - U: 'CRA Update', + U: 'Update', } const TRANSACTION_SOURCE_LABELS: Record = { @@ -30,9 +30,37 @@ const BIRTH_COUNTRY_LABELS: Record = { EX: 'Outside Canada', } +function normalizeCraStatusValue(value: string): string { + const tokens = value + .trim() + .toLowerCase() + .replace(/[^a-z0-9]+/g, ' ') + .trim() + .split(/\s+/) + .filter(Boolean) + + if (!tokens.length) return '' + + const compact = tokens.join('') + if (compact.startsWith('complete')) return 'completed' + if (compact.startsWith('abandon')) return 'abandoned' + if (compact.startsWith('inprogress')) return 'in-progress' + if (compact.startsWith('updated')) return 'updated' + + return tokens.join('-') +} + +export function normalizeCraStatusLabel(value: string): string { + return toCraStatusDisplayLabel(normalizeCraStatusValue(value)) +} + /** Display label for a stored CRA status (e.g. "in-progress" → "IN PROGRESS"). */ export function toCraStatusDisplayLabel(stored: string): string { - return stored.trim().toUpperCase().replace(/-/g, ' ') + return stored + .trim() + .replace(/[^a-zA-Z0-9]+/g, ' ') + .trim() + .toUpperCase() } const TRANSACTION_TYPE_CODES = Object.keys(TRANSACTION_TYPE_LABELS) @@ -46,7 +74,7 @@ export function filterAllowedTransactionTypes(values: string[]): string[] { /** Whitelist filter params to values stored in cra_status. */ export function filterAllowedCraStatuses(values: string[]): string[] { const allowed = new Set(WKL_STATUS_STORED_VALUES) - return [...new Set(values.map((v) => v.trim().toLowerCase()).filter((v) => allowed.has(v)))] + return [...new Set(values.map(normalizeCraStatusValue).filter((v) => allowed.has(v)))] } export interface WeeklyFileCounts { @@ -188,7 +216,7 @@ function formatWklDateString(value: string | undefined): string | null { function formatCraStatus(status: string | undefined): string { if (!status?.trim()) return '' - return toCraStatusDisplayLabel(status) + return normalizeCraStatusLabel(status) } function formatTransactionType(value: string | undefined): string { diff --git a/backend/src/api/weekly-files/weekly-files.controller.ts b/backend/src/api/weekly-files/weekly-files.controller.ts index 270a5762..8ca13458 100644 --- a/backend/src/api/weekly-files/weekly-files.controller.ts +++ b/backend/src/api/weekly-files/weekly-files.controller.ts @@ -51,20 +51,21 @@ export class WeeklyFilesController { name: 'csaMatchFound', required: false, type: String, - description: 'Comma-separated filter values for CSA Match Found: "Yes" and/or "No"', + description: 'Comma-separated filter values for CSA Match Found: "Yes", "No", and/or "N/A"', }) @ApiQuery({ name: 'transactionType', required: false, type: String, - description: 'Comma-separated stored transaction type codes: "A", "C", "U"', + description: + 'Comma-separated normalized transaction type labels: "Application", "Cancellation", "Update"', }) @ApiQuery({ name: 'craStatus', required: false, type: String, description: - 'Comma-separated stored CRA status values: "completed", "abandoned", "in-progress", "updated"', + 'Comma-separated normalized CRA status labels: "COMPLETED", "ABANDONED", "IN PROGRESS", "UPDATED"', }) @ApiQuery({ name: 'matchedBy', @@ -84,6 +85,13 @@ export class WeeklyFilesController { type: String, description: 'Text filter for Transaction Source (minimum 3 characters)', }) + @ApiQuery({ + name: 'sort', + required: false, + type: String, + description: + 'JSON array of sort objects: [{"field":"asc|desc"}]. Example: [{"craStatus":"asc"}]. Allowed fields: csaMatchFound, matchedBy, batchNumber, transactionType, transactionSource, craStatus', + }) @ApiResponse({ status: 200, description: 'Paginated detail records for a weekly file' }) @ApiResponse({ status: 404, description: 'Weekly file not found' }) findRecords( @@ -96,6 +104,7 @@ export class WeeklyFilesController { @Query('matchedBy') matchedBy?: string, @Query('batchNumber') batchNumber?: string, @Query('transactionSource') transactionSource?: string, + @Query('sort') sort?: string, ): Promise> { const filters = { csaMatchFound: csaMatchFound @@ -125,6 +134,7 @@ export class WeeklyFilesController { this.parsePage(page), this.parseLimit(limit), filters, + sort, ) } diff --git a/backend/src/api/weekly-files/weekly-files.service.spec.ts b/backend/src/api/weekly-files/weekly-files.service.spec.ts index ca97b7e7..07777748 100644 --- a/backend/src/api/weekly-files/weekly-files.service.spec.ts +++ b/backend/src/api/weekly-files/weekly-files.service.spec.ts @@ -1,6 +1,6 @@ import { BadRequestException, NotFoundException } from '@nestjs/common' -import { beforeEach, describe, expect, it, vi } from 'vitest' import { CRA_DATA_HANDLING_CONSTANT } from 'src/cra/cra.constant' +import { beforeEach, describe, expect, it, vi } from 'vitest' import { WeeklyFilesService } from './weekly-files.service' const { WKL_MATCH_STATUS } = CRA_DATA_HANDLING_CONSTANT @@ -175,8 +175,6 @@ describe('WeeklyFilesService', () => { contact: null, }, ]) - mockPrisma.wklFileRecord.count.mockResolvedValue(1) - const result = await service.findRecords(1, 1, 10) expect(result.total).toBe(1) @@ -189,6 +187,251 @@ describe('WeeklyFilesService', () => { }) }) + it('filters detail records using normalized display labels', async () => { + mockPrisma.transferFile.findFirst.mockResolvedValue({ id: 1 }) + mockPrisma.wklFileRecord.findMany.mockResolvedValue([ + { + id: 5, + recordIndex: 0, + matchStatus: 'matched', + matchedBy: 'SYSTEM', + processedAt: null, + recordData: { + ...electronicRecordData, + transactionType: 'U', + status: ';COMPLETED', + }, + contact: null, + batchDetail: { batch: { batchNumber: 1042 } }, + }, + { + id: 6, + recordIndex: 1, + matchStatus: 'matched', + matchedBy: 'SYSTEM', + processedAt: null, + recordData: { + ...electronicRecordData, + transactionType: 'C', + status: 'abandoned', + }, + contact: null, + batchDetail: { batch: { batchNumber: 1043 } }, + }, + ]) + + const result = await service.findRecords(1, 1, 10, { + transactionType: ['Update'], + craStatus: ['COMPLETED'], + }) + + expect(result.total).toBe(1) + expect(result.data).toHaveLength(1) + expect(result.data[0]).toMatchObject({ + id: 5, + transactionType: 'Update', + craStatus: 'COMPLETED', + }) + }) + + it('filters N/A csa match found values', async () => { + mockPrisma.transferFile.findFirst.mockResolvedValue({ id: 1 }) + mockPrisma.wklFileRecord.findMany.mockResolvedValue([ + { + id: 5, + recordIndex: 0, + matchStatus: 'matched', + matchedBy: 'SYSTEM', + processedAt: null, + recordData: { + ...electronicRecordData, + status: 'completed', + }, + contact: null, + batchDetail: { batch: { batchNumber: 1042 } }, + }, + { + id: 6, + recordIndex: 1, + matchStatus: 'skipped', + matchedBy: null, + processedAt: null, + recordData: { + ...electronicRecordData, + status: 'completed', + }, + contact: null, + batchDetail: null, + }, + ]) + + const result = await service.findRecords(1, 1, 10, { + csaMatchFound: ['N/A'], + }) + + expect(result.total).toBe(1) + expect(result.data).toHaveLength(1) + expect(result.data[0]).toMatchObject({ + id: 6, + csaMatchFound: 'N/A', + }) + }) + + it('filters IN PROGRESS cra status using normalized display label', async () => { + mockPrisma.transferFile.findFirst.mockResolvedValue({ id: 1 }) + mockPrisma.wklFileRecord.findMany.mockResolvedValue([ + { + id: 5, + recordIndex: 0, + matchStatus: 'matched', + matchedBy: 'SYSTEM', + processedAt: null, + recordData: { + ...electronicRecordData, + status: 'IN PROGRESS', + }, + contact: null, + batchDetail: { batch: { batchNumber: 1042 } }, + }, + { + id: 6, + recordIndex: 1, + matchStatus: 'matched', + matchedBy: 'SYSTEM', + processedAt: null, + recordData: { + ...electronicRecordData, + status: ';inprogress', + }, + contact: null, + batchDetail: { batch: { batchNumber: 1043 } }, + }, + { + id: 7, + recordIndex: 2, + matchStatus: 'matched', + matchedBy: 'SYSTEM', + processedAt: null, + recordData: { + ...electronicRecordData, + status: 'completed', + }, + contact: null, + batchDetail: { batch: { batchNumber: 1044 } }, + }, + ]) + + const result = await service.findRecords(1, 1, 10, { + craStatus: ['IN PROGRESS'], + }) + + expect(result.total).toBe(2) + expect(result.data.map((record) => record.id)).toEqual([5, 6]) + expect(result.data.every((record) => record.craStatus === 'IN PROGRESS')).toBe(true) + }) + + it('filters COMPLETED cra status and includes ;COMPLETED while excluding other statuses', async () => { + mockPrisma.transferFile.findFirst.mockResolvedValue({ id: 1 }) + mockPrisma.wklFileRecord.findMany.mockResolvedValue([ + { + id: 8, + recordIndex: 0, + matchStatus: 'matched', + matchedBy: 'SYSTEM', + processedAt: null, + recordData: { + ...electronicRecordData, + status: ';COMPLETED', + }, + contact: null, + batchDetail: { batch: { batchNumber: 2001 } }, + }, + { + id: 9, + recordIndex: 1, + matchStatus: 'matched', + matchedBy: 'SYSTEM', + processedAt: null, + recordData: { + ...electronicRecordData, + status: 'complete', + }, + contact: null, + batchDetail: { batch: { batchNumber: 2002 } }, + }, + { + id: 10, + recordIndex: 2, + matchStatus: 'matched', + matchedBy: 'SYSTEM', + processedAt: null, + recordData: { + ...electronicRecordData, + status: 'IN PROGRESS', + }, + contact: null, + batchDetail: { batch: { batchNumber: 2003 } }, + }, + { + id: 11, + recordIndex: 3, + matchStatus: 'matched', + matchedBy: 'SYSTEM', + processedAt: null, + recordData: { + ...electronicRecordData, + status: 'abandoned', + }, + contact: null, + batchDetail: { batch: { batchNumber: 2004 } }, + }, + ]) + + const result = await service.findRecords(1, 1, 10, { + craStatus: ['COMPLETED'], + }) + + expect(result.total).toBe(2) + expect(result.data.map((record) => record.id)).toEqual([8, 9]) + expect(result.data.every((record) => record.craStatus === 'COMPLETED')).toBe(true) + }) + + it('sorts detail records using normalized CRA status labels', async () => { + mockPrisma.transferFile.findFirst.mockResolvedValue({ id: 1 }) + mockPrisma.wklFileRecord.findMany.mockResolvedValue([ + { + id: 5, + recordIndex: 0, + matchStatus: 'matched', + matchedBy: 'SYSTEM', + processedAt: null, + recordData: { + ...electronicRecordData, + status: 'completed', + }, + contact: null, + batchDetail: { batch: { batchNumber: 1042 } }, + }, + { + id: 6, + recordIndex: 1, + matchStatus: 'matched', + matchedBy: 'SYSTEM', + processedAt: null, + recordData: { + ...electronicRecordData, + status: ';ABANDONED', + }, + contact: null, + batchDetail: { batch: { batchNumber: 1043 } }, + }, + ]) + + const result = await service.findRecords(1, 1, 10, undefined, '[{"craStatus":"asc"}]') + + expect(result.data.map((record) => record.craStatus)).toEqual(['ABANDONED', 'COMPLETED']) + }) + describe('associateRecord', () => { it('associates an unmatched electronic record with a contact', async () => { mockPrisma.transferFile.findFirst.mockResolvedValue({ id: 1 }) diff --git a/backend/src/api/weekly-files/weekly-files.service.ts b/backend/src/api/weekly-files/weekly-files.service.ts index 495b2e1c..fe03a649 100644 --- a/backend/src/api/weekly-files/weekly-files.service.ts +++ b/backend/src/api/weekly-files/weekly-files.service.ts @@ -12,10 +12,10 @@ import type { ReprocessWeeklyFileResultDto } from './dto/associate-wkl-record.dt import type { WeeklyFileRecordDto, WeeklyFileSummaryDto } from './dto/weekly-file.dto' import { aggregateWeeklyFileCounts, + normalizeCraStatusLabel, toWeeklyFileRecordDto, toWeeklyFileSummaryDto, } from './weekly-file.mapper' -import { buildWklRecordWhereInput } from './weekly-file-record-filters' import { assertCanAssociate, assertCanDissociate, @@ -25,17 +25,213 @@ import { const { FILE_DIRECTION, FILE_TYPE, WKL_MATCH_STATUS } = CRA_DATA_HANDLING_CONSTANT export interface WeeklyFileRecordFilters { - /** Semantic filter: "Yes" or "No" (maps to match_status groups). */ + /** Semantic filter: "Yes", "No", or "N/A" (maps to match_status groups). */ csaMatchFound?: string[] - /** Stored transaction_type codes: A, C, U. */ + /** Normalized display labels: Application, Cancellation, Update. */ transactionType?: string[] - /** Stored cra_status values: completed, in-progress, abandoned, updated. */ + /** Normalized display labels: COMPLETED, ABANDONED, IN PROGRESS, UPDATED. */ craStatus?: string[] matchedBy?: string batchNumber?: string transactionSource?: string } +type WeeklyFileRecordSortColumn = + | 'csaMatchFound' + | 'matchedBy' + | 'batchNumber' + | 'transactionType' + | 'transactionSource' + | 'craStatus' + +type WeeklyFileRecordSort = { + column: WeeklyFileRecordSortColumn + direction: 'asc' | 'desc' +} + +const TRANSACTION_TYPE_FILTER_LABELS: Record = { + A: 'Application', + APPLICATION: 'Application', + C: 'Cancellation', + CANCELLATION: 'Cancellation', + U: 'Update', + UPDATE: 'Update', + 'CRA UPDATE': 'Update', +} + +const ALLOWED_WEEKLY_RECORD_SORT_COLUMNS: WeeklyFileRecordSortColumn[] = [ + 'csaMatchFound', + 'matchedBy', + 'batchNumber', + 'transactionType', + 'transactionSource', + 'craStatus', +] + +function normalizeTransactionTypeLabel(value: string): string { + const normalized = value.trim().toUpperCase() + return TRANSACTION_TYPE_FILTER_LABELS[normalized] ?? value.trim() +} + +function normalizeCsaMatchFoundLabel(value: string): string { + const normalized = value.trim().toLowerCase() + if (normalized === 'yes') return 'Yes' + if (normalized === 'no') return 'No' + if (normalized === 'n/a') return 'N/A' + return value.trim() +} + +function parseWeeklyFileRecordSort(sort?: string): WeeklyFileRecordSort | null { + if (!sort) { + return null + } + + let sortArray: unknown + try { + sortArray = JSON.parse(sort) + } catch { + throw new BadRequestException('Invalid JSON format for sort parameter') + } + + if (!Array.isArray(sortArray) || sortArray.length === 0) { + return null + } + + const sortItem = sortArray[0] + if (!sortItem || typeof sortItem !== 'object' || Array.isArray(sortItem)) { + throw new BadRequestException('Sort parameter must contain objects with field directions') + } + + const field = Object.keys(sortItem)[0] as WeeklyFileRecordSortColumn | undefined + const direction = + field && field in sortItem + ? (sortItem[field as keyof typeof sortItem] as 'asc' | 'desc') + : undefined + + if (!field || !ALLOWED_WEEKLY_RECORD_SORT_COLUMNS.includes(field)) { + throw new BadRequestException( + `Invalid sort field: ${field}. Allowed fields: ${ALLOWED_WEEKLY_RECORD_SORT_COLUMNS.join(', ')}`, + ) + } + + if (direction !== 'asc' && direction !== 'desc') { + throw new BadRequestException(`Invalid sort direction: ${direction}. Allowed values: asc, desc`) + } + + return { column: field, direction } +} + +function matchesTextFilter( + value: string | null | undefined, + term: string | undefined, + minLength: number, +) { + const normalizedTerm = term?.trim().toLowerCase() ?? '' + if (!normalizedTerm || normalizedTerm.length < minLength) { + return true + } + + return (value ?? '').toLowerCase().includes(normalizedTerm) +} + +function applyWeeklyFileRecordFilters( + records: WeeklyFileRecordDto[], + filters?: WeeklyFileRecordFilters, +): WeeklyFileRecordDto[] { + const csaMatchFound = new Set((filters?.csaMatchFound ?? []).map(normalizeCsaMatchFoundLabel)) + const transactionTypes = new Set( + (filters?.transactionType ?? []).map(normalizeTransactionTypeLabel).filter(Boolean), + ) + const craStatuses = new Set( + (filters?.craStatus ?? []).map(normalizeCraStatusLabel).filter(Boolean), + ) + const batchNumberTerm = filters?.batchNumber?.trim().toLowerCase() ?? '' + + return records.filter((record) => { + if (csaMatchFound.size > 0 && !csaMatchFound.has(record.csaMatchFound)) { + return false + } + + if ( + transactionTypes.size > 0 && + !transactionTypes.has(normalizeTransactionTypeLabel(record.transactionType)) + ) { + return false + } + + if (craStatuses.size > 0 && !craStatuses.has(normalizeCraStatusLabel(record.craStatus))) { + return false + } + + if (!matchesTextFilter(record.matchedBy, filters?.matchedBy, 3)) { + return false + } + + if (!matchesTextFilter(record.transactionSource, filters?.transactionSource, 3)) { + return false + } + + if (batchNumberTerm && batchNumberTerm.length >= 1) { + const batchNumber = record.batchNumber?.toString() ?? '' + if (!batchNumber.toLowerCase().includes(batchNumberTerm)) { + return false + } + } + + return true + }) +} + +function getWeeklyFileRecordSortValue( + record: WeeklyFileRecordDto, + column: WeeklyFileRecordSortColumn, +) { + switch (column) { + case 'batchNumber': + return record.batchNumber ?? Number.POSITIVE_INFINITY + case 'csaMatchFound': + return record.csaMatchFound + case 'matchedBy': + return record.matchedBy ?? '' + case 'transactionType': + return record.transactionType + case 'transactionSource': + return record.transactionSource + case 'craStatus': + return record.craStatus + } +} + +function compareWeeklyFileRecordSortValues(left: string | number, right: string | number): number { + if (typeof left === 'number' && typeof right === 'number') { + return left - right + } + + return String(left).localeCompare(String(right), undefined, { sensitivity: 'base' }) +} + +function sortWeeklyFileRecords( + records: WeeklyFileRecordDto[], + sort: WeeklyFileRecordSort | null, +): WeeklyFileRecordDto[] { + if (!sort) { + return records + } + + return [...records].sort((left, right) => { + const comparison = compareWeeklyFileRecordSortValues( + getWeeklyFileRecordSortValue(left, sort.column), + getWeeklyFileRecordSortValue(right, sort.column), + ) + + if (comparison !== 0) { + return sort.direction === 'asc' ? comparison : -comparison + } + + return left.recordIndex - right.recordIndex + }) +} + const weeklyFileWhere = { fileType: FILE_TYPE.WKL, direction: FILE_DIRECTION.INBOUND, @@ -155,6 +351,7 @@ export class WeeklyFilesService { page = 1, limit = 10, filters?: WeeklyFileRecordFilters, + sort?: string, ): Promise> { await this.assertWeeklyFileExists(id) @@ -162,21 +359,21 @@ export class WeeklyFilesService { const safeLimit = limit >= 1 ? Math.min(limit, 200) : 10 const offset = (safePage - 1) * safeLimit - const where = await buildWklRecordWhereInput(this.prisma, id, filters) + const parsedSort = parseWeeklyFileRecordSort(sort) + const recordsWithRelations = await this.prisma.wklFileRecord.findMany({ + where: { transferFileId: id }, + orderBy: { recordIndex: 'asc' }, + include: wklRecordDtoInclude, + }) - const [total, recordsWithRelations] = await Promise.all([ - this.prisma.wklFileRecord.count({ where }), - this.prisma.wklFileRecord.findMany({ - where, - orderBy: { recordIndex: 'asc' }, - skip: offset, - take: safeLimit, - include: wklRecordDtoInclude, - }), - ]) + const normalizedRecords = recordsWithRelations.map(toWeeklyFileRecordDto) + const filteredRecords = applyWeeklyFileRecordFilters(normalizedRecords, filters) + const sortedRecords = sortWeeklyFileRecords(filteredRecords, parsedSort) + const paginatedRecords = sortedRecords.slice(offset, offset + safeLimit) + const total = sortedRecords.length return { - data: recordsWithRelations.map(toWeeklyFileRecordDto), + data: paginatedRecords, page: safePage, limit: safeLimit, total, diff --git a/frontend/src/components/WeeklyFileProcessingTab.tsx b/frontend/src/components/WeeklyFileProcessingTab.tsx index 41cdd38a..30056a2b 100644 --- a/frontend/src/components/WeeklyFileProcessingTab.tsx +++ b/frontend/src/components/WeeklyFileProcessingTab.tsx @@ -120,6 +120,15 @@ const CHILD_SEARCH_COLUMN_LABELS: Record = { birthPlace: 'Birth Place', } +const WEEKLY_DETAILS_COLUMN_LABELS: Record = { + csaMatchFound: 'CSA Match Found?', + matchedBy: 'Matched By', + batchNumber: 'Batch Req ID', + transactionType: 'Transaction Type', + transactionSource: 'Transaction Source', + craStatus: 'CRA Status', +} + type SortConfig = { column: T direction: SortDirection @@ -426,6 +435,9 @@ export default function WeeklyFileProcessingTab() { matchedBy: detailsBackendTextFilters.matchedBy, batchNumber: detailsBackendTextFilters.batchNumber, transactionSource: detailsBackendTextFilters.transactionSource, + sort: detailsSortConfig + ? [{ [detailsSortConfig.column]: detailsSortConfig.direction }] + : undefined, }, ) setRecords(response.data) @@ -459,6 +471,7 @@ export default function WeeklyFileProcessingTab() { detailsBackendTextFilters.matchedBy, detailsBackendTextFilters.batchNumber, detailsBackendTextFilters.transactionSource, + detailsSortConfig, ]) useEffect(() => { @@ -599,7 +612,7 @@ export default function WeeklyFileProcessingTab() { getDetailsFieldValue, '', clientColumnFilters, - detailsSortConfig, + null, // Server-side sorting is now applied - don't sort client-side ) if (!detailsShowSelectedOnly) { @@ -611,7 +624,7 @@ export default function WeeklyFileProcessingTab() { } return recordsAfterSearchFilterSort.filter((record) => record.id === selectedRecordId) - }, [records, detailsColumnFilters, detailsSortConfig, detailsShowSelectedOnly, selectedRecordId]) + }, [records, detailsColumnFilters, detailsShowSelectedOnly, selectedRecordId]) const filteredSearchedChildren = useMemo(() => { return filterAndSortRows( @@ -1720,7 +1733,7 @@ export default function WeeklyFileProcessingTab() { sx={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center', mb: 1 }} > - Filter by {detailsFilterAnchor.column} + Filter by {WEEKLY_DETAILS_COLUMN_LABELS[detailsFilterAnchor.column]}