From a3258b11ae06641dc9b5534dce639aab80711238 Mon Sep 17 00:00:00 2001 From: Samer Melhem Date: Wed, 8 Jul 2026 16:15:18 +0300 Subject: [PATCH 1/7] FINERACT-2682: Fix bulk client upload failure when custom datatables are configured Ports the fix from an internal fork (foohelpdesk CBS-1: "Bulk Client Upload fails when additional (custom) fields are configured"). Bulk client import via Excel previously failed once an Entity Datatable Check made a custom datatable mandatory for client creation, because the import handlers had no way to validate, populate, or parse datatable columns in the workbook. This adds that support end-to-end: - ClientEntityWorkbookPopulator / ClientPersonWorkbookPopulator now render columns for all datatables configured against the client entity (including empty ones), with dropdown, date, datetime and boolean column support, and mark single-row datatable columns as mandatory via a "*" suffix. - ImportHandlerUtils gains parsing/validation helpers for datatable columns, and validateRequiredDatatables() now skips multi-row (repeatable child) datatables since they are optional, using the new DatatableData.isMultiRow() flag. - ClientEntityImportHandler / ClientPersonImportHandler validate and submit datatable data as part of client creation, with clearer error reporting on failure. - DatatableData / DatatableReadServiceImpl / ReadSurveyServiceImpl carry the multi-row flag through DatatableData.create(...). Includes unit tests for single-row vs multi-row DatatableData handling. --- .../dataqueries/data/DatatableData.java | 17 +- .../data/ResultsetColumnHeaderData.java | 4 + .../data/ResultsetColumnValueData.java | 8 + .../portfolio/search/service/SearchUtil.java | 45 +- .../dataqueries/data/DatatableDataTest.java | 89 +++ .../constants/ClientEntityConstants.java | 42 +- .../constants/ClientPersonConstants.java | 38 +- .../TemplatePopulateImportConstants.java | 1 + .../importhandler/ImportHandlerUtils.java | 734 ++++++++++++++++++ .../client/ClientEntityImportHandler.java | 111 ++- .../client/ClientPersonImportHandler.java | 34 + .../client/ClientEntityWorkbookPopulator.java | 584 ++++++++++++-- .../client/ClientPersonWorkbookPopulator.java | 541 +++++++++++-- ...ulkImportWorkbookPopulatorServiceImpl.java | 62 +- .../service/DatatableReadServiceImpl.java | 6 +- .../survey/service/ReadSurveyServiceImpl.java | 8 +- .../service/ReadSurveyServiceImplTest.java | 6 +- 17 files changed, 2173 insertions(+), 157 deletions(-) create mode 100644 fineract-core/src/test/java/org/apache/fineract/infrastructure/dataqueries/data/DatatableDataTest.java diff --git a/fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/DatatableData.java b/fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/DatatableData.java index 8e22cbbca4e..f0d0f4f996e 100644 --- a/fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/DatatableData.java +++ b/fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/DatatableData.java @@ -34,18 +34,21 @@ public final class DatatableData implements Serializable { private final String entitySubType; @SuppressWarnings("unused") private final List columnHeaderData; + @SuppressWarnings("unused") + private final boolean multiRow; public static DatatableData create(final String applicationTableName, final String registeredTableName, final String entitySubType, - final List columnHeaderData) { - return new DatatableData(applicationTableName, registeredTableName, entitySubType, columnHeaderData); + final List columnHeaderData, final boolean multiRow) { + return new DatatableData(applicationTableName, registeredTableName, entitySubType, columnHeaderData, multiRow); } private DatatableData(final String applicationTableName, final String registeredTableName, final String entitySubType, - final List columnHeaderData) { + final List columnHeaderData, final boolean multiRow) { this.applicationTableName = applicationTableName; this.registeredTableName = registeredTableName; this.entitySubType = entitySubType; this.columnHeaderData = columnHeaderData; + this.multiRow = multiRow; } @@ -65,4 +68,12 @@ public String getRegisteredTableName() { return registeredTableName; } + public List getColumnHeaderData() { + return columnHeaderData; + } + + public boolean isMultiRow() { + return multiRow; + } + } diff --git a/fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/ResultsetColumnHeaderData.java b/fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/ResultsetColumnHeaderData.java index c196954c6a9..592a39fe316 100644 --- a/fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/ResultsetColumnHeaderData.java +++ b/fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/ResultsetColumnHeaderData.java @@ -149,6 +149,10 @@ public boolean hasColumnValues() { return columnValues != null && !columnValues.isEmpty(); } + public List getColumnValues() { + return columnValues; + } + public boolean isColumnValueAllowed(final String match) { for (final ResultsetColumnValueData allowedValue : this.columnValues) { if (allowedValue.matches(match)) { diff --git a/fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/ResultsetColumnValueData.java b/fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/ResultsetColumnValueData.java index 9a9e8a10074..b0436c68a72 100644 --- a/fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/ResultsetColumnValueData.java +++ b/fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/ResultsetColumnValueData.java @@ -49,4 +49,12 @@ public boolean matches(final String match) { public boolean codeMatches(final Integer match) { return match.intValue() == this.id; } + + public int getId() { + return id; + } + + public String getValue() { + return value; + } } diff --git a/fineract-core/src/main/java/org/apache/fineract/portfolio/search/service/SearchUtil.java b/fineract-core/src/main/java/org/apache/fineract/portfolio/search/service/SearchUtil.java index 0107308eb6a..0c5b1ef6b11 100644 --- a/fineract-core/src/main/java/org/apache/fineract/portfolio/search/service/SearchUtil.java +++ b/fineract-core/src/main/java/org/apache/fineract/portfolio/search/service/SearchUtil.java @@ -271,7 +271,15 @@ public Object parseColumnValue(@NonNull ResultsetColumnHeaderData columnHeader, } return columnValue; } else if (columnHeader.isCodeLookupDisplayType()) { - final Integer codeLookup = Integer.valueOf(columnValue); + // Extract ID from display value format: "Label (ID)" if present + String extractedId = extractIdFromDisplayValue(columnValue); + Integer codeLookup; + try { + codeLookup = Integer.valueOf(extractedId); + } catch (NumberFormatException e) { + // If extraction didn't change the value or parsing failed, try original value + codeLookup = Integer.valueOf(columnValue); + } if (!columnHeader.isColumnCodeAllowed(codeLookup)) { ApiParameterError error = ApiParameterError.parameterError("error.msg.invalid.columnValue", "Value not found in Allowed Value list", columnHeader.getColumnName(), columnValue); @@ -293,6 +301,17 @@ public Object parseColumnValue(@NonNull ResultsetColumnHeaderData columnHeader, return JsonParserHelper.convertDateTimeFrom(columnValue, columnHeader.getColumnName(), format, locale); } if (colType.isAnyIntegerType()) { + // Extract ID from display value format: "Label (ID)" if present + // This handles FK/INTEGER columns that show human-readable values in Excel + String extractedId = extractIdFromDisplayValue(columnValue); + if (!extractedId.equals(columnValue)) { + // ID was extracted, try to parse it + try { + return Integer.parseInt(extractedId); + } catch (NumberFormatException e) { + // If parsing fails, fall back to original conversion + } + } return helper.convertToInteger(columnValue, columnHeader.getColumnName(), locale); } if (colType.isDecimalType()) { @@ -323,4 +342,28 @@ public String camelToSnake(final String camelStr) { return camelStr == null ? null : camelStr.replaceAll("([A-Z]+)([A-Z][a-z])", "$1_$2").replaceAll("([a-z])([A-Z])", "$1_$2").toLowerCase(); } + + /** + * Extracts numeric ID from strings ending with "(id)" format. + * Example: "Permanent (32)" -> "32" + * If the string doesn't match the pattern, returns the original value. + * + * @param value The string value that may contain an ID in parentheses + * @return The extracted ID as a string, or the original value if no ID pattern is found + */ + private static String extractIdFromDisplayValue(String value) { + if (value == null || value.trim().isEmpty()) { + return value; + } + // Match pattern: ".*(\d+)$" where the number is in parentheses at the end + String trimmed = value.trim(); + int lastParenIndex = trimmed.lastIndexOf('('); + if (lastParenIndex > 0 && trimmed.endsWith(")")) { + String idPart = trimmed.substring(lastParenIndex + 1, trimmed.length() - 1); + if (idPart.matches("\\d+")) { + return idPart; + } + } + return value; + } } diff --git a/fineract-core/src/test/java/org/apache/fineract/infrastructure/dataqueries/data/DatatableDataTest.java b/fineract-core/src/test/java/org/apache/fineract/infrastructure/dataqueries/data/DatatableDataTest.java new file mode 100644 index 00000000000..25f91698d55 --- /dev/null +++ b/fineract-core/src/test/java/org/apache/fineract/infrastructure/dataqueries/data/DatatableDataTest.java @@ -0,0 +1,89 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed on an "AS IS" BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.fineract.infrastructure.dataqueries.data; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.List; +import org.apache.fineract.infrastructure.core.service.database.DatabaseType; +import org.junit.jupiter.api.Test; + +class DatatableDataTest { + + @Test + void testSingleRowDatatable() { + // Given: A single-row datatable (no "id" column) + String appTableName = "m_client"; + String registeredTableName = "extra_client_details"; + String entitySubType = null; + List columnHeaders = new ArrayList<>(); + // Single-row datatable has client_id as primary key, no "id" column + columnHeaders.add(ResultsetColumnHeaderData.basic("client_id", "bigint", DatabaseType.POSTGRESQL)); + columnHeaders.add(ResultsetColumnHeaderData.basic("field1", "text", DatabaseType.POSTGRESQL)); + boolean multiRow = false; + + // When: Creating DatatableData + DatatableData datatableData = DatatableData.create(appTableName, registeredTableName, entitySubType, + columnHeaders, multiRow); + + // Then: It should be identified as single-row + assertFalse(datatableData.isMultiRow(), "Single-row datatable should return false for isMultiRow()"); + assertEquals(registeredTableName, datatableData.getRegisteredTableName()); + assertEquals(columnHeaders, datatableData.getColumnHeaderData()); + } + + @Test + void testMultiRowDatatable() { + // Given: A multi-row datatable (has "id" column) + String appTableName = "m_client"; + String registeredTableName = "extra_family_details"; + String entitySubType = null; + List columnHeaders = new ArrayList<>(); + // Multi-row datatable has "id" as primary key + columnHeaders.add(ResultsetColumnHeaderData.basic("id", "bigint", DatabaseType.POSTGRESQL)); + columnHeaders.add(ResultsetColumnHeaderData.basic("client_id", "bigint", DatabaseType.POSTGRESQL)); + columnHeaders.add(ResultsetColumnHeaderData.basic("field1", "text", DatabaseType.POSTGRESQL)); + boolean multiRow = true; + + // When: Creating DatatableData + DatatableData datatableData = DatatableData.create(appTableName, registeredTableName, entitySubType, + columnHeaders, multiRow); + + // Then: It should be identified as multi-row + assertTrue(datatableData.isMultiRow(), "Multi-row datatable should return true for isMultiRow()"); + assertEquals(registeredTableName, datatableData.getRegisteredTableName()); + assertEquals(columnHeaders, datatableData.getColumnHeaderData()); + } + + @Test + void testHasColumn() { + // Given: A datatable with columns + List columnHeaders = new ArrayList<>(); + columnHeaders.add(ResultsetColumnHeaderData.basic("client_id", "bigint", DatabaseType.POSTGRESQL)); + columnHeaders.add(ResultsetColumnHeaderData.basic("field1", "text", DatabaseType.POSTGRESQL)); + DatatableData datatableData = DatatableData.create("m_client", "test_table", null, columnHeaders, false); + + // When/Then: Checking for column existence + assertTrue(datatableData.hasColumn("client_id"), "Should find existing column"); + assertTrue(datatableData.hasColumn("field1"), "Should find existing column"); + assertFalse(datatableData.hasColumn("nonexistent"), "Should not find non-existent column"); + } +} diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/constants/ClientEntityConstants.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/constants/ClientEntityConstants.java index a06eac907f5..73c1e772c29 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/constants/ClientEntityConstants.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/constants/ClientEntityConstants.java @@ -51,15 +51,55 @@ private ClientEntityConstants() { public static final int COUNTRY_COL = 24;// Y public static final int POSTAL_CODE_COL = 25;// Z public static final int IS_ACTIVE_ADDRESS_COL = 26;// AA - public static final int WARNING_COL = 26;// AA public static final int STATUS_COL = 27;// AB + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated + public static final int WARNING_COL = 26;// AA + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated public static final int RELATIONAL_OFFICE_NAME_COL = 35;// AJ + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated public static final int RELATIONAL_OFFICE_OPENING_DATE_COL = 36;// AK + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated public static final int LOOKUP_CONSTITUTION_COL = 37;// AL + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated public static final int LOOKUP_CLIENT_CLASSIFICATION = 38;// AM + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated public static final int LOOKUP_CLIENT_TYPES = 39;// AN + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated public static final int LOOKUP_ADDRESS_TYPE = 40;// AO + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated public static final int LOOKUP_STATE_PROVINCE = 41;// AP + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated public static final int LOOKUP_COUNTRY = 42;// AQ + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated public static final int LOOKUP_MAIN_BUSINESS_LINE = 43;// AR } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/constants/ClientPersonConstants.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/constants/ClientPersonConstants.java index ed4a6fb7bd4..12110b06976 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/constants/ClientPersonConstants.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/constants/ClientPersonConstants.java @@ -50,14 +50,50 @@ private ClientPersonConstants() { public static final int COUNTRY_COL = 23;// X public static final int POSTAL_CODE_COL = 24;// Y public static final int IS_ACTIVE_ADDRESS_COL = 25;// Z - public static final int WARNING_COL = 26;// AA public static final int STATUS_COL = 27;// AB + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated + public static final int WARNING_COL = 26;// AA + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated public static final int RELATIONAL_OFFICE_NAME_COL = 35;// AJ + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated public static final int RELATIONAL_OFFICE_OPENING_DATE_COL = 36;// AK + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated public static final int LOOKUP_GENDER_COL = 37;// AL + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated public static final int LOOKUP_CLIENT_CLASSIFICATION_COL = 38;// AM + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated public static final int LOOKUP_CLIENT_TYPES_COL = 39;// AN + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated public static final int LOOKUP_ADDRESS_TYPE_COL = 40;// AO + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated public static final int LOOKUP_STATE_PROVINCE_COL = 41;// AP + /** + * @deprecated Lookup columns have been moved to a hidden sheet. Use dynamic column resolution instead. + */ + @Deprecated public static final int LOOKUP_COUNTRY_COL = 42;// AQ } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/constants/TemplatePopulateImportConstants.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/constants/TemplatePopulateImportConstants.java index c7226e1c771..a3b90d59b15 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/constants/TemplatePopulateImportConstants.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/constants/TemplatePopulateImportConstants.java @@ -57,6 +57,7 @@ private TemplatePopulateImportConstants() { public static final String EMPLOYEE_SHEET_NAME = "Employee"; public static final String ROLES_SHEET_NAME = "Roles"; public static final String USER_SHEET_NAME = "Users"; + public static final String CLIENT_LOOKUPS_SHEET_NAME = "_CLIENT_LOOKUPS"; public static final int ROWHEADER_INDEX = 0; public static final short ROW_HEADER_HEIGHT = 500; diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/ImportHandlerUtils.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/ImportHandlerUtils.java index 1090dcbe543..86059d1b284 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/ImportHandlerUtils.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/ImportHandlerUtils.java @@ -29,6 +29,18 @@ import org.apache.fineract.infrastructure.core.exception.AbstractPlatformException; import org.apache.fineract.infrastructure.core.exception.PlatformApiDataValidationException; import org.apache.fineract.infrastructure.core.exception.UnsupportedParameterException; +import org.apache.fineract.infrastructure.dataqueries.data.DatatableData; +import org.apache.fineract.infrastructure.dataqueries.data.ResultsetColumnHeaderData; +import org.apache.fineract.infrastructure.dataqueries.domain.EntityDatatableChecks; +import org.apache.fineract.infrastructure.dataqueries.domain.EntityDatatableChecksRepository; +import org.apache.fineract.infrastructure.dataqueries.data.EntityTables; +import org.apache.fineract.infrastructure.dataqueries.data.StatusEnum; +import org.apache.fineract.infrastructure.dataqueries.service.DatatableReadService; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; import org.apache.fineract.infrastructure.core.service.DateUtils; import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.CellStyle; @@ -145,6 +157,31 @@ public static String trimEmptyDecimalPortion(String result) { } } + /** + * Extracts numeric ID from strings ending with "(id)" format. + * Example: "Permanent (32)" -> "32" + * If the string doesn't match the pattern, returns the original value. + * + * @param value The string value that may contain an ID in parentheses + * @return The extracted ID as a string, or the original value if no ID pattern is found + */ + public static String extractIdFromDisplayValue(String value) { + if (value == null || value.trim().isEmpty()) { + return value; + } + // Match pattern: ".*(\d+)$" where the number is in parentheses at the end + // More specifically: ".*\\(\\d+\\)$" + String trimmed = value.trim(); + int lastParenIndex = trimmed.lastIndexOf('('); + if (lastParenIndex > 0 && trimmed.endsWith(")")) { + String idPart = trimmed.substring(lastParenIndex + 1, trimmed.length() - 1); + if (idPart.matches("\\d+")) { + return idPart; + } + } + return value; + } + public static LocalDate readAsDate(int colIndex, Row row) { Cell c = row.getCell(colIndex); if (c == null || c.getCellType() == CellType.BLANK) { @@ -447,4 +484,701 @@ public static String getRepeatsOnDayId(String repeatsOnDay) { return null; } } + + /** + * Validates that all required datatables are present in the Excel workbook for the given row. + * + * Validation logic: + * 1. Multi-row datatables (1:N with entity) are always skipped - they are optional/repeatable child data. + * 2. If NO column header exists for a required datatable, validation is skipped entirely. + * This allows required datatables to be optional in the Excel sheet if no columns are present. + * 3. If column headers EXIST for a required datatable: + * a. Retrieve the datatable schema to identify mandatory (non-nullable) columns. + * b. If the datatable has ZERO mandatory columns (all columns nullable), validation passes even if all values are blank. + * c. If there are mandatory columns, validate that each mandatory column either: + * - Exists in the Excel header AND has a non-blank value, OR + * - Is missing from the Excel header (treated as missing field). + * d. System columns (id, client_id, created_at, updated_at) and locale/dateFormat are ignored. + * + * This ensures that: + * - Optional datatables (not present in Excel) → no validation error + * - Required datatables with all nullable columns (present but empty) → no validation error + * - Required datatables with mandatory columns (present but missing mandatory values) → validation error + * - Multi-row datatables → always skipped + * + * @param workbook The Excel workbook + * @param sheet The sheet containing the client data + * @param row The data row to validate + * @param entitySubtype The entity subtype (PERSON or ENTITY) + * @param entityDatatableChecksRepository Repository to fetch required datatables + * @param datatableReadService Service to retrieve datatable metadata (can be null, will skip multi-row and mandatory field checks if null) + * @return Error message if validation fails (format: "Missing required datatable fields in : , "), null if validation passes + */ + public static String validateRequiredDatatables(Workbook workbook, Sheet sheet, Row row, String entitySubtype, + EntityDatatableChecksRepository entityDatatableChecksRepository, DatatableReadService datatableReadService) { + if (entityDatatableChecksRepository == null) { + return null; + } + + // Get required datatables for CLIENT entity with CREATE status + List requiredChecks; + if (entitySubtype != null) { + requiredChecks = entityDatatableChecksRepository.findByEntityAndStatusAndSubtype( + EntityTables.CLIENT.getName(), StatusEnum.CREATE.getValue(), entitySubtype); + } else { + requiredChecks = entityDatatableChecksRepository.findByEntityAndStatus( + EntityTables.CLIENT.getName(), StatusEnum.CREATE.getValue()); + } + + if (requiredChecks == null || requiredChecks.isEmpty()) { + return null; // No required datatables + } + + // Get header row (row 0) + Row headerRow = sheet.getRow(0); + if (headerRow == null) { + return "Missing required datatable(s): " + getDatatableNames(requiredChecks); + } + + // Build a map of column headers to column indices + Map headerColumnMap = new HashMap<>(); + for (int colIndex = 0; colIndex <= headerRow.getLastCellNum(); colIndex++) { + Cell headerCell = headerRow.getCell(colIndex); + if (headerCell != null && headerCell.getCellType() == CellType.STRING) { + String headerValue = headerCell.getStringCellValue(); + if (headerValue != null) { + headerColumnMap.put(headerValue.trim(), colIndex); + } + } + } + + // Check each required datatable + List missingDatatables = new ArrayList<>(); + for (EntityDatatableChecks check : requiredChecks) { + String datatableName = check.getDatatableName(); + + // Skip validation for multi-row datatables - they are optional/repeatable child data + if (datatableReadService != null) { + try { + DatatableData datatableData = datatableReadService.retrieveDatatable(datatableName); + if (datatableData != null && datatableData.isMultiRow()) { + // Multi-row datatables are optional, skip validation + log.debug("Skipping validation for multi-row datatable '{}' - treated as optional/repeatable child data", + datatableName); + continue; + } + } catch (Exception e) { + // If we can't retrieve datatable metadata, continue with validation + // This preserves backward compatibility + log.debug("Could not retrieve datatable metadata for '{}' to check multi-row status: {}", + datatableName, e.getMessage()); + } + } + + // First, check if ANY column header exists for this datatable (presence check) + boolean headerExists = false; + List matchingColumnIndices = new ArrayList<>(); + + // Check if any column header matches the datatable name pattern + // Prefer dot notation (new format), fallback to underscore notation (legacy) + for (Map.Entry entry : headerColumnMap.entrySet()) { + String header = entry.getKey(); + int colIndex = entry.getValue(); + + // Remove trailing * if present + String headerWithoutStar = header.endsWith("*") ? header.substring(0, header.length() - 1) : header; + + boolean matches = false; + + // Check dot notation first (preferred format): registeredTableName.columnName + if (headerWithoutStar.contains(".")) { + int dotIndex = headerWithoutStar.indexOf("."); + if (dotIndex > 0) { + String headerDatatableName = headerWithoutStar.substring(0, dotIndex); + if (headerDatatableName.equals(datatableName)) { + matches = true; + } + } + } + // Fallback to underscore notation (legacy format) for backward compatibility + else if (headerWithoutStar.contains("_")) { + int lastUnderscoreIndex = headerWithoutStar.lastIndexOf("_"); + if (lastUnderscoreIndex > 0) { + String headerDatatableName = headerWithoutStar.substring(0, lastUnderscoreIndex); + if (headerDatatableName.equals(datatableName)) { + matches = true; + } + } + } + + if (matches) { + headerExists = true; + matchingColumnIndices.add(colIndex); + } + } + + // If no header exists for this datatable, skip validation entirely + // This handles the case where a required datatable is configured but not present in the Excel sheet + if (!headerExists) { + log.debug("Skipping validation for required datatable '{}' - no column headers found in Excel sheet. " + + "Validation only applies when at least one column header for the datatable is present.", + datatableName); + continue; + } + + // Header exists, so now validate mandatory columns + // Build a map of Excel column names (without datatable prefix) to column indices + Map excelColumnMap = new HashMap<>(); + for (Map.Entry entry : headerColumnMap.entrySet()) { + String header = entry.getKey(); + int colIndex = entry.getValue(); + + // Remove trailing * if present + String headerWithoutStar = header.endsWith("*") ? header.substring(0, header.length() - 1) : header; + + String columnName = null; + // Check dot notation first (preferred format): registeredTableName.columnName + if (headerWithoutStar.contains(".")) { + int dotIndex = headerWithoutStar.indexOf("."); + if (dotIndex > 0 && dotIndex < headerWithoutStar.length() - 1) { + String headerDatatableName = headerWithoutStar.substring(0, dotIndex); + if (headerDatatableName.equals(datatableName)) { + columnName = headerWithoutStar.substring(dotIndex + 1); + } + } + } + // Fallback to underscore notation (legacy format) for backward compatibility + else if (headerWithoutStar.contains("_")) { + int lastUnderscoreIndex = headerWithoutStar.lastIndexOf("_"); + if (lastUnderscoreIndex > 0 && lastUnderscoreIndex < headerWithoutStar.length() - 1) { + String headerDatatableName = headerWithoutStar.substring(0, lastUnderscoreIndex); + if (headerDatatableName.equals(datatableName)) { + columnName = headerWithoutStar.substring(lastUnderscoreIndex + 1); + } + } + } + + if (columnName != null) { + excelColumnMap.put(columnName, colIndex); + } + } + + // Retrieve datatable schema to check for mandatory columns + List missingMandatoryFields = new ArrayList<>(); + if (datatableReadService != null) { + try { + DatatableData datatableData = datatableReadService.retrieveDatatable(datatableName); + if (datatableData != null && datatableData.getColumnHeaderData() != null) { + // Find all mandatory columns (non-nullable, excluding system columns) + List mandatoryColumns = new ArrayList<>(); + for (ResultsetColumnHeaderData column : datatableData.getColumnHeaderData()) { + String columnName = column.getColumnName(); + + // Ignore system columns + if (columnName.equalsIgnoreCase("id") + || columnName.equalsIgnoreCase("client_id") + || columnName.equalsIgnoreCase("created_at") + || columnName.equalsIgnoreCase("updated_at") + || columnName.equalsIgnoreCase("locale") + || columnName.equalsIgnoreCase("dateFormat")) { + continue; + } + + // Check if column is mandatory (not nullable) + if (!column.getIsColumnNullable()) { + mandatoryColumns.add(columnName); + } + } + + // If there are no mandatory columns, skip validation (all columns are nullable) + if (mandatoryColumns.isEmpty()) { + log.debug("Skipping validation for required datatable '{}' - no mandatory fields found. " + + "All columns are nullable, so blank values are allowed.", + datatableName); + continue; + } + + // Check each mandatory column + for (String mandatoryColumn : mandatoryColumns) { + Integer colIndex = excelColumnMap.get(mandatoryColumn); + + if (colIndex == null) { + // Mandatory column is missing from Excel header + missingMandatoryFields.add(mandatoryColumn); + } else { + // Check if the cell has a non-blank value + Cell dataCell = row.getCell(colIndex); + boolean hasValue = false; + if (dataCell != null && dataCell.getCellType() != CellType.BLANK) { + String cellValue = readAsString(colIndex, row); + if (cellValue != null && !cellValue.trim().isEmpty()) { + hasValue = true; + } + } + + if (!hasValue) { + missingMandatoryFields.add(mandatoryColumn); + } + } + } + + // If mandatory fields are missing, add to error list + if (!missingMandatoryFields.isEmpty()) { + String friendlyName = getFriendlyDatatableName(datatableName); + String fieldsList = String.join(", ", missingMandatoryFields); + missingDatatables.add(friendlyName + ": " + fieldsList); + log.debug("Failing validation for required datatable '{}' - missing mandatory fields: {}", + datatableName, fieldsList); + } + } else { + // If we can't retrieve datatable schema, fall back to old behavior + // Check if at least one column has a non-empty value + boolean hasValue = false; + for (Integer colIndex : matchingColumnIndices) { + Cell dataCell = row.getCell(colIndex); + if (dataCell != null && dataCell.getCellType() != CellType.BLANK) { + String cellValue = readAsString(colIndex, row); + if (cellValue != null && !cellValue.trim().isEmpty()) { + hasValue = true; + break; + } + } + } + + if (!hasValue) { + String friendlyName = getFriendlyDatatableName(datatableName); + missingDatatables.add(friendlyName); + log.debug("Required datatable '{}' has column headers in Excel but no values provided " + + "(datatable schema unavailable for mandatory field check)", + datatableName); + } + } + } catch (Exception e) { + // If we can't retrieve datatable schema, fall back to old behavior + // Check if at least one column has a non-empty value + boolean hasValue = false; + for (Integer colIndex : matchingColumnIndices) { + Cell dataCell = row.getCell(colIndex); + if (dataCell != null && dataCell.getCellType() != CellType.BLANK) { + String cellValue = readAsString(colIndex, row); + if (cellValue != null && !cellValue.trim().isEmpty()) { + hasValue = true; + break; + } + } + } + + if (!hasValue) { + String friendlyName = getFriendlyDatatableName(datatableName); + missingDatatables.add(friendlyName); + log.debug("Required datatable '{}' has column headers in Excel but no values provided " + + "(error retrieving datatable schema: {})", + datatableName, e.getMessage()); + } + } + } else { + // If datatableReadService is null, fall back to old behavior + // Check if at least one column has a non-empty value + boolean hasValue = false; + for (Integer colIndex : matchingColumnIndices) { + Cell dataCell = row.getCell(colIndex); + if (dataCell != null && dataCell.getCellType() != CellType.BLANK) { + String cellValue = readAsString(colIndex, row); + if (cellValue != null && !cellValue.trim().isEmpty()) { + hasValue = true; + break; + } + } + } + + if (!hasValue) { + String friendlyName = getFriendlyDatatableName(datatableName); + missingDatatables.add(friendlyName); + log.debug("Required datatable '{}' has column headers in Excel but no values provided " + + "(datatableReadService unavailable for mandatory field check)", + datatableName); + } + } + } + + if (!missingDatatables.isEmpty()) { + // Format: "Missing required datatable fields in : , " + // or "Missing required datatable(s): " if no field details available + List errorMessages = new ArrayList<>(); + for (String missing : missingDatatables) { + if (missing.contains(": ")) { + // Has field details: "Missing required datatable fields in : " + errorMessages.add("Missing required datatable fields in " + missing); + } else { + // No field details: fallback to old format + errorMessages.add("Missing required datatable(s): " + missing); + } + } + return String.join("; ", errorMessages); + } + + return null; + } + + private static String getDatatableNames(List checks) { + List names = new ArrayList<>(); + for (EntityDatatableChecks check : checks) { + names.add(getFriendlyDatatableName(check.getDatatableName())); + } + return String.join(", ", names); + } + + private static String getFriendlyDatatableName(String datatableName) { + // Convert datatable name to a more friendly format + // e.g., "extra_client_details" -> "Extra Client Details" + if (datatableName == null || datatableName.isEmpty()) { + return datatableName; + } + // Replace underscores with spaces and capitalize words + String[] parts = datatableName.split("_"); + StringBuilder friendly = new StringBuilder(); + for (int i = 0; i < parts.length; i++) { + if (i > 0) { + friendly.append(" "); + } + if (parts[i].length() > 0) { + friendly.append(Character.toUpperCase(parts[i].charAt(0))); + if (parts[i].length() > 1) { + friendly.append(parts[i].substring(1)); + } + } + } + return friendly.toString(); + } + + /** + * Reads datatable columns from the Excel row and groups them by registered table name. + * + * Preferred format (new): . or .* + * Legacy format (backward compatibility): _ or _* + * + * For dot notation: splits at the first dot (only one is expected). + * For legacy underscore notation: splits at the last underscore. + * + * @param sheet The sheet containing the client data + * @param row The data row to read from + * @param locale The locale for datatable data + * @param dateFormat The date format for datatable data + * @return List of maps representing datatables, each with "registeredTableName" and "data" keys. + * Only includes datatables with at least one non-empty value. + */ + public static List> readDatatablesFromRow(Sheet sheet, Row row, String locale, String dateFormat) { + List> datatablesList = new ArrayList<>(); + + if (sheet == null || row == null) { + return datatablesList; + } + + // Get header row (row 0) + Row headerRow = sheet.getRow(0); + if (headerRow == null) { + return datatablesList; + } + + // Build a map of column headers to column indices + Map headerColumnMap = new HashMap<>(); + for (int colIndex = 0; colIndex <= headerRow.getLastCellNum(); colIndex++) { + Cell headerCell = headerRow.getCell(colIndex); + if (headerCell != null && headerCell.getCellType() == CellType.STRING) { + String headerValue = headerCell.getStringCellValue(); + if (headerValue != null) { + headerColumnMap.put(headerValue.trim(), colIndex); + } + } + } + + // Group datatable columns by datatable name + Map> datatablesMap = new HashMap<>(); + boolean legacyHeaderDetected = false; + + for (Map.Entry entry : headerColumnMap.entrySet()) { + String header = entry.getKey(); + int colIndex = entry.getValue(); + + // Remove trailing * if present + String headerWithoutStar = header.endsWith("*") ? header.substring(0, header.length() - 1) : header; + + String registeredTableName; + String columnName = null; + + // Prefer dot notation (new format): registeredTableName.columnName + if (headerWithoutStar.contains(".")) { + int dotIndex = headerWithoutStar.indexOf("."); + if (dotIndex > 0 && dotIndex < headerWithoutStar.length() - 1) { + registeredTableName = headerWithoutStar.substring(0, dotIndex); + columnName = headerWithoutStar.substring(dotIndex + 1); + } else { + registeredTableName = null; + } + } + // Fallback to underscore notation (legacy format) for backward compatibility + else if (headerWithoutStar.contains("_")) { + legacyHeaderDetected = true; + // Use lastIndexOf to split at the last underscore + // Everything before the last _ is the registeredTableName + // Everything after the last _ is the columnName + int lastUnderscoreIndex = headerWithoutStar.lastIndexOf("_"); + if (lastUnderscoreIndex > 0 && lastUnderscoreIndex < headerWithoutStar.length() - 1) { + registeredTableName = headerWithoutStar.substring(0, lastUnderscoreIndex); + columnName = headerWithoutStar.substring(lastUnderscoreIndex + 1); + } else { + registeredTableName = null; + } + } else { + registeredTableName = null; + } + + // Process if we have a valid datatable column + if (registeredTableName != null && columnName != null) { + // Skip system columns + if (columnName.equalsIgnoreCase("id") || columnName.equalsIgnoreCase("client_id") + || columnName.equalsIgnoreCase("created_at") || columnName.equalsIgnoreCase("updated_at")) { + continue; + } + + // Read cell value + Cell dataCell = row.getCell(colIndex); + Object cellValue = null; + if (dataCell != null && dataCell.getCellType() != CellType.BLANK) { + // Try to read as different types + if (dataCell.getCellType() == CellType.NUMERIC) { + if (org.apache.poi.ss.usermodel.DateUtil.isCellDateFormatted(dataCell)) { + // Date value - format as string using dateFormat + LocalDate dateValue = readAsDate(colIndex, row); + if (dateValue != null && dateFormat != null) { + try { + DateTimeFormatter formatter = new DateTimeFormatterBuilder() + .appendPattern(dateFormat) + .toFormatter(); + cellValue = dateValue.format(formatter); + } catch (Exception e) { + // Fallback to ISO format if dateFormat is invalid + cellValue = dateValue.toString(); + } + } else if (dateValue != null) { + cellValue = dateValue.toString(); + } + } else { + // Numeric value - check if it's a whole number + double numValue = dataCell.getNumericCellValue(); + if (numValue == Math.floor(numValue)) { + cellValue = (long) numValue; + } else { + cellValue = numValue; + } + } + } else if (dataCell.getCellType() == CellType.BOOLEAN) { + cellValue = dataCell.getBooleanCellValue(); + } else { + // String value + String stringValue = readAsString(colIndex, row); + if (stringValue != null && !stringValue.trim().isEmpty()) { + // Try to extract ID from display value format: "Label (ID)" + // This handles CODELOOKUP and FK/INTEGER columns that show human-readable values + String extractedId = extractIdFromDisplayValue(stringValue.trim()); + // If extraction succeeded (returned a different value), try to parse as integer + if (!extractedId.equals(stringValue.trim())) { + try { + cellValue = Integer.parseInt(extractedId); + } catch (NumberFormatException e) { + // If parsing fails, use the original string value + cellValue = stringValue.trim(); + } + } else { + // No ID pattern found, use original value + cellValue = stringValue.trim(); + } + } + } + } + + // Only add if value is not null/empty + if (cellValue != null) { + // Get or create datatable entry using registeredTableName + Map datatableData = datatablesMap.computeIfAbsent(registeredTableName, k -> { + Map newDatatable = new HashMap<>(); + newDatatable.put("registeredTableName", registeredTableName); + Map data = new HashMap<>(); + data.put("locale", locale); + data.put("dateFormat", dateFormat); + newDatatable.put("data", data); + return newDatatable; + }); + + // Add column value to data + @SuppressWarnings("unchecked") + Map data = (Map) datatableData.get("data"); + data.put(columnName, cellValue); + } + } + } + + // Log warning if legacy underscore-based headers were detected + if (legacyHeaderDetected) { + log.warn("Legacy underscore-based datatable column headers detected. Please update template to use dot notation (registeredTableName.columnName) for unambiguous parsing."); + } + + // Convert map values to list (only include datatables with at least one data field beyond locale/dateFormat) + for (Map datatable : datatablesMap.values()) { + @SuppressWarnings("unchecked") + Map data = (Map) datatable.get("data"); + // Check if there are any fields beyond locale and dateFormat + if (data.size() > 2) { + datatablesList.add(datatable); + } + } + + return datatablesList; + } + + /** + * Ensures all configured client datatables are included in the payload, even if they have no values. + * This is required for bulk import to match Fineract's contract where all entity-linked datatables must be present. + * Only includes datatables that are explicitly linked to Client via EntityDatatableChecks. + * + * Construction rules: + * - If datatable has at least one required column (non-nullable): existing behavior applies (must be filled by user) + * - If datatable has NO required columns: always construct the datatable in payload, even if multi-row + * - Multi-row datatables with required columns are skipped (user must provide data) + * - Multi-row datatables with no required columns are included with empty entry + * + * @param sheet The sheet containing the client data + * @param row The data row to read from + * @param locale The locale for datatable data + * @param dateFormat The date format for datatable data + * @param entityDatatableChecksRepository Repository to fetch datatables linked to Client entity (can be null, will skip if null) + * @param datatableReadService Service to retrieve datatable metadata (can be null, will skip multi-row check if null) + * @param entitySubtype The entity subtype (PERSON or ENTITY), can be null + * @return List of maps representing datatables, each with "registeredTableName" and "data" keys. + * Includes only datatables linked via EntityDatatableChecks, with empty entries for those without values. + */ + public static List> readDatatablesFromRowWithAllConfigured( + Sheet sheet, Row row, String locale, String dateFormat, + EntityDatatableChecksRepository entityDatatableChecksRepository, + DatatableReadService datatableReadService, String entitySubtype) { + // First, read datatables that have values from Excel + List> datatablesWithValues = readDatatablesFromRow(sheet, row, locale, dateFormat); + + // Create a map of registeredTableName -> datatable for quick lookup + Map> datatablesMap = new HashMap<>(); + for (Map datatable : datatablesWithValues) { + String registeredTableName = (String) datatable.get("registeredTableName"); + if (registeredTableName != null) { + datatablesMap.put(registeredTableName, datatable); + } + } + + // If entityDatatableChecksRepository is available, ensure all linked client datatables are included + if (entityDatatableChecksRepository != null) { + try { + // Retrieve only datatables linked to Client via EntityDatatableChecks with CREATE status + List linkedChecks; + if (entitySubtype != null) { + linkedChecks = entityDatatableChecksRepository.findByEntityAndStatusAndSubtype( + EntityTables.CLIENT.getName(), StatusEnum.CREATE.getValue(), entitySubtype); + } else { + linkedChecks = entityDatatableChecksRepository.findByEntityAndStatus( + EntityTables.CLIENT.getName(), StatusEnum.CREATE.getValue()); + } + + // For each linked datatable, ensure it's in the result + for (EntityDatatableChecks check : linkedChecks) { + String registeredTableName = check.getDatatableName(); + + // Skip if already included (has values) + if (datatablesMap.containsKey(registeredTableName)) { + continue; + } + + // Retrieve datatable schema to check multi-row status and required columns + boolean isMultiRow = false; + boolean hasRequiredColumns = false; + org.apache.fineract.infrastructure.dataqueries.data.DatatableData datatableData = null; + + if (datatableReadService != null) { + try { + datatableData = datatableReadService.retrieveDatatable(registeredTableName); + if (datatableData != null) { + isMultiRow = datatableData.isMultiRow(); + + // Check if datatable has at least one required column + if (datatableData.getColumnHeaderData() != null) { + for (ResultsetColumnHeaderData column : datatableData.getColumnHeaderData()) { + String columnName = column.getColumnName(); + + // Skip system columns + if (columnName.equalsIgnoreCase("id") + || columnName.equalsIgnoreCase("client_id") + || columnName.equalsIgnoreCase("created_at") + || columnName.equalsIgnoreCase("updated_at") + || columnName.equalsIgnoreCase("locale") + || columnName.equalsIgnoreCase("dateFormat")) { + continue; + } + + // Check if column is required (not nullable) + if (!column.getIsColumnNullable()) { + hasRequiredColumns = true; + break; + } + } + } + } + } catch (Exception e) { + // If we can't retrieve datatable metadata, continue (assume not multi-row, no required columns) + log.debug("Could not retrieve datatable metadata for '{}': {}", + registeredTableName, e.getMessage()); + } + } + + // Skip multi-row datatables ONLY if they have required columns + // Multi-row datatables with NO required columns should be included (empty entry) + if (isMultiRow && hasRequiredColumns) { + // Multi-row with required columns: skip (user must provide data) + continue; + } + + // Create empty datatable entry with locale and dateFormat + Map emptyDatatable = new HashMap<>(); + emptyDatatable.put("registeredTableName", registeredTableName); + Map data = new HashMap<>(); + data.put("locale", locale); + data.put("dateFormat", dateFormat); + + // Add empty strings for known columns if schema is available + if (datatableData != null && datatableData.getColumnHeaderData() != null) { + for (ResultsetColumnHeaderData column : datatableData.getColumnHeaderData()) { + String columnName = column.getColumnName(); + // Skip system columns + if (columnName.equalsIgnoreCase("id") + || columnName.equalsIgnoreCase("client_id") + || columnName.equalsIgnoreCase("created_at") + || columnName.equalsIgnoreCase("updated_at") + || columnName.equalsIgnoreCase("locale") + || columnName.equalsIgnoreCase("dateFormat")) { + continue; + } + // Add empty string for known columns + data.put(columnName, ""); + } + } + + emptyDatatable.put("data", data); + datatablesMap.put(registeredTableName, emptyDatatable); + } + } catch (Exception e) { + // If we can't retrieve linked datatables, log and continue with what we have + log.debug("Could not retrieve linked client datatables for bulk import payload: {}", e.getMessage()); + } + } + + // Return all datatables (with values and empty ones) + return new ArrayList<>(datatablesMap.values()); + } + } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientEntityImportHandler.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientEntityImportHandler.java index 9b08efaa04a..af8be5057d7 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientEntityImportHandler.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientEntityImportHandler.java @@ -23,8 +23,11 @@ import java.time.LocalDate; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import lombok.AllArgsConstructor; import org.apache.fineract.commands.domain.CommandWrapper; import org.apache.fineract.commands.service.CommandWrapperBuilder; @@ -35,12 +38,19 @@ import org.apache.fineract.infrastructure.bulkimport.importhandler.ImportHandler; import org.apache.fineract.infrastructure.bulkimport.importhandler.ImportHandlerUtils; import org.apache.fineract.infrastructure.bulkimport.importhandler.helper.DateSerializer; +import org.apache.fineract.infrastructure.core.data.ApiParameterError; import org.apache.fineract.infrastructure.core.domain.ExternalId; +import org.apache.fineract.infrastructure.core.exception.PlatformApiDataValidationException; import org.apache.fineract.infrastructure.core.serialization.GoogleGsonSerializerHelper; import org.apache.fineract.infrastructure.core.service.ExternalIdFactory; +import org.apache.fineract.infrastructure.dataqueries.domain.EntityDatatableChecksRepository; +import org.apache.fineract.infrastructure.dataqueries.service.DatatableReadService; import org.apache.fineract.portfolio.address.data.AddressData; import org.apache.fineract.portfolio.client.data.ClientData; import org.apache.fineract.portfolio.client.data.ClientNonPersonData; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import java.util.Map; import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.IndexedColors; import org.apache.poi.ss.usermodel.Row; @@ -58,6 +68,8 @@ public class ClientEntityImportHandler implements ImportHandler { private static final Logger LOG = LoggerFactory.getLogger(ClientEntityImportHandler.class); private final PortfolioCommandSourceWritePlatformService commandsSourceWritePlatformService; private final ExternalIdFactory externalIdFactory; + private final EntityDatatableChecksRepository entityDatatableChecksRepository; + private final DatatableReadService datatableReadService; @Override public Count process(final Workbook workbook, final String locale, final String dateFormat) { @@ -73,6 +85,9 @@ private List readExcelFile(final Workbook workbook, final String loc for (int rowIndex = 1; rowIndex <= noOfEntries; rowIndex++) { Row row; row = clientSheet.getRow(rowIndex); + if (row != null) { + LOG.info("Processing bulk upload row {} with {} cells", row.getRowNum(), row.getLastCellNum()); + } if (ImportHandlerUtils.isNotImported(row, ClientEntityConstants.STATUS_COL)) { clients.add(readClient(workbook, row, locale, dateFormat)); } @@ -104,16 +119,24 @@ private ClientData readClient(final Workbook workbook, final Row row, final Stri Long clientTypeId = null; if (clientType != null) { List clientTypeAr = Splitter.on(SEPARATOR).splitToList(clientType); - if (clientTypeAr.get(1) != null) { - clientTypeId = Long.parseLong(clientTypeAr.get(1)); + if (clientTypeAr.size() > 1) { + if (clientTypeAr.get(1) != null) { + clientTypeId = Long.parseLong(clientTypeAr.get(1)); + } + } else { + LOG.error("Bulk upload error: Missing expected column at index 1 in clientType. Row data: {}", clientTypeAr); } } String clientClassification = ImportHandlerUtils.readAsString(ClientEntityConstants.CLIENT_CLASSIFICATION_COL, row); Long clientClassicationId = null; if (clientClassification != null) { List clientClassificationAr = Splitter.on(SEPARATOR).splitToList(clientClassification); - if (clientClassificationAr.get(1) != null) { - clientClassicationId = Long.parseLong(clientClassificationAr.get(1)); + if (clientClassificationAr.size() > 1) { + if (clientClassificationAr.get(1) != null) { + clientClassicationId = Long.parseLong(clientClassificationAr.get(1)); + } + } else { + LOG.error("Bulk upload error: Missing expected column at index 1 in clientClassification. Row data: {}", clientClassificationAr); } } String incorporationNo = ImportHandlerUtils.readAsString(ClientEntityConstants.INCOPORATION_NUMBER_COL, row); @@ -123,18 +146,34 @@ private ClientData readClient(final Workbook workbook, final Row row, final Stri if (mainBusinessLine != null) { List mainBusinessLineAr = Splitter.on(SEPARATOR) .splitToList(Objects.requireNonNull(ImportHandlerUtils.readAsString(ClientEntityConstants.MAIN_BUSINESS_LINE, row))); - if (mainBusinessLineAr.get(1) != null) { - mainBusinessId = Long.parseLong(mainBusinessLineAr.get(1)); + if (mainBusinessLineAr.size() > 1) { + if (mainBusinessLineAr.get(1) != null) { + mainBusinessId = Long.parseLong(mainBusinessLineAr.get(1)); + } + } else { + LOG.error("Bulk upload error: Missing expected column at index 1 in mainBusinessLine. Row data: {}", mainBusinessLineAr); } } - String constitution = ImportHandlerUtils.readAsString(ClientEntityConstants.CONSTITUTION_COL, row); + String constitutionCell = ImportHandlerUtils.readAsString(ClientEntityConstants.CONSTITUTION_COL, row); + Long constitutionId = null; - if (constitution != null) { - List constitutionAr = Splitter.on(SEPARATOR).splitToList(constitution); - if (constitutionAr.get(1) != null) { - constitutionId = Long.parseLong(constitutionAr.get(1)); + + if (constitutionCell != null && constitutionCell.contains("(") && constitutionCell.contains(")")) { + Pattern pattern = Pattern.compile("\\((\\d+)\\)"); + Matcher matcher = pattern.matcher(constitutionCell); + + if (matcher.find()) { + constitutionId = Long.valueOf(matcher.group(1)); } } + + if (constitutionId == null && constitutionCell != null && !constitutionCell.trim().isEmpty()) { + LOG.error("Invalid constitution format in row {}: {}", row.getRowNum(), constitutionCell); + ApiParameterError error = ApiParameterError.parameterError("error.msg.bulk.import.constitution.invalid", + "Invalid Constitution format. Expected format: Name (ID)", "constitution", constitutionCell); + throw new PlatformApiDataValidationException("error.msg.bulk.import.constitution.invalid", + "Invalid Constitution format. Expected format: Name (ID)", Collections.singletonList(error)); + } String remarks = ImportHandlerUtils.readAsString(ClientEntityConstants.REMARKS_COL, row); ClientNonPersonData clientNonPersonData = ClientNonPersonData.importInstance(incorporationNo, incorporationTill, remarks, @@ -157,8 +196,12 @@ private ClientData readClient(final Workbook workbook, final Row row, final Stri Long addressTypeId = null; if (addressType != null) { List addressTypeAr = Splitter.on(SEPARATOR).splitToList(addressType); - if (addressTypeAr.get(1) != null) { - addressTypeId = Long.parseLong(addressTypeAr.get(1)); + if (addressTypeAr.size() > 1) { + if (addressTypeAr.get(1) != null) { + addressTypeId = Long.parseLong(addressTypeAr.get(1)); + } + } else { + LOG.error("Bulk upload error: Missing expected column at index 1 in addressType. Row data: {}", addressTypeAr); } } String street = ImportHandlerUtils.readAsString(ClientEntityConstants.STREET_COL, row); @@ -174,16 +217,24 @@ private ClientData readClient(final Workbook workbook, final Row row, final Stri Long stateProvinceId = null; if (stateProvince != null) { List stateProvinceAr = Splitter.on(SEPARATOR).splitToList(stateProvince); - if (stateProvinceAr.get(1) != null) { - stateProvinceId = Long.parseLong(stateProvinceAr.get(1)); + if (stateProvinceAr.size() > 1) { + if (stateProvinceAr.get(1) != null) { + stateProvinceId = Long.parseLong(stateProvinceAr.get(1)); + } + } else { + LOG.error("Bulk upload error: Missing expected column at index 1 in stateProvince. Row data: {}", stateProvinceAr); } } String country = ImportHandlerUtils.readAsString(ClientEntityConstants.COUNTRY_COL, row); Long countryId = null; if (country != null) { List countryAr = Splitter.on(SEPARATOR).splitToList(country); - if (countryAr.get(1) != null) { - countryId = Long.parseLong(countryAr.get(1)); + if (countryAr.size() > 1) { + if (countryAr.get(1) != null) { + countryId = Long.parseLong(countryAr.get(1)); + } + } else { + LOG.error("Bulk upload error: Missing expected column at index 1 in country. Row data: {}", countryAr); } } addressDataObj = new AddressData(addressTypeId, street, addressLine1, addressLine2, addressLine3, city, postalCode, @@ -207,7 +258,33 @@ private Count importEntity(final Workbook workbook, final List clien for (ClientData client : clients) { try { + // Get the client row once for validation and datatable reading + Row clientRow = clientSheet.getRow(client.getRowIndex()); + if (clientRow != null) { + // Validate required datatables before attempting client creation + String validationError = ImportHandlerUtils.validateRequiredDatatables( + workbook, clientSheet, clientRow, "Entity", entityDatatableChecksRepository, datatableReadService); + if (validationError != null) { + throw new RuntimeException(validationError); + } + } + String payload = gsonBuilder.create().toJson(client); + + // Read datatables from the Excel row and add to payload + // Ensure all configured client datatables are included, even if empty + if (clientRow != null) { + List> datatables = ImportHandlerUtils.readDatatablesFromRowWithAllConfigured( + clientSheet, clientRow, locale, dateFormat, entityDatatableChecksRepository, + datatableReadService, "Entity"); + if (datatables != null && !datatables.isEmpty()) { + // Parse JSON and add datatables array + JsonObject jsonObject = JsonParser.parseString(payload).getAsJsonObject(); + jsonObject.add("datatables", gsonBuilder.create().toJsonTree(datatables)); + payload = gsonBuilder.create().toJson(jsonObject); + } + } + final CommandWrapper commandRequest = new CommandWrapperBuilder() // .createClient() // .withJson(payload) // diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientPersonImportHandler.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientPersonImportHandler.java index e35027a192c..f7349de475a 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientPersonImportHandler.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientPersonImportHandler.java @@ -38,8 +38,14 @@ import org.apache.fineract.infrastructure.core.domain.ExternalId; import org.apache.fineract.infrastructure.core.serialization.GoogleGsonSerializerHelper; import org.apache.fineract.infrastructure.core.service.ExternalIdFactory; +import org.apache.fineract.infrastructure.dataqueries.domain.EntityDatatableChecksRepository; +import org.apache.fineract.infrastructure.dataqueries.service.DatatableReadService; import org.apache.fineract.portfolio.address.data.AddressData; import org.apache.fineract.portfolio.client.data.ClientData; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import java.util.List; +import java.util.Map; import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.IndexedColors; import org.apache.poi.ss.usermodel.Row; @@ -57,6 +63,8 @@ public class ClientPersonImportHandler implements ImportHandler { private static final Logger LOG = LoggerFactory.getLogger(ClientPersonImportHandler.class); private final PortfolioCommandSourceWritePlatformService commandsSourceWritePlatformService; private final ExternalIdFactory externalIdFactory; + private final EntityDatatableChecksRepository entityDatatableChecksRepository; + private final DatatableReadService datatableReadService; @Override public Count process(final Workbook workbook, final String locale, final String dateFormat) { @@ -192,7 +200,33 @@ public Count importEntity(final Workbook workbook, final List client gsonBuilder.registerTypeAdapter(LocalDate.class, new DateSerializer(dateFormat, locale)); for (ClientData client : clients) { try { + // Get the client row once for validation and datatable reading + Row clientRow = clientSheet.getRow(client.getRowIndex()); + if (clientRow != null) { + // Validate required datatables before attempting client creation + String validationError = ImportHandlerUtils.validateRequiredDatatables( + workbook, clientSheet, clientRow, "Person", entityDatatableChecksRepository, datatableReadService); + if (validationError != null) { + throw new RuntimeException(validationError); + } + } + String payload = gsonBuilder.create().toJson(client); + + // Read datatables from the Excel row and add to payload + // Ensure all configured client datatables are included, even if empty + if (clientRow != null) { + List> datatables = ImportHandlerUtils.readDatatablesFromRowWithAllConfigured( + clientSheet, clientRow, locale, dateFormat, entityDatatableChecksRepository, + datatableReadService, "Person"); + if (datatables != null && !datatables.isEmpty()) { + // Parse JSON and add datatables array + JsonObject jsonObject = JsonParser.parseString(payload).getAsJsonObject(); + jsonObject.add("datatables", gsonBuilder.create().toJsonTree(datatables)); + payload = gsonBuilder.create().toJson(jsonObject); + } + } + final CommandWrapper commandRequest = new CommandWrapperBuilder() // .createClient() // .withJson(payload) // diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/client/ClientEntityWorkbookPopulator.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/client/ClientEntityWorkbookPopulator.java index 985d209f89b..b4c5693ae86 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/client/ClientEntityWorkbookPopulator.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/client/ClientEntityWorkbookPopulator.java @@ -25,7 +25,10 @@ import org.apache.fineract.infrastructure.bulkimport.populator.OfficeSheetPopulator; import org.apache.fineract.infrastructure.bulkimport.populator.PersonnelSheetPopulator; import org.apache.fineract.infrastructure.codes.data.CodeValueData; +import org.apache.fineract.infrastructure.dataqueries.data.DatatableData; +import org.apache.fineract.infrastructure.dataqueries.data.ResultsetColumnHeaderData; import org.apache.fineract.organisation.office.data.OfficeData; +import java.util.ArrayList; import org.apache.poi.hssf.usermodel.HSSFDataValidationHelper; import org.apache.poi.hssf.usermodel.HSSFSheet; import org.apache.poi.ss.SpreadsheetVersion; @@ -38,11 +41,20 @@ import org.apache.poi.ss.usermodel.Name; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.SheetVisibility; import org.apache.poi.ss.usermodel.Workbook; import org.apache.poi.ss.util.CellRangeAddressList; +import org.apache.poi.ss.util.CellReference; +import org.apache.fineract.infrastructure.dataqueries.data.ResultsetColumnValueData; +import java.util.Map; +import java.util.HashMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class ClientEntityWorkbookPopulator extends AbstractWorkbookPopulator { + private static final Logger log = LoggerFactory.getLogger(ClientEntityWorkbookPopulator.class); + private final OfficeSheetPopulator officeSheetPopulator; private final PersonnelSheetPopulator personnelSheetPopulator; private final List clientTypeCodeValues; @@ -52,11 +64,53 @@ public class ClientEntityWorkbookPopulator extends AbstractWorkbookPopulator { private final List stateProvinceCodeValues; private final List countryCodeValues; private final List mainBusinesslineCodeValues; + private final List requiredDatatables; + // Track datatable dropdown columns: namedRangeName -> (clientSheetColumnIndex, lookupSheetColumnIndex, values) + private final Map datatableDropdowns = new HashMap<>(); + + // Track datatable date columns: columnIndex -> columnInfo + private final Map datatableDateColumns = new HashMap<>(); + + // Track datatable boolean columns: columnIndex -> columnInfo + private final Map datatableBooleanColumns = new HashMap<>(); + + // Inner class to track datatable dropdown information + private static class DatatableDropdownInfo { + final int clientSheetColumnIndex; + final int lookupSheetColumnIndex; + final List values; + + DatatableDropdownInfo(int clientSheetColumnIndex, int lookupSheetColumnIndex, List values) { + this.clientSheetColumnIndex = clientSheetColumnIndex; + this.lookupSheetColumnIndex = lookupSheetColumnIndex; + this.values = values; + } + } + + // Inner class to track datatable date column information + private static class DatatableDateInfo { + final int clientSheetColumnIndex; + final boolean isDateTime; + + DatatableDateInfo(int clientSheetColumnIndex, boolean isDateTime) { + this.clientSheetColumnIndex = clientSheetColumnIndex; + this.isDateTime = isDateTime; + } + } + + // Inner class to track datatable boolean column information + private static class DatatableBooleanInfo { + final int clientSheetColumnIndex; + + DatatableBooleanInfo(int clientSheetColumnIndex) { + this.clientSheetColumnIndex = clientSheetColumnIndex; + } + } public ClientEntityWorkbookPopulator(OfficeSheetPopulator officeSheetPopulator, PersonnelSheetPopulator personnelSheetPopulator, List clientTypeCodeValues, List constitutionCodeValues, List mainBusinessline, List clientClassification, List addressTypesCodeValues, - List stateProvinceCodeValues, List countryCodeValues) { + List stateProvinceCodeValues, List countryCodeValues, List requiredDatatables) { this.officeSheetPopulator = officeSheetPopulator; this.personnelSheetPopulator = personnelSheetPopulator; this.clientTypeCodeValues = clientTypeCodeValues; @@ -66,6 +120,7 @@ public ClientEntityWorkbookPopulator(OfficeSheetPopulator officeSheetPopulator, this.stateProvinceCodeValues = stateProvinceCodeValues; this.countryCodeValues = countryCodeValues; this.mainBusinesslineCodeValues = mainBusinessline; + this.requiredDatatables = requiredDatatables != null ? requiredDatatables : new ArrayList<>(); } @Override @@ -73,10 +128,14 @@ public void populate(Workbook workbook, String dateFormat) { Sheet clientSheet = workbook.createSheet(TemplatePopulateImportConstants.CLIENT_ENTITY_SHEET_NAME); personnelSheetPopulator.populate(workbook, dateFormat); officeSheetPopulator.populate(workbook, dateFormat); - setLayout(clientSheet); - setOfficeDateLookupTable(clientSheet, officeSheetPopulator.getOffices(), ClientEntityConstants.RELATIONAL_OFFICE_NAME_COL, - ClientEntityConstants.RELATIONAL_OFFICE_OPENING_DATE_COL, dateFormat); - setClientDataLookupTable(clientSheet); + + // Create hidden lookup sheet for lookup data + Sheet lookupSheet = workbook.createSheet(TemplatePopulateImportConstants.CLIENT_LOOKUPS_SHEET_NAME); + // TODO: For debugging, temporarily set to VISIBLE. Revert to VERY_HIDDEN once validated. + workbook.setSheetVisibility(workbook.getSheetIndex(lookupSheet), SheetVisibility.VERY_HIDDEN); + + setLayout(clientSheet); // fills datatableDropdowns + setClientDataLookupTable(lookupSheet); // now values exist setFormatStyle(workbook, clientSheet); setRules(clientSheet, dateFormat); } @@ -96,6 +155,10 @@ private void setFormatStyle(Workbook workbook, Sheet worksheet) { setFormatActivationAndSubmittedDate(row, ClientEntityConstants.SUBMITTED_ON_COL, dateCellStyle); setFormatActivationAndSubmittedDate(row, ClientEntityConstants.INCOPORATION_VALID_TILL_COL, dateCellStyle); setFormatActivationAndSubmittedDate(row, ClientEntityConstants.INCOPORATION_DATE_COL, dateCellStyle); + // Apply date formatting to datatable date columns + for (ClientEntityWorkbookPopulator.DatatableDateInfo dateInfo : datatableDateColumns.values()) { + setFormatActivationAndSubmittedDate(row, dateInfo.clientSheetColumnIndex, dateCellStyle); + } } } @@ -107,68 +170,91 @@ private void setFormatActivationAndSubmittedDate(Row row, int columnIndex, CellS cell.setCellStyle(cellStyle); } - private void setClientDataLookupTable(Sheet clientSheet) { - int rowIndex = 0; + private void setClientDataLookupTable(Sheet lookupSheet) { + // Data starts at row 2 (row index 1) to match named range references + // Column 0 (A): Client Types + int rowIndex = 1; // Start at row 2 (Excel row 2, POI index 1) for (CodeValueData clientTypeCodeValue : clientTypeCodeValues) { - Row row = clientSheet.getRow(++rowIndex); + Row row = lookupSheet.getRow(rowIndex); if (row == null) { - row = clientSheet.createRow(rowIndex); + row = lookupSheet.createRow(rowIndex); } - writeString(ClientEntityConstants.LOOKUP_CLIENT_TYPES, row, clientTypeCodeValue.getName() + "-" + clientTypeCodeValue.getId()); + writeString(0, row, clientTypeCodeValue.getName() + " (" + clientTypeCodeValue.getId() + ")"); + rowIndex++; } - rowIndex = 0; + + // Column 1 (B): Client Classification + rowIndex = 1; for (CodeValueData clientClassificationCodeValue : clientClassificationCodeValues) { - Row row = clientSheet.getRow(++rowIndex); + Row row = lookupSheet.getRow(rowIndex); if (row == null) { - row = clientSheet.createRow(rowIndex); + row = lookupSheet.createRow(rowIndex); } - writeString(ClientEntityConstants.LOOKUP_CLIENT_CLASSIFICATION, row, - clientClassificationCodeValue.getName() + "-" + clientClassificationCodeValue.getId()); + writeString(1, row, clientClassificationCodeValue.getName() + " (" + clientClassificationCodeValue.getId() + ")"); + rowIndex++; } - rowIndex = 0; + + // Column 2 (C): Constitution + rowIndex = 1; for (CodeValueData constitutionCodeValue : constitutionCodeValues) { - Row row = clientSheet.getRow(++rowIndex); + Row row = lookupSheet.getRow(rowIndex); if (row == null) { - row = clientSheet.createRow(rowIndex); + row = lookupSheet.createRow(rowIndex); } - writeString(ClientEntityConstants.LOOKUP_CONSTITUTION_COL, row, - constitutionCodeValue.getName() + "-" + constitutionCodeValue.getId()); + writeString(2, row, constitutionCodeValue.getName() + " (" + constitutionCodeValue.getId() + ")"); + rowIndex++; } - rowIndex = 0; + + // Column 3 (D): Main Business Line + rowIndex = 1; for (CodeValueData mainBusinessCodeValue : mainBusinesslineCodeValues) { - Row row = clientSheet.getRow(++rowIndex); + Row row = lookupSheet.getRow(rowIndex); if (row == null) { - row = clientSheet.createRow(rowIndex); + row = lookupSheet.createRow(rowIndex); } - writeString(ClientEntityConstants.LOOKUP_MAIN_BUSINESS_LINE, row, - mainBusinessCodeValue.getName() + "-" + mainBusinessCodeValue.getId()); + writeString(3, row, mainBusinessCodeValue.getName() + " (" + mainBusinessCodeValue.getId() + ")"); + rowIndex++; } - rowIndex = 0; + + // Column 4 (E): Address Type + rowIndex = 1; for (CodeValueData addressTypeCodeValue : addressTypesCodeValues) { - Row row = clientSheet.getRow(++rowIndex); + Row row = lookupSheet.getRow(rowIndex); if (row == null) { - row = clientSheet.createRow(rowIndex); + row = lookupSheet.createRow(rowIndex); } - writeString(ClientEntityConstants.LOOKUP_ADDRESS_TYPE, row, - addressTypeCodeValue.getName() + "-" + addressTypeCodeValue.getId()); + writeString(4, row, addressTypeCodeValue.getName() + " (" + addressTypeCodeValue.getId() + ")"); + rowIndex++; } - rowIndex = 0; + + // Column 5 (F): State/Province + rowIndex = 1; for (CodeValueData stateCodeValue : stateProvinceCodeValues) { - Row row = clientSheet.getRow(++rowIndex); + Row row = lookupSheet.getRow(rowIndex); if (row == null) { - row = clientSheet.createRow(rowIndex); + row = lookupSheet.createRow(rowIndex); } - writeString(ClientEntityConstants.LOOKUP_STATE_PROVINCE, row, stateCodeValue.getName() + "-" + stateCodeValue.getId()); + writeString(5, row, stateCodeValue.getName() + " (" + stateCodeValue.getId() + ")"); + rowIndex++; } - rowIndex = 0; + + // Column 6 (G): Country + rowIndex = 1; for (CodeValueData countryCodeValue : countryCodeValues) { - Row row = clientSheet.getRow(++rowIndex); + Row row = lookupSheet.getRow(rowIndex); if (row == null) { - row = clientSheet.createRow(rowIndex); + row = lookupSheet.createRow(rowIndex); } - writeString(ClientEntityConstants.LOOKUP_COUNTRY, row, countryCodeValue.getName() + "-" + countryCodeValue.getId()); + writeString(6, row, countryCodeValue.getName() + " (" + countryCodeValue.getId() + ")"); + rowIndex++; } - + + } + + private String sanitizeNamedRangeName(String name) { + // Excel named ranges cannot contain certain characters + // Replace invalid characters with underscore + return name.replaceAll("[ @#&()<>,;.:$£€§°\\\\/=!\\?\\-\\+\\*\"\\[\\]]", "_"); } private void setLayout(Sheet worksheet) { @@ -202,17 +288,7 @@ private void setLayout(Sheet worksheet) { worksheet.setColumnWidth(ClientEntityConstants.COUNTRY_COL, TemplatePopulateImportConstants.SMALL_COL_SIZE); worksheet.setColumnWidth(ClientEntityConstants.POSTAL_CODE_COL, TemplatePopulateImportConstants.SMALL_COL_SIZE); worksheet.setColumnWidth(ClientEntityConstants.IS_ACTIVE_ADDRESS_COL, TemplatePopulateImportConstants.SMALL_COL_SIZE); - worksheet.setColumnWidth(ClientEntityConstants.WARNING_COL, TemplatePopulateImportConstants.MEDIUM_COL_SIZE); - - worksheet.setColumnWidth(ClientEntityConstants.RELATIONAL_OFFICE_NAME_COL, TemplatePopulateImportConstants.MEDIUM_COL_SIZE); - worksheet.setColumnWidth(ClientEntityConstants.RELATIONAL_OFFICE_OPENING_DATE_COL, TemplatePopulateImportConstants.SMALL_COL_SIZE); - worksheet.setColumnWidth(ClientEntityConstants.LOOKUP_CONSTITUTION_COL, TemplatePopulateImportConstants.SMALL_COL_SIZE); - worksheet.setColumnWidth(ClientEntityConstants.LOOKUP_CLIENT_TYPES, TemplatePopulateImportConstants.SMALL_COL_SIZE); - worksheet.setColumnWidth(ClientEntityConstants.LOOKUP_CLIENT_CLASSIFICATION, TemplatePopulateImportConstants.SMALL_COL_SIZE); - worksheet.setColumnWidth(ClientEntityConstants.LOOKUP_ADDRESS_TYPE, TemplatePopulateImportConstants.SMALL_COL_SIZE); - worksheet.setColumnWidth(ClientEntityConstants.LOOKUP_STATE_PROVINCE, TemplatePopulateImportConstants.SMALL_COL_SIZE); - worksheet.setColumnWidth(ClientEntityConstants.LOOKUP_COUNTRY, TemplatePopulateImportConstants.SMALL_COL_SIZE); - worksheet.setColumnWidth(ClientEntityConstants.LOOKUP_MAIN_BUSINESS_LINE, TemplatePopulateImportConstants.MEDIUM_COL_SIZE); + worksheet.setColumnWidth(ClientEntityConstants.STATUS_COL, TemplatePopulateImportConstants.SMALL_COL_SIZE); writeString(ClientEntityConstants.NAME_COL, rowHeader, "Name"); writeString(ClientEntityConstants.OFFICE_NAME_COL, rowHeader, "Office Name*"); writeString(ClientEntityConstants.STAFF_NAME_COL, rowHeader, "Staff Name"); @@ -240,18 +316,322 @@ private void setLayout(Sheet worksheet) { writeString(ClientEntityConstants.COUNTRY_COL, rowHeader, "Country"); writeString(ClientEntityConstants.POSTAL_CODE_COL, rowHeader, "Postal Code"); writeString(ClientEntityConstants.IS_ACTIVE_ADDRESS_COL, rowHeader, "Is active Address ? "); - writeString(ClientEntityConstants.WARNING_COL, rowHeader, "All * marked fields are compulsory."); - - writeString(ClientEntityConstants.RELATIONAL_OFFICE_NAME_COL, rowHeader, "Lookup office Name "); - writeString(ClientEntityConstants.RELATIONAL_OFFICE_OPENING_DATE_COL, rowHeader, "Lookup Office Opened Date "); - writeString(ClientEntityConstants.LOOKUP_CONSTITUTION_COL, rowHeader, "Lookup Constitution "); - writeString(ClientEntityConstants.LOOKUP_CLIENT_TYPES, rowHeader, "Lookup Client Types "); - writeString(ClientEntityConstants.LOOKUP_CLIENT_CLASSIFICATION, rowHeader, "Lookup Client Classification "); - writeString(ClientEntityConstants.LOOKUP_ADDRESS_TYPE, rowHeader, "Lookup AddressType "); - writeString(ClientEntityConstants.LOOKUP_STATE_PROVINCE, rowHeader, "Lookup State/Province "); - writeString(ClientEntityConstants.LOOKUP_COUNTRY, rowHeader, "Lookup Country "); - writeString(ClientEntityConstants.LOOKUP_MAIN_BUSINESS_LINE, rowHeader, "Lookup Business Line"); + writeString(ClientEntityConstants.STATUS_COL, rowHeader, TemplatePopulateImportConstants.STATUS_COLUMN_HEADER); + + // Handle datatable columns (headers will be inserted here) + // This is called from within setLayout to ensure proper column positioning + int currentCol = handleDatatableColumnHeaders(worksheet, rowHeader); + + // Add warning message after all dynamic columns + writeString(currentCol, rowHeader, "All * marked fields are compulsory."); + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.MEDIUM_COL_SIZE); + currentCol++; + + // Insert 8 empty columns as a gap before lookup headers + currentCol += 8; + + // Add lookup columns dynamically after datatable columns + // These columns are for user reference and data validation + // Data values remain on the hidden lookup sheet + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.MEDIUM_COL_SIZE); + writeString(currentCol, rowHeader, "Lookup office Name "); + int lookupOfficeNameCol = currentCol++; + + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.SMALL_COL_SIZE); + writeString(currentCol, rowHeader, "Lookup Office Opened Date "); + int lookupOfficeDateCol = currentCol++; + + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.SMALL_COL_SIZE); + writeString(currentCol, rowHeader, "Lookup Constitution "); + int lookupConstitutionCol = currentCol++; + + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.SMALL_COL_SIZE); + writeString(currentCol, rowHeader, "Lookup Client Types "); + int lookupClientTypesCol = currentCol++; + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.SMALL_COL_SIZE); + writeString(currentCol, rowHeader, "Lookup Client Classification "); + int lookupClientClassificationCol = currentCol++; + + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.SMALL_COL_SIZE); + writeString(currentCol, rowHeader, "Lookup AddressType "); + int lookupAddressTypeCol = currentCol++; + + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.SMALL_COL_SIZE); + writeString(currentCol, rowHeader, "Lookup State/Province "); + int lookupStateProvinceCol = currentCol++; + + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.SMALL_COL_SIZE); + writeString(currentCol, rowHeader, "Lookup Country "); + int lookupCountryCol = currentCol++; + + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.MEDIUM_COL_SIZE); + writeString(currentCol, rowHeader, "Lookup Business Line"); + int lookupBusinessLineCol = currentCol++; + + // Populate visible lookup values under lookup headers (for user reference) + // These are read-only reference values; validations still use the hidden lookup sheet + populateLookupValues(worksheet, lookupOfficeNameCol, lookupOfficeDateCol, lookupConstitutionCol, + lookupClientTypesCol, lookupClientClassificationCol, lookupAddressTypeCol, + lookupStateProvinceCol, lookupCountryCol, lookupBusinessLineCol); + } + + private void populateLookupValues(Sheet worksheet, int lookupOfficeNameCol, int lookupOfficeDateCol, + int lookupConstitutionCol, int lookupClientTypesCol, int lookupClientClassificationCol, + int lookupAddressTypeCol, int lookupStateProvinceCol, int lookupCountryCol, int lookupBusinessLineCol) { + + // Populate Office Name lookup values + int rowIndex = 1; // Start at Excel row 2 (POI index 1) + List offices = officeSheetPopulator.getOffices(); + for (OfficeData office : offices) { + Row row = worksheet.getRow(rowIndex); + if (row == null) { + row = worksheet.createRow(rowIndex); + } + writeString(lookupOfficeNameCol, row, office.getName() + " (" + office.getId() + ")"); + if (office.getOpeningDate() != null) { + writeString(lookupOfficeDateCol, row, office.getOpeningDate().toString()); + } + rowIndex++; + } + + // Populate Constitution lookup values + rowIndex = 1; + for (CodeValueData constitutionCodeValue : constitutionCodeValues) { + Row row = worksheet.getRow(rowIndex); + if (row == null) { + row = worksheet.createRow(rowIndex); + } + writeString(lookupConstitutionCol, row, constitutionCodeValue.getName() + " (" + constitutionCodeValue.getId() + ")"); + rowIndex++; + } + + // Populate Client Types lookup values + rowIndex = 1; + for (CodeValueData clientTypeCodeValue : clientTypeCodeValues) { + Row row = worksheet.getRow(rowIndex); + if (row == null) { + row = worksheet.createRow(rowIndex); + } + writeString(lookupClientTypesCol, row, clientTypeCodeValue.getName() + " (" + clientTypeCodeValue.getId() + ")"); + rowIndex++; + } + + // Populate Client Classification lookup values + rowIndex = 1; + for (CodeValueData clientClassificationCodeValue : clientClassificationCodeValues) { + Row row = worksheet.getRow(rowIndex); + if (row == null) { + row = worksheet.createRow(rowIndex); + } + writeString(lookupClientClassificationCol, row, clientClassificationCodeValue.getName() + " (" + clientClassificationCodeValue.getId() + ")"); + rowIndex++; + } + + // Populate Address Type lookup values + rowIndex = 1; + for (CodeValueData addressTypeCodeValue : addressTypesCodeValues) { + Row row = worksheet.getRow(rowIndex); + if (row == null) { + row = worksheet.createRow(rowIndex); + } + writeString(lookupAddressTypeCol, row, addressTypeCodeValue.getName() + " (" + addressTypeCodeValue.getId() + ")"); + rowIndex++; + } + + // Populate State/Province lookup values + rowIndex = 1; + for (CodeValueData stateCodeValue : stateProvinceCodeValues) { + Row row = worksheet.getRow(rowIndex); + if (row == null) { + row = worksheet.createRow(rowIndex); + } + writeString(lookupStateProvinceCol, row, stateCodeValue.getName() + " (" + stateCodeValue.getId() + ")"); + rowIndex++; + } + + // Populate Country lookup values + rowIndex = 1; + for (CodeValueData countryCodeValue : countryCodeValues) { + Row row = worksheet.getRow(rowIndex); + if (row == null) { + row = worksheet.createRow(rowIndex); + } + writeString(lookupCountryCol, row, countryCodeValue.getName() + " (" + countryCodeValue.getId() + ")"); + rowIndex++; + } + + // Populate Business Line lookup values + rowIndex = 1; + for (CodeValueData mainBusinessCodeValue : mainBusinesslineCodeValues) { + Row row = worksheet.getRow(rowIndex); + if (row == null) { + row = worksheet.createRow(rowIndex); + } + writeString(lookupBusinessLineCol, row, mainBusinessCodeValue.getName() + " (" + mainBusinessCodeValue.getId() + ")"); + rowIndex++; + } + } + + /** + * Handles datatable column header creation and dropdown tracking. + * This is called from within setLayout() to insert headers at the correct position. + * + * @param worksheet The client sheet + * @param rowHeader The header row + * @return The next column index after datatable columns (or STATUS_COL + 1 if no datatables) + */ + private int handleDatatableColumnHeaders(Sheet worksheet, Row rowHeader) { + int startCol = ClientEntityConstants.STATUS_COL + 1; + try { + int currentCol = startCol; + int lookupSheetCol = 7; // Start after Country (column G = index 6), so next available is H = index 7 + + for (DatatableData datatable : requiredDatatables) { + if (datatable == null) { + continue; + } + String datatableName = datatable.getRegisteredTableName(); + List columns = datatable.getColumnHeaderData(); + if (columns != null) { + for (ResultsetColumnHeaderData column : columns) { + try { + // Skip system columns (id, client_id, etc.) + String columnName = column.getColumnName(); + if (columnName == null || columnName.equalsIgnoreCase("id") + || columnName.equalsIgnoreCase("client_id") + || columnName.equalsIgnoreCase("created_at") + || columnName.equalsIgnoreCase("updated_at")) { + continue; + } + + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.MEDIUM_COL_SIZE); + // Use dot notation: registeredTableName.columnName for unambiguous parsing + // For single-row datatables: treat as mandatory inline data (append "*" if column is not nullable) + // For multi-row datatables: treat as optional/repeatable child data (no "*" even if not nullable) + String headerLabel = datatableName + "." + columnName; + if (!datatable.isMultiRow() && !column.getIsColumnNullable()) { + // Single-row datatables: mark mandatory columns with "*" + headerLabel += "*"; + } + // Multi-row datatables are always optional (no "*" marker) + writeString(currentCol, rowHeader, headerLabel); + + // Check if this is a dropdown column (CODELOOKUP) with values + if (column.isCodeLookupDisplayType() && column.hasColumnValues()) { + String namedRangeName = sanitizeNamedRangeName(datatableName + "_" + columnName); + List columnValues = column.getColumnValues(); + if (columnValues != null && !columnValues.isEmpty()) { + datatableDropdowns.put(namedRangeName, new DatatableDropdownInfo(currentCol, lookupSheetCol, columnValues)); + lookupSheetCol++; + } + } + // Check if this is a date or datetime column + else if (column.isDateDisplayType() || column.isDateTimeDisplayType()) { + datatableDateColumns.put(currentCol, new ClientEntityWorkbookPopulator.DatatableDateInfo(currentCol, column.isDateTimeDisplayType())); + } + // Check if this is a boolean column + else if (column.isBooleanDisplayType()) { + datatableBooleanColumns.put(currentCol, new ClientEntityWorkbookPopulator.DatatableBooleanInfo(currentCol)); + } + + currentCol++; + } catch (Exception e) { + String datatableNameForLog = datatableName != null ? datatableName : "unknown"; + String columnNameForLog = column != null && column.getColumnName() != null ? column.getColumnName() : "unknown"; + log.warn("Failed to process datatable column '{}' in datatable '{}': {}", + columnNameForLog, datatableNameForLog, e.getMessage()); + // Continue with next column + } + } + } + } + return currentCol; + } catch (Exception e) { + log.warn("Failed to handle datatable column headers in client bulk import template: {}. Template generation will continue without datatable columns.", + e.getMessage()); + return startCol; // Return start position if datatables failed + } + } + + /** + * Completes datatable setup: lookup sheet population, named ranges, and validation. + * This is called after setLayout() to complete datatable enhancements. + * All operations are wrapped in try/catch to ensure datatable failures never break template generation. + * + * @param worksheet The client sheet + * @param rowHeader The header row + * @param lookupSheet The hidden lookup sheet + * @param workbook The workbook + * @param dateFormat The date format + */ + private void handleDatatableColumnsComplete(Sheet worksheet, Row rowHeader, Sheet lookupSheet, Workbook workbook, String dateFormat) { + try { + // Step 1: Populate datatable dropdown values into lookup sheet + for (Map.Entry entry : datatableDropdowns.entrySet()) { + try { + DatatableDropdownInfo dropdownInfo = entry.getValue(); + int rowIndex = 1; // Start at row 2 (Excel row 2, POI index 1) + for (ResultsetColumnValueData valueData : dropdownInfo.values) { + Row row = lookupSheet.getRow(rowIndex); + if (row == null) { + row = lookupSheet.createRow(rowIndex); + } + writeString(dropdownInfo.lookupSheetColumnIndex, row, valueData.getValue() + " (" + valueData.getId() + ")"); + rowIndex++; + } + } catch (Exception e) { + String namedRangeName = entry.getKey(); + log.warn("Failed to populate dropdown values for datatable named range '{}': {}", + namedRangeName, e.getMessage()); + // Continue with next dropdown + } + } + + // Step 2: Create named ranges for datatable dropdown columns + String lookupSheetName = TemplatePopulateImportConstants.CLIENT_LOOKUPS_SHEET_NAME; + Workbook clientWorkbook = worksheet.getWorkbook(); + for (Map.Entry entry : datatableDropdowns.entrySet()) { + try { + String namedRangeName = entry.getKey(); + DatatableDropdownInfo dropdownInfo = entry.getValue(); + Name datatableDropdownGroup = clientWorkbook.createName(); + setSanitized(datatableDropdownGroup, namedRangeName); + int dropdownLastRow = dropdownInfo.values.size() + 1; // +1 because data starts at row 2 + String dropdownCol = CellReference.convertNumToColString(dropdownInfo.lookupSheetColumnIndex); + datatableDropdownGroup.setRefersToFormula( + "'" + lookupSheetName + "'!$" + dropdownCol + "$2:$" + dropdownCol + "$" + dropdownLastRow); + } catch (Exception e) { + String namedRangeName = entry.getKey(); + log.warn("Failed to create named range '{}' for datatable dropdown: {}", + namedRangeName, e.getMessage()); + // Continue with next named range + } + } + + // Step 3: Add data validation for datatable dropdown columns + DataValidationHelper validationHelper = new HSSFDataValidationHelper((HSSFSheet) worksheet); + for (Map.Entry entry : datatableDropdowns.entrySet()) { + try { + String namedRangeName = entry.getKey(); + DatatableDropdownInfo dropdownInfo = entry.getValue(); + CellRangeAddressList datatableDropdownRange = new CellRangeAddressList(1, SpreadsheetVersion.EXCEL97.getLastRowIndex(), + dropdownInfo.clientSheetColumnIndex, dropdownInfo.clientSheetColumnIndex); + DataValidationConstraint datatableDropdownConstraint = validationHelper.createFormulaListConstraint(namedRangeName); + DataValidation datatableDropdownValidation = validationHelper.createValidation(datatableDropdownConstraint, datatableDropdownRange); + worksheet.addValidationData(datatableDropdownValidation); + } catch (Exception e) { + String namedRangeName = entry.getKey(); + log.warn("Failed to add data validation for datatable dropdown '{}': {}", + namedRangeName, e.getMessage()); + // Continue with next validation + } + } + } catch (Exception e) { + log.warn("Failed to complete datatable setup in client bulk import template: {}. Template generation will continue without datatable enhancements.", + e.getMessage()); + // Do not throw - allow template generation to continue + } } private void setRules(Sheet worksheet, String dateFormat) { @@ -350,48 +730,116 @@ private void setRules(Sheet worksheet, String dateFormat) { worksheet.addValidationData(activeAddressValidation); worksheet.addValidationData(incorporateDateValidation); worksheet.addValidationData(incorporateDateTillValidation); + + + // Add date validation for datatable date columns + for (ClientEntityWorkbookPopulator.DatatableDateInfo dateInfo : datatableDateColumns.values()) { + try { + CellRangeAddressList datatableDateRange = new CellRangeAddressList(1, SpreadsheetVersion.EXCEL97.getLastRowIndex(), + dateInfo.clientSheetColumnIndex, dateInfo.clientSheetColumnIndex); + // Use same date constraint as submittedOnDate (less than or equal to today) + DataValidationConstraint datatableDateConstraint = validationHelper + .createDateConstraint(DataValidationConstraint.OperatorType.LESS_OR_EQUAL, "=TODAY()", null, dateFormat); + DataValidation datatableDateValidation = validationHelper.createValidation(datatableDateConstraint, datatableDateRange); + worksheet.addValidationData(datatableDateValidation); + } catch (Exception e) { + log.warn("Failed to add date validation for datatable date column at index {}: {}", + dateInfo.clientSheetColumnIndex, e.getMessage()); + } + } + + // Add boolean validation for datatable boolean columns + for (ClientEntityWorkbookPopulator.DatatableBooleanInfo booleanInfo : datatableBooleanColumns.values()) { + try { + CellRangeAddressList datatableBooleanRange = new CellRangeAddressList(1, SpreadsheetVersion.EXCEL97.getLastRowIndex(), + booleanInfo.clientSheetColumnIndex, booleanInfo.clientSheetColumnIndex); + // Use same boolean constraint as active field (True/False) + DataValidationConstraint datatableBooleanConstraint = validationHelper.createExplicitListConstraint(new String[] { "True", "False" }); + DataValidation datatableBooleanValidation = validationHelper.createValidation(datatableBooleanConstraint, datatableBooleanRange); + worksheet.addValidationData(datatableBooleanValidation); + } catch (Exception e) { + log.warn("Failed to add boolean validation for datatable boolean column at index {}: {}", + booleanInfo.clientSheetColumnIndex, e.getMessage()); + } + } } private void setNames(Sheet worksheet, List offices) { Workbook clientWorkbook = worksheet.getWorkbook(); + String lookupSheetName = TemplatePopulateImportConstants.CLIENT_LOOKUPS_SHEET_NAME; + Name officeGroup = clientWorkbook.createName(); officeGroup.setNameName("Office"); officeGroup.setRefersToFormula(TemplatePopulateImportConstants.OFFICE_SHEET_NAME + "!$B$2:$B$" + (offices.size() + 1)); + // All lookup named ranges explicitly reference the hidden lookup sheet + // Column 0 (A): Client Types Name clientTypeGroup = clientWorkbook.createName(); clientTypeGroup.setNameName("ClientTypes"); + int clientTypesLastRow = clientTypeCodeValues.size() + 1; // +1 because data starts at row 2 + String clientTypesCol = CellReference.convertNumToColString(0); clientTypeGroup.setRefersToFormula( - TemplatePopulateImportConstants.CLIENT_ENTITY_SHEET_NAME + "!$AN$2:$AN$" + (clientTypeCodeValues.size() + 1)); + "'" + lookupSheetName + "'!$" + clientTypesCol + "$2:$" + clientTypesCol + "$" + clientTypesLastRow); + // Column 2 (C): Constitution Name constitutionGroup = clientWorkbook.createName(); constitutionGroup.setNameName("Constitution"); + int constitutionLastRow = constitutionCodeValues.size() + 1; + String constitutionCol = CellReference.convertNumToColString(2); constitutionGroup.setRefersToFormula( - TemplatePopulateImportConstants.CLIENT_ENTITY_SHEET_NAME + "!$AL$2:$AL$" + (constitutionCodeValues.size() + 1)); + "'" + lookupSheetName + "'!$" + constitutionCol + "$2:$" + constitutionCol + "$" + constitutionLastRow); + // Column 3 (D): Main Business Line Name mainBusinessLineGroup = clientWorkbook.createName(); mainBusinessLineGroup.setNameName("MainBusinessLine"); + int mainBusinessLineLastRow = mainBusinesslineCodeValues.size() + 1; + String mainBusinessLineCol = CellReference.convertNumToColString(3); mainBusinessLineGroup.setRefersToFormula( - TemplatePopulateImportConstants.CLIENT_ENTITY_SHEET_NAME + "!$AR$2:$AR$" + (mainBusinesslineCodeValues.size() + 1)); + "'" + lookupSheetName + "'!$" + mainBusinessLineCol + "$2:$" + mainBusinessLineCol + "$" + mainBusinessLineLastRow); + // Column 1 (B): Client Classification Name clientClassficationGroup = clientWorkbook.createName(); clientClassficationGroup.setNameName("ClientClassification"); + int clientClassificationLastRow = clientClassificationCodeValues.size() + 1; + String clientClassificationCol = CellReference.convertNumToColString(1); clientClassficationGroup.setRefersToFormula( - TemplatePopulateImportConstants.CLIENT_ENTITY_SHEET_NAME + "!$AM$2:$AM$" + (clientClassificationCodeValues.size() + 1)); + "'" + lookupSheetName + "'!$" + clientClassificationCol + "$2:$" + clientClassificationCol + "$" + clientClassificationLastRow); + // Column 4 (E): Address Type Name addressTypeGroup = clientWorkbook.createName(); addressTypeGroup.setNameName("AddressType"); + int addressTypeLastRow = addressTypesCodeValues.size() + 1; + String addressTypeCol = CellReference.convertNumToColString(4); addressTypeGroup.setRefersToFormula( - TemplatePopulateImportConstants.CLIENT_ENTITY_SHEET_NAME + "!$AO$2:$AO$" + (addressTypesCodeValues.size() + 1)); + "'" + lookupSheetName + "'!$" + addressTypeCol + "$2:$" + addressTypeCol + "$" + addressTypeLastRow); + // Column 5 (F): State/Province Name stateProvinceGroup = clientWorkbook.createName(); stateProvinceGroup.setNameName("StateProvince"); + int stateProvinceLastRow = stateProvinceCodeValues.size() + 1; + String stateProvinceCol = CellReference.convertNumToColString(5); stateProvinceGroup.setRefersToFormula( - TemplatePopulateImportConstants.CLIENT_ENTITY_SHEET_NAME + "!$AP$2:$AP$" + (stateProvinceCodeValues.size() + 1)); + "'" + lookupSheetName + "'!$" + stateProvinceCol + "$2:$" + stateProvinceCol + "$" + stateProvinceLastRow); + // Column 6 (G): Country Name countryGroup = clientWorkbook.createName(); countryGroup.setNameName("Country"); + int countryLastRow = countryCodeValues.size() + 1; + String countryCol = CellReference.convertNumToColString(6); countryGroup.setRefersToFormula( - TemplatePopulateImportConstants.CLIENT_ENTITY_SHEET_NAME + "!$AQ$2:$AQ$" + (countryCodeValues.size() + 1)); + "'" + lookupSheetName + "'!$" + countryCol + "$2:$" + countryCol + "$" + countryLastRow); + + // Create named ranges for datatable dropdown columns + for (Map.Entry entry : datatableDropdowns.entrySet()) { + String namedRangeName = entry.getKey(); + DatatableDropdownInfo dropdownInfo = entry.getValue(); + Name datatableDropdownGroup = clientWorkbook.createName(); + setSanitized(datatableDropdownGroup, namedRangeName); + int dropdownLastRow = dropdownInfo.values.size() + 1; + String dropdownCol = CellReference.convertNumToColString(dropdownInfo.lookupSheetColumnIndex); + datatableDropdownGroup.setRefersToFormula( + "'" + lookupSheetName + "'!$" + dropdownCol + "$2:$" + dropdownCol + "$" + dropdownLastRow); + } for (Integer i = 0; i < offices.size(); i++) { Integer[] officeNameToBeginEndIndexesOfStaff = personnelSheetPopulator.getOfficeNameToBeginEndIndexesOfStaff().get(i); diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/client/ClientPersonWorkbookPopulator.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/client/ClientPersonWorkbookPopulator.java index d3d43f45642..02bc0004a3d 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/client/ClientPersonWorkbookPopulator.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/client/ClientPersonWorkbookPopulator.java @@ -25,7 +25,10 @@ import org.apache.fineract.infrastructure.bulkimport.populator.OfficeSheetPopulator; import org.apache.fineract.infrastructure.bulkimport.populator.PersonnelSheetPopulator; import org.apache.fineract.infrastructure.codes.data.CodeValueData; +import org.apache.fineract.infrastructure.dataqueries.data.DatatableData; +import org.apache.fineract.infrastructure.dataqueries.data.ResultsetColumnHeaderData; import org.apache.fineract.organisation.office.data.OfficeData; +import java.util.ArrayList; import org.apache.poi.hssf.usermodel.HSSFDataValidationHelper; import org.apache.poi.hssf.usermodel.HSSFSheet; import org.apache.poi.ss.SpreadsheetVersion; @@ -38,11 +41,20 @@ import org.apache.poi.ss.usermodel.Name; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.SheetVisibility; import org.apache.poi.ss.usermodel.Workbook; import org.apache.poi.ss.util.CellRangeAddressList; +import org.apache.poi.ss.util.CellReference; +import org.apache.fineract.infrastructure.dataqueries.data.ResultsetColumnValueData; +import java.util.Map; +import java.util.HashMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class ClientPersonWorkbookPopulator extends AbstractWorkbookPopulator { + private static final Logger log = LoggerFactory.getLogger(ClientPersonWorkbookPopulator.class); + private final OfficeSheetPopulator officeSheetPopulator; private final PersonnelSheetPopulator personnelSheetPopulator; private final List clientTypeCodeValues; @@ -51,11 +63,54 @@ public class ClientPersonWorkbookPopulator extends AbstractWorkbookPopulator { private final List addressTypesCodeValues; private final List stateProvinceCodeValues; private final List countryCodeValues; + private final List requiredDatatables; + + // Track datatable dropdown columns: namedRangeName -> (clientSheetColumnIndex, lookupSheetColumnIndex, values) + private final Map datatableDropdowns = new HashMap<>(); + + // Track datatable date columns: columnIndex -> columnInfo + private final Map datatableDateColumns = new HashMap<>(); + + // Track datatable boolean columns: columnIndex -> columnInfo + private final Map datatableBooleanColumns = new HashMap<>(); + + // Inner class to track datatable dropdown information + private static class DatatableDropdownInfo { + final int clientSheetColumnIndex; + final int lookupSheetColumnIndex; + final List values; + + DatatableDropdownInfo(int clientSheetColumnIndex, int lookupSheetColumnIndex, List values) { + this.clientSheetColumnIndex = clientSheetColumnIndex; + this.lookupSheetColumnIndex = lookupSheetColumnIndex; + this.values = values; + } + } + + // Inner class to track datatable date column information + private static class DatatableDateInfo { + final int clientSheetColumnIndex; + final boolean isDateTime; + + DatatableDateInfo(int clientSheetColumnIndex, boolean isDateTime) { + this.clientSheetColumnIndex = clientSheetColumnIndex; + this.isDateTime = isDateTime; + } + } + + // Inner class to track datatable boolean column information + private static class DatatableBooleanInfo { + final int clientSheetColumnIndex; + + DatatableBooleanInfo(int clientSheetColumnIndex) { + this.clientSheetColumnIndex = clientSheetColumnIndex; + } + } public ClientPersonWorkbookPopulator(OfficeSheetPopulator officeSheetPopulator, PersonnelSheetPopulator personnelSheetPopulator, List clientTypeCodeValues, List genderCodeValues, List clientClassification, List addressTypesCodeValues, List stateProvinceCodeValues, - List countryCodeValues) { + List countryCodeValues, List requiredDatatables) { this.officeSheetPopulator = officeSheetPopulator; this.personnelSheetPopulator = personnelSheetPopulator; this.clientTypeCodeValues = clientTypeCodeValues; @@ -64,6 +119,7 @@ public ClientPersonWorkbookPopulator(OfficeSheetPopulator officeSheetPopulator, this.addressTypesCodeValues = addressTypesCodeValues; this.stateProvinceCodeValues = stateProvinceCodeValues; this.countryCodeValues = countryCodeValues; + this.requiredDatatables = requiredDatatables != null ? requiredDatatables : new ArrayList<>(); } @Override @@ -71,12 +127,22 @@ public void populate(Workbook workbook, String dateFormat) { Sheet clientSheet = workbook.createSheet(TemplatePopulateImportConstants.CLIENT_PERSON_SHEET_NAME); personnelSheetPopulator.populate(workbook, dateFormat); officeSheetPopulator.populate(workbook, dateFormat); + // Create hidden lookup sheet for lookup data + Sheet lookupSheet = workbook.createSheet(TemplatePopulateImportConstants.CLIENT_LOOKUPS_SHEET_NAME); + // TODO: For debugging, temporarily set to VISIBLE. Revert to VERY_HIDDEN once validated. + workbook.setSheetVisibility(workbook.getSheetIndex(lookupSheet), SheetVisibility.VERY_HIDDEN); + setLayout(clientSheet); - setOfficeDateLookupTable(clientSheet, officeSheetPopulator.getOffices(), ClientPersonConstants.RELATIONAL_OFFICE_NAME_COL, - ClientPersonConstants.RELATIONAL_OFFICE_OPENING_DATE_COL, dateFormat); - setClientDataLookupTable(clientSheet); + setClientDataLookupTable(lookupSheet); setFormatStyle(workbook, clientSheet); setRules(clientSheet, dateFormat); + + // Complete datatable handling (lookup population, named ranges, validation) + // This is called after setLayout to complete datatable setup + Row rowHeader = clientSheet.getRow(TemplatePopulateImportConstants.ROWHEADER_INDEX); + if (rowHeader != null) { + handleDatatableColumnsComplete(clientSheet, rowHeader, lookupSheet, workbook, dateFormat); + } } private void setFormatStyle(Workbook workbook, Sheet worksheet) { @@ -92,6 +158,11 @@ private void setFormatStyle(Workbook workbook, Sheet worksheet) { setFormatActivationAndSubmittedDate(row, ClientPersonConstants.ACTIVATION_DATE_COL, dateCellStyle); setFormatActivationAndSubmittedDate(row, ClientPersonConstants.SUBMITTED_ON_COL, dateCellStyle); + + // Apply date formatting to datatable date columns + for (DatatableDateInfo dateInfo : datatableDateColumns.values()) { + setFormatActivationAndSubmittedDate(row, dateInfo.clientSheetColumnIndex, dateCellStyle); + } } } @@ -103,59 +174,80 @@ private void setFormatActivationAndSubmittedDate(Row row, int columnIndex, CellS cell.setCellStyle(cellStyle); } - private void setClientDataLookupTable(Sheet clientSheet) { - int rowIndex = 0; + private void setClientDataLookupTable(Sheet lookupSheet) { + // Data starts at row 2 (row index 1) to match named range references + // Column 0 (A): Client Types + int rowIndex = 1; // Start at row 2 (Excel row 2, POI index 1) for (CodeValueData clientTypeCodeValue : clientTypeCodeValues) { - Row row = clientSheet.getRow(++rowIndex); + Row row = lookupSheet.getRow(rowIndex); if (row == null) { - row = clientSheet.createRow(rowIndex); + row = lookupSheet.createRow(rowIndex); } - writeString(ClientPersonConstants.LOOKUP_CLIENT_TYPES_COL, row, - clientTypeCodeValue.getName() + "-" + clientTypeCodeValue.getId()); + writeString(0, row, clientTypeCodeValue.getName() + " (" + clientTypeCodeValue.getId() + ")"); + rowIndex++; } - rowIndex = 0; + + // Column 1 (B): Client Classification + rowIndex = 1; for (CodeValueData clientClassificationCodeValue : clientClassificationCodeValues) { - Row row = clientSheet.getRow(++rowIndex); + Row row = lookupSheet.getRow(rowIndex); if (row == null) { - row = clientSheet.createRow(rowIndex); + row = lookupSheet.createRow(rowIndex); } - writeString(ClientPersonConstants.LOOKUP_CLIENT_CLASSIFICATION_COL, row, - clientClassificationCodeValue.getName() + "-" + clientClassificationCodeValue.getId()); + writeString(1, row, clientClassificationCodeValue.getName() + " (" + clientClassificationCodeValue.getId() + ")"); + rowIndex++; } - rowIndex = 0; + + // Column 2 (C): Gender + rowIndex = 1; for (CodeValueData genderCodeValue : genderCodeValues) { - Row row = clientSheet.getRow(++rowIndex); + Row row = lookupSheet.getRow(rowIndex); if (row == null) { - row = clientSheet.createRow(rowIndex); + row = lookupSheet.createRow(rowIndex); } - writeString(ClientPersonConstants.LOOKUP_GENDER_COL, row, genderCodeValue.getName() + "-" + genderCodeValue.getId()); + writeString(2, row, genderCodeValue.getName() + " (" + genderCodeValue.getId() + ")"); + rowIndex++; } - rowIndex = 0; + + // Column 3 (D): Address Type + rowIndex = 1; for (CodeValueData addressTypeCodeValue : addressTypesCodeValues) { - Row row = clientSheet.getRow(++rowIndex); + Row row = lookupSheet.getRow(rowIndex); if (row == null) { - row = clientSheet.createRow(rowIndex); + row = lookupSheet.createRow(rowIndex); } - writeString(ClientPersonConstants.LOOKUP_ADDRESS_TYPE_COL, row, - addressTypeCodeValue.getName() + "-" + addressTypeCodeValue.getId()); + writeString(3, row, addressTypeCodeValue.getName() + " (" + addressTypeCodeValue.getId() + ")"); + rowIndex++; } - rowIndex = 0; + + // Column 4 (E): State/Province + rowIndex = 1; for (CodeValueData stateCodeValue : stateProvinceCodeValues) { - Row row = clientSheet.getRow(++rowIndex); + Row row = lookupSheet.getRow(rowIndex); if (row == null) { - row = clientSheet.createRow(rowIndex); + row = lookupSheet.createRow(rowIndex); } - writeString(ClientPersonConstants.LOOKUP_STATE_PROVINCE_COL, row, stateCodeValue.getName() + "-" + stateCodeValue.getId()); + writeString(4, row, stateCodeValue.getName() + " (" + stateCodeValue.getId() + ")"); + rowIndex++; } - rowIndex = 0; + + // Column 5 (F): Country + rowIndex = 1; for (CodeValueData countryCodeValue : countryCodeValues) { - Row row = clientSheet.getRow(++rowIndex); + Row row = lookupSheet.getRow(rowIndex); if (row == null) { - row = clientSheet.createRow(rowIndex); + row = lookupSheet.createRow(rowIndex); } - writeString(ClientPersonConstants.LOOKUP_COUNTRY_COL, row, countryCodeValue.getName() + "-" + countryCodeValue.getId()); + writeString(5, row, countryCodeValue.getName() + " (" + countryCodeValue.getId() + ")"); + rowIndex++; } - + + } + + private String sanitizeNamedRangeName(String name) { + // Excel named ranges cannot contain certain characters + // Replace invalid characters with underscore + return name.replaceAll("[ @#&()<>,;.:$£€§°\\\\/=!\\?\\-\\+\\*\"\\[\\]]", "_"); } private void setLayout(Sheet worksheet) { @@ -190,16 +282,7 @@ private void setLayout(Sheet worksheet) { worksheet.setColumnWidth(ClientPersonConstants.COUNTRY_COL, TemplatePopulateImportConstants.SMALL_COL_SIZE); worksheet.setColumnWidth(ClientPersonConstants.POSTAL_CODE_COL, TemplatePopulateImportConstants.SMALL_COL_SIZE); worksheet.setColumnWidth(ClientPersonConstants.IS_ACTIVE_ADDRESS_COL, TemplatePopulateImportConstants.SMALL_COL_SIZE); - worksheet.setColumnWidth(ClientPersonConstants.WARNING_COL, TemplatePopulateImportConstants.MEDIUM_COL_SIZE); - - worksheet.setColumnWidth(ClientPersonConstants.RELATIONAL_OFFICE_NAME_COL, TemplatePopulateImportConstants.MEDIUM_COL_SIZE); - worksheet.setColumnWidth(ClientPersonConstants.RELATIONAL_OFFICE_OPENING_DATE_COL, TemplatePopulateImportConstants.SMALL_COL_SIZE); - worksheet.setColumnWidth(ClientPersonConstants.LOOKUP_GENDER_COL, TemplatePopulateImportConstants.SMALL_COL_SIZE); - worksheet.setColumnWidth(ClientPersonConstants.LOOKUP_CLIENT_TYPES_COL, TemplatePopulateImportConstants.SMALL_COL_SIZE); - worksheet.setColumnWidth(ClientPersonConstants.LOOKUP_CLIENT_CLASSIFICATION_COL, TemplatePopulateImportConstants.SMALL_COL_SIZE); - worksheet.setColumnWidth(ClientPersonConstants.LOOKUP_ADDRESS_TYPE_COL, TemplatePopulateImportConstants.SMALL_COL_SIZE); - worksheet.setColumnWidth(ClientPersonConstants.LOOKUP_STATE_PROVINCE_COL, TemplatePopulateImportConstants.SMALL_COL_SIZE); - worksheet.setColumnWidth(ClientPersonConstants.LOOKUP_COUNTRY_COL, TemplatePopulateImportConstants.SMALL_COL_SIZE); + worksheet.setColumnWidth(ClientPersonConstants.STATUS_COL, TemplatePopulateImportConstants.SMALL_COL_SIZE); writeString(ClientPersonConstants.OFFICE_NAME_COL, rowHeader, "Office Name*"); writeString(ClientPersonConstants.STAFF_NAME_COL, rowHeader, "Staff Name"); writeString(ClientPersonConstants.EXTERNAL_ID_COL, rowHeader, "External ID "); @@ -223,17 +306,307 @@ private void setLayout(Sheet worksheet) { writeString(ClientPersonConstants.COUNTRY_COL, rowHeader, "Country "); writeString(ClientPersonConstants.POSTAL_CODE_COL, rowHeader, "Postal Code "); writeString(ClientPersonConstants.IS_ACTIVE_ADDRESS_COL, rowHeader, "Is active Address ? "); - writeString(ClientPersonConstants.WARNING_COL, rowHeader, "All * marked fields are compulsory."); + writeString(ClientPersonConstants.STATUS_COL, rowHeader, TemplatePopulateImportConstants.STATUS_COLUMN_HEADER); + + // Handle datatable columns (headers will be inserted here) + // This is called from within setLayout to ensure proper column positioning + int currentCol = handleDatatableColumnHeaders(worksheet, rowHeader); + + // Add warning message after all dynamic columns + writeString(currentCol, rowHeader, "All * marked fields are compulsory."); + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.MEDIUM_COL_SIZE); + currentCol++; + + // Insert 8 empty columns as a gap before lookup headers + currentCol += 8; + + // Add lookup columns dynamically after datatable columns + // These columns are for user reference and data validation + // Data values remain on the hidden lookup sheet + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.MEDIUM_COL_SIZE); + writeString(currentCol, rowHeader, "Lookup office Name "); + int lookupOfficeNameCol = currentCol++; + + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.SMALL_COL_SIZE); + writeString(currentCol, rowHeader, "Lookup Office Opened Date "); + int lookupOfficeDateCol = currentCol++; + + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.SMALL_COL_SIZE); + writeString(currentCol, rowHeader, "Lookup Gender "); + int lookupGenderCol = currentCol++; + + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.SMALL_COL_SIZE); + writeString(currentCol, rowHeader, "Lookup Client Types "); + int lookupClientTypesCol = currentCol++; + + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.SMALL_COL_SIZE); + writeString(currentCol, rowHeader, "Lookup Client Classification "); + int lookupClientClassificationCol = currentCol++; + + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.SMALL_COL_SIZE); + writeString(currentCol, rowHeader, "Lookup AddressType "); + int lookupAddressTypeCol = currentCol++; + + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.SMALL_COL_SIZE); + writeString(currentCol, rowHeader, "Lookup State/Province "); + int lookupStateProvinceCol = currentCol++; + + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.SMALL_COL_SIZE); + writeString(currentCol, rowHeader, "Lookup Country "); + int lookupCountryCol = currentCol++; + + // Populate visible lookup values under lookup headers (for user reference) + // These are read-only reference values; validations still use the hidden lookup sheet + populateLookupValues(worksheet, lookupOfficeNameCol, lookupOfficeDateCol, lookupGenderCol, + lookupClientTypesCol, lookupClientClassificationCol, lookupAddressTypeCol, + lookupStateProvinceCol, lookupCountryCol); + } + + private void populateLookupValues(Sheet worksheet, int lookupOfficeNameCol, int lookupOfficeDateCol, + int lookupGenderCol, int lookupClientTypesCol, int lookupClientClassificationCol, + int lookupAddressTypeCol, int lookupStateProvinceCol, int lookupCountryCol) { + + // Populate Office Name lookup values + int rowIndex = 1; // Start at Excel row 2 (POI index 1) + List offices = officeSheetPopulator.getOffices(); + for (OfficeData office : offices) { + Row row = worksheet.getRow(rowIndex); + if (row == null) { + row = worksheet.createRow(rowIndex); + } + writeString(lookupOfficeNameCol, row, office.getName() + " (" + office.getId() + ")"); + if (office.getOpeningDate() != null) { + writeString(lookupOfficeDateCol, row, office.getOpeningDate().toString()); + } + rowIndex++; + } + + // Populate Gender lookup values + rowIndex = 1; + for (CodeValueData genderCodeValue : genderCodeValues) { + Row row = worksheet.getRow(rowIndex); + if (row == null) { + row = worksheet.createRow(rowIndex); + } + writeString(lookupGenderCol, row, genderCodeValue.getName() + " (" + genderCodeValue.getId() + ")"); + rowIndex++; + } + + // Populate Client Types lookup values + rowIndex = 1; + for (CodeValueData clientTypeCodeValue : clientTypeCodeValues) { + Row row = worksheet.getRow(rowIndex); + if (row == null) { + row = worksheet.createRow(rowIndex); + } + writeString(lookupClientTypesCol, row, clientTypeCodeValue.getName() + " (" + clientTypeCodeValue.getId() + ")"); + rowIndex++; + } + + // Populate Client Classification lookup values + rowIndex = 1; + for (CodeValueData clientClassificationCodeValue : clientClassificationCodeValues) { + Row row = worksheet.getRow(rowIndex); + if (row == null) { + row = worksheet.createRow(rowIndex); + } + writeString(lookupClientClassificationCol, row, clientClassificationCodeValue.getName() + " (" + clientClassificationCodeValue.getId() + ")"); + rowIndex++; + } + + // Populate Address Type lookup values + rowIndex = 1; + for (CodeValueData addressTypeCodeValue : addressTypesCodeValues) { + Row row = worksheet.getRow(rowIndex); + if (row == null) { + row = worksheet.createRow(rowIndex); + } + writeString(lookupAddressTypeCol, row, addressTypeCodeValue.getName() + " (" + addressTypeCodeValue.getId() + ")"); + rowIndex++; + } + + // Populate State/Province lookup values + rowIndex = 1; + for (CodeValueData stateCodeValue : stateProvinceCodeValues) { + Row row = worksheet.getRow(rowIndex); + if (row == null) { + row = worksheet.createRow(rowIndex); + } + writeString(lookupStateProvinceCol, row, stateCodeValue.getName() + " (" + stateCodeValue.getId() + ")"); + rowIndex++; + } + + // Populate Country lookup values + rowIndex = 1; + for (CodeValueData countryCodeValue : countryCodeValues) { + Row row = worksheet.getRow(rowIndex); + if (row == null) { + row = worksheet.createRow(rowIndex); + } + writeString(lookupCountryCol, row, countryCodeValue.getName() + " (" + countryCodeValue.getId() + ")"); + rowIndex++; + } + } - writeString(ClientPersonConstants.RELATIONAL_OFFICE_NAME_COL, rowHeader, "Lookup office Name "); - writeString(ClientPersonConstants.RELATIONAL_OFFICE_OPENING_DATE_COL, rowHeader, "Lookup Office Opened Date "); - writeString(ClientPersonConstants.LOOKUP_GENDER_COL, rowHeader, "Lookup Gender "); - writeString(ClientPersonConstants.LOOKUP_CLIENT_TYPES_COL, rowHeader, "Lookup Client Types "); - writeString(ClientPersonConstants.LOOKUP_CLIENT_CLASSIFICATION_COL, rowHeader, "Lookup Client Classification "); - writeString(ClientPersonConstants.LOOKUP_ADDRESS_TYPE_COL, rowHeader, "Lookup AddressType "); - writeString(ClientPersonConstants.LOOKUP_STATE_PROVINCE_COL, rowHeader, "Lookup State/Province "); - writeString(ClientPersonConstants.LOOKUP_COUNTRY_COL, rowHeader, "Lookup Country "); + /** + * Handles datatable column header creation and dropdown tracking. + * This is called from within setLayout() to insert headers at the correct position. + * + * @param worksheet The client sheet + * @param rowHeader The header row + * @return The next column index after datatable columns (or STATUS_COL + 1 if no datatables) + */ + private int handleDatatableColumnHeaders(Sheet worksheet, Row rowHeader) { + int startCol = ClientPersonConstants.STATUS_COL + 1; + try { + int currentCol = startCol; + int lookupSheetCol = 6; // Start after Country (column F = index 5), so next available is G = index 6 + + for (DatatableData datatable : requiredDatatables) { + if (datatable == null) { + continue; + } + String datatableName = datatable.getRegisteredTableName(); + List columns = datatable.getColumnHeaderData(); + if (columns != null) { + for (ResultsetColumnHeaderData column : columns) { + try { + // Skip system columns (id, client_id, etc.) + String columnName = column.getColumnName(); + if (columnName == null || columnName.equalsIgnoreCase("id") + || columnName.equalsIgnoreCase("client_id") + || columnName.equalsIgnoreCase("created_at") + || columnName.equalsIgnoreCase("updated_at")) { + continue; + } + + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.MEDIUM_COL_SIZE); + // Use dot notation: registeredTableName.columnName for unambiguous parsing + // For single-row datatables: treat as mandatory inline data (append "*" if column is not nullable) + // For multi-row datatables: treat as optional/repeatable child data (no "*" even if not nullable) + String headerLabel = datatableName + "." + columnName; + if (!datatable.isMultiRow() && !column.getIsColumnNullable()) { + // Single-row datatables: mark mandatory columns with "*" + headerLabel += "*"; + } + // Multi-row datatables are always optional (no "*" marker) + writeString(currentCol, rowHeader, headerLabel); + + // Check if this is a dropdown column (CODELOOKUP) with values + if (column.isCodeLookupDisplayType() && column.hasColumnValues()) { + String namedRangeName = sanitizeNamedRangeName(datatableName + "_" + columnName); + List columnValues = column.getColumnValues(); + if (columnValues != null && !columnValues.isEmpty()) { + datatableDropdowns.put(namedRangeName, new DatatableDropdownInfo(currentCol, lookupSheetCol, columnValues)); + lookupSheetCol++; + } + } + // Check if this is a date or datetime column + else if (column.isDateDisplayType() || column.isDateTimeDisplayType()) { + datatableDateColumns.put(currentCol, new DatatableDateInfo(currentCol, column.isDateTimeDisplayType())); + } + // Check if this is a boolean column + else if (column.isBooleanDisplayType()) { + datatableBooleanColumns.put(currentCol, new DatatableBooleanInfo(currentCol)); + } + + currentCol++; + } catch (Exception e) { + String datatableNameForLog = datatableName != null ? datatableName : "unknown"; + String columnNameForLog = column != null && column.getColumnName() != null ? column.getColumnName() : "unknown"; + log.warn("Failed to process datatable column '{}' in datatable '{}': {}", + columnNameForLog, datatableNameForLog, e.getMessage()); + // Continue with next column + } + } + } + } + return currentCol; + } catch (Exception e) { + log.warn("Failed to handle datatable column headers in client bulk import template: {}. Template generation will continue without datatable columns.", + e.getMessage()); + return startCol; // Return start position if datatables failed + } + } + /** + * Completes datatable setup: lookup sheet population, named ranges, and validation. + * This is called after setLayout() to complete datatable enhancements. + * All operations are wrapped in try/catch to ensure datatable failures never break template generation. + * + * @param worksheet The client sheet + * @param rowHeader The header row + * @param lookupSheet The hidden lookup sheet + * @param workbook The workbook + * @param dateFormat The date format + */ + private void handleDatatableColumnsComplete(Sheet worksheet, Row rowHeader, Sheet lookupSheet, Workbook workbook, String dateFormat) { + try { + // Step 1: Populate datatable dropdown values into lookup sheet + for (Map.Entry entry : datatableDropdowns.entrySet()) { + try { + DatatableDropdownInfo dropdownInfo = entry.getValue(); + int rowIndex = 1; // Start at row 2 (Excel row 2, POI index 1) + for (ResultsetColumnValueData valueData : dropdownInfo.values) { + Row row = lookupSheet.getRow(rowIndex); + if (row == null) { + row = lookupSheet.createRow(rowIndex); + } + writeString(dropdownInfo.lookupSheetColumnIndex, row, valueData.getValue() + " (" + valueData.getId() + ")"); + rowIndex++; + } + } catch (Exception e) { + String namedRangeName = entry.getKey(); + log.warn("Failed to populate dropdown values for datatable named range '{}': {}", + namedRangeName, e.getMessage()); + // Continue with next dropdown + } + } + + // Step 3: Create named ranges for datatable dropdown columns + String lookupSheetName = TemplatePopulateImportConstants.CLIENT_LOOKUPS_SHEET_NAME; + Workbook clientWorkbook = worksheet.getWorkbook(); + for (Map.Entry entry : datatableDropdowns.entrySet()) { + try { + String namedRangeName = entry.getKey(); + DatatableDropdownInfo dropdownInfo = entry.getValue(); + Name datatableDropdownGroup = clientWorkbook.createName(); + setSanitized(datatableDropdownGroup, namedRangeName); + int dropdownLastRow = dropdownInfo.values.size() + 1; // +1 because data starts at row 2 + String dropdownCol = CellReference.convertNumToColString(dropdownInfo.lookupSheetColumnIndex); + datatableDropdownGroup.setRefersToFormula( + "'" + lookupSheetName + "'!$" + dropdownCol + "$2:$" + dropdownCol + "$" + dropdownLastRow); + } catch (Exception e) { + String namedRangeName = entry.getKey(); + log.warn("Failed to create named range '{}' for datatable dropdown: {}", + namedRangeName, e.getMessage()); + // Continue with next named range + } + } + + // Step 4: Add data validation for datatable dropdown columns + DataValidationHelper validationHelper = new HSSFDataValidationHelper((HSSFSheet) worksheet); + for (Map.Entry entry : datatableDropdowns.entrySet()) { + try { + String namedRangeName = entry.getKey(); + DatatableDropdownInfo dropdownInfo = entry.getValue(); + CellRangeAddressList datatableDropdownRange = new CellRangeAddressList(1, SpreadsheetVersion.EXCEL97.getLastRowIndex(), + dropdownInfo.clientSheetColumnIndex, dropdownInfo.clientSheetColumnIndex); + DataValidationConstraint datatableDropdownConstraint = validationHelper.createFormulaListConstraint(namedRangeName); + DataValidation datatableDropdownValidation = validationHelper.createValidation(datatableDropdownConstraint, datatableDropdownRange); + worksheet.addValidationData(datatableDropdownValidation); + } catch (Exception e) { + String namedRangeName = entry.getKey(); + log.warn("Failed to add data validation for datatable dropdown '{}': {}", + namedRangeName, e.getMessage()); + // Continue with next validation + } + } + } catch (Exception e) { + log.warn("Failed to complete datatable setup in client bulk import template: {}. Template generation will continue without datatable enhancements.", + e.getMessage()); + // Do not throw - allow template generation to continue + } } private void setRules(Sheet worksheet, String dateformat) { @@ -325,43 +698,95 @@ private void setRules(Sheet worksheet, String dateformat) { worksheet.addValidationData(stateProvinceValidation); worksheet.addValidationData(countryValidation); worksheet.addValidationData(activeAddressValidation); + + // Add date validation for datatable date columns + for (DatatableDateInfo dateInfo : datatableDateColumns.values()) { + try { + CellRangeAddressList datatableDateRange = new CellRangeAddressList(1, SpreadsheetVersion.EXCEL97.getLastRowIndex(), + dateInfo.clientSheetColumnIndex, dateInfo.clientSheetColumnIndex); + // Use same date constraint as submittedOnDate (less than or equal to today) + DataValidationConstraint datatableDateConstraint = validationHelper + .createDateConstraint(DataValidationConstraint.OperatorType.LESS_OR_EQUAL, "=TODAY()", null, dateformat); + DataValidation datatableDateValidation = validationHelper.createValidation(datatableDateConstraint, datatableDateRange); + worksheet.addValidationData(datatableDateValidation); + } catch (Exception e) { + log.warn("Failed to add date validation for datatable date column at index {}: {}", + dateInfo.clientSheetColumnIndex, e.getMessage()); + } + } + + // Add boolean validation for datatable boolean columns + for (DatatableBooleanInfo booleanInfo : datatableBooleanColumns.values()) { + try { + CellRangeAddressList datatableBooleanRange = new CellRangeAddressList(1, SpreadsheetVersion.EXCEL97.getLastRowIndex(), + booleanInfo.clientSheetColumnIndex, booleanInfo.clientSheetColumnIndex); + // Use same boolean constraint as active field (True/False) + DataValidationConstraint datatableBooleanConstraint = validationHelper.createExplicitListConstraint(new String[] { "True", "False" }); + DataValidation datatableBooleanValidation = validationHelper.createValidation(datatableBooleanConstraint, datatableBooleanRange); + worksheet.addValidationData(datatableBooleanValidation); + } catch (Exception e) { + log.warn("Failed to add boolean validation for datatable boolean column at index {}: {}", + booleanInfo.clientSheetColumnIndex, e.getMessage()); + } + } } private void setNames(Sheet worksheet, List offices) { Workbook clientWorkbook = worksheet.getWorkbook(); + String lookupSheetName = TemplatePopulateImportConstants.CLIENT_LOOKUPS_SHEET_NAME; + Name officeGroup = clientWorkbook.createName(); officeGroup.setNameName("Office"); officeGroup.setRefersToFormula(TemplatePopulateImportConstants.OFFICE_SHEET_NAME + "!$B$2:$B$" + (offices.size() + 1)); + // All lookup named ranges explicitly reference the hidden lookup sheet + // Column 0 (A): Client Types Name clientTypeGroup = clientWorkbook.createName(); clientTypeGroup.setNameName("ClientTypes"); + int clientTypesLastRow = clientTypeCodeValues.size() + 1; // +1 because data starts at row 2 + String clientTypesCol = CellReference.convertNumToColString(0); clientTypeGroup.setRefersToFormula( - TemplatePopulateImportConstants.CLIENT_PERSON_SHEET_NAME + "!$AN$2:$AN$" + (clientTypeCodeValues.size() + 1)); + "'" + lookupSheetName + "'!$" + clientTypesCol + "$2:$" + clientTypesCol + "$" + clientTypesLastRow); + // Column 2 (C): Gender Name genderGroup = clientWorkbook.createName(); genderGroup.setNameName("Gender"); + int genderLastRow = genderCodeValues.size() + 1; + String genderCol = CellReference.convertNumToColString(2); genderGroup.setRefersToFormula( - TemplatePopulateImportConstants.CLIENT_PERSON_SHEET_NAME + "!$AL$2:$AL$" + (genderCodeValues.size() + 1)); + "'" + lookupSheetName + "'!$" + genderCol + "$2:$" + genderCol + "$" + genderLastRow); + // Column 1 (B): Client Classification Name clientClassficationGroup = clientWorkbook.createName(); clientClassficationGroup.setNameName("ClientClassification"); + int clientClassificationLastRow = clientClassificationCodeValues.size() + 1; + String clientClassificationCol = CellReference.convertNumToColString(1); clientClassficationGroup.setRefersToFormula( - TemplatePopulateImportConstants.CLIENT_PERSON_SHEET_NAME + "!$AM$2:$AM$" + (clientClassificationCodeValues.size() + 1)); + "'" + lookupSheetName + "'!$" + clientClassificationCol + "$2:$" + clientClassificationCol + "$" + clientClassificationLastRow); + // Column 3 (D): Address Type Name addressTypeGroup = clientWorkbook.createName(); addressTypeGroup.setNameName("AddressType"); + int addressTypeLastRow = addressTypesCodeValues.size() + 1; + String addressTypeCol = CellReference.convertNumToColString(3); addressTypeGroup.setRefersToFormula( - TemplatePopulateImportConstants.CLIENT_PERSON_SHEET_NAME + "!$AO$2:$AO$" + (addressTypesCodeValues.size() + 1)); + "'" + lookupSheetName + "'!$" + addressTypeCol + "$2:$" + addressTypeCol + "$" + addressTypeLastRow); + // Column 4 (E): State/Province Name stateProvinceGroup = clientWorkbook.createName(); stateProvinceGroup.setNameName("StateProvince"); + int stateProvinceLastRow = stateProvinceCodeValues.size() + 1; + String stateProvinceCol = CellReference.convertNumToColString(4); stateProvinceGroup.setRefersToFormula( - TemplatePopulateImportConstants.CLIENT_PERSON_SHEET_NAME + "!$AP$2:$AP$" + (stateProvinceCodeValues.size() + 1)); + "'" + lookupSheetName + "'!$" + stateProvinceCol + "$2:$" + stateProvinceCol + "$" + stateProvinceLastRow); + // Column 5 (F): Country Name countryGroup = clientWorkbook.createName(); countryGroup.setNameName("Country"); + int countryLastRow = countryCodeValues.size() + 1; + String countryCol = CellReference.convertNumToColString(5); countryGroup.setRefersToFormula( - TemplatePopulateImportConstants.CLIENT_PERSON_SHEET_NAME + "!$AQ$2:$AQ$" + (countryCodeValues.size() + 1)); + "'" + lookupSheetName + "'!$" + countryCol + "$2:$" + countryCol + "$" + countryLastRow); for (Integer i = 0; i < offices.size(); i++) { Integer[] officeNameToBeginEndIndexesOfStaff = personnelSheetPopulator.getOfficeNameToBeginEndIndexesOfStaff().get(i); diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/service/BulkImportWorkbookPopulatorServiceImpl.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/service/BulkImportWorkbookPopulatorServiceImpl.java index 14351c97b5b..e01fb67f2b6 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/service/BulkImportWorkbookPopulatorServiceImpl.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/service/BulkImportWorkbookPopulatorServiceImpl.java @@ -104,6 +104,14 @@ import org.apache.fineract.portfolio.savings.service.DepositProductReadPlatformService; import org.apache.fineract.portfolio.savings.service.SavingsAccountReadPlatformService; import org.apache.fineract.portfolio.savings.service.SavingsProductReadPlatformService; +import org.apache.fineract.infrastructure.dataqueries.data.DatatableData; +import org.apache.fineract.infrastructure.dataqueries.data.EntityTables; +import org.apache.fineract.infrastructure.dataqueries.data.StatusEnum; +import org.apache.fineract.infrastructure.dataqueries.domain.EntityDatatableChecks; +import org.apache.fineract.infrastructure.dataqueries.domain.EntityDatatableChecksRepository; +import org.apache.fineract.infrastructure.dataqueries.service.DatatableReadService; +import org.apache.fineract.infrastructure.dataqueries.service.EntityDatatableChecksReadService; +import org.apache.fineract.portfolio.client.domain.LegalForm; import org.apache.fineract.portfolio.shareproducts.data.ShareProductData; import org.apache.fineract.useradministration.data.RoleData; import org.apache.fineract.useradministration.service.RoleReadPlatformService; @@ -137,6 +145,9 @@ public class BulkImportWorkbookPopulatorServiceImpl implements BulkImportWorkboo private final ChargeReadPlatformService chargeReadPlatformService; private final DepositProductReadPlatformService depositProductReadPlatformService; private final RoleReadPlatformService roleReadPlatformService; + private final EntityDatatableChecksReadService entityDatatableChecksReadService; + private final DatatableReadService datatableReadService; + private final EntityDatatableChecksRepository entityDatatableChecksRepository; @Autowired public BulkImportWorkbookPopulatorServiceImpl(final PlatformSecurityContext context, @@ -153,7 +164,10 @@ public BulkImportWorkbookPopulatorServiceImpl(final PlatformSecurityContext cont final ShareProductReadPlatformService shareProductReadPlatformService, final ChargeReadPlatformService chargeReadPlatformService, final DepositProductReadPlatformService depositProductReadPlatformService, - final RoleReadPlatformService roleReadPlatformService) { + final RoleReadPlatformService roleReadPlatformService, + final EntityDatatableChecksReadService entityDatatableChecksReadService, + final DatatableReadService datatableReadService, + final EntityDatatableChecksRepository entityDatatableChecksRepository) { this.officeReadPlatformService = officeReadPlatformService; this.staffReadPlatformService = staffReadPlatformService; this.context = context; @@ -173,6 +187,9 @@ public BulkImportWorkbookPopulatorServiceImpl(final PlatformSecurityContext cont this.chargeReadPlatformService = chargeReadPlatformService; this.depositProductReadPlatformService = depositProductReadPlatformService; this.roleReadPlatformService = roleReadPlatformService; + this.entityDatatableChecksReadService = entityDatatableChecksReadService; + this.datatableReadService = datatableReadService; + this.entityDatatableChecksRepository = entityDatatableChecksRepository; } @Override @@ -237,20 +254,59 @@ private WorkbookPopulator populateClientWorkbook(final String entityType, final List addressTypesCodeValues = fetchCodeValuesByCodeName("ADDRESS_TYPE"); List stateProvinceCodeValues = fetchCodeValuesByCodeName("STATE"); List countryCodeValues = fetchCodeValuesByCodeName("COUNTRY"); + + // Fetch required datatables for CLIENT entity with CREATE status + String entitySubtype = null; + if (entityType.trim().equalsIgnoreCase(GlobalEntityType.CLIENTS_PERSON.toString())) { + entitySubtype = "Person"; + } else if (entityType.trim().equalsIgnoreCase(GlobalEntityType.CLIENTS_ENTITY.toString())) { + entitySubtype = "Entity"; + } + List requiredDatatables = fetchRequiredDatatables(entitySubtype); + if (entityType.trim().equalsIgnoreCase(GlobalEntityType.CLIENTS_PERSON.toString())) { List genderCodeValues = fetchCodeValuesByCodeName("Gender"); return new ClientPersonWorkbookPopulator(new OfficeSheetPopulator(offices), new PersonnelSheetPopulator(staff, offices), clientTypeCodeValues, genderCodeValues, clientClassification, addressTypesCodeValues, stateProvinceCodeValues, - countryCodeValues); + countryCodeValues, requiredDatatables); } else if (entityType.trim().equalsIgnoreCase(GlobalEntityType.CLIENTS_ENTITY.toString())) { List constitutionCodeValues = fetchCodeValuesByCodeName("Constitution"); List mainBusinessline = fetchCodeValuesByCodeName("Main Business Line"); return new ClientEntityWorkbookPopulator(new OfficeSheetPopulator(offices), new PersonnelSheetPopulator(staff, offices), clientTypeCodeValues, constitutionCodeValues, mainBusinessline, clientClassification, addressTypesCodeValues, - stateProvinceCodeValues, countryCodeValues); + stateProvinceCodeValues, countryCodeValues, requiredDatatables); } return null; } + + private List fetchRequiredDatatables(final String entitySubtype) { + List requiredDatatables = new ArrayList<>(); + try { + // Get required datatables for CLIENT entity with CREATE status + List entityDatatableChecks; + if (entitySubtype != null) { + // Filter by entity subtype + entityDatatableChecks = entityDatatableChecksRepository.findByEntityAndStatusAndSubtype( + EntityTables.CLIENT.getName(), StatusEnum.CREATE.getValue(), entitySubtype); + } else { + // Get all required datatables regardless of subtype + entityDatatableChecks = entityDatatableChecksRepository.findByEntityAndStatus( + EntityTables.CLIENT.getName(), StatusEnum.CREATE.getValue()); + } + + // Convert EntityDatatableChecks to DatatableData + for (EntityDatatableChecks check : entityDatatableChecks) { + DatatableData datatable = datatableReadService.retrieveDatatable(check.getDatatableName()); + if (datatable != null) { + requiredDatatables.add(datatable); + } + } + } catch (Exception e) { + LOG.warn("Error fetching required datatables for bulk import template", e); + // Continue without datatables if there's an error + } + return requiredDatatables; + } private Response buildResponse(final Workbook workbook, final String entity) { String filename = entity + DateUtils.getBusinessLocalDate().toString() + ".xls"; diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableReadServiceImpl.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableReadServiceImpl.java index 5d38c4a9a2e..26cd4cc7fc7 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableReadServiceImpl.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/dataqueries/service/DatatableReadServiceImpl.java @@ -92,8 +92,9 @@ public List retrieveDatatableNames(final String appTable) { final String registeredDatatableName = rowSet.getString("registered_table_name"); final String entitySubType = rowSet.getString("entity_subtype"); final List columnHeaderData = genericDataService.fillResultsetColumnHeaders(registeredDatatableName); + final boolean multiRow = datatableUtil.isMultirowDatatable(columnHeaderData); - datatables.add(DatatableData.create(appTableName, registeredDatatableName, entitySubType, columnHeaderData)); + datatables.add(DatatableData.create(appTableName, registeredDatatableName, entitySubType, columnHeaderData, multiRow)); } return datatables; @@ -119,8 +120,9 @@ public DatatableData retrieveDatatable(final String datatable) { final String entitySubType = rowSet.getString("entity_subtype"); final List columnHeaderData = this.genericDataService .fillResultsetColumnHeaders(registeredDatatableName); + final boolean multiRow = datatableUtil.isMultirowDatatable(columnHeaderData); - datatableData = DatatableData.create(appTableName, registeredDatatableName, entitySubType, columnHeaderData); + datatableData = DatatableData.create(appTableName, registeredDatatableName, entitySubType, columnHeaderData, multiRow); } return datatableData; diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/survey/service/ReadSurveyServiceImpl.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/survey/service/ReadSurveyServiceImpl.java index 3d3607b575b..18d2fc16ff7 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/survey/service/ReadSurveyServiceImpl.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/survey/service/ReadSurveyServiceImpl.java @@ -27,6 +27,7 @@ import org.apache.fineract.infrastructure.dataqueries.data.GenericResultsetData; import org.apache.fineract.infrastructure.dataqueries.data.ResultsetColumnHeaderData; import org.apache.fineract.infrastructure.dataqueries.service.DatatableReadService; +import org.apache.fineract.infrastructure.dataqueries.service.DatatableUtil; import org.apache.fineract.infrastructure.dataqueries.service.GenericDataService; import org.apache.fineract.infrastructure.security.service.PlatformSecurityContext; import org.apache.fineract.infrastructure.security.service.SqlValidator; @@ -47,6 +48,7 @@ public class ReadSurveyServiceImpl implements ReadSurveyService { private final SqlValidator sqlValidator; private final GenericDataService genericDataService; private final DatatableReadService datatableReadService; + private final DatatableUtil datatableUtil; @Override public List retrieveAllSurveys() { @@ -63,9 +65,10 @@ public List retrieveAllSurveys() { final boolean enabled = rs.getBoolean("enabled"); final List columnHeaderData = this.genericDataService .fillResultsetColumnHeaders(registeredDatatableName); + final boolean multiRow = datatableUtil.isMultirowDatatable(columnHeaderData); surveyDataTables.add(SurveyDataTableData - .create(DatatableData.create(appTableName, registeredDatatableName, entitySubType, columnHeaderData), enabled)); + .create(DatatableData.create(appTableName, registeredDatatableName, entitySubType, columnHeaderData, multiRow), enabled)); } return surveyDataTables; @@ -104,8 +107,9 @@ public SurveyDataTableData retrieveSurvey(String surveyName) { final boolean enabled = rs.getBoolean("enabled"); final List columnHeaderData = this.genericDataService .fillResultsetColumnHeaders(registeredDatatableName); + final boolean multiRow = datatableUtil.isMultirowDatatable(columnHeaderData); datatableData = SurveyDataTableData - .create(DatatableData.create(appTableName, registeredDatatableName, entitySubType, columnHeaderData), enabled); + .create(DatatableData.create(appTableName, registeredDatatableName, entitySubType, columnHeaderData, multiRow), enabled); } diff --git a/fineract-provider/src/test/java/org/apache/fineract/infrastructure/survey/service/ReadSurveyServiceImplTest.java b/fineract-provider/src/test/java/org/apache/fineract/infrastructure/survey/service/ReadSurveyServiceImplTest.java index 54118681247..a3f6b968d30 100644 --- a/fineract-provider/src/test/java/org/apache/fineract/infrastructure/survey/service/ReadSurveyServiceImplTest.java +++ b/fineract-provider/src/test/java/org/apache/fineract/infrastructure/survey/service/ReadSurveyServiceImplTest.java @@ -27,6 +27,7 @@ import java.util.List; import org.apache.fineract.infrastructure.dataqueries.service.DatatableReadService; +import org.apache.fineract.infrastructure.dataqueries.service.DatatableUtil; import org.apache.fineract.infrastructure.dataqueries.service.GenericDataService; import org.apache.fineract.infrastructure.security.service.PlatformSecurityContext; import org.apache.fineract.infrastructure.security.service.SqlValidator; @@ -64,6 +65,8 @@ class ReadSurveyServiceImplTest { @Mock private DatatableReadService datatableReadService; @Mock + private DatatableUtil datatableUtil; + @Mock private AppUser appUser; private ReadSurveyServiceImpl underTest; @@ -72,7 +75,8 @@ class ReadSurveyServiceImplTest { void setUp() { when(context.authenticatedUser()).thenReturn(appUser); when(appUser.getId()).thenReturn(USER_ID); - underTest = new ReadSurveyServiceImpl(context, jdbcTemplate, sqlValidator, genericDataService, datatableReadService); + underTest = new ReadSurveyServiceImpl(context, jdbcTemplate, sqlValidator, genericDataService, datatableReadService, + datatableUtil); } @Test From c2d68a2c48964f040ea523f000656e05f9f031c1 Mon Sep 17 00:00:00 2001 From: Samer Melhem Date: Fri, 10 Jul 2026 13:13:20 +0300 Subject: [PATCH 2/7] FINERACT-2682: Fix incorrect Apache license header in DatatableDataTest The header text omitted "under the License is distributed", causing the license-check plugin to flag the file as missing a header. --- .../dataqueries/data/DatatableDataTest.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/fineract-core/src/test/java/org/apache/fineract/infrastructure/dataqueries/data/DatatableDataTest.java b/fineract-core/src/test/java/org/apache/fineract/infrastructure/dataqueries/data/DatatableDataTest.java index 25f91698d55..0aef602bac9 100644 --- a/fineract-core/src/test/java/org/apache/fineract/infrastructure/dataqueries/data/DatatableDataTest.java +++ b/fineract-core/src/test/java/org/apache/fineract/infrastructure/dataqueries/data/DatatableDataTest.java @@ -10,10 +10,11 @@ * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, - * software distributed on an "AS IS" BASIS, WITHOUT WARRANTIES - * OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and - * limitations under the License. + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package org.apache.fineract.infrastructure.dataqueries.data; From 0226070b9a67bf2e452989dc6979eb3c640c9960 Mon Sep 17 00:00:00 2001 From: Samer Melhem Date: Fri, 10 Jul 2026 14:58:45 +0300 Subject: [PATCH 3/7] FINERACT-2682: Fix error-prone build failures in bulk client import - Remove unused LegalForm import in BulkImportWorkbookPopulatorServiceImpl. - Use Guava Splitter instead of String.split in ImportHandlerUtils.getFriendlyDatatableName, since split() has surprising behavior with trailing delimiters (StringSplitter). - Wire up ClientEntityWorkbookPopulator.handleDatatableColumnsComplete, which was never called (UnusedMethod), so datatable dropdown columns in the bulk import template get their lookup values and Excel validation, matching ClientPersonWorkbookPopulator's existing wiring. Removed its duplicate named-range creation, since setNames() already creates those ranges. --- .../importhandler/ImportHandlerUtils.java | 13 +++--- .../client/ClientEntityWorkbookPopulator.java | 46 +++++-------------- ...ulkImportWorkbookPopulatorServiceImpl.java | 1 - 3 files changed, 19 insertions(+), 41 deletions(-) diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/ImportHandlerUtils.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/ImportHandlerUtils.java index 86059d1b284..ff8841f2273 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/ImportHandlerUtils.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/ImportHandlerUtils.java @@ -835,16 +835,17 @@ private static String getFriendlyDatatableName(String datatableName) { return datatableName; } // Replace underscores with spaces and capitalize words - String[] parts = datatableName.split("_"); + List parts = Splitter.on('_').splitToList(datatableName); StringBuilder friendly = new StringBuilder(); - for (int i = 0; i < parts.length; i++) { + for (int i = 0; i < parts.size(); i++) { if (i > 0) { friendly.append(" "); } - if (parts[i].length() > 0) { - friendly.append(Character.toUpperCase(parts[i].charAt(0))); - if (parts[i].length() > 1) { - friendly.append(parts[i].substring(1)); + String part = parts.get(i); + if (!part.isEmpty()) { + friendly.append(Character.toUpperCase(part.charAt(0))); + if (part.length() > 1) { + friendly.append(part.substring(1)); } } } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/client/ClientEntityWorkbookPopulator.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/client/ClientEntityWorkbookPopulator.java index b4c5693ae86..82a8b3a84be 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/client/ClientEntityWorkbookPopulator.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/client/ClientEntityWorkbookPopulator.java @@ -138,6 +138,7 @@ public void populate(Workbook workbook, String dateFormat) { setClientDataLookupTable(lookupSheet); // now values exist setFormatStyle(workbook, clientSheet); setRules(clientSheet, dateFormat); + handleDatatableColumnsComplete(clientSheet, lookupSheet); } private void setFormatStyle(Workbook workbook, Sheet worksheet) { @@ -555,19 +556,17 @@ else if (column.isBooleanDisplayType()) { } /** - * Completes datatable setup: lookup sheet population, named ranges, and validation. - * This is called after setLayout() to complete datatable enhancements. + * Populates datatable dropdown values into the hidden lookup sheet and adds data validation on the + * client sheet's dropdown columns. Named ranges for these columns are already created in + * {@link #setNames(Sheet, List)}; this only fills in the values they refer to and wires up validation. * All operations are wrapped in try/catch to ensure datatable failures never break template generation. - * + * * @param worksheet The client sheet - * @param rowHeader The header row * @param lookupSheet The hidden lookup sheet - * @param workbook The workbook - * @param dateFormat The date format */ - private void handleDatatableColumnsComplete(Sheet worksheet, Row rowHeader, Sheet lookupSheet, Workbook workbook, String dateFormat) { + private void handleDatatableColumnsComplete(Sheet worksheet, Sheet lookupSheet) { try { - // Step 1: Populate datatable dropdown values into lookup sheet + // Populate datatable dropdown values into lookup sheet for (Map.Entry entry : datatableDropdowns.entrySet()) { try { DatatableDropdownInfo dropdownInfo = entry.getValue(); @@ -582,34 +581,13 @@ private void handleDatatableColumnsComplete(Sheet worksheet, Row rowHeader, Shee } } catch (Exception e) { String namedRangeName = entry.getKey(); - log.warn("Failed to populate dropdown values for datatable named range '{}': {}", + log.warn("Failed to populate dropdown values for datatable named range '{}': {}", namedRangeName, e.getMessage()); // Continue with next dropdown } } - - // Step 2: Create named ranges for datatable dropdown columns - String lookupSheetName = TemplatePopulateImportConstants.CLIENT_LOOKUPS_SHEET_NAME; - Workbook clientWorkbook = worksheet.getWorkbook(); - for (Map.Entry entry : datatableDropdowns.entrySet()) { - try { - String namedRangeName = entry.getKey(); - DatatableDropdownInfo dropdownInfo = entry.getValue(); - Name datatableDropdownGroup = clientWorkbook.createName(); - setSanitized(datatableDropdownGroup, namedRangeName); - int dropdownLastRow = dropdownInfo.values.size() + 1; // +1 because data starts at row 2 - String dropdownCol = CellReference.convertNumToColString(dropdownInfo.lookupSheetColumnIndex); - datatableDropdownGroup.setRefersToFormula( - "'" + lookupSheetName + "'!$" + dropdownCol + "$2:$" + dropdownCol + "$" + dropdownLastRow); - } catch (Exception e) { - String namedRangeName = entry.getKey(); - log.warn("Failed to create named range '{}' for datatable dropdown: {}", - namedRangeName, e.getMessage()); - // Continue with next named range - } - } - - // Step 3: Add data validation for datatable dropdown columns + + // Add data validation for datatable dropdown columns DataValidationHelper validationHelper = new HSSFDataValidationHelper((HSSFSheet) worksheet); for (Map.Entry entry : datatableDropdowns.entrySet()) { try { @@ -622,13 +600,13 @@ private void handleDatatableColumnsComplete(Sheet worksheet, Row rowHeader, Shee worksheet.addValidationData(datatableDropdownValidation); } catch (Exception e) { String namedRangeName = entry.getKey(); - log.warn("Failed to add data validation for datatable dropdown '{}': {}", + log.warn("Failed to add data validation for datatable dropdown '{}': {}", namedRangeName, e.getMessage()); // Continue with next validation } } } catch (Exception e) { - log.warn("Failed to complete datatable setup in client bulk import template: {}. Template generation will continue without datatable enhancements.", + log.warn("Failed to complete datatable setup in client bulk import template: {}. Template generation will continue without datatable enhancements.", e.getMessage()); // Do not throw - allow template generation to continue } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/service/BulkImportWorkbookPopulatorServiceImpl.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/service/BulkImportWorkbookPopulatorServiceImpl.java index e01fb67f2b6..d3ddaef73c5 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/service/BulkImportWorkbookPopulatorServiceImpl.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/service/BulkImportWorkbookPopulatorServiceImpl.java @@ -111,7 +111,6 @@ import org.apache.fineract.infrastructure.dataqueries.domain.EntityDatatableChecksRepository; import org.apache.fineract.infrastructure.dataqueries.service.DatatableReadService; import org.apache.fineract.infrastructure.dataqueries.service.EntityDatatableChecksReadService; -import org.apache.fineract.portfolio.client.domain.LegalForm; import org.apache.fineract.portfolio.shareproducts.data.ShareProductData; import org.apache.fineract.useradministration.data.RoleData; import org.apache.fineract.useradministration.service.RoleReadPlatformService; From 0c4e4cfbd2ee88ef6c6692c91dc7d6c76547bdd9 Mon Sep 17 00:00:00 2001 From: Samer Melhem Date: Fri, 17 Jul 2026 14:06:13 +0300 Subject: [PATCH 4/7] FINERACT-2682: Fix spotless and checkstyle violations - SearchUtil#extractIdFromDisplayValue: rewrap javadoc to the project's palantir-java-format style and drop trailing whitespace on the blank comment line (checkstyle RegexpSingleline). - DatatableDataTest: collapse two DatatableData.create(...) calls back onto a single line per spotless. --- .../fineract/portfolio/search/service/SearchUtil.java | 10 +++++----- .../dataqueries/data/DatatableDataTest.java | 6 ++---- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/fineract-core/src/main/java/org/apache/fineract/portfolio/search/service/SearchUtil.java b/fineract-core/src/main/java/org/apache/fineract/portfolio/search/service/SearchUtil.java index 0c5b1ef6b11..6c9a988c495 100644 --- a/fineract-core/src/main/java/org/apache/fineract/portfolio/search/service/SearchUtil.java +++ b/fineract-core/src/main/java/org/apache/fineract/portfolio/search/service/SearchUtil.java @@ -344,11 +344,11 @@ public String camelToSnake(final String camelStr) { } /** - * Extracts numeric ID from strings ending with "(id)" format. - * Example: "Permanent (32)" -> "32" - * If the string doesn't match the pattern, returns the original value. - * - * @param value The string value that may contain an ID in parentheses + * Extracts numeric ID from strings ending with "(id)" format. Example: "Permanent (32)" -> "32" If the string + * doesn't match the pattern, returns the original value. + * + * @param value + * The string value that may contain an ID in parentheses * @return The extracted ID as a string, or the original value if no ID pattern is found */ private static String extractIdFromDisplayValue(String value) { diff --git a/fineract-core/src/test/java/org/apache/fineract/infrastructure/dataqueries/data/DatatableDataTest.java b/fineract-core/src/test/java/org/apache/fineract/infrastructure/dataqueries/data/DatatableDataTest.java index 0aef602bac9..1ed181f659b 100644 --- a/fineract-core/src/test/java/org/apache/fineract/infrastructure/dataqueries/data/DatatableDataTest.java +++ b/fineract-core/src/test/java/org/apache/fineract/infrastructure/dataqueries/data/DatatableDataTest.java @@ -42,8 +42,7 @@ void testSingleRowDatatable() { boolean multiRow = false; // When: Creating DatatableData - DatatableData datatableData = DatatableData.create(appTableName, registeredTableName, entitySubType, - columnHeaders, multiRow); + DatatableData datatableData = DatatableData.create(appTableName, registeredTableName, entitySubType, columnHeaders, multiRow); // Then: It should be identified as single-row assertFalse(datatableData.isMultiRow(), "Single-row datatable should return false for isMultiRow()"); @@ -65,8 +64,7 @@ void testMultiRowDatatable() { boolean multiRow = true; // When: Creating DatatableData - DatatableData datatableData = DatatableData.create(appTableName, registeredTableName, entitySubType, - columnHeaders, multiRow); + DatatableData datatableData = DatatableData.create(appTableName, registeredTableName, entitySubType, columnHeaders, multiRow); // Then: It should be identified as multi-row assertTrue(datatableData.isMultiRow(), "Multi-row datatable should return true for isMultiRow()"); From b20e15b77a37f93291acefb8c2ecc66f6742740e Mon Sep 17 00:00:00 2001 From: Samer Melhem Date: Fri, 17 Jul 2026 14:06:37 +0300 Subject: [PATCH 5/7] FINERACT-2682: Fix client/client-person bulk import dropdown ID parsing ClientEntityWorkbookPopulator and ClientPersonWorkbookPopulator write lookup dropdown cells (client type, classification, main business line, address type, state/province, country) in "Name (Id)" format, but ClientEntityImportHandler and ClientPersonImportHandler were still parsing them with the legacy "Name-Id" dash-splitter, so every one of those fields silently resolved to a null ID on import (only constitution had been updated to the new format). Both handlers now use ImportHandlerUtils#extractIdFromDisplayValue, the helper already added for this purpose but left unused. Also fixes ClientEntityImportHandlerTest#testClientImport, which built its dummy row from ClientEntityConstants.LOOKUP_* column indices on the visible sheet. Those columns are now dynamically positioned after the datatable columns and were marked @Deprecated in favor of the hidden "_CLIENT_LOOKUPS" sheet, but the test wasn't updated to match, so it picked up an unrelated lookup's value/ID and the import failed downstream. --- .../client/ClientEntityImportHandler.java | 91 +++++-------------- .../client/ClientPersonImportHandler.java | 70 +++++--------- .../client/ClientEntityImportHandlerTest.java | 17 ++-- 3 files changed, 54 insertions(+), 124 deletions(-) diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientEntityImportHandler.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientEntityImportHandler.java index af8be5057d7..780425ae432 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientEntityImportHandler.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientEntityImportHandler.java @@ -18,7 +18,6 @@ */ package org.apache.fineract.infrastructure.bulkimport.importhandler.client; -import com.google.common.base.Splitter; import com.google.gson.GsonBuilder; import java.time.LocalDate; import java.util.ArrayList; @@ -64,7 +63,6 @@ @AllArgsConstructor public class ClientEntityImportHandler implements ImportHandler { - public static final String SEPARATOR = "-"; private static final Logger LOG = LoggerFactory.getLogger(ClientEntityImportHandler.class); private final PortfolioCommandSourceWritePlatformService commandsSourceWritePlatformService; private final ExternalIdFactory externalIdFactory; @@ -95,6 +93,22 @@ private List readExcelFile(final Workbook workbook, final String loc return clients; } + /** + * Extracts the code value ID out of a "Name (id)" formatted dropdown cell, as written by + * {@link org.apache.fineract.infrastructure.bulkimport.populator.client.ClientEntityWorkbookPopulator}. + */ + private Long parseIdFromDisplayValue(final String displayValue, final String fieldName) { + if (displayValue == null) { + return null; + } + String idPart = ImportHandlerUtils.extractIdFromDisplayValue(displayValue); + if (idPart != null && idPart.matches("\\d+")) { + return Long.valueOf(idPart); + } + LOG.error("Bulk upload error: Could not extract ID from {}. Row data: {}", fieldName, displayValue); + return null; + } + private ClientData readClient(final Workbook workbook, final Row row, final String locale, final String dateFormat) { Long legalFormId = 2L; String name = ImportHandlerUtils.readAsString(ClientEntityConstants.NAME_COL, row); @@ -116,44 +130,13 @@ private ClientData readClient(final Workbook workbook, final Row row, final Stri } String clientType = ImportHandlerUtils.readAsString(ClientEntityConstants.CLIENT_TYPE_COL, row); - Long clientTypeId = null; - if (clientType != null) { - List clientTypeAr = Splitter.on(SEPARATOR).splitToList(clientType); - if (clientTypeAr.size() > 1) { - if (clientTypeAr.get(1) != null) { - clientTypeId = Long.parseLong(clientTypeAr.get(1)); - } - } else { - LOG.error("Bulk upload error: Missing expected column at index 1 in clientType. Row data: {}", clientTypeAr); - } - } + Long clientTypeId = parseIdFromDisplayValue(clientType, "clientType"); String clientClassification = ImportHandlerUtils.readAsString(ClientEntityConstants.CLIENT_CLASSIFICATION_COL, row); - Long clientClassicationId = null; - if (clientClassification != null) { - List clientClassificationAr = Splitter.on(SEPARATOR).splitToList(clientClassification); - if (clientClassificationAr.size() > 1) { - if (clientClassificationAr.get(1) != null) { - clientClassicationId = Long.parseLong(clientClassificationAr.get(1)); - } - } else { - LOG.error("Bulk upload error: Missing expected column at index 1 in clientClassification. Row data: {}", clientClassificationAr); - } - } + Long clientClassicationId = parseIdFromDisplayValue(clientClassification, "clientClassification"); String incorporationNo = ImportHandlerUtils.readAsString(ClientEntityConstants.INCOPORATION_NUMBER_COL, row); String mainBusinessLine = ImportHandlerUtils.readAsString(ClientEntityConstants.MAIN_BUSINESS_LINE, row); - Long mainBusinessId = null; - if (mainBusinessLine != null) { - List mainBusinessLineAr = Splitter.on(SEPARATOR) - .splitToList(Objects.requireNonNull(ImportHandlerUtils.readAsString(ClientEntityConstants.MAIN_BUSINESS_LINE, row))); - if (mainBusinessLineAr.size() > 1) { - if (mainBusinessLineAr.get(1) != null) { - mainBusinessId = Long.parseLong(mainBusinessLineAr.get(1)); - } - } else { - LOG.error("Bulk upload error: Missing expected column at index 1 in mainBusinessLine. Row data: {}", mainBusinessLineAr); - } - } + Long mainBusinessId = parseIdFromDisplayValue(mainBusinessLine, "mainBusinessLine"); String constitutionCell = ImportHandlerUtils.readAsString(ClientEntityConstants.CONSTITUTION_COL, row); Long constitutionId = null; @@ -193,17 +176,7 @@ private ClientData readClient(final Workbook workbook, final Row row, final Stri Collection addressList = null; if (ImportHandlerUtils.readAsBoolean(ClientEntityConstants.ADDRESS_ENABLED, row)) { String addressType = ImportHandlerUtils.readAsString(ClientEntityConstants.ADDRESS_TYPE_COL, row); - Long addressTypeId = null; - if (addressType != null) { - List addressTypeAr = Splitter.on(SEPARATOR).splitToList(addressType); - if (addressTypeAr.size() > 1) { - if (addressTypeAr.get(1) != null) { - addressTypeId = Long.parseLong(addressTypeAr.get(1)); - } - } else { - LOG.error("Bulk upload error: Missing expected column at index 1 in addressType. Row data: {}", addressTypeAr); - } - } + Long addressTypeId = parseIdFromDisplayValue(addressType, "addressType"); String street = ImportHandlerUtils.readAsString(ClientEntityConstants.STREET_COL, row); String addressLine1 = ImportHandlerUtils.readAsString(ClientEntityConstants.ADDRESS_LINE_1_COL, row); String addressLine2 = ImportHandlerUtils.readAsString(ClientEntityConstants.ADDRESS_LINE_2_COL, row); @@ -214,29 +187,9 @@ private ClientData readClient(final Workbook workbook, final Row row, final Stri Boolean isActiveAddress = ImportHandlerUtils.readAsBoolean(ClientEntityConstants.IS_ACTIVE_ADDRESS_COL, row); String stateProvince = ImportHandlerUtils.readAsString(ClientEntityConstants.STATE_PROVINCE_COL, row); - Long stateProvinceId = null; - if (stateProvince != null) { - List stateProvinceAr = Splitter.on(SEPARATOR).splitToList(stateProvince); - if (stateProvinceAr.size() > 1) { - if (stateProvinceAr.get(1) != null) { - stateProvinceId = Long.parseLong(stateProvinceAr.get(1)); - } - } else { - LOG.error("Bulk upload error: Missing expected column at index 1 in stateProvince. Row data: {}", stateProvinceAr); - } - } + Long stateProvinceId = parseIdFromDisplayValue(stateProvince, "stateProvince"); String country = ImportHandlerUtils.readAsString(ClientEntityConstants.COUNTRY_COL, row); - Long countryId = null; - if (country != null) { - List countryAr = Splitter.on(SEPARATOR).splitToList(country); - if (countryAr.size() > 1) { - if (countryAr.get(1) != null) { - countryId = Long.parseLong(countryAr.get(1)); - } - } else { - LOG.error("Bulk upload error: Missing expected column at index 1 in country. Row data: {}", countryAr); - } - } + Long countryId = parseIdFromDisplayValue(country, "country"); addressDataObj = new AddressData(addressTypeId, street, addressLine1, addressLine2, addressLine3, city, postalCode, isActiveAddress, stateProvinceId, countryId); addressList = new ArrayList<>(List.of(addressDataObj)); diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientPersonImportHandler.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientPersonImportHandler.java index f7349de475a..761ad63f40b 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientPersonImportHandler.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientPersonImportHandler.java @@ -18,7 +18,6 @@ */ package org.apache.fineract.infrastructure.bulkimport.importhandler.client; -import com.google.common.base.Splitter; import com.google.gson.GsonBuilder; import java.time.LocalDate; import java.util.ArrayList; @@ -59,7 +58,6 @@ @AllArgsConstructor public class ClientPersonImportHandler implements ImportHandler { - public static final String SEPARATOR = "-"; private static final Logger LOG = LoggerFactory.getLogger(ClientPersonImportHandler.class); private final PortfolioCommandSourceWritePlatformService commandsSourceWritePlatformService; private final ExternalIdFactory externalIdFactory; @@ -87,6 +85,22 @@ public List readExcelFile(final Workbook workbook, final String loca return clients; } + /** + * Extracts the code value ID out of a "Name (id)" formatted dropdown cell, as written by + * {@link org.apache.fineract.infrastructure.bulkimport.populator.client.ClientPersonWorkbookPopulator}. + */ + private Long parseIdFromDisplayValue(final String displayValue, final String fieldName) { + if (displayValue == null) { + return null; + } + String idPart = ImportHandlerUtils.extractIdFromDisplayValue(displayValue); + if (idPart != null && idPart.matches("\\d+")) { + return Long.valueOf(idPart); + } + LOG.error("Bulk upload error: Could not extract ID from {}. Row data: {}", fieldName, displayValue); + return null; + } + private ClientData readClient(final Workbook workbook, final Row row, final String locale, final String dateFormat) { Long legalFormId = 1L; String firstName = ImportHandlerUtils.readAsString(ClientPersonConstants.FIRST_NAME_COL, row); @@ -116,43 +130,18 @@ private ClientData readClient(final Workbook workbook, final Row row, final Stri LocalDate dob = ImportHandlerUtils.readAsDate(ClientPersonConstants.DOB_COL, row); String clientType = ImportHandlerUtils.readAsString(ClientPersonConstants.CLIENT_TYPE_COL, row); - Long clientTypeId = null; - if (clientType != null) { - List clientTypeAr = Splitter.on(SEPARATOR).splitToList(clientType); - if (clientTypeAr.size() > 1 && clientTypeAr.get(1) != null) { - clientTypeId = Long.parseLong(clientTypeAr.get(1)); - } - } + Long clientTypeId = parseIdFromDisplayValue(clientType, "clientType"); String gender = ImportHandlerUtils.readAsString(ClientPersonConstants.GENDER_COL, row); - Long genderId = null; - if (gender != null) { - List genderAr = Splitter.on(SEPARATOR).splitToList(gender); - if (genderAr.size() > 1 && genderAr.get(1) != null) { - genderId = Long.parseLong(genderAr.get(1)); - } - } + Long genderId = parseIdFromDisplayValue(gender, "gender"); String clientClassification = ImportHandlerUtils.readAsString(ClientPersonConstants.CLIENT_CLASSIFICATION_COL, row); - Long clientClassificationId = null; - if (clientClassification != null) { - List clientClassificationAr = Splitter.on(SEPARATOR).splitToList(clientClassification); - if (clientClassificationAr.size() > 1 && clientClassificationAr.get(1) != null) { - clientClassificationId = Long.parseLong(clientClassificationAr.get(1)); - } - } + Long clientClassificationId = parseIdFromDisplayValue(clientClassification, "clientClassification"); Boolean isStaff = ImportHandlerUtils.readAsBoolean(ClientPersonConstants.IS_STAFF_COL, row); AddressData addressDataObj = null; Collection addressList = null; if (ImportHandlerUtils.readAsBoolean(ClientPersonConstants.ADDRESS_ENABLED_COL, row)) { String addressType = ImportHandlerUtils.readAsString(ClientPersonConstants.ADDRESS_TYPE_COL, row); - Long addressTypeId = null; - if (addressType != null) { - List addressTypeAr = Splitter.on(SEPARATOR).splitToList(addressType); - - if (addressTypeAr.size() > 1 && addressTypeAr.get(1) != null) { - addressTypeId = Long.parseLong(addressTypeAr.get(1)); - } - } + Long addressTypeId = parseIdFromDisplayValue(addressType, "addressType"); String street = ImportHandlerUtils.readAsString(ClientPersonConstants.STREET_COL, row); String addressLine1 = ImportHandlerUtils.readAsString(ClientPersonConstants.ADDRESS_LINE_1_COL, row); String addressLine2 = ImportHandlerUtils.readAsString(ClientPersonConstants.ADDRESS_LINE_2_COL, row); @@ -163,24 +152,9 @@ private ClientData readClient(final Workbook workbook, final Row row, final Stri Boolean isActiveAddress = ImportHandlerUtils.readAsBoolean(ClientPersonConstants.IS_ACTIVE_ADDRESS_COL, row); String stateProvince = ImportHandlerUtils.readAsString(ClientPersonConstants.STATE_PROVINCE_COL, row); - Long stateProvinceId = null; - if (stateProvince != null) { - List stateProvinceAr = Splitter.on(SEPARATOR).splitToList(stateProvince); - // Arkansas-AL <-- expected format of the cell - // but probably it's either an empty cell or it is missing a - // hyphen - if (stateProvinceAr.size() > 1 && stateProvinceAr.get(1) != null) { - stateProvinceId = Long.parseLong(stateProvinceAr.get(1)); - } - } + Long stateProvinceId = parseIdFromDisplayValue(stateProvince, "stateProvince"); String country = ImportHandlerUtils.readAsString(ClientPersonConstants.COUNTRY_COL, row); - Long countryId = null; - if (country != null) { - List countryAr = Splitter.on(SEPARATOR).splitToList(country); - if (countryAr.size() > 1 && countryAr.get(1) != null) { - countryId = Long.parseLong(countryAr.get(1)); - } - } + Long countryId = parseIdFromDisplayValue(country, "country"); addressDataObj = new AddressData(addressTypeId, street, addressLine1, addressLine2, addressLine3, city, postalCode, isActiveAddress, stateProvinceId, countryId); addressList = new ArrayList<>(List.of(addressDataObj)); diff --git a/integration-tests/src/test/java/org/apache/fineract/integrationtests/bulkimport/importhandler/client/ClientEntityImportHandlerTest.java b/integration-tests/src/test/java/org/apache/fineract/integrationtests/bulkimport/importhandler/client/ClientEntityImportHandlerTest.java index 61f9a6958ee..56c72f6add5 100644 --- a/integration-tests/src/test/java/org/apache/fineract/integrationtests/bulkimport/importhandler/client/ClientEntityImportHandlerTest.java +++ b/integration-tests/src/test/java/org/apache/fineract/integrationtests/bulkimport/importhandler/client/ClientEntityImportHandlerTest.java @@ -101,6 +101,12 @@ public void testClientImport() throws InterruptedException, IOException, ParseEx // insert dummy data into client entity sheet Sheet clientEntitySheet = workbook.getSheet(TemplatePopulateImportConstants.CLIENT_ENTITY_SHEET_NAME); + // lookup values (client type, classification, constitution, main business line, ...) live on the hidden + // lookup sheet at fixed column positions; the visible sheet's lookup reference columns are dynamically + // positioned depending on the datatable columns configured for the Client entity, so they can't be + // addressed via the deprecated ClientEntityConstants.LOOKUP_* column indices anymore. + Sheet lookupSheet = workbook.getSheet(TemplatePopulateImportConstants.CLIENT_LOOKUPS_SHEET_NAME); + Row lookupRow = lookupSheet.getRow(1); Row firstClientRow = clientEntitySheet.getRow(1); firstClientRow.createCell(ClientEntityConstants.NAME_COL).setCellValue(Utils.randomStringGenerator("C_E_", 6)); Sheet staffSheet = workbook.getSheet(TemplatePopulateImportConstants.STAFF_SHEET_NAME); @@ -112,15 +118,12 @@ public void testClientImport() throws InterruptedException, IOException, ParseEx Date validTill = simpleDateFormat.parse("14 May 2019"); firstClientRow.createCell(ClientEntityConstants.INCOPORATION_VALID_TILL_COL).setCellValue(validTill); firstClientRow.createCell(ClientEntityConstants.MOBILE_NO_COL).setCellValue(Utils.uniqueRandomNumberGenerator(9)); - firstClientRow.createCell(ClientEntityConstants.CLIENT_TYPE_COL) - .setCellValue(clientEntitySheet.getRow(1).getCell(ClientEntityConstants.LOOKUP_CLIENT_TYPES).getStringCellValue()); + firstClientRow.createCell(ClientEntityConstants.CLIENT_TYPE_COL).setCellValue(lookupRow.getCell(0).getStringCellValue()); firstClientRow.createCell(ClientEntityConstants.CLIENT_CLASSIFICATION_COL) - .setCellValue(clientEntitySheet.getRow(1).getCell(ClientEntityConstants.LOOKUP_CLIENT_CLASSIFICATION).getStringCellValue()); + .setCellValue(lookupRow.getCell(1).getStringCellValue()); firstClientRow.createCell(ClientEntityConstants.INCOPORATION_NUMBER_COL).setCellValue(Utils.randomNumberGenerator(6)); - firstClientRow.createCell(ClientEntityConstants.MAIN_BUSINESS_LINE) - .setCellValue(clientEntitySheet.getRow(1).getCell(ClientEntityConstants.LOOKUP_MAIN_BUSINESS_LINE).getStringCellValue()); - firstClientRow.createCell(ClientEntityConstants.CONSTITUTION_COL) - .setCellValue(clientEntitySheet.getRow(1).getCell(ClientEntityConstants.LOOKUP_CONSTITUTION_COL).getStringCellValue()); + firstClientRow.createCell(ClientEntityConstants.MAIN_BUSINESS_LINE).setCellValue(lookupRow.getCell(3).getStringCellValue()); + firstClientRow.createCell(ClientEntityConstants.CONSTITUTION_COL).setCellValue(lookupRow.getCell(2).getStringCellValue()); firstClientRow.createCell(ClientEntityConstants.ACTIVE_COL).setCellValue("False"); Date submittedDate = simpleDateFormat.parse("28 September 2017"); firstClientRow.createCell(ClientEntityConstants.SUBMITTED_ON_COL).setCellValue(submittedDate); From 708c22dc0babc30f1f921ebe99fb9d8bc986fb50 Mon Sep 17 00:00:00 2001 From: Samer Melhem Date: Thu, 23 Jul 2026 17:40:51 +0300 Subject: [PATCH 6/7] FINERACT-2682: Fix spotless and checkstyle violations --- .../importhandler/ImportHandlerUtils.java | 351 +++++++++--------- .../client/ClientEntityImportHandler.java | 19 +- .../client/ClientPersonImportHandler.java | 20 +- .../client/ClientEntityWorkbookPopulator.java | 170 +++++---- .../client/ClientPersonWorkbookPopulator.java | 217 ++++++----- ...ulkImportWorkbookPopulatorServiceImpl.java | 36 +- .../survey/service/ReadSurveyServiceImpl.java | 8 +- .../client/ClientEntityImportHandlerTest.java | 3 +- 8 files changed, 418 insertions(+), 406 deletions(-) diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/ImportHandlerUtils.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/ImportHandlerUtils.java index ff8841f2273..9e99ba3b2fc 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/ImportHandlerUtils.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/ImportHandlerUtils.java @@ -20,7 +20,12 @@ import com.google.common.base.Splitter; import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.fineract.infrastructure.bulkimport.constants.TemplatePopulateImportConstants; @@ -29,19 +34,14 @@ import org.apache.fineract.infrastructure.core.exception.AbstractPlatformException; import org.apache.fineract.infrastructure.core.exception.PlatformApiDataValidationException; import org.apache.fineract.infrastructure.core.exception.UnsupportedParameterException; +import org.apache.fineract.infrastructure.core.service.DateUtils; import org.apache.fineract.infrastructure.dataqueries.data.DatatableData; +import org.apache.fineract.infrastructure.dataqueries.data.EntityTables; import org.apache.fineract.infrastructure.dataqueries.data.ResultsetColumnHeaderData; +import org.apache.fineract.infrastructure.dataqueries.data.StatusEnum; import org.apache.fineract.infrastructure.dataqueries.domain.EntityDatatableChecks; import org.apache.fineract.infrastructure.dataqueries.domain.EntityDatatableChecksRepository; -import org.apache.fineract.infrastructure.dataqueries.data.EntityTables; -import org.apache.fineract.infrastructure.dataqueries.data.StatusEnum; import org.apache.fineract.infrastructure.dataqueries.service.DatatableReadService; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeFormatterBuilder; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Map; -import org.apache.fineract.infrastructure.core.service.DateUtils; import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.CellStyle; import org.apache.poi.ss.usermodel.CellType; @@ -158,11 +158,11 @@ public static String trimEmptyDecimalPortion(String result) { } /** - * Extracts numeric ID from strings ending with "(id)" format. - * Example: "Permanent (32)" -> "32" - * If the string doesn't match the pattern, returns the original value. - * - * @param value The string value that may contain an ID in parentheses + * Extracts numeric ID from strings ending with "(id)" format. Example: "Permanent (32)" -> "32" If the string + * doesn't match the pattern, returns the original value. + * + * @param value + * The string value that may contain an ID in parentheses * @return The extracted ID as a string, or the original value if no ID pattern is found */ public static String extractIdFromDisplayValue(String value) { @@ -487,32 +487,35 @@ public static String getRepeatsOnDayId(String repeatsOnDay) { /** * Validates that all required datatables are present in the Excel workbook for the given row. - * - * Validation logic: - * 1. Multi-row datatables (1:N with entity) are always skipped - they are optional/repeatable child data. - * 2. If NO column header exists for a required datatable, validation is skipped entirely. - * This allows required datatables to be optional in the Excel sheet if no columns are present. - * 3. If column headers EXIST for a required datatable: - * a. Retrieve the datatable schema to identify mandatory (non-nullable) columns. - * b. If the datatable has ZERO mandatory columns (all columns nullable), validation passes even if all values are blank. - * c. If there are mandatory columns, validate that each mandatory column either: - * - Exists in the Excel header AND has a non-blank value, OR - * - Is missing from the Excel header (treated as missing field). - * d. System columns (id, client_id, created_at, updated_at) and locale/dateFormat are ignored. - * - * This ensures that: - * - Optional datatables (not present in Excel) → no validation error - * - Required datatables with all nullable columns (present but empty) → no validation error - * - Required datatables with mandatory columns (present but missing mandatory values) → validation error - * - Multi-row datatables → always skipped - * - * @param workbook The Excel workbook - * @param sheet The sheet containing the client data - * @param row The data row to validate - * @param entitySubtype The entity subtype (PERSON or ENTITY) - * @param entityDatatableChecksRepository Repository to fetch required datatables - * @param datatableReadService Service to retrieve datatable metadata (can be null, will skip multi-row and mandatory field checks if null) - * @return Error message if validation fails (format: "Missing required datatable fields in : , "), null if validation passes + * + * Validation logic: 1. Multi-row datatables (1:N with entity) are always skipped - they are optional/repeatable + * child data. 2. If NO column header exists for a required datatable, validation is skipped entirely. This allows + * required datatables to be optional in the Excel sheet if no columns are present. 3. If column headers EXIST for a + * required datatable: a. Retrieve the datatable schema to identify mandatory (non-nullable) columns. b. If the + * datatable has ZERO mandatory columns (all columns nullable), validation passes even if all values are blank. c. + * If there are mandatory columns, validate that each mandatory column either: - Exists in the Excel header AND has + * a non-blank value, OR - Is missing from the Excel header (treated as missing field). d. System columns (id, + * client_id, created_at, updated_at) and locale/dateFormat are ignored. + * + * This ensures that: - Optional datatables (not present in Excel) → no validation error - Required datatables with + * all nullable columns (present but empty) → no validation error - Required datatables with mandatory columns + * (present but missing mandatory values) → validation error - Multi-row datatables → always skipped + * + * @param workbook + * The Excel workbook + * @param sheet + * The sheet containing the client data + * @param row + * The data row to validate + * @param entitySubtype + * The entity subtype (PERSON or ENTITY) + * @param entityDatatableChecksRepository + * Repository to fetch required datatables + * @param datatableReadService + * Service to retrieve datatable metadata (can be null, will skip multi-row and mandatory field checks if + * null) + * @return Error message if validation fails (format: "Missing required datatable fields in : , + * "), null if validation passes */ public static String validateRequiredDatatables(Workbook workbook, Sheet sheet, Row row, String entitySubtype, EntityDatatableChecksRepository entityDatatableChecksRepository, DatatableReadService datatableReadService) { @@ -523,11 +526,11 @@ public static String validateRequiredDatatables(Workbook workbook, Sheet sheet, // Get required datatables for CLIENT entity with CREATE status List requiredChecks; if (entitySubtype != null) { - requiredChecks = entityDatatableChecksRepository.findByEntityAndStatusAndSubtype( - EntityTables.CLIENT.getName(), StatusEnum.CREATE.getValue(), entitySubtype); + requiredChecks = entityDatatableChecksRepository.findByEntityAndStatusAndSubtype(EntityTables.CLIENT.getName(), + StatusEnum.CREATE.getValue(), entitySubtype); } else { - requiredChecks = entityDatatableChecksRepository.findByEntityAndStatus( - EntityTables.CLIENT.getName(), StatusEnum.CREATE.getValue()); + requiredChecks = entityDatatableChecksRepository.findByEntityAndStatus(EntityTables.CLIENT.getName(), + StatusEnum.CREATE.getValue()); } if (requiredChecks == null || requiredChecks.isEmpty()) { @@ -556,40 +559,40 @@ public static String validateRequiredDatatables(Workbook workbook, Sheet sheet, List missingDatatables = new ArrayList<>(); for (EntityDatatableChecks check : requiredChecks) { String datatableName = check.getDatatableName(); - + // Skip validation for multi-row datatables - they are optional/repeatable child data if (datatableReadService != null) { try { DatatableData datatableData = datatableReadService.retrieveDatatable(datatableName); if (datatableData != null && datatableData.isMultiRow()) { // Multi-row datatables are optional, skip validation - log.debug("Skipping validation for multi-row datatable '{}' - treated as optional/repeatable child data", + log.debug("Skipping validation for multi-row datatable '{}' - treated as optional/repeatable child data", datatableName); continue; } } catch (Exception e) { // If we can't retrieve datatable metadata, continue with validation // This preserves backward compatibility - log.debug("Could not retrieve datatable metadata for '{}' to check multi-row status: {}", - datatableName, e.getMessage()); + log.debug("Could not retrieve datatable metadata for '{}' to check multi-row status: {}", datatableName, + e.getMessage()); } } - + // First, check if ANY column header exists for this datatable (presence check) boolean headerExists = false; List matchingColumnIndices = new ArrayList<>(); - + // Check if any column header matches the datatable name pattern // Prefer dot notation (new format), fallback to underscore notation (legacy) for (Map.Entry entry : headerColumnMap.entrySet()) { String header = entry.getKey(); int colIndex = entry.getValue(); - + // Remove trailing * if present String headerWithoutStar = header.endsWith("*") ? header.substring(0, header.length() - 1) : header; - + boolean matches = false; - + // Check dot notation first (preferred format): registeredTableName.columnName if (headerWithoutStar.contains(".")) { int dotIndex = headerWithoutStar.indexOf("."); @@ -610,32 +613,31 @@ else if (headerWithoutStar.contains("_")) { } } } - + if (matches) { headerExists = true; matchingColumnIndices.add(colIndex); } } - + // If no header exists for this datatable, skip validation entirely // This handles the case where a required datatable is configured but not present in the Excel sheet if (!headerExists) { - log.debug("Skipping validation for required datatable '{}' - no column headers found in Excel sheet. " + - "Validation only applies when at least one column header for the datatable is present.", - datatableName); + log.debug("Skipping validation for required datatable '{}' - no column headers found in Excel sheet. " + + "Validation only applies when at least one column header for the datatable is present.", datatableName); continue; } - + // Header exists, so now validate mandatory columns // Build a map of Excel column names (without datatable prefix) to column indices Map excelColumnMap = new HashMap<>(); for (Map.Entry entry : headerColumnMap.entrySet()) { String header = entry.getKey(); int colIndex = entry.getValue(); - + // Remove trailing * if present String headerWithoutStar = header.endsWith("*") ? header.substring(0, header.length() - 1) : header; - + String columnName = null; // Check dot notation first (preferred format): registeredTableName.columnName if (headerWithoutStar.contains(".")) { @@ -657,12 +659,12 @@ else if (headerWithoutStar.contains("_")) { } } } - + if (columnName != null) { excelColumnMap.put(columnName, colIndex); } } - + // Retrieve datatable schema to check for mandatory columns List missingMandatoryFields = new ArrayList<>(); if (datatableReadService != null) { @@ -673,35 +675,31 @@ else if (headerWithoutStar.contains("_")) { List mandatoryColumns = new ArrayList<>(); for (ResultsetColumnHeaderData column : datatableData.getColumnHeaderData()) { String columnName = column.getColumnName(); - + // Ignore system columns - if (columnName.equalsIgnoreCase("id") - || columnName.equalsIgnoreCase("client_id") - || columnName.equalsIgnoreCase("created_at") - || columnName.equalsIgnoreCase("updated_at") - || columnName.equalsIgnoreCase("locale") - || columnName.equalsIgnoreCase("dateFormat")) { + if (columnName.equalsIgnoreCase("id") || columnName.equalsIgnoreCase("client_id") + || columnName.equalsIgnoreCase("created_at") || columnName.equalsIgnoreCase("updated_at") + || columnName.equalsIgnoreCase("locale") || columnName.equalsIgnoreCase("dateFormat")) { continue; } - + // Check if column is mandatory (not nullable) if (!column.getIsColumnNullable()) { mandatoryColumns.add(columnName); } } - + // If there are no mandatory columns, skip validation (all columns are nullable) if (mandatoryColumns.isEmpty()) { - log.debug("Skipping validation for required datatable '{}' - no mandatory fields found. " + - "All columns are nullable, so blank values are allowed.", - datatableName); + log.debug("Skipping validation for required datatable '{}' - no mandatory fields found. " + + "All columns are nullable, so blank values are allowed.", datatableName); continue; } - + // Check each mandatory column for (String mandatoryColumn : mandatoryColumns) { Integer colIndex = excelColumnMap.get(mandatoryColumn); - + if (colIndex == null) { // Mandatory column is missing from Excel header missingMandatoryFields.add(mandatoryColumn); @@ -715,20 +713,20 @@ else if (headerWithoutStar.contains("_")) { hasValue = true; } } - + if (!hasValue) { missingMandatoryFields.add(mandatoryColumn); } } } - + // If mandatory fields are missing, add to error list if (!missingMandatoryFields.isEmpty()) { String friendlyName = getFriendlyDatatableName(datatableName); String fieldsList = String.join(", ", missingMandatoryFields); missingDatatables.add(friendlyName + ": " + fieldsList); - log.debug("Failing validation for required datatable '{}' - missing mandatory fields: {}", - datatableName, fieldsList); + log.debug("Failing validation for required datatable '{}' - missing mandatory fields: {}", datatableName, + fieldsList); } } else { // If we can't retrieve datatable schema, fall back to old behavior @@ -744,13 +742,12 @@ else if (headerWithoutStar.contains("_")) { } } } - + if (!hasValue) { String friendlyName = getFriendlyDatatableName(datatableName); missingDatatables.add(friendlyName); - log.debug("Required datatable '{}' has column headers in Excel but no values provided " + - "(datatable schema unavailable for mandatory field check)", - datatableName); + log.debug("Required datatable '{}' has column headers in Excel but no values provided " + + "(datatable schema unavailable for mandatory field check)", datatableName); } } } catch (Exception e) { @@ -767,13 +764,12 @@ else if (headerWithoutStar.contains("_")) { } } } - + if (!hasValue) { String friendlyName = getFriendlyDatatableName(datatableName); missingDatatables.add(friendlyName); - log.debug("Required datatable '{}' has column headers in Excel but no values provided " + - "(error retrieving datatable schema: {})", - datatableName, e.getMessage()); + log.debug("Required datatable '{}' has column headers in Excel but no values provided " + + "(error retrieving datatable schema: {})", datatableName, e.getMessage()); } } } else { @@ -790,13 +786,12 @@ else if (headerWithoutStar.contains("_")) { } } } - + if (!hasValue) { String friendlyName = getFriendlyDatatableName(datatableName); missingDatatables.add(friendlyName); - log.debug("Required datatable '{}' has column headers in Excel but no values provided " + - "(datatableReadService unavailable for mandatory field check)", - datatableName); + log.debug("Required datatable '{}' has column headers in Excel but no values provided " + + "(datatableReadService unavailable for mandatory field check)", datatableName); } } } @@ -854,23 +849,27 @@ private static String getFriendlyDatatableName(String datatableName) { /** * Reads datatable columns from the Excel row and groups them by registered table name. - * - * Preferred format (new): . or .* - * Legacy format (backward compatibility): _ or _* - * - * For dot notation: splits at the first dot (only one is expected). - * For legacy underscore notation: splits at the last underscore. - * - * @param sheet The sheet containing the client data - * @param row The data row to read from - * @param locale The locale for datatable data - * @param dateFormat The date format for datatable data - * @return List of maps representing datatables, each with "registeredTableName" and "data" keys. - * Only includes datatables with at least one non-empty value. + * + * Preferred format (new): . or .* Legacy format + * (backward compatibility): _ or _* + * + * For dot notation: splits at the first dot (only one is expected). For legacy underscore notation: splits at the + * last underscore. + * + * @param sheet + * The sheet containing the client data + * @param row + * The data row to read from + * @param locale + * The locale for datatable data + * @param dateFormat + * The date format for datatable data + * @return List of maps representing datatables, each with "registeredTableName" and "data" keys. Only includes + * datatables with at least one non-empty value. */ public static List> readDatatablesFromRow(Sheet sheet, Row row, String locale, String dateFormat) { List> datatablesList = new ArrayList<>(); - + if (sheet == null || row == null) { return datatablesList; } @@ -896,17 +895,17 @@ public static List> readDatatablesFromRow(Sheet sheet, Row r // Group datatable columns by datatable name Map> datatablesMap = new HashMap<>(); boolean legacyHeaderDetected = false; - + for (Map.Entry entry : headerColumnMap.entrySet()) { String header = entry.getKey(); int colIndex = entry.getValue(); - + // Remove trailing * if present String headerWithoutStar = header.endsWith("*") ? header.substring(0, header.length() - 1) : header; - + String registeredTableName; String columnName = null; - + // Prefer dot notation (new format): registeredTableName.columnName if (headerWithoutStar.contains(".")) { int dotIndex = headerWithoutStar.indexOf("."); @@ -916,7 +915,7 @@ public static List> readDatatablesFromRow(Sheet sheet, Row r } else { registeredTableName = null; } - } + } // Fallback to underscore notation (legacy format) for backward compatibility else if (headerWithoutStar.contains("_")) { legacyHeaderDetected = true; @@ -941,7 +940,7 @@ else if (headerWithoutStar.contains("_")) { || columnName.equalsIgnoreCase("created_at") || columnName.equalsIgnoreCase("updated_at")) { continue; } - + // Read cell value Cell dataCell = row.getCell(colIndex); Object cellValue = null; @@ -953,9 +952,7 @@ else if (headerWithoutStar.contains("_")) { LocalDate dateValue = readAsDate(colIndex, row); if (dateValue != null && dateFormat != null) { try { - DateTimeFormatter formatter = new DateTimeFormatterBuilder() - .appendPattern(dateFormat) - .toFormatter(); + DateTimeFormatter formatter = new DateTimeFormatterBuilder().appendPattern(dateFormat).toFormatter(); cellValue = dateValue.format(formatter); } catch (Exception e) { // Fallback to ISO format if dateFormat is invalid @@ -997,7 +994,7 @@ else if (headerWithoutStar.contains("_")) { } } } - + // Only add if value is not null/empty if (cellValue != null) { // Get or create datatable entry using registeredTableName @@ -1010,7 +1007,7 @@ else if (headerWithoutStar.contains("_")) { newDatatable.put("data", data); return newDatatable; }); - + // Add column value to data @SuppressWarnings("unchecked") Map data = (Map) datatableData.get("data"); @@ -1018,12 +1015,13 @@ else if (headerWithoutStar.contains("_")) { } } } - + // Log warning if legacy underscore-based headers were detected if (legacyHeaderDetected) { - log.warn("Legacy underscore-based datatable column headers detected. Please update template to use dot notation (registeredTableName.columnName) for unambiguous parsing."); + log.warn( + "Legacy underscore-based datatable column headers detected. Please update template to use dot notation (registeredTableName.columnName) for unambiguous parsing."); } - + // Convert map values to list (only include datatables with at least one data field beyond locale/dateFormat) for (Map datatable : datatablesMap.values()) { @SuppressWarnings("unchecked") @@ -1033,38 +1031,43 @@ else if (headerWithoutStar.contains("_")) { datatablesList.add(datatable); } } - + return datatablesList; } /** - * Ensures all configured client datatables are included in the payload, even if they have no values. - * This is required for bulk import to match Fineract's contract where all entity-linked datatables must be present. - * Only includes datatables that are explicitly linked to Client via EntityDatatableChecks. - * - * Construction rules: - * - If datatable has at least one required column (non-nullable): existing behavior applies (must be filled by user) - * - If datatable has NO required columns: always construct the datatable in payload, even if multi-row - * - Multi-row datatables with required columns are skipped (user must provide data) - * - Multi-row datatables with no required columns are included with empty entry - * - * @param sheet The sheet containing the client data - * @param row The data row to read from - * @param locale The locale for datatable data - * @param dateFormat The date format for datatable data - * @param entityDatatableChecksRepository Repository to fetch datatables linked to Client entity (can be null, will skip if null) - * @param datatableReadService Service to retrieve datatable metadata (can be null, will skip multi-row check if null) - * @param entitySubtype The entity subtype (PERSON or ENTITY), can be null - * @return List of maps representing datatables, each with "registeredTableName" and "data" keys. - * Includes only datatables linked via EntityDatatableChecks, with empty entries for those without values. + * Ensures all configured client datatables are included in the payload, even if they have no values. This is + * required for bulk import to match Fineract's contract where all entity-linked datatables must be present. Only + * includes datatables that are explicitly linked to Client via EntityDatatableChecks. + * + * Construction rules: - If datatable has at least one required column (non-nullable): existing behavior applies + * (must be filled by user) - If datatable has NO required columns: always construct the datatable in payload, even + * if multi-row - Multi-row datatables with required columns are skipped (user must provide data) - Multi-row + * datatables with no required columns are included with empty entry + * + * @param sheet + * The sheet containing the client data + * @param row + * The data row to read from + * @param locale + * The locale for datatable data + * @param dateFormat + * The date format for datatable data + * @param entityDatatableChecksRepository + * Repository to fetch datatables linked to Client entity (can be null, will skip if null) + * @param datatableReadService + * Service to retrieve datatable metadata (can be null, will skip multi-row check if null) + * @param entitySubtype + * The entity subtype (PERSON or ENTITY), can be null + * @return List of maps representing datatables, each with "registeredTableName" and "data" keys. Includes only + * datatables linked via EntityDatatableChecks, with empty entries for those without values. */ - public static List> readDatatablesFromRowWithAllConfigured( - Sheet sheet, Row row, String locale, String dateFormat, - EntityDatatableChecksRepository entityDatatableChecksRepository, - DatatableReadService datatableReadService, String entitySubtype) { + public static List> readDatatablesFromRowWithAllConfigured(Sheet sheet, Row row, String locale, String dateFormat, + EntityDatatableChecksRepository entityDatatableChecksRepository, DatatableReadService datatableReadService, + String entitySubtype) { // First, read datatables that have values from Excel List> datatablesWithValues = readDatatablesFromRow(sheet, row, locale, dateFormat); - + // Create a map of registeredTableName -> datatable for quick lookup Map> datatablesMap = new HashMap<>(); for (Map datatable : datatablesWithValues) { @@ -1073,55 +1076,52 @@ public static List> readDatatablesFromRowWithAllConfigured( datatablesMap.put(registeredTableName, datatable); } } - + // If entityDatatableChecksRepository is available, ensure all linked client datatables are included if (entityDatatableChecksRepository != null) { try { // Retrieve only datatables linked to Client via EntityDatatableChecks with CREATE status List linkedChecks; if (entitySubtype != null) { - linkedChecks = entityDatatableChecksRepository.findByEntityAndStatusAndSubtype( - EntityTables.CLIENT.getName(), StatusEnum.CREATE.getValue(), entitySubtype); + linkedChecks = entityDatatableChecksRepository.findByEntityAndStatusAndSubtype(EntityTables.CLIENT.getName(), + StatusEnum.CREATE.getValue(), entitySubtype); } else { - linkedChecks = entityDatatableChecksRepository.findByEntityAndStatus( - EntityTables.CLIENT.getName(), StatusEnum.CREATE.getValue()); + linkedChecks = entityDatatableChecksRepository.findByEntityAndStatus(EntityTables.CLIENT.getName(), + StatusEnum.CREATE.getValue()); } - + // For each linked datatable, ensure it's in the result for (EntityDatatableChecks check : linkedChecks) { String registeredTableName = check.getDatatableName(); - + // Skip if already included (has values) if (datatablesMap.containsKey(registeredTableName)) { continue; } - + // Retrieve datatable schema to check multi-row status and required columns boolean isMultiRow = false; boolean hasRequiredColumns = false; org.apache.fineract.infrastructure.dataqueries.data.DatatableData datatableData = null; - + if (datatableReadService != null) { try { datatableData = datatableReadService.retrieveDatatable(registeredTableName); if (datatableData != null) { isMultiRow = datatableData.isMultiRow(); - + // Check if datatable has at least one required column if (datatableData.getColumnHeaderData() != null) { for (ResultsetColumnHeaderData column : datatableData.getColumnHeaderData()) { String columnName = column.getColumnName(); - + // Skip system columns - if (columnName.equalsIgnoreCase("id") - || columnName.equalsIgnoreCase("client_id") - || columnName.equalsIgnoreCase("created_at") - || columnName.equalsIgnoreCase("updated_at") - || columnName.equalsIgnoreCase("locale") - || columnName.equalsIgnoreCase("dateFormat")) { + if (columnName.equalsIgnoreCase("id") || columnName.equalsIgnoreCase("client_id") + || columnName.equalsIgnoreCase("created_at") || columnName.equalsIgnoreCase("updated_at") + || columnName.equalsIgnoreCase("locale") || columnName.equalsIgnoreCase("dateFormat")) { continue; } - + // Check if column is required (not nullable) if (!column.getIsColumnNullable()) { hasRequiredColumns = true; @@ -1131,44 +1131,41 @@ public static List> readDatatablesFromRowWithAllConfigured( } } } catch (Exception e) { - // If we can't retrieve datatable metadata, continue (assume not multi-row, no required columns) - log.debug("Could not retrieve datatable metadata for '{}': {}", - registeredTableName, e.getMessage()); + // If we can't retrieve datatable metadata, continue (assume not multi-row, no required + // columns) + log.debug("Could not retrieve datatable metadata for '{}': {}", registeredTableName, e.getMessage()); } } - + // Skip multi-row datatables ONLY if they have required columns // Multi-row datatables with NO required columns should be included (empty entry) if (isMultiRow && hasRequiredColumns) { // Multi-row with required columns: skip (user must provide data) continue; } - + // Create empty datatable entry with locale and dateFormat Map emptyDatatable = new HashMap<>(); emptyDatatable.put("registeredTableName", registeredTableName); Map data = new HashMap<>(); data.put("locale", locale); data.put("dateFormat", dateFormat); - + // Add empty strings for known columns if schema is available if (datatableData != null && datatableData.getColumnHeaderData() != null) { for (ResultsetColumnHeaderData column : datatableData.getColumnHeaderData()) { String columnName = column.getColumnName(); // Skip system columns - if (columnName.equalsIgnoreCase("id") - || columnName.equalsIgnoreCase("client_id") - || columnName.equalsIgnoreCase("created_at") - || columnName.equalsIgnoreCase("updated_at") - || columnName.equalsIgnoreCase("locale") - || columnName.equalsIgnoreCase("dateFormat")) { + if (columnName.equalsIgnoreCase("id") || columnName.equalsIgnoreCase("client_id") + || columnName.equalsIgnoreCase("created_at") || columnName.equalsIgnoreCase("updated_at") + || columnName.equalsIgnoreCase("locale") || columnName.equalsIgnoreCase("dateFormat")) { continue; } // Add empty string for known columns data.put(columnName, ""); } } - + emptyDatatable.put("data", data); datatablesMap.put(registeredTableName, emptyDatatable); } @@ -1177,7 +1174,7 @@ public static List> readDatatablesFromRowWithAllConfigured( log.debug("Could not retrieve linked client datatables for bulk import payload: {}", e.getMessage()); } } - + // Return all datatables (with values and empty ones) return new ArrayList<>(datatablesMap.values()); } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientEntityImportHandler.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientEntityImportHandler.java index 780425ae432..706717db4c2 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientEntityImportHandler.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientEntityImportHandler.java @@ -19,11 +19,14 @@ package org.apache.fineract.infrastructure.bulkimport.importhandler.client; import com.google.gson.GsonBuilder; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import java.time.LocalDate; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -47,9 +50,6 @@ import org.apache.fineract.portfolio.address.data.AddressData; import org.apache.fineract.portfolio.client.data.ClientData; import org.apache.fineract.portfolio.client.data.ClientNonPersonData; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import java.util.Map; import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.IndexedColors; import org.apache.poi.ss.usermodel.Row; @@ -215,21 +215,20 @@ private Count importEntity(final Workbook workbook, final List clien Row clientRow = clientSheet.getRow(client.getRowIndex()); if (clientRow != null) { // Validate required datatables before attempting client creation - String validationError = ImportHandlerUtils.validateRequiredDatatables( - workbook, clientSheet, clientRow, "Entity", entityDatatableChecksRepository, datatableReadService); + String validationError = ImportHandlerUtils.validateRequiredDatatables(workbook, clientSheet, clientRow, "Entity", + entityDatatableChecksRepository, datatableReadService); if (validationError != null) { throw new RuntimeException(validationError); } } String payload = gsonBuilder.create().toJson(client); - + // Read datatables from the Excel row and add to payload // Ensure all configured client datatables are included, even if empty if (clientRow != null) { - List> datatables = ImportHandlerUtils.readDatatablesFromRowWithAllConfigured( - clientSheet, clientRow, locale, dateFormat, entityDatatableChecksRepository, - datatableReadService, "Entity"); + List> datatables = ImportHandlerUtils.readDatatablesFromRowWithAllConfigured(clientSheet, clientRow, + locale, dateFormat, entityDatatableChecksRepository, datatableReadService, "Entity"); if (datatables != null && !datatables.isEmpty()) { // Parse JSON and add datatables array JsonObject jsonObject = JsonParser.parseString(payload).getAsJsonObject(); @@ -237,7 +236,7 @@ private Count importEntity(final Workbook workbook, final List clien payload = gsonBuilder.create().toJson(jsonObject); } } - + final CommandWrapper commandRequest = new CommandWrapperBuilder() // .createClient() // .withJson(payload) // diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientPersonImportHandler.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientPersonImportHandler.java index 761ad63f40b..11c2052a62f 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientPersonImportHandler.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientPersonImportHandler.java @@ -19,10 +19,13 @@ package org.apache.fineract.infrastructure.bulkimport.importhandler.client; import com.google.gson.GsonBuilder; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import java.time.LocalDate; import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.Objects; import lombok.AllArgsConstructor; import org.apache.fineract.commands.domain.CommandWrapper; @@ -41,10 +44,6 @@ import org.apache.fineract.infrastructure.dataqueries.service.DatatableReadService; import org.apache.fineract.portfolio.address.data.AddressData; import org.apache.fineract.portfolio.client.data.ClientData; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import java.util.List; -import java.util.Map; import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.IndexedColors; import org.apache.poi.ss.usermodel.Row; @@ -178,21 +177,20 @@ public Count importEntity(final Workbook workbook, final List client Row clientRow = clientSheet.getRow(client.getRowIndex()); if (clientRow != null) { // Validate required datatables before attempting client creation - String validationError = ImportHandlerUtils.validateRequiredDatatables( - workbook, clientSheet, clientRow, "Person", entityDatatableChecksRepository, datatableReadService); + String validationError = ImportHandlerUtils.validateRequiredDatatables(workbook, clientSheet, clientRow, "Person", + entityDatatableChecksRepository, datatableReadService); if (validationError != null) { throw new RuntimeException(validationError); } } String payload = gsonBuilder.create().toJson(client); - + // Read datatables from the Excel row and add to payload // Ensure all configured client datatables are included, even if empty if (clientRow != null) { - List> datatables = ImportHandlerUtils.readDatatablesFromRowWithAllConfigured( - clientSheet, clientRow, locale, dateFormat, entityDatatableChecksRepository, - datatableReadService, "Person"); + List> datatables = ImportHandlerUtils.readDatatablesFromRowWithAllConfigured(clientSheet, clientRow, + locale, dateFormat, entityDatatableChecksRepository, datatableReadService, "Person"); if (datatables != null && !datatables.isEmpty()) { // Parse JSON and add datatables array JsonObject jsonObject = JsonParser.parseString(payload).getAsJsonObject(); @@ -200,7 +198,7 @@ public Count importEntity(final Workbook workbook, final List client payload = gsonBuilder.create().toJson(jsonObject); } } - + final CommandWrapper commandRequest = new CommandWrapperBuilder() // .createClient() // .withJson(payload) // diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/client/ClientEntityWorkbookPopulator.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/client/ClientEntityWorkbookPopulator.java index 82a8b3a84be..f37798751c2 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/client/ClientEntityWorkbookPopulator.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/client/ClientEntityWorkbookPopulator.java @@ -18,7 +18,10 @@ */ package org.apache.fineract.infrastructure.bulkimport.populator.client; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.apache.fineract.infrastructure.bulkimport.constants.ClientEntityConstants; import org.apache.fineract.infrastructure.bulkimport.constants.TemplatePopulateImportConstants; import org.apache.fineract.infrastructure.bulkimport.populator.AbstractWorkbookPopulator; @@ -27,8 +30,8 @@ import org.apache.fineract.infrastructure.codes.data.CodeValueData; import org.apache.fineract.infrastructure.dataqueries.data.DatatableData; import org.apache.fineract.infrastructure.dataqueries.data.ResultsetColumnHeaderData; +import org.apache.fineract.infrastructure.dataqueries.data.ResultsetColumnValueData; import org.apache.fineract.organisation.office.data.OfficeData; -import java.util.ArrayList; import org.apache.poi.hssf.usermodel.HSSFDataValidationHelper; import org.apache.poi.hssf.usermodel.HSSFSheet; import org.apache.poi.ss.SpreadsheetVersion; @@ -45,9 +48,6 @@ import org.apache.poi.ss.usermodel.Workbook; import org.apache.poi.ss.util.CellRangeAddressList; import org.apache.poi.ss.util.CellReference; -import org.apache.fineract.infrastructure.dataqueries.data.ResultsetColumnValueData; -import java.util.Map; -import java.util.HashMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -76,6 +76,7 @@ public class ClientEntityWorkbookPopulator extends AbstractWorkbookPopulator { // Inner class to track datatable dropdown information private static class DatatableDropdownInfo { + final int clientSheetColumnIndex; final int lookupSheetColumnIndex; final List values; @@ -89,6 +90,7 @@ private static class DatatableDropdownInfo { // Inner class to track datatable date column information private static class DatatableDateInfo { + final int clientSheetColumnIndex; final boolean isDateTime; @@ -100,6 +102,7 @@ private static class DatatableDateInfo { // Inner class to track datatable boolean column information private static class DatatableBooleanInfo { + final int clientSheetColumnIndex; DatatableBooleanInfo(int clientSheetColumnIndex) { @@ -128,14 +131,14 @@ public void populate(Workbook workbook, String dateFormat) { Sheet clientSheet = workbook.createSheet(TemplatePopulateImportConstants.CLIENT_ENTITY_SHEET_NAME); personnelSheetPopulator.populate(workbook, dateFormat); officeSheetPopulator.populate(workbook, dateFormat); - + // Create hidden lookup sheet for lookup data Sheet lookupSheet = workbook.createSheet(TemplatePopulateImportConstants.CLIENT_LOOKUPS_SHEET_NAME); // TODO: For debugging, temporarily set to VISIBLE. Revert to VERY_HIDDEN once validated. workbook.setSheetVisibility(workbook.getSheetIndex(lookupSheet), SheetVisibility.VERY_HIDDEN); - setLayout(clientSheet); // fills datatableDropdowns - setClientDataLookupTable(lookupSheet); // now values exist + setLayout(clientSheet); // fills datatableDropdowns + setClientDataLookupTable(lookupSheet); // now values exist setFormatStyle(workbook, clientSheet); setRules(clientSheet, dateFormat); handleDatatableColumnsComplete(clientSheet, lookupSheet); @@ -183,7 +186,7 @@ private void setClientDataLookupTable(Sheet lookupSheet) { writeString(0, row, clientTypeCodeValue.getName() + " (" + clientTypeCodeValue.getId() + ")"); rowIndex++; } - + // Column 1 (B): Client Classification rowIndex = 1; for (CodeValueData clientClassificationCodeValue : clientClassificationCodeValues) { @@ -194,7 +197,7 @@ private void setClientDataLookupTable(Sheet lookupSheet) { writeString(1, row, clientClassificationCodeValue.getName() + " (" + clientClassificationCodeValue.getId() + ")"); rowIndex++; } - + // Column 2 (C): Constitution rowIndex = 1; for (CodeValueData constitutionCodeValue : constitutionCodeValues) { @@ -205,7 +208,7 @@ private void setClientDataLookupTable(Sheet lookupSheet) { writeString(2, row, constitutionCodeValue.getName() + " (" + constitutionCodeValue.getId() + ")"); rowIndex++; } - + // Column 3 (D): Main Business Line rowIndex = 1; for (CodeValueData mainBusinessCodeValue : mainBusinesslineCodeValues) { @@ -216,7 +219,7 @@ private void setClientDataLookupTable(Sheet lookupSheet) { writeString(3, row, mainBusinessCodeValue.getName() + " (" + mainBusinessCodeValue.getId() + ")"); rowIndex++; } - + // Column 4 (E): Address Type rowIndex = 1; for (CodeValueData addressTypeCodeValue : addressTypesCodeValues) { @@ -227,7 +230,7 @@ private void setClientDataLookupTable(Sheet lookupSheet) { writeString(4, row, addressTypeCodeValue.getName() + " (" + addressTypeCodeValue.getId() + ")"); rowIndex++; } - + // Column 5 (F): State/Province rowIndex = 1; for (CodeValueData stateCodeValue : stateProvinceCodeValues) { @@ -238,7 +241,7 @@ private void setClientDataLookupTable(Sheet lookupSheet) { writeString(5, row, stateCodeValue.getName() + " (" + stateCodeValue.getId() + ")"); rowIndex++; } - + // Column 6 (G): Country rowIndex = 1; for (CodeValueData countryCodeValue : countryCodeValues) { @@ -249,9 +252,9 @@ private void setClientDataLookupTable(Sheet lookupSheet) { writeString(6, row, countryCodeValue.getName() + " (" + countryCodeValue.getId() + ")"); rowIndex++; } - + } - + private String sanitizeNamedRangeName(String name) { // Excel named ranges cannot contain certain characters // Replace invalid characters with underscore @@ -372,15 +375,14 @@ private void setLayout(Sheet worksheet) { // Populate visible lookup values under lookup headers (for user reference) // These are read-only reference values; validations still use the hidden lookup sheet - populateLookupValues(worksheet, lookupOfficeNameCol, lookupOfficeDateCol, lookupConstitutionCol, - lookupClientTypesCol, lookupClientClassificationCol, lookupAddressTypeCol, - lookupStateProvinceCol, lookupCountryCol, lookupBusinessLineCol); + populateLookupValues(worksheet, lookupOfficeNameCol, lookupOfficeDateCol, lookupConstitutionCol, lookupClientTypesCol, + lookupClientClassificationCol, lookupAddressTypeCol, lookupStateProvinceCol, lookupCountryCol, lookupBusinessLineCol); } - private void populateLookupValues(Sheet worksheet, int lookupOfficeNameCol, int lookupOfficeDateCol, - int lookupConstitutionCol, int lookupClientTypesCol, int lookupClientClassificationCol, - int lookupAddressTypeCol, int lookupStateProvinceCol, int lookupCountryCol, int lookupBusinessLineCol) { - + private void populateLookupValues(Sheet worksheet, int lookupOfficeNameCol, int lookupOfficeDateCol, int lookupConstitutionCol, + int lookupClientTypesCol, int lookupClientClassificationCol, int lookupAddressTypeCol, int lookupStateProvinceCol, + int lookupCountryCol, int lookupBusinessLineCol) { + // Populate Office Name lookup values int rowIndex = 1; // Start at Excel row 2 (POI index 1) List offices = officeSheetPopulator.getOffices(); @@ -395,7 +397,7 @@ private void populateLookupValues(Sheet worksheet, int lookupOfficeNameCol, int } rowIndex++; } - + // Populate Constitution lookup values rowIndex = 1; for (CodeValueData constitutionCodeValue : constitutionCodeValues) { @@ -406,7 +408,7 @@ private void populateLookupValues(Sheet worksheet, int lookupOfficeNameCol, int writeString(lookupConstitutionCol, row, constitutionCodeValue.getName() + " (" + constitutionCodeValue.getId() + ")"); rowIndex++; } - + // Populate Client Types lookup values rowIndex = 1; for (CodeValueData clientTypeCodeValue : clientTypeCodeValues) { @@ -417,7 +419,7 @@ private void populateLookupValues(Sheet worksheet, int lookupOfficeNameCol, int writeString(lookupClientTypesCol, row, clientTypeCodeValue.getName() + " (" + clientTypeCodeValue.getId() + ")"); rowIndex++; } - + // Populate Client Classification lookup values rowIndex = 1; for (CodeValueData clientClassificationCodeValue : clientClassificationCodeValues) { @@ -425,10 +427,11 @@ private void populateLookupValues(Sheet worksheet, int lookupOfficeNameCol, int if (row == null) { row = worksheet.createRow(rowIndex); } - writeString(lookupClientClassificationCol, row, clientClassificationCodeValue.getName() + " (" + clientClassificationCodeValue.getId() + ")"); + writeString(lookupClientClassificationCol, row, + clientClassificationCodeValue.getName() + " (" + clientClassificationCodeValue.getId() + ")"); rowIndex++; } - + // Populate Address Type lookup values rowIndex = 1; for (CodeValueData addressTypeCodeValue : addressTypesCodeValues) { @@ -439,7 +442,7 @@ private void populateLookupValues(Sheet worksheet, int lookupOfficeNameCol, int writeString(lookupAddressTypeCol, row, addressTypeCodeValue.getName() + " (" + addressTypeCodeValue.getId() + ")"); rowIndex++; } - + // Populate State/Province lookup values rowIndex = 1; for (CodeValueData stateCodeValue : stateProvinceCodeValues) { @@ -450,7 +453,7 @@ private void populateLookupValues(Sheet worksheet, int lookupOfficeNameCol, int writeString(lookupStateProvinceCol, row, stateCodeValue.getName() + " (" + stateCodeValue.getId() + ")"); rowIndex++; } - + // Populate Country lookup values rowIndex = 1; for (CodeValueData countryCodeValue : countryCodeValues) { @@ -461,7 +464,7 @@ private void populateLookupValues(Sheet worksheet, int lookupOfficeNameCol, int writeString(lookupCountryCol, row, countryCodeValue.getName() + " (" + countryCodeValue.getId() + ")"); rowIndex++; } - + // Populate Business Line lookup values rowIndex = 1; for (CodeValueData mainBusinessCodeValue : mainBusinesslineCodeValues) { @@ -475,11 +478,13 @@ private void populateLookupValues(Sheet worksheet, int lookupOfficeNameCol, int } /** - * Handles datatable column header creation and dropdown tracking. - * This is called from within setLayout() to insert headers at the correct position. - * - * @param worksheet The client sheet - * @param rowHeader The header row + * Handles datatable column header creation and dropdown tracking. This is called from within setLayout() to insert + * headers at the correct position. + * + * @param worksheet + * The client sheet + * @param rowHeader + * The header row * @return The next column index after datatable columns (or STATUS_COL + 1 if no datatables) */ private int handleDatatableColumnHeaders(Sheet worksheet, Row rowHeader) { @@ -487,7 +492,7 @@ private int handleDatatableColumnHeaders(Sheet worksheet, Row rowHeader) { try { int currentCol = startCol; int lookupSheetCol = 7; // Start after Country (column G = index 6), so next available is H = index 7 - + for (DatatableData datatable : requiredDatatables) { if (datatable == null) { continue; @@ -499,17 +504,17 @@ private int handleDatatableColumnHeaders(Sheet worksheet, Row rowHeader) { try { // Skip system columns (id, client_id, etc.) String columnName = column.getColumnName(); - if (columnName == null || columnName.equalsIgnoreCase("id") - || columnName.equalsIgnoreCase("client_id") - || columnName.equalsIgnoreCase("created_at") - || columnName.equalsIgnoreCase("updated_at")) { + if (columnName == null || columnName.equalsIgnoreCase("id") || columnName.equalsIgnoreCase("client_id") + || columnName.equalsIgnoreCase("created_at") || columnName.equalsIgnoreCase("updated_at")) { continue; } - + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.MEDIUM_COL_SIZE); // Use dot notation: registeredTableName.columnName for unambiguous parsing - // For single-row datatables: treat as mandatory inline data (append "*" if column is not nullable) - // For multi-row datatables: treat as optional/repeatable child data (no "*" even if not nullable) + // For single-row datatables: treat as mandatory inline data (append "*" if column is not + // nullable) + // For multi-row datatables: treat as optional/repeatable child data (no "*" even if not + // nullable) String headerLabel = datatableName + "." + columnName; if (!datatable.isMultiRow() && !column.getIsColumnNullable()) { // Single-row datatables: mark mandatory columns with "*" @@ -517,31 +522,33 @@ private int handleDatatableColumnHeaders(Sheet worksheet, Row rowHeader) { } // Multi-row datatables are always optional (no "*" marker) writeString(currentCol, rowHeader, headerLabel); - + // Check if this is a dropdown column (CODELOOKUP) with values if (column.isCodeLookupDisplayType() && column.hasColumnValues()) { String namedRangeName = sanitizeNamedRangeName(datatableName + "_" + columnName); List columnValues = column.getColumnValues(); if (columnValues != null && !columnValues.isEmpty()) { - datatableDropdowns.put(namedRangeName, new DatatableDropdownInfo(currentCol, lookupSheetCol, columnValues)); + datatableDropdowns.put(namedRangeName, + new DatatableDropdownInfo(currentCol, lookupSheetCol, columnValues)); lookupSheetCol++; } } // Check if this is a date or datetime column else if (column.isDateDisplayType() || column.isDateTimeDisplayType()) { - datatableDateColumns.put(currentCol, new ClientEntityWorkbookPopulator.DatatableDateInfo(currentCol, column.isDateTimeDisplayType())); + datatableDateColumns.put(currentCol, + new ClientEntityWorkbookPopulator.DatatableDateInfo(currentCol, column.isDateTimeDisplayType())); } // Check if this is a boolean column else if (column.isBooleanDisplayType()) { datatableBooleanColumns.put(currentCol, new ClientEntityWorkbookPopulator.DatatableBooleanInfo(currentCol)); } - + currentCol++; } catch (Exception e) { String datatableNameForLog = datatableName != null ? datatableName : "unknown"; String columnNameForLog = column != null && column.getColumnName() != null ? column.getColumnName() : "unknown"; - log.warn("Failed to process datatable column '{}' in datatable '{}': {}", - columnNameForLog, datatableNameForLog, e.getMessage()); + log.warn("Failed to process datatable column '{}' in datatable '{}': {}", columnNameForLog, datatableNameForLog, + e.getMessage()); // Continue with next column } } @@ -549,20 +556,23 @@ else if (column.isBooleanDisplayType()) { } return currentCol; } catch (Exception e) { - log.warn("Failed to handle datatable column headers in client bulk import template: {}. Template generation will continue without datatable columns.", + log.warn( + "Failed to handle datatable column headers in client bulk import template: {}. Template generation will continue without datatable columns.", e.getMessage()); return startCol; // Return start position if datatables failed } } /** - * Populates datatable dropdown values into the hidden lookup sheet and adds data validation on the - * client sheet's dropdown columns. Named ranges for these columns are already created in - * {@link #setNames(Sheet, List)}; this only fills in the values they refer to and wires up validation. - * All operations are wrapped in try/catch to ensure datatable failures never break template generation. + * Populates datatable dropdown values into the hidden lookup sheet and adds data validation on the client sheet's + * dropdown columns. Named ranges for these columns are already created in {@link #setNames(Sheet, List)}; this only + * fills in the values they refer to and wires up validation. All operations are wrapped in try/catch to ensure + * datatable failures never break template generation. * - * @param worksheet The client sheet - * @param lookupSheet The hidden lookup sheet + * @param worksheet + * The client sheet + * @param lookupSheet + * The hidden lookup sheet */ private void handleDatatableColumnsComplete(Sheet worksheet, Sheet lookupSheet) { try { @@ -581,8 +591,7 @@ private void handleDatatableColumnsComplete(Sheet worksheet, Sheet lookupSheet) } } catch (Exception e) { String namedRangeName = entry.getKey(); - log.warn("Failed to populate dropdown values for datatable named range '{}': {}", - namedRangeName, e.getMessage()); + log.warn("Failed to populate dropdown values for datatable named range '{}': {}", namedRangeName, e.getMessage()); // Continue with next dropdown } } @@ -596,17 +605,18 @@ private void handleDatatableColumnsComplete(Sheet worksheet, Sheet lookupSheet) CellRangeAddressList datatableDropdownRange = new CellRangeAddressList(1, SpreadsheetVersion.EXCEL97.getLastRowIndex(), dropdownInfo.clientSheetColumnIndex, dropdownInfo.clientSheetColumnIndex); DataValidationConstraint datatableDropdownConstraint = validationHelper.createFormulaListConstraint(namedRangeName); - DataValidation datatableDropdownValidation = validationHelper.createValidation(datatableDropdownConstraint, datatableDropdownRange); + DataValidation datatableDropdownValidation = validationHelper.createValidation(datatableDropdownConstraint, + datatableDropdownRange); worksheet.addValidationData(datatableDropdownValidation); } catch (Exception e) { String namedRangeName = entry.getKey(); - log.warn("Failed to add data validation for datatable dropdown '{}': {}", - namedRangeName, e.getMessage()); + log.warn("Failed to add data validation for datatable dropdown '{}': {}", namedRangeName, e.getMessage()); // Continue with next validation } } } catch (Exception e) { - log.warn("Failed to complete datatable setup in client bulk import template: {}. Template generation will continue without datatable enhancements.", + log.warn( + "Failed to complete datatable setup in client bulk import template: {}. Template generation will continue without datatable enhancements.", e.getMessage()); // Do not throw - allow template generation to continue } @@ -709,7 +719,6 @@ private void setRules(Sheet worksheet, String dateFormat) { worksheet.addValidationData(incorporateDateValidation); worksheet.addValidationData(incorporateDateTillValidation); - // Add date validation for datatable date columns for (ClientEntityWorkbookPopulator.DatatableDateInfo dateInfo : datatableDateColumns.values()) { try { @@ -721,8 +730,8 @@ private void setRules(Sheet worksheet, String dateFormat) { DataValidation datatableDateValidation = validationHelper.createValidation(datatableDateConstraint, datatableDateRange); worksheet.addValidationData(datatableDateValidation); } catch (Exception e) { - log.warn("Failed to add date validation for datatable date column at index {}: {}", - dateInfo.clientSheetColumnIndex, e.getMessage()); + log.warn("Failed to add date validation for datatable date column at index {}: {}", dateInfo.clientSheetColumnIndex, + e.getMessage()); } } @@ -732,8 +741,10 @@ private void setRules(Sheet worksheet, String dateFormat) { CellRangeAddressList datatableBooleanRange = new CellRangeAddressList(1, SpreadsheetVersion.EXCEL97.getLastRowIndex(), booleanInfo.clientSheetColumnIndex, booleanInfo.clientSheetColumnIndex); // Use same boolean constraint as active field (True/False) - DataValidationConstraint datatableBooleanConstraint = validationHelper.createExplicitListConstraint(new String[] { "True", "False" }); - DataValidation datatableBooleanValidation = validationHelper.createValidation(datatableBooleanConstraint, datatableBooleanRange); + DataValidationConstraint datatableBooleanConstraint = validationHelper + .createExplicitListConstraint(new String[] { "True", "False" }); + DataValidation datatableBooleanValidation = validationHelper.createValidation(datatableBooleanConstraint, + datatableBooleanRange); worksheet.addValidationData(datatableBooleanValidation); } catch (Exception e) { log.warn("Failed to add boolean validation for datatable boolean column at index {}: {}", @@ -745,7 +756,7 @@ private void setRules(Sheet worksheet, String dateFormat) { private void setNames(Sheet worksheet, List offices) { Workbook clientWorkbook = worksheet.getWorkbook(); String lookupSheetName = TemplatePopulateImportConstants.CLIENT_LOOKUPS_SHEET_NAME; - + Name officeGroup = clientWorkbook.createName(); officeGroup.setNameName("Office"); officeGroup.setRefersToFormula(TemplatePopulateImportConstants.OFFICE_SHEET_NAME + "!$B$2:$B$" + (offices.size() + 1)); @@ -756,16 +767,16 @@ private void setNames(Sheet worksheet, List offices) { clientTypeGroup.setNameName("ClientTypes"); int clientTypesLastRow = clientTypeCodeValues.size() + 1; // +1 because data starts at row 2 String clientTypesCol = CellReference.convertNumToColString(0); - clientTypeGroup.setRefersToFormula( - "'" + lookupSheetName + "'!$" + clientTypesCol + "$2:$" + clientTypesCol + "$" + clientTypesLastRow); + clientTypeGroup + .setRefersToFormula("'" + lookupSheetName + "'!$" + clientTypesCol + "$2:$" + clientTypesCol + "$" + clientTypesLastRow); // Column 2 (C): Constitution Name constitutionGroup = clientWorkbook.createName(); constitutionGroup.setNameName("Constitution"); int constitutionLastRow = constitutionCodeValues.size() + 1; String constitutionCol = CellReference.convertNumToColString(2); - constitutionGroup.setRefersToFormula( - "'" + lookupSheetName + "'!$" + constitutionCol + "$2:$" + constitutionCol + "$" + constitutionLastRow); + constitutionGroup + .setRefersToFormula("'" + lookupSheetName + "'!$" + constitutionCol + "$2:$" + constitutionCol + "$" + constitutionLastRow); // Column 3 (D): Main Business Line Name mainBusinessLineGroup = clientWorkbook.createName(); @@ -780,16 +791,16 @@ private void setNames(Sheet worksheet, List offices) { clientClassficationGroup.setNameName("ClientClassification"); int clientClassificationLastRow = clientClassificationCodeValues.size() + 1; String clientClassificationCol = CellReference.convertNumToColString(1); - clientClassficationGroup.setRefersToFormula( - "'" + lookupSheetName + "'!$" + clientClassificationCol + "$2:$" + clientClassificationCol + "$" + clientClassificationLastRow); + clientClassficationGroup.setRefersToFormula("'" + lookupSheetName + "'!$" + clientClassificationCol + "$2:$" + + clientClassificationCol + "$" + clientClassificationLastRow); // Column 4 (E): Address Type Name addressTypeGroup = clientWorkbook.createName(); addressTypeGroup.setNameName("AddressType"); int addressTypeLastRow = addressTypesCodeValues.size() + 1; String addressTypeCol = CellReference.convertNumToColString(4); - addressTypeGroup.setRefersToFormula( - "'" + lookupSheetName + "'!$" + addressTypeCol + "$2:$" + addressTypeCol + "$" + addressTypeLastRow); + addressTypeGroup + .setRefersToFormula("'" + lookupSheetName + "'!$" + addressTypeCol + "$2:$" + addressTypeCol + "$" + addressTypeLastRow); // Column 5 (F): State/Province Name stateProvinceGroup = clientWorkbook.createName(); @@ -804,8 +815,7 @@ private void setNames(Sheet worksheet, List offices) { countryGroup.setNameName("Country"); int countryLastRow = countryCodeValues.size() + 1; String countryCol = CellReference.convertNumToColString(6); - countryGroup.setRefersToFormula( - "'" + lookupSheetName + "'!$" + countryCol + "$2:$" + countryCol + "$" + countryLastRow); + countryGroup.setRefersToFormula("'" + lookupSheetName + "'!$" + countryCol + "$2:$" + countryCol + "$" + countryLastRow); // Create named ranges for datatable dropdown columns for (Map.Entry entry : datatableDropdowns.entrySet()) { @@ -815,8 +825,8 @@ private void setNames(Sheet worksheet, List offices) { setSanitized(datatableDropdownGroup, namedRangeName); int dropdownLastRow = dropdownInfo.values.size() + 1; String dropdownCol = CellReference.convertNumToColString(dropdownInfo.lookupSheetColumnIndex); - datatableDropdownGroup.setRefersToFormula( - "'" + lookupSheetName + "'!$" + dropdownCol + "$2:$" + dropdownCol + "$" + dropdownLastRow); + datatableDropdownGroup + .setRefersToFormula("'" + lookupSheetName + "'!$" + dropdownCol + "$2:$" + dropdownCol + "$" + dropdownLastRow); } for (Integer i = 0; i < offices.size(); i++) { diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/client/ClientPersonWorkbookPopulator.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/client/ClientPersonWorkbookPopulator.java index 02bc0004a3d..2ea0bae1cae 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/client/ClientPersonWorkbookPopulator.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/populator/client/ClientPersonWorkbookPopulator.java @@ -18,7 +18,10 @@ */ package org.apache.fineract.infrastructure.bulkimport.populator.client; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.apache.fineract.infrastructure.bulkimport.constants.ClientPersonConstants; import org.apache.fineract.infrastructure.bulkimport.constants.TemplatePopulateImportConstants; import org.apache.fineract.infrastructure.bulkimport.populator.AbstractWorkbookPopulator; @@ -27,8 +30,8 @@ import org.apache.fineract.infrastructure.codes.data.CodeValueData; import org.apache.fineract.infrastructure.dataqueries.data.DatatableData; import org.apache.fineract.infrastructure.dataqueries.data.ResultsetColumnHeaderData; +import org.apache.fineract.infrastructure.dataqueries.data.ResultsetColumnValueData; import org.apache.fineract.organisation.office.data.OfficeData; -import java.util.ArrayList; import org.apache.poi.hssf.usermodel.HSSFDataValidationHelper; import org.apache.poi.hssf.usermodel.HSSFSheet; import org.apache.poi.ss.SpreadsheetVersion; @@ -45,9 +48,6 @@ import org.apache.poi.ss.usermodel.Workbook; import org.apache.poi.ss.util.CellRangeAddressList; import org.apache.poi.ss.util.CellReference; -import org.apache.fineract.infrastructure.dataqueries.data.ResultsetColumnValueData; -import java.util.Map; -import java.util.HashMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,44 +64,47 @@ public class ClientPersonWorkbookPopulator extends AbstractWorkbookPopulator { private final List stateProvinceCodeValues; private final List countryCodeValues; private final List requiredDatatables; - + // Track datatable dropdown columns: namedRangeName -> (clientSheetColumnIndex, lookupSheetColumnIndex, values) private final Map datatableDropdowns = new HashMap<>(); - + // Track datatable date columns: columnIndex -> columnInfo private final Map datatableDateColumns = new HashMap<>(); - + // Track datatable boolean columns: columnIndex -> columnInfo private final Map datatableBooleanColumns = new HashMap<>(); - + // Inner class to track datatable dropdown information private static class DatatableDropdownInfo { + final int clientSheetColumnIndex; final int lookupSheetColumnIndex; final List values; - + DatatableDropdownInfo(int clientSheetColumnIndex, int lookupSheetColumnIndex, List values) { this.clientSheetColumnIndex = clientSheetColumnIndex; this.lookupSheetColumnIndex = lookupSheetColumnIndex; this.values = values; } } - + // Inner class to track datatable date column information private static class DatatableDateInfo { + final int clientSheetColumnIndex; final boolean isDateTime; - + DatatableDateInfo(int clientSheetColumnIndex, boolean isDateTime) { this.clientSheetColumnIndex = clientSheetColumnIndex; this.isDateTime = isDateTime; } } - + // Inner class to track datatable boolean column information private static class DatatableBooleanInfo { + final int clientSheetColumnIndex; - + DatatableBooleanInfo(int clientSheetColumnIndex) { this.clientSheetColumnIndex = clientSheetColumnIndex; } @@ -109,8 +112,8 @@ private static class DatatableBooleanInfo { public ClientPersonWorkbookPopulator(OfficeSheetPopulator officeSheetPopulator, PersonnelSheetPopulator personnelSheetPopulator, List clientTypeCodeValues, List genderCodeValues, List clientClassification, - List addressTypesCodeValues, List stateProvinceCodeValues, - List countryCodeValues, List requiredDatatables) { + List addressTypesCodeValues, List stateProvinceCodeValues, List countryCodeValues, + List requiredDatatables) { this.officeSheetPopulator = officeSheetPopulator; this.personnelSheetPopulator = personnelSheetPopulator; this.clientTypeCodeValues = clientTypeCodeValues; @@ -136,7 +139,7 @@ public void populate(Workbook workbook, String dateFormat) { setClientDataLookupTable(lookupSheet); setFormatStyle(workbook, clientSheet); setRules(clientSheet, dateFormat); - + // Complete datatable handling (lookup population, named ranges, validation) // This is called after setLayout to complete datatable setup Row rowHeader = clientSheet.getRow(TemplatePopulateImportConstants.ROWHEADER_INDEX); @@ -158,7 +161,7 @@ private void setFormatStyle(Workbook workbook, Sheet worksheet) { setFormatActivationAndSubmittedDate(row, ClientPersonConstants.ACTIVATION_DATE_COL, dateCellStyle); setFormatActivationAndSubmittedDate(row, ClientPersonConstants.SUBMITTED_ON_COL, dateCellStyle); - + // Apply date formatting to datatable date columns for (DatatableDateInfo dateInfo : datatableDateColumns.values()) { setFormatActivationAndSubmittedDate(row, dateInfo.clientSheetColumnIndex, dateCellStyle); @@ -186,7 +189,7 @@ private void setClientDataLookupTable(Sheet lookupSheet) { writeString(0, row, clientTypeCodeValue.getName() + " (" + clientTypeCodeValue.getId() + ")"); rowIndex++; } - + // Column 1 (B): Client Classification rowIndex = 1; for (CodeValueData clientClassificationCodeValue : clientClassificationCodeValues) { @@ -197,7 +200,7 @@ private void setClientDataLookupTable(Sheet lookupSheet) { writeString(1, row, clientClassificationCodeValue.getName() + " (" + clientClassificationCodeValue.getId() + ")"); rowIndex++; } - + // Column 2 (C): Gender rowIndex = 1; for (CodeValueData genderCodeValue : genderCodeValues) { @@ -208,7 +211,7 @@ private void setClientDataLookupTable(Sheet lookupSheet) { writeString(2, row, genderCodeValue.getName() + " (" + genderCodeValue.getId() + ")"); rowIndex++; } - + // Column 3 (D): Address Type rowIndex = 1; for (CodeValueData addressTypeCodeValue : addressTypesCodeValues) { @@ -219,7 +222,7 @@ private void setClientDataLookupTable(Sheet lookupSheet) { writeString(3, row, addressTypeCodeValue.getName() + " (" + addressTypeCodeValue.getId() + ")"); rowIndex++; } - + // Column 4 (E): State/Province rowIndex = 1; for (CodeValueData stateCodeValue : stateProvinceCodeValues) { @@ -230,7 +233,7 @@ private void setClientDataLookupTable(Sheet lookupSheet) { writeString(4, row, stateCodeValue.getName() + " (" + stateCodeValue.getId() + ")"); rowIndex++; } - + // Column 5 (F): Country rowIndex = 1; for (CodeValueData countryCodeValue : countryCodeValues) { @@ -241,9 +244,9 @@ private void setClientDataLookupTable(Sheet lookupSheet) { writeString(5, row, countryCodeValue.getName() + " (" + countryCodeValue.getId() + ")"); rowIndex++; } - + } - + private String sanitizeNamedRangeName(String name) { // Excel named ranges cannot contain certain characters // Replace invalid characters with underscore @@ -326,46 +329,45 @@ private void setLayout(Sheet worksheet) { worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.MEDIUM_COL_SIZE); writeString(currentCol, rowHeader, "Lookup office Name "); int lookupOfficeNameCol = currentCol++; - + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.SMALL_COL_SIZE); writeString(currentCol, rowHeader, "Lookup Office Opened Date "); int lookupOfficeDateCol = currentCol++; - + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.SMALL_COL_SIZE); writeString(currentCol, rowHeader, "Lookup Gender "); int lookupGenderCol = currentCol++; - + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.SMALL_COL_SIZE); writeString(currentCol, rowHeader, "Lookup Client Types "); int lookupClientTypesCol = currentCol++; - + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.SMALL_COL_SIZE); writeString(currentCol, rowHeader, "Lookup Client Classification "); int lookupClientClassificationCol = currentCol++; - + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.SMALL_COL_SIZE); writeString(currentCol, rowHeader, "Lookup AddressType "); int lookupAddressTypeCol = currentCol++; - + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.SMALL_COL_SIZE); writeString(currentCol, rowHeader, "Lookup State/Province "); int lookupStateProvinceCol = currentCol++; - + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.SMALL_COL_SIZE); writeString(currentCol, rowHeader, "Lookup Country "); int lookupCountryCol = currentCol++; // Populate visible lookup values under lookup headers (for user reference) // These are read-only reference values; validations still use the hidden lookup sheet - populateLookupValues(worksheet, lookupOfficeNameCol, lookupOfficeDateCol, lookupGenderCol, - lookupClientTypesCol, lookupClientClassificationCol, lookupAddressTypeCol, - lookupStateProvinceCol, lookupCountryCol); + populateLookupValues(worksheet, lookupOfficeNameCol, lookupOfficeDateCol, lookupGenderCol, lookupClientTypesCol, + lookupClientClassificationCol, lookupAddressTypeCol, lookupStateProvinceCol, lookupCountryCol); } - private void populateLookupValues(Sheet worksheet, int lookupOfficeNameCol, int lookupOfficeDateCol, - int lookupGenderCol, int lookupClientTypesCol, int lookupClientClassificationCol, - int lookupAddressTypeCol, int lookupStateProvinceCol, int lookupCountryCol) { - + private void populateLookupValues(Sheet worksheet, int lookupOfficeNameCol, int lookupOfficeDateCol, int lookupGenderCol, + int lookupClientTypesCol, int lookupClientClassificationCol, int lookupAddressTypeCol, int lookupStateProvinceCol, + int lookupCountryCol) { + // Populate Office Name lookup values int rowIndex = 1; // Start at Excel row 2 (POI index 1) List offices = officeSheetPopulator.getOffices(); @@ -380,7 +382,7 @@ private void populateLookupValues(Sheet worksheet, int lookupOfficeNameCol, int } rowIndex++; } - + // Populate Gender lookup values rowIndex = 1; for (CodeValueData genderCodeValue : genderCodeValues) { @@ -391,7 +393,7 @@ private void populateLookupValues(Sheet worksheet, int lookupOfficeNameCol, int writeString(lookupGenderCol, row, genderCodeValue.getName() + " (" + genderCodeValue.getId() + ")"); rowIndex++; } - + // Populate Client Types lookup values rowIndex = 1; for (CodeValueData clientTypeCodeValue : clientTypeCodeValues) { @@ -402,7 +404,7 @@ private void populateLookupValues(Sheet worksheet, int lookupOfficeNameCol, int writeString(lookupClientTypesCol, row, clientTypeCodeValue.getName() + " (" + clientTypeCodeValue.getId() + ")"); rowIndex++; } - + // Populate Client Classification lookup values rowIndex = 1; for (CodeValueData clientClassificationCodeValue : clientClassificationCodeValues) { @@ -410,10 +412,11 @@ private void populateLookupValues(Sheet worksheet, int lookupOfficeNameCol, int if (row == null) { row = worksheet.createRow(rowIndex); } - writeString(lookupClientClassificationCol, row, clientClassificationCodeValue.getName() + " (" + clientClassificationCodeValue.getId() + ")"); + writeString(lookupClientClassificationCol, row, + clientClassificationCodeValue.getName() + " (" + clientClassificationCodeValue.getId() + ")"); rowIndex++; } - + // Populate Address Type lookup values rowIndex = 1; for (CodeValueData addressTypeCodeValue : addressTypesCodeValues) { @@ -424,7 +427,7 @@ private void populateLookupValues(Sheet worksheet, int lookupOfficeNameCol, int writeString(lookupAddressTypeCol, row, addressTypeCodeValue.getName() + " (" + addressTypeCodeValue.getId() + ")"); rowIndex++; } - + // Populate State/Province lookup values rowIndex = 1; for (CodeValueData stateCodeValue : stateProvinceCodeValues) { @@ -435,7 +438,7 @@ private void populateLookupValues(Sheet worksheet, int lookupOfficeNameCol, int writeString(lookupStateProvinceCol, row, stateCodeValue.getName() + " (" + stateCodeValue.getId() + ")"); rowIndex++; } - + // Populate Country lookup values rowIndex = 1; for (CodeValueData countryCodeValue : countryCodeValues) { @@ -449,11 +452,13 @@ private void populateLookupValues(Sheet worksheet, int lookupOfficeNameCol, int } /** - * Handles datatable column header creation and dropdown tracking. - * This is called from within setLayout() to insert headers at the correct position. - * - * @param worksheet The client sheet - * @param rowHeader The header row + * Handles datatable column header creation and dropdown tracking. This is called from within setLayout() to insert + * headers at the correct position. + * + * @param worksheet + * The client sheet + * @param rowHeader + * The header row * @return The next column index after datatable columns (or STATUS_COL + 1 if no datatables) */ private int handleDatatableColumnHeaders(Sheet worksheet, Row rowHeader) { @@ -461,7 +466,7 @@ private int handleDatatableColumnHeaders(Sheet worksheet, Row rowHeader) { try { int currentCol = startCol; int lookupSheetCol = 6; // Start after Country (column F = index 5), so next available is G = index 6 - + for (DatatableData datatable : requiredDatatables) { if (datatable == null) { continue; @@ -473,17 +478,17 @@ private int handleDatatableColumnHeaders(Sheet worksheet, Row rowHeader) { try { // Skip system columns (id, client_id, etc.) String columnName = column.getColumnName(); - if (columnName == null || columnName.equalsIgnoreCase("id") - || columnName.equalsIgnoreCase("client_id") - || columnName.equalsIgnoreCase("created_at") - || columnName.equalsIgnoreCase("updated_at")) { + if (columnName == null || columnName.equalsIgnoreCase("id") || columnName.equalsIgnoreCase("client_id") + || columnName.equalsIgnoreCase("created_at") || columnName.equalsIgnoreCase("updated_at")) { continue; } - + worksheet.setColumnWidth(currentCol, TemplatePopulateImportConstants.MEDIUM_COL_SIZE); // Use dot notation: registeredTableName.columnName for unambiguous parsing - // For single-row datatables: treat as mandatory inline data (append "*" if column is not nullable) - // For multi-row datatables: treat as optional/repeatable child data (no "*" even if not nullable) + // For single-row datatables: treat as mandatory inline data (append "*" if column is not + // nullable) + // For multi-row datatables: treat as optional/repeatable child data (no "*" even if not + // nullable) String headerLabel = datatableName + "." + columnName; if (!datatable.isMultiRow() && !column.getIsColumnNullable()) { // Single-row datatables: mark mandatory columns with "*" @@ -491,13 +496,14 @@ private int handleDatatableColumnHeaders(Sheet worksheet, Row rowHeader) { } // Multi-row datatables are always optional (no "*" marker) writeString(currentCol, rowHeader, headerLabel); - + // Check if this is a dropdown column (CODELOOKUP) with values if (column.isCodeLookupDisplayType() && column.hasColumnValues()) { String namedRangeName = sanitizeNamedRangeName(datatableName + "_" + columnName); List columnValues = column.getColumnValues(); if (columnValues != null && !columnValues.isEmpty()) { - datatableDropdowns.put(namedRangeName, new DatatableDropdownInfo(currentCol, lookupSheetCol, columnValues)); + datatableDropdowns.put(namedRangeName, + new DatatableDropdownInfo(currentCol, lookupSheetCol, columnValues)); lookupSheetCol++; } } @@ -509,13 +515,13 @@ else if (column.isDateDisplayType() || column.isDateTimeDisplayType()) { else if (column.isBooleanDisplayType()) { datatableBooleanColumns.put(currentCol, new DatatableBooleanInfo(currentCol)); } - + currentCol++; } catch (Exception e) { String datatableNameForLog = datatableName != null ? datatableName : "unknown"; String columnNameForLog = column != null && column.getColumnName() != null ? column.getColumnName() : "unknown"; - log.warn("Failed to process datatable column '{}' in datatable '{}': {}", - columnNameForLog, datatableNameForLog, e.getMessage()); + log.warn("Failed to process datatable column '{}' in datatable '{}': {}", columnNameForLog, datatableNameForLog, + e.getMessage()); // Continue with next column } } @@ -523,22 +529,28 @@ else if (column.isBooleanDisplayType()) { } return currentCol; } catch (Exception e) { - log.warn("Failed to handle datatable column headers in client bulk import template: {}. Template generation will continue without datatable columns.", + log.warn( + "Failed to handle datatable column headers in client bulk import template: {}. Template generation will continue without datatable columns.", e.getMessage()); return startCol; // Return start position if datatables failed } } /** - * Completes datatable setup: lookup sheet population, named ranges, and validation. - * This is called after setLayout() to complete datatable enhancements. - * All operations are wrapped in try/catch to ensure datatable failures never break template generation. - * - * @param worksheet The client sheet - * @param rowHeader The header row - * @param lookupSheet The hidden lookup sheet - * @param workbook The workbook - * @param dateFormat The date format + * Completes datatable setup: lookup sheet population, named ranges, and validation. This is called after + * setLayout() to complete datatable enhancements. All operations are wrapped in try/catch to ensure datatable + * failures never break template generation. + * + * @param worksheet + * The client sheet + * @param rowHeader + * The header row + * @param lookupSheet + * The hidden lookup sheet + * @param workbook + * The workbook + * @param dateFormat + * The date format */ private void handleDatatableColumnsComplete(Sheet worksheet, Row rowHeader, Sheet lookupSheet, Workbook workbook, String dateFormat) { try { @@ -557,12 +569,11 @@ private void handleDatatableColumnsComplete(Sheet worksheet, Row rowHeader, Shee } } catch (Exception e) { String namedRangeName = entry.getKey(); - log.warn("Failed to populate dropdown values for datatable named range '{}': {}", - namedRangeName, e.getMessage()); + log.warn("Failed to populate dropdown values for datatable named range '{}': {}", namedRangeName, e.getMessage()); // Continue with next dropdown } } - + // Step 3: Create named ranges for datatable dropdown columns String lookupSheetName = TemplatePopulateImportConstants.CLIENT_LOOKUPS_SHEET_NAME; Workbook clientWorkbook = worksheet.getWorkbook(); @@ -574,16 +585,15 @@ private void handleDatatableColumnsComplete(Sheet worksheet, Row rowHeader, Shee setSanitized(datatableDropdownGroup, namedRangeName); int dropdownLastRow = dropdownInfo.values.size() + 1; // +1 because data starts at row 2 String dropdownCol = CellReference.convertNumToColString(dropdownInfo.lookupSheetColumnIndex); - datatableDropdownGroup.setRefersToFormula( - "'" + lookupSheetName + "'!$" + dropdownCol + "$2:$" + dropdownCol + "$" + dropdownLastRow); + datatableDropdownGroup + .setRefersToFormula("'" + lookupSheetName + "'!$" + dropdownCol + "$2:$" + dropdownCol + "$" + dropdownLastRow); } catch (Exception e) { String namedRangeName = entry.getKey(); - log.warn("Failed to create named range '{}' for datatable dropdown: {}", - namedRangeName, e.getMessage()); + log.warn("Failed to create named range '{}' for datatable dropdown: {}", namedRangeName, e.getMessage()); // Continue with next named range } } - + // Step 4: Add data validation for datatable dropdown columns DataValidationHelper validationHelper = new HSSFDataValidationHelper((HSSFSheet) worksheet); for (Map.Entry entry : datatableDropdowns.entrySet()) { @@ -593,17 +603,18 @@ private void handleDatatableColumnsComplete(Sheet worksheet, Row rowHeader, Shee CellRangeAddressList datatableDropdownRange = new CellRangeAddressList(1, SpreadsheetVersion.EXCEL97.getLastRowIndex(), dropdownInfo.clientSheetColumnIndex, dropdownInfo.clientSheetColumnIndex); DataValidationConstraint datatableDropdownConstraint = validationHelper.createFormulaListConstraint(namedRangeName); - DataValidation datatableDropdownValidation = validationHelper.createValidation(datatableDropdownConstraint, datatableDropdownRange); + DataValidation datatableDropdownValidation = validationHelper.createValidation(datatableDropdownConstraint, + datatableDropdownRange); worksheet.addValidationData(datatableDropdownValidation); } catch (Exception e) { String namedRangeName = entry.getKey(); - log.warn("Failed to add data validation for datatable dropdown '{}': {}", - namedRangeName, e.getMessage()); + log.warn("Failed to add data validation for datatable dropdown '{}': {}", namedRangeName, e.getMessage()); // Continue with next validation } } } catch (Exception e) { - log.warn("Failed to complete datatable setup in client bulk import template: {}. Template generation will continue without datatable enhancements.", + log.warn( + "Failed to complete datatable setup in client bulk import template: {}. Template generation will continue without datatable enhancements.", e.getMessage()); // Do not throw - allow template generation to continue } @@ -698,7 +709,7 @@ private void setRules(Sheet worksheet, String dateformat) { worksheet.addValidationData(stateProvinceValidation); worksheet.addValidationData(countryValidation); worksheet.addValidationData(activeAddressValidation); - + // Add date validation for datatable date columns for (DatatableDateInfo dateInfo : datatableDateColumns.values()) { try { @@ -710,22 +721,24 @@ private void setRules(Sheet worksheet, String dateformat) { DataValidation datatableDateValidation = validationHelper.createValidation(datatableDateConstraint, datatableDateRange); worksheet.addValidationData(datatableDateValidation); } catch (Exception e) { - log.warn("Failed to add date validation for datatable date column at index {}: {}", - dateInfo.clientSheetColumnIndex, e.getMessage()); + log.warn("Failed to add date validation for datatable date column at index {}: {}", dateInfo.clientSheetColumnIndex, + e.getMessage()); } } - + // Add boolean validation for datatable boolean columns for (DatatableBooleanInfo booleanInfo : datatableBooleanColumns.values()) { try { CellRangeAddressList datatableBooleanRange = new CellRangeAddressList(1, SpreadsheetVersion.EXCEL97.getLastRowIndex(), booleanInfo.clientSheetColumnIndex, booleanInfo.clientSheetColumnIndex); // Use same boolean constraint as active field (True/False) - DataValidationConstraint datatableBooleanConstraint = validationHelper.createExplicitListConstraint(new String[] { "True", "False" }); - DataValidation datatableBooleanValidation = validationHelper.createValidation(datatableBooleanConstraint, datatableBooleanRange); + DataValidationConstraint datatableBooleanConstraint = validationHelper + .createExplicitListConstraint(new String[] { "True", "False" }); + DataValidation datatableBooleanValidation = validationHelper.createValidation(datatableBooleanConstraint, + datatableBooleanRange); worksheet.addValidationData(datatableBooleanValidation); } catch (Exception e) { - log.warn("Failed to add boolean validation for datatable boolean column at index {}: {}", + log.warn("Failed to add boolean validation for datatable boolean column at index {}: {}", booleanInfo.clientSheetColumnIndex, e.getMessage()); } } @@ -734,7 +747,7 @@ private void setRules(Sheet worksheet, String dateformat) { private void setNames(Sheet worksheet, List offices) { Workbook clientWorkbook = worksheet.getWorkbook(); String lookupSheetName = TemplatePopulateImportConstants.CLIENT_LOOKUPS_SHEET_NAME; - + Name officeGroup = clientWorkbook.createName(); officeGroup.setNameName("Office"); officeGroup.setRefersToFormula(TemplatePopulateImportConstants.OFFICE_SHEET_NAME + "!$B$2:$B$" + (offices.size() + 1)); @@ -745,32 +758,31 @@ private void setNames(Sheet worksheet, List offices) { clientTypeGroup.setNameName("ClientTypes"); int clientTypesLastRow = clientTypeCodeValues.size() + 1; // +1 because data starts at row 2 String clientTypesCol = CellReference.convertNumToColString(0); - clientTypeGroup.setRefersToFormula( - "'" + lookupSheetName + "'!$" + clientTypesCol + "$2:$" + clientTypesCol + "$" + clientTypesLastRow); + clientTypeGroup + .setRefersToFormula("'" + lookupSheetName + "'!$" + clientTypesCol + "$2:$" + clientTypesCol + "$" + clientTypesLastRow); // Column 2 (C): Gender Name genderGroup = clientWorkbook.createName(); genderGroup.setNameName("Gender"); int genderLastRow = genderCodeValues.size() + 1; String genderCol = CellReference.convertNumToColString(2); - genderGroup.setRefersToFormula( - "'" + lookupSheetName + "'!$" + genderCol + "$2:$" + genderCol + "$" + genderLastRow); + genderGroup.setRefersToFormula("'" + lookupSheetName + "'!$" + genderCol + "$2:$" + genderCol + "$" + genderLastRow); // Column 1 (B): Client Classification Name clientClassficationGroup = clientWorkbook.createName(); clientClassficationGroup.setNameName("ClientClassification"); int clientClassificationLastRow = clientClassificationCodeValues.size() + 1; String clientClassificationCol = CellReference.convertNumToColString(1); - clientClassficationGroup.setRefersToFormula( - "'" + lookupSheetName + "'!$" + clientClassificationCol + "$2:$" + clientClassificationCol + "$" + clientClassificationLastRow); + clientClassficationGroup.setRefersToFormula("'" + lookupSheetName + "'!$" + clientClassificationCol + "$2:$" + + clientClassificationCol + "$" + clientClassificationLastRow); // Column 3 (D): Address Type Name addressTypeGroup = clientWorkbook.createName(); addressTypeGroup.setNameName("AddressType"); int addressTypeLastRow = addressTypesCodeValues.size() + 1; String addressTypeCol = CellReference.convertNumToColString(3); - addressTypeGroup.setRefersToFormula( - "'" + lookupSheetName + "'!$" + addressTypeCol + "$2:$" + addressTypeCol + "$" + addressTypeLastRow); + addressTypeGroup + .setRefersToFormula("'" + lookupSheetName + "'!$" + addressTypeCol + "$2:$" + addressTypeCol + "$" + addressTypeLastRow); // Column 4 (E): State/Province Name stateProvinceGroup = clientWorkbook.createName(); @@ -785,8 +797,7 @@ private void setNames(Sheet worksheet, List offices) { countryGroup.setNameName("Country"); int countryLastRow = countryCodeValues.size() + 1; String countryCol = CellReference.convertNumToColString(5); - countryGroup.setRefersToFormula( - "'" + lookupSheetName + "'!$" + countryCol + "$2:$" + countryCol + "$" + countryLastRow); + countryGroup.setRefersToFormula("'" + lookupSheetName + "'!$" + countryCol + "$2:$" + countryCol + "$" + countryLastRow); for (Integer i = 0; i < offices.size(); i++) { Integer[] officeNameToBeginEndIndexesOfStaff = personnelSheetPopulator.getOfficeNameToBeginEndIndexesOfStaff().get(i); diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/service/BulkImportWorkbookPopulatorServiceImpl.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/service/BulkImportWorkbookPopulatorServiceImpl.java index d3ddaef73c5..2a7a4d3806a 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/service/BulkImportWorkbookPopulatorServiceImpl.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/service/BulkImportWorkbookPopulatorServiceImpl.java @@ -70,6 +70,13 @@ import org.apache.fineract.infrastructure.core.service.DateUtils; import org.apache.fineract.infrastructure.core.service.Page; import org.apache.fineract.infrastructure.core.service.SearchParameters; +import org.apache.fineract.infrastructure.dataqueries.data.DatatableData; +import org.apache.fineract.infrastructure.dataqueries.data.EntityTables; +import org.apache.fineract.infrastructure.dataqueries.data.StatusEnum; +import org.apache.fineract.infrastructure.dataqueries.domain.EntityDatatableChecks; +import org.apache.fineract.infrastructure.dataqueries.domain.EntityDatatableChecksRepository; +import org.apache.fineract.infrastructure.dataqueries.service.DatatableReadService; +import org.apache.fineract.infrastructure.dataqueries.service.EntityDatatableChecksReadService; import org.apache.fineract.infrastructure.security.service.PlatformSecurityContext; import org.apache.fineract.organisation.monetary.data.CurrencyData; import org.apache.fineract.organisation.monetary.service.CurrencyReadPlatformService; @@ -104,13 +111,6 @@ import org.apache.fineract.portfolio.savings.service.DepositProductReadPlatformService; import org.apache.fineract.portfolio.savings.service.SavingsAccountReadPlatformService; import org.apache.fineract.portfolio.savings.service.SavingsProductReadPlatformService; -import org.apache.fineract.infrastructure.dataqueries.data.DatatableData; -import org.apache.fineract.infrastructure.dataqueries.data.EntityTables; -import org.apache.fineract.infrastructure.dataqueries.data.StatusEnum; -import org.apache.fineract.infrastructure.dataqueries.domain.EntityDatatableChecks; -import org.apache.fineract.infrastructure.dataqueries.domain.EntityDatatableChecksRepository; -import org.apache.fineract.infrastructure.dataqueries.service.DatatableReadService; -import org.apache.fineract.infrastructure.dataqueries.service.EntityDatatableChecksReadService; import org.apache.fineract.portfolio.shareproducts.data.ShareProductData; import org.apache.fineract.useradministration.data.RoleData; import org.apache.fineract.useradministration.service.RoleReadPlatformService; @@ -163,10 +163,8 @@ public BulkImportWorkbookPopulatorServiceImpl(final PlatformSecurityContext cont final ShareProductReadPlatformService shareProductReadPlatformService, final ChargeReadPlatformService chargeReadPlatformService, final DepositProductReadPlatformService depositProductReadPlatformService, - final RoleReadPlatformService roleReadPlatformService, - final EntityDatatableChecksReadService entityDatatableChecksReadService, - final DatatableReadService datatableReadService, - final EntityDatatableChecksRepository entityDatatableChecksRepository) { + final RoleReadPlatformService roleReadPlatformService, final EntityDatatableChecksReadService entityDatatableChecksReadService, + final DatatableReadService datatableReadService, final EntityDatatableChecksRepository entityDatatableChecksRepository) { this.officeReadPlatformService = officeReadPlatformService; this.staffReadPlatformService = staffReadPlatformService; this.context = context; @@ -253,7 +251,7 @@ private WorkbookPopulator populateClientWorkbook(final String entityType, final List addressTypesCodeValues = fetchCodeValuesByCodeName("ADDRESS_TYPE"); List stateProvinceCodeValues = fetchCodeValuesByCodeName("STATE"); List countryCodeValues = fetchCodeValuesByCodeName("COUNTRY"); - + // Fetch required datatables for CLIENT entity with CREATE status String entitySubtype = null; if (entityType.trim().equalsIgnoreCase(GlobalEntityType.CLIENTS_PERSON.toString())) { @@ -262,7 +260,7 @@ private WorkbookPopulator populateClientWorkbook(final String entityType, final entitySubtype = "Entity"; } List requiredDatatables = fetchRequiredDatatables(entitySubtype); - + if (entityType.trim().equalsIgnoreCase(GlobalEntityType.CLIENTS_PERSON.toString())) { List genderCodeValues = fetchCodeValuesByCodeName("Gender"); return new ClientPersonWorkbookPopulator(new OfficeSheetPopulator(offices), new PersonnelSheetPopulator(staff, offices), @@ -277,7 +275,7 @@ private WorkbookPopulator populateClientWorkbook(final String entityType, final } return null; } - + private List fetchRequiredDatatables(final String entitySubtype) { List requiredDatatables = new ArrayList<>(); try { @@ -285,14 +283,14 @@ private List fetchRequiredDatatables(final String entitySubtype) List entityDatatableChecks; if (entitySubtype != null) { // Filter by entity subtype - entityDatatableChecks = entityDatatableChecksRepository.findByEntityAndStatusAndSubtype( - EntityTables.CLIENT.getName(), StatusEnum.CREATE.getValue(), entitySubtype); + entityDatatableChecks = entityDatatableChecksRepository.findByEntityAndStatusAndSubtype(EntityTables.CLIENT.getName(), + StatusEnum.CREATE.getValue(), entitySubtype); } else { // Get all required datatables regardless of subtype - entityDatatableChecks = entityDatatableChecksRepository.findByEntityAndStatus( - EntityTables.CLIENT.getName(), StatusEnum.CREATE.getValue()); + entityDatatableChecks = entityDatatableChecksRepository.findByEntityAndStatus(EntityTables.CLIENT.getName(), + StatusEnum.CREATE.getValue()); } - + // Convert EntityDatatableChecks to DatatableData for (EntityDatatableChecks check : entityDatatableChecks) { DatatableData datatable = datatableReadService.retrieveDatatable(check.getDatatableName()); diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/survey/service/ReadSurveyServiceImpl.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/survey/service/ReadSurveyServiceImpl.java index a11e076fc50..cafa4356c77 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/survey/service/ReadSurveyServiceImpl.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/survey/service/ReadSurveyServiceImpl.java @@ -69,8 +69,8 @@ public List retrieveAllSurveys() { .fillResultsetColumnHeaders(registeredDatatableName); final boolean multiRow = datatableUtil.isMultirowDatatable(columnHeaderData); - surveyDataTables.add(SurveyDataTableData - .create(DatatableData.create(appTableName, registeredDatatableName, entitySubType, columnHeaderData, multiRow), enabled)); + surveyDataTables.add(SurveyDataTableData.create( + DatatableData.create(appTableName, registeredDatatableName, entitySubType, columnHeaderData, multiRow), enabled)); } return surveyDataTables; @@ -110,8 +110,8 @@ public SurveyDataTableData retrieveSurvey(String surveyName) { final List columnHeaderData = this.genericDataService .fillResultsetColumnHeaders(registeredDatatableName); final boolean multiRow = datatableUtil.isMultirowDatatable(columnHeaderData); - datatableData = SurveyDataTableData - .create(DatatableData.create(appTableName, registeredDatatableName, entitySubType, columnHeaderData, multiRow), enabled); + datatableData = SurveyDataTableData.create( + DatatableData.create(appTableName, registeredDatatableName, entitySubType, columnHeaderData, multiRow), enabled); } diff --git a/integration-tests/src/test/java/org/apache/fineract/integrationtests/bulkimport/importhandler/client/ClientEntityImportHandlerTest.java b/integration-tests/src/test/java/org/apache/fineract/integrationtests/bulkimport/importhandler/client/ClientEntityImportHandlerTest.java index 56c72f6add5..9a2a8dbc0bf 100644 --- a/integration-tests/src/test/java/org/apache/fineract/integrationtests/bulkimport/importhandler/client/ClientEntityImportHandlerTest.java +++ b/integration-tests/src/test/java/org/apache/fineract/integrationtests/bulkimport/importhandler/client/ClientEntityImportHandlerTest.java @@ -119,8 +119,7 @@ public void testClientImport() throws InterruptedException, IOException, ParseEx firstClientRow.createCell(ClientEntityConstants.INCOPORATION_VALID_TILL_COL).setCellValue(validTill); firstClientRow.createCell(ClientEntityConstants.MOBILE_NO_COL).setCellValue(Utils.uniqueRandomNumberGenerator(9)); firstClientRow.createCell(ClientEntityConstants.CLIENT_TYPE_COL).setCellValue(lookupRow.getCell(0).getStringCellValue()); - firstClientRow.createCell(ClientEntityConstants.CLIENT_CLASSIFICATION_COL) - .setCellValue(lookupRow.getCell(1).getStringCellValue()); + firstClientRow.createCell(ClientEntityConstants.CLIENT_CLASSIFICATION_COL).setCellValue(lookupRow.getCell(1).getStringCellValue()); firstClientRow.createCell(ClientEntityConstants.INCOPORATION_NUMBER_COL).setCellValue(Utils.randomNumberGenerator(6)); firstClientRow.createCell(ClientEntityConstants.MAIN_BUSINESS_LINE).setCellValue(lookupRow.getCell(3).getStringCellValue()); firstClientRow.createCell(ClientEntityConstants.CONSTITUTION_COL).setCellValue(lookupRow.getCell(2).getStringCellValue()); From 5e56b054b4cb9024306dd80ef86c521d1cf883f8 Mon Sep 17 00:00:00 2001 From: Samer Melhem Date: Fri, 24 Jul 2026 14:11:12 +0300 Subject: [PATCH 7/7] FINERACT-2682: Address review feedback - use Lombok getters, remove duplicate ID-extraction logic --- .../data/ResultsetColumnHeaderData.java | 4 --- .../data/ResultsetColumnValueData.java | 6 ++-- .../portfolio/search/service/SearchUtil.java | 2 +- .../importhandler/ImportHandlerUtils.java | 28 ++----------------- .../client/ClientEntityImportHandler.java | 3 +- .../client/ClientPersonImportHandler.java | 3 +- 6 files changed, 9 insertions(+), 37 deletions(-) diff --git a/fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/ResultsetColumnHeaderData.java b/fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/ResultsetColumnHeaderData.java index 592a39fe316..c196954c6a9 100644 --- a/fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/ResultsetColumnHeaderData.java +++ b/fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/ResultsetColumnHeaderData.java @@ -149,10 +149,6 @@ public boolean hasColumnValues() { return columnValues != null && !columnValues.isEmpty(); } - public List getColumnValues() { - return columnValues; - } - public boolean isColumnValueAllowed(final String match) { for (final ResultsetColumnValueData allowedValue : this.columnValues) { if (allowedValue.matches(match)) { diff --git a/fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/ResultsetColumnValueData.java b/fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/ResultsetColumnValueData.java index b0436c68a72..40353a1bb4b 100644 --- a/fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/ResultsetColumnValueData.java +++ b/fineract-core/src/main/java/org/apache/fineract/infrastructure/dataqueries/data/ResultsetColumnValueData.java @@ -19,12 +19,14 @@ package org.apache.fineract.infrastructure.dataqueries.data; import java.io.Serializable; +import lombok.Getter; /** * Immutable data object representing a possible value for a given resultset column. */ public class ResultsetColumnValueData implements Serializable { + @Getter private final int id; private final String value; @SuppressWarnings("unused") @@ -50,10 +52,6 @@ public boolean codeMatches(final Integer match) { return match.intValue() == this.id; } - public int getId() { - return id; - } - public String getValue() { return value; } diff --git a/fineract-core/src/main/java/org/apache/fineract/portfolio/search/service/SearchUtil.java b/fineract-core/src/main/java/org/apache/fineract/portfolio/search/service/SearchUtil.java index 6c9a988c495..7189f61a7a1 100644 --- a/fineract-core/src/main/java/org/apache/fineract/portfolio/search/service/SearchUtil.java +++ b/fineract-core/src/main/java/org/apache/fineract/portfolio/search/service/SearchUtil.java @@ -351,7 +351,7 @@ public String camelToSnake(final String camelStr) { * The string value that may contain an ID in parentheses * @return The extracted ID as a string, or the original value if no ID pattern is found */ - private static String extractIdFromDisplayValue(String value) { + public static String extractIdFromDisplayValue(String value) { if (value == null || value.trim().isEmpty()) { return value; } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/ImportHandlerUtils.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/ImportHandlerUtils.java index 9e99ba3b2fc..a069ef83589 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/ImportHandlerUtils.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/ImportHandlerUtils.java @@ -42,6 +42,7 @@ import org.apache.fineract.infrastructure.dataqueries.domain.EntityDatatableChecks; import org.apache.fineract.infrastructure.dataqueries.domain.EntityDatatableChecksRepository; import org.apache.fineract.infrastructure.dataqueries.service.DatatableReadService; +import org.apache.fineract.portfolio.search.service.SearchUtil; import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.CellStyle; import org.apache.poi.ss.usermodel.CellType; @@ -157,31 +158,6 @@ public static String trimEmptyDecimalPortion(String result) { } } - /** - * Extracts numeric ID from strings ending with "(id)" format. Example: "Permanent (32)" -> "32" If the string - * doesn't match the pattern, returns the original value. - * - * @param value - * The string value that may contain an ID in parentheses - * @return The extracted ID as a string, or the original value if no ID pattern is found - */ - public static String extractIdFromDisplayValue(String value) { - if (value == null || value.trim().isEmpty()) { - return value; - } - // Match pattern: ".*(\d+)$" where the number is in parentheses at the end - // More specifically: ".*\\(\\d+\\)$" - String trimmed = value.trim(); - int lastParenIndex = trimmed.lastIndexOf('('); - if (lastParenIndex > 0 && trimmed.endsWith(")")) { - String idPart = trimmed.substring(lastParenIndex + 1, trimmed.length() - 1); - if (idPart.matches("\\d+")) { - return idPart; - } - } - return value; - } - public static LocalDate readAsDate(int colIndex, Row row) { Cell c = row.getCell(colIndex); if (c == null || c.getCellType() == CellType.BLANK) { @@ -978,7 +954,7 @@ else if (headerWithoutStar.contains("_")) { if (stringValue != null && !stringValue.trim().isEmpty()) { // Try to extract ID from display value format: "Label (ID)" // This handles CODELOOKUP and FK/INTEGER columns that show human-readable values - String extractedId = extractIdFromDisplayValue(stringValue.trim()); + String extractedId = SearchUtil.extractIdFromDisplayValue(stringValue.trim()); // If extraction succeeded (returned a different value), try to parse as integer if (!extractedId.equals(stringValue.trim())) { try { diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientEntityImportHandler.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientEntityImportHandler.java index 706717db4c2..95661e24ae0 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientEntityImportHandler.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientEntityImportHandler.java @@ -50,6 +50,7 @@ import org.apache.fineract.portfolio.address.data.AddressData; import org.apache.fineract.portfolio.client.data.ClientData; import org.apache.fineract.portfolio.client.data.ClientNonPersonData; +import org.apache.fineract.portfolio.search.service.SearchUtil; import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.IndexedColors; import org.apache.poi.ss.usermodel.Row; @@ -101,7 +102,7 @@ private Long parseIdFromDisplayValue(final String displayValue, final String fie if (displayValue == null) { return null; } - String idPart = ImportHandlerUtils.extractIdFromDisplayValue(displayValue); + String idPart = SearchUtil.extractIdFromDisplayValue(displayValue); if (idPart != null && idPart.matches("\\d+")) { return Long.valueOf(idPart); } diff --git a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientPersonImportHandler.java b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientPersonImportHandler.java index 11c2052a62f..e2a3fc8c6df 100644 --- a/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientPersonImportHandler.java +++ b/fineract-provider/src/main/java/org/apache/fineract/infrastructure/bulkimport/importhandler/client/ClientPersonImportHandler.java @@ -44,6 +44,7 @@ import org.apache.fineract.infrastructure.dataqueries.service.DatatableReadService; import org.apache.fineract.portfolio.address.data.AddressData; import org.apache.fineract.portfolio.client.data.ClientData; +import org.apache.fineract.portfolio.search.service.SearchUtil; import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.IndexedColors; import org.apache.poi.ss.usermodel.Row; @@ -92,7 +93,7 @@ private Long parseIdFromDisplayValue(final String displayValue, final String fie if (displayValue == null) { return null; } - String idPart = ImportHandlerUtils.extractIdFromDisplayValue(displayValue); + String idPart = SearchUtil.extractIdFromDisplayValue(displayValue); if (idPart != null && idPart.matches("\\d+")) { return Long.valueOf(idPart); }