diff --git a/README.md b/README.md index c350153c..5c11c2ac 100644 --- a/README.md +++ b/README.md @@ -162,8 +162,7 @@ The service can be configured using environment variables or a configuration fil |--------------|------|-------------|---------| | EXPORT_JOB_TYPE | string | Export job type | `"Export"` | | EXPORT_CLEANUP_EXPIRATION_DAYS | number | Days until exported files are cleaned up | `7` | -| EXPORT_GPKGS_PATH | string | Path to store geopackage files | `"/tmp/gpkgs"` | -| EXPORT_DOWNLOAD_PATH | string | Download path for exported files | `"/downloads"` | +| EXPORT_GPKGS_ROOT_DIR | string | Root directory for storing exported geopackage (GPKG) files | `"gpkgs"` | | TILES_EXPORTING_TASK_TYPE | string | Tiles exporting task type | `"tiles-exporting"` | # Core Functionality diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index 5a13e965..4046c36a 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -186,8 +186,7 @@ "__name": "EXPORT_CLEANUP_EXPIRATION_DAYS", "__format": "number" }, - "gpkgsPath": "EXPORT_GPKGS_PATH", - "downloadPath": "EXPORT_DOWNLOAD_PATH" + "gpkgsRootDir": "EXPORT_GPKGS_ROOT_DIR" } }, "tasks": { diff --git a/config/default.json b/config/default.json index 944aa6dc..0a531e7c 100644 --- a/config/default.json +++ b/config/default.json @@ -122,8 +122,7 @@ "export": { "type": "Export", "cleanupExpirationDays": 14, - "gpkgsPath": "/gpkgs", - "downloadPath": "downloads" + "gpkgsRootDir": "gpkgs" } }, "tasks": { diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index 834e1080..89ab0836 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -4,8 +4,6 @@ {{- $serviceUrls := (include "common.serviceUrls.merged" .) | fromYaml }} {{- $storage := (include "common.storage.merged" .) | fromYaml }} {{- $jobDefinitions := (include "common.jobDefinitions.merged" .) | fromYaml }} -{{- $gpkgPath := (printf "%s/%s" $storage.fs.internalPvc.mountPath $storage.fs.internalPvc.gpkgSubPath) }} - {{- if .Values.enabled -}} apiVersion: v1 @@ -56,8 +54,7 @@ data: INGESTION_SEED_JOB_TYPE : {{ $jobDefinitions.jobs.seed.type | quote }} EXPORT_JOB_TYPE: {{ $jobDefinitions.jobs.export.type | quote }} EXPORT_CLEANUP_EXPIRATION_DAYS: {{ $jobDefinitions.jobs.export.cleanupExpirationDays | quote }} - EXPORT_GPKGS_PATH: {{ $gpkgPath | quote }} - EXPORT_DOWNLOAD_PATH: {{ $jobDefinitions.jobs.export.downloadPath | quote }} + EXPORT_GPKGS_ROOT_DIR: {{ $jobDefinitions.jobs.export.gpkgsRootDir | quote }} TILES_MERGING_TASK_TYPE: {{ $jobDefinitions.tasks.merge.type | quote }} TILES_MERGING_TILE_BATCH_SIZE: {{ $jobDefinitions.tasks.merge.tileBatchSize | quote }} TILES_MERGING_TASK_BATCH_SIZE: {{ $jobDefinitions.tasks.merge.taskBatchSize | quote }} diff --git a/helm/values.yaml b/helm/values.yaml index 0ceb8334..ea7d2d8c 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -141,7 +141,7 @@ jobDefinitions: export: type: "" cleanupExpirationDays: 14 - downloadPath: "" + gpkgsRootDir: "gpkgs" tasks: createTasks: type: "" diff --git a/src/job/models/export/exportJobHandler.ts b/src/job/models/export/exportJobHandler.ts index d0117dfb..5c3c3299 100644 --- a/src/job/models/export/exportJobHandler.ts +++ b/src/job/models/export/exportJobHandler.ts @@ -19,7 +19,6 @@ import { EXPORT_FAILURE_MESSAGE, EXPORT_SUCCESS_MESSAGE, GPKG_CONTENT_TYPE, - GPKGS_PREFIX, JSON_CONTENT_TYPE, SERVICES, StorageProvider, @@ -29,6 +28,7 @@ import { TaskMetrics } from '../../../utils/metrics/taskMetrics'; import { AggregationLayerMetadata, ExportFinalizeExecutionContext, + ExportFinalizeGpkgPaths, ExportTask, ExportTaskParameters, GpkgArtifactProperties, @@ -47,14 +47,15 @@ import { CallbackClient } from '../../../httpClients/callbackClient'; import { JobTrackerClient } from '../../../httpClients/jobTrackerClient'; import { PolygonPartsMangerClient } from '../../../httpClients/polygonPartsMangerClient'; import { convertObjectKeysToSnakeCase } from '../../../utils/db/dbUtils'; +import { buildUrl } from '../../../utils/url'; @injectable() export class ExportJobHandler extends JobHandler implements IJobHandler { private readonly exportTaskType: string; - private readonly gpkgsPath: string; + private readonly gpkgsRootDir: string; private readonly isS3GpkgProvider: boolean; private readonly cleanupExpirationDays: number; - private readonly downloadUrl: string; + private readonly downloadServerUrl: string; public constructor( @inject(SERVICES.LOGGER) logger: Logger, @inject(SERVICES.CONFIG) config: IConfig, @@ -71,14 +72,12 @@ export class ExportJobHandler extends JobHandler implements IJobHandler('jobManagement.export.tasks.tilesExporting.type'); - this.gpkgsPath = config.get('jobManagement.export.pollingJobs.export.gpkgsPath'); + this.gpkgsRootDir = config.get('jobManagement.export.pollingJobs.export.gpkgsRootDir'); // eslint-disable-next-line @typescript-eslint/naming-convention const gpkgProvider = config.get('gpkgStorageProvider'); this.isS3GpkgProvider = gpkgProvider === StorageProvider.S3; this.cleanupExpirationDays = config.get('jobManagement.export.pollingJobs.export.cleanupExpirationDays'); - const downloadServerUrl = config.get('servicesUrl.downloadServerPublicDNS'); - const downloadPath = config.get('jobManagement.export.pollingJobs.export.downloadPath'); - this.downloadUrl = `${downloadServerUrl}/${downloadPath}`; + this.downloadServerUrl = config.get('servicesUrl.downloadServerPublicDNS'); } public async handleJobInit(job: ExportJob, task: ExportInitTask): Promise { await context.with(trace.setSpan(context.active(), this.tracer.startSpan(`${ExportJobHandler.name}.${this.handleJobInit.name}`)), async () => { @@ -150,7 +149,7 @@ export class ExportJobHandler extends JobHandler implements IJobHandler { const { job, task, paths, telemetry, logger } = context; const { taskTracker, tracingSpan: activeSpan } = telemetry; - const { gpkgFilePath, gpkgRelativePath } = paths; + const { gpkgFilePath } = paths; let fullProcessingParams = taskParams; let gpkgProcessingComplete = false; @@ -220,7 +219,7 @@ export class ExportJobHandler extends JobHandler implements IJobHandler { - const gpkgS3Key = `${GPKGS_PREFIX}/${gpkgRelativePath}`; + const { gpkgFilePath } = paths; + const gpkgS3Key = path.posix.join(this.gpkgsRootDir, paths.gpkgRelativePath); + const jsonFilePath = gpkgFilePath.replace(/\.gpkg$/, '.json'); //TODO: In future, we will remove the json metadata file and support only gpkg const jsonS3Key = gpkgS3Key.replace(/\.gpkg$/, '.json'); //TODO: In future, we will remove the json metadata file and support only gpkg - const jsonPath = gpkgPath.replace(/\.gpkg$/, '.json'); //TODO: In future, we will remove the json metadata file and support only gpkg await this.s3Service.uploadFiles([ - { filePath: gpkgPath, s3Key: gpkgS3Key, contentType: GPKG_CONTENT_TYPE }, - { filePath: jsonPath, s3Key: jsonS3Key, contentType: JSON_CONTENT_TYPE }, + { filePath: gpkgFilePath, s3Key: gpkgS3Key, contentType: GPKG_CONTENT_TYPE }, + { filePath: jsonFilePath, s3Key: jsonS3Key, contentType: JSON_CONTENT_TYPE }, ]); - await this.fsService.deleteFileAndParentDir(gpkgPath); + await this.fsService.deleteFileAndParentDir(gpkgFilePath); const updatedParams = await this.markFinalizeStepAsCompleted(jobId, taskId, taskParams, 'gpkgUploadedToS3'); return updatedParams; } @@ -390,11 +389,11 @@ export class ExportJobHandler extends JobHandler implements IJobHandler { + private async sendCallbacks(job: ExportJob, taskId: string, taskParams: ExportFinalizeTaskParams, paths: ExportFinalizeGpkgPaths): Promise { try { const cleanupExpirationTimeUTC = createExpirationDate(this.cleanupExpirationDays); - const cleanupDataParams: CleanupData = { cleanupExpirationTimeUTC, directoryPath: dirPath }; - const callbackParams = this.createCallbacksParams(job, cleanupExpirationTimeUTC); + const cleanupDataParams: CleanupData = { cleanupExpirationTimeUTC, directoryPath: paths.gpkgDirPath }; + const callbackParams = this.createCallbacksParams(job, cleanupExpirationTimeUTC, paths); await this.queueClient.jobManagerClient.updateJob(job.id, { parameters: { ...job.parameters, cleanupDataParams, callbackParams }, @@ -429,9 +428,9 @@ export class ExportJobHandler extends JobHandler implements IJobHandler { + const url = new URL(baseUrl); + url.pathname = path.posix.join(url.pathname, ...pathSegments); + return url.href; +}; diff --git a/tests/configurations/unit/jest.config.js b/tests/configurations/unit/jest.config.js index 1e0df038..94a2ba8d 100644 --- a/tests/configurations/unit/jest.config.js +++ b/tests/configurations/unit/jest.config.js @@ -14,8 +14,8 @@ module.exports = { '!**/routes/**', '!/src/*', ], - modulePathIgnorePatterns: ['/src/utils/metrics/taskMetrics.ts'], - coveragePathIgnorePatterns: ['/src/utils/metrics/taskMetrics.ts'], + modulePathIgnorePatterns: ['/src/utils/metrics/taskMetrics.ts', '/src/utils/url.ts'], + coveragePathIgnorePatterns: ['/src/utils/metrics/taskMetrics.ts', '/src/utils/url.ts'], coverageDirectory: '/coverage', reporters: [ 'default', diff --git a/tests/unit/job/exportJobHandler/exportJobHandler.spec.ts b/tests/unit/job/exportJobHandler/exportJobHandler.spec.ts index 11c67dc0..2561f27a 100644 --- a/tests/unit/job/exportJobHandler/exportJobHandler.spec.ts +++ b/tests/unit/job/exportJobHandler/exportJobHandler.spec.ts @@ -112,9 +112,9 @@ describe('ExportJobHandler', () => { }); describe('handleJobFinalize', () => { - const gpkgsPath = '/gpkgs'; + const gpkgsRootDir = 'gpkgs'; const gpkgRelativePath = 'package.gpkg'; - const gpkgFilePath = `${gpkgsPath}/${gpkgRelativePath}`; + const gpkgFilePath = path.join('/', gpkgsRootDir, gpkgRelativePath); const jsonFilePath = gpkgFilePath.replace('.gpkg', '.json'); const gpkgDirPath = '/path/to/gpkgs'; let joinSpy: jest.SpyInstance; @@ -165,7 +165,7 @@ describe('ExportJobHandler', () => { await exportJobHandler.handleJobFinalize(job, task); // Verify path methods were called correctly - expect(joinSpy).toHaveBeenCalledWith(gpkgsPath, gpkgRelativePath); + expect(joinSpy).toHaveBeenCalledWith('/', gpkgsRootDir, gpkgRelativePath); expect(dirnameSpy).toHaveBeenCalledWith(gpkgFilePath); // Verify metadata processing @@ -229,7 +229,6 @@ describe('ExportJobHandler', () => { describe('when handling S3 upload', () => { it('should upload GPKG to S3 and delete local file when storage provider is S3', async () => { setValue('gpkgStorageProvider', 'S3'); - setValue('jobManagement.jobs.export.gpkgsPath', '/gpkgs'); const { exportJobHandler, s3ServiceMock, fsServiceMock, jobManagerClientMock } = setupExportJobHandlerTest(); const job = { @@ -258,7 +257,7 @@ describe('ExportJobHandler', () => { await exportJobHandler.handleJobFinalize(job, task); - expect(joinSpy).toHaveBeenCalledWith(gpkgsPath, gpkgRelativePath); + expect(joinSpy).toHaveBeenCalledWith('/', gpkgsRootDir, gpkgRelativePath); expect(s3ServiceMock.uploadFiles).toHaveBeenCalledWith([ { diff --git a/tests/unit/mocks/configMock.ts b/tests/unit/mocks/configMock.ts index 2e7c2bc4..de4d56af 100644 --- a/tests/unit/mocks/configMock.ts +++ b/tests/unit/mocks/configMock.ts @@ -108,6 +108,7 @@ const registerDefaultConfig = (): void => { mapproxyDns: 'http://mapproxy', polygonPartsManager: 'http://polygon-parts-manager', geoserverDns: 'http://geoserver', + downloadServerPublicDNS: 'http://download-server', jobTracker: 'http://job-tracker', }, geoserver: { @@ -169,7 +170,7 @@ const registerDefaultConfig = (): void => { pollingJobs: { export: { type: 'Export', - gpkgsPath: '/gpkgs', + gpkgsRootDir: 'gpkgs', }, }, tasks: {