diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index a2875f5..440f884 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -1,4 +1,5 @@ { + "domain": "JOB_DOMAIN", "openapiConfig": { "filePath": "OPENAPI_FILE_PATH" }, @@ -42,6 +43,21 @@ } } }, + "jobDefinitions": { + "tasks": { + "export": { + "type": "EXPORT_JOB_TYPE" + } + }, + "jobs": { + "export": { + "type": "TILES_TASK_TYPE" + }, + "finalize": { + "type": "FINALIZE_TASK_TYPE" + } + } + }, "storageEstimation": { "jpegTileEstimatedSizeInBytes": { "__name": "JPEG_TILE_ESTIMATED_SIZE_IN_BYTES", @@ -75,7 +91,6 @@ "clientsUrls": { "jobManager": { "url": "JOB_MANAGER_URL", - "jobDomain": "JOB_DOMAIN", "dequeueFinalizeIntervalMs": { "__name": "QUEUE_DEQUEUE_FINALIZE_INTERVAL_MS", "__format": "number" @@ -96,11 +111,6 @@ } } }, - "exportJobAndTaskTypes": { - "jobType": "EXPORT_JOB_TYPE", - "taskTilesType": "TILES_TASK_TYPE", - "taskFinalizeType": "FINALIZE_TASK_TYPE" - }, "httpRetry": { "attempts": { "__name": "HTTP_RETRY_ATTEMPTS", diff --git a/config/default.json b/config/default.json index 4706a08..13dc183 100644 --- a/config/default.json +++ b/config/default.json @@ -1,4 +1,5 @@ { + "domain": "RASTER", "openapiConfig": { "filePath": "./openapi3.yaml", "basePath": "/docs", @@ -45,7 +46,6 @@ "clientsUrls": { "jobManager": { "url": "http://job-manager-job-manager", - "jobDomain": "RASTER", "dequeueFinalizeIntervalMs": 1000, "finalizeTasksAttempts": 5 }, @@ -57,11 +57,6 @@ "heartbeatIntervalMs": 300 } }, - "exportJobAndTaskTypes": { - "jobType": "rasterTilesExporter", - "taskTilesType": "rasterTilesExporter", - "taskFinalizeType": "rasterFinalizeExporter" - }, "httpRetry": { "attempts": 5, "delay": "exponential", @@ -73,6 +68,21 @@ "gpkgsLocation": "/app/tiles_outputs/gpkgs", "downloadServerUrl": "http://download-service", "finalizePollingTimeMS": 2000, + "jobDefinitions": { + "tasks": { + "export": { + "type": "export" + }, + "finalize": { + "type": "finalize" + } + }, + "jobs": { + "export": { + "type": "Export" + } + } + }, "roiBufferMeter": 5, "minContainedPercentage": 80 } diff --git a/helm/templates/_tplValues.tpl b/helm/templates/_tplValues.tpl index bdd0cb8..8cd0497 100644 --- a/helm/templates/_tplValues.tpl +++ b/helm/templates/_tplValues.tpl @@ -57,8 +57,8 @@ Custom definitions {{- include "common.tplvalues.merge" ( dict "values" ( list .Values.serviceUrls .Values.global.serviceUrls ) "context" . ) }} {{- end -}} -{{- define "common.jobManagement.merged" -}} -{{- include "common.tplvalues.merge" ( dict "values" ( list .Values.jobManagement .Values.global.jobManagement ) "context" . ) }} +{{- define "common.jobDefinitions.merged" -}} +{{- include "common.tplvalues.merge" ( dict "values" ( list .Values.jobDefinitions .Values.global.jobDefinitions ) "context" . ) }} {{- end -}} {{- define "common.ingestion.merged" -}} diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index db4e8cd..007f3b8 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -2,9 +2,11 @@ {{- $tracing := (include "common.tracing.merged" .) | fromYaml }} {{- $metrics := (include "common.metrics.merged" .) | fromYaml }} {{- $serviceUrls := (include "common.serviceUrls.merged" .) | fromYaml }} -{{- $jobManagement := (include "common.jobManagement.merged" .) | fromYaml }} {{- $storage := (include "common.storage.merged" .) | fromYaml }} {{- $configmapName := include "configmap.fullname" . }} +{{- $jobDefinitions := (include "common.jobDefinitions.merged" .) | fromYaml }} + + {{- if .Values.enabled -}} apiVersion: v1 kind: ConfigMap @@ -26,9 +28,9 @@ data: JOB_MANAGER_URL: {{ $serviceUrls.jobManager | quote }} CLEANUP_EXPIRATION_DAYS: {{ .Values.env.cleanupExpirationDays | quote }} RASTER_CATALOG_MANAGER_URL: {{ $serviceUrls.catalogManager | quote }} - JOB_DOMAIN: {{ $jobManagement.jobDomain | quote }} - EXPORT_JOB_TYPE: {{ $jobManagement.exporter.jobType | quote }} - TILES_TASK_TYPE: {{ $jobManagement.exporter.taskTilesType | quote }} + JOB_DOMAIN: {{ .Values.global.domain | quote }} + EXPORT_JOB_TYPE: {{ $jobDefinitions.jobs.export.type | quote }} + TILES_TASK_TYPE: {{ $jobDefinitions.tasks.export.type | quote }} HTTP_RETRY_ATTEMPTS: {{ .Values.env.httpRetry.attempts | quote }} HTTP_RETRY_DELAY: {{ .Values.env.httpRetry.delay | quote }} HTTP_RETRY_SHOULD_RESET_TIMEOUT: {{ .Values.env.httpRetry.shouldResetTimeout | quote }} @@ -44,7 +46,7 @@ data: QUEUE_HEART_BEAT_MANAGER_BASE_URL: {{ $serviceUrls.heartbeatManager | quote }} QUEUE_DEQUEUE_FINALIZE_INTERVAL_MS: {{ .Values.env.queue.dequeueFinalizeIntervalMs | quote }} QUEUE_HEART_BEAT_INTERVAL_MS: {{ .Values.env.queue.heartbeatIntervalMs | quote }} - FINALIZE_TASK_TYPE: {{ $jobManagement.exporter.taskFinalizeType | quote }} + FINALIZE_TASK_TYPE: {{ $jobDefinitions.tasks.finalize.type | quote }} FINALIZE_TASKS_ATTEMPTS: {{ .Values.env.queue.finalizeTasksAttempts | quote }} ROI_BUFFER_METER: {{ .Values.env.roiBufferMeter | quote }} MIN_CONTAINED_PERCENTAGE: {{ .Values.env.minContainedPercentage | quote }} diff --git a/helm/values.yaml b/helm/values.yaml index ee49bec..2a059a2 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -8,12 +8,6 @@ global: heartbeatManager: "" jobManager: "" downloadServerPublicDNS: "" - jobManagement: - jobDomain: "" - exporter: - jobType: "" - taskTilesType: "" - taskFinalizeType: "" storage: tilesStorageProvider: "" fs: @@ -22,19 +16,25 @@ global: name: "" gpkgSubPath: "" ingress: - domain: 'apps.aroapp.io' + domain: 'apps.aroapp.io' + jobDefinitions: {} + domain: '' + +jobDefinitions: + jobs: + export: + type: "" + tasks: + export: + type: "" + finalize: + type: "" serviceUrls: catalogManager: "" heartbeatManager: "" jobManager: "" downloadServerPublicDNS: "" -jobManagement: - jobDomain: RASTER - exporter: - jobType: "" - taskTilesType: "" - taskFinalizeType: "" storage: tilesStorageProvider: '' fs: diff --git a/src/clients/jobManagerWrapper.ts b/src/clients/jobManagerWrapper.ts index 7b299b3..a565f7e 100644 --- a/src/clients/jobManagerWrapper.ts +++ b/src/clients/jobManagerWrapper.ts @@ -28,16 +28,16 @@ export class JobManagerWrapper extends JobManagerClient { public constructor(@inject(SERVICES.LOGGER) protected readonly logger: Logger, @inject(SERVICES.TRACER) public readonly tracer: Tracer) { super( logger, - config.get('externalClientsConfig.exportJobAndTaskTypes.jobType'), + config.get('jobDefinitions.jobs.export.type'), config.get('externalClientsConfig.clientsUrls.jobManager.url'), config.get('externalClientsConfig.httpRetry'), 'jobManagerClient', config.get('externalClientsConfig.disableHttpClientLogs') ); this.expirationDays = config.get('cleanupExpirationDays'); - this.tilesJobType = config.get('externalClientsConfig.exportJobAndTaskTypes.jobType'); - this.tilesTaskType = config.get('externalClientsConfig.exportJobAndTaskTypes.taskTilesType'); - this.jobDomain = config.get('externalClientsConfig.clientsUrls.jobManager.jobDomain'); + this.tilesJobType = config.get('jobDefinitions.jobs.export.type'); + this.tilesTaskType = config.get('jobDefinitions.tasks.export.type'); + this.jobDomain = config.get('domain'); } @withSpanAsyncV4 diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index 7791af5..ab465a5 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -214,11 +214,16 @@ export interface IExternalClientsConfig { heartbeatManager: IHeartbeatManager; finalizeTasksAttempts: number; }; - exportJobAndTaskTypes: { - jobType: string; - taskTilesType: string; - taskFinalizeType: string; - }; httpRetry: IHttpRetryConfig; disableHttpClientLogs: boolean; } + +//consider changing from nested interface +export interface IJobDefinitions { + jobs: { + export: { type: string }; + }; + tasks: { + export: { type: string }; + }; +} diff --git a/src/containerConfig.ts b/src/containerConfig.ts index b7bb867..011da50 100644 --- a/src/containerConfig.ts +++ b/src/containerConfig.ts @@ -9,7 +9,7 @@ import { createPackageRouterFactory, CREATE_PACKAGE_ROUTER_SYMBOL } from './crea import { InjectionObject, registerDependencies } from './common/dependencyRegistration'; import { tasksRouterFactory, TASKS_ROUTER_SYMBOL } from './tasks/routes/tasksRouter'; import { FinalizationManager, FINALIZATION_MANGER_SYMBOL } from './finalizationManager'; -import { IQueueConfig, IExternalClientsConfig } from './common/interfaces'; +import { IQueueConfig, IExternalClientsConfig, IJobDefinitions } from './common/interfaces'; import { storageRouterFactory, STORAGE_ROUTER_SYMBOL } from './storage/routes/storageRouter'; export interface RegisterOptions { @@ -20,13 +20,14 @@ export interface RegisterOptions { export const registerExternalValues = (options?: RegisterOptions): DependencyContainer => { const loggerConfig = config.get('telemetry.logger'); const externalClientsConfig = config.get('externalClientsConfig'); + const jobDefinitionsConfig = config.get('jobDefinitions'); const queueConfig: IQueueConfig = { jobManagerBaseUrl: externalClientsConfig.clientsUrls.jobManager.url, heartbeatManagerBaseUrl: externalClientsConfig.clientsUrls.heartbeatManager.url, dequeueFinalizeIntervalMs: externalClientsConfig.clientsUrls.jobManager.dequeueFinalizeIntervalMs, heartbeatIntervalMs: externalClientsConfig.clientsUrls.heartbeatManager.heartbeatIntervalMs, - jobType: externalClientsConfig.exportJobAndTaskTypes.jobType, - tilesTaskType: externalClientsConfig.exportJobAndTaskTypes.taskFinalizeType, + jobType: jobDefinitionsConfig.jobs.export.type, + tilesTaskType: jobDefinitionsConfig.tasks.export.type, }; // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment const logger = jsLogger({ ...loggerConfig, prettyPrint: loggerConfig.prettyPrint, hooks: { logMethod } }); diff --git a/src/finalizationManager.ts b/src/finalizationManager.ts index e8ef4ee..50d173b 100644 --- a/src/finalizationManager.ts +++ b/src/finalizationManager.ts @@ -29,7 +29,7 @@ export class FinalizationManager { @inject(JobManagerWrapper) private readonly jobManagerClient: JobManagerWrapper ) { this.expirationDays = config.get('cleanupExpirationDays'); - this.finalizeTaskType = config.get('externalClientsConfig.exportJobAndTaskTypes.taskFinalizeType'); + this.finalizeTaskType = config.get('jobDefinitions.tasks.finalize.type'); this.finalizeAttempts = config.get('externalClientsConfig.clientsUrls.jobManager.finalizeTasksAttempts'); } diff --git a/src/tasks/models/tasksManager.ts b/src/tasks/models/tasksManager.ts index 74b9d39..ee48078 100644 --- a/src/tasks/models/tasksManager.ts +++ b/src/tasks/models/tasksManager.ts @@ -45,7 +45,7 @@ export class TasksManager { ) { this.gpkgsLocation = config.get('gpkgsLocation'); this.downloadServerUrl = config.get('downloadServerUrl'); - this.tilesJobType = config.get('externalClientsConfig.exportJobAndTaskTypes.jobType'); + this.tilesJobType = config.get('jobDefinitions.jobs.export.type'); } @withSpanAsyncV4 diff --git a/tests/integration/createPackage/tasks.spec.ts b/tests/integration/createPackage/tasks.spec.ts index 1312410..877cbb1 100644 --- a/tests/integration/createPackage/tasks.spec.ts +++ b/tests/integration/createPackage/tasks.spec.ts @@ -36,7 +36,7 @@ describe('tasks', function () { { id: '0a5552f7-01eb-40af-a912-eed8fa9e1561', jobId: '0a5552f7-01eb-40af-a912-eed8fa9e1568', - type: 'rasterTilesExporterd', + type: 'export', description: '', parameters: {} as unknown as ITaskParameters, status: OperationStatus.IN_PROGRESS, diff --git a/tests/integration/testContainerConfig.ts b/tests/integration/testContainerConfig.ts index c454b84..d0ef256 100644 --- a/tests/integration/testContainerConfig.ts +++ b/tests/integration/testContainerConfig.ts @@ -4,18 +4,19 @@ import { container } from 'tsyringe'; import { SERVICES } from '../../src/common/constants'; import { configMock, registerDefaultConfig, getMock, hasMock } from '../mocks/config'; import { InjectionObject } from '../../src/common/dependencyRegistration'; -import { IExternalClientsConfig, IQueueConfig } from '../../src/common/interfaces'; +import { IExternalClientsConfig, IJobDefinitions, IQueueConfig } from '../../src/common/interfaces'; function getContainerConfig(): InjectionObject[] { registerDefaultConfig(); const externalClientsConfig = configMock.get('externalClientsConfig'); + const jobDefinitionsConfig = configMock.get('jobDefinitions'); const queueConfig: IQueueConfig = { jobManagerBaseUrl: externalClientsConfig.clientsUrls.jobManager.url, heartbeatManagerBaseUrl: externalClientsConfig.clientsUrls.heartbeatManager.url, dequeueFinalizeIntervalMs: externalClientsConfig.clientsUrls.jobManager.dequeueFinalizeIntervalMs, heartbeatIntervalMs: externalClientsConfig.clientsUrls.heartbeatManager.heartbeatIntervalMs, - jobType: externalClientsConfig.exportJobAndTaskTypes.jobType, - tilesTaskType: externalClientsConfig.exportJobAndTaskTypes.taskFinalizeType, + jobType: jobDefinitionsConfig.jobs.export.type, + tilesTaskType: jobDefinitionsConfig.tasks.export.type, }; const testTracer = trace.getTracer('testTracer'); diff --git a/tests/mocks/config.ts b/tests/mocks/config.ts index 1c071c2..d5e8ba1 100644 --- a/tests/mocks/config.ts +++ b/tests/mocks/config.ts @@ -20,6 +20,7 @@ const setConfigValues = (values: Record): void => { const registerDefaultConfig = (): void => { const config = { + domain: 'RASTER', openapiConfig: { filePath: './openapi3.yaml', basePath: '/docs', @@ -65,7 +66,6 @@ const registerDefaultConfig = (): void => { clientsUrls: { jobManager: { url: 'http://raster-catalog-manager', - jobDomain: 'RASTER', dequeueFinalizeIntervalMs: 1000, finalizeTasksAttempts: 5, }, @@ -77,11 +77,6 @@ const registerDefaultConfig = (): void => { heartbeatIntervalMs: 300, }, }, - exportJobAndTaskTypes: { - jobType: 'rasterTilesExporter', - taskTilesType: 'rasterTilesExporter', - taskFinalizeType: 'rasterTilesExporter', - }, httpRetry: { attempts: 5, delay: 'exponential', @@ -91,6 +86,21 @@ const registerDefaultConfig = (): void => { roiBufferMeter: 5, minContainedPercentage: 75, }, + jobDefinitions: { + tasks: { + export: { + type: 'export', + }, + finalize: { + type: 'finalize', + }, + }, + jobs: { + export: { + type: 'Export', + }, + }, + }, }; setConfigValues(config); }; diff --git a/tests/mocks/data.ts b/tests/mocks/data.ts index e47e2b7..8a0030d 100644 --- a/tests/mocks/data.ts +++ b/tests/mocks/data.ts @@ -493,7 +493,7 @@ const completedExportJob: IJobResponse = id: 'afbdd5e6-25db-4567-a81f-71e0e7d30761', resourceId: 'string_completed', version: '1.0', - type: 'rasterTilesExporter', + type: 'Export', domain: 'testDomain', description: '', parameters: { @@ -549,7 +549,7 @@ const completedExportJob: IJobResponse = { id: '542ebbfd-f4d1-4c77-bd4d-97ca121f0de7', jobId: 'b0b19b88-aecb-4e74-b694-dfa7eada8bf7', - type: 'rasterTilesExporter', + type: 'export', description: '', parameters: { isNewTarget: true, @@ -575,7 +575,7 @@ const inProgressExportJob: IJobResponse = domain: 'testDomain', resourceId: 'string_inprogress', version: '1.0', - type: 'rasterTilesExporter', + type: 'Export', percentage: 0, description: '', parameters: { @@ -608,7 +608,7 @@ const inProgressExportJob: IJobResponse = { id: '1f765695-338b-4752-b182-a8cbae3c610e', jobId: 'b0b19b88-aecb-4e74-b694-dfa7eada8bf7', - type: 'rasterTilesExporter', + type: 'export', description: '', parameters: { isNewTarget: true, @@ -908,10 +908,10 @@ const jobPayloadWithMixedForFixedStrategyCheck = { ], targetFormat: undefined, }, - type: 'rasterTilesExporter', + type: 'export', }, ], - type: 'rasterTilesExporter', + type: 'Export', version: '1.0', }; diff --git a/tests/mocks/data/mockJob.ts b/tests/mocks/data/mockJob.ts index f75ec82..b816102 100644 --- a/tests/mocks/data/mockJob.ts +++ b/tests/mocks/data/mockJob.ts @@ -8,7 +8,7 @@ export const mockCompletedJob: JobFinalizeResponse = { resourceId: 'testCompleted', domain: 'testDomain', version: '1.0', - type: 'rasterTilesExporter', + type: 'Export', description: 'test job', parameters: { crs: 'EPSG:4326', diff --git a/tests/unit/clients/jobManagerClient.spec.ts b/tests/unit/clients/jobManagerClient.spec.ts index fb2e61f..31185c1 100644 --- a/tests/unit/clients/jobManagerClient.spec.ts +++ b/tests/unit/clients/jobManagerClient.spec.ts @@ -116,7 +116,7 @@ describe('JobManagerClient', () => { describe('Find Job by Status', () => { it('should findExportCompletedJobs successfully', async () => { - const tilesJobType = configMock.get('externalClientsConfig.exportJobAndTaskTypes.jobType'); + const tilesJobType = configMock.get('jobDefinitions.jobs.export.type'); getExportJobs = jest.fn(); const jobManager = jobManagerClient as unknown as { getExportJobs: unknown }; jobManager.getExportJobs = getExportJobs.mockResolvedValue([completedExportJob]); @@ -151,7 +151,7 @@ describe('JobManagerClient', () => { }); it('should findExportInProgressJobs successfully', async () => { - const tilesJobType = configMock.get('externalClientsConfig.exportJobAndTaskTypes.jobType'); + const tilesJobType = configMock.get('jobDefinitions.jobs.export.type'); getExportJobs = jest.fn(); const jobManager = jobManagerClient as unknown as { getExportJobs: unknown }; jobManager.getExportJobs = getExportJobs.mockResolvedValue([inProgressExportJob]); diff --git a/tests/unit/createPackage/models/tasksModel.spec.ts b/tests/unit/createPackage/models/tasksModel.spec.ts index 6602296..f7257ac 100644 --- a/tests/unit/createPackage/models/tasksModel.spec.ts +++ b/tests/unit/createPackage/models/tasksModel.spec.ts @@ -55,7 +55,7 @@ describe('TasksManager', () => { { id: '0a5552f7-01eb-40af-a912-eed8fa9e1561', jobId: '0a5552f7-01eb-40af-a912-eed8fa9e1568', - type: 'rasterTilesExporterd', + type: 'export', description: '', parameters: {} as unknown as ITaskParameters, status: OperationStatus.IN_PROGRESS, @@ -245,7 +245,7 @@ describe('TasksManager', () => { describe('#createFinalizeTask', () => { it('should create new success finalize task', async () => { - const finalizeTaskType = configMock.get('externalClientsConfig.exportJobAndTaskTypes.taskFinalizeType'); + const finalizeTaskType = configMock.get('jobDefinitions.tasks.finalize.type'); const expectedCreateTaskRequest: CreateFinalizeTaskBody = { type: finalizeTaskType, parameters: { exporterTaskStatus: OperationStatus.COMPLETED }, @@ -262,7 +262,7 @@ describe('TasksManager', () => { }); it('should create new failed finalize task', async () => { - const finalizeTaskType = configMock.get('externalClientsConfig.exportJobAndTaskTypes.taskFinalizeType'); + const finalizeTaskType = configMock.get('jobDefinitions.tasks.finalize.type'); const expectedCreateTaskRequest: CreateFinalizeTaskBody = { type: finalizeTaskType, parameters: { reason: 'GPKG corrupted', exporterTaskStatus: OperationStatus.FAILED },