refactor: change jobManagement to new jobDefentions (MAPCO-5091) (#98)

* refactor: change jobManagemnt to new jobDefentions * feat: pr flow to v2 * helm: configuration * refactor: all pointers to JobDefenitions * style: lint * feat: add comment * fix: fix default json
MapColonies · Oct 31, 2024 · e3d26ef · e3d26ef
1 parent b2d49f2
commit e3d26ef
Show file tree

Hide file tree

Showing 17 changed files with 107 additions and 68 deletions.
diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json
@@ -1,4 +1,5 @@
 {
+  "domain": "JOB_DOMAIN",
   "openapiConfig": {
     "filePath": "OPENAPI_FILE_PATH"
   },
@@ -42,6 +43,21 @@
       }
     }
   },
+  "jobDefinitions": {
+    "tasks": {
+      "export": {
+        "type": "EXPORT_JOB_TYPE"
+      }
+    },
+    "jobs": {
+      "export": {
+        "type": "TILES_TASK_TYPE"
+      },
+      "finalize": {
+        "type": "FINALIZE_TASK_TYPE"
+      }
+    }
+  },
   "storageEstimation": {
     "jpegTileEstimatedSizeInBytes": {
       "__name": "JPEG_TILE_ESTIMATED_SIZE_IN_BYTES",
@@ -75,7 +91,6 @@
     "clientsUrls": {
       "jobManager": {
         "url": "JOB_MANAGER_URL",
-        "jobDomain": "JOB_DOMAIN",
         "dequeueFinalizeIntervalMs": {
           "__name": "QUEUE_DEQUEUE_FINALIZE_INTERVAL_MS",
           "__format": "number"
@@ -96,11 +111,6 @@
         }
       }
     },
-    "exportJobAndTaskTypes": {
-      "jobType": "EXPORT_JOB_TYPE",
-      "taskTilesType": "TILES_TASK_TYPE",
-      "taskFinalizeType": "FINALIZE_TASK_TYPE"
-    },
     "httpRetry": {
       "attempts": {
         "__name": "HTTP_RETRY_ATTEMPTS",

diff --git a/config/default.json b/config/default.json
@@ -1,4 +1,5 @@
 {
+  "domain": "RASTER",
   "openapiConfig": {
     "filePath": "./openapi3.yaml",
     "basePath": "/docs",
@@ -45,7 +46,6 @@
     "clientsUrls": {
       "jobManager": {
         "url": "http://job-manager-job-manager",
-        "jobDomain": "RASTER",
         "dequeueFinalizeIntervalMs": 1000,
         "finalizeTasksAttempts": 5
       },
@@ -57,11 +57,6 @@
         "heartbeatIntervalMs": 300
       }
     },
-    "exportJobAndTaskTypes": {
-      "jobType": "rasterTilesExporter",
-      "taskTilesType": "rasterTilesExporter",
-      "taskFinalizeType": "rasterFinalizeExporter"
-    },
     "httpRetry": {
       "attempts": 5,
       "delay": "exponential",
@@ -73,6 +68,21 @@
   "gpkgsLocation": "/app/tiles_outputs/gpkgs",
   "downloadServerUrl": "http://download-service",
   "finalizePollingTimeMS": 2000,
+  "jobDefinitions": {
+    "tasks": {
+      "export": {
+        "type": "export"
+      },
+      "finalize": {
+        "type": "finalize"
+      }
+    },
+    "jobs": {
+      "export": {
+        "type": "Export"
+      }
+    }
+  },
   "roiBufferMeter": 5,
   "minContainedPercentage": 80
 }
diff --git a/helm/templates/_tplValues.tpl b/helm/templates/_tplValues.tpl
@@ -57,8 +57,8 @@ Custom definitions
 {{- include "common.tplvalues.merge" ( dict "values" ( list .Values.serviceUrls .Values.global.serviceUrls ) "context" . ) }}
 {{- end -}}
 
-{{- define "common.jobManagement.merged" -}}
-{{- include "common.tplvalues.merge" ( dict "values" ( list .Values.jobManagement .Values.global.jobManagement ) "context" . ) }}
+{{- define "common.jobDefinitions.merged" -}}
+{{- include "common.tplvalues.merge" ( dict "values" ( list .Values.jobDefinitions .Values.global.jobDefinitions ) "context" . ) }}
 {{- end -}}
 
 {{- define "common.ingestion.merged" -}}

diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml
@@ -2,9 +2,11 @@
 {{- $tracing := (include "common.tracing.merged" .) | fromYaml }}
 {{- $metrics := (include "common.metrics.merged" .) | fromYaml }}
 {{- $serviceUrls := (include "common.serviceUrls.merged" .) | fromYaml }}
-{{- $jobManagement := (include "common.jobManagement.merged" .) | fromYaml }}
 {{- $storage := (include "common.storage.merged" .) | fromYaml }}
 {{- $configmapName := include "configmap.fullname" . }}
+{{- $jobDefinitions := (include "common.jobDefinitions.merged" .) | fromYaml }}
+
+
 {{- if .Values.enabled -}}
 apiVersion: v1
 kind: ConfigMap
@@ -26,9 +28,9 @@ data:
   JOB_MANAGER_URL: {{ $serviceUrls.jobManager | quote }}
   CLEANUP_EXPIRATION_DAYS: {{ .Values.env.cleanupExpirationDays | quote }}
   RASTER_CATALOG_MANAGER_URL: {{ $serviceUrls.catalogManager | quote }}
-  JOB_DOMAIN: {{ $jobManagement.jobDomain | quote }}
-  EXPORT_JOB_TYPE: {{ $jobManagement.exporter.jobType | quote }}
-  TILES_TASK_TYPE: {{ $jobManagement.exporter.taskTilesType | quote }}
+  JOB_DOMAIN: {{ .Values.global.domain | quote }}
+  EXPORT_JOB_TYPE: {{ $jobDefinitions.jobs.export.type | quote }}
+  TILES_TASK_TYPE: {{ $jobDefinitions.tasks.export.type | quote }}
   HTTP_RETRY_ATTEMPTS: {{ .Values.env.httpRetry.attempts | quote }}
   HTTP_RETRY_DELAY: {{ .Values.env.httpRetry.delay | quote }}
   HTTP_RETRY_SHOULD_RESET_TIMEOUT: {{ .Values.env.httpRetry.shouldResetTimeout | quote }}
@@ -44,7 +46,7 @@ data:
   QUEUE_HEART_BEAT_MANAGER_BASE_URL: {{ $serviceUrls.heartbeatManager | quote }}
   QUEUE_DEQUEUE_FINALIZE_INTERVAL_MS: {{ .Values.env.queue.dequeueFinalizeIntervalMs | quote }}
   QUEUE_HEART_BEAT_INTERVAL_MS: {{ .Values.env.queue.heartbeatIntervalMs | quote }}
-  FINALIZE_TASK_TYPE: {{ $jobManagement.exporter.taskFinalizeType | quote }}
+  FINALIZE_TASK_TYPE: {{ $jobDefinitions.tasks.finalize.type | quote }}
   FINALIZE_TASKS_ATTEMPTS: {{ .Values.env.queue.finalizeTasksAttempts | quote }}
   ROI_BUFFER_METER: {{ .Values.env.roiBufferMeter | quote }}
   MIN_CONTAINED_PERCENTAGE: {{ .Values.env.minContainedPercentage | quote }}

diff --git a/helm/values.yaml b/helm/values.yaml
@@ -8,12 +8,6 @@ global:
     heartbeatManager: ""
     jobManager: ""
     downloadServerPublicDNS: ""
-  jobManagement:
-    jobDomain: ""
-    exporter:
-      jobType: ""
-      taskTilesType: ""
-      taskFinalizeType: ""
   storage:
     tilesStorageProvider: ""
     fs:
@@ -22,19 +16,25 @@ global:
         name: ""
         gpkgSubPath: ""
   ingress:
-    domain: 'apps.aroapp.io'          
+    domain: 'apps.aroapp.io'  
+  jobDefinitions: {}
+  domain: ''        
+
+jobDefinitions:
+  jobs:
+    export:
+      type: ""
+  tasks:
+    export:
+      type: ""
+    finalize:
+      type: ""
 
 serviceUrls:
   catalogManager: ""
   heartbeatManager: ""
   jobManager: ""
   downloadServerPublicDNS: ""
-jobManagement:
-  jobDomain: RASTER
-  exporter:
-    jobType: ""
-    taskTilesType: ""
-    taskFinalizeType: ""
 storage:
   tilesStorageProvider: ''
   fs:

diff --git a/src/clients/jobManagerWrapper.ts b/src/clients/jobManagerWrapper.ts
@@ -28,16 +28,16 @@ export class JobManagerWrapper extends JobManagerClient {
   public constructor(@inject(SERVICES.LOGGER) protected readonly logger: Logger, @inject(SERVICES.TRACER) public readonly tracer: Tracer) {
     super(
       logger,
-      config.get<string>('externalClientsConfig.exportJobAndTaskTypes.jobType'),
+      config.get<string>('jobDefinitions.jobs.export.type'),
       config.get<string>('externalClientsConfig.clientsUrls.jobManager.url'),
       config.get<IHttpRetryConfig>('externalClientsConfig.httpRetry'),
       'jobManagerClient',
       config.get<boolean>('externalClientsConfig.disableHttpClientLogs')
     );
     this.expirationDays = config.get<number>('cleanupExpirationDays');
-    this.tilesJobType = config.get<string>('externalClientsConfig.exportJobAndTaskTypes.jobType');
-    this.tilesTaskType = config.get<string>('externalClientsConfig.exportJobAndTaskTypes.taskTilesType');
-    this.jobDomain = config.get<string>('externalClientsConfig.clientsUrls.jobManager.jobDomain');
+    this.tilesJobType = config.get<string>('jobDefinitions.jobs.export.type');
+    this.tilesTaskType = config.get<string>('jobDefinitions.tasks.export.type');
+    this.jobDomain = config.get<string>('domain');
   }
 
   @withSpanAsyncV4

diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts
@@ -214,11 +214,16 @@ export interface IExternalClientsConfig {
     heartbeatManager: IHeartbeatManager;
     finalizeTasksAttempts: number;
   };
-  exportJobAndTaskTypes: {
-    jobType: string;
-    taskTilesType: string;
-    taskFinalizeType: string;
-  };
   httpRetry: IHttpRetryConfig;
   disableHttpClientLogs: boolean;
 }
+
+//consider changing from nested interface
+export interface IJobDefinitions {
+  jobs: {
+    export: { type: string };
+  };
+  tasks: {
+    export: { type: string };
+  };
+}
diff --git a/src/containerConfig.ts b/src/containerConfig.ts
@@ -9,7 +9,7 @@ import { createPackageRouterFactory, CREATE_PACKAGE_ROUTER_SYMBOL } from './crea
 import { InjectionObject, registerDependencies } from './common/dependencyRegistration';
 import { tasksRouterFactory, TASKS_ROUTER_SYMBOL } from './tasks/routes/tasksRouter';
 import { FinalizationManager, FINALIZATION_MANGER_SYMBOL } from './finalizationManager';
-import { IQueueConfig, IExternalClientsConfig } from './common/interfaces';
+import { IQueueConfig, IExternalClientsConfig, IJobDefinitions } from './common/interfaces';
 import { storageRouterFactory, STORAGE_ROUTER_SYMBOL } from './storage/routes/storageRouter';
 
 export interface RegisterOptions {
@@ -20,13 +20,14 @@ export interface RegisterOptions {
 export const registerExternalValues = (options?: RegisterOptions): DependencyContainer => {
   const loggerConfig = config.get<LoggerOptions>('telemetry.logger');
   const externalClientsConfig = config.get<IExternalClientsConfig>('externalClientsConfig');
+  const jobDefinitionsConfig = config.get<IJobDefinitions>('jobDefinitions');
   const queueConfig: IQueueConfig = {
     jobManagerBaseUrl: externalClientsConfig.clientsUrls.jobManager.url,
     heartbeatManagerBaseUrl: externalClientsConfig.clientsUrls.heartbeatManager.url,
     dequeueFinalizeIntervalMs: externalClientsConfig.clientsUrls.jobManager.dequeueFinalizeIntervalMs,
     heartbeatIntervalMs: externalClientsConfig.clientsUrls.heartbeatManager.heartbeatIntervalMs,
-    jobType: externalClientsConfig.exportJobAndTaskTypes.jobType,
-    tilesTaskType: externalClientsConfig.exportJobAndTaskTypes.taskFinalizeType,
+    jobType: jobDefinitionsConfig.jobs.export.type,
+    tilesTaskType: jobDefinitionsConfig.tasks.export.type,
   };
   // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
   const logger = jsLogger({ ...loggerConfig, prettyPrint: loggerConfig.prettyPrint, hooks: { logMethod } });

diff --git a/src/finalizationManager.ts b/src/finalizationManager.ts
@@ -29,7 +29,7 @@ export class FinalizationManager {
     @inject(JobManagerWrapper) private readonly jobManagerClient: JobManagerWrapper
   ) {
     this.expirationDays = config.get<number>('cleanupExpirationDays');
-    this.finalizeTaskType = config.get<string>('externalClientsConfig.exportJobAndTaskTypes.taskFinalizeType');
+    this.finalizeTaskType = config.get<string>('jobDefinitions.tasks.finalize.type');
     this.finalizeAttempts = config.get<number>('externalClientsConfig.clientsUrls.jobManager.finalizeTasksAttempts');
   }
 

diff --git a/src/tasks/models/tasksManager.ts b/src/tasks/models/tasksManager.ts
@@ -45,7 +45,7 @@ export class TasksManager {
   ) {
     this.gpkgsLocation = config.get<string>('gpkgsLocation');
     this.downloadServerUrl = config.get<string>('downloadServerUrl');
-    this.tilesJobType = config.get<string>('externalClientsConfig.exportJobAndTaskTypes.jobType');
+    this.tilesJobType = config.get<string>('jobDefinitions.jobs.export.type');
   }
 
   @withSpanAsyncV4

diff --git a/tests/integration/createPackage/tasks.spec.ts b/tests/integration/createPackage/tasks.spec.ts
@@ -36,7 +36,7 @@ describe('tasks', function () {
         {
           id: '0a5552f7-01eb-40af-a912-eed8fa9e1561',
           jobId: '0a5552f7-01eb-40af-a912-eed8fa9e1568',
-          type: 'rasterTilesExporterd',
+          type: 'export',
           description: '',
           parameters: {} as unknown as ITaskParameters,
           status: OperationStatus.IN_PROGRESS,

diff --git a/tests/integration/testContainerConfig.ts b/tests/integration/testContainerConfig.ts
@@ -4,18 +4,19 @@ import { container } from 'tsyringe';
 import { SERVICES } from '../../src/common/constants';
 import { configMock, registerDefaultConfig, getMock, hasMock } from '../mocks/config';
 import { InjectionObject } from '../../src/common/dependencyRegistration';
-import { IExternalClientsConfig, IQueueConfig } from '../../src/common/interfaces';
+import { IExternalClientsConfig, IJobDefinitions, IQueueConfig } from '../../src/common/interfaces';
 
 function getContainerConfig(): InjectionObject<unknown>[] {
   registerDefaultConfig();
   const externalClientsConfig = configMock.get<IExternalClientsConfig>('externalClientsConfig');
+  const jobDefinitionsConfig = configMock.get<IJobDefinitions>('jobDefinitions');
   const queueConfig: IQueueConfig = {
     jobManagerBaseUrl: externalClientsConfig.clientsUrls.jobManager.url,
     heartbeatManagerBaseUrl: externalClientsConfig.clientsUrls.heartbeatManager.url,
     dequeueFinalizeIntervalMs: externalClientsConfig.clientsUrls.jobManager.dequeueFinalizeIntervalMs,
     heartbeatIntervalMs: externalClientsConfig.clientsUrls.heartbeatManager.heartbeatIntervalMs,
-    jobType: externalClientsConfig.exportJobAndTaskTypes.jobType,
-    tilesTaskType: externalClientsConfig.exportJobAndTaskTypes.taskFinalizeType,
+    jobType: jobDefinitionsConfig.jobs.export.type,
+    tilesTaskType: jobDefinitionsConfig.tasks.export.type,
   };
 
   const testTracer = trace.getTracer('testTracer');

diff --git a/tests/mocks/config.ts b/tests/mocks/config.ts
@@ -20,6 +20,7 @@ const setConfigValues = (values: Record<string, unknown>): void => {
 
 const registerDefaultConfig = (): void => {
   const config = {
+    domain: 'RASTER',
     openapiConfig: {
       filePath: './openapi3.yaml',
       basePath: '/docs',
@@ -65,7 +66,6 @@ const registerDefaultConfig = (): void => {
       clientsUrls: {
         jobManager: {
           url: 'http://raster-catalog-manager',
-          jobDomain: 'RASTER',
           dequeueFinalizeIntervalMs: 1000,
           finalizeTasksAttempts: 5,
         },
@@ -77,11 +77,6 @@ const registerDefaultConfig = (): void => {
           heartbeatIntervalMs: 300,
         },
       },
-      exportJobAndTaskTypes: {
-        jobType: 'rasterTilesExporter',
-        taskTilesType: 'rasterTilesExporter',
-        taskFinalizeType: 'rasterTilesExporter',
-      },
       httpRetry: {
         attempts: 5,
         delay: 'exponential',
@@ -91,6 +86,21 @@ const registerDefaultConfig = (): void => {
       roiBufferMeter: 5,
       minContainedPercentage: 75,
     },
+    jobDefinitions: {
+      tasks: {
+        export: {
+          type: 'export',
+        },
+        finalize: {
+          type: 'finalize',
+        },
+      },
+      jobs: {
+        export: {
+          type: 'Export',
+        },
+      },
+    },
   };
   setConfigValues(config);
 };