feat: first commit for new cleanup managment logic (MAPCO-2877) (MAPCO-2878)

src/clients/jobManagerWrapper.ts

@@ -66,7 +66,6 @@ export class JobManagerWrapper extends JobManagerClient {
       resourceId: data.cswProductId,
       version: data.version,
       type: this.tilesJobType,
-      expirationDate,
       domain: this.jobDomain,
       parameters: {
         sanitizedBbox: data.sanitizedBbox,
@@ -122,7 +121,6 @@ export class JobManagerWrapper extends JobManagerClient {
       resourceId: data.cswProductId,
       version: data.version,
       type: this.tilesJobType,
-      expirationDate,
       domain: this.jobDomain,
       parameters: jobParameters,
       internalId: data.dbId,
@@ -272,11 +270,18 @@ export class JobManagerWrapper extends JobManagerClient {
 
     const job = await this.get<JobResponse | JobExportResponse | undefined>(getOrUpdateURL);
     if (job) {
-      const oldExpirationDate = new Date(job.expirationDate as Date);
+      const oldExpirationDate = new Date(job.parameters.cleanupData?.cleanupExpirationTimeUTC as Date);
       if (oldExpirationDate < newExpirationDate) {
-        this.logger.info({ jobId, oldExpirationDate, newExpirationDate }, 'Will execute update for expirationDate');
+        this.logger.info({ jobId, oldExpirationDate, newExpirationDate, msg: 'update expirationDate' });
         await this.put(getOrUpdateURL, {
-          expirationDate: newExpirationDate,
+          parameters: {
+            ...job.parameters,
+            cleanupData: {
+              ...job.parameters.cleanupData,
+              cleanupExpirationTimeUTC: newExpirationDate,
+              directoryPath: job.parameters.relativeDirectoryPath,
+            },
+          },
         });
       } else {
         const msg = 'Will not update expiration date, as current expiration date is later than current expiration date';

src/common/interfaces.ts

@@ -21,6 +21,11 @@ export interface IBaseCreatePackage {
   priority?: number;
 }
 
+export interface ICleanupData {
+  directoryPath?: string;
+  cleanupExpirationTimeUTC?: Date;
+}
+
 /**
  * @deprecated GetMap API - will be deprecated on future
  */
@@ -174,6 +179,7 @@ export interface IJobParameters {
   callbackParams?: ICallbackDataBase;
   fileName: string;
   gpkgEstimatedSize?: number;
+  cleanupData?: ICleanupData;
 }
 
 export interface IJobExportParameters {
@@ -185,6 +191,7 @@ export interface IJobExportParameters {
   callbackParams?: ICallbackExportResponse;
   fileNamesTemplates: ILinkDefinition;
   gpkgEstimatedSize?: number;
+  cleanupData?: ICleanupData;
 }
 
 export declare type MergerSourceType = 'S3' | 'GPKG' | 'FS';

src/createPackage/models/createPackageManager.ts

@@ -638,7 +638,6 @@ export class CreatePackageManager {
     const processingJob = (await this.jobManagerClient.findInProgressJob(dupParams)) ?? (await this.jobManagerClient.findPendingJob(dupParams));
     if (processingJob) {
       await this.updateCallbackURLs(processingJob, newCallbacks);
-      await this.jobManagerClient.validateAndUpdateExpiration(processingJob.id);
       return {
         id: processingJob.id,
         taskIds: (processingJob.tasks as unknown as IJobResponse<IJobParameters, ITaskParameters>[]).map((t) => t.id),
@@ -657,7 +656,6 @@ export class CreatePackageManager {
       (await this.jobManagerClient.findExportJob(OperationStatus.PENDING, dupParams, true));
     if (processingJob) {
       await this.updateExportCallbackURLs(processingJob, newCallbacks);
-      await this.jobManagerClient.validateAndUpdateExpiration(processingJob.id);
       return {
         id: processingJob.id,
         taskIds: (processingJob.tasks as unknown as IJobResponse<IJobExportParameters, ITaskParameters>[]).map((t) => t.id),

src/tasks/models/tasksManager.ts

@@ -12,6 +12,7 @@ import {
   ICallbackDataExportBase,
   ICallbackExportData,
   ICallbackExportResponse,
+  ICleanupData,
   IExportJobStatusResponse,
   IJobExportParameters,
   IJobParameters,
@@ -171,8 +172,10 @@ export class TasksManager {
       reason,
       /* eslint-disable-next-line @typescript-eslint/no-magic-numbers */
       percentage: isSuccess ? 100 : undefined,
-      expirationDate: expirationDate,
     };
+
+    const cleanupData: ICleanupData = this.generateCleanupEntity(job, expirationDate);
+
     try {
       this.logger.info({ jobId: job.id, msg: `Finalize Job` });
       const packageName = job.parameters.fileName;
@@ -181,7 +184,7 @@ export class TasksManager {
         await this.packageManager.createJsonMetadata(packageFullPath, job);
       }
       const callbackParams = await this.sendCallbacks(job, expirationDate, reason);
-      updateJobParams = { ...updateJobParams, parameters: { ...job.parameters, callbackParams } };
+      updateJobParams = { ...updateJobParams, parameters: { ...job.parameters, callbackParams, cleanupData } };
 
       this.logger.info({ jobId: job.id, status: isSuccess, msg: `Update Job status` });
       await this.jobManagerClient.updateJob(job.id, updateJobParams);
@@ -193,7 +196,7 @@ export class TasksManager {
         msg: `Could not finalize job, will updating to status failed`,
       });
       const callbackParams = await this.sendCallbacks(job, expirationDate, reason);
-      updateJobParams = { ...updateJobParams, status: OperationStatus.FAILED, parameters: { ...job.parameters, callbackParams } };
+      updateJobParams = { ...updateJobParams, status: OperationStatus.FAILED, parameters: { ...job.parameters, callbackParams, cleanupData } };
       await this.jobManagerClient.updateJob(job.id, updateJobParams);
     }
   }
@@ -204,8 +207,10 @@ export class TasksManager {
       /* eslint-disable-next-line @typescript-eslint/no-magic-numbers */
       percentage: isSuccess ? 100 : undefined,
       status: isSuccess ? OperationStatus.COMPLETED : OperationStatus.FAILED,
-      expirationDate: expirationDate,
     };
+
+    const cleanupData: ICleanupData = this.generateCleanupEntity(job, expirationDate);
+
     try {
       this.logger.info({ jobId: job.id, isSuccess, msg: `Finalize Job` });
       if (isSuccess) {
@@ -230,16 +235,27 @@ export class TasksManager {
         errorReason: reason,
       };
 
-      updateJobParams = { ...updateJobParams, parameters: { ...job.parameters, callbackParams } };
+      updateJobParams = { ...updateJobParams, parameters: { ...job.parameters, callbackParams, cleanupData } };
       this.logger.info({ finalizeStatus, jobId: job.id, msg: `Updating job finalizing status` });
     } catch (error) {
       this.logger.error({ jobId: job.id, err: error, reason: `${(error as Error).message}`, msg: `Could not finalize job` });
-      updateJobParams = { ...updateJobParams, reason: JSON.stringify(error as Error), status: OperationStatus.FAILED };
+      updateJobParams = {
+        ...updateJobParams,
+        reason: JSON.stringify(error as Error),
+        status: OperationStatus.FAILED,
+        parameters: { ...job.parameters, cleanupData },
+      };
     } finally {
       await this.jobManagerClient.updateJob(job.id, updateJobParams);
     }
   }
 
+  private generateCleanupEntity(job: JobResponse | JobExportResponse, expirationDate: Date): ICleanupData {
+    const cleanupData = { directoryPath: job.parameters.relativeDirectoryPath, cleanupExpirationTimeUTC: expirationDate };
+    this.logger.info({ jobId: job.id, cleanupData, msg: `Generated new cleanupData param for job parameters` });
+    return cleanupData;
+  }
+
   private async generateCallbackParam(job: JobExportResponse, expirationDate: Date, errorReason?: string): Promise<ICallbackDataExportBase> {
     let links: ILinkDefinition = { ...job.parameters.fileNamesTemplates }; // default file names in case of failure
     this.logger.info({ jobId: job.id, msg: `generate callback body for job: ${job.id}` });

tests/mocks/data.ts

@@ -277,6 +277,10 @@ const completedJob: IJobResponse<IJobParameters, ITaskParameters> = {
       expirationTime: new Date(),
       targetResolution: 0.0439453125,
     },
+    cleanupData: {
+      directoryPath: 'test',
+      cleanupExpirationTimeUTC: new Date(),
+    },
 
     targetResolution: 0.0439453125,
   },
@@ -693,6 +697,10 @@ const completedExportJob: IJobResponse<IJobExportParameters, ITaskParameters> =
       expirationTime: new Date(),
       recordCatalogId: 'b0b19b88-aecb-4e74-b694-dfa7eada8bf7',
     },
+    cleanupData: {
+      directoryPath: 'b0b19b88-aecb-4e74-b694-dfa7eada8bf7',
+      cleanupExpirationTimeUTC: new Date(),
+    },
     gpkgEstimatedSize: 187500,
     fileNamesTemplates: {
       dataURI: 'Orthophoto_testArea_1_0_2023_02_28T15_09_50_924Z.gpkg',

tests/mocks/data/mockJob.ts

@@ -88,7 +88,7 @@ export const mockCompletedJob: JobExportResponse = {
   reason: '',
   isCleaned: false,
   priority: 0,
-  expirationDate: new Date(),
+  expirationDate: undefined,
   internalId: '880a9316-0f10-4874-92e2-a62d587a1169',
   producerName: undefined,
   productName: 'test',

tests/unit/clients/jobManagerClient.spec.ts

@@ -6,6 +6,7 @@ import { JobResponse, ICreateJobResponse as JobInProgressResponse, JobExportDupl
 import { configMock, registerDefaultConfig } from '../../mocks/config';
 import {
   completedExportJob,
+  completedJob,
   fc1,
   inProgressExportJob,
   inProgressJob,
@@ -131,14 +132,20 @@ describe('JobManagerClient', () => {
         putFun = jest.fn();
         (jobManagerClient as unknown as { put: unknown }).put = putFun.mockResolvedValue(undefined);
         const jobManager = jobManagerClient as unknown as { get: unknown };
-        jobManager.get = get.mockResolvedValue({ ...inProgressJob, expirationDate: testExpirationDate });
+        jobManager.get = get.mockResolvedValue({
+          ...completedJob,
+          parameters: {
+            ...completedJob.parameters,
+            cleanupData: { ...completedJob.parameters.cleanupData, cleanupExpirationTimeUTC: testExpirationDate },
+          },
+        });
 
-        await jobManagerClient.validateAndUpdateExpiration(inProgressJob.id);
+        await jobManagerClient.validateAndUpdateExpiration(completedJob.id);
 
         expect(get).toHaveBeenCalledTimes(1);
         expect(putFun).toHaveBeenCalledTimes(1);
         // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access
-        const expirationParamCall: Date = putFun.mock.calls[0][1].expirationDate;
+        const expirationParamCall: Date = putFun.mock.calls[0][1].parameters.cleanupData.cleanupExpirationTimeUTC;
         expirationParamCall.setSeconds(0, 0);
         expect(JSON.stringify(expirationParamCall)).toBe(JSON.stringify(expectedNewExpirationDate));
       });
@@ -272,7 +279,7 @@ describe('JobManagerClient', () => {
         });
       });
       describe('Update Jobs', () => {
-        it('should successfully update running Export job (already in progress) expirationDate (old expirationDate lower)', async () => {
+        it('should successfully update completed Export job (Naive cache) expirationDate (old expirationDate lower)', async () => {
           const expirationDays: number = configMock.get('jobManager.expirationDays');
           const testExpirationDate = getUTCDate();
           const expectedNewExpirationDate = getUTCDate();
@@ -284,19 +291,19 @@ describe('JobManagerClient', () => {
           putFun = jest.fn();
           (jobManagerClient as unknown as { put: unknown }).put = putFun.mockResolvedValue(undefined);
           const jobManager = jobManagerClient as unknown as { get: unknown };
-          jobManager.get = get.mockResolvedValue({ ...inProgressExportJob, expirationDate: testExpirationDate });
+          jobManager.get = get.mockResolvedValue({ ...completedExportJob });
 
-          await jobManagerClient.validateAndUpdateExpiration(inProgressExportJob.id);
+          await jobManagerClient.validateAndUpdateExpiration(completedExportJob.id);
 
           expect(get).toHaveBeenCalledTimes(1);
           expect(putFun).toHaveBeenCalledTimes(1);
           // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access
-          const expirationParamCall: Date = putFun.mock.calls[0][1].expirationDate;
+          const expirationParamCall: Date = putFun.mock.calls[0][1].parameters.cleanupData.cleanupExpirationTimeUTC;
           expirationParamCall.setSeconds(0, 0);
           expect(JSON.stringify(expirationParamCall)).toBe(JSON.stringify(expectedNewExpirationDate));
         });
 
-        it('should not update running Export job (already in progress) expirationDate (old expirationDate higher)', async () => {
+        it('should not update completed Export job (naive cache) expirationDate (old expirationDate higher)', async () => {
           const expirationDays: number = configMock.get('jobManager.expirationDays');
           const testExpirationDate = getUTCDate();
           const expectedNewExpirationDate = getUTCDate();
@@ -308,9 +315,15 @@ describe('JobManagerClient', () => {
           putFun = jest.fn();
           (jobManagerClient as unknown as { put: unknown }).put = putFun.mockResolvedValue(undefined);
           const jobManager = jobManagerClient as unknown as { get: unknown };
-          jobManager.get = get.mockResolvedValue({ ...inProgressExportJob, expirationDate: testExpirationDate });
+          jobManager.get = get.mockResolvedValue({
+            ...completedExportJob,
+            parameters: {
+              ...completedExportJob.parameters,
+              cleanupData: { ...completedExportJob.parameters.cleanupData, cleanupExpirationTimeUTC: testExpirationDate },
+            },
+          });
 
-          await jobManagerClient.validateAndUpdateExpiration(inProgressExportJob.id);
+          await jobManagerClient.validateAndUpdateExpiration(completedExportJob.id);
 
           expect(get).toHaveBeenCalledTimes(1);
           expect(putFun).toHaveBeenCalledTimes(0);

tests/unit/createPackage/models/tasksModel.spec.ts

@@ -451,10 +451,10 @@ describe('TasksManager', () => {
           reason: undefined,
           percentage: 100,
           status: OperationStatus.COMPLETED,
-          expirationDate: expirationTime,
           parameters: {
             ...mockCompletedJob.parameters,
             callbackParams: { ...expectedCallbackParamData, roi: mockCompletedJob.parameters.roi, status: OperationStatus.COMPLETED },
+            cleanupData: { directoryPath: mockCompletedJob.parameters.relativeDirectoryPath, cleanupExpirationTimeUTC: expirationTime },
           },
         };
         const action = async () => tasksManager.finalizeExportJob(mockCompletedJob, expirationTime);
@@ -495,10 +495,10 @@ describe('TasksManager', () => {
           reason: undefined,
           percentage: 100,
           status: OperationStatus.COMPLETED,
-          expirationDate: expirationTime,
           parameters: {
             ...mockCompletedJob.parameters,
             callbackParams: { ...expectedCallbackParamData, roi: mockCompletedJob.parameters.roi, status: OperationStatus.COMPLETED },
+            cleanupData: { directoryPath: mockCompletedJob.parameters.relativeDirectoryPath, cleanupExpirationTimeUTC: expirationTime },
           },
         };
         const action = async () => tasksManager.finalizeExportJob(mockCompletedJob, expirationTime);
@@ -526,7 +526,10 @@ describe('TasksManager', () => {
           reason: JSON.stringify({ message: 'failed generate metadata.json' }),
           percentage: 100,
           status: OperationStatus.FAILED,
-          expirationDate: expirationTime,
+          parameters: {
+            ...mockCompletedJob.parameters,
+            cleanupData: { directoryPath: mockCompletedJob.parameters.relativeDirectoryPath, cleanupExpirationTimeUTC: expirationTime },
+          },
         };
         const action = async () => tasksManager.finalizeExportJob(mockCompletedJob, expirationTime);
         await expect(action()).resolves.not.toThrow();
@@ -560,10 +563,10 @@ describe('TasksManager', () => {
           reason: 'testError',
           percentage: undefined,
           status: OperationStatus.FAILED,
-          expirationDate: expirationTime,
           parameters: {
             ...mockCompletedJob.parameters,
             callbackParams: { ...expectedCallbackParamData, roi: mockCompletedJob.parameters.roi, status: OperationStatus.FAILED },
+            cleanupData: { directoryPath: mockCompletedJob.parameters.relativeDirectoryPath, cleanupExpirationTimeUTC: expirationTime },
           },
         };
         const action = async () => tasksManager.finalizeExportJob(mockCompletedJob, expirationTime, false, 'testError');