From d3b47b767067c598a746b42f43bd54e92cab0bf7 Mon Sep 17 00:00:00 2001 From: ronenkapelian <72082238+ronenkapelian@users.noreply.github.com> Date: Sun, 12 Mar 2023 11:40:48 +0200 Subject: [PATCH] feat: first commit for new cleanup managment logic (MAPCO-2877) (MAPCO-2878) (#56) * feat: first commit for new cleanup managment logic * feat: test updating * feat: fix tests * fix: naming for cleanupExpirationUTC --- src/clients/jobManagerWrapper.ts | 15 ++++++--- src/common/interfaces.ts | 7 ++++ .../models/createPackageManager.ts | 2 -- src/tasks/models/tasksManager.ts | 28 ++++++++++++---- tests/mocks/data.ts | 8 +++++ tests/mocks/data/mockJob.ts | 2 +- tests/unit/clients/jobManagerClient.spec.ts | 33 +++++++++++++------ .../createPackage/models/tasksModel.spec.ts | 11 ++++--- 8 files changed, 78 insertions(+), 28 deletions(-) diff --git a/src/clients/jobManagerWrapper.ts b/src/clients/jobManagerWrapper.ts index 4923355..3c313be 100644 --- a/src/clients/jobManagerWrapper.ts +++ b/src/clients/jobManagerWrapper.ts @@ -66,7 +66,6 @@ export class JobManagerWrapper extends JobManagerClient { resourceId: data.cswProductId, version: data.version, type: this.tilesJobType, - expirationDate, domain: this.jobDomain, parameters: { sanitizedBbox: data.sanitizedBbox, @@ -122,7 +121,6 @@ export class JobManagerWrapper extends JobManagerClient { resourceId: data.cswProductId, version: data.version, type: this.tilesJobType, - expirationDate, domain: this.jobDomain, parameters: jobParameters, internalId: data.dbId, @@ -272,11 +270,18 @@ export class JobManagerWrapper extends JobManagerClient { const job = await this.get(getOrUpdateURL); if (job) { - const oldExpirationDate = new Date(job.expirationDate as Date); + const oldExpirationDate = new Date(job.parameters.cleanupData?.cleanupExpirationTimeUTC as Date); if (oldExpirationDate < newExpirationDate) { - this.logger.info({ jobId, oldExpirationDate, newExpirationDate }, 'Will execute update for expirationDate'); + this.logger.info({ jobId, oldExpirationDate, newExpirationDate, msg: 'update expirationDate' }); await this.put(getOrUpdateURL, { - expirationDate: newExpirationDate, + parameters: { + ...job.parameters, + cleanupData: { + ...job.parameters.cleanupData, + cleanupExpirationTimeUTC: newExpirationDate, + directoryPath: job.parameters.relativeDirectoryPath, + }, + }, }); } else { const msg = 'Will not update expiration date, as current expiration date is later than current expiration date'; diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index daa9e79..ab3071b 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -21,6 +21,11 @@ export interface IBaseCreatePackage { priority?: number; } +export interface ICleanupData { + directoryPath?: string; + cleanupExpirationTimeUTC?: Date; +} + /** * @deprecated GetMap API - will be deprecated on future */ @@ -174,6 +179,7 @@ export interface IJobParameters { callbackParams?: ICallbackDataBase; fileName: string; gpkgEstimatedSize?: number; + cleanupData?: ICleanupData; } export interface IJobExportParameters { @@ -185,6 +191,7 @@ export interface IJobExportParameters { callbackParams?: ICallbackExportResponse; fileNamesTemplates: ILinkDefinition; gpkgEstimatedSize?: number; + cleanupData?: ICleanupData; } export declare type MergerSourceType = 'S3' | 'GPKG' | 'FS'; diff --git a/src/createPackage/models/createPackageManager.ts b/src/createPackage/models/createPackageManager.ts index 1345c98..6711208 100644 --- a/src/createPackage/models/createPackageManager.ts +++ b/src/createPackage/models/createPackageManager.ts @@ -638,7 +638,6 @@ export class CreatePackageManager { const processingJob = (await this.jobManagerClient.findInProgressJob(dupParams)) ?? (await this.jobManagerClient.findPendingJob(dupParams)); if (processingJob) { await this.updateCallbackURLs(processingJob, newCallbacks); - await this.jobManagerClient.validateAndUpdateExpiration(processingJob.id); return { id: processingJob.id, taskIds: (processingJob.tasks as unknown as IJobResponse[]).map((t) => t.id), @@ -657,7 +656,6 @@ export class CreatePackageManager { (await this.jobManagerClient.findExportJob(OperationStatus.PENDING, dupParams, true)); if (processingJob) { await this.updateExportCallbackURLs(processingJob, newCallbacks); - await this.jobManagerClient.validateAndUpdateExpiration(processingJob.id); return { id: processingJob.id, taskIds: (processingJob.tasks as unknown as IJobResponse[]).map((t) => t.id), diff --git a/src/tasks/models/tasksManager.ts b/src/tasks/models/tasksManager.ts index 6c27669..1c98765 100644 --- a/src/tasks/models/tasksManager.ts +++ b/src/tasks/models/tasksManager.ts @@ -12,6 +12,7 @@ import { ICallbackDataExportBase, ICallbackExportData, ICallbackExportResponse, + ICleanupData, IExportJobStatusResponse, IJobExportParameters, IJobParameters, @@ -171,8 +172,10 @@ export class TasksManager { reason, /* eslint-disable-next-line @typescript-eslint/no-magic-numbers */ percentage: isSuccess ? 100 : undefined, - expirationDate: expirationDate, }; + + const cleanupData: ICleanupData = this.generateCleanupEntity(job, expirationDate); + try { this.logger.info({ jobId: job.id, msg: `Finalize Job` }); const packageName = job.parameters.fileName; @@ -181,7 +184,7 @@ export class TasksManager { await this.packageManager.createJsonMetadata(packageFullPath, job); } const callbackParams = await this.sendCallbacks(job, expirationDate, reason); - updateJobParams = { ...updateJobParams, parameters: { ...job.parameters, callbackParams } }; + updateJobParams = { ...updateJobParams, parameters: { ...job.parameters, callbackParams, cleanupData } }; this.logger.info({ jobId: job.id, status: isSuccess, msg: `Update Job status` }); await this.jobManagerClient.updateJob(job.id, updateJobParams); @@ -193,7 +196,7 @@ export class TasksManager { msg: `Could not finalize job, will updating to status failed`, }); const callbackParams = await this.sendCallbacks(job, expirationDate, reason); - updateJobParams = { ...updateJobParams, status: OperationStatus.FAILED, parameters: { ...job.parameters, callbackParams } }; + updateJobParams = { ...updateJobParams, status: OperationStatus.FAILED, parameters: { ...job.parameters, callbackParams, cleanupData } }; await this.jobManagerClient.updateJob(job.id, updateJobParams); } } @@ -204,8 +207,10 @@ export class TasksManager { /* eslint-disable-next-line @typescript-eslint/no-magic-numbers */ percentage: isSuccess ? 100 : undefined, status: isSuccess ? OperationStatus.COMPLETED : OperationStatus.FAILED, - expirationDate: expirationDate, }; + + const cleanupData: ICleanupData = this.generateCleanupEntity(job, expirationDate); + try { this.logger.info({ jobId: job.id, isSuccess, msg: `Finalize Job` }); if (isSuccess) { @@ -230,16 +235,27 @@ export class TasksManager { errorReason: reason, }; - updateJobParams = { ...updateJobParams, parameters: { ...job.parameters, callbackParams } }; + updateJobParams = { ...updateJobParams, parameters: { ...job.parameters, callbackParams, cleanupData } }; this.logger.info({ finalizeStatus, jobId: job.id, msg: `Updating job finalizing status` }); } catch (error) { this.logger.error({ jobId: job.id, err: error, reason: `${(error as Error).message}`, msg: `Could not finalize job` }); - updateJobParams = { ...updateJobParams, reason: JSON.stringify(error as Error), status: OperationStatus.FAILED }; + updateJobParams = { + ...updateJobParams, + reason: JSON.stringify(error as Error), + status: OperationStatus.FAILED, + parameters: { ...job.parameters, cleanupData }, + }; } finally { await this.jobManagerClient.updateJob(job.id, updateJobParams); } } + private generateCleanupEntity(job: JobResponse | JobExportResponse, expirationDate: Date): ICleanupData { + const cleanupData = { directoryPath: job.parameters.relativeDirectoryPath, cleanupExpirationTimeUTC: expirationDate }; + this.logger.info({ jobId: job.id, cleanupData, msg: `Generated new cleanupData param for job parameters` }); + return cleanupData; + } + private async generateCallbackParam(job: JobExportResponse, expirationDate: Date, errorReason?: string): Promise { let links: ILinkDefinition = { ...job.parameters.fileNamesTemplates }; // default file names in case of failure this.logger.info({ jobId: job.id, msg: `generate callback body for job: ${job.id}` }); diff --git a/tests/mocks/data.ts b/tests/mocks/data.ts index f9825be..0a6ae53 100644 --- a/tests/mocks/data.ts +++ b/tests/mocks/data.ts @@ -277,6 +277,10 @@ const completedJob: IJobResponse = { expirationTime: new Date(), targetResolution: 0.0439453125, }, + cleanupData: { + directoryPath: 'test', + cleanupExpirationTimeUTC: new Date(), + }, targetResolution: 0.0439453125, }, @@ -693,6 +697,10 @@ const completedExportJob: IJobResponse = expirationTime: new Date(), recordCatalogId: 'b0b19b88-aecb-4e74-b694-dfa7eada8bf7', }, + cleanupData: { + directoryPath: 'b0b19b88-aecb-4e74-b694-dfa7eada8bf7', + cleanupExpirationTimeUTC: new Date(), + }, gpkgEstimatedSize: 187500, fileNamesTemplates: { dataURI: 'Orthophoto_testArea_1_0_2023_02_28T15_09_50_924Z.gpkg', diff --git a/tests/mocks/data/mockJob.ts b/tests/mocks/data/mockJob.ts index 66200ce..bfcb0d3 100644 --- a/tests/mocks/data/mockJob.ts +++ b/tests/mocks/data/mockJob.ts @@ -88,7 +88,7 @@ export const mockCompletedJob: JobExportResponse = { reason: '', isCleaned: false, priority: 0, - expirationDate: new Date(), + expirationDate: undefined, internalId: '880a9316-0f10-4874-92e2-a62d587a1169', producerName: undefined, productName: 'test', diff --git a/tests/unit/clients/jobManagerClient.spec.ts b/tests/unit/clients/jobManagerClient.spec.ts index 003bbe7..852fc3a 100644 --- a/tests/unit/clients/jobManagerClient.spec.ts +++ b/tests/unit/clients/jobManagerClient.spec.ts @@ -6,6 +6,7 @@ import { JobResponse, ICreateJobResponse as JobInProgressResponse, JobExportDupl import { configMock, registerDefaultConfig } from '../../mocks/config'; import { completedExportJob, + completedJob, fc1, inProgressExportJob, inProgressJob, @@ -131,14 +132,20 @@ describe('JobManagerClient', () => { putFun = jest.fn(); (jobManagerClient as unknown as { put: unknown }).put = putFun.mockResolvedValue(undefined); const jobManager = jobManagerClient as unknown as { get: unknown }; - jobManager.get = get.mockResolvedValue({ ...inProgressJob, expirationDate: testExpirationDate }); + jobManager.get = get.mockResolvedValue({ + ...completedJob, + parameters: { + ...completedJob.parameters, + cleanupData: { ...completedJob.parameters.cleanupData, cleanupExpirationTimeUTC: testExpirationDate }, + }, + }); - await jobManagerClient.validateAndUpdateExpiration(inProgressJob.id); + await jobManagerClient.validateAndUpdateExpiration(completedJob.id); expect(get).toHaveBeenCalledTimes(1); expect(putFun).toHaveBeenCalledTimes(1); // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access - const expirationParamCall: Date = putFun.mock.calls[0][1].expirationDate; + const expirationParamCall: Date = putFun.mock.calls[0][1].parameters.cleanupData.cleanupExpirationTimeUTC; expirationParamCall.setSeconds(0, 0); expect(JSON.stringify(expirationParamCall)).toBe(JSON.stringify(expectedNewExpirationDate)); }); @@ -272,7 +279,7 @@ describe('JobManagerClient', () => { }); }); describe('Update Jobs', () => { - it('should successfully update running Export job (already in progress) expirationDate (old expirationDate lower)', async () => { + it('should successfully update completed Export job (Naive cache) expirationDate (old expirationDate lower)', async () => { const expirationDays: number = configMock.get('jobManager.expirationDays'); const testExpirationDate = getUTCDate(); const expectedNewExpirationDate = getUTCDate(); @@ -284,19 +291,19 @@ describe('JobManagerClient', () => { putFun = jest.fn(); (jobManagerClient as unknown as { put: unknown }).put = putFun.mockResolvedValue(undefined); const jobManager = jobManagerClient as unknown as { get: unknown }; - jobManager.get = get.mockResolvedValue({ ...inProgressExportJob, expirationDate: testExpirationDate }); + jobManager.get = get.mockResolvedValue({ ...completedExportJob }); - await jobManagerClient.validateAndUpdateExpiration(inProgressExportJob.id); + await jobManagerClient.validateAndUpdateExpiration(completedExportJob.id); expect(get).toHaveBeenCalledTimes(1); expect(putFun).toHaveBeenCalledTimes(1); // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access - const expirationParamCall: Date = putFun.mock.calls[0][1].expirationDate; + const expirationParamCall: Date = putFun.mock.calls[0][1].parameters.cleanupData.cleanupExpirationTimeUTC; expirationParamCall.setSeconds(0, 0); expect(JSON.stringify(expirationParamCall)).toBe(JSON.stringify(expectedNewExpirationDate)); }); - it('should not update running Export job (already in progress) expirationDate (old expirationDate higher)', async () => { + it('should not update completed Export job (naive cache) expirationDate (old expirationDate higher)', async () => { const expirationDays: number = configMock.get('jobManager.expirationDays'); const testExpirationDate = getUTCDate(); const expectedNewExpirationDate = getUTCDate(); @@ -308,9 +315,15 @@ describe('JobManagerClient', () => { putFun = jest.fn(); (jobManagerClient as unknown as { put: unknown }).put = putFun.mockResolvedValue(undefined); const jobManager = jobManagerClient as unknown as { get: unknown }; - jobManager.get = get.mockResolvedValue({ ...inProgressExportJob, expirationDate: testExpirationDate }); + jobManager.get = get.mockResolvedValue({ + ...completedExportJob, + parameters: { + ...completedExportJob.parameters, + cleanupData: { ...completedExportJob.parameters.cleanupData, cleanupExpirationTimeUTC: testExpirationDate }, + }, + }); - await jobManagerClient.validateAndUpdateExpiration(inProgressExportJob.id); + await jobManagerClient.validateAndUpdateExpiration(completedExportJob.id); expect(get).toHaveBeenCalledTimes(1); expect(putFun).toHaveBeenCalledTimes(0); diff --git a/tests/unit/createPackage/models/tasksModel.spec.ts b/tests/unit/createPackage/models/tasksModel.spec.ts index 8582e45..8c45439 100644 --- a/tests/unit/createPackage/models/tasksModel.spec.ts +++ b/tests/unit/createPackage/models/tasksModel.spec.ts @@ -451,10 +451,10 @@ describe('TasksManager', () => { reason: undefined, percentage: 100, status: OperationStatus.COMPLETED, - expirationDate: expirationTime, parameters: { ...mockCompletedJob.parameters, callbackParams: { ...expectedCallbackParamData, roi: mockCompletedJob.parameters.roi, status: OperationStatus.COMPLETED }, + cleanupData: { directoryPath: mockCompletedJob.parameters.relativeDirectoryPath, cleanupExpirationTimeUTC: expirationTime }, }, }; const action = async () => tasksManager.finalizeExportJob(mockCompletedJob, expirationTime); @@ -495,10 +495,10 @@ describe('TasksManager', () => { reason: undefined, percentage: 100, status: OperationStatus.COMPLETED, - expirationDate: expirationTime, parameters: { ...mockCompletedJob.parameters, callbackParams: { ...expectedCallbackParamData, roi: mockCompletedJob.parameters.roi, status: OperationStatus.COMPLETED }, + cleanupData: { directoryPath: mockCompletedJob.parameters.relativeDirectoryPath, cleanupExpirationTimeUTC: expirationTime }, }, }; const action = async () => tasksManager.finalizeExportJob(mockCompletedJob, expirationTime); @@ -526,7 +526,10 @@ describe('TasksManager', () => { reason: JSON.stringify({ message: 'failed generate metadata.json' }), percentage: 100, status: OperationStatus.FAILED, - expirationDate: expirationTime, + parameters: { + ...mockCompletedJob.parameters, + cleanupData: { directoryPath: mockCompletedJob.parameters.relativeDirectoryPath, cleanupExpirationTimeUTC: expirationTime }, + }, }; const action = async () => tasksManager.finalizeExportJob(mockCompletedJob, expirationTime); await expect(action()).resolves.not.toThrow(); @@ -560,10 +563,10 @@ describe('TasksManager', () => { reason: 'testError', percentage: undefined, status: OperationStatus.FAILED, - expirationDate: expirationTime, parameters: { ...mockCompletedJob.parameters, callbackParams: { ...expectedCallbackParamData, roi: mockCompletedJob.parameters.roi, status: OperationStatus.FAILED }, + cleanupData: { directoryPath: mockCompletedJob.parameters.relativeDirectoryPath, cleanupExpirationTimeUTC: expirationTime }, }, }; const action = async () => tasksManager.finalizeExportJob(mockCompletedJob, expirationTime, false, 'testError');