From 18294638fa17fab711c36a1c2b617296fd4daa2b Mon Sep 17 00:00:00 2001 From: almog8k Date: Sun, 20 Oct 2024 14:37:41 +0300 Subject: [PATCH 1/9] feat: update job init task hanling --- .vscode/settings.json | 3 +- package-lock.json | 3 +- package.json | 3 +- src/job/models/updateJobHandler.ts | 65 +++++++++++-- src/utils/zod/schemas/jobParametersSchema.ts | 9 ++ .../updateJobHandler/updateJobHandler.spec.ts | 92 +++++++++++++++++++ .../updateJobHandler/updateJobHandlerSetup.ts | 52 +++++++++++ tests/unit/mocks/jobsMockData.ts | 15 ++- 8 files changed, 230 insertions(+), 12 deletions(-) create mode 100644 src/utils/zod/schemas/jobParametersSchema.ts create mode 100644 tests/unit/job/updateJobHandler/updateJobHandler.spec.ts create mode 100644 tests/unit/job/updateJobHandler/updateJobHandlerSetup.ts diff --git a/.vscode/settings.json b/.vscode/settings.json index 995b79f..3d240ee 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,6 @@ { "typescript.tsdk": "node_modules/typescript/lib", "files.eol": "\n", - "cSpell.words": ["geoserver", "Mapproxy"] + "cSpell.words": ["geoserver", "Mapproxy"], + "liveServer.settings.port": 5501 } diff --git a/package-lock.json b/package-lock.json index 5055e27..39f85b3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -32,7 +32,8 @@ "lodash.has": "^4.5.2", "prom-client": "^15.1.2", "reflect-metadata": "^0.1.13", - "tsyringe": "^4.8.0" + "tsyringe": "^4.8.0", + "zod": "^3.23.8" }, "devDependencies": { "@commitlint/cli": "^17.6.6", diff --git a/package.json b/package.json index 391e6f7..459dfc8 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,8 @@ "lodash.has": "^4.5.2", "prom-client": "^15.1.2", "reflect-metadata": "^0.1.13", - "tsyringe": "^4.8.0" + "tsyringe": "^4.8.0", + "zod": "^3.23.8" }, "devDependencies": { "@commitlint/cli": "^17.6.6", diff --git a/src/job/models/updateJobHandler.ts b/src/job/models/updateJobHandler.ts index 707ab74..105488d 100644 --- a/src/job/models/updateJobHandler.ts +++ b/src/job/models/updateJobHandler.ts @@ -1,18 +1,64 @@ +import { ZodError } from 'zod'; import { inject, injectable } from 'tsyringe'; import { Logger } from '@map-colonies/js-logger'; -import { IJobResponse, ITaskResponse } from '@map-colonies/mc-priority-queue'; -import { UpdateRasterLayer } from '@map-colonies/mc-model-types'; -import { IJobHandler } from '../../common/interfaces'; +import { OperationStatus, TaskHandler as QueueClient, IJobResponse, ITaskResponse } from '@map-colonies/mc-priority-queue'; +import { IngestionUpdateJobParams, UpdateRasterLayer } from '@map-colonies/mc-model-types'; +import { Grid, IConfig, IJobHandler, MergeTilesTaskParams } from '../../common/interfaces'; import { SERVICES } from '../../common/constants'; +import { UpdateAdditionalParams, updateAdditionalParamsSchema } from '../../utils/zod/schemas/jobParametersSchema'; +import { TileMergeTaskManager } from '../../task/models/tileMergeTaskManager'; @injectable() export class UpdateJobHandler implements IJobHandler { - public constructor(@inject(SERVICES.LOGGER) private readonly logger: Logger) {} + private readonly isNewTarget: boolean; + public constructor( + @inject(SERVICES.LOGGER) private readonly logger: Logger, + @inject(SERVICES.CONFIG) private readonly config: IConfig, + @inject(TileMergeTaskManager) private readonly taskBuilder: TileMergeTaskManager, + @inject(SERVICES.QUEUE_CLIENT) private readonly queueClient: QueueClient + ) { + this.isNewTarget = config.get('jobManagement.ingestion.tasks.tilesMerging.useNewTargetFlagInUpdate'); + } - public async handleJobInit(job: IJobResponse, taskId: string): Promise { + public async handleJobInit(job: IJobResponse, taskId: string): Promise { const logger = this.logger.child({ jobId: job.id, taskId }); - logger.info({ msg: `handling ${job.type} job with "init" task` }); - await Promise.reject('not implemented'); + try { + logger.info({ msg: `handling ${job.type} job with "init" task` }); + const { inputFiles, partData, additionalParams } = job.parameters; + + const validAdditionalParams = this.validateAdditionalParams(additionalParams); + + const taskBuildParams: MergeTilesTaskParams = { + inputFiles, + taskMetadata: { + layerRelativePath: `${job.internalId}/${validAdditionalParams.displayPath}`, + tileOutputFormat: validAdditionalParams.tileOutputFormat, + isNewTarget: this.isNewTarget, + grid: Grid.TWO_ON_ONE, + }, + partsData: partData, + }; + + logger.info({ msg: 'building tasks' }); + const mergeTasks = this.taskBuilder.buildTasks(taskBuildParams); + + logger.info({ msg: 'pushing tasks' }); + await this.taskBuilder.pushTasks(job.id, mergeTasks); + + logger.info({ msg: 'Acking task' }); + await this.queueClient.ack(job.id, taskId); + } catch (err) { + if (err instanceof ZodError) { + const errorMsg = `Failed to validate additionalParams: ${err.message}`; + logger.error({ msg: errorMsg, err }); + await this.queueClient.reject(job.id, taskId, false, err.message); + return await this.queueClient.jobManagerClient.updateJob(job.id, { status: OperationStatus.FAILED, reason: errorMsg }); + } + if (err instanceof Error) { + logger.error({ msg: 'Failed to handle job init', error: err }); + await this.queueClient.reject(job.id, taskId, true, err.message); + } + } } public async handleJobFinalize(job: IJobResponse, task: ITaskResponse): Promise { @@ -20,4 +66,9 @@ export class UpdateJobHandler implements IJobHandler { logger.info({ msg: `handling ${job.type} job with "finalize" task` }); await Promise.reject('not implemented'); } + + private validateAdditionalParams(additionalParams: Record): UpdateAdditionalParams { + const validatedParams = updateAdditionalParamsSchema.parse(additionalParams); + return validatedParams; + } } diff --git a/src/utils/zod/schemas/jobParametersSchema.ts b/src/utils/zod/schemas/jobParametersSchema.ts new file mode 100644 index 0000000..5682645 --- /dev/null +++ b/src/utils/zod/schemas/jobParametersSchema.ts @@ -0,0 +1,9 @@ +import { TileOutputFormat } from '@map-colonies/mc-model-types'; +import { z } from 'zod'; + +export const updateAdditionalParamsSchema = z.object({ + displayPath: z.string().uuid(), + tileOutputFormat: z.nativeEnum(TileOutputFormat), +}); + +export type UpdateAdditionalParams = z.infer; diff --git a/tests/unit/job/updateJobHandler/updateJobHandler.spec.ts b/tests/unit/job/updateJobHandler/updateJobHandler.spec.ts new file mode 100644 index 0000000..afbd525 --- /dev/null +++ b/tests/unit/job/updateJobHandler/updateJobHandler.spec.ts @@ -0,0 +1,92 @@ +/* eslint-disable @typescript-eslint/unbound-method */ +import { ZodError } from 'zod'; +import { OperationStatus } from '@map-colonies/mc-priority-queue'; +import { TileOutputFormat } from '@map-colonies/mc-model-types'; +import { Grid, IMergeTaskParameters } from '../../../../src/common/interfaces'; +import { finalizeTaskForIngestionUpdate } from '../../mocks/tasksMockData'; +import { updateAdditionalParamsSchema } from '../../../../src/utils/zod/schemas/jobParametersSchema'; +import { registerDefaultConfig } from '../../mocks/configMock'; +import { ingestionUpdateJob } from '../../mocks/jobsMockData'; +import { setupUpdateJobHandlerTest } from './updateJobHandlerSetup'; + +describe('updateJobHandler', () => { + beforeEach(() => { + jest.resetAllMocks(); + registerDefaultConfig(); + }); + + describe('handleJobInit', () => { + it('should handle job init successfully', async () => { + const { updateJobHandler, queueClientMock, taskBuilderMock } = setupUpdateJobHandlerTest(); + const job = ingestionUpdateJob; + const taskId = '291bf779-efe0-42bd-8357-aaede47e4d37'; + + const additionalParams = updateAdditionalParamsSchema.parse(job.parameters.additionalParams); + + const taskBuildParams = { + inputFiles: job.parameters.inputFiles, + taskMetadata: { + layerRelativePath: `${job.internalId}/${additionalParams.displayPath}`, + tileOutputFormat: additionalParams.tileOutputFormat, + isNewTarget: true, + grid: Grid.TWO_ON_ONE, + }, + partsData: job.parameters.partData, + }; + + const mergeTasks: AsyncGenerator = (async function* () {})(); + + taskBuilderMock.buildTasks.mockReturnValue(mergeTasks); + taskBuilderMock.pushTasks.mockResolvedValue(undefined); + queueClientMock.ack.mockResolvedValue(undefined); + + await updateJobHandler.handleJobInit(job, taskId); + + expect(taskBuilderMock.buildTasks).toHaveBeenCalledWith(taskBuildParams); + expect(taskBuilderMock.pushTasks).toHaveBeenCalledWith(job.id, mergeTasks); + expect(queueClientMock.ack).toHaveBeenCalledWith(job.id, taskId); + }); + + it('should handle job init failure and reject the task', async () => { + const { updateJobHandler, taskBuilderMock, queueClientMock } = setupUpdateJobHandlerTest(); + + const job = ingestionUpdateJob; + job.parameters.additionalParams = { displayPath: 'df2fce09-a97c-462f-a537-9a8edca643e9', tileOutputFormat: TileOutputFormat.PNG }; + + const taskId = '7e630dea-ea29-4b30-a88e-5407bf67d1bc'; + const tasks: AsyncGenerator = (async function* () {})(); + + const error = new Error('Test error'); + + taskBuilderMock.buildTasks.mockReturnValue(tasks); + taskBuilderMock.pushTasks.mockRejectedValue(error); + queueClientMock.reject.mockResolvedValue(undefined); + + await updateJobHandler.handleJobInit(job, taskId); + + expect(queueClientMock.reject).toHaveBeenCalledWith(job.id, taskId, true, error.message); + }); + + it('should handle job init failure with ZodError and Failed the job', async () => { + const { updateJobHandler, jobManagerClientMock, queueClientMock } = setupUpdateJobHandlerTest(); + const job = ingestionUpdateJob; + job.parameters.additionalParams = { wrongField: 'wrongValue' }; + const taskId = '291bf779-efe0-42bd-8357-aaede47e4d37'; + const validAdditionalParamsSpy = jest.spyOn(updateAdditionalParamsSchema, 'parse'); + + await updateJobHandler.handleJobInit(job, taskId); + + expect(validAdditionalParamsSpy).toThrow(ZodError); + expect(queueClientMock.reject).toHaveBeenCalledWith(job.id, taskId, false, expect.any(String)); + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + expect(jobManagerClientMock.updateJob).toHaveBeenCalledWith(job.id, { status: OperationStatus.FAILED, reason: expect.any(String) }); + }); + }); + describe('handleJobFinalize', () => { + it('should throw not implemented Error', async () => { + const { updateJobHandler } = setupUpdateJobHandlerTest(); + + await expect(updateJobHandler.handleJobFinalize(ingestionUpdateJob, finalizeTaskForIngestionUpdate)).rejects.toBe('not implemented'); + }); + }); +}); diff --git a/tests/unit/job/updateJobHandler/updateJobHandlerSetup.ts b/tests/unit/job/updateJobHandler/updateJobHandlerSetup.ts new file mode 100644 index 0000000..4ea244a --- /dev/null +++ b/tests/unit/job/updateJobHandler/updateJobHandlerSetup.ts @@ -0,0 +1,52 @@ +import jsLogger from '@map-colonies/js-logger'; +import { JobManagerClient, TaskHandler as QueueClient } from '@map-colonies/mc-priority-queue'; +import { configMock } from '../../mocks/configMock'; +import { TileMergeTaskManager } from '../../../../src/task/models/tileMergeTaskManager'; +import { MapproxyApiClient } from '../../../../src/httpClients/mapproxyClient'; +import { GeoserverClient } from '../../../../src/httpClients/geoserverClient'; +import { CatalogClient } from '../../../../src/httpClients/catalogClient'; +import { UpdateJobHandler } from '../../../../src/job/models/updateJobHandler'; + +export interface UpdateJobHandlerTestContext { + updateJobHandler: UpdateJobHandler; + taskBuilderMock: jest.Mocked; + queueClientMock: jest.Mocked; + jobManagerClientMock: jest.Mocked; + mapproxyClientMock: jest.Mocked; + geoserverClientMock: jest.Mocked; + catalogClientMock: jest.Mocked; +} + +export const setupUpdateJobHandlerTest = (): UpdateJobHandlerTestContext => { + const taskBuilderMock = { + buildTasks: jest.fn(), + pushTasks: jest.fn(), + } as unknown as jest.Mocked; + + const jobManagerClientMock = { + updateJob: jest.fn(), + updateTask: jest.fn(), + } as unknown as jest.Mocked; + + const queueClientMock = { + jobManagerClient: jobManagerClientMock, + ack: jest.fn(), + reject: jest.fn(), + } as unknown as jest.Mocked; + + const mapproxyClientMock = { publish: jest.fn() } as unknown as jest.Mocked; + const geoserverClientMock = { publish: jest.fn() } as unknown as jest.Mocked; + const catalogClientMock = { publish: jest.fn() } as unknown as jest.Mocked; + + const updateJobHandler = new UpdateJobHandler(jsLogger({ enabled: false }), configMock, taskBuilderMock, queueClientMock); + + return { + updateJobHandler, + taskBuilderMock, + queueClientMock, + jobManagerClientMock, + mapproxyClientMock, + geoserverClientMock, + catalogClientMock, + }; +}; diff --git a/tests/unit/mocks/jobsMockData.ts b/tests/unit/mocks/jobsMockData.ts index d3f7a3b..1b034e6 100644 --- a/tests/unit/mocks/jobsMockData.ts +++ b/tests/unit/mocks/jobsMockData.ts @@ -1,4 +1,11 @@ -import { NewRasterLayer, ProductType, TileOutputFormat, Transparency, UpdateRasterLayer } from '@map-colonies/mc-model-types'; +import { + IngestionUpdateJobParams, + NewRasterLayer, + ProductType, + TileOutputFormat, + Transparency, + UpdateRasterLayer, +} from '@map-colonies/mc-model-types'; import { IJobResponse, OperationStatus } from '@map-colonies/mc-priority-queue'; import { ExtendedNewRasterLayer, Grid } from '../../../src/common/interfaces'; import { partsData } from './partsMockData'; @@ -71,7 +78,7 @@ export const ingestionNewJobExtended: IJobResponse = { +export const ingestionUpdateJob: IJobResponse = { id: 'd027b3aa-272b-4dc9-91d7-ba8343af5ed1', resourceId: 'another-product-id', version: '1.0', @@ -86,6 +93,10 @@ export const ingestionUpdateJob: IJobResponse = { fileNames: ['blueMarble.gpkg'], originDirectory: 'tests', }, + additionalParams: { + displayPath: 'd1e9fe74-2a8f-425f-ac46-d65bb5c5756d', + tileOutputFormat: TileOutputFormat.PNG, + }, }, status: OperationStatus.PENDING, percentage: 0, From 82deb970ac96d5047824544023f5651f3e7fe5fc Mon Sep 17 00:00:00 2001 From: almog8k Date: Sun, 20 Oct 2024 14:44:35 +0300 Subject: [PATCH 2/9] fix: remove liveServer form settings.json --- .vscode/settings.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 3d240ee..995b79f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,5 @@ { "typescript.tsdk": "node_modules/typescript/lib", "files.eol": "\n", - "cSpell.words": ["geoserver", "Mapproxy"], - "liveServer.settings.port": 5501 + "cSpell.words": ["geoserver", "Mapproxy"] } From ab7d036823c477b6b7a642dcd79dd8453420935b Mon Sep 17 00:00:00 2001 From: almog8k Date: Sun, 27 Oct 2024 17:02:36 +0200 Subject: [PATCH 3/9] fix: custom-environment-variables.json typo mistake task->tasks --- config/custom-environment-variables.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index 3cef8c5..33c4e6c 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -93,7 +93,7 @@ "type": "INGESTION_SWAP_UPDATE_JOB_TYPE" } }, - "task": { + "tasks": { "tilesMerging": { "type": "TILES_MERGING_TASK_TYPE", "tileBatchSize": "TILES_MERGING_TILE_BATCH_SIZE", From 82eb1717fc42e1a9aca21a5addfc97d58ced408b Mon Sep 17 00:00:00 2001 From: almog8k Date: Sun, 27 Oct 2024 17:23:50 +0200 Subject: [PATCH 4/9] fix: adjust to new partsData property from mc-model-types --- package-lock.json | 8 ++++---- package.json | 2 +- src/job/models/newJobHandler.ts | 6 +++--- src/job/models/updateJobHandler.ts | 4 ++-- tests/unit/job/updateJobHandler/updateJobHandler.spec.ts | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/package-lock.json b/package-lock.json index 69cc26f..740b978 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,7 @@ "@map-colonies/error-express-handler": "^2.1.0", "@map-colonies/express-access-log-middleware": "^2.0.1", "@map-colonies/js-logger": "^1.1.0", - "@map-colonies/mc-model-types": "^17.6.0", + "@map-colonies/mc-model-types": "^17.7.2", "@map-colonies/mc-priority-queue": "^8.1.1", "@map-colonies/mc-utils": "^3.1.0", "@map-colonies/read-pkg": "0.0.1", @@ -3351,9 +3351,9 @@ } }, "node_modules/@map-colonies/mc-model-types": { - "version": "17.6.0", - "resolved": "https://registry.npmjs.org/@map-colonies/mc-model-types/-/mc-model-types-17.6.0.tgz", - "integrity": "sha512-xuopwKFGfgl18BkKMLl9/cb5/prWDVRJYzQSX3wu8G5TcPVlOPNjq0g2TAa97Mr9OEWJyMG98qlXifUz2wxrpg==", + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/@map-colonies/mc-model-types/-/mc-model-types-17.7.2.tgz", + "integrity": "sha512-BGuOEscs0vGAUpy3upZEfs/AlkPJwgEOLSpZ1LG5jysetCPvyee/l6KwS+UfDAFKgtE1GsUpLllhEBq7KpVZAw==", "license": "MIT", "dependencies": { "@types/geojson": "^7946.0.7", diff --git a/package.json b/package.json index 58c7a67..f10ac83 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "@map-colonies/error-express-handler": "^2.1.0", "@map-colonies/express-access-log-middleware": "^2.0.1", "@map-colonies/js-logger": "^1.1.0", - "@map-colonies/mc-model-types": "^17.6.0", + "@map-colonies/mc-model-types": "^17.7.2", "@map-colonies/mc-priority-queue": "^8.1.1", "@map-colonies/mc-utils": "^3.1.0", "@map-colonies/read-pkg": "0.0.1", diff --git a/src/job/models/newJobHandler.ts b/src/job/models/newJobHandler.ts index 5fe0957..50bf4f2 100644 --- a/src/job/models/newJobHandler.ts +++ b/src/job/models/newJobHandler.ts @@ -29,7 +29,7 @@ export class NewJobHandler implements IJobHandler { try { logger.info({ msg: `handling ${job.type} job with "init" task` }); - const { inputFiles, metadata, partData } = job.parameters; + const { inputFiles, metadata, partsData } = job.parameters; const extendedLayerMetadata = this.mapToExtendedNewLayerMetadata(metadata); const taskBuildParams: MergeTilesTaskParams = { @@ -40,7 +40,7 @@ export class NewJobHandler implements IJobHandler { isNewTarget: true, grid: extendedLayerMetadata.grid, }, - partsData: partData, + partsData, }; logger.info({ msg: 'building tasks' }); @@ -52,7 +52,7 @@ export class NewJobHandler implements IJobHandler { logger.info({ msg: 'Updating job with new metadata', ...metadata, extendedLayerMetadata }); await this.queueClient.jobManagerClient.updateJob(job.id, { internalId: extendedLayerMetadata.catalogId, - parameters: { metadata: extendedLayerMetadata, partData, inputFiles }, + parameters: { metadata: extendedLayerMetadata, partsData, inputFiles }, }); logger.info({ msg: 'Acking task' }); diff --git a/src/job/models/updateJobHandler.ts b/src/job/models/updateJobHandler.ts index 105488d..a0bef5d 100644 --- a/src/job/models/updateJobHandler.ts +++ b/src/job/models/updateJobHandler.ts @@ -24,7 +24,7 @@ export class UpdateJobHandler implements IJobHandler { const logger = this.logger.child({ jobId: job.id, taskId }); try { logger.info({ msg: `handling ${job.type} job with "init" task` }); - const { inputFiles, partData, additionalParams } = job.parameters; + const { inputFiles, partsData, additionalParams } = job.parameters; const validAdditionalParams = this.validateAdditionalParams(additionalParams); @@ -36,7 +36,7 @@ export class UpdateJobHandler implements IJobHandler { isNewTarget: this.isNewTarget, grid: Grid.TWO_ON_ONE, }, - partsData: partData, + partsData, }; logger.info({ msg: 'building tasks' }); diff --git a/tests/unit/job/updateJobHandler/updateJobHandler.spec.ts b/tests/unit/job/updateJobHandler/updateJobHandler.spec.ts index afbd525..d8bf60f 100644 --- a/tests/unit/job/updateJobHandler/updateJobHandler.spec.ts +++ b/tests/unit/job/updateJobHandler/updateJobHandler.spec.ts @@ -31,7 +31,7 @@ describe('updateJobHandler', () => { isNewTarget: true, grid: Grid.TWO_ON_ONE, }, - partsData: job.parameters.partData, + partsData: job.parameters.partsData, }; const mergeTasks: AsyncGenerator = (async function* () {})(); From 44b5c55aa1a600bdd6a710abdc03b57d3211800e Mon Sep 17 00:00:00 2001 From: almog8k Date: Sun, 27 Oct 2024 17:26:49 +0200 Subject: [PATCH 5/9] fix: adjust to new partsData property from mc-model-types --- src/httpClients/catalogClient.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/httpClients/catalogClient.ts b/src/httpClients/catalogClient.ts index 26fff49..71a1025 100644 --- a/src/httpClients/catalogClient.ts +++ b/src/httpClients/catalogClient.ts @@ -51,9 +51,9 @@ export class CatalogClient extends HttpClient { private mapToCatalogRecordMetadata(job: IJobResponse): LayerMetadata { const { parameters, version } = job; - const { partData, metadata } = parameters; + const { partsData, metadata } = parameters; - const aggregatedPartData = this.polygonPartMangerClient.getAggregatedPartData(partData); + const aggregatedPartData = this.polygonPartMangerClient.getAggregatedPartData(partsData); return { id: metadata.catalogId, From 9335d091f7d442e2c27e939f90304a4914479e2d Mon Sep 17 00:00:00 2001 From: almog8k Date: Sun, 27 Oct 2024 17:30:13 +0200 Subject: [PATCH 6/9] fix: adjust to new partsData property from mc-model-types --- tests/unit/mocks/jobsMockData.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/mocks/jobsMockData.ts b/tests/unit/mocks/jobsMockData.ts index 1b034e6..ecf3338 100644 --- a/tests/unit/mocks/jobsMockData.ts +++ b/tests/unit/mocks/jobsMockData.ts @@ -33,7 +33,7 @@ export const ingestionNewJob: IJobResponse = { classification: '6', productSubType: 'string', }, - partData: partsData, + partsData, inputFiles: { fileNames: ['blueMarble.gpkg'], originDirectory: 'tests', @@ -88,7 +88,7 @@ export const ingestionUpdateJob: IJobResponse metadata: { classification: '6', }, - partData: partsData, + partsData, inputFiles: { fileNames: ['blueMarble.gpkg'], originDirectory: 'tests', @@ -131,7 +131,7 @@ export const ingestionSwapUpdateJob: IJobResponse = metadata: { classification: '6', }, - partData: partsData, + partsData, inputFiles: { fileNames: ['blueMarble.gpkg'], originDirectory: 'tests', From 6cb2cd167e6060dc39fea3c0399069faabd8f16f Mon Sep 17 00:00:00 2001 From: almog8k Date: Sun, 27 Oct 2024 18:34:06 +0200 Subject: [PATCH 7/9] fix: pr fixes --- src/job/models/updateJobHandler.ts | 2 +- src/utils/zod/schemas/jobParametersSchema.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/job/models/updateJobHandler.ts b/src/job/models/updateJobHandler.ts index a0bef5d..b1fb87f 100644 --- a/src/job/models/updateJobHandler.ts +++ b/src/job/models/updateJobHandler.ts @@ -21,7 +21,7 @@ export class UpdateJobHandler implements IJobHandler { } public async handleJobInit(job: IJobResponse, taskId: string): Promise { - const logger = this.logger.child({ jobId: job.id, taskId }); + const logger = this.logger.child({ jobId: job.id, taskId, jobType: job.type }); try { logger.info({ msg: `handling ${job.type} job with "init" task` }); const { inputFiles, partsData, additionalParams } = job.parameters; diff --git a/src/utils/zod/schemas/jobParametersSchema.ts b/src/utils/zod/schemas/jobParametersSchema.ts index 5682645..5d846c0 100644 --- a/src/utils/zod/schemas/jobParametersSchema.ts +++ b/src/utils/zod/schemas/jobParametersSchema.ts @@ -2,6 +2,7 @@ import { TileOutputFormat } from '@map-colonies/mc-model-types'; import { z } from 'zod'; export const updateAdditionalParamsSchema = z.object({ + jobTrackerServiceUrl: z.string().url(), displayPath: z.string().uuid(), tileOutputFormat: z.nativeEnum(TileOutputFormat), }); From 2af4fb047e0e26fb4dcf7ff87d77230ee74c55f7 Mon Sep 17 00:00:00 2001 From: almog8k Date: Sun, 27 Oct 2024 18:54:58 +0200 Subject: [PATCH 8/9] fix: pr fixes --- package-lock.json | 8 ++++---- package.json | 2 +- tests/unit/job/updateJobHandler/updateJobHandler.spec.ts | 3 +-- tests/unit/mocks/jobsMockData.ts | 1 + 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index 740b978..c8fd79d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,7 @@ "@map-colonies/error-express-handler": "^2.1.0", "@map-colonies/express-access-log-middleware": "^2.0.1", "@map-colonies/js-logger": "^1.1.0", - "@map-colonies/mc-model-types": "^17.7.2", + "@map-colonies/mc-model-types": "^17.8.0", "@map-colonies/mc-priority-queue": "^8.1.1", "@map-colonies/mc-utils": "^3.1.0", "@map-colonies/read-pkg": "0.0.1", @@ -3351,9 +3351,9 @@ } }, "node_modules/@map-colonies/mc-model-types": { - "version": "17.7.2", - "resolved": "https://registry.npmjs.org/@map-colonies/mc-model-types/-/mc-model-types-17.7.2.tgz", - "integrity": "sha512-BGuOEscs0vGAUpy3upZEfs/AlkPJwgEOLSpZ1LG5jysetCPvyee/l6KwS+UfDAFKgtE1GsUpLllhEBq7KpVZAw==", + "version": "17.8.0", + "resolved": "https://registry.npmjs.org/@map-colonies/mc-model-types/-/mc-model-types-17.8.0.tgz", + "integrity": "sha512-Y5Vi/odDtBOF1xE+8E0qtvbj+840JHecl2zphz3CKprUeLPkde9yVDFRLlezS8uaXtJauG25+7RYcpZZJP4SaA==", "license": "MIT", "dependencies": { "@types/geojson": "^7946.0.7", diff --git a/package.json b/package.json index f10ac83..c757c61 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "@map-colonies/error-express-handler": "^2.1.0", "@map-colonies/express-access-log-middleware": "^2.0.1", "@map-colonies/js-logger": "^1.1.0", - "@map-colonies/mc-model-types": "^17.7.2", + "@map-colonies/mc-model-types": "^17.8.0", "@map-colonies/mc-priority-queue": "^8.1.1", "@map-colonies/mc-utils": "^3.1.0", "@map-colonies/read-pkg": "0.0.1", diff --git a/tests/unit/job/updateJobHandler/updateJobHandler.spec.ts b/tests/unit/job/updateJobHandler/updateJobHandler.spec.ts index d8bf60f..5285002 100644 --- a/tests/unit/job/updateJobHandler/updateJobHandler.spec.ts +++ b/tests/unit/job/updateJobHandler/updateJobHandler.spec.ts @@ -50,8 +50,7 @@ describe('updateJobHandler', () => { it('should handle job init failure and reject the task', async () => { const { updateJobHandler, taskBuilderMock, queueClientMock } = setupUpdateJobHandlerTest(); - const job = ingestionUpdateJob; - job.parameters.additionalParams = { displayPath: 'df2fce09-a97c-462f-a537-9a8edca643e9', tileOutputFormat: TileOutputFormat.PNG }; + const job = { ...ingestionUpdateJob }; const taskId = '7e630dea-ea29-4b30-a88e-5407bf67d1bc'; const tasks: AsyncGenerator = (async function* () {})(); diff --git a/tests/unit/mocks/jobsMockData.ts b/tests/unit/mocks/jobsMockData.ts index ecf3338..edf16f0 100644 --- a/tests/unit/mocks/jobsMockData.ts +++ b/tests/unit/mocks/jobsMockData.ts @@ -94,6 +94,7 @@ export const ingestionUpdateJob: IJobResponse originDirectory: 'tests', }, additionalParams: { + jobTrackerServiceUrl: 'http://job-tracker-service', displayPath: 'd1e9fe74-2a8f-425f-ac46-d65bb5c5756d', tileOutputFormat: TileOutputFormat.PNG, }, From d758e4c68b0b4141324737a6186c415b33be6b98 Mon Sep 17 00:00:00 2001 From: almog8k Date: Sun, 27 Oct 2024 18:57:38 +0200 Subject: [PATCH 9/9] fix: pr fixes --- tests/unit/job/updateJobHandler/updateJobHandler.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/job/updateJobHandler/updateJobHandler.spec.ts b/tests/unit/job/updateJobHandler/updateJobHandler.spec.ts index 5285002..51c12cb 100644 --- a/tests/unit/job/updateJobHandler/updateJobHandler.spec.ts +++ b/tests/unit/job/updateJobHandler/updateJobHandler.spec.ts @@ -1,7 +1,6 @@ /* eslint-disable @typescript-eslint/unbound-method */ import { ZodError } from 'zod'; import { OperationStatus } from '@map-colonies/mc-priority-queue'; -import { TileOutputFormat } from '@map-colonies/mc-model-types'; import { Grid, IMergeTaskParameters } from '../../../../src/common/interfaces'; import { finalizeTaskForIngestionUpdate } from '../../mocks/tasksMockData'; import { updateAdditionalParamsSchema } from '../../../../src/utils/zod/schemas/jobParametersSchema';