From 62520345262073bdb3dee096973d50d37975a8df Mon Sep 17 00:00:00 2001 From: razbroc Date: Mon, 18 Mar 2024 14:39:09 +0200 Subject: [PATCH 01/19] chore: remove create api --- openapi3.yaml | 44 --- src/common/interfaces.ts | 3 - .../controllers/createPackageController.ts | 14 +- .../models/createPackageManager.ts | 107 ------- .../routes/createPackageRouter.ts | 1 - .../createPackage/createPackage.spec.ts | 284 ------------------ .../helpers/createPackageSender.ts | 6 +- tests/mocks/data.ts | 4 +- .../models/createPackageModel.spec.ts | 275 ----------------- 9 files changed, 4 insertions(+), 734 deletions(-) delete mode 100644 tests/integration/createPackage/createPackage.spec.ts diff --git a/openapi3.yaml b/openapi3.yaml index c3306d0..460fe8d 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -26,50 +26,6 @@ paths: application/json: schema: $ref: '#/components/schemas/internalError' - /create: - post: - tags: - - createGpkg - summary: Trigger export geopackage process - operationId: exportTilesToGpkg - deprecated: true - requestBody: - $ref: '#/components/requestBodies/ExportGetmapBody' - responses: - '200': - description: OK - content: - application/json: - schema: - oneOf: - - $ref: '#/components/schemas/createGpkgJobResponse' - - $ref: '#/components/schemas/naiveCacheJobResponse' - discriminator: - propertyName: response - '400': - description: Bad Request - content: - application/json: - schema: - $ref: '#/components/schemas/error' - '404': - description: Could not find layer with matched dbId - content: - application/json: - schema: - $ref: '#/components/schemas/error' - '500': - description: Internal Server Error - content: - application/json: - schema: - $ref: '#/components/schemas/internalError' - '507': - description: Insufficient Storage on disk for exporting - content: - application/json: - schema: - $ref: '#/components/schemas/error' /create/roi: post: tags: diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index 5718283..3d5875c 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -27,9 +27,6 @@ export interface ICleanupData { cleanupExpirationTimeUTC?: Date; } -/** - * @deprecated GetMap API - will be deprecated on future - */ export interface ICreatePackage extends IBaseCreatePackage { targetResolution?: number; bbox?: BBox | Polygon | MultiPolygon; diff --git a/src/createPackage/controllers/createPackageController.ts b/src/createPackage/controllers/createPackageController.ts index 596c900..c32da55 100644 --- a/src/createPackage/controllers/createPackageController.ts +++ b/src/createPackage/controllers/createPackageController.ts @@ -6,7 +6,6 @@ import { SERVICES } from '../../common/constants'; import { CreatePackageManager } from '../models/createPackageManager'; import { IBasicResponse, - ICreatePackage, ICreateJobResponse, ICallbackResposne, ICreatePackageRoi, @@ -17,7 +16,7 @@ import { type CreatePackageHandler = RequestHandler< undefined, IBasicResponse | ICreateJobResponse | ICreateExportJobResponse | ICallbackResposne | ICallbackExportResponse, - ICreatePackage | ICreatePackageRoi + ICreatePackageRoi >; @injectable() @@ -27,17 +26,6 @@ export class CreatePackageController { @inject(CreatePackageManager) private readonly manager: CreatePackageManager ) {} - public create: CreatePackageHandler = async (req, res, next) => { - const userInput: ICreatePackage = req.body as ICreatePackage; - try { - this.logger.debug(userInput, `Creating package with user input`); - const jobCreated = await this.manager.createPackage(userInput); - return res.status(httpStatus.OK).json(jobCreated); - } catch (err) { - next(err); - } - }; - public createPackageRoi: CreatePackageHandler = async (req, res, next) => { const userInput: ICreatePackageRoi = req.body; try { diff --git a/src/createPackage/models/createPackageManager.ts b/src/createPackage/models/createPackageManager.ts index 778d0c5..c9e834c 100644 --- a/src/createPackage/models/createPackageManager.ts +++ b/src/createPackage/models/createPackageManager.ts @@ -60,7 +60,6 @@ import { import { RasterCatalogManagerClient } from '../../clients/rasterCatalogManagerClient'; import { DEFAULT_CRS, DEFAULT_PRIORITY, METADA_JSON_FILE_EXTENSION as METADATA_JSON_FILE_EXTENSION, SERVICES } from '../../common/constants'; import { - ICreatePackage, ICreateJobResponse, IWorkerInput, JobDuplicationParams, @@ -386,112 +385,6 @@ export class CreatePackageManager { } } - /** - * @deprecated GetMap API - will be deprecated on future - */ - public async createPackage(userInput: ICreatePackage): Promise { - const layer = await this.rasterCatalogManager.findLayer(userInput.dbId); - const layerMetadata = layer.metadata; - let { productId: resourceId, productVersion: version, productType } = layerMetadata; - const { dbId, crs, priority, bbox: bboxFromUser, callbackURLs } = userInput; - const normalizedPolygon = this.normalize2Polygon(bboxFromUser); - const polygon = normalizedPolygon ?? layerMetadata.footprint; - const targetResolution = (userInput.targetResolution ?? layerMetadata.maxResolutionDeg) as number; - const zoomLevel = degreesPerPixelToZoomLevel(targetResolution); - const tileEstimatedSize = this.getTileEstimatedSize(layerMetadata.tileOutputFormat as TileOutputFormat); - - resourceId = resourceId as string; - version = version as string; - productType = productType as ProductType; - - const srcRes = layerMetadata.maxResolutionDeg as number; - const maxZoom = degreesPerPixelToZoomLevel(srcRes); - if (zoomLevel > maxZoom) { - throw new BadRequestError(`The requested resolution ${targetResolution} is larger than product resolution ${srcRes}`); - } - - const sanitizedBbox = this.sanitizeBbox(polygon as Polygon, layerMetadata.footprint as Polygon | MultiPolygon, zoomLevel); - if (sanitizedBbox === null) { - throw new BadRequestError( - `Requested ${JSON.stringify(polygon as Polygon)} has no intersection with requested layer ${layer.metadata.id as string}` - ); - } - - const dupParams: JobDuplicationParams = { - resourceId, - version, - dbId, - zoomLevel, - sanitizedBbox, - crs: crs ?? DEFAULT_CRS, - }; - - const callbacks = callbackURLs.map((url) => { url, bbox: bboxFromUser ?? sanitizedBbox }); - const duplicationExist = await this.checkForDuplicate(dupParams, callbacks); - if (duplicationExist && duplicationExist.status === OperationStatus.COMPLETED) { - const completeResponseData = duplicationExist as ICallbackResponse; - completeResponseData.bbox = bboxFromUser ?? sanitizedBbox; - return duplicationExist; - } else if (duplicationExist) { - return duplicationExist; - } - - // TODO: remove and replace with `generateTileGroups` that is commented, when multiple tasks for GPKG target is possible - const batches: ITileRange[] = []; - for (let i = 0; i <= zoomLevel; i++) { - batches.push(bboxToTileRange(sanitizedBbox as BBox2d, i)); - } - // const batches = this.generateTileGroups(polygon as Polygon, layerMetadata.footprint as Polygon | MultiPolygon, zoomLevel); - const estimatesGpkgSize = calculateEstimateGpkgSize(batches, tileEstimatedSize); // size of requested gpkg export - if (this.storageEstimation.validateStorageSize) { - const isEnoughStorage = await this.validateFreeSpace(estimatesGpkgSize); - if (!isEnoughStorage) { - throw new InsufficientStorage(`There isn't enough free disk space to executing export`); - } - } - const separator = this.getSeparator(); - const packageName = this.generatePackageName(productType, resourceId, version, zoomLevel, sanitizedBbox); - const packageRelativePath = getGpkgRelativePath(packageName, separator); - const sources: IMapSource[] = [ - { - path: packageRelativePath, - type: 'GPKG', - extent: { - minX: sanitizedBbox[0], - minY: sanitizedBbox[1], - maxX: sanitizedBbox[2], - maxY: sanitizedBbox[3], - }, - }, - { - path: `${layerMetadata.id as string}${separator}${layerMetadata.displayPath as string}`, //tiles path - type: this.tilesProvider, - }, - ]; - - const workerInput: IWorkerInput = { - sanitizedBbox, - targetResolution, - fileName: packageName, - relativeDirectoryPath: getGpkgNameWithoutExt(packageName), - zoomLevel, - dbId, - exportVersion: ExportVersion.GETMAP, - version: version, - cswProductId: resourceId, - crs: crs ?? DEFAULT_CRS, - productType, - batches, - sources, - priority: priority ?? DEFAULT_PRIORITY, - callbacks: callbacks, - gpkgEstimatedSize: estimatesGpkgSize, - targetFormat: layerMetadata.tileOutputFormat, - }; - const jobCreated = await this.jobManagerClient.create(workerInput); - return jobCreated; - } - private featuresFootprintIntersects( features: Feature[], footprint: Polygon | MultiPolygon diff --git a/src/createPackage/routes/createPackageRouter.ts b/src/createPackage/routes/createPackageRouter.ts index 1749a1b..50e46e5 100644 --- a/src/createPackage/routes/createPackageRouter.ts +++ b/src/createPackage/routes/createPackageRouter.ts @@ -6,7 +6,6 @@ const createPackageRouterFactory: FactoryFunction = (dependencyContainer const router = Router(); const controller = dependencyContainer.resolve(CreatePackageController); - router.post('/', controller.create); router.post('/roi', controller.createPackageRoi); return router; diff --git a/tests/integration/createPackage/createPackage.spec.ts b/tests/integration/createPackage/createPackage.spec.ts deleted file mode 100644 index bf4f3db..0000000 --- a/tests/integration/createPackage/createPackage.spec.ts +++ /dev/null @@ -1,284 +0,0 @@ -import httpStatusCodes from 'http-status-codes'; -import { OperationStatus } from '@map-colonies/mc-priority-queue'; -import { getApp } from '../../../src/app'; -import { RasterCatalogManagerClient } from '../../../src/clients/rasterCatalogManagerClient'; -import { getContainerConfig, resetContainer } from '../testContainerConfig'; -import { ICreateJobResponse, ICreatePackage } from '../../../src/common/interfaces'; -import { layerFromCatalog } from '../../mocks/data'; -import { JobManagerWrapper } from '../../../src/clients/jobManagerWrapper'; -import { CreatePackageManager } from '../../../src/createPackage/models/createPackageManager'; -import { CreatePackageSender } from './helpers/createPackageSender'; - -describe('tiles', function () { - let requestSender: CreatePackageSender; - let findLayerSpy: jest.SpyInstance; - let createJobSpy: jest.SpyInstance; - let checkForDuplicateSpy: jest.SpyInstance; - let validateFreeSpaceSpy: jest.SpyInstance; - let checkForCompletedSpy: jest.SpyInstance; - let checkForProcessingSpy: jest.SpyInstance; - let normalize2Polygon: jest.SpyInstance; - let generateTileGroupsSpy: jest.SpyInstance; - - beforeEach(function () { - const app = getApp({ - override: [...getContainerConfig()], - useChild: true, - }); - requestSender = new CreatePackageSender(app); - checkForDuplicateSpy = jest.spyOn(CreatePackageManager.prototype as unknown as { checkForDuplicate: jest.Mock }, 'checkForDuplicate'); - normalize2Polygon = jest.spyOn(CreatePackageManager.prototype as unknown as { normalize2Polygon: jest.Mock }, 'normalize2Polygon'); - checkForCompletedSpy = jest.spyOn(CreatePackageManager.prototype as unknown as { checkForCompleted: jest.Mock }, 'checkForCompleted'); - checkForProcessingSpy = jest.spyOn(CreatePackageManager.prototype as unknown as { checkForProcessing: jest.Mock }, 'checkForProcessing'); - validateFreeSpaceSpy = jest.spyOn(CreatePackageManager.prototype as unknown as { validateFreeSpace: jest.Mock }, 'validateFreeSpace'); - generateTileGroupsSpy = jest.spyOn(CreatePackageManager.prototype as unknown as { generateTileGroups: jest.Mock }, 'generateTileGroups'); - findLayerSpy = jest.spyOn(RasterCatalogManagerClient.prototype, 'findLayer'); - createJobSpy = jest.spyOn(JobManagerWrapper.prototype, 'createJob'); - }); - - afterEach(function () { - resetContainer(); - jest.resetAllMocks(); - }); - - describe('Happy Path', function () { - it('should return 200 status code and the job created details', async function () { - const body: ICreatePackage = { - dbId: layerFromCatalog.id, - bbox: [34.811938017107494, 31.95475033759175, 34.82237261707599, 31.96426962177354], - targetResolution: 0.00034332275390625, - callbackURLs: ['http://example.getmap.com/callback'], - crs: 'EPSG:4326', - priority: 0, - }; - findLayerSpy.mockResolvedValue(layerFromCatalog); - createJobSpy.mockResolvedValue({ id: 'b1c59730-c31d-4e44-9c67-4dbbb3b1c812', taskIds: ['6556896a-113c-4397-a48b-0cb2c99658f5'] }); - checkForDuplicateSpy.mockResolvedValue(undefined); - validateFreeSpaceSpy.mockResolvedValue(true); - - const resposne = await requestSender.create(body); - expect(resposne).toSatisfyApiSpec(); - expect(findLayerSpy).toHaveBeenCalledTimes(1); - expect(createJobSpy).toHaveBeenCalledTimes(1); - expect(resposne.status).toBe(httpStatusCodes.OK); - }); - - it('should return 200 status code and the job created details even if bbox and resolution were not supplied', async function () { - const body: ICreatePackage = { - dbId: layerFromCatalog.id, - callbackURLs: ['http://example.getmap.com/callback'], - crs: 'EPSG:4326', - priority: 0, - }; - generateTileGroupsSpy.mockReturnValue([]); - findLayerSpy.mockResolvedValue(layerFromCatalog); - createJobSpy.mockResolvedValue({ id: 'b1c59730-c31d-4e44-9c67-4dbbb3b1c812', taskIds: ['6556896a-113c-4397-a48b-0cb2c99658f5'] }); - checkForDuplicateSpy.mockResolvedValue(undefined); - validateFreeSpaceSpy.mockResolvedValue(true); - - const resposne = await requestSender.create(body); - - expect(resposne).toSatisfyApiSpec(); - expect(findLayerSpy).toHaveBeenCalledTimes(1); - expect(createJobSpy).toHaveBeenCalledTimes(1); - expect(resposne.status).toBe(httpStatusCodes.OK); - }); - - it(`should return 200 status code and the exists un-cleaned completed job's callback (with original bbox of request)`, async function () { - checkForDuplicateSpy.mockRestore(); - - const expirationTime = new Date(); - const body: ICreatePackage = { - dbId: layerFromCatalog.id, - bbox: [34.811938017107494, 31.95475033759175, 34.82237261707599, 31.96426962177354], - targetResolution: 0.00034332275390625, - callbackURLs: ['http://example.getmap.com/callback'], - crs: 'EPSG:4326', - priority: 0, - }; - const origCallback = { - dbId: layerFromCatalog.id, - fileUri: 'http://example.getmap.com/callback', - success: true, - fileSize: 10, - requestId: 'string', - errorReason: '', - packageName: 'string', - expirationTime: expirationTime, - targetResolution: 0.123, - status: OperationStatus.COMPLETED, - bbox: [-180, -90, 90, 180], - }; - - const expectedCompletedCallback = { - dbId: layerFromCatalog.id, - fileUri: 'http://example.getmap.com/callback', - success: true, - fileSize: 10, - requestId: 'string', - errorReason: '', - packageName: 'string', - expirationTime: expirationTime, - targetResolution: 0.123, - status: OperationStatus.COMPLETED, - bbox: body.bbox, - }; - findLayerSpy.mockResolvedValue(layerFromCatalog); - checkForCompletedSpy.mockResolvedValue(origCallback); - validateFreeSpaceSpy.mockResolvedValue(true); - - const response = await requestSender.create(body); - - expect(response).toSatisfyApiSpec(); - expect(findLayerSpy).toHaveBeenCalledTimes(1); - expect(createJobSpy).toHaveBeenCalledTimes(0); - expect(checkForCompletedSpy).toHaveBeenCalledTimes(1); - expect(validateFreeSpaceSpy).toHaveBeenCalledTimes(0); - expect(JSON.stringify(response.body)).toBe(JSON.stringify(expectedCompletedCallback)); - expect(response.status).toBe(httpStatusCodes.OK); - }); - }); - - it(`should return 200 status code and the exists un-cleaned completed job's callback (with original bbox as footprint of request)`, async function () { - checkForDuplicateSpy.mockRestore(); - - const expirationTime = new Date(); - const body: ICreatePackage = { - dbId: layerFromCatalog.id, - bbox: { - type: 'Polygon', - coordinates: [ - [ - [25, 15], - [50, 15], - [50, 40], - [25, 40], - [25, 15], - ], - ], - }, - targetResolution: 0.00034332275390625, - callbackURLs: ['http://example.getmap.com/callback'], - crs: 'EPSG:4326', - priority: 0, - }; - const origCallback = { - dbId: layerFromCatalog.id, - fileUri: 'http://example.getmap.com/callback', - success: true, - fileSize: 10, - requestId: 'string', - errorReason: '', - packageName: 'string', - expirationTime: expirationTime, - targetResolution: 0.123, - status: OperationStatus.COMPLETED, - bbox: [-180, -90, 90, 180], - }; - - const expectedCompletedCallback = { - dbId: layerFromCatalog.id, - fileUri: 'http://example.getmap.com/callback', - success: true, - fileSize: 10, - requestId: 'string', - errorReason: '', - packageName: 'string', - expirationTime: expirationTime, - targetResolution: 0.123, - status: OperationStatus.COMPLETED, - bbox: body.bbox, - }; - findLayerSpy.mockResolvedValue(layerFromCatalog); - checkForCompletedSpy.mockResolvedValue(origCallback); - validateFreeSpaceSpy.mockResolvedValue(true); - - const response = await requestSender.create(body); - - expect(response).toSatisfyApiSpec(); - expect(findLayerSpy).toHaveBeenCalledTimes(1); - expect(normalize2Polygon).toHaveBeenCalledTimes(1); - expect(checkForCompletedSpy).toHaveBeenCalledTimes(1); - expect(createJobSpy).toHaveBeenCalledTimes(0); - expect(validateFreeSpaceSpy).toHaveBeenCalledTimes(0); - expect(JSON.stringify(response.body)).toBe(JSON.stringify(expectedCompletedCallback)); - expect(response.status).toBe(httpStatusCodes.OK); - }); - - it(`should return 200 status code and the exists in-progress job (id, task and status)`, async function () { - checkForDuplicateSpy.mockRestore(); - - const body: ICreatePackage = { - dbId: layerFromCatalog.id, - bbox: [34.811938017107494, 31.95475033759175, 34.82237261707599, 31.96426962177354], - targetResolution: 0.00034332275390625, - callbackURLs: ['http://example.getmap.com/callback'], - crs: 'EPSG:4326', - priority: 0, - }; - - const expectedInProgressJobResponse: ICreateJobResponse = { - id: 'b1c59730-c31d-4e44-9c67-4dbbb3b1c812', - taskIds: ['6556896a-113c-4397-a48b-0cb2c99658f5'], - status: OperationStatus.IN_PROGRESS, - }; - - findLayerSpy.mockResolvedValue(layerFromCatalog); - checkForCompletedSpy.mockResolvedValue(undefined); - checkForProcessingSpy.mockResolvedValue(expectedInProgressJobResponse); - validateFreeSpaceSpy.mockResolvedValue(true); - - const response = await requestSender.create(body); - - expect(response).toSatisfyApiSpec(); - expect(findLayerSpy).toHaveBeenCalledTimes(1); - expect(createJobSpy).toHaveBeenCalledTimes(0); - expect(checkForCompletedSpy).toHaveBeenCalledTimes(2); - expect(checkForProcessingSpy).toHaveBeenCalledTimes(1); - expect(validateFreeSpaceSpy).toHaveBeenCalledTimes(0); - expect(response.body).toStrictEqual(expectedInProgressJobResponse); - expect(response.status).toBe(httpStatusCodes.OK); - }); - - describe('Sad Path', function () { - it('should return 400 status code because of bad data - no "dbId" field', async function () { - const body = { - targetResolution: 0.0000429153442382812, - bbox: [34.811938017107494, 31.95475033759175, 34.82237261707599, 31.96426962177354], - callbackURLs: ['http://example.getmap.com/callback'], - crs: 'EPSG:4326', - priority: 0, - } as unknown as ICreatePackage; - - const resposne = await requestSender.create(body); - - expect(resposne).toSatisfyApiSpec(); - expect(findLayerSpy).toHaveBeenCalledTimes(0); - expect(createJobSpy).toHaveBeenCalledTimes(0); - expect(resposne.status).toBe(httpStatusCodes.BAD_REQUEST); - }); - }); - - describe('Bad Path', function () { - it('should return 507 status code for insufficient storage to gpkg creation', async function () { - const body: ICreatePackage = { - dbId: layerFromCatalog.id, - bbox: [34.811938017107494, 31.95475033759175, 34.82237261707599, 31.96426962177354], - targetResolution: 0.00034332275390625, - callbackURLs: ['http://example.getmap.com/callback'], - crs: 'EPSG:4326', - priority: 0, - }; - generateTileGroupsSpy.mockReturnValue([]); - findLayerSpy.mockResolvedValue(layerFromCatalog); - checkForDuplicateSpy.mockResolvedValue(undefined); - validateFreeSpaceSpy.mockResolvedValue(false); - const resposne = await requestSender.create(body); - - expect(resposne).toSatisfyApiSpec(); - expect(findLayerSpy).toHaveBeenCalledTimes(1); - expect(createJobSpy).toHaveBeenCalledTimes(0); - expect(resposne.status).toBe(httpStatusCodes.INSUFFICIENT_STORAGE); - }); - }); -}); diff --git a/tests/integration/createPackage/helpers/createPackageSender.ts b/tests/integration/createPackage/helpers/createPackageSender.ts index 3502a3a..e4b47bb 100644 --- a/tests/integration/createPackage/helpers/createPackageSender.ts +++ b/tests/integration/createPackage/helpers/createPackageSender.ts @@ -1,13 +1,9 @@ import * as supertest from 'supertest'; -import { ICreatePackage, ICreatePackageRoi } from '../../../../src/common/interfaces'; +import { ICreatePackageRoi } from '../../../../src/common/interfaces'; export class CreatePackageSender { public constructor(private readonly app: Express.Application) {} - public async create(body: ICreatePackage): Promise { - return supertest.agent(this.app).post(`/create`).set('Content-Type', 'application/json').send(body); - } - public async createPackageRoi(body: ICreatePackageRoi): Promise { return supertest.agent(this.app).post(`/create/roi`).set('Content-Type', 'application/json').send(body); } diff --git a/tests/mocks/data.ts b/tests/mocks/data.ts index 6115d41..3e41075 100644 --- a/tests/mocks/data.ts +++ b/tests/mocks/data.ts @@ -4,7 +4,7 @@ import { IJobResponse, OperationStatus } from '@map-colonies/mc-priority-queue'; import { FeatureCollection } from '@turf/helpers'; import { ExportVersion, - ICreatePackage, +ICreatePackage, IJobExportParameters, IJobParameters, ITaskParameters, @@ -1080,7 +1080,7 @@ export { workerInput, workerExportInput, jobs, - userInput, +userInput, completedJob, inProgressJob, fc1, diff --git a/tests/unit/createPackage/models/createPackageModel.spec.ts b/tests/unit/createPackage/models/createPackageModel.spec.ts index 80cf99a..2c88a1c 100644 --- a/tests/unit/createPackage/models/createPackageModel.spec.ts +++ b/tests/unit/createPackage/models/createPackageModel.spec.ts @@ -24,7 +24,6 @@ import { ExportVersion, ICreateExportJobResponse, ICreateJobResponse, - ICreatePackage, ICreatePackageRoi, IJobParameters, ITaskParameters, @@ -83,280 +82,6 @@ describe('CreatePackageManager', () => { * @deprecated GetMap API - will be deprecated on future */ describe('GetMAP', () => { - describe('#create', () => { - it('should create job and return its job and task ids', async () => { - const req: ICreatePackage = { - dbId: layerFromCatalog.id, - bbox: [0, 1, 3, 5], - callbackURLs: ['testUrl'], - targetResolution: 0.0439453125, - crs: 'EPSG:4326', - }; - - const expectedsanitizedBbox: BBox = [0, 0, 11.25, 11.25]; - const jobDupParams: JobDuplicationParams = { - resourceId: 'string', - version: '1.0', - dbId: layerFromCatalog.id, - zoomLevel: 4, - sanitizedBbox: expectedsanitizedBbox, - crs: 'EPSG:4326', - }; - - const expectedCreateJobResponse = { - jobId: '09e29fa8-7283-4334-b3a4-99f75922de59', - taskIds: ['66aa1e2e-784c-4178-b5a0-af962937d561'], - status: OperationStatus.IN_PROGRESS, - }; - const validateFreeSpaceSpy = jest.spyOn(CreatePackageManager.prototype as unknown as { validateFreeSpace: jest.Mock }, 'validateFreeSpace'); - - findLayerMock.mockResolvedValue(layerFromCatalog); - createMock.mockResolvedValue(expectedCreateJobResponse); - findCompletedJobMock.mockResolvedValue(undefined); - findInProgressJobMock.mockResolvedValue(undefined); - findPendingJobMock.mockResolvedValue(undefined); - validateFreeSpaceSpy.mockResolvedValue(true); - const res = await createPackageManager.createPackage(req); - - expect(res).toEqual(expectedCreateJobResponse); - expect(findLayerMock).toHaveBeenCalledWith(req.dbId); - expect(findLayerMock).toHaveBeenCalledTimes(1); - expect(createMock).toHaveBeenCalledTimes(1); - expect(findCompletedJobMock).toHaveBeenCalledWith(jobDupParams); - expect(findCompletedJobMock).toHaveBeenCalledTimes(1); - expect(findInProgressJobMock).toHaveBeenCalledWith(jobDupParams); - expect(findInProgressJobMock).toHaveBeenCalledTimes(1); - }); - - it('should create job and convert provided footprint to bbox', async () => { - const footprint: Polygon = { - type: 'Polygon', - coordinates: [ - [ - [25, 15], - [50, 15], - [50, 40], - [25, 40], - [25, 15], - ], - ], - }; - const req: ICreatePackage = { - dbId: layerFromCatalog.id, - bbox: footprint, - callbackURLs: ['testUrl'], - targetResolution: 0.0439453125, - crs: 'EPSG:4326', - }; - - const expectedsanitizedBbox: BBox = [22.5, 11.25, 56.25, 45]; - const jobDupParams: JobDuplicationParams = { - resourceId: 'string', - version: '1.0', - dbId: layerFromCatalog.id, - zoomLevel: 4, - sanitizedBbox: expectedsanitizedBbox, - crs: 'EPSG:4326', - }; - - const expectedCreateJobResponse = { - jobId: '09e29fa8-7283-4334-b3a4-99f75922de59', - taskIds: ['66aa1e2e-784c-4178-b5a0-af962937d561'], - status: OperationStatus.IN_PROGRESS, - }; - const validateFreeSpaceSpy = jest.spyOn(CreatePackageManager.prototype as unknown as { validateFreeSpace: jest.Mock }, 'validateFreeSpace'); - const normalize2PolygonSpy = jest.spyOn(CreatePackageManager.prototype as unknown as { normalize2Polygon: jest.Mock }, 'normalize2Polygon'); - - findLayerMock.mockResolvedValue(layerFromCatalog); - createMock.mockResolvedValue(expectedCreateJobResponse); - findCompletedJobMock.mockResolvedValue(undefined); - findInProgressJobMock.mockResolvedValue(undefined); - findPendingJobMock.mockResolvedValue(undefined); - validateFreeSpaceSpy.mockResolvedValue(true); - const res = await createPackageManager.createPackage(req); - - expect(res).toEqual(expectedCreateJobResponse); - expect(findLayerMock).toHaveBeenCalledWith(req.dbId); - expect(findLayerMock).toHaveBeenCalledTimes(1); - expect(normalize2PolygonSpy).toHaveBeenCalledTimes(1); - expect(createMock).toHaveBeenCalledTimes(1); - expect(findCompletedJobMock).toHaveBeenCalledWith(jobDupParams); - expect(findCompletedJobMock).toHaveBeenCalledTimes(1); - expect(findInProgressJobMock).toHaveBeenCalledWith(jobDupParams); - expect(findInProgressJobMock).toHaveBeenCalledTimes(1); - }); - - it(`should create job and take original layer's resolution and sanitized bbox`, async () => { - const req: ICreatePackage = { - dbId: layerFromCatalog.id, - callbackURLs: ['testUrl'], - crs: 'EPSG:4326', - }; - - const expectedCreateJobResponse = { - jobId: '09e29fa8-7283-4334-b3a4-99f75922de59', - taskIds: ['66aa1e2e-784c-4178-b5a0-af962937d561'], - status: OperationStatus.IN_PROGRESS, - }; - - const validateFreeSpaceSpy = jest.spyOn(CreatePackageManager.prototype as unknown as { validateFreeSpace: jest.Mock }, 'validateFreeSpace'); - - const expectedsanitizedBbox: BBox = [0, -90, 180, 90]; - const expectedTargetResolution = layerFromCatalog.metadata.maxResolutionDeg; - - findLayerMock.mockResolvedValue(layerFromCatalog); - createMock.mockResolvedValue(expectedCreateJobResponse); - findCompletedJobMock.mockResolvedValue(undefined); - findInProgressJobMock.mockResolvedValue(undefined); - findPendingJobMock.mockResolvedValue(undefined); - validateFreeSpaceSpy.mockResolvedValue(true); - - const res = await createPackageManager.createPackage(req); - - expect(res).toEqual(expectedCreateJobResponse); - expect(findLayerMock).toHaveBeenCalledWith(req.dbId); - expect(findLayerMock).toHaveBeenCalledTimes(1); - expect(createMock).toHaveBeenCalledTimes(1); - expect(createMock).toHaveBeenCalledWith( - expect.objectContaining({ - targetResolution: expectedTargetResolution, - sanitizedBbox: expectedsanitizedBbox, - }) - ); - }); - - it('should return job and task-ids of existing in pending job', async () => { - const expectedsanitizedBbox: BBox = [0, 2.8125, 25.3125, 42.1875]; - const jobDupParams: JobDuplicationParams = { - resourceId: layerFromCatalog.metadata.productId as string, - version: layerFromCatalog.metadata.productVersion as string, - dbId: layerFromCatalog.id, - zoomLevel: 7, - sanitizedBbox: expectedsanitizedBbox, - crs: userInput.crs as string, - }; - - findLayerMock.mockResolvedValue(layerFromCatalog); - createMock.mockResolvedValue(undefined); - updateJobMock.mockResolvedValue(undefined); - findCompletedJobMock.mockResolvedValue(undefined); - findInProgressJobMock.mockResolvedValue(undefined); - findPendingJobMock.mockResolvedValue(JSON.parse(JSON.stringify(inProgressJob))); - - await createPackageManager.createPackage(userInput); - - expect(findLayerMock).toHaveBeenCalledWith(layerFromCatalog.id); - expect(findLayerMock).toHaveBeenCalledTimes(1); - expect(createMock).toHaveBeenCalledTimes(0); - expect(findCompletedJobMock).toHaveBeenNthCalledWith(1, jobDupParams); - expect(findCompletedJobMock).toHaveBeenNthCalledWith(2, jobDupParams); - expect(findCompletedJobMock).toHaveBeenCalledTimes(2); - expect(findInProgressJobMock).toHaveBeenCalledWith(jobDupParams); - expect(findInProgressJobMock).toHaveBeenCalledTimes(1); - expect(findPendingJobMock).toHaveBeenCalledWith(jobDupParams); - expect(findPendingJobMock).toHaveBeenCalledTimes(1); - }); - - it('should return job and task-ids of existing in progress job', async () => { - const expectedsanitizedBbox: BBox = [0, 2.8125, 25.3125, 42.1875]; - const jobDupParams: JobDuplicationParams = { - resourceId: layerFromCatalog.metadata.productId as string, - version: layerFromCatalog.metadata.productVersion as string, - dbId: layerFromCatalog.id, - zoomLevel: 7, - sanitizedBbox: expectedsanitizedBbox, - crs: userInput.crs as string, - }; - - findLayerMock.mockResolvedValue(layerFromCatalog); - createMock.mockResolvedValue(undefined); - updateJobMock.mockResolvedValue(undefined); - findCompletedJobMock.mockResolvedValue(undefined); - findInProgressJobMock.mockResolvedValue(JSON.parse(JSON.stringify(inProgressJob))); - - const res = await createPackageManager.createPackage(userInput); - const expectedReturn: ICreateJobResponse = { - id: inProgressJob.id, - taskIds: [(inProgressJob.tasks as unknown as IJobResponse[])[0].id], - status: OperationStatus.IN_PROGRESS, - }; - - expect(res).toEqual(expectedReturn); - expect(findLayerMock).toHaveBeenCalledWith(jobDupParams.dbId); - expect(findLayerMock).toHaveBeenCalledTimes(1); - expect(createMock).toHaveBeenCalledTimes(0); - expect(findCompletedJobMock).toHaveBeenNthCalledWith(1, jobDupParams); - expect(findCompletedJobMock).toHaveBeenNthCalledWith(2, jobDupParams); - expect(findCompletedJobMock).toHaveBeenCalledTimes(2); - expect(findInProgressJobMock).toHaveBeenCalledWith(jobDupParams); - expect(findInProgressJobMock).toHaveBeenCalledTimes(1); - expect(findPendingJobMock).toHaveBeenCalledTimes(0); - }); - - it('should increase callbacks array of existing in progress job', async () => { - const expectedsanitizedBbox: BBox = [0, 2.8125, 25.3125, 42.1875]; - const jobDupParams: JobDuplicationParams = { - resourceId: layerFromCatalog.metadata.productId as string, - version: layerFromCatalog.metadata.productVersion as string, - dbId: layerFromCatalog.id, - zoomLevel: 7, - sanitizedBbox: expectedsanitizedBbox, - crs: userInput.crs as string, - }; - const expirationDays: number = configMock.get('cleanupExpirationDays'); - const testExpirationDate = new Date(); - testExpirationDate.setDate(testExpirationDate.getDate() - expirationDays); - findLayerMock.mockResolvedValue(layerFromCatalog); - createMock.mockResolvedValue(undefined); - updateJobMock.mockResolvedValue(undefined); - findCompletedJobMock.mockResolvedValue(undefined); - findInProgressJobMock.mockResolvedValue(JSON.parse(JSON.stringify({ ...inProgressJob, expirationDate: testExpirationDate }))); - const jobUpdateParams = { - parameters: { - fileName: 'test.gpkg', - relativeDirectoryPath: 'test', - crs: 'EPSG:4326', - sanitizedBbox: [0, 0, 25, 41], - zoomLevel: 4, - exportVersion: ExportVersion.GETMAP, - callbacks: [ - { url: 'http://localhost:6969', bbox: [0, 0, 25, 41] }, - { url: 'http://new-added-callback-url.com', bbox: [-5, 3, 25, 41] }, - ], - targetResolution: 0.0439453125, - }, - }; - const res = await createPackageManager.createPackage({ ...userInput, callbackURLs: ['http://new-added-callback-url.com'] }); - const expectedReturn: ICreateJobResponse = { - id: inProgressJob.id, - taskIds: [(inProgressJob.tasks as unknown as IJobResponse[])[0].id], - status: OperationStatus.IN_PROGRESS, - }; - - expect(res).toEqual(expectedReturn); - expect(findLayerMock).toHaveBeenCalledWith(jobDupParams.dbId); - expect(findLayerMock).toHaveBeenCalledTimes(1); - expect(createMock).toHaveBeenCalledTimes(0); - expect(findCompletedJobMock).toHaveBeenNthCalledWith(1, jobDupParams); - expect(findCompletedJobMock).toHaveBeenNthCalledWith(2, jobDupParams); - expect(findCompletedJobMock).toHaveBeenCalledTimes(2); - expect(findInProgressJobMock).toHaveBeenCalledWith(jobDupParams); - expect(findInProgressJobMock).toHaveBeenCalledTimes(1); - expect(updateJobMock).toHaveBeenCalledWith('fa3ab609-377a-4d96-bf0b-e0bb72f683b8', jobUpdateParams); - expect(findPendingJobMock).toHaveBeenCalledTimes(0); - }); - - it('should throw bad request error when requested resolution is higher than the layer resolution', async () => { - const layer = { ...layerFromCatalog, metadata: { ...layerFromCatalog.metadata, maxResolutionDeg: 0.072 } }; - findLayerMock.mockResolvedValue(layer); - - const action = async () => createPackageManager.createPackage(userInput); - - await expect(action).rejects.toThrow(BadRequestError); - expect(findLayerMock).toHaveBeenCalledTimes(1); - expect(findLayerMock).toHaveBeenCalledWith(layer.id); - }); - }); describe('#createMetadata', () => { it('should create metadata.json file with the correct parameters', async () => { const fileName = 'file'; From 93288702a0ffb9a06520b803df111761b7a09daa Mon Sep 17 00:00:00 2001 From: razbroc Date: Mon, 18 Mar 2024 14:54:33 +0200 Subject: [PATCH 02/19] chore: remove deprications --- .../models/createPackageManager.ts | 148 ------------------ .../createPackage/createExportPackage.spec.ts | 3 - 2 files changed, 151 deletions(-) diff --git a/src/createPackage/models/createPackageManager.ts b/src/createPackage/models/createPackageManager.ts index c9e834c..663e13c 100644 --- a/src/createPackage/models/createPackageManager.ts +++ b/src/createPackage/models/createPackageManager.ts @@ -459,90 +459,6 @@ export class CreatePackageManager { } } - private generateTileGroups(polygon: Polygon | MultiPolygon, footprint: Polygon | MultiPolygon, zoom: number): ITileRange[] { - let intersaction: Feature | null; - - try { - intersaction = intersect(polygon, footprint); - if (intersaction === null) { - throw new BadRequestError( - `Requested ${JSON.stringify(polygon)} has no intersection with requested layer footprint: ${JSON.stringify(footprint)}` - ); - } - } catch (error) { - const message = `Error occurred while trying to generate tiles batches - intersaction error: ${JSON.stringify(error)}`; - this.logger.error({ - firstPolygon: polygon, - secondPolygon: footprint, - zoom: zoom, - message: message, - }); - throw new Error(message); - } - - try { - const tileRanger = new TileRanger(); - const tilesGroups: ITileRange[] = []; - - for (let i = 0; i <= zoom; i++) { - const zoomTilesGroups = tileRanger.encodeFootprint(intersaction, i); - for (const group of zoomTilesGroups) { - tilesGroups.push(group); - } - } - return tilesGroups; - } catch (error) { - const message = `Error occurred while trying to generate tiles batches - encodeFootprint error: ${JSON.stringify(error)}`; - this.logger.error({ - firstPolygon: polygon, - secondPolygon: footprint, - zoom: zoom, - message: message, - }); - throw new Error(message); - } - } - - /** - * @deprecated GetMap API - will be deprecated on future - */ - private async checkForDuplicate( - dupParams: JobDuplicationParams, - callbackUrls: ICallbackTarget[] - ): Promise { - let completedExists = await this.checkForCompleted(dupParams); - if (completedExists) { - return completedExists; - } - - const processingExists = await this.checkForProcessing(dupParams, callbackUrls); - if (processingExists) { - // For race condition - completedExists = await this.checkForCompleted(dupParams); - if (completedExists) { - return completedExists; - } - return processingExists; - } - - return undefined; - } - - /** - * @deprecated GetMap API - will be deprecated on future - */ - private async checkForCompleted(dupParams: JobDuplicationParams): Promise { - this.logger.info(dupParams, `Checking for COMPLETED duplications with parameters`); - const responseJob = await this.jobManagerClient.findCompletedJob(dupParams); - if (responseJob) { - await this.jobManagerClient.validateAndUpdateExpiration(responseJob.id); - return { - ...responseJob.parameters.callbackParams, - status: OperationStatus.COMPLETED, - } as ICallbackResponse; - } - } - private async checkForExportCompleted(dupParams: JobExportDuplicationParams): Promise { this.logger.info({ ...dupParams, roi: undefined, msg: `Checking for COMPLETED duplications with parameters` }); const responseJob = await this.jobManagerClient.findExportJob(OperationStatus.COMPLETED, dupParams); @@ -555,22 +471,6 @@ export class CreatePackageManager { } } - /** - * @deprecated GetMap API - will be deprecated on future - */ - private async checkForProcessing(dupParams: JobDuplicationParams, newCallbacks: ICallbackTarget[]): Promise { - this.logger.info(dupParams, `Checking for PROCESSING duplications with parameters`); - const processingJob = (await this.jobManagerClient.findInProgressJob(dupParams)) ?? (await this.jobManagerClient.findPendingJob(dupParams)); - if (processingJob) { - await this.updateCallbackURLs(processingJob, newCallbacks); - return { - id: processingJob.id, - taskIds: (processingJob.tasks as unknown as IJobResponse[]).map((t) => t.id), - status: OperationStatus.IN_PROGRESS, - }; - } - } - private async checkForExportProcessing( dupParams: JobExportDuplicationParams, newCallbacks?: ICallbackTargetExport[] @@ -589,44 +489,6 @@ export class CreatePackageManager { } } - /** - * @deprecated GetMap API - will be deprecated on future - */ - private async updateCallbackURLs(processingJob: JobResponse, newCallbacks: ICallbackTarget[]): Promise { - const callbacks = processingJob.parameters.callbacks; - for (const newCallback of newCallbacks) { - const hasCallback = callbacks.findIndex((callback) => { - const exist = callback.url === newCallback.url; - if (!exist) { - return false; - } - - if (this.isAPolygon(callback.bbox) && this.isAPolygon(newCallback.bbox)) { - return booleanEqual(newCallback.bbox, callback.bbox); - } else if (this.isAPolygon(callback.bbox) || this.isAPolygon(newCallback.bbox)) { - return false; - } - // else both BBoxes - let sameBboxCoordinate = false; - for (let i = 0; i < callback.bbox.length; i++) { - sameBboxCoordinate = callback.bbox[i] === newCallback.bbox[i]; - if (!sameBboxCoordinate) { - sameBboxCoordinate = false; - break; - } - } - return sameBboxCoordinate; - }); - // eslint-disable-next-line @typescript-eslint/no-magic-numbers - if (hasCallback === -1) { - callbacks.push(newCallback); - } - } - await this.jobManagerClient.updateJob(processingJob.id, { - parameters: processingJob.parameters, - }); - } - private async updateExportCallbackURLs(processingJob: JobExportResponse, newCallbacks?: ICallbackTargetExport[]): Promise { if (!newCallbacks) { return; @@ -657,16 +519,6 @@ export class CreatePackageManager { }); } - /** - * @deprecated GetMap API - will be deprecated on future - */ - private generatePackageName(productType: string, productId: string, productVersion: string, zoomLevel: number, bbox: BBox): string { - const numberOfDecimals = 5; - const bboxToString = bbox.map((val) => String(val.toFixed(numberOfDecimals)).replace('.', '_').replace(/-/g, 'm')).join(''); - const productVersionConvention = productVersion.replace('.', '_'); - return `${productType}_${productId}_${productVersionConvention}_${zoomLevel}_${bboxToString}.gpkg`; - } - private generateExportFileNames(productType: string, productId: string, productVersion: string, featuresRecords: IGeometryRecord[]): string { const maxZoom = Math.max(...featuresRecords.map((feature) => feature.zoomLevel)); let currentDateStr = new Date().toJSON(); diff --git a/tests/integration/createPackage/createExportPackage.spec.ts b/tests/integration/createPackage/createExportPackage.spec.ts index 7672e6f..a487788 100644 --- a/tests/integration/createPackage/createExportPackage.spec.ts +++ b/tests/integration/createPackage/createExportPackage.spec.ts @@ -18,7 +18,6 @@ describe('Export by ROI', function () { let validateFreeSpaceSpy: jest.SpyInstance; let checkForExportCompletedSpy: jest.SpyInstance; let checkForExportProcessingSpy: jest.SpyInstance; - let generateTileGroupsSpy: jest.SpyInstance; beforeEach(function () { const app = getApp({ @@ -39,7 +38,6 @@ describe('Export by ROI', function () { 'checkForExportProcessing' ); validateFreeSpaceSpy = jest.spyOn(CreatePackageManager.prototype as unknown as { validateFreeSpace: jest.Mock }, 'validateFreeSpace'); - generateTileGroupsSpy = jest.spyOn(CreatePackageManager.prototype as unknown as { generateTileGroups: jest.Mock }, 'generateTileGroups'); findLayerSpy = jest.spyOn(RasterCatalogManagerClient.prototype, 'findLayer'); createJobSpy = jest.spyOn(JobManagerWrapper.prototype, 'createJob'); }); @@ -335,7 +333,6 @@ describe('Export by ROI', function () { crs: 'EPSG:4326', priority: 0, }; - generateTileGroupsSpy.mockReturnValue([]); findLayerSpy.mockResolvedValue(layerFromCatalog); checkForExportDuplicateSpy.mockResolvedValue(undefined); validateFreeSpaceSpy.mockResolvedValue(false); From a5223a75c6715d5d9532f09d33faedab30d1212c Mon Sep 17 00:00:00 2001 From: razbroc Date: Mon, 18 Mar 2024 16:11:13 +0200 Subject: [PATCH 03/19] chore: remove deprication --- src/clients/jobManagerWrapper.ts | 50 ------------------- .../models/createPackageManager.ts | 37 -------------- src/finalizationManager.ts | 28 +---------- src/tasks/models/tasksManager.ts | 38 -------------- tests/unit/clients/jobManagerClient.spec.ts | 11 ---- .../createPackage/models/tasksModel.spec.ts | 42 ---------------- tests/unit/finalizationManager.spec.ts | 5 -- 7 files changed, 1 insertion(+), 210 deletions(-) diff --git a/src/clients/jobManagerWrapper.ts b/src/clients/jobManagerWrapper.ts index 36d2393..a941549 100644 --- a/src/clients/jobManagerWrapper.ts +++ b/src/clients/jobManagerWrapper.ts @@ -103,56 +103,6 @@ export class JobManagerWrapper extends JobManagerClient { return createJobResponse; } - /** - * @deprecated The method should not be used - */ - public async create(data: IWorkerInput): Promise { - const expirationDate = new Date(); - expirationDate.setDate(expirationDate.getDate() + this.expirationDays); - - const createJobRequest: CreateJobBody = { - resourceId: data.cswProductId, - version: data.version, - type: this.tilesJobType, - domain: this.jobDomain, - parameters: { - sanitizedBbox: data.sanitizedBbox, - targetResolution: data.targetResolution, - exportVersion: ExportVersion.GETMAP, - zoomLevel: data.zoomLevel, - callbacks: data.callbacks, - crs: data.crs, - fileName: data.fileName, - relativeDirectoryPath: data.relativeDirectoryPath, - gpkgEstimatedSize: data.gpkgEstimatedSize, - }, - internalId: data.dbId, - productType: data.productType, - productName: data.cswProductId, - priority: data.priority, - status: OperationStatus.IN_PROGRESS, - additionalIdentifiers: data.sanitizedBbox.toString() + String(data.zoomLevel), - tasks: [ - { - type: this.tilesTaskType, - parameters: { - isNewTarget: true, - targetFormat: data.targetFormat, - batches: data.batches, - sources: data.sources, - }, - }, - ], - }; - - const res = await this.createJob(createJobRequest); - return { - id: res.id, - taskIds: res.taskIds, - status: OperationStatus.IN_PROGRESS, - }; - } - /** * @deprecated The method should not be used */ diff --git a/src/createPackage/models/createPackageManager.ts b/src/createPackage/models/createPackageManager.ts index 663e13c..226d14d 100644 --- a/src/createPackage/models/createPackageManager.ts +++ b/src/createPackage/models/createPackageManager.ts @@ -409,43 +409,6 @@ export class CreatePackageManager { return this.tilesProvider === 'S3' ? '/' : sep; } - private normalize2Polygon(bboxFromUser: Polygon | MultiPolygon | BBox | undefined): Polygon | undefined { - try { - if (isArray(bboxFromUser) && bboxFromUser.length === CreatePackageManager.bboxLength2d) { - this.logger.debug({ ...bboxFromUser, msg: `Export will be executed by provided BBox from request input` }); - const resultPolygon = bboxPolygon(bboxFromUser as BBox); - return resultPolygon.geometry; - } else if (this.isAPolygon(bboxFromUser)) { - this.logger.debug({ ...bboxFromUser, msg: `Export will be executed by provided Footprint from request input` }); - return bboxFromUser; - } else if (!bboxFromUser) { - this.logger.debug(`Export will be executed on entire layer's footprint`); - return undefined; - } else { - this.logger.warn({ ...bboxFromUser, msg: `Input bbox param illegal - should be bbox | polygon | null types` }); - throw new BadRequestError('Input bbox param illegal - should be bbox | polygon | null types'); - } - } catch (error) { - this.logger.error({ bboxFromUser, msg: `Failed with error ${(error as Error).message}` }); - throw new BadRequestError('Input bbox param illegal - should be bbox | polygon | null types'); - } - } - - private isAPolygon(obj?: object): obj is Polygon { - if (obj === undefined) { - return false; - } - const isPolygon = 'type' in obj && 'coordinates' in obj && (obj as { type: string }).type === 'Polygon'; - if (isPolygon) { - const errors = geojsonhint.hint(obj); - if (!isEmpty(errors)) { - this.logger.warn({ bboxFromUser: obj, errors }, `Not a polygon`); - return false; - } - } - return isPolygon; - } - private sanitizeBbox(polygon: Polygon | MultiPolygon, footprint: Polygon | MultiPolygon, zoom: number): BBox | null { try { const intersaction = intersect(polygon, footprint); diff --git a/src/finalizationManager.ts b/src/finalizationManager.ts index 934c781..d8e820a 100644 --- a/src/finalizationManager.ts +++ b/src/finalizationManager.ts @@ -156,34 +156,8 @@ export class FinalizationManager { } public async jobStatusPoll(): Promise { - const getMapExistsJobs = await this.handleGetMapJobs(); const roiExistsJobs = await this.handleROIJobs(); - return getMapExistsJobs || roiExistsJobs; - } - - private async handleGetMapJobs(): Promise { - let existsJobs = false; - const getMapJobs = await this.taskManager.getJobsByTaskStatus(); // for old getmap api - will be removed - const expirationDateUTC = getUTCDate(); - expirationDateUTC.setDate(expirationDateUTC.getDate() + this.expirationDays); - this.logger.debug({ ...getMapJobs, msg: `Handling GetMap jobs` }); - if (getMapJobs.completedJobs && getMapJobs.completedJobs.length > 0) { - existsJobs = true; - this.logger.info({ msg: `GETMAP Completed GetMap jobs detected, running finalize job` }); - for (const job of getMapJobs.completedJobs) { - this.logger.info({ jobId: job.id, msg: `GETMAP Execute completed job finalizing on BBOX (GetMap) exporting for job: ${job.id}` }); - await this.taskManager.finalizeJob(job, expirationDateUTC); - } - } else if (getMapJobs.failedJobs && getMapJobs.failedJobs.length > 0) { - existsJobs = true; - this.logger.info({ msg: `GETMAP Failed jobs detected, running finalize job` }); - for (const job of getMapJobs.failedJobs) { - this.logger.info({ jobId: job.id, msg: `GETMAP Execute Failed job finalizing on BBOX (GetMap) exporting for job: ${job.id}` }); - const gpkgFailedErr = `failed to create gpkg, job: ${job.id}`; - await this.taskManager.finalizeJob(job, expirationDateUTC, false, gpkgFailedErr); - } - } - return existsJobs; + return roiExistsJobs; } private async handleROIJobs(): Promise { diff --git a/src/tasks/models/tasksManager.ts b/src/tasks/models/tasksManager.ts index 4506a2f..28cd579 100644 --- a/src/tasks/models/tasksManager.ts +++ b/src/tasks/models/tasksManager.ts @@ -206,44 +206,6 @@ export class TasksManager { } } - /** - * @deprecated GetMap API - will be deprecated on future - */ - public async finalizeJob(job: JobResponse, expirationDate: Date, isSuccess = true, reason?: string): Promise { - let updateJobParams: IUpdateJobBody = { - status: isSuccess ? OperationStatus.COMPLETED : OperationStatus.FAILED, - reason, - /* eslint-disable-next-line @typescript-eslint/no-magic-numbers */ - percentage: isSuccess ? 100 : undefined, - }; - - const cleanupData: ICleanupData = this.generateCleanupEntity(job, expirationDate); - - try { - this.logger.info({ jobId: job.id, msg: `GetMap Finalize Job` }); - const packageName = job.parameters.fileName; - if (isSuccess) { - const packageFullPath = getGpkgFullPath(this.gpkgsLocation, packageName); - await this.packageManager.createJsonMetadata(packageFullPath, job); - } - const callbackParams = await this.sendCallbacks(job, expirationDate, reason); - updateJobParams = { ...updateJobParams, parameters: { ...job.parameters, callbackParams, cleanupData } }; - - this.logger.info({ jobId: job.id, status: isSuccess, msg: `GetMap Update Job status` }); - await this.jobManagerClient.updateJob(job.id, updateJobParams); - } catch (error) { - this.logger.error({ - jobId: job.id, - err: error, - errorReason: (error as Error).message, - msg: `GetMap Could not finalize job, will updating to status failed`, - }); - const callbackParams = await this.sendCallbacks(job, expirationDate, reason); - updateJobParams = { ...updateJobParams, status: OperationStatus.FAILED, parameters: { ...job.parameters, callbackParams, cleanupData } }; - await this.jobManagerClient.updateJob(job.id, updateJobParams); - } - } - public async finalizeGPKGSuccess(job: JobFinalizeResponse, expirationDateUTC: Date): Promise> { // initialization to mutual finalized params const cleanupData: ICleanupData = this.generateCleanupEntity(job, expirationDateUTC); diff --git a/tests/unit/clients/jobManagerClient.spec.ts b/tests/unit/clients/jobManagerClient.spec.ts index e70cdb2..778e62f 100644 --- a/tests/unit/clients/jobManagerClient.spec.ts +++ b/tests/unit/clients/jobManagerClient.spec.ts @@ -39,17 +39,6 @@ describe('JobManagerClient', () => { }); describe('getMap', () => { - /** - * @deprecated GetMap API - will be deprecated on future - */ - it('should create job successfully', async () => { - postFun = jest.fn(); - (jobManagerClient as unknown as { post: unknown }).post = postFun.mockResolvedValue({ id: '123', taskIds: ['123'] }); - await jobManagerClient.create(workerInput); - - expect(postFun).toHaveBeenCalledTimes(1); - }); - it('should update job successfully', async () => { putFun = jest.fn(); (jobManagerClient as unknown as { put: unknown }).put = putFun.mockResolvedValue(undefined); diff --git a/tests/unit/createPackage/models/tasksModel.spec.ts b/tests/unit/createPackage/models/tasksModel.spec.ts index a67308e..a5c3ba9 100644 --- a/tests/unit/createPackage/models/tasksModel.spec.ts +++ b/tests/unit/createPackage/models/tasksModel.spec.ts @@ -197,48 +197,6 @@ describe('TasksManager', () => { }); }); - describe('#finalizeJob', () => { - let sendCallbacksSpy: jest.SpyInstance; - - it('should successfully finalize a job with status completed', async () => { - const expirationTime = new Date(); - createJsonMetadataMock.mockResolvedValue({}); - updateJobMock.mockResolvedValue({}); - sendCallbacksSpy = jest.spyOn(tasksManager, 'sendCallbacks'); - - const action = async () => tasksManager.finalizeJob(mockJob, expirationTime); - await expect(action()).resolves.not.toThrow(); - expect(createJsonMetadataMock).toHaveBeenCalledTimes(1); - expect(sendCallbacksSpy).toHaveBeenCalledTimes(1); - expect(updateJobMock).toHaveBeenCalledTimes(1); - }); - - it('should successfully finalize a job with status failed due to error while create json metadata file', async () => { - const expirationTime = new Date(); - createJsonMetadataMock.mockRejectedValue({}); - updateJobMock.mockResolvedValue({}); - sendCallbacksSpy = jest.spyOn(tasksManager, 'sendCallbacks'); - - const action = async () => tasksManager.finalizeJob(mockJob, expirationTime); - await expect(action()).resolves.not.toThrow(); - expect(createJsonMetadataMock).toHaveBeenCalledTimes(1); - expect(sendCallbacksSpy).toHaveBeenCalledTimes(1); - expect(updateJobMock).toHaveBeenCalledTimes(1); - }); - - it('should successfully finalize a job with job status failed without create json metadata file due to failed in task', async () => { - const expirationTime = new Date(); - updateJobMock.mockResolvedValue({}); - sendCallbacksSpy = jest.spyOn(tasksManager, 'sendCallbacks'); - - const errMessage = 'gpkg failed to create'; - const action = async () => tasksManager.finalizeJob(mockJob, expirationTime, false, errMessage); - await expect(action()).resolves.not.toThrow(); - expect(createJsonMetadataMock).toHaveBeenCalledTimes(0); - expect(sendCallbacksSpy).toHaveBeenCalledTimes(1); - expect(updateJobMock).toHaveBeenCalledTimes(1); - }); - }); }); describe('ROI', () => { describe('#getTaskStatusByJobId', () => { diff --git a/tests/unit/finalizationManager.spec.ts b/tests/unit/finalizationManager.spec.ts index ee4a233..ad59ece 100644 --- a/tests/unit/finalizationManager.spec.ts +++ b/tests/unit/finalizationManager.spec.ts @@ -65,9 +65,7 @@ describe('FinalizationManager', () => { await finalizationManager.jobStatusPoll(); - expect(getJobsByTaskStatusMock).toHaveBeenCalledTimes(1); expect(getExportJobsByTaskStatusMock).toHaveBeenCalledTimes(1); - expect(finalizeJobMock).toHaveBeenCalledTimes(2); expect(createFinalizeTaskMock).toHaveBeenCalledTimes(3); }); @@ -92,9 +90,6 @@ describe('FinalizationManager', () => { await finalizationManager.jobStatusPoll(); - expect(getJobsByTaskStatusMock).toHaveBeenCalledTimes(1); - expect(getExportJobsByTaskStatusMock).toHaveBeenCalledTimes(1); - expect(finalizeJobMock).toHaveBeenCalledTimes(2); expect(createFinalizeTaskMock).toHaveBeenCalledTimes(3); }); }); From a42a56e6956f174acd4ac772f7b35ef0b0a551b4 Mon Sep 17 00:00:00 2001 From: razbroc Date: Mon, 18 Mar 2024 16:19:15 +0200 Subject: [PATCH 04/19] chore: remove deprecated createMetadata --- .../models/createPackageManager.ts | 23 ---------- .../models/createPackageModel.spec.ts | 42 ------------------- 2 files changed, 65 deletions(-) diff --git a/src/createPackage/models/createPackageManager.ts b/src/createPackage/models/createPackageManager.ts index 226d14d..b51c797 100644 --- a/src/createPackage/models/createPackageManager.ts +++ b/src/createPackage/models/createPackageManager.ts @@ -312,29 +312,6 @@ export class CreatePackageManager { return undefined; } - /** - * @deprecated GetMap API - will be deprecated on future - */ - public async createJsonMetadata(fullGpkgPath: string, job: JobResponse): Promise { - this.logger.info({ - jobId: job.id, - msg: `Creating metadata.json file for gpkg in path "${this.gpkgsLocation}/${fullGpkgPath}" for jobId ${job.id}`, - }); - const record = await this.rasterCatalogManager.findLayer(job.internalId as string); - - const parsedPath = parsePath(fullGpkgPath); - const directoryName = parsedPath.dir; - const metadataFileName = parsedPath.name.concat(METADATA_JSON_FILE_EXTENSION); - const metadataFilePath = `${directoryName}${sep}${metadataFileName}`; - const sanitizedBboxToPolygon = bboxPolygon(job.parameters.sanitizedBbox); - - record.metadata.footprint = sanitizedBboxToPolygon; - record.metadata.maxResolutionDeg = job.parameters.targetResolution; - - const recordMetadata = JSON.stringify(record.metadata); - await fsPromise.writeFile(metadataFilePath, recordMetadata); - } - public async createExportJsonMetadata(job: JobExportResponse | JobFinalizeResponse): Promise { this.logger.info({ jobId: job.id, diff --git a/tests/unit/createPackage/models/createPackageModel.spec.ts b/tests/unit/createPackage/models/createPackageModel.spec.ts index 2c88a1c..8c6042f 100644 --- a/tests/unit/createPackage/models/createPackageModel.spec.ts +++ b/tests/unit/createPackage/models/createPackageModel.spec.ts @@ -78,49 +78,7 @@ describe('CreatePackageManager', () => { jest.resetAllMocks(); jest.restoreAllMocks(); }); - /** - * @deprecated GetMap API - will be deprecated on future - */ - describe('GetMAP', () => { - describe('#createMetadata', () => { - it('should create metadata.json file with the correct parameters', async () => { - const fileName = 'file'; - const directoryName = '/tmp/gpkgDir'; - - const mockGgpkgPath = `${directoryName}/${fileName}`; - - findLayerMock.mockResolvedValue(layerFromCatalog); - - await createPackageManager.createJsonMetadata(mockGgpkgPath, completedJob); - - const expectedFileName = `${directoryName}${sep}${fileName}${METADA_JSON_FILE_EXTENSION}`; - const expectedMetadata: LayerMetadata = { - ...layerFromCatalog.metadata, - maxResolutionDeg: completedJob.parameters.targetResolution, - footprint: { - type: 'Feature', - bbox: [0, 0, 25, 41], - properties: {}, - geometry: { - type: 'Polygon', - coordinates: [ - [ - [0, 0], - [25, 0], - [25, 41], - [0, 41], - [0, 0], - ], - ], - }, - }, - }; - expect(fs.promises.writeFile).toHaveBeenCalledTimes(1); - expect(fs.promises.writeFile).toHaveBeenCalledWith(expectedFileName, JSON.stringify(expectedMetadata)); - }); - }); - }); describe('ROI', () => { describe('#createExportMetadata', () => { it('should create metadata.json file with the correct parameters', async () => { From 87b03a1adfc29008eb68fc76e6c92a25f0c3ac09 Mon Sep 17 00:00:00 2001 From: razbroc Date: Mon, 18 Mar 2024 16:32:12 +0200 Subject: [PATCH 05/19] chore: remove sendCallbacks --- src/tasks/models/tasksManager.ts | 64 -------- .../createPackage/models/tasksModel.spec.ts | 154 ------------------ 2 files changed, 218 deletions(-) diff --git a/src/tasks/models/tasksManager.ts b/src/tasks/models/tasksManager.ts index 28cd579..62ca627 100644 --- a/src/tasks/models/tasksManager.ts +++ b/src/tasks/models/tasksManager.ts @@ -92,70 +92,6 @@ export class TasksManager { } } - /** - * @deprecated GetMap API - will be deprecated on future - */ - public async sendCallbacks(job: JobResponse, expirationDate: Date, errorReason?: string): Promise { - let fileUri = ''; - let fileRelativePath = ''; - try { - this.logger.info(`Sending callback for job: ${job.id}`); - const packageName = job.parameters.fileName; - const success = errorReason === undefined; - let fileSize = 0; - if (success) { - fileRelativePath = getGpkgRelativePath(packageName); - const packageFullPath = getGpkgFullPath(this.gpkgsLocation, packageName); - fileUri = `${this.downloadServerUrl}/downloads/${fileRelativePath}`; - fileSize = await getFileSize(packageFullPath); - } - const callbackParams: ICallbackDataBase = { - fileUri, - expirationTime: expirationDate, - fileSize, - dbId: job.internalId as string, - packageName: packageName, - requestId: job.id, - targetResolution: job.parameters.targetResolution, - success, - errorReason, - }; - - const targetCallbacks = job.parameters.callbacks; - const callbackPromises: Promise[] = []; - for (const target of targetCallbacks) { - const params: ICallbackData = { ...callbackParams, bbox: target.bbox }; - callbackPromises.push(this.callbackClient.send(target.url, params)); - } - - const promisesResponse = await Promise.allSettled(callbackPromises); - promisesResponse.forEach((response, index) => { - if (response.status === 'rejected') { - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - this.logger.error({ reason: response.reason, url: targetCallbacks[index].url, jobId: job.id, msg: `Failed to send callback to url` }); - } - }); - - return callbackParams; - } catch (error) { - this.logger.error({ jobId: job.id, err: error, reason: (error as Error).message, msg: `Sending callback has failed` }); - } - } - - /** - * @deprecated GetMap API - will be deprecated on future - */ - public async getJobsByTaskStatus(): Promise { - const jobs = await this.jobManagerClient.getInProgressJobs(); - const completedJobs = jobs?.filter((job) => job.completedTasks === job.taskCount); - const failedJobs = jobs?.filter((job) => job.failedTasks === job.taskCount); - const jobsStatus = { - completedJobs: completedJobs, - failedJobs: failedJobs, - }; - return jobsStatus; - } - public async getFinalizeJobById(jobId: string): Promise> { const job = await this.jobManagerClient.getJob(jobId); return job; diff --git a/tests/unit/createPackage/models/tasksModel.spec.ts b/tests/unit/createPackage/models/tasksModel.spec.ts index a5c3ba9..7f0f542 100644 --- a/tests/unit/createPackage/models/tasksModel.spec.ts +++ b/tests/unit/createPackage/models/tasksModel.spec.ts @@ -43,161 +43,7 @@ describe('TasksManager', () => { jest.resetAllMocks(); jest.restoreAllMocks(); }); - describe('GetMap', () => { - /** - * @deprecated GetMap API - will be deprecated on future - */ - describe('#getJobsByTaskStatus', () => { - it('should return completed job with no failed jobs', async () => { - const jobs: JobResponse[] = []; - const completedMockJob = { ...mockJob, completedTasks: 1 }; - jobs.push(completedMockJob); - getInProgressJobsMock.mockResolvedValue(jobs); - - const jobsStatus = await tasksManager.getJobsByTaskStatus(); - - expect(jobsStatus.completedJobs?.length).toBe(1); - expect(jobsStatus.failedJobs?.length).toBe(0); - expect(getInProgressJobsMock).toHaveBeenCalledTimes(1); - }); - - it('should return failed job with no completed jobs', async () => { - const jobs: JobResponse[] = []; - const failedMockJob = { ...mockJob, failedTasks: 1 }; - jobs.push(failedMockJob); - getInProgressJobsMock.mockResolvedValue(jobs); - - const jobsStatus = await tasksManager.getJobsByTaskStatus(); - - expect(jobsStatus.completedJobs?.length).toBe(0); - expect(jobsStatus.failedJobs?.length).toBe(1); - expect(getInProgressJobsMock).toHaveBeenCalledTimes(1); - }); - - it('should return completed job and failed job', async () => { - const jobs: JobResponse[] = []; - const completedMockJob = { ...mockJob, completedTasks: 1 }; - const failedMockJob = { ...mockJob, failedTasks: 1 }; - jobs.push(completedMockJob, failedMockJob); - getInProgressJobsMock.mockResolvedValue(jobs); - - const jobsStatus = await tasksManager.getJobsByTaskStatus(); - - expect(jobsStatus.completedJobs?.length).toBe(1); - expect(jobsStatus.failedJobs?.length).toBe(1); - expect(getInProgressJobsMock).toHaveBeenCalledTimes(1); - }); - - it('should return an empty jobs response if task is in progress', async () => { - const jobs: JobResponse[] = []; - - const inProgressMockJob = { ...mockJob, inProgressTasks: 1 }; - jobs.push(inProgressMockJob); - getInProgressJobsMock.mockResolvedValue(jobs); - - const jobsStatus = await tasksManager.getJobsByTaskStatus(); - - expect(jobsStatus.completedJobs?.length).toBe(0); - expect(jobsStatus.failedJobs?.length).toBe(0); - expect(getInProgressJobsMock).toHaveBeenCalledTimes(1); - }); - - it('should return an empty jobs response if task is in pending', async () => { - const jobs: JobResponse[] = []; - const pendingMockJob = { ...mockJob, pendingTasks: 1 }; - jobs.push(pendingMockJob); - getInProgressJobsMock.mockResolvedValue(jobs); - - const jobsStatus = await tasksManager.getJobsByTaskStatus(); - - expect(jobsStatus.completedJobs?.length).toBe(0); - expect(jobsStatus.failedJobs?.length).toBe(0); - expect(getInProgressJobsMock).toHaveBeenCalledTimes(1); - }); - - it('should return an empty jobs response if task is in expired', async () => { - const jobs: JobResponse[] = []; - const expiredMockJob = { ...mockJob, expiredTasks: 1 }; - jobs.push(expiredMockJob); - getInProgressJobsMock.mockResolvedValue(jobs); - const jobsStatus = await tasksManager.getJobsByTaskStatus(); - - expect(jobsStatus.completedJobs?.length).toBe(0); - expect(jobsStatus.failedJobs?.length).toBe(0); - expect(getInProgressJobsMock).toHaveBeenCalledTimes(1); - }); - - it('should return an empty jobs response if task is in aborted', async () => { - const jobs: JobResponse[] = []; - const abortedMockJob = { ...mockJob, abortedTasks: 1 }; - jobs.push(abortedMockJob); - getInProgressJobsMock.mockResolvedValue(jobs); - - const jobsStatus = await tasksManager.getJobsByTaskStatus(); - - expect(jobsStatus.completedJobs?.length).toBe(0); - expect(jobsStatus.failedJobs?.length).toBe(0); - expect(getInProgressJobsMock).toHaveBeenCalledTimes(1); - }); - }); - - /** - * @deprecated GetMap API - will be deprecated on future - */ - describe('#sendCallbacks', () => { - it('should return callback data with the expected params for success jobs', async () => { - sendMock.mockResolvedValue(200); - const getFileSizeSpy = jest.spyOn(utils, 'getFileSize'); - getFileSizeSpy.mockResolvedValue(2000); - const expirationTime = new Date(); - const expectedCallbackData: ICallbackDataBase = { - fileUri: `http://download-service/downloads/test${sep}test.gpkg`, - expirationTime: expirationTime, - fileSize: 2000, - dbId: '880a9316-0f10-4874-92e2-a62d587a1169', - packageName: 'test.gpkg', - requestId: 'b729f0e0-af64-4c2c-ba4e-e799e2f3df0f', - targetResolution: 0.072, - success: true, - errorReason: undefined, - }; - - const callbackData = await tasksManager.sendCallbacks(mockJob, expirationTime); - expect(callbackData).toEqual(expectedCallbackData); - }); - - it('should return callback data with the expected params for failed jobs', async () => { - sendMock.mockResolvedValue(200); - const getFileSizeSpy = jest.spyOn(utils, 'getFileSize'); - getFileSizeSpy.mockResolvedValue(2000); - const expirationTime = new Date(); - const errMessage = 'gpkg failed to create'; - const expectedCallbackData: ICallbackDataBase = { - fileUri: '', - expirationTime: expirationTime, - fileSize: 0, - dbId: '880a9316-0f10-4874-92e2-a62d587a1169', - packageName: 'test.gpkg', - requestId: 'b729f0e0-af64-4c2c-ba4e-e799e2f3df0f', - targetResolution: 0.072, - success: false, - errorReason: errMessage, - }; - const callbackData = await tasksManager.sendCallbacks(mockJob, expirationTime, errMessage); - expect(callbackData).toEqual(expectedCallbackData); - }); - - it('should return callback data even if callback response got rejected', async () => { - sendMock.mockRejectedValue({}); - const expirationTime = new Date(); - - const action = async () => tasksManager.sendCallbacks(mockJob, expirationTime); - await expect(action()).resolves.not.toThrow(); - }); - }); - - }); describe('ROI', () => { describe('#getTaskStatusByJobId', () => { it('should throw NotFoundError when jobId is not exists', async () => { From 086627a0caae02ba23eb4458ec1068d6fd652549 Mon Sep 17 00:00:00 2001 From: razbroc Date: Mon, 18 Mar 2024 16:53:07 +0200 Subject: [PATCH 06/19] fix: getInProgressJobs was using only get map jobs + removed deprications --- src/clients/jobManagerWrapper.ts | 87 +------------------ .../models/createPackageManager.ts | 2 +- tests/unit/clients/jobManagerClient.spec.ts | 59 ------------- 3 files changed, 4 insertions(+), 144 deletions(-) diff --git a/src/clients/jobManagerWrapper.ts b/src/clients/jobManagerWrapper.ts index a941549..d6b51d9 100644 --- a/src/clients/jobManagerWrapper.ts +++ b/src/clients/jobManagerWrapper.ts @@ -103,27 +103,6 @@ export class JobManagerWrapper extends JobManagerClient { return createJobResponse; } - /** - * @deprecated The method should not be used - */ - public async findCompletedJob(jobParams: JobDuplicationParams): Promise { - const queryParams: IFindJobsRequest = { - resourceId: jobParams.resourceId, - version: jobParams.version, - isCleaned: false, - type: this.tilesJobType, - shouldReturnTasks: false, - status: OperationStatus.COMPLETED, - }; - const jobs = await this.getGetMapJobs(queryParams); - if (jobs) { - const matchingJob = this.findJobWithMatchingParams(jobs, jobParams); - return matchingJob; - } - - return undefined; - } - public async findExportJob( status: OperationStatus, jobParams: JobExportDuplicationParams, @@ -146,61 +125,14 @@ export class JobManagerWrapper extends JobManagerClient { return undefined; } - /** - * @deprecated The method should not be used - */ - public async findInProgressJob(jobParams: JobDuplicationParams): Promise { - const queryParams: IFindJobsRequest = { - resourceId: jobParams.resourceId, - version: jobParams.version, - isCleaned: false, - type: this.tilesJobType, - shouldReturnTasks: true, - status: OperationStatus.IN_PROGRESS, - }; - - const jobs = await this.getGetMapJobs(queryParams); - if (jobs) { - const matchingJob = this.findJobWithMatchingParams(jobs, jobParams); - return matchingJob; - } - - return undefined; - } - - /** - * @deprecated The method should not be used - */ - public async findPendingJob(jobParams: JobDuplicationParams): Promise { - const queryParams: IFindJobsRequest = { - resourceId: jobParams.resourceId, - version: jobParams.version, - isCleaned: false, - type: this.tilesJobType, - shouldReturnTasks: true, - status: OperationStatus.PENDING, - }; - - const jobs = await this.getGetMapJobs(queryParams); - if (jobs) { - const matchingJob = this.findJobWithMatchingParams(jobs, jobParams); - return matchingJob; - } - - return undefined; - } - - /** - * @deprecated GetMap API - will be deprecated on future - */ - public async getInProgressJobs(shouldReturnTasks = false): Promise { + public async getInProgressJobs(shouldReturnTasks = false): Promise { const queryParams: IFindJobsRequest = { isCleaned: false, type: this.tilesJobType, shouldReturnTasks, status: OperationStatus.IN_PROGRESS, }; - const jobs = await this.getGetMapJobs(queryParams); + const jobs = await this.getExportJobs(queryParams); return jobs; } @@ -246,6 +178,7 @@ export class JobManagerWrapper extends JobManagerClient { } } + //TODO: once will be only one kind of exported jobs, no need to filter by ROI's public async getExportJobs(queryParams: IFindJobsRequest): Promise { this.logger.debug({ ...queryParams }, `Getting jobs that match these parameters`); const jobs = await this.get('/jobs', queryParams as unknown as Record); @@ -257,20 +190,6 @@ export class JobManagerWrapper extends JobManagerClient { return exportJobs; } - /** - * @deprecated GetMap API - will be deprecated on future - */ - private async getGetMapJobs(queryParams: IFindJobsRequest): Promise { - this.logger.debug({ ...queryParams }, `Getting jobs that match these parameters`); - const jobs = await this.get('/jobs', queryParams as unknown as Record); - const exportJobs = jobs?.filter((job) => { - if (job.parameters.exportVersion === ExportVersion.GETMAP) { - return job; - } - }); - return exportJobs; - } - /** * @deprecated GetMap API - will be deprecated on future */ diff --git a/src/createPackage/models/createPackageManager.ts b/src/createPackage/models/createPackageManager.ts index b51c797..09872f7 100644 --- a/src/createPackage/models/createPackageManager.ts +++ b/src/createPackage/models/createPackageManager.ts @@ -273,7 +273,7 @@ export class CreatePackageManager { const storageStatus: IStorageStatusResponse = await getStorageStatus(this.gpkgsLocation); let otherRunningJobsSize = 0; - const inProcessingJobs: JobResponse[] | undefined = await this.jobManagerClient.getInProgressJobs(); + const inProcessingJobs: JobExportResponse[] | undefined = await this.jobManagerClient.getInProgressJobs(); if (inProcessingJobs !== undefined && inProcessingJobs.length !== 0) { inProcessingJobs.forEach((job) => { let jobGpkgEstimatedSize = job.parameters.gpkgEstimatedSize as number; diff --git a/tests/unit/clients/jobManagerClient.spec.ts b/tests/unit/clients/jobManagerClient.spec.ts index 778e62f..ba5f7e7 100644 --- a/tests/unit/clients/jobManagerClient.spec.ts +++ b/tests/unit/clients/jobManagerClient.spec.ts @@ -47,65 +47,6 @@ describe('JobManagerClient', () => { expect(putFun).toHaveBeenCalledTimes(1); }); - /** - * @deprecated GetMap API - will be deprecated on future - */ - it('should findCompletedJobs successfully', async () => { - getGetMapJobs = jest.fn(); - - const jobManager = jobManagerClient as unknown as { getGetMapJobs: unknown }; - jobManager.getGetMapJobs = getGetMapJobs.mockResolvedValue(jobs); - const completedJobs = await jobManagerClient.findCompletedJob({ - resourceId: jobs[0].resourceId, - version: jobs[0].version, - dbId: jobs[0].internalId as string, - zoomLevel: jobs[0].parameters.zoomLevel, - crs: 'EPSG:4326', - sanitizedBbox: jobs[0].parameters.sanitizedBbox, - }); - expect(getGetMapJobs).toHaveBeenCalledTimes(1); - expect(completedJobs).toBeDefined(); - }); - - /** - * @deprecated GetMap API - will be deprecated on future - */ - it('should findInProgressJob successfully', async () => { - getGetMapJobs = jest.fn(); - - const jobManager = jobManagerClient as unknown as { getGetMapJobs: unknown }; - jobManager.getGetMapJobs = getGetMapJobs.mockResolvedValue(jobs); - - const completedJobs = await jobManagerClient.findInProgressJob({ - resourceId: jobs[0].resourceId, - version: jobs[0].version, - dbId: jobs[0].internalId as string, - zoomLevel: jobs[0].parameters.zoomLevel, - crs: 'EPSG:4326', - sanitizedBbox: jobs[0].parameters.sanitizedBbox, - }); - - expect(getGetMapJobs).toHaveBeenCalledTimes(1); - expect(completedJobs).toBeDefined(); - }); - - /** - * @deprecated GetMap API - will be deprecated on future - */ - it('should get In-Progress jobs status successfully', async () => { - getGetMapJobs = jest.fn(); - const jobs: JobResponse[] = []; - jobs.push(inProgressJob); - const jobManager = jobManagerClient as unknown as { getGetMapJobs: unknown }; - jobManager.getGetMapJobs = getGetMapJobs.mockResolvedValue(jobs); - - const result = await jobManagerClient.getInProgressJobs(); - - expect(getGetMapJobs).toHaveBeenCalledTimes(1); - expect(result).toBeDefined(); - expect(result).toEqual(jobs); - }); - /** * @deprecated GetMap API - will be deprecated on future */ From 26c1c6d4588e1bd1b5ab458282ca2f46b192946c Mon Sep 17 00:00:00 2001 From: razbroc Date: Mon, 18 Mar 2024 16:59:27 +0200 Subject: [PATCH 07/19] chore: remove ICallbackTarget --- src/clients/jobManagerWrapper.ts | 15 --------------- src/common/interfaces.ts | 9 --------- src/createPackage/models/createPackageManager.ts | 1 - tests/mocks/data.ts | 13 ------------- tests/mocks/data/mockJob.ts | 10 ---------- 5 files changed, 48 deletions(-) diff --git a/src/clients/jobManagerWrapper.ts b/src/clients/jobManagerWrapper.ts index d6b51d9..08f69a2 100644 --- a/src/clients/jobManagerWrapper.ts +++ b/src/clients/jobManagerWrapper.ts @@ -190,21 +190,6 @@ export class JobManagerWrapper extends JobManagerClient { return exportJobs; } - /** - * @deprecated GetMap API - will be deprecated on future - */ - private findJobWithMatchingParams(jobs: JobResponse[], jobParams: JobDuplicationParams): JobResponse | undefined { - const matchingJob = jobs.find( - (job) => - job.internalId === jobParams.dbId && - job.version === jobParams.version && - job.parameters.zoomLevel === jobParams.zoomLevel && - job.parameters.crs === jobParams.crs && - booleanEqual(bboxPolygon(job.parameters.sanitizedBbox), bboxPolygon(jobParams.sanitizedBbox)) - ); - return matchingJob; - } - private findExportJobWithMatchingParams(jobs: JobExportResponse[], jobParams: JobExportDuplicationParams): JobExportResponse | undefined { const matchingJob = jobs.find( (job) => diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index 3d5875c..d56f5f7 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -43,13 +43,6 @@ export interface ICallbackBase { url: string; } -/** - * @deprecated GetMap API - will be deprecated on future - */ -export interface ICallbackTarget extends ICallbackBase { - bbox: BBox | Polygon; -} - export interface ICallbackTargetExport extends ICallbackBase { roi: FeatureCollection; } @@ -75,7 +68,6 @@ export interface IWorkerInputBase { export interface IWorkerInput extends IWorkerInputBase { targetResolution: number; fileName: string; - callbacks: ICallbackTarget[]; sanitizedBbox: BBox; zoomLevel: number; } @@ -199,7 +191,6 @@ export interface IJobParameters { relativeDirectoryPath: string; crs: string; exportVersion: ExportVersion; - callbacks: ICallbackTarget[]; sanitizedBbox: BBox; zoomLevel: number; callbackParams?: ICallbackDataBase; diff --git a/src/createPackage/models/createPackageManager.ts b/src/createPackage/models/createPackageManager.ts index 09872f7..a6ef343 100644 --- a/src/createPackage/models/createPackageManager.ts +++ b/src/createPackage/models/createPackageManager.ts @@ -68,7 +68,6 @@ import { JobResponse, MergerSourceType, IMapSource, - ICallbackTarget, ITaskParameters, IStorageStatusResponse, } from '../../common/interfaces'; diff --git a/tests/mocks/data.ts b/tests/mocks/data.ts index 3e41075..c109c6a 100644 --- a/tests/mocks/data.ts +++ b/tests/mocks/data.ts @@ -218,12 +218,6 @@ const completedJob: IJobResponse = { fileName: 'test.gpkg', relativeDirectoryPath: 'test', zoomLevel: 4, - callbacks: [ - { - url: 'http://localhost:1234', - bbox: [0, 0, 25, 41], - }, - ], callbackParams: { dbId: '0c3e455f-4aeb-4258-982d-f7773469a92d', fileUri: 'http://localhost:4515/downloads/gm_0c3e455f_4aeb_4258_982d_f7773469a92d_4_0_000000_0000025_0000041_00000.gpkg', @@ -300,7 +294,6 @@ const inProgressJob: IJobResponse = { crs: 'EPSG:4326', sanitizedBbox: [0, 0, 25, 41], zoomLevel: 4, - callbacks: [{ url: 'http://localhost:6969', bbox: [0, 0, 25, 41] }], targetResolution: 0.0439453125, }, status: OperationStatus.IN_PROGRESS, @@ -353,12 +346,6 @@ const workerInput: IWorkerInput = { targetResolution: 0.0000429153442382812, zoomLevel: 15, dbId: '0c3e455f-4aeb-4258-982d-f7773469a92d', - callbacks: [ - { - bbox: [0, 3, 25, 41], - url: 'http://localhost:6969', - }, - ], sources: [ { path: 'test.gpkg', diff --git a/tests/mocks/data/mockJob.ts b/tests/mocks/data/mockJob.ts index ce31ccb..3e38ac2 100644 --- a/tests/mocks/data/mockJob.ts +++ b/tests/mocks/data/mockJob.ts @@ -17,16 +17,6 @@ export const mockJob: JobResponse = { crs: 'EPSG:4326', fileName: 'test.gpkg', exportVersion: ExportVersion.GETMAP, - callbacks: [ - { - url: 'http://example.getmap.com/callback', - bbox: [34.811938017107494, 31.95475033759175, 34.82237261707599, 31.96426962177354], - }, - { - url: 'http://example.getmap.com/callback2', - bbox: [34.811938017107494, 31.95475033759175, 34.82237261707599, 31.96426962177354], - }, - ], zoomLevel: 3, sanitizedBbox: [0, -90, 180, 90], targetResolution: 0.072, From 90f43e7888a1a4dfe42233a2d69ef1d5c49f1281 Mon Sep 17 00:00:00 2001 From: razbroc Date: Mon, 18 Mar 2024 17:10:03 +0200 Subject: [PATCH 08/19] chore: remove interface deprications --- src/clients/callbackClient.ts | 4 +- src/clients/jobManagerWrapper.ts | 2 - src/common/interfaces.ts | 33 ------------- .../controllers/createPackageController.ts | 4 +- .../models/createPackageManager.ts | 3 -- src/tasks/models/tasksManager.ts | 2 - tests/mocks/data.ts | 48 ------------------- tests/unit/clients/jobManagerClient.spec.ts | 1 - .../models/createPackageModel.spec.ts | 1 - 9 files changed, 3 insertions(+), 95 deletions(-) diff --git a/src/clients/callbackClient.ts b/src/clients/callbackClient.ts index 91c674b..06feedf 100644 --- a/src/clients/callbackClient.ts +++ b/src/clients/callbackClient.ts @@ -2,7 +2,7 @@ import { inject, singleton } from 'tsyringe'; import { HttpClient, IHttpRetryConfig } from '@map-colonies/mc-utils'; import { Logger } from '@map-colonies/js-logger'; import { SERVICES } from '../common/constants'; -import { ICallbackData, ICallbackExportData, IConfig } from '../common/interfaces'; +import { ICallbackExportData, IConfig } from '../common/interfaces'; @singleton() export class CallbackClient extends HttpClient { @@ -16,7 +16,7 @@ export class CallbackClient extends HttpClient { ); } - public async send(callbackUrl: string, data: ICallbackData | ICallbackExportData): Promise { + public async send(callbackUrl: string, data: ICallbackExportData): Promise { this.logger.info({ data, msg: `Sending callback request to URL: "${callbackUrl}"` }); await this.post(callbackUrl, data); } diff --git a/src/clients/jobManagerWrapper.ts b/src/clients/jobManagerWrapper.ts index 08f69a2..9732df5 100644 --- a/src/clients/jobManagerWrapper.ts +++ b/src/clients/jobManagerWrapper.ts @@ -12,13 +12,11 @@ import { CreateExportJobBody, CreateJobBody, ExportVersion, - ICreateJobResponse, ICreateExportJobResponse, IJobExportParameters, IJobParameters, ITaskParameters, IWorkerExportInput, - IWorkerInput, JobDuplicationParams, JobExportDuplicationParams, JobExportResponse, diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index d56f5f7..b30e1af 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -62,16 +62,6 @@ export interface IWorkerInputBase { targetFormat?: TileOutputFormat; } -/** - * @deprecated GetMap API - will be deprecated on future - */ -export interface IWorkerInput extends IWorkerInputBase { - targetResolution: number; - fileName: string; - sanitizedBbox: BBox; - zoomLevel: number; -} - export interface IWorkerExportInput extends IWorkerInputBase { callbacks?: ICallbackTargetExport[]; roi: FeatureCollection; @@ -83,15 +73,6 @@ export interface IBasicResponse { message: string; } -/** - * @deprecated GetMap API - will be deprecated on future - */ -export interface ICreateJobResponse { - id: string; - taskIds: string[]; - status: OperationStatus.IN_PROGRESS | OperationStatus.COMPLETED; -} - export interface ICreateExportJobResponse { jobId: string; taskIds: string[]; @@ -125,24 +106,10 @@ export interface ICallbackDataExportBase { artifacts?: IArtifactDefinition[]; } -/** - * @deprecated GetMap API - will be deprecated on future - */ -export interface ICallbackData extends ICallbackDataBase { - bbox: BBox | Polygon | MultiPolygon; -} - export interface ICallbackExportData extends ICallbackDataExportBase { roi: FeatureCollection; } -/** - * @deprecated GetMap API - will be deprecated on future - */ -export interface ICallbackResposne extends ICallbackData { - status: OperationStatus.IN_PROGRESS | OperationStatus.COMPLETED; -} - //todo - should be replaced and imported from exporter SDK export interface IArtifactDefinition { name: string; diff --git a/src/createPackage/controllers/createPackageController.ts b/src/createPackage/controllers/createPackageController.ts index c32da55..e1f7fcd 100644 --- a/src/createPackage/controllers/createPackageController.ts +++ b/src/createPackage/controllers/createPackageController.ts @@ -6,8 +6,6 @@ import { SERVICES } from '../../common/constants'; import { CreatePackageManager } from '../models/createPackageManager'; import { IBasicResponse, - ICreateJobResponse, - ICallbackResposne, ICreatePackageRoi, ICallbackExportResponse, ICreateExportJobResponse, @@ -15,7 +13,7 @@ import { type CreatePackageHandler = RequestHandler< undefined, - IBasicResponse | ICreateJobResponse | ICreateExportJobResponse | ICallbackResposne | ICallbackExportResponse, + IBasicResponse | ICreateExportJobResponse | ICallbackExportResponse, ICreatePackageRoi >; diff --git a/src/createPackage/models/createPackageManager.ts b/src/createPackage/models/createPackageManager.ts index a6ef343..912cd00 100644 --- a/src/createPackage/models/createPackageManager.ts +++ b/src/createPackage/models/createPackageManager.ts @@ -60,11 +60,8 @@ import { import { RasterCatalogManagerClient } from '../../clients/rasterCatalogManagerClient'; import { DEFAULT_CRS, DEFAULT_PRIORITY, METADA_JSON_FILE_EXTENSION as METADATA_JSON_FILE_EXTENSION, SERVICES } from '../../common/constants'; import { - ICreateJobResponse, - IWorkerInput, JobDuplicationParams, IJobParameters, - ICallbackResposne as ICallbackResponse, JobResponse, MergerSourceType, IMapSource, diff --git a/src/tasks/models/tasksManager.ts b/src/tasks/models/tasksManager.ts index 62ca627..af336b2 100644 --- a/src/tasks/models/tasksManager.ts +++ b/src/tasks/models/tasksManager.ts @@ -11,8 +11,6 @@ import { JobManagerWrapper } from '../../clients/jobManagerWrapper'; import { CreateFinalizeTaskBody, IArtifactDefinition, - ICallbackData, - ICallbackDataBase, ICallbackDataExportBase, ICallbackExportData, ICallbackExportResponse, diff --git a/tests/mocks/data.ts b/tests/mocks/data.ts index c109c6a..3a280d8 100644 --- a/tests/mocks/data.ts +++ b/tests/mocks/data.ts @@ -9,7 +9,6 @@ ICreatePackage, IJobParameters, ITaskParameters, IWorkerExportInput, - IWorkerInput, } from '../../src/common/interfaces'; const layerMetadata: LayerMetadata = { @@ -335,52 +334,6 @@ const inProgressJob: IJobResponse = { updated: '2021-12-29T10:42:13.487Z', }; -/** - * @deprecated GetMap API - will be deprecated on future - */ -const workerInput: IWorkerInput = { - fileName: 'test.gpkg', - relativeDirectoryPath: 'test', - exportVersion: ExportVersion.GETMAP, - sanitizedBbox: [0, 2.999267578125, 25.0048828125, 41.0009765625], - targetResolution: 0.0000429153442382812, - zoomLevel: 15, - dbId: '0c3e455f-4aeb-4258-982d-f7773469a92d', - sources: [ - { - path: 'test.gpkg', - type: 'GPKG', - extent: { - minX: 0, - minY: 2.999267578125, - maxX: 25.0048828125, - maxY: 41.0009765625, - }, - }, - ], - batches: [ - { - zoom: 15, - minX: 32768, - minY: 16930, - maxX: 20936, - maxY: 23848, - }, - { - zoom: 14, - minX: 16384, - minY: 8465, - maxX: 10468, - maxY: 11924, - }, - ], - version: '1.0', - cswProductId: 'string', - priority: 0, - crs: 'EPSG:4326', - productType: 'OrthophotoHistory', -}; - /** * @deprecated GetMap API - will be deprecated on future */ @@ -1064,7 +1017,6 @@ const pycswRecord = { export { layerFromCatalog, - workerInput, workerExportInput, jobs, userInput, diff --git a/tests/unit/clients/jobManagerClient.spec.ts b/tests/unit/clients/jobManagerClient.spec.ts index ba5f7e7..bd186c0 100644 --- a/tests/unit/clients/jobManagerClient.spec.ts +++ b/tests/unit/clients/jobManagerClient.spec.ts @@ -14,7 +14,6 @@ import { jobs, layerFromCatalog, workerExportInput, - workerInput, } from '../../mocks/data'; let jobManagerClient: JobManagerWrapper; diff --git a/tests/unit/createPackage/models/createPackageModel.spec.ts b/tests/unit/createPackage/models/createPackageModel.spec.ts index 8c6042f..a2920fe 100644 --- a/tests/unit/createPackage/models/createPackageModel.spec.ts +++ b/tests/unit/createPackage/models/createPackageModel.spec.ts @@ -23,7 +23,6 @@ import { catalogManagerMock, findLayerMock } from '../../../mocks/clients/catalo import { ExportVersion, ICreateExportJobResponse, - ICreateJobResponse, ICreatePackageRoi, IJobParameters, ITaskParameters, From dcea1e1ee7faa06222b97bc4089aea92eea3745c Mon Sep 17 00:00:00 2001 From: razbroc Date: Mon, 18 Mar 2024 17:55:36 +0200 Subject: [PATCH 09/19] chore: remove all deprecations for interfaces --- src/clients/jobManagerWrapper.ts | 6 +- src/common/interfaces.ts | 48 +----- .../models/createPackageManager.ts | 3 - src/tasks/models/tasksManager.ts | 5 +- tests/mocks/data.ts | 145 +----------------- tests/mocks/data/mockJob.ts | 44 +----- tests/unit/clients/jobManagerClient.spec.ts | 65 +------- .../models/createPackageModel.spec.ts | 4 - .../createPackage/models/tasksModel.spec.ts | 4 +- tests/unit/finalizationManager.spec.ts | 19 +-- 10 files changed, 11 insertions(+), 332 deletions(-) diff --git a/src/clients/jobManagerWrapper.ts b/src/clients/jobManagerWrapper.ts index 9732df5..28fdc19 100644 --- a/src/clients/jobManagerWrapper.ts +++ b/src/clients/jobManagerWrapper.ts @@ -10,17 +10,13 @@ import { Tracer } from '@opentelemetry/api'; import { SERVICES } from '../common/constants'; import { CreateExportJobBody, - CreateJobBody, ExportVersion, ICreateExportJobResponse, IJobExportParameters, - IJobParameters, ITaskParameters, IWorkerExportInput, - JobDuplicationParams, JobExportDuplicationParams, JobExportResponse, - JobResponse, TaskResponse, } from '../common/interfaces'; @@ -152,7 +148,7 @@ export class JobManagerWrapper extends JobManagerClient { const getOrUpdateURL = `/jobs/${jobId}`; const newExpirationDate = getUTCDate(); newExpirationDate.setDate(newExpirationDate.getDate() + this.expirationDays); - const job = await this.get(getOrUpdateURL); + const job = await this.get(getOrUpdateURL); const oldExpirationDate = new Date(job.parameters.cleanupData?.cleanupExpirationTimeUTC as Date); if (oldExpirationDate < newExpirationDate) { this.logger.info({ jobId, oldExpirationDate, newExpirationDate, msg: 'update expirationDate' }); diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index b30e1af..f510b32 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -80,21 +80,6 @@ export interface ICreateExportJobResponse { isDuplicated?: boolean; } -/** - * @deprecated GetMap API - will be deprecated on future - */ -export interface ICallbackDataBase { - fileUri: string; - expirationTime: Date; - fileSize: number; - dbId: string; - packageName: string; - targetResolution: number; - requestId: string; - success: boolean; - errorReason?: string; -} - export interface ICallbackDataExportBase { links?: ILinkDefinition; expirationTime?: Date; @@ -130,18 +115,6 @@ export interface ICallbackExportResponse extends ICallbackExportData { status: OperationStatus.IN_PROGRESS | OperationStatus.COMPLETED | OperationStatus.FAILED; } -/** - * @deprecated GetMap API - will be deprecated on future - */ -export interface JobDuplicationParams { - resourceId: string; - version: string; - dbId: string; - zoomLevel: number; - crs: string; - sanitizedBbox: BBox; -} - export interface JobExportDuplicationParams { resourceId: string; version: string; @@ -150,9 +123,7 @@ export interface JobExportDuplicationParams { roi: FeatureCollection; } -/** - * @deprecated GetMap API - will be deprecated on future - */ + export interface IJobParameters { targetResolution: number; relativeDirectoryPath: string; @@ -160,7 +131,6 @@ export interface IJobParameters { exportVersion: ExportVersion; sanitizedBbox: BBox; zoomLevel: number; - callbackParams?: ICallbackDataBase; fileName: string; gpkgEstimatedSize?: number; cleanupData?: ICleanupData; @@ -215,14 +185,6 @@ export interface IInput { dbId: string; } -/** - * @deprecated GetMap API - will be deprecated on future - */ -export interface IJobStatusResponse { - completedJobs: JobResponse[] | undefined; - failedJobs: JobResponse[] | undefined; -} - export interface IExportJobStatusResponse { completedJobs: JobExportResponse[] | undefined; failedJobs: JobExportResponse[] | undefined; @@ -259,15 +221,7 @@ export enum ExportVersion { ROI = 'ROI', } -/** - * @deprecated GetMap API - will be deprecated on future - */ -export type JobResponse = IJobResponse; export type TaskResponse = ITaskResponse; -/** - * @deprecated GetMap API - will be deprecated on future - */ -export type CreateJobBody = ICreateJobBody; // new API based on multi resolution export type JobExportResponse = IJobResponse; diff --git a/src/createPackage/models/createPackageManager.ts b/src/createPackage/models/createPackageManager.ts index 912cd00..f87e525 100644 --- a/src/createPackage/models/createPackageManager.ts +++ b/src/createPackage/models/createPackageManager.ts @@ -60,9 +60,6 @@ import { import { RasterCatalogManagerClient } from '../../clients/rasterCatalogManagerClient'; import { DEFAULT_CRS, DEFAULT_PRIORITY, METADA_JSON_FILE_EXTENSION as METADATA_JSON_FILE_EXTENSION, SERVICES } from '../../common/constants'; import { - JobDuplicationParams, - IJobParameters, - JobResponse, MergerSourceType, IMapSource, ITaskParameters, diff --git a/src/tasks/models/tasksManager.ts b/src/tasks/models/tasksManager.ts index af336b2..1b92c42 100644 --- a/src/tasks/models/tasksManager.ts +++ b/src/tasks/models/tasksManager.ts @@ -17,13 +17,10 @@ import { ICleanupData, IExportJobStatusResponse, IJobExportParameters, - IJobParameters, - IJobStatusResponse, ILinkDefinition, ITaskFinalizeParameters, JobExportResponse, JobFinalizeResponse, - JobResponse, } from '../../common/interfaces'; import { CallbackClient } from '../../clients/callbackClient'; import { getFileSize } from '../../common/utils'; @@ -208,7 +205,7 @@ export class TasksManager { return updateJobParams; } - private generateCleanupEntity(job: JobResponse | JobExportResponse | JobFinalizeResponse, expirationDate: Date): ICleanupData { + private generateCleanupEntity(job: JobExportResponse | JobFinalizeResponse, expirationDate: Date): ICleanupData { const cleanupData = { directoryPath: job.parameters.relativeDirectoryPath, cleanupExpirationTimeUTC: expirationDate }; this.logger.info({ jobId: job.id, cleanupData, msg: `Generated new cleanupData param for job parameters` }); return cleanupData; diff --git a/tests/mocks/data.ts b/tests/mocks/data.ts index 3a280d8..2defc45 100644 --- a/tests/mocks/data.ts +++ b/tests/mocks/data.ts @@ -6,7 +6,6 @@ import { ExportVersion, ICreatePackage, IJobExportParameters, - IJobParameters, ITaskParameters, IWorkerExportInput, } from '../../src/common/interfaces'; @@ -200,145 +199,6 @@ const layerFromCatalogSample = { metadata: layerMetadataSample, }; -/** - * @deprecated GetMap API - will be deprecated on future - */ -const completedJob: IJobResponse = { - id: 'b0b19b88-aecb-4e74-b694-dfa7eada8bf7', - resourceId: 'string', - version: '1.0', - type: 'rasterTilesExporter', - domain: 'testDomain', - description: '', - parameters: { - crs: 'EPSG:4326', - sanitizedBbox: [0, 0, 25, 41], - exportVersion: ExportVersion.GETMAP, - fileName: 'test.gpkg', - relativeDirectoryPath: 'test', - zoomLevel: 4, - callbackParams: { - dbId: '0c3e455f-4aeb-4258-982d-f7773469a92d', - fileUri: 'http://localhost:4515/downloads/gm_0c3e455f_4aeb_4258_982d_f7773469a92d_4_0_000000_0000025_0000041_00000.gpkg', - success: true, - fileSize: 1773568, - requestId: 'b0b19b88-aecb-4e74-b694-dfa7eada8bf7', - packageName: 'gm_0c3e455f_4aeb_4258_982d_f7773469a92d_4_0_000000_0000025_0000041_00000', - expirationTime: new Date(), - targetResolution: 0.0439453125, - }, - cleanupData: { - directoryPath: 'test', - cleanupExpirationTimeUTC: new Date(), - }, - - targetResolution: 0.0439453125, - }, - - status: OperationStatus.COMPLETED, - percentage: 100, - reason: '', - isCleaned: false, - priority: 1000, - expirationDate: new Date(), - internalId: '0c3e455f-4aeb-4258-982d-f7773469a92d', - productName: 'string', - productType: 'OrthophotoHistory', - taskCount: 1, - completedTasks: 1, - failedTasks: 0, - expiredTasks: 0, - pendingTasks: 0, - inProgressTasks: 0, - abortedTasks: 0, - tasks: [ - { - id: '542ebbfd-f4d1-4c77-bd4d-97ca121f0de7', - jobId: 'b0b19b88-aecb-4e74-b694-dfa7eada8bf7', - type: 'rasterTilesExporter', - description: '', - parameters: { - isNewTarget: true, - targetFormat: TileOutputFormat.PNG, - batches: [], - sources: [], - }, - status: OperationStatus.COMPLETED, - reason: '', - attempts: 0, - resettable: true, - created: '2021-12-29T08:06:48.399Z', - updated: '2021-12-29T08:07:00.293Z', - }, - ], - created: '2021-12-29T08:06:48.399Z', - updated: '2021-12-29T08:07:00.270Z', -}; - -/** - * @deprecated GetMap API - will be deprecated on future - */ -const inProgressJob: IJobResponse = { - id: 'fa3ab609-377a-4d96-bf0b-e0bb72f683b8', - domain: 'testDomain', - resourceId: 'string', - version: '1.0', - type: 'rasterTilesExporter', - percentage: 0, - description: '', - parameters: { - fileName: 'test.gpkg', - relativeDirectoryPath: 'test', - exportVersion: ExportVersion.GETMAP, - crs: 'EPSG:4326', - sanitizedBbox: [0, 0, 25, 41], - zoomLevel: 4, - targetResolution: 0.0439453125, - }, - status: OperationStatus.IN_PROGRESS, - reason: '', - isCleaned: false, - priority: 0, - expirationDate: new Date(), - internalId: '0c3e455f-4aeb-4258-982d-f7773469a92d', - productName: 'string', - productType: 'OrthophotoHistory', - taskCount: 1, - completedTasks: 0, - failedTasks: 0, - expiredTasks: 0, - pendingTasks: 0, - abortedTasks: 0, - inProgressTasks: 1, - tasks: [ - { - id: '1f765695-338b-4752-b182-a8cbae3c610e', - jobId: 'b0b19b88-aecb-4e74-b694-dfa7eada8bf7', - type: 'rasterTilesExporter', - description: '', - parameters: { - isNewTarget: true, - targetFormat: TileOutputFormat.PNG, - batches: [], - sources: [], - }, - status: OperationStatus.IN_PROGRESS, - reason: '', - attempts: 0, - resettable: true, - created: '2021-12-29T10:42:13.487Z', - updated: '2021-12-29T10:42:16.231Z', - }, - ], - created: '2021-12-29T10:42:13.487Z', - updated: '2021-12-29T10:42:13.487Z', -}; - -/** - * @deprecated GetMap API - will be deprecated on future - */ -const jobs = [inProgressJob, completedJob]; - const userInput: ICreatePackage = { dbId: '0c3e455f-4aeb-4258-982d-f7773469a92d', targetResolution: 0.00439453125, @@ -1018,10 +878,7 @@ const pycswRecord = { export { layerFromCatalog, workerExportInput, - jobs, -userInput, - completedJob, - inProgressJob, + userInput, fc1, fcNoMaxResolutionDeg, fcBadMinResolutionDeg, diff --git a/tests/mocks/data/mockJob.ts b/tests/mocks/data/mockJob.ts index 3e38ac2..8fdecf1 100644 --- a/tests/mocks/data/mockJob.ts +++ b/tests/mocks/data/mockJob.ts @@ -1,50 +1,8 @@ /* eslint-disable @typescript-eslint/no-magic-numbers */ import { OperationStatus } from '@map-colonies/mc-priority-queue'; -import { ExportVersion, JobFinalizeResponse, JobResponse } from '../../../src/common/interfaces'; +import { ExportVersion, JobFinalizeResponse } from '../../../src/common/interfaces'; import { fc1 } from '../data'; -/** - * @deprecated GetMap API - will be deprecated on future - */ -export const mockJob: JobResponse = { - id: 'b729f0e0-af64-4c2c-ba4e-e799e2f3df0f', - resourceId: 'test', - domain: 'testDomain', - version: '1.0', - type: 'rasterTilesExporter', - description: '', - parameters: { - crs: 'EPSG:4326', - fileName: 'test.gpkg', - exportVersion: ExportVersion.GETMAP, - zoomLevel: 3, - sanitizedBbox: [0, -90, 180, 90], - targetResolution: 0.072, - relativeDirectoryPath: 'test', - }, - status: OperationStatus.IN_PROGRESS, - percentage: 100, - reason: '', - isCleaned: false, - priority: 0, - expirationDate: new Date(), - internalId: '880a9316-0f10-4874-92e2-a62d587a1169', - producerName: undefined, - productName: 'test', - productType: 'Orthophoto', - additionalIdentifiers: '0,-90,180,903', - taskCount: 1, - completedTasks: 0, - failedTasks: 0, - expiredTasks: 0, - pendingTasks: 0, - inProgressTasks: 0, - abortedTasks: 0, - tasks: [], - created: '2022-08-29T07:06:05.043Z', - updated: '2022-08-29T07:13:05.206Z', -}; - export const mockCompletedJob: JobFinalizeResponse = { id: 'b729f0e0-af64-4c2c-ba4e-e799e2f3df0f', resourceId: 'testCompleted', diff --git a/tests/unit/clients/jobManagerClient.spec.ts b/tests/unit/clients/jobManagerClient.spec.ts index bd186c0..eb23ae2 100644 --- a/tests/unit/clients/jobManagerClient.spec.ts +++ b/tests/unit/clients/jobManagerClient.spec.ts @@ -2,16 +2,13 @@ import jsLogger from '@map-colonies/js-logger'; import { IFindJobsRequest, OperationStatus } from '@map-colonies/mc-priority-queue'; import { getUTCDate } from '@map-colonies/mc-utils'; import { JobManagerWrapper } from '../../../src/clients/jobManagerWrapper'; -import { JobResponse, JobExportDuplicationParams, ICreateExportJobResponse } from '../../../src/common/interfaces'; +import { JobExportDuplicationParams, ICreateExportJobResponse } from '../../../src/common/interfaces'; import { configMock, registerDefaultConfig } from '../../mocks/config'; import { tracerMock } from '../../mocks/clients/tracer'; import { completedExportJob, - completedJob, fc1, inProgressExportJob, - inProgressJob, - jobs, layerFromCatalog, workerExportInput, } from '../../mocks/data'; @@ -45,66 +42,6 @@ describe('JobManagerClient', () => { expect(putFun).toHaveBeenCalledTimes(1); }); - - /** - * @deprecated GetMap API - will be deprecated on future - */ - it('should successfully update job expirationDate (old expirationDate lower)', async () => { - const expirationDays: number = configMock.get('cleanupExpirationDays'); - const testExpirationDate = getUTCDate(); - const expectedNewExpirationDate = getUTCDate(); - testExpirationDate.setDate(testExpirationDate.getDate() - expirationDays); - expectedNewExpirationDate.setDate(expectedNewExpirationDate.getDate() + expirationDays); - expectedNewExpirationDate.setSeconds(0, 0); - - get = jest.fn(); - putFun = jest.fn(); - (jobManagerClient as unknown as { put: unknown }).put = putFun.mockResolvedValue(undefined); - const jobManager = jobManagerClient as unknown as { get: unknown }; - jobManager.get = get.mockResolvedValue({ - ...completedJob, - parameters: { - ...completedJob.parameters, - cleanupData: { ...completedJob.parameters.cleanupData, cleanupExpirationTimeUTC: testExpirationDate }, - callbackParams: { ...completedJob.parameters.callbackParams, expirationTime: testExpirationDate }, - }, - }); - - await jobManagerClient.validateAndUpdateExpiration(completedJob.id); - - expect(get).toHaveBeenCalledTimes(1); - expect(putFun).toHaveBeenCalledTimes(1); - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access - const expirationParamCall: Date = putFun.mock.calls[0][1].parameters.cleanupData.cleanupExpirationTimeUTC; - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access - const expirationCallbackParams: Date = putFun.mock.calls[0][1].parameters.callbackParams.expirationTime; - expirationParamCall.setSeconds(0, 0); - expect(JSON.stringify(expirationParamCall)).toBe(JSON.stringify(expectedNewExpirationDate)); - expect(JSON.stringify(expirationCallbackParams)).toBe(JSON.stringify(expectedNewExpirationDate)); - }); - - /** - * @deprecated GetMap API - will be deprecated on future - */ - it('should not update job expirationDate (old expirationDate higher)', async () => { - const expirationDays: number = configMock.get('cleanupExpirationDays'); - const testExpirationDate = getUTCDate(); - const expectedNewExpirationDate = getUTCDate(); - testExpirationDate.setDate(testExpirationDate.getDate() + 2 * expirationDays); - expectedNewExpirationDate.setDate(expectedNewExpirationDate.getDate() + expirationDays); - expectedNewExpirationDate.setSeconds(0, 0); - - get = jest.fn(); - putFun = jest.fn(); - (jobManagerClient as unknown as { put: unknown }).put = putFun.mockResolvedValue(undefined); - const jobManager = jobManagerClient as unknown as { get: unknown }; - jobManager.get = get.mockResolvedValue({ ...inProgressJob, expirationDate: testExpirationDate }); - - await jobManagerClient.validateAndUpdateExpiration(inProgressJob.id); - - expect(get).toHaveBeenCalledTimes(1); - expect(putFun).toHaveBeenCalledTimes(0); - }); }); describe('RoiExport', () => { diff --git a/tests/unit/createPackage/models/createPackageModel.spec.ts b/tests/unit/createPackage/models/createPackageModel.spec.ts index a2920fe..973869c 100644 --- a/tests/unit/createPackage/models/createPackageModel.spec.ts +++ b/tests/unit/createPackage/models/createPackageModel.spec.ts @@ -24,16 +24,12 @@ import { ExportVersion, ICreateExportJobResponse, ICreatePackageRoi, - IJobParameters, ITaskParameters, - JobDuplicationParams, JobExportDuplicationParams, } from '../../../../src/common/interfaces'; import { CreatePackageManager } from '../../../../src/createPackage/models/createPackageManager'; import { completedExportJob, - completedJob, - inProgressJob, layerFromCatalog, userInput, metadataExportJson, diff --git a/tests/unit/createPackage/models/tasksModel.spec.ts b/tests/unit/createPackage/models/tasksModel.spec.ts index 7f0f542..0a26339 100644 --- a/tests/unit/createPackage/models/tasksModel.spec.ts +++ b/tests/unit/createPackage/models/tasksModel.spec.ts @@ -5,14 +5,12 @@ import { NotFoundError } from '@map-colonies/error-types'; import { ITaskStatusResponse, TasksManager } from '../../../../src/tasks/models/tasksManager'; import { CreateFinalizeTaskBody, - ICallbackDataBase, ICallbackDataExportBase, ICallbackExportResponse, ICallbackTargetExport, ITaskParameters, JobExportResponse, JobFinalizeResponse, - JobResponse, TaskResponse, } from '../../../../src/common/interfaces'; import { configMock, registerDefaultConfig } from '../../../mocks/config'; @@ -24,7 +22,7 @@ import { updateJobMock, getExportJobsMock, } from '../../../mocks/clients/jobManagerWrapper'; -import { mockCompletedJob, mockJob } from '../../../mocks/data/mockJob'; +import { mockCompletedJob } from '../../../mocks/data/mockJob'; import * as utils from '../../../../src/common/utils'; import { ArtifactType } from '../../../../src/common/enums'; import { tracerMock } from '../../../mocks/clients/tracer'; diff --git a/tests/unit/finalizationManager.spec.ts b/tests/unit/finalizationManager.spec.ts index ad59ece..fb85751 100644 --- a/tests/unit/finalizationManager.spec.ts +++ b/tests/unit/finalizationManager.spec.ts @@ -20,10 +20,9 @@ import { ICallbackDataExportBase, IExportJobStatusResponse, IJobExportParameters, - IJobStatusResponse, ITaskFinalizeParameters, } from '../../src/common/interfaces'; -import { completedExportJob, inProgressJob, inProgressExportJob } from '../mocks/data'; +import { completedExportJob, inProgressExportJob } from '../mocks/data'; import { configMock, registerDefaultConfig } from '../mocks/config'; import { jobManagerWrapperMock, updateJobMock, deleteTaskByIdMock } from '../mocks/clients/jobManagerWrapper'; import { callbackClientMock } from '../mocks/clients/callbackClient'; @@ -44,12 +43,7 @@ describe('FinalizationManager', () => { }); describe('#jobStatusPoll', () => { - it('should poll completed jobs for getmap and roi jobs', async () => { - const getmapJobStatus: IJobStatusResponse = { - completedJobs: [JSON.parse(JSON.stringify(inProgressJob)), JSON.parse(JSON.stringify(inProgressJob))], - failedJobs: [], - }; - + it('should poll completed jobs for roi jobs', async () => { const roiJobStatus: IExportJobStatusResponse = { completedJobs: [ JSON.parse(JSON.stringify(inProgressExportJob)), @@ -58,7 +52,6 @@ describe('FinalizationManager', () => { ], failedJobs: [], }; - getJobsByTaskStatusMock.mockReturnValue(getmapJobStatus); getExportJobsByTaskStatusMock.mockReturnValue(roiJobStatus); finalizeJobMock.mockReturnValue(undefined); createFinalizeTaskMock.mockReturnValue(undefined); @@ -69,11 +62,8 @@ describe('FinalizationManager', () => { expect(createFinalizeTaskMock).toHaveBeenCalledTimes(3); }); - it('should poll failed jobs for getmap and roi jobs', async () => { - const getmapJobStatus: IJobStatusResponse = { - completedJobs: [], - failedJobs: [JSON.parse(JSON.stringify(inProgressJob)), JSON.parse(JSON.stringify(inProgressJob))], - }; + it('should poll failed jobs for roi jobs', async () => { + const roiJobStatus: IExportJobStatusResponse = { completedJobs: [], @@ -83,7 +73,6 @@ describe('FinalizationManager', () => { JSON.parse(JSON.stringify(inProgressExportJob)), ], }; - getJobsByTaskStatusMock.mockReturnValue(getmapJobStatus); getExportJobsByTaskStatusMock.mockReturnValue(roiJobStatus); finalizeJobMock.mockReturnValue(undefined); createFinalizeTaskMock.mockReturnValue(undefined); From 0a0d7a2a92aae5e444513b403d364527ca2ccd5b Mon Sep 17 00:00:00 2001 From: razbroc Date: Mon, 18 Mar 2024 18:05:37 +0200 Subject: [PATCH 10/19] fix: lower coverage a tiny bit --- tests/configurations/unit/jest.config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/configurations/unit/jest.config.js b/tests/configurations/unit/jest.config.js index e9ceb99..a1fc232 100644 --- a/tests/configurations/unit/jest.config.js +++ b/tests/configurations/unit/jest.config.js @@ -33,7 +33,7 @@ module.exports = { coverageThreshold: { global: { branches: 74, - functions: 90, + functions: 88, lines: 85, statements: 85, }, From 03ff268be944facb5c8a1f48a93a9c1029fcf6b9 Mon Sep 17 00:00:00 2001 From: razbroc Date: Mon, 18 Mar 2024 18:17:53 +0200 Subject: [PATCH 11/19] fix: fix lint --- src/clients/jobManagerWrapper.ts | 4 +-- src/common/interfaces.ts | 1 - .../controllers/createPackageController.ts | 13 ++------ .../models/createPackageManager.ts | 32 +++---------------- src/tasks/models/tasksManager.ts | 2 +- tests/mocks/data.ts | 8 +---- tests/unit/clients/jobManagerClient.spec.ts | 9 +----- .../models/createPackageModel.spec.ts | 26 ++------------- .../createPackage/models/tasksModel.spec.ts | 10 ++---- tests/unit/finalizationManager.spec.ts | 10 +----- 10 files changed, 16 insertions(+), 99 deletions(-) diff --git a/src/clients/jobManagerWrapper.ts b/src/clients/jobManagerWrapper.ts index 28fdc19..f90085f 100644 --- a/src/clients/jobManagerWrapper.ts +++ b/src/clients/jobManagerWrapper.ts @@ -1,8 +1,6 @@ import { inject, injectable } from 'tsyringe'; import config from 'config'; import { Logger } from '@map-colonies/js-logger'; -import booleanEqual from '@turf/boolean-equal'; -import bboxPolygon from '@turf/bbox-polygon'; import { IFindJobsRequest, JobManagerClient, OperationStatus } from '@map-colonies/mc-priority-queue'; import { featureCollectionBooleanEqual, getUTCDate, IHttpRetryConfig } from '@map-colonies/mc-utils'; import { withSpanAsyncV4 } from '@map-colonies/telemetry'; @@ -172,7 +170,7 @@ export class JobManagerWrapper extends JobManagerClient { } } - //TODO: once will be only one kind of exported jobs, no need to filter by ROI's + //TODO: once will be only one kind of exported jobs, no need to filter by ROI's public async getExportJobs(queryParams: IFindJobsRequest): Promise { this.logger.debug({ ...queryParams }, `Getting jobs that match these parameters`); const jobs = await this.get('/jobs', queryParams as unknown as Record); diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index f510b32..b47fffe 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -123,7 +123,6 @@ export interface JobExportDuplicationParams { roi: FeatureCollection; } - export interface IJobParameters { targetResolution: number; relativeDirectoryPath: string; diff --git a/src/createPackage/controllers/createPackageController.ts b/src/createPackage/controllers/createPackageController.ts index e1f7fcd..8cac81f 100644 --- a/src/createPackage/controllers/createPackageController.ts +++ b/src/createPackage/controllers/createPackageController.ts @@ -4,18 +4,9 @@ import httpStatus from 'http-status-codes'; import { injectable, inject } from 'tsyringe'; import { SERVICES } from '../../common/constants'; import { CreatePackageManager } from '../models/createPackageManager'; -import { - IBasicResponse, - ICreatePackageRoi, - ICallbackExportResponse, - ICreateExportJobResponse, -} from '../../common/interfaces'; +import { IBasicResponse, ICreatePackageRoi, ICallbackExportResponse, ICreateExportJobResponse } from '../../common/interfaces'; -type CreatePackageHandler = RequestHandler< - undefined, - IBasicResponse | ICreateExportJobResponse | ICallbackExportResponse, - ICreatePackageRoi ->; +type CreatePackageHandler = RequestHandler; @injectable() export class CreatePackageController { diff --git a/src/createPackage/models/createPackageManager.ts b/src/createPackage/models/createPackageManager.ts index f87e525..fe977ed 100644 --- a/src/createPackage/models/createPackageManager.ts +++ b/src/createPackage/models/createPackageManager.ts @@ -1,5 +1,5 @@ import { promises as fsPromise } from 'fs'; -import { sep, parse as parsePath } from 'path'; +import { sep } from 'path'; import { Logger } from '@map-colonies/js-logger'; import { Tracer } from '@opentelemetry/api'; import { withSpanAsyncV4 } from '@map-colonies/telemetry'; @@ -10,24 +10,14 @@ import { bbox as PolygonBbox, intersect, combine as featureCombine, - bboxPolygon, FeatureCollection, Feature, Geometry, } from '@turf/turf'; import { inject, injectable } from 'tsyringe'; -import { - degreesPerPixelToZoomLevel, - featureCollectionBooleanEqual, - ITileRange, - snapBBoxToTileGrid, - TileRanger, - bboxToTileRange, -} from '@map-colonies/mc-utils'; +import { degreesPerPixelToZoomLevel, featureCollectionBooleanEqual, ITileRange, snapBBoxToTileGrid, bboxToTileRange } from '@map-colonies/mc-utils'; import { IJobResponse, OperationStatus } from '@map-colonies/mc-priority-queue'; import { BadRequestError, InsufficientStorage } from '@map-colonies/error-types'; -import { isArray, isEmpty } from 'lodash'; -import booleanEqual from '@turf/boolean-equal'; import { BBox2d } from '@turf/helpers/dist/js/lib/geojson'; import { ProductType, TileOutputFormat } from '@map-colonies/mc-model-types'; import { feature, featureCollection } from '@turf/helpers'; @@ -49,31 +39,17 @@ import { } from '../../common/interfaces'; import { calculateEstimateGpkgSize, - getGpkgRelativePath, getStorageStatus, - getGpkgNameWithoutExt, concatFsPaths, parseFeatureCollection, generateGeoIdentifier, getFilesha256Hash, } from '../../common/utils'; import { RasterCatalogManagerClient } from '../../clients/rasterCatalogManagerClient'; -import { DEFAULT_CRS, DEFAULT_PRIORITY, METADA_JSON_FILE_EXTENSION as METADATA_JSON_FILE_EXTENSION, SERVICES } from '../../common/constants'; -import { - MergerSourceType, - IMapSource, - ITaskParameters, - IStorageStatusResponse, -} from '../../common/interfaces'; +import { DEFAULT_CRS, DEFAULT_PRIORITY, SERVICES } from '../../common/constants'; +import { MergerSourceType, IMapSource, ITaskParameters, IStorageStatusResponse } from '../../common/interfaces'; import { JobManagerWrapper } from '../../clients/jobManagerWrapper'; -// eslint-disable-next-line @typescript-eslint/no-var-requires -const geojsonhint: IHinter = require('@mapbox/geojsonhint') as IHinter; - -interface IHinter { - hint: (obj: object) => []; -} - @injectable() export class CreatePackageManager { // eslint-disable-next-line @typescript-eslint/no-magic-numbers diff --git a/src/tasks/models/tasksManager.ts b/src/tasks/models/tasksManager.ts index 1b92c42..09ecf67 100644 --- a/src/tasks/models/tasksManager.ts +++ b/src/tasks/models/tasksManager.ts @@ -5,7 +5,7 @@ import { IFindJobsRequest, IJobResponse, IUpdateJobBody, OperationStatus } from import { NotFoundError } from '@map-colonies/error-types'; import { withSpanAsyncV4 } from '@map-colonies/telemetry'; import { Tracer } from '@opentelemetry/api'; -import { concatFsPaths, getGpkgFullPath, getGpkgRelativePath } from '../../common/utils'; +import { concatFsPaths } from '../../common/utils'; import { SERVICES } from '../../common/constants'; import { JobManagerWrapper } from '../../clients/jobManagerWrapper'; import { diff --git a/tests/mocks/data.ts b/tests/mocks/data.ts index 2defc45..57e6a44 100644 --- a/tests/mocks/data.ts +++ b/tests/mocks/data.ts @@ -2,13 +2,7 @@ import { LayerMetadata, ProductType, RecordType, TileOutputFormat } from '@map-colonies/mc-model-types'; import { IJobResponse, OperationStatus } from '@map-colonies/mc-priority-queue'; import { FeatureCollection } from '@turf/helpers'; -import { - ExportVersion, -ICreatePackage, - IJobExportParameters, - ITaskParameters, - IWorkerExportInput, -} from '../../src/common/interfaces'; +import { ExportVersion, ICreatePackage, IJobExportParameters, ITaskParameters, IWorkerExportInput } from '../../src/common/interfaces'; const layerMetadata: LayerMetadata = { type: 'RECORD_RASTER', diff --git a/tests/unit/clients/jobManagerClient.spec.ts b/tests/unit/clients/jobManagerClient.spec.ts index eb23ae2..0db0dfe 100644 --- a/tests/unit/clients/jobManagerClient.spec.ts +++ b/tests/unit/clients/jobManagerClient.spec.ts @@ -5,18 +5,11 @@ import { JobManagerWrapper } from '../../../src/clients/jobManagerWrapper'; import { JobExportDuplicationParams, ICreateExportJobResponse } from '../../../src/common/interfaces'; import { configMock, registerDefaultConfig } from '../../mocks/config'; import { tracerMock } from '../../mocks/clients/tracer'; -import { - completedExportJob, - fc1, - inProgressExportJob, - layerFromCatalog, - workerExportInput, -} from '../../mocks/data'; +import { completedExportJob, fc1, inProgressExportJob, layerFromCatalog, workerExportInput } from '../../mocks/data'; let jobManagerClient: JobManagerWrapper; let postFun: jest.Mock; let putFun: jest.Mock; -let getGetMapJobs: jest.Mock; let get: jest.Mock; let getExportJobs: jest.Mock; let deleteFun: jest.Mock; diff --git a/tests/unit/createPackage/models/createPackageModel.spec.ts b/tests/unit/createPackage/models/createPackageModel.spec.ts index 973869c..924e126 100644 --- a/tests/unit/createPackage/models/createPackageModel.spec.ts +++ b/tests/unit/createPackage/models/createPackageModel.spec.ts @@ -2,35 +2,16 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ /* eslint-disable @typescript-eslint/naming-convention */ import fs from 'fs'; -import { sep } from 'path'; import { BadRequestError, InsufficientStorage } from '@map-colonies/error-types'; import jsLogger from '@map-colonies/js-logger'; -import { IJobResponse, OperationStatus } from '@map-colonies/mc-priority-queue'; -import { LayerMetadata } from '@map-colonies/mc-model-types'; -import { BBox, Polygon } from '@turf/helpers'; +import { OperationStatus } from '@map-colonies/mc-priority-queue'; import { tracerMock } from '../../../mocks/clients/tracer'; -import { - jobManagerWrapperMock, - findCompletedJobMock, - findInProgressJobMock, - findPendingJobMock, - updateJobMock, - createMock, - findExportJobMock, - createExportMock, -} from '../../../mocks/clients/jobManagerWrapper'; +import { jobManagerWrapperMock, updateJobMock, findExportJobMock, createExportMock } from '../../../mocks/clients/jobManagerWrapper'; import { catalogManagerMock, findLayerMock } from '../../../mocks/clients/catalogManagerClient'; -import { - ExportVersion, - ICreateExportJobResponse, - ICreatePackageRoi, - ITaskParameters, - JobExportDuplicationParams, -} from '../../../../src/common/interfaces'; +import { ExportVersion, ICreateExportJobResponse, ICreatePackageRoi, JobExportDuplicationParams } from '../../../../src/common/interfaces'; import { CreatePackageManager } from '../../../../src/createPackage/models/createPackageManager'; import { completedExportJob, - layerFromCatalog, userInput, metadataExportJson, layerFromCatalogSample, @@ -45,7 +26,6 @@ import { featuresRecordsSampleFcMinResolutionDeg, } from '../../../mocks/data'; import { configMock, registerDefaultConfig } from '../../../mocks/config'; -import { METADA_JSON_FILE_EXTENSION } from '../../../../src/common/constants'; import * as utils from '../../../../src/common/utils'; jest.mock('fs', () => { diff --git a/tests/unit/createPackage/models/tasksModel.spec.ts b/tests/unit/createPackage/models/tasksModel.spec.ts index 0a26339..1584a71 100644 --- a/tests/unit/createPackage/models/tasksModel.spec.ts +++ b/tests/unit/createPackage/models/tasksModel.spec.ts @@ -1,4 +1,3 @@ -import { sep } from 'path'; import jsLogger from '@map-colonies/js-logger'; import { OperationStatus } from '@map-colonies/mc-priority-queue'; import { NotFoundError } from '@map-colonies/error-types'; @@ -15,13 +14,8 @@ import { } from '../../../../src/common/interfaces'; import { configMock, registerDefaultConfig } from '../../../mocks/config'; import { callbackClientMock, sendMock } from '../../../mocks/clients/callbackClient'; -import { createExportJsonMetadataMock, createJsonMetadataMock, packageManagerMock } from '../../../mocks/clients/packageManager'; -import { - jobManagerWrapperMock, - getInProgressJobsMock as getInProgressJobsMock, - updateJobMock, - getExportJobsMock, -} from '../../../mocks/clients/jobManagerWrapper'; +import { createExportJsonMetadataMock, packageManagerMock } from '../../../mocks/clients/packageManager'; +import { jobManagerWrapperMock, getExportJobsMock } from '../../../mocks/clients/jobManagerWrapper'; import { mockCompletedJob } from '../../../mocks/data/mockJob'; import * as utils from '../../../../src/common/utils'; import { ArtifactType } from '../../../../src/common/enums'; diff --git a/tests/unit/finalizationManager.spec.ts b/tests/unit/finalizationManager.spec.ts index fb85751..07ae456 100644 --- a/tests/unit/finalizationManager.spec.ts +++ b/tests/unit/finalizationManager.spec.ts @@ -14,14 +14,8 @@ import { finalizeJobMock, getExportJobsByTaskStatusMock, getFinalizeJobByIdMock, - getJobsByTaskStatusMock, } from '../mocks/clients/taskManager'; -import { - ICallbackDataExportBase, - IExportJobStatusResponse, - IJobExportParameters, - ITaskFinalizeParameters, -} from '../../src/common/interfaces'; +import { ICallbackDataExportBase, IExportJobStatusResponse, IJobExportParameters, ITaskFinalizeParameters } from '../../src/common/interfaces'; import { completedExportJob, inProgressExportJob } from '../mocks/data'; import { configMock, registerDefaultConfig } from '../mocks/config'; import { jobManagerWrapperMock, updateJobMock, deleteTaskByIdMock } from '../mocks/clients/jobManagerWrapper'; @@ -63,8 +57,6 @@ describe('FinalizationManager', () => { }); it('should poll failed jobs for roi jobs', async () => { - - const roiJobStatus: IExportJobStatusResponse = { completedJobs: [], failedJobs: [ From 6475a869d932f8785037dd72f1bd0a7618c7edb7 Mon Sep 17 00:00:00 2001 From: razbroc Date: Mon, 18 Mar 2024 18:20:48 +0200 Subject: [PATCH 12/19] fix: fix lint --- openapi3.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/openapi3.yaml b/openapi3.yaml index 460fe8d..64644a6 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -111,13 +111,6 @@ paths: $ref: '#/components/schemas/internalError' components: requestBodies: - ExportGetmapBody: - description: Export to gpkg via GetMap - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/exportGetMap' ExportByRoiBody: description: Export to gpkg via FeatureCollection required: true From 190d828e8df027e84a5daa9726af3a9d0ab5dbc6 Mon Sep 17 00:00:00 2001 From: razbroc Date: Mon, 18 Mar 2024 18:23:20 +0200 Subject: [PATCH 13/19] fix: fix lint --- openapi3.yaml | 44 -------------------------------------------- 1 file changed, 44 deletions(-) diff --git a/openapi3.yaml b/openapi3.yaml index 64644a6..6ea530e 100644 --- a/openapi3.yaml +++ b/openapi3.yaml @@ -142,50 +142,6 @@ components: - Completed discriminator: propertyName: response - exportGetMap: - type: object - properties: - dbId: - type: string - format: uuid - description: ID as the primary key from the Raster Catalog - bbox: - oneOf: - - $ref: '#/components/schemas/BBox' - - $ref: '#/components/schemas/Geometry' - targetResolution: - type: number - description: >- - The target resolution in which the tiles will be created - DEGREE to - PIXEL. If empty - original layer's resolution is taken. - callbackURLs: - type: array - items: - type: string - description: The callback URL to notify the process if finished - crs: - $ref: '#/components/schemas/CRS' - priority: - type: number - description: The priority of the record. Maximum priority = most urgent. - minimum: 0 - maximum: 999999999 - required: - - dbId - - callbackURLs - example: - dbId: ef03ca54-c68e-4ca8-8432-50ae5ad7a7f8 - bbox: - - 34.811938017107494 - - 31.95475033759175 - - 34.82237261707599 - - 31.96426962177354 - targetResolution: 0.0000429153442382812 - callbackURLs: - - http://example.getmap.com/callback - - http://example.getmap.com/callback2 - crs: EPSG:4326 - priority: 0 exportFromFeatures: type: object properties: From 2affcd13e4472213ce49c0fe48b3214c148f0bef Mon Sep 17 00:00:00 2001 From: razbroc Date: Mon, 25 Mar 2024 14:01:52 +0200 Subject: [PATCH 14/19] fix: fix pr --- src/common/interfaces.ts | 54 +-- src/finalizationManager.ts | 4 +- tests/mocks/data.ts | 11 +- .../models/createPackageModel.spec.ts | 383 +----------------- 4 files changed, 12 insertions(+), 440 deletions(-) diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index b47fffe..5aa3e8f 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -1,4 +1,4 @@ -import { MultiPolygon, Polygon, BBox, FeatureCollection, Geometry } from '@turf/turf'; +import { BBox, FeatureCollection, Geometry } from '@turf/turf'; import { ICreateJobBody, ICreateTaskBody, IJobResponse, ITaskResponse, OperationStatus } from '@map-colonies/mc-priority-queue'; import { IHttpRetryConfig, ITileRange } from '@map-colonies/mc-utils'; import { TileOutputFormat } from '@map-colonies/mc-model-types'; @@ -16,38 +16,26 @@ export interface OpenApiConfig { uiPath: string; } -export interface IBaseCreatePackage { - dbId: string; - crs?: string; - priority?: number; -} - export interface ICleanupData { directoryPath?: string; cleanupExpirationTimeUTC?: Date; } -export interface ICreatePackage extends IBaseCreatePackage { - targetResolution?: number; - bbox?: BBox | Polygon | MultiPolygon; - callbackURLs: string[]; -} - -export interface ICreatePackageRoi extends IBaseCreatePackage { +export interface ICreatePackageRoi { + dbId: string; + crs?: string; + priority?: number; roi?: FeatureCollection; callbackURLs?: string[]; description?: string; } -export interface ICallbackBase { +export interface ICallbackTargetExport { url: string; -} - -export interface ICallbackTargetExport extends ICallbackBase { roi: FeatureCollection; } -export interface IWorkerInputBase { +export interface IWorkerExportInput { dbId: string; relativeDirectoryPath: string; exportVersion: ExportVersion; @@ -60,9 +48,6 @@ export interface IWorkerInputBase { sources: IMapSource[]; gpkgEstimatedSize?: number; targetFormat?: TileOutputFormat; -} - -export interface IWorkerExportInput extends IWorkerInputBase { callbacks?: ICallbackTargetExport[]; roi: FeatureCollection; fileNamesTemplates: ILinkDefinition; @@ -123,18 +108,6 @@ export interface JobExportDuplicationParams { roi: FeatureCollection; } -export interface IJobParameters { - targetResolution: number; - relativeDirectoryPath: string; - crs: string; - exportVersion: ExportVersion; - sanitizedBbox: BBox; - zoomLevel: number; - fileName: string; - gpkgEstimatedSize?: number; - cleanupData?: ICleanupData; -} - export interface IJobExportParameters { relativeDirectoryPath: string; crs: string; @@ -171,19 +144,6 @@ export interface ITaskParameters { sources: IMapSource[]; } -/** - * @deprecated GetMap API - will be deprecated on future - */ -export interface IInput { - jobId: string; - footprint?: Polygon | MultiPolygon; - bbox: BBox | true; - zoomLevel: number; - packageName: string; - callbackURLs: string[]; - dbId: string; -} - export interface IExportJobStatusResponse { completedJobs: JobExportResponse[] | undefined; failedJobs: JobExportResponse[] | undefined; diff --git a/src/finalizationManager.ts b/src/finalizationManager.ts index d8e820a..c4cbe77 100644 --- a/src/finalizationManager.ts +++ b/src/finalizationManager.ts @@ -156,11 +156,11 @@ export class FinalizationManager { } public async jobStatusPoll(): Promise { - const roiExistsJobs = await this.handleROIJobs(); + const roiExistsJobs = await this.handleExportJobs(); return roiExistsJobs; } - private async handleROIJobs(): Promise { + private async handleExportJobs(): Promise { let existsJobs = false; const roiJobs = await this.taskManager.getExportJobsByTaskStatus(); // new api by roi, this.logger.debug({ ...roiJobs, msg: `Handling ROI jobs` }); diff --git a/tests/mocks/data.ts b/tests/mocks/data.ts index 57e6a44..020feda 100644 --- a/tests/mocks/data.ts +++ b/tests/mocks/data.ts @@ -2,7 +2,7 @@ import { LayerMetadata, ProductType, RecordType, TileOutputFormat } from '@map-colonies/mc-model-types'; import { IJobResponse, OperationStatus } from '@map-colonies/mc-priority-queue'; import { FeatureCollection } from '@turf/helpers'; -import { ExportVersion, ICreatePackage, IJobExportParameters, ITaskParameters, IWorkerExportInput } from '../../src/common/interfaces'; +import { ExportVersion, IJobExportParameters, ITaskParameters, IWorkerExportInput } from '../../src/common/interfaces'; const layerMetadata: LayerMetadata = { type: 'RECORD_RASTER', @@ -193,14 +193,6 @@ const layerFromCatalogSample = { metadata: layerMetadataSample, }; -const userInput: ICreatePackage = { - dbId: '0c3e455f-4aeb-4258-982d-f7773469a92d', - targetResolution: 0.00439453125, - callbackURLs: ['http://callback-url.com'], - bbox: [-5, 3, 25, 41], - crs: 'EPSG:4326', -}; - const fc1: FeatureCollection = { type: 'FeatureCollection', features: [ @@ -872,7 +864,6 @@ const pycswRecord = { export { layerFromCatalog, workerExportInput, - userInput, fc1, fcNoMaxResolutionDeg, fcBadMinResolutionDeg, diff --git a/tests/unit/createPackage/models/createPackageModel.spec.ts b/tests/unit/createPackage/models/createPackageModel.spec.ts index 924e126..d8996fe 100644 --- a/tests/unit/createPackage/models/createPackageModel.spec.ts +++ b/tests/unit/createPackage/models/createPackageModel.spec.ts @@ -2,29 +2,12 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ /* eslint-disable @typescript-eslint/naming-convention */ import fs from 'fs'; -import { BadRequestError, InsufficientStorage } from '@map-colonies/error-types'; import jsLogger from '@map-colonies/js-logger'; -import { OperationStatus } from '@map-colonies/mc-priority-queue'; import { tracerMock } from '../../../mocks/clients/tracer'; -import { jobManagerWrapperMock, updateJobMock, findExportJobMock, createExportMock } from '../../../mocks/clients/jobManagerWrapper'; +import { jobManagerWrapperMock } from '../../../mocks/clients/jobManagerWrapper'; import { catalogManagerMock, findLayerMock } from '../../../mocks/clients/catalogManagerClient'; -import { ExportVersion, ICreateExportJobResponse, ICreatePackageRoi, JobExportDuplicationParams } from '../../../../src/common/interfaces'; import { CreatePackageManager } from '../../../../src/createPackage/models/createPackageManager'; -import { - completedExportJob, - userInput, - metadataExportJson, - layerFromCatalogSample, - fc1, - featuresRecordsSampleFc1, - layerMetadataRoi, - pycswRecord, - inProgressExportJob, - fcNoIntersection, - fcTooHighResolution, - fcMinResolutionDeg, - featuresRecordsSampleFcMinResolutionDeg, -} from '../../../mocks/data'; +import { completedExportJob, metadataExportJson, layerFromCatalogSample } from '../../../mocks/data'; import { configMock, registerDefaultConfig } from '../../../mocks/config'; import * as utils from '../../../../src/common/utils'; @@ -94,367 +77,5 @@ describe('CreatePackageManager', () => { expect(fs.promises.writeFile).toHaveBeenCalledTimes(0); }); }); - describe('#create', () => { - it('should create job and return its job and task ids', async () => { - const req: ICreatePackageRoi = { - dbId: pycswRecord.id, - roi: fc1, - callbackURLs: ['testUrl'], - crs: 'EPSG:4326', - }; - - const jobDupParams: JobExportDuplicationParams = { - resourceId: 'string', - version: '1.0', - dbId: pycswRecord.id, - roi: fc1, - crs: 'EPSG:4326', - }; - - const expectedCreateJobResponse = { - jobId: '09e29fa8-7283-4334-b3a4-99f75922de59', - taskIds: ['66aa1e2e-784c-4178-b5a0-af962937d561'], - status: OperationStatus.IN_PROGRESS, - }; - const validateFreeSpaceSpy = jest.spyOn(CreatePackageManager.prototype as unknown as { validateFreeSpace: jest.Mock }, 'validateFreeSpace'); - const generateExportFileNamesSpy = jest.spyOn( - CreatePackageManager.prototype as unknown as { generateExportFileNames: jest.Mock }, - 'generateExportFileNames' - ); - - const testPycswRecord = JSON.parse(JSON.stringify(pycswRecord)); - findLayerMock.mockResolvedValue({ ...testPycswRecord }); - createExportMock.mockResolvedValue(expectedCreateJobResponse); - findExportJobMock.mockResolvedValue(undefined); - - validateFreeSpaceSpy.mockResolvedValue(true); - const res = await createPackageManager.createPackageRoi(req); - - expect(res).toEqual(expectedCreateJobResponse); - expect(findLayerMock).toHaveBeenCalledWith(req.dbId); - expect(findLayerMock).toHaveBeenCalledTimes(1); - expect(createExportMock).toHaveBeenCalledTimes(1); - expect(findExportJobMock).toHaveBeenCalledTimes(3); - expect(findExportJobMock.mock.calls[0]).toEqual([OperationStatus.COMPLETED, jobDupParams]); - expect(findExportJobMock.mock.calls[1]).toEqual([OperationStatus.IN_PROGRESS, jobDupParams, true]); - expect(findExportJobMock.mock.calls[2]).toEqual([OperationStatus.PENDING, jobDupParams, true]); - expect(generateExportFileNamesSpy).toHaveBeenCalledTimes(1); - expect(generateExportFileNamesSpy).toHaveBeenCalledWith( - testPycswRecord.metadata.productType, - testPycswRecord.metadata.productId, - testPycswRecord.metadata.productVersion, - featuresRecordsSampleFc1 - ); - expect(res).toBe(expectedCreateJobResponse); - }); - - it(`should create job and take original layer's resolution and footprint as ROI`, async () => { - const req: ICreatePackageRoi = { - dbId: pycswRecord.id, - callbackURLs: ['testUrl'], - crs: 'EPSG:4326', - }; - - const expectedCreateJobResponse = { - jobId: '09e29fa8-7283-4334-b3a4-99f75922de59', - taskIds: ['66aa1e2e-784c-4178-b5a0-af962937d561'], - status: OperationStatus.IN_PROGRESS, - }; - - const testPycswRecord = JSON.parse(JSON.stringify(pycswRecord)); - - const validateFreeSpaceSpy = jest.spyOn(CreatePackageManager.prototype as unknown as { validateFreeSpace: jest.Mock }, 'validateFreeSpace'); - createExportMock.mockResolvedValue(expectedCreateJobResponse); - findLayerMock.mockResolvedValue({ ...testPycswRecord }); - findExportJobMock.mockResolvedValue(undefined); - - validateFreeSpaceSpy.mockResolvedValue(true); - - const res = await createPackageManager.createPackageRoi(req); - expect(res).toEqual(expectedCreateJobResponse); - expect(findLayerMock).toHaveBeenCalledWith(req.dbId); - expect(findLayerMock).toHaveBeenCalledTimes(1); - expect(createExportMock).toHaveBeenCalledTimes(1); - expect(findExportJobMock).toHaveBeenCalledTimes(3); - expect(createExportMock).toHaveBeenCalledWith( - expect.objectContaining({ - roi: layerMetadataRoi, - }) - ); - }); - - it('should create job with batches from provided minResolutionDeg', async () => { - const req: ICreatePackageRoi = { - dbId: pycswRecord.id, - roi: fcMinResolutionDeg, - callbackURLs: ['testUrl'], - crs: 'EPSG:4326', - }; - - const expectedCreateJobResponse = { - jobId: '09e29fa8-7283-4334-b3a4-99f75922de59', - taskIds: ['66aa1e2e-784c-4178-b5a0-af962937d561'], - status: OperationStatus.IN_PROGRESS, - }; - const validateFreeSpaceSpy = jest.spyOn(CreatePackageManager.prototype as unknown as { validateFreeSpace: jest.Mock }, 'validateFreeSpace'); - const generateExportFileNamesSpy = jest.spyOn( - CreatePackageManager.prototype as unknown as { generateExportFileNames: jest.Mock }, - 'generateExportFileNames' - ); - - const calculateEstimateGpkgSizeSpy = jest.spyOn(utils, 'calculateEstimateGpkgSize'); - const testPycswRecord = JSON.parse(JSON.stringify(pycswRecord)); - findLayerMock.mockResolvedValue({ ...testPycswRecord }); - createExportMock.mockResolvedValue(expectedCreateJobResponse); - findExportJobMock.mockResolvedValue(undefined); - - validateFreeSpaceSpy.mockResolvedValue(true); - const res = await createPackageManager.createPackageRoi(req); - // eslint-disable-next-line @typescript-eslint/no-unused-expressions - const calculatedBatches = calculateEstimateGpkgSizeSpy.mock.calls[0][0]; - expect(res).toEqual(expectedCreateJobResponse); - expect(generateExportFileNamesSpy).toHaveBeenCalledTimes(1); - expect(generateExportFileNamesSpy).toHaveBeenCalledWith( - testPycswRecord.metadata.productType, - testPycswRecord.metadata.productId, - testPycswRecord.metadata.productVersion, - featuresRecordsSampleFcMinResolutionDeg - ); - expect(calculatedBatches).toHaveLength(7); - expect(res).toBe(expectedCreateJobResponse); - }); - - it('should return callbackParam(webhook) for existing completed job', async () => { - const req: ICreatePackageRoi = { - dbId: pycswRecord.id, - callbackURLs: ['testUrl'], - crs: 'EPSG:4326', - roi: fc1, - }; - const jobDupParams: JobExportDuplicationParams = { - resourceId: pycswRecord.metadata.productId as string, - version: pycswRecord.metadata.productVersion as string, - dbId: pycswRecord.id, - crs: userInput.crs as string, - roi: fc1, - }; - - const testPycswRecord = JSON.parse(JSON.stringify(pycswRecord)); - - findLayerMock.mockResolvedValue({ ...testPycswRecord }); - createExportMock.mockResolvedValue(undefined); - findExportJobMock.mockResolvedValue(JSON.parse(JSON.stringify(completedExportJob))); - updateJobMock.mockResolvedValue(undefined); - - const res = await createPackageManager.createPackageRoi(req); - expect(findLayerMock).toHaveBeenCalledWith(pycswRecord.id); - expect(findLayerMock).toHaveBeenCalledTimes(1); - expect(findExportJobMock).toHaveBeenCalledTimes(1); - expect(findExportJobMock.mock.calls[0]).toEqual([OperationStatus.COMPLETED, jobDupParams]); - expect(createExportMock).toHaveBeenCalledTimes(0); - expect(JSON.stringify(res)).toStrictEqual( - JSON.stringify({ ...completedExportJob.parameters.callbackParams, status: OperationStatus.COMPLETED }) - ); - }); - - it('should return job and task-ids of existing pending job', async () => { - const req: ICreatePackageRoi = { - dbId: pycswRecord.id, - callbackURLs: ['testUrl'], - crs: 'EPSG:4326', - roi: fc1, - }; - const jobDupParams: JobExportDuplicationParams = { - resourceId: pycswRecord.metadata.productId as string, - version: pycswRecord.metadata.productVersion as string, - dbId: pycswRecord.id, - crs: userInput.crs as string, - roi: fc1, - }; - - const testPycswRecord = JSON.parse(JSON.stringify(pycswRecord)); - - findLayerMock.mockResolvedValue({ ...testPycswRecord }); - createExportMock.mockResolvedValue(undefined); - findExportJobMock.mockResolvedValueOnce(undefined); - findExportJobMock.mockResolvedValueOnce(undefined); - findExportJobMock.mockResolvedValueOnce(JSON.parse(JSON.stringify(inProgressExportJob))); - findExportJobMock.mockResolvedValueOnce(undefined); - updateJobMock.mockResolvedValue(undefined); - - const res = await createPackageManager.createPackageRoi(req); - expect(findLayerMock).toHaveBeenCalledWith(pycswRecord.id); - expect(findLayerMock).toHaveBeenCalledTimes(1); - expect(findExportJobMock).toHaveBeenCalledTimes(4); - expect(findExportJobMock.mock.calls[0]).toEqual([OperationStatus.COMPLETED, jobDupParams]); - expect(findExportJobMock.mock.calls[1]).toEqual([OperationStatus.IN_PROGRESS, jobDupParams, true]); - expect(findExportJobMock.mock.calls[2]).toEqual([OperationStatus.PENDING, jobDupParams, true]); - expect(findExportJobMock.mock.calls[3]).toEqual([OperationStatus.COMPLETED, jobDupParams]); - expect(createExportMock).toHaveBeenCalledTimes(0); - expect(res).toStrictEqual({ - jobId: inProgressExportJob.id, - taskIds: ['1f765695-338b-4752-b182-a8cbae3c610e'], - status: OperationStatus.IN_PROGRESS, - isDuplicated: true, - }); - }); - - it('should return job and task-ids of existing in progress job', async () => { - const req: ICreatePackageRoi = { - dbId: pycswRecord.id, - callbackURLs: ['testUrl'], - crs: 'EPSG:4326', - roi: fc1, - }; - const jobDupParams: JobExportDuplicationParams = { - resourceId: pycswRecord.metadata.productId as string, - version: pycswRecord.metadata.productVersion as string, - dbId: pycswRecord.id, - crs: userInput.crs as string, - roi: fc1, - }; - - const testPycswRecord = JSON.parse(JSON.stringify(pycswRecord)); - - findLayerMock.mockResolvedValue({ ...testPycswRecord }); - createExportMock.mockResolvedValue(undefined); - findExportJobMock.mockResolvedValueOnce(undefined); - findExportJobMock.mockResolvedValueOnce(JSON.parse(JSON.stringify(inProgressExportJob))); - findExportJobMock.mockResolvedValueOnce(undefined); - updateJobMock.mockResolvedValue(undefined); - - const res = await createPackageManager.createPackageRoi(req); - expect(findLayerMock).toHaveBeenCalledWith(pycswRecord.id); - expect(findLayerMock).toHaveBeenCalledTimes(1); - expect(findExportJobMock).toHaveBeenCalledTimes(3); - expect(findExportJobMock.mock.calls[0]).toEqual([OperationStatus.COMPLETED, jobDupParams]); - expect(findExportJobMock.mock.calls[1]).toEqual([OperationStatus.IN_PROGRESS, jobDupParams, true]); - expect(findExportJobMock.mock.calls[2]).toEqual([OperationStatus.COMPLETED, jobDupParams]); - expect(createExportMock).toHaveBeenCalledTimes(0); - expect(res).toStrictEqual({ - jobId: inProgressExportJob.id, - taskIds: ['1f765695-338b-4752-b182-a8cbae3c610e'], - status: OperationStatus.IN_PROGRESS, - isDuplicated: true, - }); - }); - - it('should increase callbacks array of existing in progress job', async () => { - const req: ICreatePackageRoi = { - dbId: pycswRecord.id, - callbackURLs: ['http://new-added-callback-url.com'], - crs: 'EPSG:4326', - roi: fc1, - }; - - const jobDupParams: JobExportDuplicationParams = { - resourceId: pycswRecord.metadata.productId as string, - version: pycswRecord.metadata.productVersion as string, - dbId: pycswRecord.id, - crs: userInput.crs as string, - roi: fc1, - }; - const expirationDays: number = configMock.get('cleanupExpirationDays'); - const testExpirationDate = new Date(); - testExpirationDate.setDate(testExpirationDate.getDate() - expirationDays); - const testPycswRecord = JSON.parse(JSON.stringify(pycswRecord)); - - findLayerMock.mockResolvedValue({ ...testPycswRecord }); - createExportMock.mockResolvedValue(undefined); - updateJobMock.mockResolvedValue(undefined); - findExportJobMock.mockResolvedValueOnce(undefined); - findExportJobMock.mockResolvedValueOnce(JSON.parse(JSON.stringify({ ...inProgressExportJob, expirationDate: testExpirationDate }))); - findExportJobMock.mockResolvedValueOnce(undefined); - const jobUpdateParams = { - parameters: { - crs: 'EPSG:4326', - exportVersion: ExportVersion.ROI, - callbacks: [ - { url: 'http://localhost:6969', roi: fc1 }, - { url: 'http://new-added-callback-url.com', roi: fc1 }, - ], - roi: fc1, - fileNamesTemplates: inProgressExportJob.parameters.fileNamesTemplates, - gpkgEstimatedSize: inProgressExportJob.parameters.gpkgEstimatedSize, - relativeDirectoryPath: inProgressExportJob.parameters.relativeDirectoryPath, - }, - }; - const res = await createPackageManager.createPackageRoi({ ...req, callbackURLs: ['http://new-added-callback-url.com'] }); - const expectedReturn: ICreateExportJobResponse = { - jobId: inProgressExportJob.id, - taskIds: ['1f765695-338b-4752-b182-a8cbae3c610e'], - status: OperationStatus.IN_PROGRESS, - isDuplicated: true, - }; - - expect(res).toEqual(expectedReturn); - expect(findLayerMock).toHaveBeenCalledWith(pycswRecord.id); - expect(findLayerMock).toHaveBeenCalledTimes(1); - expect(findExportJobMock).toHaveBeenCalledTimes(3); - expect(findExportJobMock.mock.calls[0]).toEqual([OperationStatus.COMPLETED, jobDupParams]); - expect(findExportJobMock.mock.calls[1]).toEqual([OperationStatus.IN_PROGRESS, jobDupParams, true]); - expect(findExportJobMock.mock.calls[2]).toEqual([OperationStatus.COMPLETED, jobDupParams]); - expect(createExportMock).toHaveBeenCalledTimes(0); - expect(updateJobMock).toHaveBeenCalledWith(inProgressExportJob.id, jobUpdateParams); - }); - - it('should throw bad request error when requested feature is without intersection with layer geometry', async () => { - const req: ICreatePackageRoi = { - dbId: pycswRecord.id, - callbackURLs: ['http://new-added-callback-url.com'], - crs: 'EPSG:4326', - roi: fcNoIntersection, - }; - - const testPycswRecord = JSON.parse(JSON.stringify(pycswRecord)); - findLayerMock.mockResolvedValue({ ...testPycswRecord }); - - const action = async () => createPackageManager.createPackageRoi(req); - - await expect(action).rejects.toThrow(BadRequestError); - expect(findLayerMock).toHaveBeenCalledTimes(1); - expect(findLayerMock).toHaveBeenCalledWith(pycswRecord.id); - }); - - it('should throw bad request error when requested feature maxDegResolution is higher than the layer resolution', async () => { - const req: ICreatePackageRoi = { - dbId: pycswRecord.id, - callbackURLs: ['http://new-added-callback-url.com'], - crs: 'EPSG:4326', - roi: fcTooHighResolution, - }; - - const testPycswRecord = JSON.parse(JSON.stringify(pycswRecord)); - findLayerMock.mockResolvedValue({ ...testPycswRecord }); - - const action = async () => createPackageManager.createPackageRoi(req); - - await expect(action).rejects.toThrow(BadRequestError); - expect(findLayerMock).toHaveBeenCalledTimes(1); - expect(findLayerMock).toHaveBeenCalledWith(pycswRecord.id); - }); - - it('should throw bad request error when requested gpkg estimated size is larger than storage', async () => { - const req: ICreatePackageRoi = { - dbId: pycswRecord.id, - roi: fc1, - callbackURLs: ['testUrl'], - crs: 'EPSG:4326', - }; - - const validateFreeSpaceSpy = jest.spyOn(CreatePackageManager.prototype as unknown as { validateFreeSpace: jest.Mock }, 'validateFreeSpace'); - - const testPycswRecord = JSON.parse(JSON.stringify(pycswRecord)); - findLayerMock.mockResolvedValue({ ...testPycswRecord }); - findExportJobMock.mockResolvedValue(undefined); - validateFreeSpaceSpy.mockResolvedValue(false); - - const action = async () => createPackageManager.createPackageRoi(req); - await expect(action).rejects.toThrow(InsufficientStorage); - expect(findLayerMock).toHaveBeenCalledTimes(1); - expect(findLayerMock).toHaveBeenCalledWith(pycswRecord.id); - expect(createExportMock).toHaveBeenCalledTimes(0); - }); - }); }); }); From d113f43750bc8438c273ea075b37b59c5f1ddced Mon Sep 17 00:00:00 2001 From: razbroc Date: Mon, 25 Mar 2024 17:19:00 +0200 Subject: [PATCH 15/19] fix: fir pr comments --- tests/mocks/data.ts | 9 +- .../models/createPackageModel.spec.ts | 383 +++++++++++++++++- 2 files changed, 389 insertions(+), 3 deletions(-) diff --git a/tests/mocks/data.ts b/tests/mocks/data.ts index 020feda..1c938cc 100644 --- a/tests/mocks/data.ts +++ b/tests/mocks/data.ts @@ -2,7 +2,7 @@ import { LayerMetadata, ProductType, RecordType, TileOutputFormat } from '@map-colonies/mc-model-types'; import { IJobResponse, OperationStatus } from '@map-colonies/mc-priority-queue'; import { FeatureCollection } from '@turf/helpers'; -import { ExportVersion, IJobExportParameters, ITaskParameters, IWorkerExportInput } from '../../src/common/interfaces'; +import { ExportVersion, ICreatePackageRoi, IJobExportParameters, ITaskParameters, IWorkerExportInput } from '../../src/common/interfaces'; const layerMetadata: LayerMetadata = { type: 'RECORD_RASTER', @@ -193,6 +193,12 @@ const layerFromCatalogSample = { metadata: layerMetadataSample, }; +const userInput: ICreatePackageRoi = { + dbId: '0c3e455f-4aeb-4258-982d-f7773469a92d', + callbackURLs: ['http://callback-url.com'], + crs: 'EPSG:4326', +}; + const fc1: FeatureCollection = { type: 'FeatureCollection', features: [ @@ -864,6 +870,7 @@ const pycswRecord = { export { layerFromCatalog, workerExportInput, + userInput, fc1, fcNoMaxResolutionDeg, fcBadMinResolutionDeg, diff --git a/tests/unit/createPackage/models/createPackageModel.spec.ts b/tests/unit/createPackage/models/createPackageModel.spec.ts index d8996fe..924e126 100644 --- a/tests/unit/createPackage/models/createPackageModel.spec.ts +++ b/tests/unit/createPackage/models/createPackageModel.spec.ts @@ -2,12 +2,29 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ /* eslint-disable @typescript-eslint/naming-convention */ import fs from 'fs'; +import { BadRequestError, InsufficientStorage } from '@map-colonies/error-types'; import jsLogger from '@map-colonies/js-logger'; +import { OperationStatus } from '@map-colonies/mc-priority-queue'; import { tracerMock } from '../../../mocks/clients/tracer'; -import { jobManagerWrapperMock } from '../../../mocks/clients/jobManagerWrapper'; +import { jobManagerWrapperMock, updateJobMock, findExportJobMock, createExportMock } from '../../../mocks/clients/jobManagerWrapper'; import { catalogManagerMock, findLayerMock } from '../../../mocks/clients/catalogManagerClient'; +import { ExportVersion, ICreateExportJobResponse, ICreatePackageRoi, JobExportDuplicationParams } from '../../../../src/common/interfaces'; import { CreatePackageManager } from '../../../../src/createPackage/models/createPackageManager'; -import { completedExportJob, metadataExportJson, layerFromCatalogSample } from '../../../mocks/data'; +import { + completedExportJob, + userInput, + metadataExportJson, + layerFromCatalogSample, + fc1, + featuresRecordsSampleFc1, + layerMetadataRoi, + pycswRecord, + inProgressExportJob, + fcNoIntersection, + fcTooHighResolution, + fcMinResolutionDeg, + featuresRecordsSampleFcMinResolutionDeg, +} from '../../../mocks/data'; import { configMock, registerDefaultConfig } from '../../../mocks/config'; import * as utils from '../../../../src/common/utils'; @@ -77,5 +94,367 @@ describe('CreatePackageManager', () => { expect(fs.promises.writeFile).toHaveBeenCalledTimes(0); }); }); + describe('#create', () => { + it('should create job and return its job and task ids', async () => { + const req: ICreatePackageRoi = { + dbId: pycswRecord.id, + roi: fc1, + callbackURLs: ['testUrl'], + crs: 'EPSG:4326', + }; + + const jobDupParams: JobExportDuplicationParams = { + resourceId: 'string', + version: '1.0', + dbId: pycswRecord.id, + roi: fc1, + crs: 'EPSG:4326', + }; + + const expectedCreateJobResponse = { + jobId: '09e29fa8-7283-4334-b3a4-99f75922de59', + taskIds: ['66aa1e2e-784c-4178-b5a0-af962937d561'], + status: OperationStatus.IN_PROGRESS, + }; + const validateFreeSpaceSpy = jest.spyOn(CreatePackageManager.prototype as unknown as { validateFreeSpace: jest.Mock }, 'validateFreeSpace'); + const generateExportFileNamesSpy = jest.spyOn( + CreatePackageManager.prototype as unknown as { generateExportFileNames: jest.Mock }, + 'generateExportFileNames' + ); + + const testPycswRecord = JSON.parse(JSON.stringify(pycswRecord)); + findLayerMock.mockResolvedValue({ ...testPycswRecord }); + createExportMock.mockResolvedValue(expectedCreateJobResponse); + findExportJobMock.mockResolvedValue(undefined); + + validateFreeSpaceSpy.mockResolvedValue(true); + const res = await createPackageManager.createPackageRoi(req); + + expect(res).toEqual(expectedCreateJobResponse); + expect(findLayerMock).toHaveBeenCalledWith(req.dbId); + expect(findLayerMock).toHaveBeenCalledTimes(1); + expect(createExportMock).toHaveBeenCalledTimes(1); + expect(findExportJobMock).toHaveBeenCalledTimes(3); + expect(findExportJobMock.mock.calls[0]).toEqual([OperationStatus.COMPLETED, jobDupParams]); + expect(findExportJobMock.mock.calls[1]).toEqual([OperationStatus.IN_PROGRESS, jobDupParams, true]); + expect(findExportJobMock.mock.calls[2]).toEqual([OperationStatus.PENDING, jobDupParams, true]); + expect(generateExportFileNamesSpy).toHaveBeenCalledTimes(1); + expect(generateExportFileNamesSpy).toHaveBeenCalledWith( + testPycswRecord.metadata.productType, + testPycswRecord.metadata.productId, + testPycswRecord.metadata.productVersion, + featuresRecordsSampleFc1 + ); + expect(res).toBe(expectedCreateJobResponse); + }); + + it(`should create job and take original layer's resolution and footprint as ROI`, async () => { + const req: ICreatePackageRoi = { + dbId: pycswRecord.id, + callbackURLs: ['testUrl'], + crs: 'EPSG:4326', + }; + + const expectedCreateJobResponse = { + jobId: '09e29fa8-7283-4334-b3a4-99f75922de59', + taskIds: ['66aa1e2e-784c-4178-b5a0-af962937d561'], + status: OperationStatus.IN_PROGRESS, + }; + + const testPycswRecord = JSON.parse(JSON.stringify(pycswRecord)); + + const validateFreeSpaceSpy = jest.spyOn(CreatePackageManager.prototype as unknown as { validateFreeSpace: jest.Mock }, 'validateFreeSpace'); + createExportMock.mockResolvedValue(expectedCreateJobResponse); + findLayerMock.mockResolvedValue({ ...testPycswRecord }); + findExportJobMock.mockResolvedValue(undefined); + + validateFreeSpaceSpy.mockResolvedValue(true); + + const res = await createPackageManager.createPackageRoi(req); + expect(res).toEqual(expectedCreateJobResponse); + expect(findLayerMock).toHaveBeenCalledWith(req.dbId); + expect(findLayerMock).toHaveBeenCalledTimes(1); + expect(createExportMock).toHaveBeenCalledTimes(1); + expect(findExportJobMock).toHaveBeenCalledTimes(3); + expect(createExportMock).toHaveBeenCalledWith( + expect.objectContaining({ + roi: layerMetadataRoi, + }) + ); + }); + + it('should create job with batches from provided minResolutionDeg', async () => { + const req: ICreatePackageRoi = { + dbId: pycswRecord.id, + roi: fcMinResolutionDeg, + callbackURLs: ['testUrl'], + crs: 'EPSG:4326', + }; + + const expectedCreateJobResponse = { + jobId: '09e29fa8-7283-4334-b3a4-99f75922de59', + taskIds: ['66aa1e2e-784c-4178-b5a0-af962937d561'], + status: OperationStatus.IN_PROGRESS, + }; + const validateFreeSpaceSpy = jest.spyOn(CreatePackageManager.prototype as unknown as { validateFreeSpace: jest.Mock }, 'validateFreeSpace'); + const generateExportFileNamesSpy = jest.spyOn( + CreatePackageManager.prototype as unknown as { generateExportFileNames: jest.Mock }, + 'generateExportFileNames' + ); + + const calculateEstimateGpkgSizeSpy = jest.spyOn(utils, 'calculateEstimateGpkgSize'); + const testPycswRecord = JSON.parse(JSON.stringify(pycswRecord)); + findLayerMock.mockResolvedValue({ ...testPycswRecord }); + createExportMock.mockResolvedValue(expectedCreateJobResponse); + findExportJobMock.mockResolvedValue(undefined); + + validateFreeSpaceSpy.mockResolvedValue(true); + const res = await createPackageManager.createPackageRoi(req); + // eslint-disable-next-line @typescript-eslint/no-unused-expressions + const calculatedBatches = calculateEstimateGpkgSizeSpy.mock.calls[0][0]; + expect(res).toEqual(expectedCreateJobResponse); + expect(generateExportFileNamesSpy).toHaveBeenCalledTimes(1); + expect(generateExportFileNamesSpy).toHaveBeenCalledWith( + testPycswRecord.metadata.productType, + testPycswRecord.metadata.productId, + testPycswRecord.metadata.productVersion, + featuresRecordsSampleFcMinResolutionDeg + ); + expect(calculatedBatches).toHaveLength(7); + expect(res).toBe(expectedCreateJobResponse); + }); + + it('should return callbackParam(webhook) for existing completed job', async () => { + const req: ICreatePackageRoi = { + dbId: pycswRecord.id, + callbackURLs: ['testUrl'], + crs: 'EPSG:4326', + roi: fc1, + }; + const jobDupParams: JobExportDuplicationParams = { + resourceId: pycswRecord.metadata.productId as string, + version: pycswRecord.metadata.productVersion as string, + dbId: pycswRecord.id, + crs: userInput.crs as string, + roi: fc1, + }; + + const testPycswRecord = JSON.parse(JSON.stringify(pycswRecord)); + + findLayerMock.mockResolvedValue({ ...testPycswRecord }); + createExportMock.mockResolvedValue(undefined); + findExportJobMock.mockResolvedValue(JSON.parse(JSON.stringify(completedExportJob))); + updateJobMock.mockResolvedValue(undefined); + + const res = await createPackageManager.createPackageRoi(req); + expect(findLayerMock).toHaveBeenCalledWith(pycswRecord.id); + expect(findLayerMock).toHaveBeenCalledTimes(1); + expect(findExportJobMock).toHaveBeenCalledTimes(1); + expect(findExportJobMock.mock.calls[0]).toEqual([OperationStatus.COMPLETED, jobDupParams]); + expect(createExportMock).toHaveBeenCalledTimes(0); + expect(JSON.stringify(res)).toStrictEqual( + JSON.stringify({ ...completedExportJob.parameters.callbackParams, status: OperationStatus.COMPLETED }) + ); + }); + + it('should return job and task-ids of existing pending job', async () => { + const req: ICreatePackageRoi = { + dbId: pycswRecord.id, + callbackURLs: ['testUrl'], + crs: 'EPSG:4326', + roi: fc1, + }; + const jobDupParams: JobExportDuplicationParams = { + resourceId: pycswRecord.metadata.productId as string, + version: pycswRecord.metadata.productVersion as string, + dbId: pycswRecord.id, + crs: userInput.crs as string, + roi: fc1, + }; + + const testPycswRecord = JSON.parse(JSON.stringify(pycswRecord)); + + findLayerMock.mockResolvedValue({ ...testPycswRecord }); + createExportMock.mockResolvedValue(undefined); + findExportJobMock.mockResolvedValueOnce(undefined); + findExportJobMock.mockResolvedValueOnce(undefined); + findExportJobMock.mockResolvedValueOnce(JSON.parse(JSON.stringify(inProgressExportJob))); + findExportJobMock.mockResolvedValueOnce(undefined); + updateJobMock.mockResolvedValue(undefined); + + const res = await createPackageManager.createPackageRoi(req); + expect(findLayerMock).toHaveBeenCalledWith(pycswRecord.id); + expect(findLayerMock).toHaveBeenCalledTimes(1); + expect(findExportJobMock).toHaveBeenCalledTimes(4); + expect(findExportJobMock.mock.calls[0]).toEqual([OperationStatus.COMPLETED, jobDupParams]); + expect(findExportJobMock.mock.calls[1]).toEqual([OperationStatus.IN_PROGRESS, jobDupParams, true]); + expect(findExportJobMock.mock.calls[2]).toEqual([OperationStatus.PENDING, jobDupParams, true]); + expect(findExportJobMock.mock.calls[3]).toEqual([OperationStatus.COMPLETED, jobDupParams]); + expect(createExportMock).toHaveBeenCalledTimes(0); + expect(res).toStrictEqual({ + jobId: inProgressExportJob.id, + taskIds: ['1f765695-338b-4752-b182-a8cbae3c610e'], + status: OperationStatus.IN_PROGRESS, + isDuplicated: true, + }); + }); + + it('should return job and task-ids of existing in progress job', async () => { + const req: ICreatePackageRoi = { + dbId: pycswRecord.id, + callbackURLs: ['testUrl'], + crs: 'EPSG:4326', + roi: fc1, + }; + const jobDupParams: JobExportDuplicationParams = { + resourceId: pycswRecord.metadata.productId as string, + version: pycswRecord.metadata.productVersion as string, + dbId: pycswRecord.id, + crs: userInput.crs as string, + roi: fc1, + }; + + const testPycswRecord = JSON.parse(JSON.stringify(pycswRecord)); + + findLayerMock.mockResolvedValue({ ...testPycswRecord }); + createExportMock.mockResolvedValue(undefined); + findExportJobMock.mockResolvedValueOnce(undefined); + findExportJobMock.mockResolvedValueOnce(JSON.parse(JSON.stringify(inProgressExportJob))); + findExportJobMock.mockResolvedValueOnce(undefined); + updateJobMock.mockResolvedValue(undefined); + + const res = await createPackageManager.createPackageRoi(req); + expect(findLayerMock).toHaveBeenCalledWith(pycswRecord.id); + expect(findLayerMock).toHaveBeenCalledTimes(1); + expect(findExportJobMock).toHaveBeenCalledTimes(3); + expect(findExportJobMock.mock.calls[0]).toEqual([OperationStatus.COMPLETED, jobDupParams]); + expect(findExportJobMock.mock.calls[1]).toEqual([OperationStatus.IN_PROGRESS, jobDupParams, true]); + expect(findExportJobMock.mock.calls[2]).toEqual([OperationStatus.COMPLETED, jobDupParams]); + expect(createExportMock).toHaveBeenCalledTimes(0); + expect(res).toStrictEqual({ + jobId: inProgressExportJob.id, + taskIds: ['1f765695-338b-4752-b182-a8cbae3c610e'], + status: OperationStatus.IN_PROGRESS, + isDuplicated: true, + }); + }); + + it('should increase callbacks array of existing in progress job', async () => { + const req: ICreatePackageRoi = { + dbId: pycswRecord.id, + callbackURLs: ['http://new-added-callback-url.com'], + crs: 'EPSG:4326', + roi: fc1, + }; + + const jobDupParams: JobExportDuplicationParams = { + resourceId: pycswRecord.metadata.productId as string, + version: pycswRecord.metadata.productVersion as string, + dbId: pycswRecord.id, + crs: userInput.crs as string, + roi: fc1, + }; + const expirationDays: number = configMock.get('cleanupExpirationDays'); + const testExpirationDate = new Date(); + testExpirationDate.setDate(testExpirationDate.getDate() - expirationDays); + const testPycswRecord = JSON.parse(JSON.stringify(pycswRecord)); + + findLayerMock.mockResolvedValue({ ...testPycswRecord }); + createExportMock.mockResolvedValue(undefined); + updateJobMock.mockResolvedValue(undefined); + findExportJobMock.mockResolvedValueOnce(undefined); + findExportJobMock.mockResolvedValueOnce(JSON.parse(JSON.stringify({ ...inProgressExportJob, expirationDate: testExpirationDate }))); + findExportJobMock.mockResolvedValueOnce(undefined); + const jobUpdateParams = { + parameters: { + crs: 'EPSG:4326', + exportVersion: ExportVersion.ROI, + callbacks: [ + { url: 'http://localhost:6969', roi: fc1 }, + { url: 'http://new-added-callback-url.com', roi: fc1 }, + ], + roi: fc1, + fileNamesTemplates: inProgressExportJob.parameters.fileNamesTemplates, + gpkgEstimatedSize: inProgressExportJob.parameters.gpkgEstimatedSize, + relativeDirectoryPath: inProgressExportJob.parameters.relativeDirectoryPath, + }, + }; + const res = await createPackageManager.createPackageRoi({ ...req, callbackURLs: ['http://new-added-callback-url.com'] }); + const expectedReturn: ICreateExportJobResponse = { + jobId: inProgressExportJob.id, + taskIds: ['1f765695-338b-4752-b182-a8cbae3c610e'], + status: OperationStatus.IN_PROGRESS, + isDuplicated: true, + }; + + expect(res).toEqual(expectedReturn); + expect(findLayerMock).toHaveBeenCalledWith(pycswRecord.id); + expect(findLayerMock).toHaveBeenCalledTimes(1); + expect(findExportJobMock).toHaveBeenCalledTimes(3); + expect(findExportJobMock.mock.calls[0]).toEqual([OperationStatus.COMPLETED, jobDupParams]); + expect(findExportJobMock.mock.calls[1]).toEqual([OperationStatus.IN_PROGRESS, jobDupParams, true]); + expect(findExportJobMock.mock.calls[2]).toEqual([OperationStatus.COMPLETED, jobDupParams]); + expect(createExportMock).toHaveBeenCalledTimes(0); + expect(updateJobMock).toHaveBeenCalledWith(inProgressExportJob.id, jobUpdateParams); + }); + + it('should throw bad request error when requested feature is without intersection with layer geometry', async () => { + const req: ICreatePackageRoi = { + dbId: pycswRecord.id, + callbackURLs: ['http://new-added-callback-url.com'], + crs: 'EPSG:4326', + roi: fcNoIntersection, + }; + + const testPycswRecord = JSON.parse(JSON.stringify(pycswRecord)); + findLayerMock.mockResolvedValue({ ...testPycswRecord }); + + const action = async () => createPackageManager.createPackageRoi(req); + + await expect(action).rejects.toThrow(BadRequestError); + expect(findLayerMock).toHaveBeenCalledTimes(1); + expect(findLayerMock).toHaveBeenCalledWith(pycswRecord.id); + }); + + it('should throw bad request error when requested feature maxDegResolution is higher than the layer resolution', async () => { + const req: ICreatePackageRoi = { + dbId: pycswRecord.id, + callbackURLs: ['http://new-added-callback-url.com'], + crs: 'EPSG:4326', + roi: fcTooHighResolution, + }; + + const testPycswRecord = JSON.parse(JSON.stringify(pycswRecord)); + findLayerMock.mockResolvedValue({ ...testPycswRecord }); + + const action = async () => createPackageManager.createPackageRoi(req); + + await expect(action).rejects.toThrow(BadRequestError); + expect(findLayerMock).toHaveBeenCalledTimes(1); + expect(findLayerMock).toHaveBeenCalledWith(pycswRecord.id); + }); + + it('should throw bad request error when requested gpkg estimated size is larger than storage', async () => { + const req: ICreatePackageRoi = { + dbId: pycswRecord.id, + roi: fc1, + callbackURLs: ['testUrl'], + crs: 'EPSG:4326', + }; + + const validateFreeSpaceSpy = jest.spyOn(CreatePackageManager.prototype as unknown as { validateFreeSpace: jest.Mock }, 'validateFreeSpace'); + + const testPycswRecord = JSON.parse(JSON.stringify(pycswRecord)); + findLayerMock.mockResolvedValue({ ...testPycswRecord }); + findExportJobMock.mockResolvedValue(undefined); + validateFreeSpaceSpy.mockResolvedValue(false); + + const action = async () => createPackageManager.createPackageRoi(req); + await expect(action).rejects.toThrow(InsufficientStorage); + expect(findLayerMock).toHaveBeenCalledTimes(1); + expect(findLayerMock).toHaveBeenCalledWith(pycswRecord.id); + expect(createExportMock).toHaveBeenCalledTimes(0); + }); + }); }); }); From d1ec5ada7985ef60066f5a2e01670f446245989d Mon Sep 17 00:00:00 2001 From: razbroc Date: Tue, 26 Mar 2024 15:41:04 +0200 Subject: [PATCH 16/19] chore: fix lint --- .../models/createPackageManager.ts | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/createPackage/models/createPackageManager.ts b/src/createPackage/models/createPackageManager.ts index fe977ed..06789b9 100644 --- a/src/createPackage/models/createPackageManager.ts +++ b/src/createPackage/models/createPackageManager.ts @@ -3,6 +3,7 @@ import { sep } from 'path'; import { Logger } from '@map-colonies/js-logger'; import { Tracer } from '@opentelemetry/api'; import { withSpanAsyncV4 } from '@map-colonies/telemetry'; +// import { TileRanger } from '@map-colonies/mc-utils'; import { Polygon, MultiPolygon, @@ -462,4 +463,49 @@ export class CreatePackageManager { } return combinedFootprint; } + + // TODO: remove and replace with generateTileGroups that is commented, when multiple tasks for GPKG target is possible + /* private generateTileGroups(polygon: Polygon | MultiPolygon, footprint: Polygon | MultiPolygon, zoom: number): ITileRange[] { + let intersaction: Feature | null; + + try { + intersaction = intersect(polygon, footprint); + if (intersaction === null) { + throw new BadRequestError( + `Requested ${JSON.stringify(polygon)} has no intersection with requested layer footprint: ${JSON.stringify(footprint)}` + ); + } + } catch (error) { + const message = `Error occurred while trying to generate tiles batches - intersaction error: ${JSON.stringify(error)}`; + this.logger.error({ + firstPolygon: polygon, + secondPolygon: footprint, + zoom: zoom, + message: message, + }); + throw new Error(message); + } + + try { + const tileRanger = new TileRanger(); + const tilesGroups: ITileRange[] = []; + + for (let i = 0; i <= zoom; i++) { + const zoomTilesGroups = tileRanger.encodeFootprint(intersaction, i); + for (const group of zoomTilesGroups) { + tilesGroups.push(group); + } + } + return tilesGroups; + } catch (error) { + const message = `Error occurred while trying to generate tiles batches - encodeFootprint error: ${JSON.stringify(error)}`; + this.logger.error({ + firstPolygon: polygon, + secondPolygon: footprint, + zoom: zoom, + message: message, + }); + throw new Error(message); + } + } */ } From dd076aea8c8c017926badf40e0bb200fa898caa1 Mon Sep 17 00:00:00 2001 From: razbroc Date: Sun, 31 Mar 2024 15:10:17 +0300 Subject: [PATCH 17/19] fix: fix pr --- src/clients/jobManagerWrapper.ts | 6 +----- src/common/interfaces.ts | 13 +------------ src/createPackage/models/createPackageManager.ts | 2 -- src/finalizationManager.ts | 14 ++------------ src/tasks/models/tasksManager.ts | 14 +++++++------- tests/mocks/data.ts | 5 +---- tests/mocks/data/mockJob.ts | 3 +-- .../models/createPackageModel.spec.ts | 3 +-- tests/unit/createPackage/models/tasksModel.spec.ts | 14 +++++++++++--- tests/unit/finalizationManager.spec.ts | 8 ++++---- 10 files changed, 29 insertions(+), 53 deletions(-) diff --git a/src/clients/jobManagerWrapper.ts b/src/clients/jobManagerWrapper.ts index f90085f..0737540 100644 --- a/src/clients/jobManagerWrapper.ts +++ b/src/clients/jobManagerWrapper.ts @@ -8,7 +8,6 @@ import { Tracer } from '@opentelemetry/api'; import { SERVICES } from '../common/constants'; import { CreateExportJobBody, - ExportVersion, ICreateExportJobResponse, IJobExportParameters, ITaskParameters, @@ -55,7 +54,6 @@ export class JobManagerWrapper extends JobManagerClient { roi: data.roi, callbacks: data.callbacks, crs: data.crs, - exportVersion: ExportVersion.ROI, fileNamesTemplates: data.fileNamesTemplates, relativeDirectoryPath: data.relativeDirectoryPath, gpkgEstimatedSize: data.gpkgEstimatedSize, @@ -175,9 +173,7 @@ export class JobManagerWrapper extends JobManagerClient { this.logger.debug({ ...queryParams }, `Getting jobs that match these parameters`); const jobs = await this.get('/jobs', queryParams as unknown as Record); const exportJobs = jobs?.filter((job) => { - if (job.parameters.exportVersion === ExportVersion.ROI) { - return job; - } + return job; }); return exportJobs; } diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index 5aa3e8f..abcb296 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -38,7 +38,6 @@ export interface ICallbackTargetExport { export interface IWorkerExportInput { dbId: string; relativeDirectoryPath: string; - exportVersion: ExportVersion; priority?: number; crs: string; version: string; @@ -65,7 +64,7 @@ export interface ICreateExportJobResponse { isDuplicated?: boolean; } -export interface ICallbackDataExportBase { +export interface ICallbackExportData { links?: ILinkDefinition; expirationTime?: Date; fileSize?: number; @@ -74,9 +73,6 @@ export interface ICallbackDataExportBase { errorReason?: string; description?: string; artifacts?: IArtifactDefinition[]; -} - -export interface ICallbackExportData extends ICallbackDataExportBase { roi: FeatureCollection; } @@ -111,7 +107,6 @@ export interface JobExportDuplicationParams { export interface IJobExportParameters { relativeDirectoryPath: string; crs: string; - exportVersion: ExportVersion; roi: FeatureCollection; callbacks?: ICallbackTargetExport[]; callbackParams?: ICallbackExportResponse; @@ -174,12 +169,6 @@ export interface IGeometryRecord extends IGeometryRecordBase { minZoomLevel: number; } -// todo - Temporary enum to define old\new api - will be removed after deleting getMap API -export enum ExportVersion { - GETMAP = 'GETMAP', - ROI = 'ROI', -} - export type TaskResponse = ITaskResponse; // new API based on multi resolution diff --git a/src/createPackage/models/createPackageManager.ts b/src/createPackage/models/createPackageManager.ts index 06789b9..4874844 100644 --- a/src/createPackage/models/createPackageManager.ts +++ b/src/createPackage/models/createPackageManager.ts @@ -23,7 +23,6 @@ import { BBox2d } from '@turf/helpers/dist/js/lib/geojson'; import { ProductType, TileOutputFormat } from '@map-colonies/mc-model-types'; import { feature, featureCollection } from '@turf/helpers'; import { - ExportVersion, ICallbackExportResponse, ICallbackTargetExport, IConfig, @@ -221,7 +220,6 @@ export class CreatePackageManager { fileNamesTemplates: fileNamesTemplates, relativeDirectoryPath: additionalIdentifiers, dbId, - exportVersion: ExportVersion.ROI, version: version, cswProductId: resourceId, crs: crs ?? DEFAULT_CRS, diff --git a/src/finalizationManager.ts b/src/finalizationManager.ts index c4cbe77..e4c61ca 100644 --- a/src/finalizationManager.ts +++ b/src/finalizationManager.ts @@ -8,14 +8,7 @@ import { withSpanAsyncV4 } from '@map-colonies/telemetry'; import { SERVICES } from './common/constants'; import { TasksManager } from './tasks/models/tasksManager'; import { QueueClient } from './clients/queueClient'; -import { - ICallbackDataExportBase, - ICallbackExportData, - ICallbackExportResponse, - ITaskFinalizeParameters, - JobExportResponse, - JobFinalizeResponse, -} from './common/interfaces'; +import { ICallbackExportData, ICallbackExportResponse, ITaskFinalizeParameters, JobExportResponse, JobFinalizeResponse } from './common/interfaces'; import { JobManagerWrapper } from './clients/jobManagerWrapper'; import { CallbackClient } from './clients/callbackClient'; @@ -119,10 +112,7 @@ export class FinalizationManager { return true; } - public async sendExportCallbacks( - job: JobExportResponse | JobFinalizeResponse, - callbackParams: ICallbackDataExportBase | ICallbackExportResponse - ): Promise { + public async sendExportCallbacks(job: JobExportResponse | JobFinalizeResponse, callbackParams: ICallbackExportResponse): Promise { try { this.logger.info({ jobId: job.id, callbacks: job.parameters.callbacks, msg: `Sending callback for job: ${job.id}` }); const targetCallbacks = job.parameters.callbacks; diff --git a/src/tasks/models/tasksManager.ts b/src/tasks/models/tasksManager.ts index 09ecf67..2954adb 100644 --- a/src/tasks/models/tasksManager.ts +++ b/src/tasks/models/tasksManager.ts @@ -11,7 +11,6 @@ import { JobManagerWrapper } from '../../clients/jobManagerWrapper'; import { CreateFinalizeTaskBody, IArtifactDefinition, - ICallbackDataExportBase, ICallbackExportData, ICallbackExportResponse, ICleanupData, @@ -109,10 +108,7 @@ export class TasksManager { return jobsStatus; } - public async sendExportCallbacks( - job: JobExportResponse | JobFinalizeResponse, - callbackParams: ICallbackDataExportBase | ICallbackExportResponse - ): Promise { + public async sendExportCallbacks(job: JobExportResponse | JobFinalizeResponse, callbackParams: ICallbackExportData): Promise { try { this.logger.info({ jobId: job.id, callbacks: job.parameters.callbacks, msg: `Sending callback for job: ${job.id}` }); const targetCallbacks = job.parameters.callbacks; @@ -216,7 +212,7 @@ export class TasksManager { job: JobExportResponse | JobFinalizeResponse, expirationDate: Date, errorReason?: string - ): Promise { + ): Promise { let links: ILinkDefinition = { ...job.parameters.fileNamesTemplates }; // default file names in case of failure this.logger.info({ jobId: job.id, msg: `generate callback body for job: ${job.id}` }); @@ -263,7 +259,7 @@ export class TasksManager { }, ]; - const callbackParams: ICallbackDataExportBase = { + const callbackParams: ICallbackExportData = { links, expirationTime: expirationDate, fileSize, @@ -272,6 +268,10 @@ export class TasksManager { errorReason, description: job.description, artifacts, + roi: { + type: 'FeatureCollection', + features: [], + }, }; this.logger.info({ links: callbackParams.links, diff --git a/tests/mocks/data.ts b/tests/mocks/data.ts index 1c938cc..6567d11 100644 --- a/tests/mocks/data.ts +++ b/tests/mocks/data.ts @@ -2,7 +2,7 @@ import { LayerMetadata, ProductType, RecordType, TileOutputFormat } from '@map-colonies/mc-model-types'; import { IJobResponse, OperationStatus } from '@map-colonies/mc-priority-queue'; import { FeatureCollection } from '@turf/helpers'; -import { ExportVersion, ICreatePackageRoi, IJobExportParameters, ITaskParameters, IWorkerExportInput } from '../../src/common/interfaces'; +import { ICreatePackageRoi, IJobExportParameters, ITaskParameters, IWorkerExportInput } from '../../src/common/interfaces'; const layerMetadata: LayerMetadata = { type: 'RECORD_RASTER', @@ -411,7 +411,6 @@ const workerExportInput: IWorkerExportInput = { }, relativeDirectoryPath: '1a26c1661df10eee54f9727fcdb8b71d', dbId: '0c3e455f-4aeb-4258-982d-f7773469a92d', - exportVersion: ExportVersion.ROI, version: '1.0', cswProductId: 'string', crs: 'EPSG:4326', @@ -499,7 +498,6 @@ const completedExportJob: IJobResponse = roi: fc1, }, ], - exportVersion: ExportVersion.ROI, callbackParams: { roi: fc1, links: { @@ -576,7 +574,6 @@ const inProgressExportJob: IJobResponse = crs: 'EPSG:4326', roi: fc1, callbacks: [{ url: 'http://localhost:6969', roi: fc1 }], - exportVersion: ExportVersion.ROI, gpkgEstimatedSize: 187500, fileNamesTemplates: { dataURI: 'Orthophoto_testArea_1_0_2023_03_01T15_09_50_924Z.gpkg', diff --git a/tests/mocks/data/mockJob.ts b/tests/mocks/data/mockJob.ts index 8fdecf1..f75ec82 100644 --- a/tests/mocks/data/mockJob.ts +++ b/tests/mocks/data/mockJob.ts @@ -1,6 +1,6 @@ /* eslint-disable @typescript-eslint/no-magic-numbers */ import { OperationStatus } from '@map-colonies/mc-priority-queue'; -import { ExportVersion, JobFinalizeResponse } from '../../../src/common/interfaces'; +import { JobFinalizeResponse } from '../../../src/common/interfaces'; import { fc1 } from '../data'; export const mockCompletedJob: JobFinalizeResponse = { @@ -23,7 +23,6 @@ export const mockCompletedJob: JobFinalizeResponse = { roi: fc1, }, ], - exportVersion: ExportVersion.ROI, gpkgEstimatedSize: 187500, fileNamesTemplates: { dataURI: 'Orthophoto_testArea_1_0_2023_02_28T15_09_50_924Z.gpkg', diff --git a/tests/unit/createPackage/models/createPackageModel.spec.ts b/tests/unit/createPackage/models/createPackageModel.spec.ts index 924e126..2ee19c4 100644 --- a/tests/unit/createPackage/models/createPackageModel.spec.ts +++ b/tests/unit/createPackage/models/createPackageModel.spec.ts @@ -8,7 +8,7 @@ import { OperationStatus } from '@map-colonies/mc-priority-queue'; import { tracerMock } from '../../../mocks/clients/tracer'; import { jobManagerWrapperMock, updateJobMock, findExportJobMock, createExportMock } from '../../../mocks/clients/jobManagerWrapper'; import { catalogManagerMock, findLayerMock } from '../../../mocks/clients/catalogManagerClient'; -import { ExportVersion, ICreateExportJobResponse, ICreatePackageRoi, JobExportDuplicationParams } from '../../../../src/common/interfaces'; +import { ICreateExportJobResponse, ICreatePackageRoi, JobExportDuplicationParams } from '../../../../src/common/interfaces'; import { CreatePackageManager } from '../../../../src/createPackage/models/createPackageManager'; import { completedExportJob, @@ -368,7 +368,6 @@ describe('CreatePackageManager', () => { const jobUpdateParams = { parameters: { crs: 'EPSG:4326', - exportVersion: ExportVersion.ROI, callbacks: [ { url: 'http://localhost:6969', roi: fc1 }, { url: 'http://new-added-callback-url.com', roi: fc1 }, diff --git a/tests/unit/createPackage/models/tasksModel.spec.ts b/tests/unit/createPackage/models/tasksModel.spec.ts index 1584a71..5e27e13 100644 --- a/tests/unit/createPackage/models/tasksModel.spec.ts +++ b/tests/unit/createPackage/models/tasksModel.spec.ts @@ -4,7 +4,7 @@ import { NotFoundError } from '@map-colonies/error-types'; import { ITaskStatusResponse, TasksManager } from '../../../../src/tasks/models/tasksManager'; import { CreateFinalizeTaskBody, - ICallbackDataExportBase, + ICallbackExportData, ICallbackExportResponse, ICallbackTargetExport, ITaskParameters, @@ -181,7 +181,7 @@ describe('TasksManager', () => { it('should send callback data with the expected params for success jobs to all clients', async () => { sendMock.mockResolvedValue(200); const expirationTime = new Date(); - const callbackData: ICallbackDataExportBase = { + const callbackData: ICallbackExportData = { links: { dataURI: 'http://download-service/downloads/test${sep}test.gpkg', metadataURI: 'http://download-service/downloads/test${sep}test.json', @@ -191,6 +191,10 @@ describe('TasksManager', () => { expirationTime: expirationTime, fileSize: 2000, errorReason: undefined, + roi: { + type: 'FeatureCollection', + features: [], + }, }; const expectedCallbacksData = mockCompletedJob.parameters.callbacks as unknown as ICallbackTargetExport[]; @@ -208,7 +212,7 @@ describe('TasksManager', () => { it('should return callback data even if callback response got rejected', async () => { sendMock.mockRejectedValue({}); const expirationTime = new Date(); - const callbackData: ICallbackDataExportBase = { + const callbackData: ICallbackExportData = { links: { dataURI: 'http://download-service/downloads/test${sep}test.gpkg', metadataURI: 'http://download-service/downloads/test${sep}test.json', @@ -218,6 +222,10 @@ describe('TasksManager', () => { expirationTime: expirationTime, fileSize: 2000, errorReason: undefined, + roi: { + type: 'FeatureCollection', + features: [], + }, }; const action = async () => tasksManager.sendExportCallbacks(mockCompletedJob, callbackData); await expect(action()).resolves.not.toThrow(); diff --git a/tests/unit/finalizationManager.spec.ts b/tests/unit/finalizationManager.spec.ts index 07ae456..bdfacf0 100644 --- a/tests/unit/finalizationManager.spec.ts +++ b/tests/unit/finalizationManager.spec.ts @@ -15,7 +15,7 @@ import { getExportJobsByTaskStatusMock, getFinalizeJobByIdMock, } from '../mocks/clients/taskManager'; -import { ICallbackDataExportBase, IExportJobStatusResponse, IJobExportParameters, ITaskFinalizeParameters } from '../../src/common/interfaces'; +import { ICallbackExportData, IExportJobStatusResponse, IJobExportParameters, ITaskFinalizeParameters } from '../../src/common/interfaces'; import { completedExportJob, inProgressExportJob } from '../mocks/data'; import { configMock, registerDefaultConfig } from '../mocks/config'; import { jobManagerWrapperMock, updateJobMock, deleteTaskByIdMock } from '../mocks/clients/jobManagerWrapper'; @@ -117,7 +117,7 @@ describe('FinalizationManager', () => { finalizeGPKGSuccessMock.mockReturnValue(expectedUpdateParams); await finalizationManager.jobFinalizePoll(); - const createdCallbackParam: ICallbackDataExportBase = sendCallbacksSpy.mock.calls[0][1] as ICallbackDataExportBase; + const createdCallbackParam: ICallbackExportData = sendCallbacksSpy.mock.calls[0][1] as ICallbackExportData; expect(sendCallbacksSpy).toHaveBeenCalledTimes(1); expect(createdCallbackParam).toStrictEqual(expectedCallbackParamData); expect(dequeueMock).toHaveBeenCalledTimes(1); @@ -173,7 +173,7 @@ describe('FinalizationManager', () => { finalizeGPKGFailureMock.mockResolvedValue(expectedUpdateParams); await finalizationManager.jobFinalizePoll(); - const createdCallbackParam: ICallbackDataExportBase = sendCallbacksSpy.mock.calls[0][1] as ICallbackDataExportBase; + const createdCallbackParam: ICallbackExportData = sendCallbacksSpy.mock.calls[0][1] as ICallbackExportData; expect(sendCallbacksSpy).toHaveBeenCalledTimes(1); expect(createdCallbackParam).toStrictEqual(expectedCallbackParamData); expect(dequeueMock).toHaveBeenCalledTimes(1); @@ -249,7 +249,7 @@ describe('FinalizationManager', () => { await finalizationManager.jobFinalizePoll(); - const createdCallbackParam: ICallbackDataExportBase = sendCallbacksSpy.mock.calls[0][1] as ICallbackDataExportBase; + const createdCallbackParam: ICallbackExportData = sendCallbacksSpy.mock.calls[0][1] as ICallbackExportData; expect(sendCallbacksSpy).toHaveBeenCalledTimes(1); expect(createdCallbackParam).toStrictEqual(expectedCallbackParamData); expect(dequeueMock).toHaveBeenCalledTimes(1); From 4b42a9e2ebfdacfd9e1d594ab2e21271d2d6c4bf Mon Sep 17 00:00:00 2001 From: razbroc <77406876+razbroc@users.noreply.github.com> Date: Wed, 15 May 2024 09:40:21 +0300 Subject: [PATCH 18/19] fix: pr comment --- src/createPackage/models/createPackageManager.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/createPackage/models/createPackageManager.ts b/src/createPackage/models/createPackageManager.ts index 4874844..903536e 100644 --- a/src/createPackage/models/createPackageManager.ts +++ b/src/createPackage/models/createPackageManager.ts @@ -3,7 +3,6 @@ import { sep } from 'path'; import { Logger } from '@map-colonies/js-logger'; import { Tracer } from '@opentelemetry/api'; import { withSpanAsyncV4 } from '@map-colonies/telemetry'; -// import { TileRanger } from '@map-colonies/mc-utils'; import { Polygon, MultiPolygon, From 840b3dcc70db13d8122770e90c8cb1e3cbf3944b Mon Sep 17 00:00:00 2001 From: razbroc Date: Thu, 23 May 2024 18:48:54 +0300 Subject: [PATCH 19/19] chore: upgrade telemetry pakcage --- package-lock.json | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/package-lock.json b/package-lock.json index 03aa374..4fb9c7e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,7 +20,7 @@ "@map-colonies/mc-utils": "^2.0.0", "@map-colonies/openapi-express-viewer": "^3.0.0", "@map-colonies/read-pkg": "0.0.1", - "@map-colonies/telemetry": "5.3.1", + "@map-colonies/telemetry": "6.0.0", "@mapbox/geojsonhint": "3.1.0", "@opentelemetry/api": "^1.4.1", "@opentelemetry/api-metrics": "0.23.0", @@ -3526,9 +3526,9 @@ } }, "node_modules/@map-colonies/telemetry": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/@map-colonies/telemetry/-/telemetry-5.3.1.tgz", - "integrity": "sha512-LVKUKbMB6xlNzIIIYMv5Dlj1CtUDySnyL4yIbkUjKVn4N4tQYfu/Sbvs2LzKC0p2YZPsZr8/MvjSAfC4hEPbNg==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/@map-colonies/telemetry/-/telemetry-6.0.0.tgz", + "integrity": "sha512-o1wNVoZkJRczQ7iVANHTN38CSHWFCKEXTEbMgV9GaLjJ9wu4weqW/VADtjrNXc20cB54UxIeDNfYEEgTzAxM6w==", "dependencies": { "@map-colonies/read-pkg": "^0.0.1", "@opentelemetry/api": "^1.7.0", @@ -10355,9 +10355,9 @@ "dev": true }, "node_modules/cookie": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", - "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==", + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz", + "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==", "engines": { "node": ">= 0.6" } @@ -12337,16 +12337,16 @@ } }, "node_modules/express": { - "version": "4.18.3", - "resolved": "https://registry.npmjs.org/express/-/express-4.18.3.tgz", - "integrity": "sha512-6VyCijWQ+9O7WuVMTRBTl+cjNNIzD5cY5mQ1WM8r/LEkI2u8EYpOotESNwzNlyCn3g+dmjKYI6BmNneSr/FSRw==", + "version": "4.19.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz", + "integrity": "sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==", "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", "body-parser": "1.20.2", "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.5.0", + "cookie": "0.6.0", "cookie-signature": "1.0.6", "debug": "2.6.9", "depd": "2.0.0", @@ -12695,9 +12695,9 @@ "dev": true }, "node_modules/follow-redirects": { - "version": "1.15.5", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.5.tgz", - "integrity": "sha512-vSFWUON1B+yAw1VN4xMfxgn5fTUiaOzAJCKBwIIgT/+7CuGy9+r+5gITvP62j3RmaD5Ph65UaERdOSRGUzZtgw==", + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", "funding": [ { "type": "individual",