diff --git a/.vscode/settings.json b/.vscode/settings.json index 1e5da04..995b79f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,4 +1,5 @@ { "typescript.tsdk": "node_modules/typescript/lib", - "files.eol": "\n" + "files.eol": "\n", + "cSpell.words": ["geoserver", "Mapproxy"] } diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index aba858a..3cef8c5 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -66,6 +66,17 @@ }, "dequeueIntervalMs": "DEQUEUE_INTERVAL_MS" }, + "servicesUrl": { + "mapproxyApi": "MAPPROXY_API_URL", + "geoserverApi": "GEOSERVER_API_URL", + "catalogManager": "CATALOG_MANAGER_URL", + "mapproxyDns": "MAPPROXY_DNS", + "polygonPartManager": "POLYGON_PART_MANAGER_URL" + }, + "geoserver": { + "workspace": "GEOSERVER_WORKSPACE", + "datastore": "GEOSERVER_DATASTORE" + }, "ingestion": { "pollingTasks": { "init": "INGESTION_POLLING_INIT_TASK", diff --git a/config/default.json b/config/default.json index ce7b58d..b90b274 100644 --- a/config/default.json +++ b/config/default.json @@ -37,11 +37,23 @@ }, "disableHttpClientLogs": true, "tilesStorageProvider": "FS", + "linkTemplatesPath": "config/linkTemplates.template", + "servicesUrl": { + "mapproxyApi": "http://localhost:8083", + "geoserverApi": "http://localhost:8084", + "catalogManager": "http://localhost:8085", + "mapproxyDns": "http://localhost:8086", + "polygonPartManager": "http://localhost:8087" + }, + "geoserver": { + "workspace": "polygonParts", + "dataStore": "polygonParts" + }, "jobManagement": { "config": { "jobManagerBaseUrl": "http://localhost:8081", "heartbeat": { - "baseUrl": "http://localhost:8083", + "baseUrl": "http://localhost:8082", "intervalMs": 3000 }, "dequeueIntervalMs": 3000 diff --git a/config/linkTemplates.template b/config/linkTemplates.template new file mode 100644 index 0000000..1fb0555 --- /dev/null +++ b/config/linkTemplates.template @@ -0,0 +1,38 @@ +[ + { + "name": "{{layerName}}", + "description": "", + "protocol": "WMS", + "url": "{{serverUrl}}/service?REQUEST=GetCapabilities" + }, + { + "name": "{{layerName}}", + "description": "", + "protocol": "WMS_BASE", + "url": "{{serverUrl}}/wms" + }, + { + "name": "{{layerName}}", + "description": "", + "protocol": "WMTS", + "url": "{{serverUrl}}/wmts/1.0.0/WMTSCapabilities.xml" + }, + { + "name": "{{layerName}}", + "description": "", + "protocol": "WMTS_KVP", + "url": "{{serverUrl}}/service?REQUEST=GetCapabilities&SERVICE=WMTS" + }, + { + "name": "{{layerName}}", + "description": "", + "protocol": "WMTS_BASE", + "url": "{{serverUrl}}/wmts" + }, + { + "name": "{{layerName}}", + "description": "", + "protocol": "WFS", + "url": "{{serverUrl}}/wfs?request=GetCapabilities" + } +] diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index 2e8e99f..6116622 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -45,4 +45,11 @@ data: TILES_MERGING_TILE_BATCH_SIZE: {{ .Values.env.jobManagement.ingestion.tasks.tilesMerging.tileBatchSize | quote }} TILES_MERGING_TASK_BATCH_SIZE: {{ .Values.env.jobManagement.ingestion.tasks.tilesMerging.taskBatchSize | quote }} TILES_MERGING_USE_NEW_TARGET_FLAG: {{ .Values.env.jobManagement.ingestion.tasks.tilesMerging.useNewTargetFlagInUpdate | quote }} + MAPPROXY_API_URL: {{ $serviceUrls.mapproxyApi | quote }} + GEOSERVER_API_URL: {{ $serviceUrls.geoserverApi | quote }} + CATALOG_MANAGER_URL: {{ $serviceUrls.catalogManager | quote }} + MAPPROXY_DNS: {{ $serviceUrls.mapServerPublicDNS | quote }} + POLYGON_PART_MANAGER_URL: {{ $serviceUrls.polygonPartManager | quote }} + GEOSERVER_WORKSPACE: {{ .Values.global.geoserver.workspace | quote }} + GEOSERVER_DATASTORE: {{ .Values.global.geoserver.dataStore | quote }} {{- end }} diff --git a/helm/values.yaml b/helm/values.yaml index a3ad8d8..0701661 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -6,6 +6,14 @@ global: serviceUrls: jobManager: "" heartbeatManager: "" + catalogManager: "" + mapServerPublicDNS: "" + mapproxyApi: "" + geoserverApi: "" + polygonPartManager: "" + geoserver: + workspace: '' + dataStore: '' ca: secretName: '' path: '/usr/local/share/ca-certificates' @@ -69,6 +77,11 @@ image: serviceUrls: jobManager: "" heartbeatManager: "" + catalogManager: "" + mapServerPublicDNS: "" + mapproxyApi: "" + geoserverApi: "" + polygonPartManager: "" tracing: enabled: false @@ -92,6 +105,8 @@ metrics: - 50 - 250 - 500 + prometheus: + scrape: false env: port: 8080 diff --git a/src/common/constants.ts b/src/common/constants.ts index 29c04ec..b9beece 100644 --- a/src/common/constants.ts +++ b/src/common/constants.ts @@ -1,3 +1,4 @@ +import { IngestionNewFinalizeTaskParams } from '@map-colonies/mc-model-types'; import { readPackageJsonSync } from '@map-colonies/read-pkg'; export const SERVICE_NAME = readPackageJsonSync().name ?? 'unknown_service'; @@ -16,4 +17,26 @@ export const SERVICES = { TILE_RANGER: Symbol('TileRanger'), } satisfies Record; +export const TilesStorageProvider = { + FS: 'FS', + S3: 'S3', +} as const; + +export type TilesStorageProvider = (typeof TilesStorageProvider)[keyof typeof TilesStorageProvider]; + +export const PublishedLayerCacheType = { + FS: 'file', + S3: 's3', + REDIS: 'redis', +} as const; + +export type PublishedLayerCacheType = (typeof PublishedLayerCacheType)[keyof typeof PublishedLayerCacheType]; + +export const storageProviderToCacheTypeMap = new Map([ + [TilesStorageProvider.FS, PublishedLayerCacheType.FS], + [TilesStorageProvider.S3, PublishedLayerCacheType.S3], +]); + +export type FinalizeSteps = keyof IngestionNewFinalizeTaskParams; + /* eslint-enable @typescript-eslint/naming-convention */ diff --git a/src/common/errors.ts b/src/common/errors.ts index c41b19b..f70926d 100644 --- a/src/common/errors.ts +++ b/src/common/errors.ts @@ -18,3 +18,18 @@ export class UnsupportedTransparencyError extends Error { this.name = UnsupportedTransparencyError.name; } } + +export class UnsupportedStorageProviderError extends Error { + public constructor(storageProvider: string) { + super(`Unsupported storage provider: ${storageProvider}`); + this.name = UnsupportedStorageProviderError.name; + } +} + +export class PublishLayerError extends Error { + public constructor(publishingClient: string, layerName: string, err: Error) { + super(`Failed to publish ${layerName} layer to ${publishingClient} client: ${err.message}`); + this.name = PublishLayerError.name; + this.stack = err.stack; + } +} diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index a998444..e886a37 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -1,8 +1,10 @@ import { IJobResponse, ITaskResponse } from '@map-colonies/mc-priority-queue'; -import { InputFiles, NewRasterLayerMetadata, PolygonPart, TileOutputFormat } from '@map-colonies/mc-model-types'; +import { GeoJSON } from 'geojson'; +import { InputFiles, NewRasterLayerMetadata, PolygonPart, TileOutputFormat, LayerData } from '@map-colonies/mc-model-types'; import { TilesMimeFormat } from '@map-colonies/types'; import { BBox, Polygon } from 'geojson'; import { Footprint, ITileRange } from '@map-colonies/mc-utils'; +import { PublishedLayerCacheType } from './constants'; //#region config interfaces export interface IConfig { @@ -49,11 +51,12 @@ export interface IngestionConfig { } //#endregion config +//#region job/task interfaces export interface IJobHandler { // eslint-disable-next-line @typescript-eslint/no-explicit-any handleJobInit: (job: IJobResponse, taskId: string) => Promise; // eslint-disable-next-line @typescript-eslint/no-explicit-any - handleJobFinalize: (job: IJobResponse, taskId: string) => Promise; + handleJobFinalize: (job: IJobResponse, task: ITaskResponse) => Promise; } export interface JobAndTaskResponse { @@ -72,23 +75,22 @@ export interface ExtendedRasterLayerMetadata extends NewRasterLayerMetadata { grid: Grid; } -export interface MergeTilesTaskParams { - inputFiles: InputFiles; - taskMetadata: MergeTilesMetadata; - partsData: PolygonPart[]; -} +export type ExtendedNewRasterLayer = { metadata: ExtendedRasterLayerMetadata } & LayerData; -export interface MergeTilesMetadata { - layerRelativePath: string; - tileOutputFormat: TileOutputFormat; - isNewTarget: boolean; - grid: Grid; -} +//#endregion job/task + +//#region merge task export enum Grid { TWO_ON_ONE = '2x1', } -//#region task + +export interface IBBox { + minX: number; + minY: number; + maxX: number; + maxY: number; +} export interface IPartSourceContext { fileName: string; tilesPath: string; @@ -129,11 +131,50 @@ export interface IntersectionState { accumulatedIntersection: Footprint | null; currentIntersection: Footprint | null; } + +export interface MergeTilesTaskParams { + inputFiles: InputFiles; + taskMetadata: MergeTilesMetadata; + partsData: PolygonPart[]; +} + +export interface MergeTilesMetadata { + layerRelativePath: string; + tileOutputFormat: TileOutputFormat; + isNewTarget: boolean; + grid: Grid; +} //#endregion task -export interface IBBox { - minX: number; - minY: number; - maxX: number; - maxY: number; +//#region mapproxyApi +export interface IPublishMapLayerRequest { + name: string; + tilesPath: string; + cacheType: PublishedLayerCacheType; + format: TileOutputFormat; } +//#endregion mapproxyApi + +//#region geoserverApi +export interface IInsertGeoserverRequest { + nativeName: string; +} +//#endregion geoserverApi + +//#region catalogClient + +export interface PartAggregatedData { + imagingTimeBeginUTC: Date; + imagingTimeEndUTC: Date; + minHorizontalAccuracyCE90: number; + maxHorizontalAccuracyCE90: number; + sensors: string[]; + maxResolutionDeg: number; + minResolutionDeg: number; + maxResolutionMeter: number; + minResolutionMeter: number; + footprint: GeoJSON; + productBoundingBox: string; +} + +//#endregion catalogClient diff --git a/src/httpClients/catalogClient.ts b/src/httpClients/catalogClient.ts new file mode 100644 index 0000000..26fff49 --- /dev/null +++ b/src/httpClients/catalogClient.ts @@ -0,0 +1,93 @@ +import { IConfig } from 'config'; +import { Logger } from '@map-colonies/js-logger'; +import { IJobResponse } from '@map-colonies/mc-priority-queue'; +import { HttpClient, IHttpRetryConfig } from '@map-colonies/mc-utils'; +import { IRasterCatalogUpsertRequestBody, LayerMetadata, Link, RecordType } from '@map-colonies/mc-model-types'; +import { inject, injectable } from 'tsyringe'; +import { SERVICES } from '../common/constants'; +import { ExtendedNewRasterLayer } from '../common/interfaces'; +import { PublishLayerError } from '../common/errors'; +import { ILinkBuilderData, LinkBuilder } from '../utils/linkBuilder'; +import { PolygonPartMangerClient } from './polygonPartMangerClient'; + +@injectable() +export class CatalogClient extends HttpClient { + private readonly mapproxyDns: string; + public constructor( + @inject(SERVICES.CONFIG) private readonly config: IConfig, + @inject(SERVICES.LOGGER) protected readonly logger: Logger, + private readonly linkBuilder: LinkBuilder, + private readonly polygonPartMangerClient: PolygonPartMangerClient + ) { + const serviceName = 'RasterCatalogManager'; + const baseUrl = config.get('servicesUrl.catalogManager'); + const httpRetryConfig = config.get('httpRetry'); + const disableHttpClientLogs = config.get('disableHttpClientLogs'); + super(logger, baseUrl, serviceName, httpRetryConfig, disableHttpClientLogs); + this.mapproxyDns = config.get('servicesUrl.mapproxyDns'); + } + + public async publish(job: IJobResponse, layerName: string): Promise { + try { + const url = '/records'; + const publishReq: IRasterCatalogUpsertRequestBody = this.createPublishReqBody(job, layerName); + await this.post(url, publishReq); + } catch (err) { + if (err instanceof Error) { + throw new PublishLayerError(this.targetService, layerName, err); + } + } + } + + private createPublishReqBody(job: IJobResponse, layerName: string): IRasterCatalogUpsertRequestBody { + const metadata = this.mapToCatalogRecordMetadata(job); + const links = this.buildLinks(layerName); + + return { + metadata, + links, + }; + } + + private mapToCatalogRecordMetadata(job: IJobResponse): LayerMetadata { + const { parameters, version } = job; + const { partData, metadata } = parameters; + + const aggregatedPartData = this.polygonPartMangerClient.getAggregatedPartData(partData); + + return { + id: metadata.catalogId, + type: RecordType.RECORD_RASTER, + classification: metadata.classification, + productName: metadata.productName, + description: metadata.description, + srs: metadata.srs, + srsName: metadata.srsName, + producerName: metadata.producerName, + region: metadata.region, + productId: metadata.productId, + productType: metadata.productType, + productSubType: metadata.productSubType, + displayPath: metadata.displayPath, + transparency: metadata.transparency, + scale: metadata.scale, + tileMimeFormat: metadata.tileMimeType, + tileOutputFormat: metadata.tileOutputFormat, + productVersion: version, + updateDateUTC: undefined, + creationDateUTC: undefined, + rms: undefined, + ingestionDate: new Date(), + ...aggregatedPartData, + }; + } + + private buildLinks(layerName: string): Link[] { + const linkBuildData: ILinkBuilderData = { + layerName, + serverUrl: this.mapproxyDns, + }; + + return this.linkBuilder.createLinks(linkBuildData); + } +} diff --git a/src/httpClients/geoserverClient.ts b/src/httpClients/geoserverClient.ts new file mode 100644 index 0000000..3cc4aa8 --- /dev/null +++ b/src/httpClients/geoserverClient.ts @@ -0,0 +1,37 @@ +import { IConfig } from 'config'; +import { Logger } from '@map-colonies/js-logger'; +import { HttpClient, IHttpRetryConfig } from '@map-colonies/mc-utils'; +import { inject, injectable } from 'tsyringe'; +import { SERVICES } from '../common/constants'; +import { IInsertGeoserverRequest } from '../common/interfaces'; +import { PublishLayerError } from '../common/errors'; + +@injectable() +export class GeoserverClient extends HttpClient { + private readonly workspace: string; + private readonly dataStore: string; + public constructor(@inject(SERVICES.CONFIG) private readonly config: IConfig, @inject(SERVICES.LOGGER) protected readonly logger: Logger) { + const serviceName = 'GeoserverApi'; + const baseUrl = config.get('servicesUrl.geoserverApi'); + const httpRetryConfig = config.get('httpRetry'); + const disableHttpClientLogs = config.get('disableHttpClientLogs'); + super(logger, baseUrl, serviceName, httpRetryConfig, disableHttpClientLogs); + this.workspace = config.get('geoserver.workspace'); + this.dataStore = config.get('geoserver.dataStore'); + } + + public async publish(layerName: string): Promise { + try { + const url = `/featureTypes/${this.workspace}/${this.dataStore}`; + const publishReq: IInsertGeoserverRequest = { + nativeName: layerName, + }; + + await this.post(url, publishReq); + } catch (err) { + if (err instanceof Error) { + throw new PublishLayerError(this.targetService, layerName, err); + } + } + } +} diff --git a/src/httpClients/mapproxyClient.ts b/src/httpClients/mapproxyClient.ts new file mode 100644 index 0000000..0e5a09f --- /dev/null +++ b/src/httpClients/mapproxyClient.ts @@ -0,0 +1,42 @@ +import { IConfig } from 'config'; +import { Logger } from '@map-colonies/js-logger'; +import { TileOutputFormat } from '@map-colonies/mc-model-types'; +import { HttpClient, IHttpRetryConfig } from '@map-colonies/mc-utils'; +import { inject, injectable } from 'tsyringe'; +import { SERVICES, storageProviderToCacheTypeMap, TilesStorageProvider } from '../common/constants'; +import { IPublishMapLayerRequest } from '../common/interfaces'; +import { PublishLayerError, UnsupportedStorageProviderError } from '../common/errors'; + +@injectable() +export class MapproxyApiClient extends HttpClient { + private readonly tilesStorageProvider: TilesStorageProvider; + public constructor(@inject(SERVICES.CONFIG) private readonly config: IConfig, @inject(SERVICES.LOGGER) protected readonly logger: Logger) { + const serviceName = 'MapproxyApi'; + const baseUrl = config.get('servicesUrl.mapproxyApi'); + const httpRetryConfig = config.get('httpRetry'); + const disableHttpClientLogs = config.get('disableHttpClientLogs'); + super(logger, baseUrl, serviceName, httpRetryConfig, disableHttpClientLogs); + this.tilesStorageProvider = config.get('tilesStorageProvider'); + } + + public async publish(layerName: string, tilesPath: string, format: TileOutputFormat): Promise { + const cacheType = storageProviderToCacheTypeMap.get(this.tilesStorageProvider); + if (!cacheType) { + throw new UnsupportedStorageProviderError(this.tilesStorageProvider); + } + try { + const publishReq: IPublishMapLayerRequest = { + name: layerName, + tilesPath, + format, + cacheType, + }; + const url = '/layer'; + await this.post(url, publishReq); + } catch (err) { + if (err instanceof Error) { + throw new PublishLayerError(this.targetService, layerName, err); + } + } + } +} diff --git a/src/httpClients/polygonPartMangerClient.ts b/src/httpClients/polygonPartMangerClient.ts new file mode 100644 index 0000000..79974e4 --- /dev/null +++ b/src/httpClients/polygonPartMangerClient.ts @@ -0,0 +1,36 @@ +import { IConfig } from 'config'; +import { Logger } from '@map-colonies/js-logger'; +import { PolygonPart } from '@map-colonies/mc-model-types'; +import { HttpClient, IHttpRetryConfig } from '@map-colonies/mc-utils'; +import { inject, injectable } from 'tsyringe'; +import { SERVICES } from '../common/constants'; +import { PartAggregatedData } from '../common/interfaces'; + +@injectable() +export class PolygonPartMangerClient extends HttpClient { + public constructor(@inject(SERVICES.CONFIG) private readonly config: IConfig, @inject(SERVICES.LOGGER) protected readonly logger: Logger) { + const serviceName = 'PolygonPartManger'; + const baseUrl = config.get('servicesUrl.polygonPartManager'); + const httpRetryConfig = config.get('httpRetry'); + const disableHttpClientLogs = config.get('disableHttpClientLogs'); + super(logger, baseUrl, serviceName, httpRetryConfig, disableHttpClientLogs); + } + + public getAggregatedPartData(partsData: PolygonPart[]): PartAggregatedData { + //later we should send request to PolygonPartsManager to get aggregated data + + return { + imagingTimeBeginUTC: partsData[0].imagingTimeBeginUTC, + imagingTimeEndUTC: partsData[0].imagingTimeEndUTC, + minHorizontalAccuracyCE90: partsData[0].horizontalAccuracyCE90, + maxHorizontalAccuracyCE90: partsData[0].horizontalAccuracyCE90, + sensors: partsData[0].sensors, + maxResolutionDeg: partsData[0].resolutionDegree, + minResolutionDeg: partsData[0].resolutionDegree, + maxResolutionMeter: partsData[0].resolutionMeter, + minResolutionMeter: partsData[0].resolutionMeter, + footprint: partsData[0].footprint, + productBoundingBox: '', + }; + } +} diff --git a/src/job/models/jobProcessor.ts b/src/job/models/jobProcessor.ts index 268f6bb..5b8a607 100644 --- a/src/job/models/jobProcessor.ts +++ b/src/job/models/jobProcessor.ts @@ -72,7 +72,7 @@ export class JobProcessor { await jobHandler.handleJobInit(job, task.id); break; case taskTypes.finalize: - await jobHandler.handleJobFinalize(job, task.id); + await jobHandler.handleJobFinalize(job, task); break; } } catch (error) { diff --git a/src/job/models/newJobHandler.ts b/src/job/models/newJobHandler.ts index 3b51527..3d98a8d 100644 --- a/src/job/models/newJobHandler.ts +++ b/src/job/models/newJobHandler.ts @@ -1,21 +1,27 @@ import { randomUUID } from 'crypto'; import { inject, injectable } from 'tsyringe'; import { Logger } from '@map-colonies/js-logger'; -import { IJobResponse } from '@map-colonies/mc-priority-queue'; +import { IJobResponse, ITaskResponse, OperationStatus } from '@map-colonies/mc-priority-queue'; import { TilesMimeFormat, lookup as mimeLookup } from '@map-colonies/types'; -import { NewRasterLayer, NewRasterLayerMetadata } from '@map-colonies/mc-model-types'; +import { IngestionNewFinalizeTaskParams, NewRasterLayer, NewRasterLayerMetadata } from '@map-colonies/mc-model-types'; import { TaskHandler as QueueClient } from '@map-colonies/mc-priority-queue'; -import { Grid, IJobHandler, MergeTilesTaskParams, ExtendedRasterLayerMetadata } from '../../common/interfaces'; -import { SERVICES } from '../../common/constants'; +import { Grid, IJobHandler, MergeTilesTaskParams, ExtendedRasterLayerMetadata, ExtendedNewRasterLayer } from '../../common/interfaces'; +import { FinalizeSteps, SERVICES } from '../../common/constants'; import { getTileOutputFormat } from '../../utils/imageFormatUtil'; import { TileMergeTaskManager } from '../../task/models/tileMergeTaskManager'; +import { MapproxyApiClient } from '../../httpClients/mapproxyClient'; +import { GeoserverClient } from '../../httpClients/geoserverClient'; +import { CatalogClient } from '../../httpClients/catalogClient'; @injectable() export class NewJobHandler implements IJobHandler { public constructor( @inject(SERVICES.LOGGER) private readonly logger: Logger, @inject(TileMergeTaskManager) private readonly taskBuilder: TileMergeTaskManager, - @inject(SERVICES.QUEUE_CLIENT) private readonly queueClient: QueueClient + @inject(SERVICES.QUEUE_CLIENT) private readonly queueClient: QueueClient, + @inject(MapproxyApiClient) private readonly mapproxyClient: MapproxyApiClient, + @inject(GeoserverClient) private readonly geoserverClient: GeoserverClient, + @inject(CatalogClient) private readonly catalogClient: CatalogClient ) {} public async handleJobInit(job: IJobResponse, taskId: string): Promise { @@ -58,10 +64,51 @@ export class NewJobHandler implements IJobHandler { } } - public async handleJobFinalize(job: IJobResponse, taskId: string): Promise { - const logger = this.logger.child({ jobId: job.id, taskId }); - logger.info({ msg: `handling ${job.type} job with "finalize"` }); - await Promise.reject('not implemented'); + public async handleJobFinalize( + job: IJobResponse, + task: ITaskResponse + ): Promise { + const logger = this.logger.child({ jobId: job.id, taskId: task.id }); + + try { + logger.info({ msg: `handling ${job.type} job with "finalize"` }); + + let finalizeTaskParams: IngestionNewFinalizeTaskParams = task.parameters; + const { insertedToMapproxy, insertedToGeoServer, insertedToCatalog } = finalizeTaskParams; + const { productName, productType, layerRelativePath, tileOutputFormat } = job.parameters.metadata; + const layerName = this.generateLayerName(productName, productType); + + if (!insertedToMapproxy) { + logger.info({ msg: 'publishing to mapproxy', layerName, layerRelativePath, tileOutputFormat }); + await this.mapproxyClient.publish(layerName, layerRelativePath, tileOutputFormat); + finalizeTaskParams = await this.markFinalizeStepAsCompleted(job.id, task.id, 'insertedToMapproxy', finalizeTaskParams); + } + + if (!insertedToGeoServer) { + const geoserverLayerName = layerName.toLowerCase(); + logger.info({ msg: 'publishing to geoserver', geoserverLayerName }); + await this.geoserverClient.publish(geoserverLayerName); + finalizeTaskParams = await this.markFinalizeStepAsCompleted(job.id, task.id, 'insertedToGeoServer', finalizeTaskParams); + } + + if (!insertedToCatalog) { + logger.info({ msg: 'publishing to catalog', layerName }); + await this.catalogClient.publish(job, layerName); + finalizeTaskParams = await this.markFinalizeStepAsCompleted(job.id, task.id, 'insertedToCatalog', finalizeTaskParams); + } + + if (this.isAllStepsCompleted(finalizeTaskParams)) { + logger.info({ msg: 'All finalize steps completed successfully', ...finalizeTaskParams }); + await this.queueClient.ack(job.id, task.id); + await this.queueClient.jobManagerClient.updateJob(job.id, { status: OperationStatus.COMPLETED, reason: 'Job completed successfully' }); + } + } catch (err) { + if (err instanceof Error) { + const errorMsg = `Failed to handle job finalize: ${err.message}`; + logger.error({ msg: errorMsg, error: err }); + await this.queueClient.reject(job.id, task.id, true, err.message); + } + } } private readonly mapToExtendedNewLayerMetadata = (metadata: NewRasterLayerMetadata): ExtendedRasterLayerMetadata => { @@ -82,4 +129,23 @@ export class NewJobHandler implements IJobHandler { grid, }; }; + + private generateLayerName(productId: string, productType: string): string { + return `${productId}_${productType}`; + } + + private isAllStepsCompleted(finalizeTaskParams: IngestionNewFinalizeTaskParams): boolean { + return Object.values(finalizeTaskParams).every((value) => value); + } + + private async markFinalizeStepAsCompleted( + jobId: string, + taskId: string, + step: FinalizeSteps, + finalizeTaskParams: IngestionNewFinalizeTaskParams + ): Promise { + const updatedParams: IngestionNewFinalizeTaskParams = { ...finalizeTaskParams, [step]: true }; + await this.queueClient.jobManagerClient.updateTask(jobId, taskId, { parameters: updatedParams }); + return updatedParams; + } } diff --git a/src/job/models/swapJobHandler.ts b/src/job/models/swapJobHandler.ts index 43c9590..84e02eb 100644 --- a/src/job/models/swapJobHandler.ts +++ b/src/job/models/swapJobHandler.ts @@ -1,6 +1,6 @@ import { inject, injectable } from 'tsyringe'; import { Logger } from '@map-colonies/js-logger'; -import { IJobResponse } from '@map-colonies/mc-priority-queue'; +import { IJobResponse, ITaskResponse } from '@map-colonies/mc-priority-queue'; import { UpdateRasterLayer } from '@map-colonies/mc-model-types'; import { IJobHandler } from '../../common/interfaces'; import { SERVICES } from '../../common/constants'; @@ -15,8 +15,8 @@ export class SwapJobHandler implements IJobHandler { await Promise.reject('not implemented'); } - public async handleJobFinalize(job: IJobResponse, taskId: string): Promise { - const logger = this.logger.child({ jobId: job.id, taskId }); + public async handleJobFinalize(job: IJobResponse, task: ITaskResponse): Promise { + const logger = this.logger.child({ jobId: job.id, taskId: task.id }); logger.info({ msg: `handling ${job.type} job with "finalize" task` }); await Promise.reject('not implemented'); } diff --git a/src/job/models/updateJobHandler.ts b/src/job/models/updateJobHandler.ts index 1377c7b..707ab74 100644 --- a/src/job/models/updateJobHandler.ts +++ b/src/job/models/updateJobHandler.ts @@ -1,6 +1,6 @@ import { inject, injectable } from 'tsyringe'; import { Logger } from '@map-colonies/js-logger'; -import { IJobResponse } from '@map-colonies/mc-priority-queue'; +import { IJobResponse, ITaskResponse } from '@map-colonies/mc-priority-queue'; import { UpdateRasterLayer } from '@map-colonies/mc-model-types'; import { IJobHandler } from '../../common/interfaces'; import { SERVICES } from '../../common/constants'; @@ -15,8 +15,8 @@ export class UpdateJobHandler implements IJobHandler { await Promise.reject('not implemented'); } - public async handleJobFinalize(job: IJobResponse, taskId: string): Promise { - const logger = this.logger.child({ jobId: job.id, taskId }); + public async handleJobFinalize(job: IJobResponse, task: ITaskResponse): Promise { + const logger = this.logger.child({ jobId: job.id, taskId: task.id }); logger.info({ msg: `handling ${job.type} job with "finalize" task` }); await Promise.reject('not implemented'); } diff --git a/src/task/models/tileMergeTaskManager.ts b/src/task/models/tileMergeTaskManager.ts index fabab49..36e251a 100644 --- a/src/task/models/tileMergeTaskManager.ts +++ b/src/task/models/tileMergeTaskManager.ts @@ -7,7 +7,7 @@ import { degreesPerPixelToZoomLevel, Footprint, multiIntersect, subGroupsGen, ti import { bbox, featureCollection, union } from '@turf/turf'; import { difference } from '@turf/difference'; import { inject, injectable } from 'tsyringe'; -import { SERVICES } from '../../common/constants'; +import { SERVICES, TilesStorageProvider } from '../../common/constants'; import { Grid, IConfig, @@ -34,7 +34,7 @@ export class TileMergeTaskManager { @inject(SERVICES.TILE_RANGER) private readonly tileRanger: TileRanger, @inject(SERVICES.QUEUE_CLIENT) private readonly queueClient: QueueClient ) { - this.tilesStorageProvider = this.config.get('tilesStorageProvider'); + this.tilesStorageProvider = this.config.get('tilesStorageProvider'); this.tileBatchSize = this.config.get('jobManagement.ingestion.tasks.tilesMerging.tileBatchSize'); this.taskBatchSize = this.config.get('jobManagement.ingestion.tasks.tilesMerging.taskBatchSize'); this.taskType = this.config.get('jobManagement.ingestion.tasks.tilesMerging.type'); diff --git a/src/utils/linkBuilder.ts b/src/utils/linkBuilder.ts new file mode 100644 index 0000000..03a478e --- /dev/null +++ b/src/utils/linkBuilder.ts @@ -0,0 +1,28 @@ +import { readFileSync } from 'fs'; +import { Link } from '@map-colonies/mc-model-types'; +import { inject, injectable } from 'tsyringe'; +import { IConfig } from 'config'; +import { compile } from 'handlebars'; +import { SERVICES } from '../common/constants'; + +export interface ILinkBuilderData { + serverUrl: string; + layerName: string; +} + +@injectable() +export class LinkBuilder { + private readonly compiledTemplate: HandlebarsTemplateDelegate; + + public constructor(@inject(SERVICES.CONFIG) private readonly config: IConfig) { + const templatePath = this.config.get('linkTemplatesPath'); + const template = readFileSync(templatePath, { encoding: 'utf8' }); + this.compiledTemplate = compile(template, { noEscape: true }); + } + + public createLinks(data: ILinkBuilderData): Link[] { + const linksJson = this.compiledTemplate(data); + const links = JSON.parse(linksJson) as Link[]; + return links; + } +} diff --git a/tests/configurations/unit/jest.config.js b/tests/configurations/unit/jest.config.js index 2829c8f..919a740 100644 --- a/tests/configurations/unit/jest.config.js +++ b/tests/configurations/unit/jest.config.js @@ -14,11 +14,7 @@ module.exports = { '!**/routes/**', '!/src/*', ], - modulePathIgnorePatterns: [ - '/src/job/models/newJobHandler.ts', - '/src/job/models/swapJobHandler.ts', - '/src/job/models/updateJobHandler.ts', - ], + modulePathIgnorePatterns: ['/src/job/models/swapJobHandler.ts', '/src/job/models/updateJobHandler.ts'], coverageDirectory: '/coverage', reporters: [ 'default', diff --git a/tests/unit/httpClients/catalogCLientSetup.ts b/tests/unit/httpClients/catalogCLientSetup.ts new file mode 100644 index 0000000..07c3e8e --- /dev/null +++ b/tests/unit/httpClients/catalogCLientSetup.ts @@ -0,0 +1,30 @@ +import { Link } from '@map-colonies/mc-model-types'; +import jsLogger from '@map-colonies/js-logger'; +import { ILinkBuilderData, LinkBuilder } from '../../../src/utils/linkBuilder'; +import { configMock, registerDefaultConfig } from '../mocks/configMock'; +import { CatalogClient } from '../../../src/httpClients/catalogClient'; +import { PolygonPartMangerClient } from '../../../src/httpClients/polygonPartMangerClient'; + +export type MockCreateLinks = jest.MockedFunction<(data: ILinkBuilderData) => Link[]>; + +export interface CatalogClientTestContext { + createLinksMock: MockCreateLinks; + catalogClient: CatalogClient; +} + +export function setupCatalogClientTest(): CatalogClientTestContext { + registerDefaultConfig(); + const createLinksMock = jest.fn() as MockCreateLinks; + const linkBuilder = { + createLinks: createLinksMock, + } as unknown as LinkBuilder; + const polygonPartManagerClient = new PolygonPartMangerClient(configMock, jsLogger({ enabled: false })); + const catalogClient = new CatalogClient(configMock, jsLogger({ enabled: false }), linkBuilder, polygonPartManagerClient); + + return { + createLinksMock, + catalogClient, + }; +} + +export const linksMockData: Link[] = []; diff --git a/tests/unit/httpClients/catalogClient.spec.ts b/tests/unit/httpClients/catalogClient.spec.ts new file mode 100644 index 0000000..b6b7cf0 --- /dev/null +++ b/tests/unit/httpClients/catalogClient.spec.ts @@ -0,0 +1,47 @@ +import nock from 'nock'; +import { clear as clearConfig, configMock, registerDefaultConfig } from '../mocks/configMock'; +import { PublishLayerError } from '../../../src/common/errors'; +import { ingestionNewJobExtended } from '../mocks/jobsMockData'; +import { setupCatalogClientTest } from './catalogCLientSetup'; + +describe('CatalogClient', () => { + beforeEach(() => { + registerDefaultConfig(); + }); + + afterEach(() => { + nock.cleanAll(); + clearConfig(); + jest.resetAllMocks(); + }); + describe('publish', () => { + it('should publish a layer to catalog', async () => { + const { catalogClient, createLinksMock } = setupCatalogClientTest(); + + createLinksMock.mockReturnValue([]); + const baseUrl = configMock.get('servicesUrl.catalogManager'); + const layerName = 'testLayer'; + + nock(baseUrl).post('/records').reply(201); + + const action = catalogClient.publish(ingestionNewJobExtended, layerName); + + await expect(action).resolves.not.toThrow(); + expect(nock.isDone()).toBe(true); + }); + + it('should throw an PublishLayerError when the catalog returns an error', async () => { + const { catalogClient, createLinksMock } = setupCatalogClientTest(); + + createLinksMock.mockReturnValue([]); + const baseUrl = configMock.get('servicesUrl.catalogManager'); + const layerName = 'testLayer'; + + nock(baseUrl).post('/records').reply(500); + + const action = catalogClient.publish(ingestionNewJobExtended, layerName); + + await expect(action).rejects.toThrow(PublishLayerError); + }); + }); +}); diff --git a/tests/unit/httpClients/geoserverClient.spec.ts b/tests/unit/httpClients/geoserverClient.spec.ts new file mode 100644 index 0000000..7921b2c --- /dev/null +++ b/tests/unit/httpClients/geoserverClient.spec.ts @@ -0,0 +1,46 @@ +import nock from 'nock'; +import jsLogger from '@map-colonies/js-logger'; +import { GeoserverClient } from '../../../src/httpClients/geoserverClient'; +import { configMock, registerDefaultConfig } from '../mocks/configMock'; +import { PublishLayerError } from '../../../src/common/errors'; + +describe('GeoserverClient', () => { + let geoServerClient: GeoserverClient; + beforeEach(() => { + registerDefaultConfig(); + geoServerClient = new GeoserverClient(configMock, jsLogger({ enabled: false })); + }); + + afterEach(() => { + nock.cleanAll(); + jest.resetAllMocks(); + }); + describe('publish', () => { + it('should publish a layer to geoserver', async () => { + const baseUrl = configMock.get('servicesUrl.geoserverApi'); + const workspace = configMock.get('geoserver.workspace'); + const dataStore = configMock.get('geoserver.dataStore'); + const layerName = 'testLayer'; + + nock(baseUrl).post(`/featureTypes/${workspace}/${dataStore}`).reply(201); + + const action = geoServerClient.publish(layerName); + + await expect(action).resolves.not.toThrow(); + expect(nock.isDone()).toBe(true); + }); + }); + + it('should throw an error when geoserver client returns an error', async () => { + const baseUrl = configMock.get('servicesUrl.geoserverApi'); + const workspace = configMock.get('geoserver.workspace'); + const dataStore = configMock.get('geoserver.dataStore'); + const layerName = 'errorTestLayer'; + + nock(baseUrl).post(`/featureTypes/${workspace}/${dataStore}`).reply(500); + + const action = geoServerClient.publish(layerName); + + await expect(action).rejects.toThrow(PublishLayerError); + }); +}); diff --git a/tests/unit/httpClients/mapproxyClient.spec.ts b/tests/unit/httpClients/mapproxyClient.spec.ts new file mode 100644 index 0000000..564e308 --- /dev/null +++ b/tests/unit/httpClients/mapproxyClient.spec.ts @@ -0,0 +1,68 @@ +import nock from 'nock'; +import jsLogger from '@map-colonies/js-logger'; +import { TileOutputFormat } from '@map-colonies/mc-model-types'; +import { MapproxyApiClient } from '../../../src/httpClients/mapproxyClient'; +import { clear as clearConfig, configMock, init, setValue } from '../mocks/configMock'; +import { PublishLayerError, UnsupportedStorageProviderError } from '../../../src/common/errors'; + +describe('mapproxyClient', () => { + let mapproxyApiClient: MapproxyApiClient; + + afterEach(() => { + nock.cleanAll(); + clearConfig(); + jest.resetAllMocks(); + }); + + describe('publish', () => { + it('should publish a layer to mapproxy', async () => { + init(); + setValue('tilesStorageProvider', 'FS'); + + mapproxyApiClient = new MapproxyApiClient(configMock, jsLogger({ enabled: false })); + const baseUrl = configMock.get('servicesUrl.mapproxyApi'); + const layerName = 'testLayer'; + const layerRelativePath = 'testLayerPath'; + const tileOutputFormat = TileOutputFormat.PNG; + + nock(baseUrl).post('/layer').reply(201); + + const action = mapproxyApiClient.publish(layerName, layerRelativePath, tileOutputFormat); + + await expect(action).resolves.not.toThrow(); + expect(nock.isDone()).toBe(true); + }); + + it('should throw an error for unsupported storage provider', async () => { + init(); + setValue('tilesStorageProvider', 'unsupported'); + mapproxyApiClient = new MapproxyApiClient(configMock, jsLogger({ enabled: false })); + + const baseUrl = configMock.get('servicesUrl.mapproxyApi'); + const layerName = 'testLayer'; + const layerRelativePath = 'testLayerPath'; + const tileOutputFormat = TileOutputFormat.PNG; + + nock(baseUrl).post('/layer').reply(201); + + await expect(mapproxyApiClient.publish(layerName, layerRelativePath, tileOutputFormat)).rejects.toThrow(UnsupportedStorageProviderError); + }); + + it('should throw an PublishLayerError when mapproxyApi client returns an error', async () => { + init(); + setValue('tilesStorageProvider', 'FS'); + + mapproxyApiClient = new MapproxyApiClient(configMock, jsLogger({ enabled: false })); + const baseUrl = configMock.get('servicesUrl.mapproxyApi'); + const layerName = 'errorTestLayer'; + const layerRelativePath = 'errorTestLayerPath'; + const tileOutputFormat = TileOutputFormat.PNG; + + nock(baseUrl).post('/layer').reply(500); + + const action = mapproxyApiClient.publish(layerName, layerRelativePath, tileOutputFormat); + + await expect(action).rejects.toThrow(PublishLayerError); + }); + }); +}); diff --git a/tests/unit/job/jobProcessor/JobProcessor.spec.ts b/tests/unit/job/jobProcessor/JobProcessor.spec.ts index aeedc1c..7b50c2f 100644 --- a/tests/unit/job/jobProcessor/JobProcessor.spec.ts +++ b/tests/unit/job/jobProcessor/JobProcessor.spec.ts @@ -100,7 +100,7 @@ describe('JobProcessor', () => { expect(mockGetJob).toHaveBeenCalledTimes(1); expect(mockGetJob).toHaveBeenCalledWith(task.jobId); expect(mockJobHandlerFactory).toHaveBeenCalledWith(job.type); - expect(mockHandler.handleJobFinalize).toHaveBeenCalledWith(job, task.id); + expect(mockHandler.handleJobFinalize).toHaveBeenCalledWith(job, task); }); it('should reject task if an error occurred during processing', async () => { diff --git a/tests/unit/job/newJobHandler/newJobHandler.spec.ts b/tests/unit/job/newJobHandler/newJobHandler.spec.ts new file mode 100644 index 0000000..1cdc32c --- /dev/null +++ b/tests/unit/job/newJobHandler/newJobHandler.spec.ts @@ -0,0 +1,130 @@ +/* eslint-disable @typescript-eslint/unbound-method */ +import { OperationStatus } from '@map-colonies/mc-priority-queue'; +import { finalizeTaskForIngestionNew } from '../../mocks/tasksMockData'; +import { ingestionNewJob, ingestionNewJobExtended } from '../../mocks/jobsMockData'; +import { IMergeTaskParameters } from '../../../../src/common/interfaces'; +import { PublishLayerError } from '../../../../src/common/errors'; +import { setupNewJobHandlerTest } from './newJobHandlerSetup'; + +describe('NewJobHandler', () => { + beforeEach(() => { + jest.resetAllMocks(); + }); + describe('handleJobInit', () => { + it('should handle job init successfully', async () => { + const { newJobHandler, taskBuilderMock, queueClientMock, jobManagerClientMock } = setupNewJobHandlerTest(); + const job = ingestionNewJob; + const taskId = '7e630dea-ea29-4b30-a88e-5407bf67d1bc'; + const tasks: AsyncGenerator = (async function* () {})(); + taskBuilderMock.buildTasks.mockReturnValue(tasks); + taskBuilderMock.pushTasks.mockResolvedValue(undefined); + jobManagerClientMock.updateJob.mockResolvedValue(undefined); + queueClientMock.ack.mockResolvedValue(undefined); + + await newJobHandler.handleJobInit(job, taskId); + + expect(taskBuilderMock.buildTasks).toHaveBeenCalled(); + expect(taskBuilderMock.pushTasks).toHaveBeenCalledWith(job.id, tasks); + expect(queueClientMock.jobManagerClient.updateJob).toHaveBeenCalled(); + expect(queueClientMock.ack).toHaveBeenCalledWith(job.id, taskId); + }); + + it('should handle job init failure and reject the task', async () => { + const { newJobHandler, taskBuilderMock, queueClientMock } = setupNewJobHandlerTest(); + + const job = ingestionNewJob; + const taskId = '7e630dea-ea29-4b30-a88e-5407bf67d1bc'; + const tasks: AsyncGenerator = (async function* () {})(); + + const error = new Error('Test error'); + + taskBuilderMock.buildTasks.mockReturnValue(tasks); + taskBuilderMock.pushTasks.mockRejectedValue(error); + queueClientMock.reject.mockResolvedValue(undefined); + + await newJobHandler.handleJobInit(job, taskId); + + expect(queueClientMock.reject).toHaveBeenCalledWith(job.id, taskId, true, error.message); + }); + }); + + describe('handleJobFinalize', () => { + it('should handle job finalize successfully', async () => { + const { newJobHandler, queueClientMock, jobManagerClientMock, mapproxyClientMock, geoserverClientMock, catalogClientMock } = + setupNewJobHandlerTest(); + const job = ingestionNewJobExtended; + const task = finalizeTaskForIngestionNew; + + jobManagerClientMock.updateJob.mockResolvedValue(undefined); + jobManagerClientMock.updateTask.mockResolvedValue(undefined); + mapproxyClientMock.publish.mockResolvedValue(undefined); + geoserverClientMock.publish.mockResolvedValue(undefined); + catalogClientMock.publish.mockResolvedValue(undefined); + queueClientMock.ack.mockResolvedValue(undefined); + + await newJobHandler.handleJobFinalize(job, task); + + expect(jobManagerClientMock.updateJob).toHaveBeenCalledWith(job.id, { + status: OperationStatus.COMPLETED, + reason: 'Job completed successfully', + }); + + expect(queueClientMock.ack).toHaveBeenCalledWith(job.id, task.id); + }); + + it('should handle job finalize failure and reject the task (mapproxyApi publish failed)', async () => { + const { newJobHandler, queueClientMock, jobManagerClientMock, mapproxyClientMock } = setupNewJobHandlerTest(); + const job = ingestionNewJobExtended; + const task = finalizeTaskForIngestionNew; + + const error = new PublishLayerError('MapproxyApi', 'testLayer', new Error('Test error')); + + jobManagerClientMock.updateJob.mockResolvedValue(undefined); + jobManagerClientMock.updateTask.mockResolvedValue(undefined); + mapproxyClientMock.publish.mockRejectedValue(error); + queueClientMock.reject.mockResolvedValue(undefined); + + await newJobHandler.handleJobFinalize(job, task); + + expect(queueClientMock.reject).toHaveBeenCalledWith(job.id, task.id, true, error.message); + }); + + it('should handle job finalize failure and reject the task (geoserverApi publish failed)', async () => { + const { newJobHandler, queueClientMock, jobManagerClientMock, mapproxyClientMock, geoserverClientMock } = setupNewJobHandlerTest(); + const job = ingestionNewJobExtended; + const task = finalizeTaskForIngestionNew; + + const error = new PublishLayerError('GeoserverApi', 'testLayer', new Error('Test error')); + + jobManagerClientMock.updateJob.mockResolvedValue(undefined); + jobManagerClientMock.updateTask.mockResolvedValue(undefined); + mapproxyClientMock.publish.mockResolvedValue(undefined); + geoserverClientMock.publish.mockRejectedValue(error); + queueClientMock.reject.mockResolvedValue(undefined); + + await newJobHandler.handleJobFinalize(job, task); + + expect(queueClientMock.reject).toHaveBeenCalledWith(job.id, task.id, true, error.message); + }); + + it('should handle job finalize failure and reject the task (catalogApi publish failed)', async () => { + const { newJobHandler, queueClientMock, jobManagerClientMock, mapproxyClientMock, geoserverClientMock, catalogClientMock } = + setupNewJobHandlerTest(); + const job = ingestionNewJobExtended; + const task = finalizeTaskForIngestionNew; + + const error = new PublishLayerError('CatalogApi', 'testLayer', new Error('Test error')); + + jobManagerClientMock.updateJob.mockResolvedValue(undefined); + jobManagerClientMock.updateTask.mockResolvedValue(undefined); + mapproxyClientMock.publish.mockResolvedValue(undefined); + geoserverClientMock.publish.mockResolvedValue(undefined); + catalogClientMock.publish.mockRejectedValue(error); + queueClientMock.reject.mockResolvedValue(undefined); + + await newJobHandler.handleJobFinalize(job, task); + + expect(queueClientMock.reject).toHaveBeenCalledWith(job.id, task.id, true, error.message); + }); + }); +}); diff --git a/tests/unit/job/newJobHandler/newJobHandlerSetup.ts b/tests/unit/job/newJobHandler/newJobHandlerSetup.ts new file mode 100644 index 0000000..f768d50 --- /dev/null +++ b/tests/unit/job/newJobHandler/newJobHandlerSetup.ts @@ -0,0 +1,58 @@ +import jsLogger from '@map-colonies/js-logger'; +import { JobManagerClient, TaskHandler as QueueClient } from '@map-colonies/mc-priority-queue'; +import { TileMergeTaskManager } from '../../../../src/task/models/tileMergeTaskManager'; +import { MapproxyApiClient } from '../../../../src/httpClients/mapproxyClient'; +import { GeoserverClient } from '../../../../src/httpClients/geoserverClient'; +import { CatalogClient } from '../../../../src/httpClients/catalogClient'; +import { NewJobHandler } from '../../../../src/job/models/newJobHandler'; + +export interface NewJobHandlerTestContext { + newJobHandler: NewJobHandler; + taskBuilderMock: jest.Mocked; + queueClientMock: jest.Mocked; + jobManagerClientMock: jest.Mocked; + mapproxyClientMock: jest.Mocked; + geoserverClientMock: jest.Mocked; + catalogClientMock: jest.Mocked; +} + +export const setupNewJobHandlerTest = (): NewJobHandlerTestContext => { + const taskBuilderMock = { + buildTasks: jest.fn(), + pushTasks: jest.fn(), + } as unknown as jest.Mocked; + + const jobManagerClientMock = { + updateJob: jest.fn(), + updateTask: jest.fn(), + } as unknown as jest.Mocked; + + const queueClientMock = { + jobManagerClient: jobManagerClientMock, + ack: jest.fn(), + reject: jest.fn(), + } as unknown as jest.Mocked; + + const mapproxyClientMock = { publish: jest.fn() } as unknown as jest.Mocked; + const geoserverClientMock = { publish: jest.fn() } as unknown as jest.Mocked; + const catalogClientMock = { publish: jest.fn() } as unknown as jest.Mocked; + + const newJobHandler = new NewJobHandler( + jsLogger({ enabled: false }), + taskBuilderMock, + queueClientMock, + mapproxyClientMock, + geoserverClientMock, + catalogClientMock + ); + + return { + newJobHandler, + taskBuilderMock, + queueClientMock, + jobManagerClientMock, + mapproxyClientMock, + geoserverClientMock, + catalogClientMock, + }; +}; diff --git a/tests/unit/mocks/configMock.ts b/tests/unit/mocks/configMock.ts index 6806d97..d71073a 100644 --- a/tests/unit/mocks/configMock.ts +++ b/tests/unit/mocks/configMock.ts @@ -79,7 +79,20 @@ const registerDefaultConfig = (): void => { delay: 'exponential', shouldResetTimeout: true, }, + tilesStorageProvider: 'FS', disableHttpClientLogs: true, + linkTemplatesPath: 'config/linkTemplates.template', + servicesUrl: { + mapproxyApi: 'http://mapproxy-api', + geoserverApi: 'http://geoserver-api', + catalogManager: 'http://catalog-manager', + mapproxyDns: 'http://mapproxy', + polygonPartsManager: 'http://polygon-parts-manager', + }, + geoserver: { + workspace: 'testWorkspace', + dataStore: 'testDataStore', + }, jobManagement: { config: { jobManagerBaseUrl: 'http://job-manager', diff --git a/tests/unit/mocks/jobsMockData.ts b/tests/unit/mocks/jobsMockData.ts index b33d628..d3f7a3b 100644 --- a/tests/unit/mocks/jobsMockData.ts +++ b/tests/unit/mocks/jobsMockData.ts @@ -1,5 +1,6 @@ -import { NewRasterLayer, ProductType, Transparency, UpdateRasterLayer } from '@map-colonies/mc-model-types'; +import { NewRasterLayer, ProductType, TileOutputFormat, Transparency, UpdateRasterLayer } from '@map-colonies/mc-model-types'; import { IJobResponse, OperationStatus } from '@map-colonies/mc-priority-queue'; +import { ExtendedNewRasterLayer, Grid } from '../../../src/common/interfaces'; import { partsData } from './partsMockData'; /* eslint-disable @typescript-eslint/no-magic-numbers */ @@ -54,6 +55,22 @@ export const ingestionNewJob: IJobResponse = { updated: '2024-07-21T10:59:23.510Z', }; +export const ingestionNewJobExtended: IJobResponse = { + ...ingestionNewJob, + parameters: { + ...ingestionNewJob.parameters, + metadata: { + catalogId: 'some-catalog-id', + displayPath: 'some-display-path', + layerRelativePath: 'some-layer-relative-path', + tileOutputFormat: TileOutputFormat.PNG, + tileMimeType: 'image/png', + grid: Grid.TWO_ON_ONE, + ...ingestionNewJob.parameters.metadata, + }, + }, +}; + export const ingestionUpdateJob: IJobResponse = { id: 'd027b3aa-272b-4dc9-91d7-ba8343af5ed1', resourceId: 'another-product-id', diff --git a/tests/unit/mocks/linksBuilderUtils.ts b/tests/unit/mocks/linksBuilderUtils.ts new file mode 100644 index 0000000..936fb6a --- /dev/null +++ b/tests/unit/mocks/linksBuilderUtils.ts @@ -0,0 +1,81 @@ +import { Link } from '@map-colonies/mc-model-types'; + +export const linksTemplate = `[ + { + "name": "{{layerName}}", + "description": "", + "protocol": "WMS", + "url": "{{serverUrl}}/service?REQUEST=GetCapabilities" + }, + { + "name": "{{layerName}}", + "description": "", + "protocol": "WMS_BASE", + "url": "{{serverUrl}}/wms" + }, + { + "name": "{{layerName}}", + "description": "", + "protocol": "WMTS", + "url": "{{serverUrl}}/wmts/1.0.0/WMTSCapabilities.xml" + }, + { + "name": "{{layerName}}", + "description": "", + "protocol": "WMTS_KVP", + "url": "{{serverUrl}}/service?REQUEST=GetCapabilities&SERVICE=WMTS" + }, + { + "name": "{{layerName}}", + "description": "", + "protocol": "WMTS_BASE", + "url": "{{serverUrl}}/wmts" + }, + { + "name": "{{layerName}}", + "description": "", + "protocol": "WFS", + "url": "{{serverUrl}}/wfs?request=GetCapabilities" + } +]`; + +export const getExpectedLinks = (serverUrl: string, layerName: string): Link[] => { + return [ + { + name: layerName, + description: '', + protocol: 'WMS', + url: `${serverUrl}/service?REQUEST=GetCapabilities`, + }, + { + name: layerName, + description: '', + protocol: 'WMS_BASE', + url: `${serverUrl}/wms`, + }, + { + name: layerName, + description: '', + protocol: 'WMTS', + url: `${serverUrl}/wmts/1.0.0/WMTSCapabilities.xml`, + }, + { + name: layerName, + description: '', + protocol: 'WMTS_KVP', + url: `${serverUrl}/service?REQUEST=GetCapabilities&SERVICE=WMTS`, + }, + { + name: layerName, + description: '', + protocol: 'WMTS_BASE', + url: `${serverUrl}/wmts`, + }, + { + name: layerName, + description: '', + protocol: 'WFS', + url: `${serverUrl}/wfs?request=GetCapabilities`, + }, + ]; +}; diff --git a/tests/unit/mocks/tasksMockData.ts b/tests/unit/mocks/tasksMockData.ts index 7be0cdd..886fe6b 100644 --- a/tests/unit/mocks/tasksMockData.ts +++ b/tests/unit/mocks/tasksMockData.ts @@ -1,3 +1,4 @@ +import { IngestionNewFinalizeTaskParams } from '@map-colonies/mc-model-types'; import { ITaskResponse, OperationStatus } from '@map-colonies/mc-priority-queue'; //copied from Ingestion-Trigger, should be moved to a shared library (Mc-Models) @@ -59,12 +60,14 @@ export const initTaskForIngestionSwapUpdate: ITaskResponse = { +export const finalizeTaskForIngestionNew: ITaskResponse = { id: '4a5486bd-6269-4898-b9b1-647fe56d6ae2', type: 'finalize', description: '', parameters: { - blockDuplication: true, + insertedToCatalog: false, + insertedToGeoServer: false, + insertedToMapproxy: false, }, status: OperationStatus.IN_PROGRESS, percentage: 0, diff --git a/tests/unit/utils/linkBuilder.spec.ts b/tests/unit/utils/linkBuilder.spec.ts new file mode 100644 index 0000000..37957f8 --- /dev/null +++ b/tests/unit/utils/linkBuilder.spec.ts @@ -0,0 +1,31 @@ +import FS from 'fs'; +import { ILinkBuilderData, LinkBuilder } from '../../../src/utils/linkBuilder'; +import { configMock, registerDefaultConfig } from '../mocks/configMock'; +import { getExpectedLinks, linksTemplate } from '../mocks/linksBuilderUtils'; + +let linkBuilder: LinkBuilder; + +describe('LinkBuilder', () => { + beforeEach(() => { + jest.resetAllMocks(); + registerDefaultConfig(); + }); + describe('createLinks', () => { + it('should return links for a given layer', () => { + linkBuilder = new LinkBuilder(configMock); + const serverUrl = configMock.get('servicesUrl.mapproxyDns'); + + const linkBuilderData: ILinkBuilderData = { + serverUrl, + layerName: 'testLayer', + }; + + const expectedLinks = getExpectedLinks(serverUrl, linkBuilderData.layerName); + + jest.spyOn(FS, 'readFileSync').mockReturnValue(linksTemplate); + const actualLinks = linkBuilder.createLinks(linkBuilderData); + + expect(actualLinks).toEqual(expectedLinks); + }); + }); +});