diff --git a/src/common/interfaces.ts b/src/common/interfaces.ts index 5a94490..2ee47c8 100644 --- a/src/common/interfaces.ts +++ b/src/common/interfaces.ts @@ -12,7 +12,7 @@ import { IngestionUpdateJobParams, } from '@map-colonies/mc-model-types'; import { TilesMimeFormat } from '@map-colonies/types'; -import { BBox, Feature, MultiPolygon, Polygon } from 'geojson'; +import { BBox, Feature, FeatureCollection, MultiPolygon, Polygon } from 'geojson'; import { ITileRange } from '@map-colonies/mc-utils'; import { LayerCacheType, SeedMode } from './constants'; @@ -117,29 +117,39 @@ export interface IBBox { export type Footprint = Polygon | MultiPolygon | Feature; -export interface PartSourceContext { - fileName: string; - tilesPath: string; - footprint: Polygon; - extent: BBox; +export interface PolygonProperties { + maxZoom: number; +} + +export type PolygonFeature = Feature; + +export type PPFeatureCollection = FeatureCollection; + +export interface FeatureCollectionWitZoomDefinitions { + ppCollection: PPFeatureCollection; + zoomDefinitions: ZoomDefinitions; +} + +export interface ZoomDefinitions { maxZoom: number; + partsZoomLevelMatch: boolean; } -export interface UnifiedPart { +export interface TilesSource { fileName: string; tilesPath: string; +} + +export type UnifiedPart = { footprint: Feature; extent: BBox; - maxZoom: number; -} +} & TilesSource; export interface MergeParameters { - parts: PartSourceContext[]; - destPath: string; - maxZoom: number; - grid: Grid; - targetFormat: TileOutputFormat; - isNewTarget: boolean; + ppCollection: PPFeatureCollection; + zoomDefinitions: ZoomDefinitions; + taskMetadata: MergeTilesMetadata; + tilesSource: TilesSource; } export interface MergeSources { @@ -157,7 +167,7 @@ export interface MergeTaskParameters { } export interface PartsIntersection { - parts: PartSourceContext[]; + parts: PolygonFeature[]; intersection: Footprint | null; } @@ -179,10 +189,6 @@ export interface MergeTilesMetadata { grid: Grid; } -export interface PartsSourceWithMaxZoom { - parts: PartSourceContext[]; - maxZoom: number; -} //#endregion task //#region finalize task diff --git a/src/task/models/tileMergeTaskManager.ts b/src/task/models/tileMergeTaskManager.ts index 2017519..dfc7cfc 100644 --- a/src/task/models/tileMergeTaskManager.ts +++ b/src/task/models/tileMergeTaskManager.ts @@ -1,10 +1,9 @@ import { join } from 'path'; -import { BBox, Feature, MultiPolygon, Polygon } from 'geojson'; import { Logger } from '@map-colonies/js-logger'; -import { InputFiles, PolygonPart, TileOutputFormat } from '@map-colonies/mc-model-types'; +import { InputFiles, PolygonPart } from '@map-colonies/mc-model-types'; import { ICreateTaskBody, TaskHandler as QueueClient } from '@map-colonies/mc-priority-queue'; import { degreesPerPixelToZoomLevel, tileBatchGenerator, TileRanger } from '@map-colonies/mc-utils'; -import { bbox, featureCollection, intersect, polygon, union } from '@turf/turf'; +import { bbox, featureCollection, polygon, union } from '@turf/turf'; import { inject, injectable } from 'tsyringe'; import { SERVICES, TilesStorageProvider } from '../../common/constants'; import { @@ -14,9 +13,12 @@ import { MergeSources, MergeTaskParameters, MergeTilesTaskParams, - PartSourceContext, - PartsSourceWithMaxZoom, + PolygonFeature, + FeatureCollectionWitZoomDefinitions, UnifiedPart, + TilesSource, + MergeTilesMetadata, + PPFeatureCollection, } from '../../common/interfaces'; import { fileExtensionExtractor } from '../../utils/fileutils'; import { TaskMetrics } from '../../utils/metrics/taskMetrics'; @@ -109,123 +111,139 @@ export class TileMergeTaskManager { logger.info({ msg: 'creating task parameters' }); - const { parts, maxZoom } = this.generatePartsSourceWithMaxZoom(partsData, inputFiles); + const { ppCollection, zoomDefinitions } = this.createFeatureCollectionWithZoomDefinitions(partsData); + const tilesSource = this.extractTilesSource(inputFiles); return { - parts, - destPath: taskMetadata.layerRelativePath, - grid: taskMetadata.grid, - targetFormat: taskMetadata.tileOutputFormat, - isNewTarget: taskMetadata.isNewTarget, - maxZoom, + ppCollection, + zoomDefinitions, + taskMetadata, + tilesSource, }; } - private generatePartsSourceWithMaxZoom(parts: PolygonPart[], inputFiles: InputFiles): PartsSourceWithMaxZoom { - this.logger.info({ msg: 'Generating parts with source context', inputFiles, numberOfParts: parts.length }); + private extractTilesSource(inputFiles: InputFiles): TilesSource { + const { originDirectory, fileNames } = inputFiles; + if (fileNames.length > 1) { + throw new Error('Multiple files ingestion is currently not supported'); + } + const fileName = fileNames[0]; + const tilesPath = join(originDirectory, fileName); - const partsContext: PartSourceContext[] = []; - let maxZoom = 0; + return { + fileName, + tilesPath, + }; + } - parts.forEach((part) => { - const currentZoom = degreesPerPixelToZoomLevel(part.resolutionDegree); - maxZoom = Math.max(maxZoom, currentZoom); - const parts = this.linkPartToInputFiles(part, inputFiles); - partsContext.push(...parts); + private createFeatureCollectionWithZoomDefinitions(parts: PolygonPart[]): FeatureCollectionWitZoomDefinitions { + this.logger.info({ + msg: 'Generating featureParts', + numberOfParts: parts.length, }); - this.logger.info({ msg: 'Calculated parts max zoom', maxZoom }); - return { parts: partsContext, maxZoom }; - } + let partsMaxZoom = 0; + + const featureParts = parts.map((part) => { + const featurePart = this.createFeaturePolygon(part); + const currentZoom = featurePart.properties.maxZoom; - private linkPartToInputFiles(part: PolygonPart, inputFiles: InputFiles): PartSourceContext[] { - this.logger.debug({ msg: 'linking parts to input files', part, inputFiles, numberOfFiles: inputFiles.fileNames.length }); - return inputFiles.fileNames.map((fileName) => this.linkPartToFile(part, fileName, inputFiles.originDirectory)); + partsMaxZoom = Math.max(partsMaxZoom, currentZoom); + return featurePart; + }); + + const partsZoomLevelMatch = featureParts.every((part) => part.properties.maxZoom === partsMaxZoom); + + const zoomDefinitions = { + maxZoom: partsMaxZoom, + partsZoomLevelMatch, + }; + + this.logger.info({ + msg: 'Calculated parts zoom definitions', + partsMaxZoom, + partsZoomLevelMatch, + }); + + return { + ppCollection: featureCollection(featureParts), + zoomDefinitions, + }; } - private linkPartToFile(part: PolygonPart, fileName: string, originDirectory: string): PartSourceContext { + private createFeaturePolygon(part: PolygonPart): PolygonFeature { const logger = this.logger.child({ partName: part.sourceName, - fileName, - originDirectory, }); - logger.debug({ msg: 'Linking part to input file' }); - const tilesPath = join(originDirectory, fileName); - const footprint = part.footprint; - const extent: BBox = bbox(footprint); const maxZoom = degreesPerPixelToZoomLevel(part.resolutionDegree); + logger.debug({ msg: `Part max zoom: ${maxZoom}` }); + const featurePolygon = polygon(part.footprint.coordinates, { maxZoom }); - return { - fileName, - tilesPath, - footprint, - extent, - maxZoom, - }; + return featurePolygon; } private async *createZoomLevelTasks(params: MergeParameters): AsyncGenerator { - const { parts, destPath, targetFormat, isNewTarget, grid, maxZoom } = params; + const { ppCollection, taskMetadata, zoomDefinitions, tilesSource } = params; + const { maxZoom, partsZoomLevelMatch } = zoomDefinitions; + + let unifiedPart: UnifiedPart | null = partsZoomLevelMatch ? this.unifyParts(ppCollection, tilesSource) : null; for (let zoom = maxZoom; zoom >= 0; zoom--) { - const filteredParts = parts.filter((part) => part.maxZoom >= zoom); - const processedParts = this.unifyParts(filteredParts); - for await (const part of processedParts) { - yield* this.createTasksForPart(part, zoom, { destPath, grid, isNewTarget, targetFormat }); + if (!partsZoomLevelMatch) { + const filteredFeatures = ppCollection.features.filter((feature) => feature.properties.maxZoom >= zoom); + const collection = featureCollection(filteredFeatures); + unifiedPart = this.unifyParts(collection, tilesSource); } + + yield* this.createTasksForPart(unifiedPart!, zoom, taskMetadata); } } - private unifyParts(parts: PartSourceContext[]): UnifiedPart[] { - const mergedParts: UnifiedPart[] = []; - // Merge parts by union and avoid duplicate overlaps. - for (const part of parts) { - let merged = false; - const currentPart = polygon(part.footprint.coordinates); - for (let i = 0; i < mergedParts.length; i++) { - const mergedPart = mergedParts[i].footprint; - if (this.doIntersect(currentPart, mergedPart)) { - const unionResult = union(featureCollection([currentPart, mergedPart])); - if (unionResult === null) { - continue; - } - mergedParts[i].footprint = unionResult; - merged = true; - break; - } - } - if (!merged) { - const processedPart: UnifiedPart = { ...part, footprint: currentPart }; - mergedParts.push(processedPart); - } + private unifyParts(featureCollection: PPFeatureCollection, tilesSource: TilesSource): UnifiedPart { + const { fileName, tilesPath } = tilesSource; + const isOnePart = featureCollection.features.length === 1; + + if (isOnePart) { + const featurePart = featureCollection.features[0]; + return { + fileName: fileName, + tilesPath: tilesPath, + footprint: featurePart, + extent: bbox(featurePart), + }; } - this.logger.debug({ msg: 'Preprocessed parts', numberOfParts: mergedParts.length }); - return mergedParts; - } - private doIntersect(footprint1: Feature, footprint2: Feature): boolean { - const intersection = intersect(featureCollection([footprint1, footprint2])); - return intersection !== null; + const mergedFootprint = union(featureCollection); + if (mergedFootprint === null) { + throw new Error('Failed to merge footprints because the union result is null'); + } + + return { + fileName: fileName, + tilesPath: tilesPath, + footprint: mergedFootprint, + extent: bbox(mergedFootprint), + }; } private async *createTasksForPart( part: UnifiedPart, zoom: number, - params: { destPath: string; targetFormat: TileOutputFormat; isNewTarget: boolean; grid: Grid } + tilesMetadata: MergeTilesMetadata ): AsyncGenerator { - const { destPath, grid, isNewTarget, targetFormat } = params; - const logger = this.logger.child({ zoomLevel: zoom, isNewTarget, destPath, targetFormat, grid }); + const { layerRelativePath, grid, isNewTarget, tileOutputFormat } = tilesMetadata; + const logger = this.logger.child({ zoomLevel: zoom, isNewTarget, layerRelativePath, tileOutputFormat, grid }); const footprint = part.footprint; const rangeGenerator = this.tileRanger.encodeFootprint(footprint, zoom); const batches = tileBatchGenerator(this.tileBatchSize, rangeGenerator); - const sources = this.createPartSources(part, grid, destPath); + const sources = this.createPartSources(part, grid, layerRelativePath); for await (const batch of batches) { logger.debug({ msg: 'Yielding batch task', batchSize: batch.length }); yield { - targetFormat, + targetFormat: tileOutputFormat, isNewTarget: isNewTarget, batches: batch, sources, diff --git a/src/utils/geoUtils.ts b/src/utils/geoUtils.ts deleted file mode 100644 index ff043ce..0000000 --- a/src/utils/geoUtils.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { Feature, Polygon } from 'geojson'; -import { Footprint } from '../common/interfaces'; - -// we need to use FootprintFeature because turf supporting this type -export type FootprintFeature = Feature; - -export const convertToFeature = (footprint: Footprint): FootprintFeature => { - if ('type' in footprint && footprint.type === 'Feature' && footprint.geometry.type === 'Polygon') { - return footprint as Feature; // making this assertion because we know for sure the FootPrint feature is of type polygon (which is the only geometry we support) - } - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - if ('type' in footprint && footprint.type === 'Polygon') { - return { - type: 'Feature', - geometry: footprint, - properties: {}, - }; - } - - throw new Error('Unsupported footprint type'); -}; diff --git a/tests/unit/mocks/partsMockData.ts b/tests/unit/mocks/partsMockData.ts index d91640a..e80f924 100644 --- a/tests/unit/mocks/partsMockData.ts +++ b/tests/unit/mocks/partsMockData.ts @@ -2,7 +2,7 @@ import { faker } from '@faker-js/faker'; import { BBox, Polygon } from 'geojson'; import { PolygonPart } from '@map-colonies/mc-model-types'; -import { PartSourceContext } from '../../../src/common/interfaces'; +import { PolygonFeature, PPFeatureCollection } from '../../../src/common/interfaces'; export function createFakeBBox(): BBox { return [ @@ -29,13 +29,20 @@ export function createFakePolygon(): Polygon { }; } -export function createFakePartSource(): PartSourceContext { +export function createFakePolygonFeature(): PolygonFeature { return { - tilesPath: `${faker.string.uuid()}/${faker.string.uuid()}`, - fileName: `${faker.string.alpha({ length: 8 })}.gpkg`, - maxZoom: faker.number.int({ min: 0, max: 21 }), - extent: createFakeBBox(), - footprint: createFakePolygon(), + geometry: createFakePolygon(), + type: 'Feature', + properties: { + maxZoom: faker.number.int({ min: 0, max: 20 }), + }, + }; +} + +export function createFakeFeatureCollection(): PPFeatureCollection { + return { + type: 'FeatureCollection', + features: Array.from({ length: faker.number.int({ min: 1, max: 10 }) }, createFakePolygonFeature), }; } diff --git a/tests/unit/task/tileMergeTaskManager/tileMergeTaskManager.spec.ts b/tests/unit/task/tileMergeTaskManager/tileMergeTaskManager.spec.ts index 23cb905..e28bff2 100644 --- a/tests/unit/task/tileMergeTaskManager/tileMergeTaskManager.spec.ts +++ b/tests/unit/task/tileMergeTaskManager/tileMergeTaskManager.spec.ts @@ -1,9 +1,9 @@ /* eslint-disable jest/no-commented-out-tests */ import { randomUUID } from 'crypto'; import nock from 'nock'; -import { faker } from '@faker-js/faker'; +import { bbox } from '@turf/turf'; import { TileOutputFormat } from '@map-colonies/mc-model-types'; -import { createFakePartSource, partsData } from '../../mocks/partsMockData'; +import { createFakeFeatureCollection, multiPartDataWithDifferentResolution, partsData } from '../../mocks/partsMockData'; import { configMock, registerDefaultConfig } from '../../mocks/configMock'; import { ingestionNewJob } from '../../mocks/jobsMockData'; import { Grid, MergeTaskParameters, MergeTilesTaskParams } from '../../../../src/common/interfaces'; @@ -21,13 +21,21 @@ describe('tileMergeTaskManager', () => { }); describe('buildTasks', () => { - it('should build tasks successfully for Ingestion New init task', async () => { - const { tileMergeTaskManager } = testContext; - const buildTasksParams: MergeTilesTaskParams = { + const successTestCases: MergeTilesTaskParams[] = [ + { taskMetadata: { layerRelativePath: 'layerRelativePath', tileOutputFormat: TileOutputFormat.PNG, isNewTarget: true, grid: Grid.TWO_ON_ONE }, partsData, - inputFiles: { originDirectory: 'originDirectory', fileNames: ['fileNames'] }, - }; + inputFiles: { originDirectory: 'originDirectory', fileNames: ['unified_zoom_level'] }, + }, + { + taskMetadata: { layerRelativePath: 'layerRelativePath', tileOutputFormat: TileOutputFormat.PNG, isNewTarget: true, grid: Grid.TWO_ON_ONE }, + partsData: multiPartDataWithDifferentResolution, + inputFiles: { originDirectory: 'originDirectory', fileNames: ['different_zoom_level'] }, + }, + ]; + + test.each(successTestCases)('should build tasks successfully', async (buildTasksParams) => { + const { tileMergeTaskManager } = testContext; const result = tileMergeTaskManager.buildTasks(buildTasksParams); const tasks: MergeTaskParameters[] = []; @@ -48,14 +56,10 @@ describe('tileMergeTaskManager', () => { it('should handle errors in buildTasks correctly', () => { const { tileMergeTaskManager } = testContext; - jest.spyOn(tileMergeTaskManager as unknown as { prepareMergeParameters: jest.Func }, 'prepareMergeParameters').mockImplementationOnce(() => { - throw new Error('Mocked error'); - }); - - const buildTasksParams: MergeTilesTaskParams = { + const buildTasksParams = { taskMetadata: { layerRelativePath: 'layerRelativePath', tileOutputFormat: TileOutputFormat.PNG, isNewTarget: true, grid: Grid.TWO_ON_ONE }, partsData, - inputFiles: { originDirectory: 'originDirectory', fileNames: ['fileNames'] }, + inputFiles: { originDirectory: 'originDirectory', fileNames: ['file1', 'file2'] }, }; let error: Error | null = null; @@ -73,10 +77,11 @@ describe('tileMergeTaskManager', () => { describe('unifyParts', () => { it('should unify parts correctly', () => { const { tileMergeTaskManager } = testContext; - const partsData = faker.helpers.multiple(createFakePartSource, { count: 10 }); - const result = tileMergeTaskManager['unifyParts'](partsData); + const partsData = createFakeFeatureCollection(); + const result = tileMergeTaskManager['unifyParts'](partsData, { fileName: 'fileName', tilesPath: 'tilesPath' }); - expect(result.length).toBeGreaterThan(0); + expect(result.extent).toEqual(bbox(result.footprint.geometry)); + expect(result.footprint).not.toBeNull(); }); }); diff --git a/tests/unit/utils/utils.spec.ts b/tests/unit/utils/utils.spec.ts index 7967ab2..b30be85 100644 --- a/tests/unit/utils/utils.spec.ts +++ b/tests/unit/utils/utils.spec.ts @@ -1,9 +1,7 @@ import { TileOutputFormat, Transparency } from '@map-colonies/mc-model-types'; import { fileExtensionExtractor } from '../../../src/utils/fileutils'; -import { convertToFeature } from '../../../src/utils/geoUtils'; import { getTileOutputFormat } from '../../../src/utils/imageFormatUtil'; import { registerDefaultConfig } from '../mocks/configMock'; -import { Footprint } from '../../../src/common/interfaces'; describe('utils', () => { beforeEach(() => { @@ -38,42 +36,4 @@ describe('utils', () => { }); }); }); - - describe('geoUtil', () => { - describe('convertToFeature', () => { - it('should convert a polygon footPrint to a feature', () => { - const footPrint: Footprint = { - type: 'Polygon', - coordinates: [ - [ - [125.6, 10.1], - [125.7, 10.1], - [125.7, 10.2], - [125.6, 10.2], - [125.6, 10.1], - ], - ], - }; - - const footPrintFeature = { - type: 'Feature', - geometry: footPrint, - properties: {}, - }; - const result = convertToFeature(footPrint); - expect(result).toEqual(footPrintFeature); - }); - - it('should throw an error for unsupported footPrint type', () => { - const footPrint = { - type: 'Point', - coordinates: [], - }; - - const action = () => convertToFeature(footPrint as Footprint); - - expect(action).toThrow(); - }); - }); - }); });