From 92f6075e688127945c31726fe897aada88f29f01 Mon Sep 17 00:00:00 2001 From: Asaf Date: Sun, 4 Dec 2022 16:05:47 +0200 Subject: [PATCH] feat: add job domain (MAPCO-2709) (#43) * feat: add job domain * fix: fix cr comments Co-authored-by: Asaf Masa --- config/custom-environment-variables.json | 1 + config/default.json | 1 + helm/templates/configmap.yaml | 1 + helm/values.yaml | 1 + package-lock.json | 14 +++++++------- package.json | 2 +- src/clients/jobManagerWrapper.ts | 3 +++ src/common/utils.ts | 10 +++++----- src/createPackage/models/createPackageManager.ts | 5 +++-- tests/mocks/config.ts | 1 + tests/mocks/data.ts | 2 ++ tests/mocks/data/mockJob.ts | 1 + tests/unit/createPackage/models/tasksModel.spec.ts | 3 ++- 13 files changed, 29 insertions(+), 16 deletions(-) diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index a368a81..9a325b2 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -52,6 +52,7 @@ }, "jobManager": { "url": "JOB_MANAGER_URL", + "jobDomain": "JOB_DOMAIN", "expirationDays": { "__name": "JOB_MANAGER_EXPIRATION_DAYS", "__format": "number" diff --git a/config/default.json b/config/default.json index b435378..f9b74c5 100644 --- a/config/default.json +++ b/config/default.json @@ -32,6 +32,7 @@ }, "jobManager": { "url": "http://job-manager-job-manager", + "jobDomain": "RASTER", "expirationDays": 30 }, "rasterCatalogManager": { diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index 12d5b4c..bae9fd9 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -22,6 +22,7 @@ data: JOB_MANAGER_URL: {{ .Values.rasterCommon.serviceUrls.jobManager | quote }} JOB_MANAGER_EXPIRATION_DAYS: {{ .Values.env.jobManager.expirationDays | quote }} RASTER_CATALOG_MANAGER_URL: {{ .Values.rasterCommon.serviceUrls.catalogManager | quote }} + JOB_DOMAIN: {{ .Values.rasterCommon.jobManagement.jobDomain | quote }} WORKER_TYPES_TILES_JOB_TYPE: {{ .Values.rasterCommon.jobManagement.exporter.jobType | quote }} WORKER_TYPES_TILES_TASK_TYPE: {{ .Values.rasterCommon.jobManagement.exporter.taskType | quote }} HTTP_RETRY_ATTEMPTS: {{ .Values.env.httpRetry.attempts | quote }} diff --git a/helm/values.yaml b/helm/values.yaml index 4310b28..9cb5ada 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -12,6 +12,7 @@ rasterCommon: jobManager: "http://job-manager-raster-dev-discrete-ingestion-db" downloadServer: "http://files-server-raster-dev-files-server" jobManagement: + jobDomain: RASTER exporter: jobType: tilesExport taskType: tilesExport diff --git a/package-lock.json b/package-lock.json index c54b520..5a1d0b5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,7 @@ "@map-colonies/express-access-log-middleware": "^0.0.3", "@map-colonies/js-logger": "^0.0.3", "@map-colonies/mc-model-types": "^13.4.2", - "@map-colonies/mc-priority-queue": "^4.0.1", + "@map-colonies/mc-priority-queue": "^4.0.2", "@map-colonies/mc-utils": "^1.5.0", "@map-colonies/openapi-express-viewer": "^2.0.1", "@map-colonies/read-pkg": "0.0.1", @@ -2903,9 +2903,9 @@ } }, "node_modules/@map-colonies/mc-priority-queue": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/@map-colonies/mc-priority-queue/-/mc-priority-queue-4.0.1.tgz", - "integrity": "sha512-Wu7xUluH8OVVShToLgZUVa3XqKj0YrfcnjhBnLn+82xhANAmj4VAg0QDxp39wxVeOK+QDc41Y6BrFx0OHOXNwQ==", + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@map-colonies/mc-priority-queue/-/mc-priority-queue-4.0.2.tgz", + "integrity": "sha512-3cvJZQDapbyx3HxyEq1djqRoAIcKpeGNahvpEIJNIHJxrJkg/Y4hVmwewcrsfrauDcLy38gav1AaxRPPzO0IeQ==", "dependencies": { "@map-colonies/mc-utils": "^1.3.0" } @@ -18019,9 +18019,9 @@ } }, "@map-colonies/mc-priority-queue": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/@map-colonies/mc-priority-queue/-/mc-priority-queue-4.0.1.tgz", - "integrity": "sha512-Wu7xUluH8OVVShToLgZUVa3XqKj0YrfcnjhBnLn+82xhANAmj4VAg0QDxp39wxVeOK+QDc41Y6BrFx0OHOXNwQ==", + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@map-colonies/mc-priority-queue/-/mc-priority-queue-4.0.2.tgz", + "integrity": "sha512-3cvJZQDapbyx3HxyEq1djqRoAIcKpeGNahvpEIJNIHJxrJkg/Y4hVmwewcrsfrauDcLy38gav1AaxRPPzO0IeQ==", "requires": { "@map-colonies/mc-utils": "^1.3.0" } diff --git a/package.json b/package.json index 412f47d..2a3a980 100644 --- a/package.json +++ b/package.json @@ -45,7 +45,7 @@ "@map-colonies/express-access-log-middleware": "^0.0.3", "@map-colonies/js-logger": "^0.0.3", "@map-colonies/mc-model-types": "^13.4.2", - "@map-colonies/mc-priority-queue": "^4.0.1", + "@map-colonies/mc-priority-queue": "^4.0.2", "@map-colonies/mc-utils": "^1.5.0", "@map-colonies/openapi-express-viewer": "^2.0.1", "@map-colonies/read-pkg": "0.0.1", diff --git a/src/clients/jobManagerWrapper.ts b/src/clients/jobManagerWrapper.ts index 8bd226e..5e41043 100644 --- a/src/clients/jobManagerWrapper.ts +++ b/src/clients/jobManagerWrapper.ts @@ -34,6 +34,7 @@ export class JobManagerWrapper extends JobManagerClient { private readonly tilesJobType: string; private readonly tilesTaskType: string; private readonly expirationDays: number; + private readonly jobDomain: string; public constructor(@inject(SERVICES.LOGGER) protected readonly logger: Logger) { super( @@ -45,6 +46,7 @@ export class JobManagerWrapper extends JobManagerClient { this.expirationDays = config.get('jobManager.expirationDays'); this.tilesJobType = config.get('workerTypes.tiles.jobType'); this.tilesTaskType = config.get('workerTypes.tiles.taskType'); + this.jobDomain = config.get('jobManager.jobDomain'); } public async create(data: IWorkerInput): Promise { @@ -56,6 +58,7 @@ export class JobManagerWrapper extends JobManagerClient { version: data.version, type: this.tilesJobType, expirationDate, + domain: this.jobDomain, parameters: { sanitizedBbox: data.sanitizedBbox, targetResolution: data.targetResolution, diff --git a/src/common/utils.ts b/src/common/utils.ts index a83033f..5e6265c 100644 --- a/src/common/utils.ts +++ b/src/common/utils.ts @@ -1,5 +1,5 @@ import { promises as fsPromise } from 'fs'; -import { join } from 'path'; +import { sep } from 'path'; import { BBox } from '@turf/turf'; import checkDiskSpace from 'check-disk-space'; import { ITileRange } from '@map-colonies/mc-utils'; @@ -22,15 +22,15 @@ export const getGpkgNameWithoutExt = (packageName: string): string => { return gpkgNameArr.join('.'); }; -export const getGpkgRelativePath = (packageName: string): string => { +export const getGpkgRelativePath = (packageName: string, separator: string = sep): string => { const packageDirectoryName = getGpkgNameWithoutExt(packageName); - const packageRelativePath = join(packageDirectoryName, packageName); + const packageRelativePath = `${packageDirectoryName}${separator}${packageName}`; return packageRelativePath; }; -export const getGpkgFullPath = (gpkgsLocation: string, packageName: string): string => { +export const getGpkgFullPath = (gpkgsLocation: string, packageName: string, separator: string = sep): string => { const packageDirectoryName = getGpkgNameWithoutExt(packageName); - const packageFullPath = join(gpkgsLocation, packageDirectoryName, packageName); + const packageFullPath = `${gpkgsLocation}${separator}${packageDirectoryName}${separator}${packageName}`; return packageFullPath; }; diff --git a/src/createPackage/models/createPackageManager.ts b/src/createPackage/models/createPackageManager.ts index 3bcb617..a1dbccc 100644 --- a/src/createPackage/models/createPackageManager.ts +++ b/src/createPackage/models/createPackageManager.ts @@ -98,7 +98,7 @@ export class CreatePackageManager { } const separator = this.getSeparator(); const packageName = generatePackageName(dbId, zoomLevel, sanitizedBbox); - const packageRelativePath = getGpkgRelativePath(packageName); + const packageRelativePath = getGpkgRelativePath(packageName, separator); const sources: IMapSource[] = [ { path: packageRelativePath, @@ -111,7 +111,7 @@ export class CreatePackageManager { }, }, { - path: (layerMetadata.id as string) + separator + (layerMetadata.displayPath as string), //tiles path + path: `${layerMetadata.id as string}${separator}${layerMetadata.displayPath as string}`, //tiles path type: this.tilesProvider, }, ]; @@ -169,6 +169,7 @@ export class CreatePackageManager { this.logger.debug(`Estimated requested gpkg size: ${estimatesGpkgSize}, Estimated free space: ${diskFreeSpace}`); return diskFreeSpace - estimatesGpkgSize >= 0; } + private getSeparator(): string { return this.tilesProvider === 'S3' ? '/' : sep; } diff --git a/tests/mocks/config.ts b/tests/mocks/config.ts index 6ac4076..39631c7 100644 --- a/tests/mocks/config.ts +++ b/tests/mocks/config.ts @@ -53,6 +53,7 @@ const registerDefaultConfig = (): void => { }, jobManager: { url: 'http://job-manager-job-manager', + jobDomain: 'testDomain', expirationDays: 30, }, rasterCatalogManager: { diff --git a/tests/mocks/data.ts b/tests/mocks/data.ts index a913d26..51cdba3 100644 --- a/tests/mocks/data.ts +++ b/tests/mocks/data.ts @@ -107,6 +107,7 @@ const completedJob: IJobResponse = { resourceId: 'string', version: '1.0', type: 'rasterTilesExporter', + domain: 'testDomain', description: '', parameters: { crs: 'EPSG:4326', @@ -173,6 +174,7 @@ const completedJob: IJobResponse = { const inProgressJob: IJobResponse = { id: 'fa3ab609-377a-4d96-bf0b-e0bb72f683b8', + domain: 'testDomain', resourceId: 'string', version: '1.0', type: 'rasterTilesExporter', diff --git a/tests/mocks/data/mockJob.ts b/tests/mocks/data/mockJob.ts index aab0392..6c9ae01 100644 --- a/tests/mocks/data/mockJob.ts +++ b/tests/mocks/data/mockJob.ts @@ -5,6 +5,7 @@ import { JobResponse } from '../../../src/common/interfaces'; export const mockJob: JobResponse = { id: 'b729f0e0-af64-4c2c-ba4e-e799e2f3df0f', resourceId: 'test', + domain: 'testDomain', version: '1.0', type: 'rasterTilesExporter', description: '', diff --git a/tests/unit/createPackage/models/tasksModel.spec.ts b/tests/unit/createPackage/models/tasksModel.spec.ts index 18cd937..347c0ec 100644 --- a/tests/unit/createPackage/models/tasksModel.spec.ts +++ b/tests/unit/createPackage/models/tasksModel.spec.ts @@ -1,3 +1,4 @@ +import { sep } from 'path'; import jsLogger from '@map-colonies/js-logger'; import { OperationStatus } from '@map-colonies/mc-priority-queue'; import { NotFoundError } from '@map-colonies/error-types'; @@ -171,7 +172,7 @@ describe('TasksManager', () => { getFileSizeSpy.mockResolvedValue(2000); const expirationTime = new Date(); const expectedCallbackData: ICallbackDataBase = { - fileUri: 'http://download-service/downloads/test/test.gpkg', + fileUri: `http://download-service/downloads/test${sep}test.gpkg`, expirationTime: expirationTime, fileSize: 2000, dbId: '880a9316-0f10-4874-92e2-a62d587a1169',