feat: making dynamic tile size calculation per tile format: jpeg/png (MAPCO-2758)

config/custom-environment-variables.json

@@ -72,6 +72,10 @@
       "__name": "JPEG_TILE_ESTIMATED_SIZE_IN_BYTES",
       "__format": "number"
     },
+    "pngTileEstimatedSizeInBytes": {
+      "__name": "PNG_TILE_ESTIMATED_SIZE_IN_BYTES",
+      "__format": "number"
+    },
     "storageFactorBuffer": {
       "__name": "STORAGE_FACTOR_BUFFER",
       "__format": "number"

config/default.json

@@ -46,6 +46,7 @@
   },
   "storageEstimation": {
     "jpegTileEstimatedSizeInBytes": 12500,
+    "pngTileEstimatedSizeInBytes": 12500,
     "storageFactorBuffer": 1.25,
     "validateStorageSize": true
   },

helm/templates/configmap.yaml

@@ -32,6 +32,7 @@ data:
   DOWNLOAD_SERVER_URL: {{ .Values.rasterCommon.serviceUrls.downloadServer | quote }}
   POLLING_TIMEOUT_MS: {{ .Values.env.pollingTimeoutMS | quote }}
   JPEG_TILE_ESTIMATED_SIZE_IN_BYTES: {{ .Values.env.estimatedStorageCalculation.jpegTileEstimatedSizeInBytes | quote }}
+  PNG_TILE_ESTIMATED_SIZE_IN_BYTES: {{ .Values.env.estimatedStorageCalculation.pngTileEstimatedSizeInBytes | quote }}
   STORAGE_FACTOR_BUFFER: {{ .Values.env.estimatedStorageCalculation.storageFactorBuffer | quote }}
   VALIDATE_STORAGE_SIZE: {{ .Values.env.estimatedStorageCalculation.validateStorageSize | quote }}
 {{- end }}

helm/values.yaml

@@ -96,6 +96,7 @@ env:
     shouldResetTimeout: true
   estimatedStorageCalculation:
     jpegTileEstimatedSizeInBytes: 12500
+    pngTileEstimatedSizeInBytes: 12500
     storageFactorBuffer: 1.25
     validateStorageSize: true
 

package.json

@@ -44,7 +44,7 @@
     "@map-colonies/error-types": "^1.1.5",
     "@map-colonies/express-access-log-middleware": "^1.0.0",
     "@map-colonies/js-logger": "^0.0.5",
-    "@map-colonies/mc-model-types": "^13.4.3",
+    "@map-colonies/mc-model-types": "^14.0.1",
     "@map-colonies/mc-priority-queue": "^4.0.2",
     "@map-colonies/mc-utils": "^1.6.0",
     "@map-colonies/openapi-express-viewer": "^2.0.1",

src/common/interfaces.ts

@@ -134,6 +134,13 @@ export interface IStorageStatusResponse {
   size: number;
 }
 
+export interface IStorageEstimation {
+  jpegTileEstimatedSizeInBytes: number;
+  pngTileEstimatedSizeInBytes: number;
+  storageFactorBuffer: number;
+  validateStorageSize: boolean;
+}
+
 export type JobResponse = IJobResponse<IJobParameters, ITaskParameters>;
 export type TaskResponse = ITaskResponse<ITaskParameters>;
 export type CreateJobBody = ICreateJobBody<IJobParameters, ITaskParameters>;

src/createPackage/models/createPackageManager.ts

@@ -20,8 +20,8 @@ import { BadRequestError, InsufficientStorage } from '@map-colonies/error-types'
 import { isArray, isEmpty } from 'lodash';
 import booleanEqual from '@turf/boolean-equal';
 import { BBox2d } from '@turf/helpers/dist/js/lib/geojson';
-import { ProductType } from '@map-colonies/mc-model-types';
-import { IConfig } from '../../../src/common/interfaces';
+import { ProductType, TileOutputFormat } from '@map-colonies/mc-model-types';
+import { IConfig, IStorageEstimation } from '../../../src/common/interfaces';
 import { calculateEstimateGpkgSize, getGpkgRelativePath, getStorageStatus, getGpkgNameWithoutExt } from '../../common/utils';
 import { RasterCatalogManagerClient } from '../../clients/rasterCatalogManagerClient';
 import { DEFAULT_CRS, DEFAULT_PRIORITY, METADA_JSON_FILE_EXTENSION as METADATA_JSON_FILE_EXTENSION, SERVICES } from '../../common/constants';
@@ -55,9 +55,8 @@ export class CreatePackageManager {
 
   private readonly tilesProvider: MergerSourceType;
   private readonly gpkgsLocation: string;
-  private readonly tileEstimatedSize: number;
-  private readonly storageFactorBuffer: number;
-  private readonly validateStorageSize: boolean;
+  private readonly storageEstimation: IStorageEstimation;
+
   public constructor(
     @inject(SERVICES.CONFIG) private readonly config: IConfig,
     @inject(SERVICES.LOGGER) private readonly logger: Logger,
@@ -66,9 +65,7 @@ export class CreatePackageManager {
   ) {
     this.tilesProvider = config.get<MergerSourceType>('tilesProvider');
     this.gpkgsLocation = config.get<string>('gpkgsLocation');
-    this.tileEstimatedSize = config.get<number>('storageEstimation.jpegTileEstimatedSizeInBytes'); // todo - should be calculated on future param from request
-    this.storageFactorBuffer = config.get<number>('storageEstimation.storageFactorBuffer');
-    this.validateStorageSize = config.get<boolean>('storageEstimation.validateStorageSize');
+    this.storageEstimation = config.get<IStorageEstimation>('storageEstimation');
 
     this.tilesProvider = this.tilesProvider.toUpperCase() as MergerSourceType;
   }
@@ -82,6 +79,7 @@ export class CreatePackageManager {
     const polygon = normalizedPolygon ?? layerMetadata.footprint;
     const targetResolution = (userInput.targetResolution ?? layerMetadata.maxResolutionDeg) as number;
     const zoomLevel = degreesPerPixelToZoomLevel(targetResolution);
+    const tileEstimatedSize = this.getTileEstimatedSize(layerMetadata.tileOutputFormat as TileOutputFormat);
 
     resourceId = resourceId as string;
     version = version as string;
@@ -121,8 +119,8 @@ export class CreatePackageManager {
       batches.push(bboxToTileRange(sanitizedBbox as BBox2d, i));
     }
 
-    const estimatesGpkgSize = calculateEstimateGpkgSize(batches, this.tileEstimatedSize); // size of requested gpkg export
-    if (this.validateStorageSize) {
+    const estimatesGpkgSize = calculateEstimateGpkgSize(batches, tileEstimatedSize); // size of requested gpkg export
+    if (this.storageEstimation.validateStorageSize) {
       const isEnoughStorage = await this.validateFreeSpace(estimatesGpkgSize); // todo - on current stage, the calculation estimated by jpeg sizes
       if (!isEnoughStorage) {
         throw new InsufficientStorage(`There isn't enough free disk space to executing export`);
@@ -205,7 +203,7 @@ export class CreatePackageManager {
         otherRunningJobsSize += jobGpkgEstimatedSize;
       });
     }
-    const actualFreeSpace = storageStatus.free - otherRunningJobsSize * this.storageFactorBuffer;
+    const actualFreeSpace = storageStatus.free - otherRunningJobsSize * this.storageEstimation.storageFactorBuffer;
     this.logger.debug(`Current storage free space for gpkgs location: ${JSON.stringify({ free: actualFreeSpace, total: storageStatus.size })}`);
     return actualFreeSpace;
   }
@@ -371,4 +369,16 @@ export class CreatePackageManager {
 
     return newPolygonLarts;
   }
+
+  private getTileEstimatedSize(tileOutputFormat: TileOutputFormat): number {
+    let tileEstimatedSize;
+    if (tileOutputFormat === TileOutputFormat.JPEG) {
+      tileEstimatedSize = this.storageEstimation.jpegTileEstimatedSizeInBytes;
+    } else {
+      tileEstimatedSize = this.storageEstimation.pngTileEstimatedSizeInBytes;
+    }
+    this.logger.debug(`single tile size defined as ${tileOutputFormat} from configuration: ${tileEstimatedSize} bytes`);
+
+    return tileEstimatedSize;
+  }
 }

tests/mocks/config.ts

@@ -71,6 +71,7 @@ const registerDefaultConfig = (): void => {
     pollingTimeoutMS: 2000,
     storageEstimation: {
       jpegTileEstimatedSizeInBytes: 12500,
+      pngTileEstimatedSizeInBytes: 12500,
       storageFactorBuffer: 1.25,
       validateStorageSize: true,
     },