From c9f8a6435cd6e86add2108b01f596669814a5ff1 Mon Sep 17 00:00:00 2001 From: vitaligi <54726763+vitaligi@users.noreply.github.com> Date: Tue, 26 Nov 2024 14:51:34 +0200 Subject: [PATCH] feat: aggregation validation (#238) * refactor: additional regex patterns and organize props alphabetically * refactor: use validation pattern * feat: aggregation zod schema * refactor: add time validation between begin, end and current timestamp * fix: openapi sensor regex pattern * fix: file name pattern validation * style: remove extra spaces * refactor: improve error messages * refactor: removed prefix of each error message * refactor: error messages for part data zod schema --- src/models/raster/constants.ts | 26 ++++-- .../aggregationLayerMetadata.schema.ts | 92 +++++++++++++++++++ .../raster/ingestion/zod/schemas/index.ts | 3 +- .../ingestion/zod/schemas/partData.schema.ts | 75 +++++++++++---- .../inputFiles/rasterLayerInputFiles.yaml | 2 +- .../partData/rasterLayerPartData.yaml | 2 +- 6 files changed, 168 insertions(+), 32 deletions(-) create mode 100644 src/models/raster/ingestion/zod/schemas/aggregationLayerMetadata.schema.ts diff --git a/src/models/raster/constants.ts b/src/models/raster/constants.ts index 5e0e203f..bf8c7bf4 100644 --- a/src/models/raster/constants.ts +++ b/src/models/raster/constants.ts @@ -2,13 +2,14 @@ import { zoomLevelToResolutionDeg, zoomLevelToResolutionMeter } from '@map-colonies/mc-utils'; export const VALIDATIONS = { - resolutionMeter: { - min: zoomLevelToResolutionMeter(22), - max: zoomLevelToResolutionMeter(0), + boundingBox: { + pattern: '^-?(0|[1-9]\\d*)(\\.\\d*)?,-?(0|[1-9]\\d*)(\\.\\d*)?,-?(0|[1-9]\\d*)(\\.\\d*)?,-?(0|[1-9]\\d*)(\\.\\d*)?$', }, - resolutionDeg: { - min: zoomLevelToResolutionDeg(22), - max: zoomLevelToResolutionDeg(0), + classification: { + pattern: '^[0-9]$|^[1-9][0-9]$|^(100)$', + }, + fileNames: { + pattern: '^.+\\.[Gg][Pp][Kk][Gg]$', }, horizontalAccuracyCE90: { min: 0.01, @@ -20,14 +21,19 @@ export const VALIDATIONS = { productVersion: { pattern: '^[1-9]\\d*(\\.(0|[1-9]\\d?))?$', }, - classification: { - pattern: '^[0-9]$|^[1-9][0-9]$|^(100)$', + resolutionDeg: { + min: zoomLevelToResolutionDeg(22), + max: zoomLevelToResolutionDeg(0), + }, + resolutionMeter: { + min: zoomLevelToResolutionMeter(22), + max: zoomLevelToResolutionMeter(0), }, scale: { min: 0, max: 100000000, }, - fileNames: { - pattern: '^.+.[Gg][Pp][Kk][Gg]$', + sensor: { + pattern: '^(?!\\s).+(? = z + .object( + { + footprint: z.custom(), + imagingTimeBeginUTC: z.coerce.date({ message: 'Imaging time begin UTC should be a datetime' }), + imagingTimeEndUTC: z.coerce.date({ message: 'Imaging time end UTC should be a datetime' }), + maxHorizontalAccuracyCE90: z + .number({ message: 'Max horizontal accuracy CE90 should be a number' }) + .min(VALIDATIONS.horizontalAccuracyCE90.min, { + message: `Max horizontal accuracy CE90 should not be less than ${VALIDATIONS.horizontalAccuracyCE90.min}`, + }) + .max(VALIDATIONS.horizontalAccuracyCE90.max, { + message: `Max horizontal accuracy CE90 should not be larger than ${VALIDATIONS.horizontalAccuracyCE90.max}`, + }), + maxResolutionDeg: z + .number({ message: 'Max resolution degree should be a number' }) + .min(VALIDATIONS.resolutionDeg.min as number, { + message: `Max resolution degree should not be less than ${VALIDATIONS.resolutionDeg.min as number}`, + }) + .max(VALIDATIONS.resolutionDeg.max as number, { + message: `Max resolution degree should not be larger than ${VALIDATIONS.resolutionDeg.max as number}`, + }), + maxResolutionMeter: z + .number({ message: 'Max resolution meter should be a number' }) + .min(VALIDATIONS.resolutionMeter.min as number, { + message: `Max resolution meter should not be less than ${VALIDATIONS.resolutionMeter.min as number}`, + }) + .max(VALIDATIONS.resolutionMeter.max as number, { + message: `Max resolution meter should not be larger than ${VALIDATIONS.resolutionMeter.max as number}`, + }), + minHorizontalAccuracyCE90: z + .number({ message: 'Min horizontal accuracy CE90 should be a number' }) + .min(VALIDATIONS.horizontalAccuracyCE90.min, { + message: `Min horizontal accuracy CE90 should not be less than ${VALIDATIONS.horizontalAccuracyCE90.min}`, + }) + .max(VALIDATIONS.horizontalAccuracyCE90.max, { + message: `Min horizontal accuracy CE90 should not be larger than ${VALIDATIONS.horizontalAccuracyCE90.max}`, + }), + minResolutionDeg: z + .number({ message: 'Min resolution degree should be a number' }) + .min(VALIDATIONS.resolutionDeg.min as number, { + message: `Min resolution degree should not be less than ${VALIDATIONS.resolutionDeg.min as number}`, + }) + .max(VALIDATIONS.resolutionDeg.max as number, { + message: `Min resolution degree should not be larger than ${VALIDATIONS.resolutionDeg.max as number}`, + }), + minResolutionMeter: z + .number({ message: 'Min resolution meter should be a number' }) + .min(VALIDATIONS.resolutionMeter.min as number, { + message: `Min resolution meter should not be less than ${VALIDATIONS.resolutionMeter.min as number}`, + }) + .max(VALIDATIONS.resolutionMeter.max as number, { + message: `Min resolution meter should not be larger than ${VALIDATIONS.resolutionMeter.max as number}`, + }), + productBoundingBox: z.string({ message: 'Product bounding box should be a string' }).regex(new RegExp(VALIDATIONS.boundingBox.pattern), { + message: 'Product bounding box must be of the shape min_x,min_y,max_x,max_y', + }), + sensors: z + .array( + z.string({ message: 'Sensors should be an array of strings' }).regex(new RegExp(VALIDATIONS.sensor.pattern), { + message: 'Sensors should be an array with items not starting or ending with whitespace characters', + }), + { message: 'Sensors should be an array' } + ) + .min(1, { message: 'Sensors should have an array length of at least 1' }), + }, + { message: 'Layer metadata should be an object' } + ) + .strict() + .refine( + (aggregationLayerMetadata) => + aggregationLayerMetadata.imagingTimeBeginUTC <= aggregationLayerMetadata.imagingTimeEndUTC && + aggregationLayerMetadata.imagingTimeEndUTC <= new Date(), + { + message: 'Imaging time begin UTC should be less than or equal to imaging time end UTC and both less than or equal to current timestamp', + } + ) + .refine((aggregationLayerMetadata) => aggregationLayerMetadata.minHorizontalAccuracyCE90 <= aggregationLayerMetadata.maxHorizontalAccuracyCE90, { + message: 'Min horizontal accuracy CE90 should be less than or equal to max horizontal accuracy CE90', + }) + .refine((aggregationLayerMetadata) => aggregationLayerMetadata.minResolutionDeg <= aggregationLayerMetadata.maxResolutionDeg, { + message: 'Min resolution degree should be less than or equal to max resolution degree', + }) + .refine((aggregationLayerMetadata) => aggregationLayerMetadata.minResolutionMeter <= aggregationLayerMetadata.maxResolutionMeter, { + message: 'Min resolution meter should be less than or equal to max resolution meter', + }) + .describe('aggregationLayerMetadataSchema'); diff --git a/src/models/raster/ingestion/zod/schemas/index.ts b/src/models/raster/ingestion/zod/schemas/index.ts index 33dc8103..f09d1692 100644 --- a/src/models/raster/ingestion/zod/schemas/index.ts +++ b/src/models/raster/ingestion/zod/schemas/index.ts @@ -1,3 +1,4 @@ -export * from './partData.schema'; +export * from './aggregationLayerMetadata.schema'; export * from './inputFiles.schema'; export * from './metadata.schema'; +export * from './partData.schema'; diff --git a/src/models/raster/ingestion/zod/schemas/partData.schema.ts b/src/models/raster/ingestion/zod/schemas/partData.schema.ts index 72325cf3..8bdc4bee 100644 --- a/src/models/raster/ingestion/zod/schemas/partData.schema.ts +++ b/src/models/raster/ingestion/zod/schemas/partData.schema.ts @@ -1,31 +1,68 @@ /* eslint-disable @typescript-eslint/no-magic-numbers */ +import type { Polygon } from 'geojson'; import { z } from 'zod'; -import { Polygon } from 'geojson'; import { VALIDATIONS } from '../../../constants'; export const partSchema = z .object({ - sourceId: z.string().optional(), - sourceName: z.string().min(1), - description: z.string().optional(), - imagingTimeBeginUTC: z.coerce.date(), - imagingTimeEndUTC: z.coerce.date(), + sourceId: z.string({ message: 'Source id should be a string' }).optional(), + sourceName: z.string({ message: 'Source name should be a string' }).min(1, { message: 'Source name should have length of at least 1' }), + description: z.string({ message: 'Description should be a string' }).optional(), + imagingTimeBeginUTC: z.coerce.date({ message: 'Imaging time begin UTC should be a datetime' }), + imagingTimeEndUTC: z.coerce.date({ message: 'Imaging time end UTC should be a datetime' }), resolutionDegree: z - .number() - .min(VALIDATIONS.resolutionDeg.min as number) - .max(VALIDATIONS.resolutionDeg.max as number), + .number({ message: 'Resolution degree should be a number' }) + .min(VALIDATIONS.resolutionDeg.min as number, { + message: `Resolution degree should not be less than ${VALIDATIONS.resolutionDeg.min as number}`, + }) + .max(VALIDATIONS.resolutionDeg.max as number, { + message: `Resolution degree should not be larger than ${VALIDATIONS.resolutionDeg.max as number}`, + }), resolutionMeter: z - .number() - .min(VALIDATIONS.resolutionMeter.min as number) - .max(VALIDATIONS.resolutionMeter.max as number), + .number({ message: 'Resolution meter should be a number' }) + .min(VALIDATIONS.resolutionMeter.min as number, { + message: `Resolution meter should not be less than ${VALIDATIONS.resolutionMeter.min as number}`, + }) + .max(VALIDATIONS.resolutionMeter.max as number, { + message: `Resolution meter should not be larger than ${VALIDATIONS.resolutionMeter.max as number}`, + }), sourceResolutionMeter: z - .number() - .min(VALIDATIONS.resolutionMeter.min as number) - .max(VALIDATIONS.resolutionMeter.max as number), - horizontalAccuracyCE90: z.number().min(VALIDATIONS.horizontalAccuracyCE90.min).max(VALIDATIONS.horizontalAccuracyCE90.max), - sensors: z.array(z.string().regex(new RegExp('^(?! ).+(?(), }) + .refine((part) => part.imagingTimeBeginUTC <= part.imagingTimeEndUTC && part.imagingTimeEndUTC <= new Date(), { + message: 'Imaging time begin UTC should be less than or equal to imaging time end UTC and both less than or equal to current timestamp', + }) .describe('partSchema'); diff --git a/src/yaml/ingestionTrigger/inputFiles/rasterLayerInputFiles.yaml b/src/yaml/ingestionTrigger/inputFiles/rasterLayerInputFiles.yaml index 127845b0..59a6a627 100644 --- a/src/yaml/ingestionTrigger/inputFiles/rasterLayerInputFiles.yaml +++ b/src/yaml/ingestionTrigger/inputFiles/rasterLayerInputFiles.yaml @@ -21,4 +21,4 @@ components: example: ['example.gpkg'] required: - originDirectory - - fileNames + - fileNames diff --git a/src/yaml/ingestionTrigger/partData/rasterLayerPartData.yaml b/src/yaml/ingestionTrigger/partData/rasterLayerPartData.yaml index 149f5044..6ab59bcb 100644 --- a/src/yaml/ingestionTrigger/partData/rasterLayerPartData.yaml +++ b/src/yaml/ingestionTrigger/partData/rasterLayerPartData.yaml @@ -66,7 +66,7 @@ components: type: array items: type: string - pattern: ^(?! ).+(?