From 928545d1980e801dc360c1536c758a6ce77e571c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daute=20Rodr=C3=ADguez=20Rodr=C3=ADguez?= Date: Fri, 1 Apr 2022 15:59:59 +0100 Subject: [PATCH] feat: scenario features data piece exporter --- .../export-resource-pieces.adapter.spec.ts | 1 + .../export-resource-pieces.adapter.ts | 1 + .../pieces-exporters.module.ts | 9 +- .../scenario-features-data.piece-exporter.ts | 209 ++++++++++++++++++ .../scenario-features-data.ts | 32 +++ .../src/scenario-features-data.geo.entity.ts | 141 +++++++++++- ...cenario-features-preparation.geo.entity.ts | 62 ------ .../geofeatures/src/geo-feature.geo.entity.ts | 4 + 8 files changed, 391 insertions(+), 68 deletions(-) create mode 100644 api/apps/geoprocessing/src/export/pieces-exporters/scenario-features-data.piece-exporter.ts diff --git a/api/apps/api/src/modules/clone/export/adapters/export-resource-pieces.adapter.spec.ts b/api/apps/api/src/modules/clone/export/adapters/export-resource-pieces.adapter.spec.ts index 8c054d7fee..f45da1f061 100644 --- a/api/apps/api/src/modules/clone/export/adapters/export-resource-pieces.adapter.spec.ts +++ b/api/apps/api/src/modules/clone/export/adapters/export-resource-pieces.adapter.spec.ts @@ -107,6 +107,7 @@ const getFixtures = async () => { ClonePiece.ScenarioProtectedAreas, ClonePiece.ScenarioPlanningUnitsData, ClonePiece.ScenarioRunResults, + ClonePiece.ScenarioFeaturesData, ]; if (!projectExport) pieces.push(ClonePiece.ExportConfig); return pieces; diff --git a/api/apps/api/src/modules/clone/export/adapters/export-resource-pieces.adapter.ts b/api/apps/api/src/modules/clone/export/adapters/export-resource-pieces.adapter.ts index 3a49655474..1cf816d118 100644 --- a/api/apps/api/src/modules/clone/export/adapters/export-resource-pieces.adapter.ts +++ b/api/apps/api/src/modules/clone/export/adapters/export-resource-pieces.adapter.ts @@ -68,6 +68,7 @@ export class ExportResourcePiecesAdapter implements ExportResourcePieces { ExportComponent.newOne(id, ClonePiece.ScenarioProtectedAreas), ExportComponent.newOne(id, ClonePiece.ScenarioPlanningUnitsData), ExportComponent.newOne(id, ClonePiece.ScenarioRunResults), + ExportComponent.newOne(id, ClonePiece.ScenarioFeaturesData), ]; if (kind === ResourceKind.Scenario) { diff --git a/api/apps/geoprocessing/src/export/pieces-exporters/pieces-exporters.module.ts b/api/apps/geoprocessing/src/export/pieces-exporters/pieces-exporters.module.ts index d38b05d845..9ad4bd06ef 100644 --- a/api/apps/geoprocessing/src/export/pieces-exporters/pieces-exporters.module.ts +++ b/api/apps/geoprocessing/src/export/pieces-exporters/pieces-exporters.module.ts @@ -1,5 +1,7 @@ import { geoprocessingConnections } from '@marxan-geoprocessing/ormconfig'; +import { ScenarioFeaturesData } from '@marxan/features'; import { FileRepositoryModule } from '@marxan/files-repository'; +import { OutputScenariosFeaturesDataGeoEntity } from '@marxan/marxan-output'; import { Logger, Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; import { ExportConfigProjectPieceExporter } from './export-config.project-piece-exporter'; @@ -12,6 +14,7 @@ import { PlanningUnitsGridPieceExporter } from './planning-units-grid.piece-expo import { ProjectCustomFeaturesPieceExporter } from './project-custom-features.piece-exporter'; import { ProjectCustomProtectedAreasPieceExporter } from './project-custom-protected-areas.piece-exporter'; import { ProjectMetadataPieceExporter } from './project-metadata.piece-exporter'; +import { ScenarioFeaturesDataPieceExporter } from './scenario-features-data.piece-exporter'; import { ScenarioMetadataPieceExporter } from './scenario-metadata.piece-exporter'; import { ScenarioPlanningUnitsDataPieceExporter } from './scenario-planning-units-data.piece-exporter'; import { ScenarioProtectedAreasPieceExporter } from './scenario-protected-areas.piece-exporter'; @@ -21,7 +24,10 @@ import { ScenarioRunResultsPieceExporter } from './scenario-run-results.piece-ex imports: [ FileRepositoryModule, TypeOrmModule.forFeature([], geoprocessingConnections.apiDB), - TypeOrmModule.forFeature([], geoprocessingConnections.default), + TypeOrmModule.forFeature( + [ScenarioFeaturesData, OutputScenariosFeaturesDataGeoEntity], + geoprocessingConnections.default, + ), ], providers: [ ProjectMetadataPieceExporter, @@ -38,6 +44,7 @@ import { ScenarioRunResultsPieceExporter } from './scenario-run-results.piece-ex ScenarioProtectedAreasPieceExporter, ScenarioRunResultsPieceExporter, ScenarioPlanningUnitsDataPieceExporter, + ScenarioFeaturesDataPieceExporter, Logger, ], }) diff --git a/api/apps/geoprocessing/src/export/pieces-exporters/scenario-features-data.piece-exporter.ts b/api/apps/geoprocessing/src/export/pieces-exporters/scenario-features-data.piece-exporter.ts new file mode 100644 index 0000000000..39a69bc2c1 --- /dev/null +++ b/api/apps/geoprocessing/src/export/pieces-exporters/scenario-features-data.piece-exporter.ts @@ -0,0 +1,209 @@ +import { geoprocessingConnections } from '@marxan-geoprocessing/ormconfig'; +import { ClonePiece, ExportJobInput, ExportJobOutput } from '@marxan/cloning'; +import { ClonePieceUrisResolver } from '@marxan/cloning/infrastructure/clone-piece-data'; +import { + FeatureDataElement, + ScenarioFeaturesDataContent, +} from '@marxan/cloning/infrastructure/clone-piece-data/scenario-features-data'; +import { ScenarioFeaturesData } from '@marxan/features'; +import { FileRepository } from '@marxan/files-repository'; +import { OutputScenariosFeaturesDataGeoEntity } from '@marxan/marxan-output'; +import { Injectable, Logger } from '@nestjs/common'; +import { InjectEntityManager } from '@nestjs/typeorm'; +import { isLeft } from 'fp-ts/Either'; +import { Readable } from 'stream'; +import { EntityManager, In } from 'typeorm'; +import { + ExportPieceProcessor, + PieceExportProvider, +} from '../pieces/export-piece-processor'; + +type FeaturesSelectResult = { + id: string; + feature_class_name: string; + is_custom: boolean; +}; + +type FeatureDataElementWithFeatureId = FeatureDataElement & { + sfdId: string; + apiFeatureId: string; +}; + +type FeatureDataElementWithIsCustom = FeatureDataElementWithFeatureId & { + isCustom: boolean; + featureClassName: string; +}; + +@Injectable() +@PieceExportProvider() +export class ScenarioFeaturesDataPieceExporter implements ExportPieceProcessor { + constructor( + private readonly fileRepository: FileRepository, + @InjectEntityManager(geoprocessingConnections.apiDB) + private readonly apiEntityManager: EntityManager, + @InjectEntityManager(geoprocessingConnections.default) + private readonly geoprocessingEntityManager: EntityManager, + private readonly logger: Logger, + ) { + this.logger.setContext(ScenarioFeaturesDataPieceExporter.name); + } + + isSupported(piece: ClonePiece): boolean { + return piece === ClonePiece.ScenarioFeaturesData; + } + + private parseScenarioFeaturesDataToFeatureDataElementWithFeatureId( + scenarioFeaturesData: ScenarioFeaturesData[], + ): FeatureDataElementWithFeatureId[] { + return scenarioFeaturesData + .filter((sfd) => sfd.featureData.featureId) + .map((sfd) => ({ + sfdId: sfd.id, + apiFeatureId: sfd.featureData.featureId!, + currentArea: sfd.currentArea, + featureDataHash: sfd.featureData.hash, + featureId: sfd.featureId, + specificationId: sfd.specificationId, + totalArea: sfd.totalArea, + fpf: sfd.fpf, + metadata: sfd.metadata, + prop: sfd.prop, + sepNum: sfd.sepNum, + target2: sfd.target2, + target: sfd.target, + targetocc: sfd.targetocc, + featureClassName: '', + outputFeaturesData: [], + })); + } + + private getScenarioFeaturesDataWithIsCustom( + featuresDataWithFeatureId: FeatureDataElementWithFeatureId[], + features: FeaturesSelectResult[], + ): FeatureDataElementWithIsCustom[] { + const featurePropertiesById: Record< + string, + Omit + > = {}; + features.forEach(({ id, feature_class_name, is_custom }) => { + featurePropertiesById[id] = { feature_class_name, is_custom }; + }); + + return featuresDataWithFeatureId.map((el) => { + const featureProperties = featurePropertiesById[el.apiFeatureId]; + + if (!featureProperties) + throw new Error( + `Feature properties not found for feature with id ${el.apiFeatureId}`, + ); + + return { + ...el, + isCustom: featureProperties.is_custom, + featureClassName: featureProperties.feature_class_name, + }; + }); + } + + private getFileContent( + scenarioFeaturesDataWithIsCustom: FeatureDataElementWithIsCustom[], + outputScenariosFeaturesData: OutputScenariosFeaturesDataGeoEntity[], + ): ScenarioFeaturesDataContent { + const customFeaturesData: FeatureDataElement[] = []; + const platformFeaturesData: FeatureDataElement[] = []; + + scenarioFeaturesDataWithIsCustom.forEach( + ({ sfdId, isCustom, apiFeatureId, ...sfd }) => { + const outputData = outputScenariosFeaturesData.filter( + (el) => el.featureScenarioId === sfdId, + ); + + const array = isCustom ? customFeaturesData : platformFeaturesData; + array.push({ + ...sfd, + outputFeaturesData: outputData.map( + ({ id, featureScenarioId, ...rest }) => ({ + ...rest, + }), + ), + }); + }, + ); + + return { + customFeaturesData, + platformFeaturesData, + }; + } + + async run(input: ExportJobInput): Promise { + const scenarioFeaturesData = await this.geoprocessingEntityManager + .getRepository(ScenarioFeaturesData) + .find({ + where: { scenarioId: input.resourceId }, + relations: ['featureData'], + }); + const outputScenariosFeaturesData = await this.geoprocessingEntityManager + .getRepository(OutputScenariosFeaturesDataGeoEntity) + .find({ + where: { + featureScenarioId: In(scenarioFeaturesData.map((el) => el.id)), + }, + }); + + const scenarioFeaturesDataWithFeatureId = this.parseScenarioFeaturesDataToFeatureDataElementWithFeatureId( + scenarioFeaturesData, + ); + const featuresIds = [ + ...new Set( + scenarioFeaturesDataWithFeatureId.map((sfd) => sfd.apiFeatureId), + ), + ]; + + let features: FeaturesSelectResult[] = []; + + if (featuresIds.length > 0) { + features = await this.apiEntityManager + .createQueryBuilder() + .select('id, feature_class_name, is_custom') + .from('features', 'f') + .where('id IN (:...featuresIds)', { + featuresIds, + }) + .execute(); + } + + const scenarioFeaturesDataWithIsCustom = this.getScenarioFeaturesDataWithIsCustom( + scenarioFeaturesDataWithFeatureId, + features, + ); + + const fileContent = this.getFileContent( + scenarioFeaturesDataWithIsCustom, + outputScenariosFeaturesData, + ); + + const outputFile = await this.fileRepository.save( + Readable.from(JSON.stringify(fileContent)), + `json`, + ); + + if (isLeft(outputFile)) { + const errorMessage = `${ScenarioFeaturesDataPieceExporter.name} - Scenario - couldn't save file - ${outputFile.left.description}`; + this.logger.error(errorMessage); + throw new Error(errorMessage); + } + + return { + ...input, + uris: ClonePieceUrisResolver.resolveFor( + ClonePiece.ScenarioFeaturesData, + outputFile.right, + { + kind: input.resourceKind, + scenarioId: input.resourceId, + }, + ), + }; + } +} diff --git a/api/libs/cloning/src/infrastructure/clone-piece-data/scenario-features-data.ts b/api/libs/cloning/src/infrastructure/clone-piece-data/scenario-features-data.ts index d8e077c995..c2dd250048 100644 --- a/api/libs/cloning/src/infrastructure/clone-piece-data/scenario-features-data.ts +++ b/api/libs/cloning/src/infrastructure/clone-piece-data/scenario-features-data.ts @@ -3,3 +3,35 @@ export const scenarioFeaturesDataRelativePath = { projectImport: (oldScenarioId: string) => `scenarios/${oldScenarioId}/scenario-features-data.json`, }; + +export type OutputFeatureDataElement = { + runId: number; + amount?: number; + totalArea?: number; + occurrences?: number; + separation?: number; + target?: boolean; + mpm?: number; +}; + +export type FeatureDataElement = { + featureClassName: string; + featureDataHash: string; + totalArea: number; + currentArea: number; + fpf?: number; + target?: number; + prop?: number; + target2?: number; + targetocc?: number; + sepNum?: number; + metadata?: Record<'sepdistance', number | string>; + featureId: number; + specificationId: string; + outputFeaturesData: OutputFeatureDataElement[]; +}; + +export type ScenarioFeaturesDataContent = { + customFeaturesData: FeatureDataElement[]; + platformFeaturesData: FeatureDataElement[]; +}; diff --git a/api/libs/features/src/scenario-features-data.geo.entity.ts b/api/libs/features/src/scenario-features-data.geo.entity.ts index df483e9e63..c6ab2f81c0 100644 --- a/api/libs/features/src/scenario-features-data.geo.entity.ts +++ b/api/libs/features/src/scenario-features-data.geo.entity.ts @@ -1,12 +1,143 @@ -import { ApiProperty } from '@nestjs/swagger'; -import { Column, Entity } from 'typeorm'; -import { ScenarioFeaturesPreparation } from './scenario-features-preparation.geo.entity'; +import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; +import { + Column, + Entity, + JoinColumn, + OneToOne, + PrimaryGeneratedColumn, +} from 'typeorm'; +import { GeoFeatureGeometry } from '../../geofeatures/src'; +import { FeatureTag } from './domain'; @Entity(`scenario_features_data`) -export class ScenarioFeaturesData extends ScenarioFeaturesPreparation { - @ApiProperty() +export class ScenarioFeaturesData { + @PrimaryGeneratedColumn('uuid') + id!: string; + + @Column({ name: 'feature_class_id' }) + featuresDataId!: string; + + @Column({ name: 'scenario_id' }) + scenarioId!: string; + + @Column({ name: 'specification_id', type: 'uuid' }) + specificationId!: string; + + @Column({ name: 'total_area' }) + /** + * total area of the feature in the study area + */ + totalArea!: number; + + @Column({ name: 'current_pa' }) + /** + * total area of the feature which is protected within the study area + */ + currentArea!: number; + + @ApiProperty({ + description: `Feature Penalty Factor for this feature run.`, + }) + @Column() + /** + * penalty factor + */ + fpf?: number; + + @ApiProperty({ + description: `Total area space, expressed in m^2`, + }) + @Column() + /** + * target to be met for protection + */ + target?: number; + + @ApiProperty({ + description: + 'Protection target for this feature, as proportion of the conservation feature to be included in the reserve system.', + }) + @Column({ + name: `prop`, + type: `float8`, + }) + prop?: number; + + @Column({ + name: `sepnum`, + type: `float8`, + }) + sepNum?: number; + + @Column({ + name: `targetocc`, + type: `float8`, + }) + targetocc?: number; + + @Column() + /** + * not used yet + * in marxan realm you can set a secondary target for a minimum clump size for the representation of conservation features in the reserve + */ + target2?: number; + + @Column({ + name: 'metadata', + type: 'jsonb', + }) + metadata?: Record<'sepdistance', number | string>; + + @OneToOne(() => GeoFeatureGeometry, (featureData) => featureData.id) + @JoinColumn({ + name: 'feature_class_id', + referencedColumnName: 'id', + }) + featureData!: GeoFeatureGeometry; + @Column({ name: `feature_id`, }) featureId!: number; + + @ApiProperty({ + description: `0-100 (%) value of target protection coverage of all available species.`, + }) + coverageTarget!: number; + + @ApiProperty({ + description: `Equivalent of \`target\` percentage in covered area, expressed in m^2`, + }) + coverageTargetArea!: number; + + @ApiProperty({ + description: `0-100 (%) value of how many species % is protected currently.`, + }) + met!: number; + + @ApiProperty({ + description: `Equivalent of \`met\` percentage in covered area, expressed in m^2`, + }) + metArea!: number; + + @ApiProperty({ + description: `Shorthand value if current \`met\` is good enough compared to \`target\`.`, + }) + onTarget!: boolean; + + // what we expose with Extend + @ApiProperty({ + enum: FeatureTag, + }) + tag!: FeatureTag; + + @ApiPropertyOptional({ + description: `Name of the feature, for example \`Lion in Deserts\`.`, + }) + name?: string | null; + + @ApiPropertyOptional({ + description: `Description of the feature.`, + }) + description?: string | null; } diff --git a/api/libs/features/src/scenario-features-preparation.geo.entity.ts b/api/libs/features/src/scenario-features-preparation.geo.entity.ts index e7fbad1337..12f483dc97 100644 --- a/api/libs/features/src/scenario-features-preparation.geo.entity.ts +++ b/api/libs/features/src/scenario-features-preparation.geo.entity.ts @@ -1,5 +1,3 @@ -import { FeatureTag } from '@marxan/features/domain'; -import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; import { Column, Entity, PrimaryGeneratedColumn } from 'typeorm'; @Entity(`scenario_features_preparation`) @@ -28,28 +26,18 @@ export class ScenarioFeaturesPreparation { */ currentArea!: number; - @ApiProperty({ - description: `Feature Penalty Factor for this feature run.`, - }) @Column() /** * penalty factor */ fpf?: number; - @ApiProperty({ - description: `Total area space, expressed in m^2`, - }) @Column() /** * target to be met for protection */ target?: number; - @ApiProperty({ - description: - 'Protection target for this feature, as proportion of the conservation feature to be included in the reserve system.', - }) @Column({ name: `prop`, type: `float8`, @@ -80,54 +68,4 @@ export class ScenarioFeaturesPreparation { type: 'jsonb', }) metadata?: Record<'sepdistance', number | string>; - - /** - * no FK - */ - // @OneToOne(() => RemoteFeaturesData, (featureData) => featureData.id) - // @JoinColumn() - // featureData!: RemoteFeaturesData; - - // what we map - - @ApiProperty({ - description: `0-100 (%) value of target protection coverage of all available species.`, - }) - coverageTarget!: number; - - @ApiProperty({ - description: `Equivalent of \`target\` percentage in covered area, expressed in m^2`, - }) - coverageTargetArea!: number; - - @ApiProperty({ - description: `0-100 (%) value of how many species % is protected currently.`, - }) - met!: number; - - @ApiProperty({ - description: `Equivalent of \`met\` percentage in covered area, expressed in m^2`, - }) - metArea!: number; - - @ApiProperty({ - description: `Shorthand value if current \`met\` is good enough compared to \`target\`.`, - }) - onTarget!: boolean; - - // what we expose with Extend - @ApiProperty({ - enum: FeatureTag, - }) - tag!: FeatureTag; - - @ApiPropertyOptional({ - description: `Name of the feature, for example \`Lion in Deserts\`.`, - }) - name?: string | null; - - @ApiPropertyOptional({ - description: `Description of the feature.`, - }) - description?: string | null; } diff --git a/api/libs/geofeatures/src/geo-feature.geo.entity.ts b/api/libs/geofeatures/src/geo-feature.geo.entity.ts index 83def4cfa7..4fe36ffb0b 100644 --- a/api/libs/geofeatures/src/geo-feature.geo.entity.ts +++ b/api/libs/geofeatures/src/geo-feature.geo.entity.ts @@ -24,4 +24,8 @@ export class GeoFeatureGeometry { @ApiProperty() @Column('uuid', { name: 'feature_id' }) featureId?: string; + + @ApiProperty() + @Column('text', { name: 'hash', nullable: false }) + hash!: string; }