diff --git a/api/apps/geoprocessing/src/legacy-project-import/legacy-piece-importers/features-specification.legacy-piece-importer.ts b/api/apps/geoprocessing/src/legacy-project-import/legacy-piece-importers/features-specification.legacy-piece-importer.ts index 6373162847..a02d68611b 100644 --- a/api/apps/geoprocessing/src/legacy-project-import/legacy-piece-importers/features-specification.legacy-piece-importer.ts +++ b/api/apps/geoprocessing/src/legacy-project-import/legacy-piece-importers/features-specification.legacy-piece-importer.ts @@ -11,9 +11,10 @@ import { LegacyProjectImportPiece, } from '@marxan/legacy-project-import'; import { HttpService, HttpStatus, Injectable, Logger } from '@nestjs/common'; -import { InjectEntityManager } from '@nestjs/typeorm'; +import { InjectEntityManager, InjectRepository } from '@nestjs/typeorm'; import { Either, isLeft, left, right } from 'fp-ts/lib/Either'; -import { EntityManager, In } from 'typeorm'; +import { chunk } from 'lodash'; +import { EntityManager, In, Repository } from 'typeorm'; import { AppConfig } from '../../utils/config.utils'; import { LegacyProjectImportPieceProcessor, @@ -54,8 +55,10 @@ export class FeaturesSpecificationLegacyProjectPieceImporter private readonly puvsprDatReader: PuvsprDatReader, @InjectEntityManager(geoprocessingConnections.apiDB) private readonly apiEntityManager: EntityManager, - @InjectEntityManager(geoprocessingConnections.default) - private readonly geoprocessingEntityManager: EntityManager, + @InjectRepository(GeoFeatureGeometry) + private readonly featuresDataRepo: Repository, + @InjectRepository(ScenarioFeaturesData) + private readonly scenarioFeaturesDataRepo: Repository, private readonly logger: Logger, private readonly httpService: HttpService, ) { @@ -148,10 +151,7 @@ export class FeaturesSpecificationLegacyProjectPieceImporter ): Promise { const featureIds = await this.getProjectFeaturesIds(projectId); const featureIdByIntegerId: FeatureIdByIntegerId = {}; - const featureDataRepo = this.geoprocessingEntityManager.getRepository( - GeoFeatureGeometry, - ); - const featuresData = await featureDataRepo.find({ + const featuresData = await this.featuresDataRepo.find({ where: { featureId: In(featureIds) }, }); @@ -189,7 +189,7 @@ export class FeaturesSpecificationLegacyProjectPieceImporter ): Promise> { const timeout = left('specification timeout'); const failure = left('specification failed'); - const intervalSeconds = 1; + const intervalSeconds = 3; let triesLeft = retries; return new Promise>((resolve) => { @@ -239,19 +239,22 @@ export class FeaturesSpecificationLegacyProjectPieceImporter 'api.url', )}/api/v1/projects/import/legacy/${projectId}/specification`, { - features: specRows.map((row) => ({ - featureId: featureIdByIntegerId[row.id], - kind: 'plain', - marxanSettings: { - fpf: row.spf, - prop: row.prop, - }, - })), + features: specRows + .filter((row) => Boolean(featureIdByIntegerId[row.id])) + .map((row) => ({ + featureId: featureIdByIntegerId[row.id], + kind: 'plain', + marxanSettings: { + fpf: row.spf, + prop: row.prop, + }, + })), }, { headers: { 'x-api-key': AppConfig.get('auth.xApiKey.secret'), }, + validateStatus: () => true, }, ) .toPromise(); @@ -277,16 +280,13 @@ export class FeaturesSpecificationLegacyProjectPieceImporter puvsprRows: PuvrsprDatRow[], scenarioId: string, ): Promise { - const sfdRepo = this.geoprocessingEntityManager.getRepository( - ScenarioFeaturesData, - ); - const scenarioFeaturesData = await sfdRepo.find({ - select: ['id', 'featureData'], - where: { - scenarioId, + const scenarioFeaturesData: ScenarioFeaturesData[] = await this.scenarioFeaturesDataRepo.find( + { + select: ['id', 'featureData'], + where: { scenarioId }, + relations: ['featureData'], }, - relations: ['featureData'], - }); + ); const amountsByIntegerIdAndPuid: Record = {}; const propertiesByIntegerId: Record = {}; @@ -319,12 +319,17 @@ export class FeaturesSpecificationLegacyProjectPieceImporter target2, targetocc, sepnum, - // TODO Update amountFromLegacyProject - // amountFromLegacyProject: amount + amountFromLegacyProject: amount, }; }); - await sfdRepo.save(updateValues); + const chunkSize = 250; + + await Promise.all( + chunk(updateValues, chunkSize).map((values) => + this.scenarioFeaturesDataRepo.save(values), + ), + ); } async run( diff --git a/api/apps/geoprocessing/src/legacy-project-import/legacy-piece-importers/legacy-piece-importers.module.ts b/api/apps/geoprocessing/src/legacy-project-import/legacy-piece-importers/legacy-piece-importers.module.ts index 3b5dfc57ea..f680cf10d9 100644 --- a/api/apps/geoprocessing/src/legacy-project-import/legacy-piece-importers/legacy-piece-importers.module.ts +++ b/api/apps/geoprocessing/src/legacy-project-import/legacy-piece-importers/legacy-piece-importers.module.ts @@ -1,4 +1,6 @@ +import { GeoLegacyProjectImportFilesRepositoryModule } from '@marxan-geoprocessing/modules/legacy-project-import-files-repository'; import { ProjectsPuEntity } from '@marxan-jobs/planning-unit-geometry'; +import { ScenarioFeaturesData } from '@marxan/features'; import { ScenariosPuCostDataGeo, ScenariosPuPaDataGeo, @@ -6,13 +8,13 @@ import { import { ShapefilesModule } from '@marxan/shapefile-converter'; import { HttpModule, Logger, Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; -import { FileReadersModule } from './file-readers/file-readers.module'; +import { GeoFeatureGeometry } from '../../../../../libs/geofeatures/src'; +import { FeaturesSpecificationLegacyProjectPieceImporter } from './features-specification.legacy-piece-importer'; import { FeaturesLegacyProjectPieceImporter } from './features.legacy-piece-importer'; +import { FileReadersModule } from './file-readers/file-readers.module'; +import { InputLegacyProjectPieceImporter } from './input.legacy-piece-importer'; import { PlanningGridLegacyProjectPieceImporter } from './planning-grid.legacy-piece-importer'; import { ScenarioPusDataLegacyProjectPieceImporter } from './scenarios-pus-data.legacy-piece-importer'; -import { InputLegacyProjectPieceImporter } from './input.legacy-piece-importer'; -import { GeoLegacyProjectImportFilesRepositoryModule } from '@marxan-geoprocessing/modules/legacy-project-import-files-repository'; -import { FeaturesSpecificationLegacyProjectPieceImporter } from './features-specification.legacy-piece-importer'; @Module({ imports: [ @@ -20,6 +22,8 @@ import { FeaturesSpecificationLegacyProjectPieceImporter } from './features-spec ProjectsPuEntity, ScenariosPuPaDataGeo, ScenariosPuCostDataGeo, + GeoFeatureGeometry, + ScenarioFeaturesData, ]), ShapefilesModule, FileReadersModule, diff --git a/api/apps/geoprocessing/test/integration/legacy-project-import/features-specification.legacy-piece-importer.e2e-spec.ts b/api/apps/geoprocessing/test/integration/legacy-project-import/features-specification.legacy-piece-importer.e2e-spec.ts index c59415d56d..6289a92ee5 100644 --- a/api/apps/geoprocessing/test/integration/legacy-project-import/features-specification.legacy-piece-importer.e2e-spec.ts +++ b/api/apps/geoprocessing/test/integration/legacy-project-import/features-specification.legacy-piece-importer.e2e-spec.ts @@ -228,7 +228,7 @@ describe(FeaturesSpecificationLegacyProjectPieceImporter, () => { .WhenPieceImporterIsInvoked(job) .AndSpecificationProcessTimeouts() .ThenASpecificationDidntFinishErrorShouldBeThrown(); - }, 20_000); + }, 40_000); it(`fails if features data records does not contain required properties`, async () => { const specDatFileType = LegacyProjectImportFileType.SpecDat;