fix: features specification legacy piece importer (#1129)

Vizzuality · Jun 8, 2022 · 6ee0d7a · 6ee0d7a
1 parent 6dc95a7
commit 6ee0d7a
Show file tree

Hide file tree

Showing 3 changed files with 43 additions and 34 deletions.
diff --git a/...acy-project-import/legacy-piece-importers/features-specification.legacy-piece-importer.ts b/...acy-project-import/legacy-piece-importers/features-specification.legacy-piece-importer.ts
@@ -11,9 +11,10 @@ import {
   LegacyProjectImportPiece,
 } from '@marxan/legacy-project-import';
 import { HttpService, HttpStatus, Injectable, Logger } from '@nestjs/common';
-import { InjectEntityManager } from '@nestjs/typeorm';
+import { InjectEntityManager, InjectRepository } from '@nestjs/typeorm';
 import { Either, isLeft, left, right } from 'fp-ts/lib/Either';
-import { EntityManager, In } from 'typeorm';
+import { chunk } from 'lodash';
+import { EntityManager, In, Repository } from 'typeorm';
 import { AppConfig } from '../../utils/config.utils';
 import {
   LegacyProjectImportPieceProcessor,
@@ -54,8 +55,10 @@ export class FeaturesSpecificationLegacyProjectPieceImporter
     private readonly puvsprDatReader: PuvsprDatReader,
     @InjectEntityManager(geoprocessingConnections.apiDB)
     private readonly apiEntityManager: EntityManager,
-    @InjectEntityManager(geoprocessingConnections.default)
-    private readonly geoprocessingEntityManager: EntityManager,
+    @InjectRepository(GeoFeatureGeometry)
+    private readonly featuresDataRepo: Repository<GeoFeatureGeometry>,
+    @InjectRepository(ScenarioFeaturesData)
+    private readonly scenarioFeaturesDataRepo: Repository<ScenarioFeaturesData>,
     private readonly logger: Logger,
     private readonly httpService: HttpService,
   ) {
@@ -148,10 +151,7 @@ export class FeaturesSpecificationLegacyProjectPieceImporter
   ): Promise<FeatureIdByIntegerId> {
     const featureIds = await this.getProjectFeaturesIds(projectId);
     const featureIdByIntegerId: FeatureIdByIntegerId = {};
-    const featureDataRepo = this.geoprocessingEntityManager.getRepository(
-      GeoFeatureGeometry,
-    );
-    const featuresData = await featureDataRepo.find({
+    const featuresData = await this.featuresDataRepo.find({
       where: { featureId: In(featureIds) },
     });
 
@@ -189,7 +189,7 @@ export class FeaturesSpecificationLegacyProjectPieceImporter
   ): Promise<Either<string, true>> {
     const timeout = left('specification timeout');
     const failure = left('specification failed');
-    const intervalSeconds = 1;
+    const intervalSeconds = 3;
     let triesLeft = retries;
 
     return new Promise<Either<string, true>>((resolve) => {
@@ -239,19 +239,22 @@ export class FeaturesSpecificationLegacyProjectPieceImporter
           'api.url',
         )}/api/v1/projects/import/legacy/${projectId}/specification`,
         {
-          features: specRows.map((row) => ({
-            featureId: featureIdByIntegerId[row.id],
-            kind: 'plain',
-            marxanSettings: {
-              fpf: row.spf,
-              prop: row.prop,
-            },
-          })),
+          features: specRows
+            .filter((row) => Boolean(featureIdByIntegerId[row.id]))
+            .map((row) => ({
+              featureId: featureIdByIntegerId[row.id],
+              kind: 'plain',
+              marxanSettings: {
+                fpf: row.spf,
+                prop: row.prop,
+              },
+            })),
         },
         {
           headers: {
             'x-api-key': AppConfig.get<string>('auth.xApiKey.secret'),
           },
+          validateStatus: () => true,
         },
       )
       .toPromise();
@@ -277,16 +280,13 @@ export class FeaturesSpecificationLegacyProjectPieceImporter
     puvsprRows: PuvrsprDatRow[],
     scenarioId: string,
   ): Promise<void> {
-    const sfdRepo = this.geoprocessingEntityManager.getRepository(
-      ScenarioFeaturesData,
-    );
-    const scenarioFeaturesData = await sfdRepo.find({
-      select: ['id', 'featureData'],
-      where: {
-        scenarioId,
+    const scenarioFeaturesData: ScenarioFeaturesData[] = await this.scenarioFeaturesDataRepo.find(
+      {
+        select: ['id', 'featureData'],
+        where: { scenarioId },
+        relations: ['featureData'],
       },
-      relations: ['featureData'],
-    });
+    );
     const amountsByIntegerIdAndPuid: Record<string, number> = {};
     const propertiesByIntegerId: Record<number, PropSpecDatRow> = {};
 
@@ -319,12 +319,17 @@ export class FeaturesSpecificationLegacyProjectPieceImporter
         target2,
         targetocc,
         sepnum,
-        // TODO Update amountFromLegacyProject
-        // amountFromLegacyProject: amount
+        amountFromLegacyProject: amount,
       };
     });
 
-    await sfdRepo.save(updateValues);
+    const chunkSize = 250;
+
+    await Promise.all(
+      chunk(updateValues, chunkSize).map((values) =>
+        this.scenarioFeaturesDataRepo.save(values),
+      ),
+    );
   }
 
   async run(

diff --git a/...cessing/src/legacy-project-import/legacy-piece-importers/legacy-piece-importers.module.ts b/...cessing/src/legacy-project-import/legacy-piece-importers/legacy-piece-importers.module.ts
@@ -1,25 +1,29 @@
+import { GeoLegacyProjectImportFilesRepositoryModule } from '@marxan-geoprocessing/modules/legacy-project-import-files-repository';
 import { ProjectsPuEntity } from '@marxan-jobs/planning-unit-geometry';
+import { ScenarioFeaturesData } from '@marxan/features';
 import {
   ScenariosPuCostDataGeo,
   ScenariosPuPaDataGeo,
 } from '@marxan/scenarios-planning-unit';
 import { ShapefilesModule } from '@marxan/shapefile-converter';
 import { HttpModule, Logger, Module } from '@nestjs/common';
 import { TypeOrmModule } from '@nestjs/typeorm';
-import { FileReadersModule } from './file-readers/file-readers.module';
+import { GeoFeatureGeometry } from '../../../../../libs/geofeatures/src';
+import { FeaturesSpecificationLegacyProjectPieceImporter } from './features-specification.legacy-piece-importer';
 import { FeaturesLegacyProjectPieceImporter } from './features.legacy-piece-importer';
+import { FileReadersModule } from './file-readers/file-readers.module';
+import { InputLegacyProjectPieceImporter } from './input.legacy-piece-importer';
 import { PlanningGridLegacyProjectPieceImporter } from './planning-grid.legacy-piece-importer';
 import { ScenarioPusDataLegacyProjectPieceImporter } from './scenarios-pus-data.legacy-piece-importer';
-import { InputLegacyProjectPieceImporter } from './input.legacy-piece-importer';
-import { GeoLegacyProjectImportFilesRepositoryModule } from '@marxan-geoprocessing/modules/legacy-project-import-files-repository';
-import { FeaturesSpecificationLegacyProjectPieceImporter } from './features-specification.legacy-piece-importer';
 
 @Module({
   imports: [
     TypeOrmModule.forFeature([
       ProjectsPuEntity,
       ScenariosPuPaDataGeo,
       ScenariosPuCostDataGeo,
+      GeoFeatureGeometry,
+      ScenarioFeaturesData,
     ]),
     ShapefilesModule,
     FileReadersModule,

diff --git a/...ntegration/legacy-project-import/features-specification.legacy-piece-importer.e2e-spec.ts b/...ntegration/legacy-project-import/features-specification.legacy-piece-importer.e2e-spec.ts
@@ -228,7 +228,7 @@ describe(FeaturesSpecificationLegacyProjectPieceImporter, () => {
       .WhenPieceImporterIsInvoked(job)
       .AndSpecificationProcessTimeouts()
       .ThenASpecificationDidntFinishErrorShouldBeThrown();
-  }, 20_000);
+  }, 40_000);
 
   it(`fails if features data records does not contain required properties`, async () => {
     const specDatFileType = LegacyProjectImportFileType.SpecDat;