Feat/marxan 1605 solutions legacy piece importer

api/apps/geoprocessing/src/legacy-project-import/legacy-piece-importers/file-readers/spec-dat.reader.spec.ts

@@ -94,6 +94,15 @@ it('fails when spec.dat contains prop values lower than zero', async () => {
   );
 });
 
+it('fails when spec.dat contains prop values equal to zero', async () => {
+  const file = fixtures.GivenAnInvalidSpecDatFile({ prop: 0 });
+  const result = await fixtures.WhenExecutingSpecDatReader(file);
+  fixtures.ThenSpecDatReadOperationFails(
+    result,
+    /prop values should between/gi,
+  );
+});
+
 it('fails when spec.dat contains non integer spf values', async () => {
   const file = fixtures.GivenAnInvalidSpecDatFile({ spf: 'invalid spf' });
   const result = await fixtures.WhenExecutingSpecDatReader(file);
@@ -182,7 +191,7 @@ const getFixtures = async () => {
   const amountOfRows = 100;
 
   const getValidRow = (index: number = 0) =>
-    `${index}\t${index / amountOfRows}\t${
+    `${index}\t${(index + 1) / amountOfRows}\t${
       index / 10
     }\t${index}\t${index}\t${index}\t${index * 2}\t${index + 4}`;
 

api/apps/geoprocessing/src/legacy-project-import/legacy-piece-importers/file-readers/spec-dat.reader.ts

@@ -87,8 +87,8 @@ export class SpecDatReader extends DatFileReader<ReadRow, SpecDatRow> {
         result:
           this.isPropRow(propOrTarget) &&
           isNumber(propOrTarget.prop) &&
-          (propOrTarget.prop < 0 || propOrTarget.prop > 1),
-        errorMessage: 'Prop values should between [0, 1]',
+          (propOrTarget.prop <= 0 || propOrTarget.prop > 1),
+        errorMessage: 'Prop values should between (0, 1]',
       },
       {
         result:

api/apps/geoprocessing/src/legacy-project-import/legacy-piece-importers/legacy-piece-importers.module.ts

@@ -1,20 +1,27 @@
 import { GeoLegacyProjectImportFilesRepositoryModule } from '@marxan-geoprocessing/modules/legacy-project-import-files-repository';
 import { ProjectsPuEntity } from '@marxan-jobs/planning-unit-geometry';
 import { ScenarioFeaturesData } from '@marxan/features';
+import { GeoFeatureGeometry } from '@marxan/geofeatures';
+import { ScenariosOutputResultsApiEntity } from '@marxan/marxan-output';
 import {
   ScenariosPuCostDataGeo,
   ScenariosPuPaDataGeo,
 } from '@marxan/scenarios-planning-unit';
-import { ShapefilesModule } from '@marxan/shapefile-converter';
+import { FilesModule, ShapefilesModule } from '@marxan/shapefile-converter';
 import { HttpModule, Logger, Module } from '@nestjs/common';
 import { TypeOrmModule } from '@nestjs/typeorm';
-import { GeoFeatureGeometry } from '../../../../../libs/geofeatures/src';
+import { ScenarioFeaturesModule } from '../../marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/scenario-features/scenario-features.module';
+import { SolutionsReaderService } from '../../marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/solutions/output-file-parsing/solutions-reader.service';
+import { PlanningUnitSelectionCalculatorService } from '../../marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/solutions/solution-aggregation/planning-unit-selection-calculator.service';
+import { SolutionsOutputModule } from '../../marxan-sandboxed-runner/adapters-single/solutions-output/solutions-output.module';
+import { geoprocessingConnections } from '../../ormconfig';
 import { FeaturesSpecificationLegacyProjectPieceImporter } from './features-specification.legacy-piece-importer';
 import { FeaturesLegacyProjectPieceImporter } from './features.legacy-piece-importer';
 import { FileReadersModule } from './file-readers/file-readers.module';
 import { InputLegacyProjectPieceImporter } from './input.legacy-piece-importer';
 import { PlanningGridLegacyProjectPieceImporter } from './planning-grid.legacy-piece-importer';
 import { ScenarioPusDataLegacyProjectPieceImporter } from './scenarios-pus-data.legacy-piece-importer';
+import { SolutionsLegacyProjectPieceImporter } from './solutions.legacy-piece-importer';
 
 @Module({
   imports: [
@@ -25,10 +32,17 @@ import { ScenarioPusDataLegacyProjectPieceImporter } from './scenarios-pus-data.
       GeoFeatureGeometry,
       ScenarioFeaturesData,
     ]),
+    TypeOrmModule.forFeature(
+      [ScenariosOutputResultsApiEntity],
+      geoprocessingConnections.apiDB,
+    ),
     ShapefilesModule,
     FileReadersModule,
     GeoLegacyProjectImportFilesRepositoryModule,
     HttpModule,
+    FilesModule,
+    ScenarioFeaturesModule,
+    SolutionsOutputModule,
   ],
   providers: [
     Logger,
@@ -37,6 +51,9 @@ import { ScenarioPusDataLegacyProjectPieceImporter } from './scenarios-pus-data.
     FeaturesLegacyProjectPieceImporter,
     InputLegacyProjectPieceImporter,
     FeaturesSpecificationLegacyProjectPieceImporter,
+    SolutionsLegacyProjectPieceImporter,
+    SolutionsReaderService,
+    PlanningUnitSelectionCalculatorService,
   ],
 })
 export class LegacyPieceImportersModule {}

api/apps/geoprocessing/src/legacy-project-import/legacy-piece-importers/solutions.legacy-piece-importer.ts

@@ -0,0 +1,227 @@
+import {
+  LegacyProjectImportFilesRepository,
+  LegacyProjectImportFileType,
+  LegacyProjectImportJobInput,
+  LegacyProjectImportJobOutput,
+  LegacyProjectImportPiece,
+} from '@marxan/legacy-project-import';
+import {
+  OutputScenariosFeaturesDataGeoEntity,
+  OutputScenariosPuDataGeoEntity,
+  ScenariosOutputResultsApiEntity,
+} from '@marxan/marxan-output';
+import { FileService } from '@marxan/shapefile-converter';
+import { Injectable, Logger } from '@nestjs/common';
+import { InjectEntityManager } from '@nestjs/typeorm';
+import { isLeft } from 'fp-ts/lib/Either';
+import { promises, readdirSync } from 'fs';
+import { chunk } from 'lodash';
+import * as path from 'path';
+import { EntityManager } from 'typeorm';
+import {
+  ScenarioFeatureRunData,
+  ScenarioFeaturesDataService,
+} from '../../marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/scenario-features';
+import { SolutionsReaderService } from '../../marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/solutions/output-file-parsing/solutions-reader.service';
+import { PlanningUnitsSelectionState } from '../../marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/solutions/planning-unit-selection-state';
+import { PlanningUnitSelectionCalculatorService } from '../../marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/solutions/solution-aggregation/planning-unit-selection-calculator.service';
+import { ResultParserService } from '../../marxan-sandboxed-runner/adapters-single/solutions-output/result-parser.service';
+import { geoprocessingConnections } from '../../ormconfig';
+import {
+  LegacyProjectImportPieceProcessor,
+  LegacyProjectImportPieceProcessorProvider,
+} from '../pieces/legacy-project-import-piece-processor';
+
+type SolutionsFileExtension = 'dat' | 'csv' | 'txt';
+
+@Injectable()
+@LegacyProjectImportPieceProcessorProvider()
+export class SolutionsLegacyProjectPieceImporter
+  implements LegacyProjectImportPieceProcessor {
+  constructor(
+    private readonly filesRepo: LegacyProjectImportFilesRepository,
+    private readonly filesService: FileService,
+    private readonly scenarioFeaturesDataService: ScenarioFeaturesDataService,
+    private readonly solutionsReader: SolutionsReaderService,
+    private readonly planningUnitsStateCalculator: PlanningUnitSelectionCalculatorService,
+    private readonly resultParserService: ResultParserService,
+    @InjectEntityManager(geoprocessingConnections.default.name)
+    private readonly geoEntityManager: EntityManager,
+    @InjectEntityManager(geoprocessingConnections.apiDB.name)
+    private readonly apiEntityManager: EntityManager,
+    private readonly logger: Logger,
+  ) {
+    this.logger.setContext(SolutionsLegacyProjectPieceImporter.name);
+  }
+
+  isSupported(piece: LegacyProjectImportPiece): boolean {
+    return piece === LegacyProjectImportPiece.Solutions;
+  }
+
+  private logAndThrow(message: string): never {
+    this.logger.error(message);
+    throw new Error(message);
+  }
+
+  private ensureThatOutputFileExists(
+    outputFolder: string,
+    fileName: string,
+  ): SolutionsFileExtension {
+    const solutionsFileExtension: SolutionsFileExtension[] = [
+      'dat',
+      'csv',
+      'txt',
+    ];
+
+    const file = readdirSync(outputFolder, {
+      encoding: `utf8`,
+      withFileTypes: true,
+    }).find((file) => file.isFile() && file.name.startsWith(fileName));
+
+    if (!file) this.logAndThrow(`output.zip does not contain ${fileName}`);
+
+    const tokens = file.name.split('.');
+    const extension = tokens[tokens.length - 1] as SolutionsFileExtension;
+
+    if (solutionsFileExtension.includes(extension)) {
+      return extension;
+    }
+
+    this.logAndThrow(`${fileName} file has an unknown extension: ${extension}`);
+  }
+
+  private async insertOutputScenarioFeaturesData(
+    em: EntityManager,
+    records: ScenarioFeatureRunData[],
+  ): Promise<void> {
+    const chunkSize = 1000;
+
+    await Promise.all(
+      chunk(records, chunkSize).map((values) =>
+        em.insert(OutputScenariosFeaturesDataGeoEntity, values),
+      ),
+    );
+  }
+
+  private async insertOutputScenariosPuData(
+    em: EntityManager,
+    planningUnitsState: PlanningUnitsSelectionState,
+  ): Promise<void> {
+    const chunkSize = 1000;
+
+    await Promise.all(
+      chunk(Object.entries(planningUnitsState.puSelectionState), chunkSize).map(
+        (ospuChunk) => {
+          const insertValues = ospuChunk.map(([scenarioPuId, data]) => ({
+            scenarioPuId,
+            values: data.values,
+            includedCount: data.usedCount,
+          }));
+
+          return em.insert(OutputScenariosPuDataGeoEntity, insertValues);
+        },
+      ),
+    );
+  }
+
+  private async insertOutputScenariosSummaries(
+    outputSumPath: string,
+    planningUnitsState: PlanningUnitsSelectionState,
+    scenarioId: string,
+  ): Promise<void> {
+    const buffer = await promises.readFile(outputSumPath);
+    const runsSummary = buffer.toString();
+
+    const results = await this.resultParserService.parse(
+      runsSummary,
+      planningUnitsState,
+    );
+
+    await this.apiEntityManager.save(
+      ScenariosOutputResultsApiEntity,
+      results.map(({ score, cost, ...runSummary }) => ({
+        ...runSummary,
+        scoreValue: score,
+        costValue: cost,
+        scenarioId,
+      })),
+    );
+  }
+
+  async run(
+    input: LegacyProjectImportJobInput,
+  ): Promise<LegacyProjectImportJobOutput> {
+    const { files, scenarioId } = input;
+
+    const outputZip = files.find(
+      (file) => file.type === LegacyProjectImportFileType.Output,
+    );
+
+    if (!outputZip)
+      this.logAndThrow('output.zip file was not found inside input file array');
+
+    const outputZipReadableOrError = await this.filesRepo.get(
+      outputZip.location,
+    );
+
+    if (isLeft(outputZipReadableOrError))
+      this.logAndThrow('output.zip file was not found in files repo');
+
+    const origin = outputZip.location;
+    const fileName = outputZip.type;
+    const destination = path.dirname(origin);
+    const outputFolder = path.join(
+      destination,
+      path.basename(fileName, '.zip'),
+    );
+
+    await this.filesService.unzipFile(origin, fileName, destination);
+
+    const missingValuesFileNames = 'output_mv';
+    const missingValuesFileNameExtension = this.ensureThatOutputFileExists(
+      outputFolder,
+      missingValuesFileNames,
+    );
+    const legacyProjectImport = true;
+    const scenarioFeatureRunData = await this.scenarioFeaturesDataService.from(
+      outputFolder,
+      scenarioId,
+      missingValuesFileNameExtension,
+      legacyProjectImport,
+    );
+
+    const solutionsMatrixFileName = 'output_solutionsmatrix';
+    const solutionsMatrixFileNameExtension = this.ensureThatOutputFileExists(
+      outputFolder,
+      solutionsMatrixFileName,
+    );
+    const solutionsStream = await this.solutionsReader.from(
+      outputFolder,
+      scenarioId,
+      solutionsMatrixFileNameExtension,
+    );
+    const planningUnitsState = await this.planningUnitsStateCalculator.consume(
+      solutionsStream,
+    );
+
+    const outputSumFileName = 'output_sum';
+    const outputSumFileNameExtension = this.ensureThatOutputFileExists(
+      outputFolder,
+      outputSumFileName,
+    );
+    const outputSumPath = `${outputFolder}/${outputSumFileName}.${outputSumFileNameExtension}`;
+
+    this.geoEntityManager.transaction(async (em) => {
+      await this.insertOutputScenarioFeaturesData(em, scenarioFeatureRunData);
+      await this.insertOutputScenariosPuData(em, planningUnitsState);
+
+      await this.insertOutputScenariosSummaries(
+        outputSumPath,
+        planningUnitsState,
+        scenarioId,
+      );
+    });
+
+    return input;
+  }
+}

api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/scenario-features/file-reader/__snapshots__/output-line-to-data.transformer.spec.ts.snap

@@ -7,11 +7,11 @@ exports[`when piece of data has incorrect values should throw 1`] = `
  - property totalArea has failed the following constraints: isNumber 
 => [NaN]. An instance of ScenarioFeatureRunData has failed the validation:
  - property featureScenarioId has failed the following constraints: isUuid 
-=> [undefined]. An instance of ScenarioFeatureRunData has failed the validation:
+=> [invalid-uuid]. An instance of ScenarioFeatureRunData has failed the validation:
  - property occurrences has failed the following constraints: isNumber, isInt 
 => [NaN]. An instance of ScenarioFeatureRunData has failed the validation:
  - property separation has failed the following constraints: isNumber 
 => [NaN]. An instance of ScenarioFeatureRunData has failed the validation:
  - property mpm has failed the following constraints: isNumber 
-=> [NaN]. Chunk: ,a,VALUE_6,string-value-1,string-value-2,string-value-3,string-value-4,string-value-5,string-value-6,,string-value-7]
+=> [NaN]. Chunk: ,7,VALUE_7,string-value-1,string-value-2,string-value-3,string-value-4,string-value-5,string-value-6,,string-value-7]
 `;

api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/scenario-features/file-reader/mv-file-reader.ts

@@ -8,14 +8,17 @@ import { createStream as lineStream } from 'byline';
 
 @Injectable()
 export class MvFileReader {
-  from(outputDirectory: string): NodeJS.ReadableStream {
+  from(
+    outputDirectory: string,
+    extension: 'csv' | 'txt' | 'dat' = 'csv',
+  ): NodeJS.ReadableStream {
     const files = readdirSync(outputDirectory, {
       encoding: `utf8`,
       withFileTypes: true,
     })
       .filter((file) => file.isFile())
       .map((file) => {
-        const matcher = new RegExp(`^output_mv(?<runId>\\d+)\\.csv$`);
+        const matcher = new RegExp(`^output_mv(?<runId>\\d+)\\.${extension}$`);
         const matches = matcher.exec(file.name);
         if (isDefined(matches?.groups?.runId)) {
           return {

api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/scenario-features/file-reader/output-line-to-data.transformer.spec.ts

@@ -21,6 +21,14 @@ describe(`when piece of data has incorrect values`, () => {
   });
 });
 
+describe(`when piece of data has unknown values`, () => {
+  it(`ignores unknown values`, async () => {
+    await expect(
+      fixtures.resolvesTo(fixtures.withUnknownFeatureIdsOutput().pipe(sut)),
+    ).resolves.toEqual([]);
+  });
+});
+
 describe(`when every piece of data is valid`, () => {
   it(`should return parsed data`, async () => {
     const results = await fixtures.resolvesTo(
@@ -90,6 +98,10 @@ const getFixtures = async () => {
         id: `4242a3d3-0433-4f1b-b264-685f6461abcf`,
         prop: 0.5,
       },
+      7: {
+        id: `invalid-uuid`,
+        prop: 0.5,
+      },
     }),
     withValidOutput: () =>
       stream.Readable.from(
@@ -101,6 +113,15 @@ const getFixtures = async () => {
         },
       ),
     withInvalidOutput: () =>
+      stream.Readable.from(
+        `,7,VALUE_7,string-value-1,string-value-2,string-value-3,string-value-4,string-value-5,string-value-6,,string-value-7`.split(
+          `\n`,
+        ),
+        {
+          objectMode: true,
+        },
+      ),
+    withUnknownFeatureIdsOutput: () =>
       stream.Readable.from(
         `,a,VALUE_6,string-value-1,string-value-2,string-value-3,string-value-4,string-value-5,string-value-6,,string-value-7`.split(
           `\n`,