diff --git a/api/apps/geoprocessing/src/legacy-project-import/legacy-piece-importers/file-readers/spec-dat.reader.spec.ts b/api/apps/geoprocessing/src/legacy-project-import/legacy-piece-importers/file-readers/spec-dat.reader.spec.ts index 8a95506742..7f802fb70b 100644 --- a/api/apps/geoprocessing/src/legacy-project-import/legacy-piece-importers/file-readers/spec-dat.reader.spec.ts +++ b/api/apps/geoprocessing/src/legacy-project-import/legacy-piece-importers/file-readers/spec-dat.reader.spec.ts @@ -94,6 +94,15 @@ it('fails when spec.dat contains prop values lower than zero', async () => { ); }); +it('fails when spec.dat contains prop values equal to zero', async () => { + const file = fixtures.GivenAnInvalidSpecDatFile({ prop: 0 }); + const result = await fixtures.WhenExecutingSpecDatReader(file); + fixtures.ThenSpecDatReadOperationFails( + result, + /prop values should between/gi, + ); +}); + it('fails when spec.dat contains non integer spf values', async () => { const file = fixtures.GivenAnInvalidSpecDatFile({ spf: 'invalid spf' }); const result = await fixtures.WhenExecutingSpecDatReader(file); @@ -182,7 +191,7 @@ const getFixtures = async () => { const amountOfRows = 100; const getValidRow = (index: number = 0) => - `${index}\t${index / amountOfRows}\t${ + `${index}\t${(index + 1) / amountOfRows}\t${ index / 10 }\t${index}\t${index}\t${index}\t${index * 2}\t${index + 4}`; diff --git a/api/apps/geoprocessing/src/legacy-project-import/legacy-piece-importers/file-readers/spec-dat.reader.ts b/api/apps/geoprocessing/src/legacy-project-import/legacy-piece-importers/file-readers/spec-dat.reader.ts index 9256ecef77..eb1fe00651 100644 --- a/api/apps/geoprocessing/src/legacy-project-import/legacy-piece-importers/file-readers/spec-dat.reader.ts +++ b/api/apps/geoprocessing/src/legacy-project-import/legacy-piece-importers/file-readers/spec-dat.reader.ts @@ -87,8 +87,8 @@ export class SpecDatReader extends DatFileReader { result: this.isPropRow(propOrTarget) && isNumber(propOrTarget.prop) && - (propOrTarget.prop < 0 || propOrTarget.prop > 1), - errorMessage: 'Prop values should between [0, 1]', + (propOrTarget.prop <= 0 || propOrTarget.prop > 1), + errorMessage: 'Prop values should between (0, 1]', }, { result: diff --git a/api/apps/geoprocessing/src/legacy-project-import/legacy-piece-importers/legacy-piece-importers.module.ts b/api/apps/geoprocessing/src/legacy-project-import/legacy-piece-importers/legacy-piece-importers.module.ts index f680cf10d9..c083903984 100644 --- a/api/apps/geoprocessing/src/legacy-project-import/legacy-piece-importers/legacy-piece-importers.module.ts +++ b/api/apps/geoprocessing/src/legacy-project-import/legacy-piece-importers/legacy-piece-importers.module.ts @@ -1,20 +1,27 @@ import { GeoLegacyProjectImportFilesRepositoryModule } from '@marxan-geoprocessing/modules/legacy-project-import-files-repository'; import { ProjectsPuEntity } from '@marxan-jobs/planning-unit-geometry'; import { ScenarioFeaturesData } from '@marxan/features'; +import { GeoFeatureGeometry } from '@marxan/geofeatures'; +import { ScenariosOutputResultsApiEntity } from '@marxan/marxan-output'; import { ScenariosPuCostDataGeo, ScenariosPuPaDataGeo, } from '@marxan/scenarios-planning-unit'; -import { ShapefilesModule } from '@marxan/shapefile-converter'; +import { FilesModule, ShapefilesModule } from '@marxan/shapefile-converter'; import { HttpModule, Logger, Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; -import { GeoFeatureGeometry } from '../../../../../libs/geofeatures/src'; +import { ScenarioFeaturesModule } from '../../marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/scenario-features/scenario-features.module'; +import { SolutionsReaderService } from '../../marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/solutions/output-file-parsing/solutions-reader.service'; +import { PlanningUnitSelectionCalculatorService } from '../../marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/solutions/solution-aggregation/planning-unit-selection-calculator.service'; +import { SolutionsOutputModule } from '../../marxan-sandboxed-runner/adapters-single/solutions-output/solutions-output.module'; +import { geoprocessingConnections } from '../../ormconfig'; import { FeaturesSpecificationLegacyProjectPieceImporter } from './features-specification.legacy-piece-importer'; import { FeaturesLegacyProjectPieceImporter } from './features.legacy-piece-importer'; import { FileReadersModule } from './file-readers/file-readers.module'; import { InputLegacyProjectPieceImporter } from './input.legacy-piece-importer'; import { PlanningGridLegacyProjectPieceImporter } from './planning-grid.legacy-piece-importer'; import { ScenarioPusDataLegacyProjectPieceImporter } from './scenarios-pus-data.legacy-piece-importer'; +import { SolutionsLegacyProjectPieceImporter } from './solutions.legacy-piece-importer'; @Module({ imports: [ @@ -25,10 +32,17 @@ import { ScenarioPusDataLegacyProjectPieceImporter } from './scenarios-pus-data. GeoFeatureGeometry, ScenarioFeaturesData, ]), + TypeOrmModule.forFeature( + [ScenariosOutputResultsApiEntity], + geoprocessingConnections.apiDB, + ), ShapefilesModule, FileReadersModule, GeoLegacyProjectImportFilesRepositoryModule, HttpModule, + FilesModule, + ScenarioFeaturesModule, + SolutionsOutputModule, ], providers: [ Logger, @@ -37,6 +51,9 @@ import { ScenarioPusDataLegacyProjectPieceImporter } from './scenarios-pus-data. FeaturesLegacyProjectPieceImporter, InputLegacyProjectPieceImporter, FeaturesSpecificationLegacyProjectPieceImporter, + SolutionsLegacyProjectPieceImporter, + SolutionsReaderService, + PlanningUnitSelectionCalculatorService, ], }) export class LegacyPieceImportersModule {} diff --git a/api/apps/geoprocessing/src/legacy-project-import/legacy-piece-importers/solutions.legacy-piece-importer.ts b/api/apps/geoprocessing/src/legacy-project-import/legacy-piece-importers/solutions.legacy-piece-importer.ts new file mode 100644 index 0000000000..a3e3ae2f40 --- /dev/null +++ b/api/apps/geoprocessing/src/legacy-project-import/legacy-piece-importers/solutions.legacy-piece-importer.ts @@ -0,0 +1,227 @@ +import { + LegacyProjectImportFilesRepository, + LegacyProjectImportFileType, + LegacyProjectImportJobInput, + LegacyProjectImportJobOutput, + LegacyProjectImportPiece, +} from '@marxan/legacy-project-import'; +import { + OutputScenariosFeaturesDataGeoEntity, + OutputScenariosPuDataGeoEntity, + ScenariosOutputResultsApiEntity, +} from '@marxan/marxan-output'; +import { FileService } from '@marxan/shapefile-converter'; +import { Injectable, Logger } from '@nestjs/common'; +import { InjectEntityManager } from '@nestjs/typeorm'; +import { isLeft } from 'fp-ts/lib/Either'; +import { promises, readdirSync } from 'fs'; +import { chunk } from 'lodash'; +import * as path from 'path'; +import { EntityManager } from 'typeorm'; +import { + ScenarioFeatureRunData, + ScenarioFeaturesDataService, +} from '../../marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/scenario-features'; +import { SolutionsReaderService } from '../../marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/solutions/output-file-parsing/solutions-reader.service'; +import { PlanningUnitsSelectionState } from '../../marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/solutions/planning-unit-selection-state'; +import { PlanningUnitSelectionCalculatorService } from '../../marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/solutions/solution-aggregation/planning-unit-selection-calculator.service'; +import { ResultParserService } from '../../marxan-sandboxed-runner/adapters-single/solutions-output/result-parser.service'; +import { geoprocessingConnections } from '../../ormconfig'; +import { + LegacyProjectImportPieceProcessor, + LegacyProjectImportPieceProcessorProvider, +} from '../pieces/legacy-project-import-piece-processor'; + +type SolutionsFileExtension = 'dat' | 'csv' | 'txt'; + +@Injectable() +@LegacyProjectImportPieceProcessorProvider() +export class SolutionsLegacyProjectPieceImporter + implements LegacyProjectImportPieceProcessor { + constructor( + private readonly filesRepo: LegacyProjectImportFilesRepository, + private readonly filesService: FileService, + private readonly scenarioFeaturesDataService: ScenarioFeaturesDataService, + private readonly solutionsReader: SolutionsReaderService, + private readonly planningUnitsStateCalculator: PlanningUnitSelectionCalculatorService, + private readonly resultParserService: ResultParserService, + @InjectEntityManager(geoprocessingConnections.default.name) + private readonly geoEntityManager: EntityManager, + @InjectEntityManager(geoprocessingConnections.apiDB.name) + private readonly apiEntityManager: EntityManager, + private readonly logger: Logger, + ) { + this.logger.setContext(SolutionsLegacyProjectPieceImporter.name); + } + + isSupported(piece: LegacyProjectImportPiece): boolean { + return piece === LegacyProjectImportPiece.Solutions; + } + + private logAndThrow(message: string): never { + this.logger.error(message); + throw new Error(message); + } + + private ensureThatOutputFileExists( + outputFolder: string, + fileName: string, + ): SolutionsFileExtension { + const solutionsFileExtension: SolutionsFileExtension[] = [ + 'dat', + 'csv', + 'txt', + ]; + + const file = readdirSync(outputFolder, { + encoding: `utf8`, + withFileTypes: true, + }).find((file) => file.isFile() && file.name.startsWith(fileName)); + + if (!file) this.logAndThrow(`output.zip does not contain ${fileName}`); + + const tokens = file.name.split('.'); + const extension = tokens[tokens.length - 1] as SolutionsFileExtension; + + if (solutionsFileExtension.includes(extension)) { + return extension; + } + + this.logAndThrow(`${fileName} file has an unknown extension: ${extension}`); + } + + private async insertOutputScenarioFeaturesData( + em: EntityManager, + records: ScenarioFeatureRunData[], + ): Promise { + const chunkSize = 1000; + + await Promise.all( + chunk(records, chunkSize).map((values) => + em.insert(OutputScenariosFeaturesDataGeoEntity, values), + ), + ); + } + + private async insertOutputScenariosPuData( + em: EntityManager, + planningUnitsState: PlanningUnitsSelectionState, + ): Promise { + const chunkSize = 1000; + + await Promise.all( + chunk(Object.entries(planningUnitsState.puSelectionState), chunkSize).map( + (ospuChunk) => { + const insertValues = ospuChunk.map(([scenarioPuId, data]) => ({ + scenarioPuId, + values: data.values, + includedCount: data.usedCount, + })); + + return em.insert(OutputScenariosPuDataGeoEntity, insertValues); + }, + ), + ); + } + + private async insertOutputScenariosSummaries( + outputSumPath: string, + planningUnitsState: PlanningUnitsSelectionState, + scenarioId: string, + ): Promise { + const buffer = await promises.readFile(outputSumPath); + const runsSummary = buffer.toString(); + + const results = await this.resultParserService.parse( + runsSummary, + planningUnitsState, + ); + + await this.apiEntityManager.save( + ScenariosOutputResultsApiEntity, + results.map(({ score, cost, ...runSummary }) => ({ + ...runSummary, + scoreValue: score, + costValue: cost, + scenarioId, + })), + ); + } + + async run( + input: LegacyProjectImportJobInput, + ): Promise { + const { files, scenarioId } = input; + + const outputZip = files.find( + (file) => file.type === LegacyProjectImportFileType.Output, + ); + + if (!outputZip) + this.logAndThrow('output.zip file was not found inside input file array'); + + const outputZipReadableOrError = await this.filesRepo.get( + outputZip.location, + ); + + if (isLeft(outputZipReadableOrError)) + this.logAndThrow('output.zip file was not found in files repo'); + + const origin = outputZip.location; + const fileName = outputZip.type; + const destination = path.dirname(origin); + const outputFolder = path.join( + destination, + path.basename(fileName, '.zip'), + ); + + await this.filesService.unzipFile(origin, fileName, destination); + + const missingValuesFileNames = 'output_mv'; + const missingValuesFileNameExtension = this.ensureThatOutputFileExists( + outputFolder, + missingValuesFileNames, + ); + const legacyProjectImport = true; + const scenarioFeatureRunData = await this.scenarioFeaturesDataService.from( + outputFolder, + scenarioId, + missingValuesFileNameExtension, + legacyProjectImport, + ); + + const solutionsMatrixFileName = 'output_solutionsmatrix'; + const solutionsMatrixFileNameExtension = this.ensureThatOutputFileExists( + outputFolder, + solutionsMatrixFileName, + ); + const solutionsStream = await this.solutionsReader.from( + outputFolder, + scenarioId, + solutionsMatrixFileNameExtension, + ); + const planningUnitsState = await this.planningUnitsStateCalculator.consume( + solutionsStream, + ); + + const outputSumFileName = 'output_sum'; + const outputSumFileNameExtension = this.ensureThatOutputFileExists( + outputFolder, + outputSumFileName, + ); + const outputSumPath = `${outputFolder}/${outputSumFileName}.${outputSumFileNameExtension}`; + + this.geoEntityManager.transaction(async (em) => { + await this.insertOutputScenarioFeaturesData(em, scenarioFeatureRunData); + await this.insertOutputScenariosPuData(em, planningUnitsState); + + await this.insertOutputScenariosSummaries( + outputSumPath, + planningUnitsState, + scenarioId, + ); + }); + + return input; + } +} diff --git a/api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/scenario-features/file-reader/__snapshots__/output-line-to-data.transformer.spec.ts.snap b/api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/scenario-features/file-reader/__snapshots__/output-line-to-data.transformer.spec.ts.snap index 19a7c49812..a95880f82b 100644 --- a/api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/scenario-features/file-reader/__snapshots__/output-line-to-data.transformer.spec.ts.snap +++ b/api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/scenario-features/file-reader/__snapshots__/output-line-to-data.transformer.spec.ts.snap @@ -7,11 +7,11 @@ exports[`when piece of data has incorrect values should throw 1`] = ` - property totalArea has failed the following constraints: isNumber => [NaN]. An instance of ScenarioFeatureRunData has failed the validation: - property featureScenarioId has failed the following constraints: isUuid -=> [undefined]. An instance of ScenarioFeatureRunData has failed the validation: +=> [invalid-uuid]. An instance of ScenarioFeatureRunData has failed the validation: - property occurrences has failed the following constraints: isNumber, isInt => [NaN]. An instance of ScenarioFeatureRunData has failed the validation: - property separation has failed the following constraints: isNumber => [NaN]. An instance of ScenarioFeatureRunData has failed the validation: - property mpm has failed the following constraints: isNumber -=> [NaN]. Chunk: ,a,VALUE_6,string-value-1,string-value-2,string-value-3,string-value-4,string-value-5,string-value-6,,string-value-7] +=> [NaN]. Chunk: ,7,VALUE_7,string-value-1,string-value-2,string-value-3,string-value-4,string-value-5,string-value-6,,string-value-7] `; diff --git a/api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/scenario-features/file-reader/mv-file-reader.ts b/api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/scenario-features/file-reader/mv-file-reader.ts index 5d0ca82c1e..bacb5265b6 100644 --- a/api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/scenario-features/file-reader/mv-file-reader.ts +++ b/api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/scenario-features/file-reader/mv-file-reader.ts @@ -8,14 +8,17 @@ import { createStream as lineStream } from 'byline'; @Injectable() export class MvFileReader { - from(outputDirectory: string): NodeJS.ReadableStream { + from( + outputDirectory: string, + extension: 'csv' | 'txt' | 'dat' = 'csv', + ): NodeJS.ReadableStream { const files = readdirSync(outputDirectory, { encoding: `utf8`, withFileTypes: true, }) .filter((file) => file.isFile()) .map((file) => { - const matcher = new RegExp(`^output_mv(?\\d+)\\.csv$`); + const matcher = new RegExp(`^output_mv(?\\d+)\\.${extension}$`); const matches = matcher.exec(file.name); if (isDefined(matches?.groups?.runId)) { return { diff --git a/api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/scenario-features/file-reader/output-line-to-data.transformer.spec.ts b/api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/scenario-features/file-reader/output-line-to-data.transformer.spec.ts index bd6604cdd0..e3fb98ee5b 100644 --- a/api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/scenario-features/file-reader/output-line-to-data.transformer.spec.ts +++ b/api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/scenario-features/file-reader/output-line-to-data.transformer.spec.ts @@ -21,6 +21,14 @@ describe(`when piece of data has incorrect values`, () => { }); }); +describe(`when piece of data has unknown values`, () => { + it(`ignores unknown values`, async () => { + await expect( + fixtures.resolvesTo(fixtures.withUnknownFeatureIdsOutput().pipe(sut)), + ).resolves.toEqual([]); + }); +}); + describe(`when every piece of data is valid`, () => { it(`should return parsed data`, async () => { const results = await fixtures.resolvesTo( @@ -90,6 +98,10 @@ const getFixtures = async () => { id: `4242a3d3-0433-4f1b-b264-685f6461abcf`, prop: 0.5, }, + 7: { + id: `invalid-uuid`, + prop: 0.5, + }, }), withValidOutput: () => stream.Readable.from( @@ -101,6 +113,15 @@ const getFixtures = async () => { }, ), withInvalidOutput: () => + stream.Readable.from( + `,7,VALUE_7,string-value-1,string-value-2,string-value-3,string-value-4,string-value-5,string-value-6,,string-value-7`.split( + `\n`, + ), + { + objectMode: true, + }, + ), + withUnknownFeatureIdsOutput: () => stream.Readable.from( `,a,VALUE_6,string-value-1,string-value-2,string-value-3,string-value-4,string-value-5,string-value-6,,string-value-7`.split( `\n`, diff --git a/api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/scenario-features/file-reader/output-line-to-data.transformer.ts b/api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/scenario-features/file-reader/output-line-to-data.transformer.ts index 0cc97142db..d27b86b2db 100644 --- a/api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/scenario-features/file-reader/output-line-to-data.transformer.ts +++ b/api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/scenario-features/file-reader/output-line-to-data.transformer.ts @@ -35,8 +35,16 @@ export class OutputLineToDataTransformer extends Transform< targetMet, mpm, ] = chunk.split(','); - const featureScenarioId: string | undefined = this.idMap[+featureId]?.id; - const totalArea = Number(target) * (1 / this.idMap[+featureId]?.prop ?? 1); + + const mapValue = this.idMap[+featureId]; + + if (!mapValue) { + callback(null, undefined); + return; + } + + const featureScenarioId: string | undefined = mapValue.id; + const totalArea = Number(target) * (1 / mapValue.prop ?? 1); const data: ScenarioFeatureRunData = plainToClass< ScenarioFeatureRunData, ScenarioFeatureRunData @@ -53,7 +61,7 @@ export class OutputLineToDataTransformer extends Transform< runId: +runId, }); - const errors = await validateSync(data); + const errors = validateSync(data); if (errors.length > 0) { return callback( new Error( diff --git a/api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/scenario-features/id-mapper/legacy-project-import-scenario-feature-id.mapper.ts b/api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/scenario-features/id-mapper/legacy-project-import-scenario-feature-id.mapper.ts new file mode 100644 index 0000000000..f9ea66ae8c --- /dev/null +++ b/api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/scenario-features/id-mapper/legacy-project-import-scenario-feature-id.mapper.ts @@ -0,0 +1,41 @@ +import { ScenarioFeaturesData } from '@marxan/features'; +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { specDatFeatureIdPropertyKey } from '../../../../../../legacy-project-import/legacy-piece-importers/features.legacy-piece-importer'; +import { FeatureIdToScenarioFeatureData } from '../feature-id-to-scenario-feature-data'; + +@Injectable() +export class LegacyProjectImportScenarioFeatureIdMapper { + constructor( + @InjectRepository(ScenarioFeaturesData) + private readonly scenarioFeatureData: Repository, + ) {} + + async getMapping(scenarioId: string) { + const records = await this.scenarioFeatureData.find({ + select: ['id', 'prop', 'featureData'], + relations: ['featureData'], + where: { + scenarioId, + }, + }); + + return records.reduce( + (previousValue, sfd) => { + const specId = + sfd.featureData.properties?.[specDatFeatureIdPropertyKey]; + + if (typeof specId === 'number') { + previousValue[specId] = { + id: sfd.id, + prop: sfd.prop ?? 0.5, + }; + } + + return previousValue; + }, + {}, + ); + } +} diff --git a/api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/scenario-features/scenario-features-data.service.ts b/api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/scenario-features/scenario-features-data.service.ts index cf8db72a27..f84b5c85de 100644 --- a/api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/scenario-features/scenario-features-data.service.ts +++ b/api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/scenario-features/scenario-features-data.service.ts @@ -1,27 +1,41 @@ import { Injectable } from '@nestjs/common'; import { pipeline } from 'stream'; -import { ScenarioFeatureRunData } from './scenario-feature-run-data'; - -import { ScenarioFeatureIdMapper } from './id-mapper/scenario-feature-id.mapper'; +import { FeatureIdToScenarioFeatureData } from './feature-id-to-scenario-feature-data'; import { MvFileReader } from './file-reader/mv-file-reader'; import { OutputLineToDataTransformer } from './file-reader/output-line-to-data.transformer'; +import { LegacyProjectImportScenarioFeatureIdMapper } from './id-mapper/legacy-project-import-scenario-feature-id.mapper'; +import { ScenarioFeatureIdMapper } from './id-mapper/scenario-feature-id.mapper'; +import { ScenarioFeatureRunData } from './scenario-feature-run-data'; @Injectable() export class ScenarioFeaturesDataService { constructor( private readonly scenarioIdMapper: ScenarioFeatureIdMapper, + private readonly legacyProjectImportScenarioIdMapper: LegacyProjectImportScenarioFeatureIdMapper, private readonly fileReader: MvFileReader, ) {} + private async getIdMap( + scenarioId: string, + legacyProjectImport: boolean, + ): Promise { + return (legacyProjectImport + ? this.legacyProjectImportScenarioIdMapper + : this.scenarioIdMapper + ).getMapping(scenarioId); + } + async from( outputDirectory: string, scenarioId: string, + extension: 'csv' | 'txt' | 'dat' = 'csv', + legacyProjectImport: boolean = false, ): Promise { return new Promise(async (resolve, reject) => { const result: ScenarioFeatureRunData[] = []; - const idMap = await this.scenarioIdMapper.getMapping(scenarioId); + const idMap = await this.getIdMap(scenarioId, legacyProjectImport); pipeline( - this.fileReader.from(outputDirectory), + this.fileReader.from(outputDirectory, extension), new OutputLineToDataTransformer(idMap), (error) => { if (error) { @@ -29,8 +43,8 @@ export class ScenarioFeaturesDataService { } return resolve(result); }, - ).on(`data`, (data: ScenarioFeatureRunData) => { - result.push(data); + ).on(`data`, (data: ScenarioFeatureRunData | undefined) => { + if (data) result.push(data); }); }); } diff --git a/api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/scenario-features/scenario-features.module.ts b/api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/scenario-features/scenario-features.module.ts index d101f2663f..a6e3990315 100644 --- a/api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/scenario-features/scenario-features.module.ts +++ b/api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/scenario-features/scenario-features.module.ts @@ -6,6 +6,7 @@ import { OutputScenariosFeaturesDataGeoEntity } from '@marxan/marxan-output'; import { ScenarioFeatureIdMapper } from './id-mapper/scenario-feature-id.mapper'; import { ScenarioFeaturesDataService } from './scenario-features-data.service'; import { MvFileReader } from './file-reader/mv-file-reader'; +import { LegacyProjectImportScenarioFeatureIdMapper } from './id-mapper/legacy-project-import-scenario-feature-id.mapper'; @Module({ imports: [ @@ -16,6 +17,7 @@ import { MvFileReader } from './file-reader/mv-file-reader'; ], providers: [ ScenarioFeatureIdMapper, + LegacyProjectImportScenarioFeatureIdMapper, ScenarioFeaturesDataService, MvFileReader, ], diff --git a/api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/solutions/output-file-parsing/solutions-reader.service.ts b/api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/solutions/output-file-parsing/solutions-reader.service.ts index 219dd726e3..8f0a609ba8 100644 --- a/api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/solutions/output-file-parsing/solutions-reader.service.ts +++ b/api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters-single/solutions-output/geo-output/solutions/output-file-parsing/solutions-reader.service.ts @@ -24,8 +24,10 @@ export class SolutionsReaderService { async from( outputsDirectory: string, scenarioId: string, + extension: 'csv' | 'txt' | 'dat' = 'csv', ): Promise> { - const solutionsFile = outputsDirectory + `/output_solutionsmatrix.csv`; + const solutionsFile = + outputsDirectory + `/output_solutionsmatrix.${extension}`; const planningUnits = await this.scenarioPuData.findAndCount({ where: { scenarioId, diff --git a/api/libs/legacy-project-import/src/domain/legacy-project-import-piece.ts b/api/libs/legacy-project-import/src/domain/legacy-project-import-piece.ts index dc97a40101..95d9424747 100644 --- a/api/libs/legacy-project-import/src/domain/legacy-project-import-piece.ts +++ b/api/libs/legacy-project-import/src/domain/legacy-project-import-piece.ts @@ -18,7 +18,7 @@ export class LegacyProjectImportPieceOrderResolver { [LegacyProjectImportPiece.Features]: 1, [LegacyProjectImportPiece.ScenarioPusData]: 1, [LegacyProjectImportPiece.FeaturesSpecification]: 2, - [LegacyProjectImportPiece.Solutions]: 2, + [LegacyProjectImportPiece.Solutions]: 3, }; static resolveFor(