feat(marxan-run): return parsed output

api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters/solutions-output/metadata/metadata-exection-metadata.repository.ts

@@ -32,8 +32,8 @@ export class MarxanExecutionMetadataRepository {
         scenarioId,
         stdOutput: metaData.stdOutput.toString(),
         stdError: metaData.stdErr?.toString(),
-        outputZip: readFileSync(inputArchivePath),
-        inputZip: readFileSync(outputArchivePath),
+        outputZip: readFileSync(outputArchivePath),
+        inputZip: readFileSync(inputArchivePath),
       }),
     );
   }

api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters/solutions-output/result-parser.service.spec.ts

@@ -0,0 +1,144 @@
+import { ResultParserService } from './result-parser.service';
+import { Test } from '@nestjs/testing';
+
+let sut: ResultParserService;
+
+beforeEach(async () => {
+  const sandbox = await Test.createTestingModule({
+    providers: [ResultParserService],
+  }).compile();
+
+  sut = sandbox.get(ResultParserService);
+});
+
+describe(`given empty content`, () => {
+  it(`should return empty array`, () => {
+    expect(sut.parse('')).toEqual([]);
+  });
+});
+
+describe(`given headers only`, () => {
+  it(`should return empty array`, () => {
+    expect(sut.parse('one,two,three')).toEqual([]);
+  });
+});
+
+describe(`given invalid data in a row (2.1 planning units)`, () => {
+  it(`should throw an error`, () => {
+    expect(() =>
+      sut.parse(`headers...
+1,16640,640,2.1,16000,5.1664e+07,0,16000,5.1648e+07,0,0,0,0,1
+
+    `),
+    ).toThrow(
+      `Unexpected values in Marxan output at value [0]: [1,16640,640,2.1,16000,5.1664e+07,0,16000,5.1648e+07,0,0,0,0,1]`,
+    );
+  });
+});
+
+describe(`given data`, () => {
+  it(`should return parsed values`, () => {
+    expect(sut.parse(content)).toMatchInlineSnapshot(`
+      Array [
+        ResultRow {
+          "best": false,
+          "connectivity": 16000,
+          "connectivityEdge": 16000,
+          "connectivityIn": 0,
+          "connectivityInFraction": 0,
+          "connectivityOut": 51648000,
+          "connectivityTotal": 51664000,
+          "cost": 640,
+          "missingValues": 0,
+          "mostDifferent": false,
+          "mpm": 1,
+          "penalty": 0,
+          "planningUnits": 2,
+          "runId": 1,
+          "score": 16640,
+          "shortfall": 0,
+        },
+        ResultRow {
+          "best": false,
+          "connectivity": 8000,
+          "connectivityEdge": 8000,
+          "connectivityIn": 0,
+          "connectivityInFraction": 0,
+          "connectivityOut": 51656000,
+          "connectivityTotal": 51664000,
+          "cost": 400,
+          "missingValues": 1,
+          "mostDifferent": false,
+          "mpm": 0,
+          "penalty": 8240,
+          "planningUnits": 1,
+          "runId": 2,
+          "score": 16640,
+          "shortfall": 0.5,
+        },
+        ResultRow {
+          "best": false,
+          "connectivity": 12000,
+          "connectivityEdge": 12000,
+          "connectivityIn": 2000,
+          "connectivityInFraction": 0.0000387117,
+          "connectivityOut": 51650000,
+          "connectivityTotal": 51664000,
+          "cost": 800,
+          "missingValues": 1,
+          "mostDifferent": false,
+          "mpm": 0,
+          "penalty": 8240,
+          "planningUnits": 2,
+          "runId": 3,
+          "score": 21040,
+          "shortfall": 0.5,
+        },
+        ResultRow {
+          "best": false,
+          "connectivity": 16000,
+          "connectivityEdge": 16000,
+          "connectivityIn": 0,
+          "connectivityInFraction": 0,
+          "connectivityOut": 51648000,
+          "connectivityTotal": 51664000,
+          "cost": 640,
+          "missingValues": 0,
+          "mostDifferent": false,
+          "mpm": 1,
+          "penalty": 0,
+          "planningUnits": 2,
+          "runId": 4,
+          "score": 16640,
+          "shortfall": 0,
+        },
+        ResultRow {
+          "best": false,
+          "connectivity": 32000,
+          "connectivityEdge": 32000,
+          "connectivityIn": 4000,
+          "connectivityInFraction": 0.0000774234,
+          "connectivityOut": 51628000,
+          "connectivityTotal": 51664000,
+          "cost": 1840,
+          "missingValues": 0,
+          "mostDifferent": false,
+          "mpm": 1,
+          "penalty": 0,
+          "planningUnits": 5,
+          "runId": 5,
+          "score": 33840,
+          "shortfall": 0,
+        },
+      ]
+    `);
+  });
+});
+
+const content = `"Run_Number","Score","Cost","Planning_Units","Connectivity","Connectivity_Total","Connectivity_In","Connectivity_Edge","Connectivity_Out","Connectivity_In_Fraction","Penalty","Shortfall","Missing_Values","MPM"
+1,16640,640,2,16000,5.1664e+07,0,16000,5.1648e+07,0,0,0,0,1
+2,16640,400,1,8000,5.1664e+07,0,8000,5.1656e+07,0,8240,0.5,1,0
+3,21040,800,2,12000,5.1664e+07,2000,12000,5.165e+07,3.87117e-05,8240,0.5,1,0
+4,16640,640,2,16000,5.1664e+07,0,16000,5.1648e+07,0,0,0,0,1
+5,33840,1840,5,32000,5.1664e+07,4000,32000,5.1628e+07,7.74234e-05,0,0,0,1
+`;

api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters/solutions-output/result-parser.service.ts

@@ -0,0 +1,60 @@
+import { Injectable } from '@nestjs/common';
+import { plainToClass } from 'class-transformer';
+import { ExecutionResult, ResultRow } from '@marxan/marxan-output';
+import { isDefined } from '@marxan/utils';
+import { validateSync } from 'class-validator';
+
+@Injectable()
+export class ResultParserService {
+  parse(csvContent: string): ExecutionResult {
+    return csvContent
+      .split('\n')
+      .slice(1)
+      .map((row, index) => {
+        if (row === '') {
+          return;
+        }
+        const [
+          runId,
+          score,
+          cost,
+          planningUnits,
+          connectivity,
+          connectivityTotal,
+          connectivityIn,
+          connectivityEdge,
+          connectivityOut,
+          connectivityInFraction,
+          penalty,
+          shortfall,
+          missingValues,
+          mpm,
+        ] = row.split(',');
+        const entry = plainToClass<ResultRow, ResultRow>(ResultRow, {
+          runId: +runId,
+          score: +score,
+          cost: +cost,
+          planningUnits: +planningUnits,
+          connectivity: +connectivity,
+          connectivityTotal: +connectivityTotal,
+          connectivityIn: +connectivityIn,
+          connectivityEdge: +connectivityEdge,
+          connectivityOut: +connectivityOut,
+          connectivityInFraction: +connectivityInFraction,
+          penalty: +penalty,
+          shortfall: +shortfall,
+          missingValues: +missingValues,
+          mpm: +mpm,
+          best: false,
+          mostDifferent: false,
+        });
+        if (validateSync(entry).length > 0) {
+          throw new Error(
+            `Unexpected values in Marxan output at value [${index}]: [${row}]`,
+          );
+        }
+        return entry;
+      })
+      .filter(isDefined);
+  }
+}

api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters/solutions-output/solutions-output.module.ts

@@ -1,7 +1,10 @@
 import { Module } from '@nestjs/common';
 import { MarxanExecutionMetadataModule } from './metadata';
+import { ResultParserService } from './result-parser.service';
 
 @Module({
   imports: [MarxanExecutionMetadataModule],
+  providers: [ResultParserService],
+  exports: [ResultParserService],
 })
 export class SolutionsOutputModule {}

api/apps/geoprocessing/src/marxan-sandboxed-runner/adapters/solutions-output/solutions-output.service.ts

@@ -1,29 +1,18 @@
 import { Injectable } from '@nestjs/common';
-
 import { existsSync, promises } from 'fs';
-import { InjectRepository } from '@nestjs/typeorm';
-import { Repository } from 'typeorm';
+
+import { ExecutionResult } from '@marxan/marxan-output';
 import { Workspace } from '../../ports/workspace';
 import { Cancellable } from '../../ports/cancellable';
 
 import { MarxanExecutionMetadataRepository } from './metadata';
-import { ScenariosOutputResultsApiEntity } from '@marxan/scenarios-planning-unit';
-import { geoprocessingConnections } from '@marxan-geoprocessing/ormconfig';
+import { ResultParserService } from './result-parser.service';
 
 @Injectable()
 export class SolutionsOutputService implements Cancellable {
-  /**
-   * load entities
-   * file streamers
-   * ...
-   */
   constructor(
-    @InjectRepository(
-      ScenariosOutputResultsApiEntity,
-      geoprocessingConnections.apiDB.name,
-    )
-    private readonly resultsRepo: Repository<ScenariosOutputResultsApiEntity>,
     private readonly metadataRepository: MarxanExecutionMetadataRepository,
+    private readonly resultParserService: ResultParserService,
   ) {
     //
   }
@@ -38,59 +27,29 @@ export class SolutionsOutputService implements Cancellable {
    * ScenariosPuOutputGeoEntity
    *
    */
-  async saveFrom(
+  async dump(
     workspace: Workspace,
     scenarioId: string,
     stdOutput: string[],
     stdErr?: string[],
-  ): Promise<void> {
+  ): Promise<ExecutionResult> {
     if (!existsSync(workspace.workingDirectory + `/output/output_sum.csv`)) {
       throw new Error(`Output is missing from the marxan run.`);
     }
     await this.metadataRepository.save(scenarioId, workspace, {
       stdOutput,
       stdErr,
     });
+    const runsSummary = (
+      await promises.readFile(
+        workspace.workingDirectory + `/output/output_sum.csv`,
+      )
+    ).toString();
 
-    // just a sample for brevity, ideally should stream into db tables & use csv streamer
-    // const runsSummary = (
-    //   await promises.readFile(
-    //     workspace.workingDirectory + `/output/output_sum.csv`,
-    //   )
-    // ).toString();
-    // await this.resultsRepo.save(
-    //   runsSummary
-    //     .split('\n')
-    //     .slice(1)
-    //     .map((row) => {
-    //       const [
-    //         runId,
-    //         score,
-    //         cost,
-    //         planningUnits,
-    //         connectivity,
-    //         connectivityTotal,
-    //         connectivityIn,
-    //         connectivityEdge,
-    //         connectivityOut,
-    //         connectivityInFraction,
-    //         penalty,
-    //         shortfall,
-    //         missingValues,
-    //         mpm,
-    //       ] = row.split(',');
-    //       return this.resultsRepo.create({
-    //         scenarioId,
-    //         scoreValue: +score,
-    //       });
-    //     }),
-    // );
-
-    return;
+    return this.resultParserService.parse(runsSummary);
   }
 
-  cancel(): Promise<void> {
-    // TODO if streaming will be involved, can be interrupted
-    return Promise.resolve(undefined);
+  async cancel(): Promise<void> {
+    return;
   }
 }

api/apps/geoprocessing/src/marxan-sandboxed-runner/marxan-sandbox-runner.service.ts

@@ -2,6 +2,8 @@ import { Injectable } from '@nestjs/common';
 import { ModuleRef } from '@nestjs/core';
 import AbortController from 'abort-controller';
 
+import { ExecutionResult } from '@marxan/marxan-output';
+
 import { MarxanRun } from './marxan-run';
 import { WorkspaceBuilder } from './ports/workspace-builder';
 import { Cancellable } from './ports/cancellable';
@@ -24,7 +26,7 @@ export class MarxanSandboxRunnerService {
     }
   }
 
-  async run(forScenarioId: string, assets: Assets): Promise<void> {
+  async run(forScenarioId: string, assets: Assets): Promise<ExecutionResult> {
     const workspace = await this.workspaceService.get();
     const inputFiles = await this.moduleRef.create(InputFilesFs);
     const outputFilesRepository = await this.moduleRef.create(
@@ -66,14 +68,14 @@ export class MarxanSandboxRunnerService {
       marxanRun.on('finished', async () => {
         try {
           await interruptIfKilled();
-          await outputFilesRepository.saveFrom(
+          const output = await outputFilesRepository.dump(
             workspace,
             forScenarioId,
             marxanRun.stdOut,
             [],
           );
           await workspace.cleanup();
-          resolve();
+          resolve(output);
         } catch (error) {
           reject(error);
         } finally {

api/apps/geoprocessing/src/marxan-sandboxed-runner/marxan-sandboxed-runner.module.ts

@@ -1,7 +1,4 @@
 import { HttpModule, Module } from '@nestjs/common';
-import { TypeOrmModule } from '@nestjs/typeorm';
-
-import { ScenariosOutputResultsApiEntity } from '@marxan/scenarios-planning-unit';
 import { MarxanConfig } from './marxan-config';
 import { MarxanSandboxRunnerService } from './marxan-sandbox-runner.service';
 
@@ -11,17 +8,14 @@ import { MarxanExecutionMetadataModule } from './adapters/solutions-output/metad
 import { FileReader } from './adapters/file-reader';
 import { AssetFetcher } from './adapters/scenario-data/asset-fetcher';
 import { FetchConfig } from './adapters/scenario-data/fetch.config';
-import { geoprocessingConnections } from '@marxan-geoprocessing/ormconfig';
+import { SolutionsOutputModule } from './adapters/solutions-output/solutions-output.module';
 
 @Module({
   imports: [
     HttpModule,
     WorkspaceModule,
-    TypeOrmModule.forFeature(
-      [ScenariosOutputResultsApiEntity],
-      geoprocessingConnections.apiDB,
-    ),
     MarxanExecutionMetadataModule,
+    SolutionsOutputModule,
   ],
   providers: [
     MarxanConfig,