feat: import marxan execution metadata (#983)

api/apps/geoprocessing/src/export/pieces-exporters/marxan-execution-metadata.piece-exporter.ts

@@ -3,7 +3,7 @@ import { ClonePiece, ExportJobInput, ExportJobOutput } from '@marxan/cloning';
 import { ComponentLocation } from '@marxan/cloning/domain';
 import { ClonePieceUrisResolver } from '@marxan/cloning/infrastructure/clone-piece-data';
 import {
-  FolderZipType,
+  MarxanExecutionMetadataFolderType,
   getMarxanExecutionMetadataFolderRelativePath,
   MarxanExecutionMetadataContent,
 } from '@marxan/cloning/infrastructure/clone-piece-data/marxan-execution-metadata';
@@ -22,7 +22,7 @@ import {
 
 type FolderZipData = {
   id: string;
-  type: FolderZipType;
+  type: MarxanExecutionMetadataFolderType;
   buffer: Buffer;
 };
 
@@ -46,7 +46,7 @@ export class MarxanExecutionMetadataPieceExporter
   private async saveZipFile(
     file: Buffer,
     executionId: string,
-    type: FolderZipType,
+    type: MarxanExecutionMetadataFolderType,
   ): Promise<string> {
     const locationOrError = await this.fileRepository.save(Readable.from(file));
     if (isLeft(locationOrError)) {
@@ -104,20 +104,14 @@ export class MarxanExecutionMetadataPieceExporter
         scenarioId: input.resourceId,
       },
     );
-    const jsonFileRelativePath = jsonFileLocation.relativePath;
-
-    const foldersZipsRelativePathPrefix = jsonFileRelativePath.substring(
-      0,
-      jsonFileRelativePath.lastIndexOf('/') + 1,
-    );
 
     const folderZipsLocations = await Promise.all(
       foldersZipsData.map(async ({ buffer, id, type }) => {
         const location = await this.saveZipFile(buffer, id, type);
         const relativePath = getMarxanExecutionMetadataFolderRelativePath(
           id,
-          foldersZipsRelativePathPrefix,
           type,
+          jsonFileLocation.relativePath,
         );
 
         return new ComponentLocation(location, relativePath);

api/apps/geoprocessing/src/import/pieces-importers/marxan-execution-metadata.piece-importer.ts

@@ -0,0 +1,178 @@
+import { geoprocessingConnections } from '@marxan-geoprocessing/ormconfig';
+import { ClonePiece, ImportJobInput, ImportJobOutput } from '@marxan/cloning';
+import {
+  MarxanExecutionMetadataFolderType,
+  getMarxanExecutionMetadataFolderRelativePath,
+  MarxanExecutionMetadataContent,
+  MarxanExecutionMetadataElement,
+} from '@marxan/cloning/infrastructure/clone-piece-data/marxan-execution-metadata';
+import { FileRepository } from '@marxan/files-repository';
+import { MarxanExecutionMetadataGeoEntity } from '@marxan/marxan-output';
+import { extractFile, extractFiles } from '@marxan/utils';
+import { Injectable, Logger } from '@nestjs/common';
+import { InjectEntityManager } from '@nestjs/typeorm';
+import { isLeft } from 'fp-ts/lib/Either';
+import { Readable } from 'stream';
+import { EntityManager } from 'typeorm/entity-manager/EntityManager';
+import {
+  ImportPieceProcessor,
+  PieceImportProvider,
+} from '../pieces/import-piece-processor';
+
+type FolderData = {
+  id: string;
+  relativePath: string;
+  type: MarxanExecutionMetadataFolderType;
+};
+
+type MetadataFolderBuffers = Record<string, Buffer>;
+
+@Injectable()
+@PieceImportProvider()
+export class MarxanExecutionMetadataPieceImporter
+  implements ImportPieceProcessor {
+  constructor(
+    private readonly fileRepository: FileRepository,
+    @InjectEntityManager(geoprocessingConnections.default)
+    private readonly entityManager: EntityManager,
+    private readonly logger: Logger,
+  ) {
+    this.logger.setContext(MarxanExecutionMetadataPieceImporter.name);
+  }
+
+  isSupported(piece: ClonePiece): boolean {
+    return piece === ClonePiece.MarxanExecutionMetadata;
+  }
+
+  private async getExportFileFromFileRepo(uri: string): Promise<Readable> {
+    const readableOrError = await this.fileRepository.get(uri);
+    if (isLeft(readableOrError)) {
+      const errorMessage = `Export file is not available at ${uri}`;
+      this.logger.error(errorMessage);
+      throw new Error(errorMessage);
+    }
+
+    return readableOrError.right;
+  }
+
+  private async getMetadataFolders(
+    exportFileUri: string,
+    marxanExecutionMetadata: MarxanExecutionMetadataElement[],
+    foldersZipsRelativePathPrefix: string,
+  ): Promise<MetadataFolderBuffers> {
+    const metadataFoldersBuffers: MetadataFolderBuffers = {};
+    const foldersData = marxanExecutionMetadata.flatMap((metadata) => {
+      const relativePaths: FolderData[] = [
+        {
+          id: metadata.id,
+          type: 'input',
+          relativePath: getMarxanExecutionMetadataFolderRelativePath(
+            metadata.id,
+            'input',
+            foldersZipsRelativePathPrefix,
+          ),
+        },
+      ];
+
+      if (metadata.includesOutputFolder)
+        relativePaths.push({
+          id: metadata.id,
+          type: 'output',
+          relativePath: getMarxanExecutionMetadataFolderRelativePath(
+            metadata.id,
+            'output',
+            foldersZipsRelativePathPrefix,
+          ),
+        });
+
+      return relativePaths;
+    });
+
+    const exportFile = await this.getExportFileFromFileRepo(exportFileUri);
+
+    const buffersOrErrors = await extractFiles(
+      exportFile,
+      foldersData.map((data) => data.relativePath),
+    );
+
+    if (isLeft(buffersOrErrors)) {
+      const errorMessage = `Error extracting input and output folders of marxan execution metadata`;
+      this.logger.error(errorMessage);
+      throw new Error(errorMessage);
+    }
+
+    foldersData.forEach((data) => {
+      const buffer = buffersOrErrors.right[data.relativePath];
+
+      if (!buffer) {
+        const errorMessage = `Error extracting ${data.type} folder of execution metadata with id ${data.id}`;
+        this.logger.error(errorMessage);
+        throw new Error(errorMessage);
+      }
+
+      metadataFoldersBuffers[`${data.id}-${data.type}`] = buffer;
+    });
+
+    return metadataFoldersBuffers;
+  }
+
+  async run(input: ImportJobInput): Promise<ImportJobOutput> {
+    const { uris, projectId, pieceResourceId: scenarioId } = input;
+
+    const marxanExecutionMetadataRepo = this.entityManager.getRepository(
+      MarxanExecutionMetadataGeoEntity,
+    );
+
+    if (uris.length !== 1) {
+      const errorMessage = `uris array has an unexpected amount of elements: ${uris.length}`;
+      this.logger.error(errorMessage);
+      throw new Error(errorMessage);
+    }
+    const [marxanExecutionMetadataLocation] = uris;
+
+    const exportFile = await this.getExportFileFromFileRepo(
+      marxanExecutionMetadataLocation.uri,
+    );
+
+    const marxanExecutionMetadataOrError = await extractFile(
+      exportFile,
+      marxanExecutionMetadataLocation.relativePath,
+    );
+    if (isLeft(marxanExecutionMetadataOrError)) {
+      const errorMessage = `Marxan execution metadata file extraction failed: ${marxanExecutionMetadataLocation.relativePath}`;
+      this.logger.error(errorMessage);
+      throw new Error(errorMessage);
+    }
+
+    const {
+      marxanExecutionMetadata,
+    }: MarxanExecutionMetadataContent = JSON.parse(
+      marxanExecutionMetadataOrError.right,
+    );
+
+    const buffers = await this.getMetadataFolders(
+      marxanExecutionMetadataLocation.uri,
+      marxanExecutionMetadata,
+      marxanExecutionMetadataLocation.relativePath,
+    );
+
+    await marxanExecutionMetadataRepo.save(
+      marxanExecutionMetadata.map((metadata) => ({
+        scenarioId,
+        stdOutput: metadata.stdOutput,
+        stdError: metadata.stdError,
+        inputZip: buffers[`${metadata.id}-input`],
+        outputZip: buffers[`${metadata.id}-output`],
+        failed: metadata.failed,
+      })),
+    );
+
+    return {
+      importId: input.importId,
+      componentId: input.componentId,
+      pieceResourceId: scenarioId,
+      projectId,
+      piece: input.piece,
+    };
+  }
+}

api/apps/geoprocessing/src/import/pieces-importers/pieces-importers.module.ts

@@ -2,6 +2,7 @@ import { geoprocessingConnections } from '@marxan-geoprocessing/ormconfig';
 import { FileRepositoryModule } from '@marxan/files-repository';
 import { Logger, Module, Scope } from '@nestjs/common';
 import { TypeOrmModule } from '@nestjs/typeorm';
+import { MarxanExecutionMetadataPieceImporter } from './marxan-execution-metadata.piece-importer';
 import { PlanningAreaCustomPieceImporter } from './planning-area-custom.piece-importer';
 import { PlanningAreaGadmPieceImporter } from './planning-area-gadm.piece-importer';
 import { PlanningUnitsGridPieceImporter } from './planning-units-grid.piece-importer';
@@ -33,6 +34,7 @@ import { ScenarioRunResultsPieceImporter } from './scenario-run-results.piece-im
     ScenarioRunResultsPieceImporter,
     ScenarioFeaturesDataPieceImporter,
     ScenarioFeaturesSpecificationPieceImporter,
+    MarxanExecutionMetadataPieceImporter,
     { provide: Logger, useClass: Logger, scope: Scope.TRANSIENT },
   ],
 })