feat(api): scenarios: cost-sufrace marxan data

* feat(api): scenarios: cost-sufrace marxan data * chore(api): scenario-cost-surface * refactor(api): use relative import inside module
Vizzuality · Jun 17, 2021 · 0696ccb · 0696ccb
1 parent 73fce92
commit 0696ccb
Show file tree

Hide file tree

Showing 17 changed files with 286 additions and 14 deletions.
diff --git a/api/apps/api/src/modules/scenarios/cost-surface-readmodel/cost-surface-view.module.ts b/api/apps/api/src/modules/scenarios/cost-surface-readmodel/cost-surface-view.module.ts
@@ -0,0 +1,25 @@
+import { Module } from '@nestjs/common';
+import { CostSurfaceViewService } from './cost-surface-view.service';
+import { TypeOrmModule } from '@nestjs/typeorm';
+import { PlanningUnitsGeom } from '@marxan-jobs/planning-unit-geometry';
+import {
+  ScenariosPlanningUnitGeoEntity,
+  ScenariosPuCostDataGeo,
+} from '@marxan/scenarios-planning-unit';
+import { DbConnections } from '@marxan-api/ormconfig.connections';
+
+@Module({
+  imports: [
+    TypeOrmModule.forFeature(
+      [
+        PlanningUnitsGeom,
+        ScenariosPlanningUnitGeoEntity,
+        ScenariosPuCostDataGeo,
+      ],
+      DbConnections.geoprocessingDB,
+    ),
+  ],
+  providers: [CostSurfaceViewService],
+  exports: [CostSurfaceViewService],
+})
+export class CostSurfaceViewModule {}
diff --git a/api/apps/api/src/modules/scenarios/cost-surface-readmodel/cost-surface-view.service.ts b/api/apps/api/src/modules/scenarios/cost-surface-readmodel/cost-surface-view.service.ts
@@ -0,0 +1,57 @@
+import { Injectable } from '@nestjs/common';
+import * as stream from 'stream';
+import { InjectRepository } from '@nestjs/typeorm';
+import { Repository } from 'typeorm';
+import { DbConnections } from '@marxan-api/ormconfig.connections';
+import { ScenariosPlanningUnitGeoEntity } from '@marxan/scenarios-planning-unit';
+
+@Injectable()
+export class CostSurfaceViewService {
+  readonly #separator = '\t';
+
+  constructor(
+    @InjectRepository(
+      ScenariosPlanningUnitGeoEntity,
+      DbConnections.geoprocessingDB,
+    )
+    private readonly spuDataRepo: Repository<ScenariosPlanningUnitGeoEntity>,
+  ) {}
+
+  async read(
+    scenarioId: string,
+    responseStream: stream.Writable,
+  ): Promise<void> {
+    responseStream.write([`id`, `cost`, `status`].join(this.#separator));
+
+    const query = await this.spuDataRepo
+      .createQueryBuilder('spu')
+      .select(['spu.puid', 'spu.lockin_status', 'spucd.cost'])
+      .leftJoin(
+        `scenarios_pu_cost_data`,
+        `spucd`,
+        `spucd.scenarios_pu_data_id = spu.id`,
+      )
+      .where(`spu.scenario_id = :scenarioId`, { scenarioId });
+
+    const queryStream = await query.stream();
+    // "pipe" does not seem to trigger
+    queryStream.on(
+      'data',
+      (data: {
+        puid: number;
+        lockin_status: number | null;
+        spucd_cost: number | null;
+      }) => {
+        const tsvRow = [data.puid, data.spucd_cost, data.lockin_status].join(
+          this.#separator,
+        );
+        responseStream.write(`\n`);
+        responseStream.write(tsvRow);
+      },
+    );
+
+    queryStream.on('end', () => {
+      responseStream.end();
+    });
+  }
+}
diff --git a/api/apps/api/src/modules/scenarios/scenarios.controller.ts b/api/apps/api/src/modules/scenarios/scenarios.controller.ts
@@ -3,20 +3,22 @@ import {
   Controller,
   Delete,
   Get,
+  Header,
   Param,
   ParseBoolPipe,
   ParseUUIDPipe,
   Patch,
   Post,
   Query,
   Req,
+  Res,
   UploadedFile,
   UseGuards,
   UseInterceptors,
   ValidationPipe,
 } from '@nestjs/common';
 import { scenarioResource, ScenarioResult } from './scenario.api.entity';
-import { Request } from 'express';
+import { Request, Response } from 'express';
 import {
   FetchSpecification,
   ProcessFetchSpecification,
@@ -260,4 +262,22 @@ export class ScenariosController {
       result.metadata,
     );
   }
+
+  @Header('Content-Type', 'text/csv')
+  @ApiOkResponse({
+    schema: {
+      type: 'string',
+    },
+  })
+  @ApiOperation({
+    description: `Uploaded cost surface data`,
+  })
+  @Get(`:id/marxan/dat/pu.dat`)
+  async getScenarioCostSurface(
+    @Param('id', ParseUUIDPipe) id: string,
+    @Res() res: Response,
+  ): Promise<void> {
+    await this.service.getCostSurfaceCsv(id, res);
+    return;
+  }
 }
diff --git a/api/apps/api/src/modules/scenarios/scenarios.module.ts b/api/apps/api/src/modules/scenarios/scenarios.module.ts
@@ -22,6 +22,7 @@ import { SolutionResultCrudService } from './solutions-result/solution-result-cr
 import { DbConnections } from '@marxan-api/ormconfig.connections';
 import { ScenariosOutputResultsGeoEntity } from '@marxan/scenarios-planning-unit';
 import { ScenarioSolutionSerializer } from './dto/scenario-solution.serializer';
+import { CostSurfaceViewModule } from './cost-surface-readmodel/cost-surface-view.module';
 
 @Module({
   imports: [
@@ -39,6 +40,7 @@ import { ScenarioSolutionSerializer } from './dto/scenario-solution.serializer';
     CostSurfaceModule,
     HttpModule,
     CostSurfaceTemplateModule,
+    CostSurfaceViewModule,
   ],
   providers: [
     ScenariosService,

diff --git a/api/apps/api/src/modules/scenarios/scenarios.service.ts b/api/apps/api/src/modules/scenarios/scenarios.service.ts
@@ -1,5 +1,6 @@
 import { HttpService, Injectable } from '@nestjs/common';
 import { FetchSpecification } from 'nestjs-base-service';
+import * as stream from 'stream';
 
 import { AppInfoDTO } from '@marxan-api/dto/info.dto';
 import { AppConfig } from '@marxan-api/utils/config.utils';
@@ -14,6 +15,7 @@ import { CreateScenarioDTO } from './dto/create.scenario.dto';
 import { UpdateScenarioDTO } from './dto/update.scenario.dto';
 import { UpdateScenarioPlanningUnitLockStatusDto } from './dto/update-scenario-planning-unit-lock-status.dto';
 import { SolutionResultCrudService } from './solutions-result/solution-result-crud.service';
+import { CostSurfaceViewService } from './cost-surface-readmodel/cost-surface-view.service';
 
 @Injectable()
 export class ScenariosService {
@@ -28,6 +30,7 @@ export class ScenariosService {
     private readonly costSurface: CostSurfaceFacade,
     private readonly httpService: HttpService,
     private readonly solutionsCrudService: SolutionResultCrudService,
+    private readonly costSurfaceView: CostSurfaceViewService,
   ) {}
 
   async findAllPaginated(fetchSpecification: FetchSpecification) {
@@ -114,6 +117,14 @@ export class ScenariosService {
     return this.solutionsCrudService.findAll(fetchSpecification);
   }
 
+  async getCostSurfaceCsv(
+    scenarioId: string,
+    stream: stream.Writable,
+  ): Promise<void> {
+    await this.assertScenario(scenarioId);
+    await this.costSurfaceView.read(scenarioId, stream);
+  }
+
   private async assertScenario(scenarioId: string) {
     await this.crudService.getById(scenarioId);
   }

diff --git a/api/apps/api/test/scenario-cost-surface/scenario-cost-surface.e2e-spec.ts b/api/apps/api/test/scenario-cost-surface/scenario-cost-surface.e2e-spec.ts
@@ -0,0 +1,33 @@
+import { PromiseType } from 'utility-types';
+import { createWorld } from './world';
+
+let world: PromiseType<ReturnType<typeof createWorld>>;
+
+beforeAll(async () => {
+  world = await createWorld();
+});
+
+describe(`When scenario has PUs with cost and lock status`, () => {
+  beforeAll(async () => {
+    await world.GivenScenarioWithPuAndLocks();
+  });
+
+  it(`returns relevant data`, async () => {
+    const result = await world.WhenGettingMarxanData();
+    const [headers, ...costAndStatus] = result.split('\n');
+
+    expect(headers).toEqual('id\tcost\tstatus');
+    expect(costAndStatus).toMatchInlineSnapshot(`
+      Array [
+        "0	200	",
+        "1	400	1",
+        "2	600	2",
+        "3	800	2",
+      ]
+    `);
+  });
+});
+
+afterAll(async () => {
+  await world?.cleanup();
+});
diff --git a/api/apps/api/test/scenario-cost-surface/world.ts b/api/apps/api/test/scenario-cost-surface/world.ts
@@ -0,0 +1,123 @@
+import { bootstrapApplication } from '../utils/api-application';
+import { GivenUserIsLoggedIn } from '../steps/given-user-is-logged-in';
+import * as request from 'supertest';
+import { In, Repository } from 'typeorm';
+import { getRepositoryToken } from '@nestjs/typeorm';
+import {
+  LockStatus,
+  ScenariosPlanningUnitGeoEntity,
+  ScenariosPuCostDataGeo,
+} from '@marxan/scenarios-planning-unit';
+import { Polygon } from 'geojson';
+import { DbConnections } from '@marxan-api/ormconfig.connections';
+import {
+  PlanningUnitsGeom,
+  ShapeType,
+} from '@marxan-jobs/planning-unit-geometry';
+import { GivenProjectExists } from '../steps/given-project';
+import { ScenariosTestUtils } from '../utils/scenarios.test.utils';
+import { ScenarioType } from '@marxan-api/modules/scenarios/scenario.api.entity';
+import { v4 } from 'uuid';
+
+export const createWorld = async () => {
+  const app = await bootstrapApplication();
+  const jwt = await GivenUserIsLoggedIn(app);
+  const { cleanup: projectCleanup, projectId } = await GivenProjectExists(
+    app,
+    jwt,
+  );
+  const scenarioId = (
+    await ScenariosTestUtils.createScenario(app, jwt, {
+      name: `scenario-name`,
+      type: ScenarioType.marxan,
+      projectId,
+    })
+  ).data.id;
+  const geometries: string[] = [];
+  const scenariosPuData: string[] = [];
+
+  const puGeometryRepo: Repository<PlanningUnitsGeom> = app.get(
+    getRepositoryToken(PlanningUnitsGeom, DbConnections.geoprocessingDB),
+  );
+  const scenarioPuDataRepo: Repository<ScenariosPlanningUnitGeoEntity> = app.get(
+    getRepositoryToken(
+      ScenariosPlanningUnitGeoEntity,
+      DbConnections.geoprocessingDB,
+    ),
+  );
+  const scenarioPuDataCostRepo: Repository<ScenariosPuCostDataGeo> = app.get(
+    getRepositoryToken(ScenariosPuCostDataGeo, DbConnections.geoprocessingDB),
+  );
+
+  return {
+    cleanup: async () => {
+      await ScenariosTestUtils.deleteScenario(app, jwt, scenarioId);
+      await projectCleanup();
+      await scenarioPuDataCostRepo.delete({
+        scenariosPuDataId: In(scenariosPuData),
+      });
+      await puGeometryRepo.delete({
+        id: In(geometries),
+      });
+      await scenarioPuDataRepo.delete({
+        scenarioId,
+      });
+      await app.close();
+    },
+    GivenScenarioWithPuAndLocks: async () => {
+      const polygons: Polygon[] = [1, 2, 3, 4].map((i) => ({
+        type: 'Polygon',
+        coordinates: [
+          [
+            [0, i],
+            [i, i],
+            [i, 0],
+            [0, 0],
+            [0, i],
+          ],
+        ],
+      }));
+      const geoRows = (
+        await puGeometryRepo.insert(
+          polygons.map((poly) => ({
+            theGeom: () =>
+              `st_multi(ST_GeomFromGeoJSON('${JSON.stringify(poly)}'))`,
+            type: ShapeType.Square,
+          })),
+        )
+      ).identifiers;
+
+      geometries.push(...geoRows.map((geo) => geo.id));
+      const scenarioPuData = await scenarioPuDataRepo.save(
+        geometries.map((id, index) =>
+          scenarioPuDataRepo.create({
+            puGeometryId: id,
+            scenarioId,
+            planningUnitMarxanId: index,
+            lockStatus:
+              index === 0
+                ? LockStatus.Unstated
+                : index === 1
+                ? LockStatus.LockedIn
+                : LockStatus.LockedOut,
+          }),
+        ),
+      );
+      scenariosPuData.push(...scenarioPuData.map((spud) => spud.id));
+      await scenarioPuDataCostRepo.save(
+        scenarioPuData.map((spud, index) => ({
+          cost: (index + 1) * 200,
+          scenariosPuDataId: spud.id,
+          scenariosPlanningUnit: spud,
+          planningUnitId: v4(),
+        })),
+      );
+    },
+    WhenGettingMarxanData: async () =>
+      (
+        await request(app.getHttpServer())
+          .get(`/api/v1/scenarios/${scenarioId}/marxan/dat/pu.dat`)
+          .set('Authorization', `Bearer ${jwt}`)
+      ).text,
+  };
+};
diff --git a/api/apps/geoprocessing/src/modules/planning-units/planning-units.module.ts b/api/apps/geoprocessing/src/modules/planning-units/planning-units.module.ts
@@ -1,10 +1,10 @@
 import { Logger, Module } from '@nestjs/common';
 import { TypeOrmModule } from '@nestjs/typeorm';
 import { TileModule } from '@marxan-geoprocessing/modules/tile/tile.module';
+import { PlanningUnitsGeom } from '@marxan-jobs/planning-unit-geometry';
 import { PlanningUnitsProcessor } from './planning-units.worker';
 import { PlanningUnitsController } from './planning-units.controller';
 import { ShapefileService } from '../shapefiles/shapefiles.service';
-import { PlanningUnitsGeom } from '@marxan-geoprocessing/modules/planning-units/planning-units.geo.entity';
 import { PlanningUnitsService } from './planning-units.service';
 import { FileService } from '../files/files.service';
 import { WorkerModule } from '../worker';

diff --git a/api/apps/geoprocessing/src/modules/planning-units/planning-units.service.ts b/api/apps/geoprocessing/src/modules/planning-units/planning-units.service.ts
@@ -10,8 +10,8 @@ import { ApiProperty } from '@nestjs/swagger';
 import { BBox } from 'geojson';
 import { Transform } from 'class-transformer';
 
-import { PlanningUnitsGeom } from '@marxan-geoprocessing/modules/planning-units/planning-units.geo.entity';
 import { nominatim2bbox } from '@marxan-geoprocessing/utils/bbox.utils';
+import { PlanningUnitsGeom } from '@marxan-jobs/planning-unit-geometry';
 
 export class tileSpecification extends TileRequest {
   @ApiProperty()

diff --git a/api/apps/geoprocessing/src/modules/surface-cost/adapters/typeorm-cost-surface.ts b/api/apps/geoprocessing/src/modules/surface-cost/adapters/typeorm-cost-surface.ts
@@ -5,7 +5,7 @@ import { flatten } from 'lodash';
 
 import { CostSurfacePersistencePort } from '../ports/persistence/cost-surface-persistence.port';
 import { PlanningUnitCost } from '../ports/planning-unit-cost';
-import { ScenariosPuCostDataGeo } from '../../scenarios/scenarios-pu-cost-data.geo.entity';
+import { ScenariosPuCostDataGeo } from '@marxan/scenarios-planning-unit';
 
 @Injectable()
 export class TypeormCostSurface implements CostSurfacePersistencePort {

diff --git a/api/apps/geoprocessing/src/modules/surface-cost/surface-cost.module.ts b/api/apps/geoprocessing/src/modules/surface-cost/surface-cost.module.ts
@@ -18,9 +18,9 @@ import { ShapefileConverterPort } from './ports/shapefile-converter/shapefile-co
 
 import { TypeormCostSurface } from './adapters/typeorm-cost-surface';
 import { ShapefileConverter } from './adapters/shapefile-converter';
-import { ScenariosPuCostDataGeo } from '../scenarios/scenarios-pu-cost-data.geo.entity';
 import { PuCostExtractor } from './adapters/pu-cost-extractor';
 import { AvailablePlanningUnitsRepository } from './adapters/available-planning-units-repository';
+import { ScenariosPuCostDataGeo } from '@marxan/scenarios-planning-unit';
 
 @Module({
   imports: [

diff --git a/api/apps/geoprocessing/test/integration/cost-surface/planning-unit-fixtures.ts b/api/apps/geoprocessing/test/integration/cost-surface/planning-unit-fixtures.ts
@@ -3,8 +3,10 @@ import { getRepositoryToken } from '@nestjs/typeorm';
 import { Repository } from 'typeorm';
 import { v4 } from 'uuid';
 
-import { ScenariosPuCostDataGeo } from '@marxan-geoprocessing/modules/scenarios/scenarios-pu-cost-data.geo.entity';
-import { ScenariosPlanningUnitGeoEntity } from '@marxan/scenarios-planning-unit';
+import {
+  ScenariosPlanningUnitGeoEntity,
+  ScenariosPuCostDataGeo,
+} from '@marxan/scenarios-planning-unit';
 
 import { GivenScenarioPuDataExists } from '../../steps/given-scenario-pu-data-exists';