diff --git a/api/apps/api/src/modules/scenarios/cost-surface-readmodel/cost-surface-view.module.ts b/api/apps/api/src/modules/scenarios/cost-surface-readmodel/cost-surface-view.module.ts new file mode 100644 index 0000000000..6672749275 --- /dev/null +++ b/api/apps/api/src/modules/scenarios/cost-surface-readmodel/cost-surface-view.module.ts @@ -0,0 +1,25 @@ +import { Module } from '@nestjs/common'; +import { CostSurfaceViewService } from './cost-surface-view.service'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { PlanningUnitsGeom } from '@marxan-jobs/planning-unit-geometry'; +import { + ScenariosPlanningUnitGeoEntity, + ScenariosPuCostDataGeo, +} from '@marxan/scenarios-planning-unit'; +import { DbConnections } from '@marxan-api/ormconfig.connections'; + +@Module({ + imports: [ + TypeOrmModule.forFeature( + [ + PlanningUnitsGeom, + ScenariosPlanningUnitGeoEntity, + ScenariosPuCostDataGeo, + ], + DbConnections.geoprocessingDB, + ), + ], + providers: [CostSurfaceViewService], + exports: [CostSurfaceViewService], +}) +export class CostSurfaceViewModule {} diff --git a/api/apps/api/src/modules/scenarios/cost-surface-readmodel/cost-surface-view.service.ts b/api/apps/api/src/modules/scenarios/cost-surface-readmodel/cost-surface-view.service.ts new file mode 100644 index 0000000000..6508cb85e2 --- /dev/null +++ b/api/apps/api/src/modules/scenarios/cost-surface-readmodel/cost-surface-view.service.ts @@ -0,0 +1,57 @@ +import { Injectable } from '@nestjs/common'; +import * as stream from 'stream'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { DbConnections } from '@marxan-api/ormconfig.connections'; +import { ScenariosPlanningUnitGeoEntity } from '@marxan/scenarios-planning-unit'; + +@Injectable() +export class CostSurfaceViewService { + readonly #separator = '\t'; + + constructor( + @InjectRepository( + ScenariosPlanningUnitGeoEntity, + DbConnections.geoprocessingDB, + ) + private readonly spuDataRepo: Repository, + ) {} + + async read( + scenarioId: string, + responseStream: stream.Writable, + ): Promise { + responseStream.write([`id`, `cost`, `status`].join(this.#separator)); + + const query = await this.spuDataRepo + .createQueryBuilder('spu') + .select(['spu.puid', 'spu.lockin_status', 'spucd.cost']) + .leftJoin( + `scenarios_pu_cost_data`, + `spucd`, + `spucd.scenarios_pu_data_id = spu.id`, + ) + .where(`spu.scenario_id = :scenarioId`, { scenarioId }); + + const queryStream = await query.stream(); + // "pipe" does not seem to trigger + queryStream.on( + 'data', + (data: { + puid: number; + lockin_status: number | null; + spucd_cost: number | null; + }) => { + const tsvRow = [data.puid, data.spucd_cost, data.lockin_status].join( + this.#separator, + ); + responseStream.write(`\n`); + responseStream.write(tsvRow); + }, + ); + + queryStream.on('end', () => { + responseStream.end(); + }); + } +} diff --git a/api/apps/api/src/modules/scenarios/scenarios.controller.ts b/api/apps/api/src/modules/scenarios/scenarios.controller.ts index 6ed817d3d7..5119ddda7a 100644 --- a/api/apps/api/src/modules/scenarios/scenarios.controller.ts +++ b/api/apps/api/src/modules/scenarios/scenarios.controller.ts @@ -3,6 +3,7 @@ import { Controller, Delete, Get, + Header, Param, ParseBoolPipe, ParseUUIDPipe, @@ -10,13 +11,14 @@ import { Post, Query, Req, + Res, UploadedFile, UseGuards, UseInterceptors, ValidationPipe, } from '@nestjs/common'; import { scenarioResource, ScenarioResult } from './scenario.api.entity'; -import { Request } from 'express'; +import { Request, Response } from 'express'; import { FetchSpecification, ProcessFetchSpecification, @@ -260,4 +262,22 @@ export class ScenariosController { result.metadata, ); } + + @Header('Content-Type', 'text/csv') + @ApiOkResponse({ + schema: { + type: 'string', + }, + }) + @ApiOperation({ + description: `Uploaded cost surface data`, + }) + @Get(`:id/marxan/dat/pu.dat`) + async getScenarioCostSurface( + @Param('id', ParseUUIDPipe) id: string, + @Res() res: Response, + ): Promise { + await this.service.getCostSurfaceCsv(id, res); + return; + } } diff --git a/api/apps/api/src/modules/scenarios/scenarios.module.ts b/api/apps/api/src/modules/scenarios/scenarios.module.ts index aa33ff3f79..6414b17822 100644 --- a/api/apps/api/src/modules/scenarios/scenarios.module.ts +++ b/api/apps/api/src/modules/scenarios/scenarios.module.ts @@ -22,6 +22,7 @@ import { SolutionResultCrudService } from './solutions-result/solution-result-cr import { DbConnections } from '@marxan-api/ormconfig.connections'; import { ScenariosOutputResultsGeoEntity } from '@marxan/scenarios-planning-unit'; import { ScenarioSolutionSerializer } from './dto/scenario-solution.serializer'; +import { CostSurfaceViewModule } from './cost-surface-readmodel/cost-surface-view.module'; @Module({ imports: [ @@ -39,6 +40,7 @@ import { ScenarioSolutionSerializer } from './dto/scenario-solution.serializer'; CostSurfaceModule, HttpModule, CostSurfaceTemplateModule, + CostSurfaceViewModule, ], providers: [ ScenariosService, diff --git a/api/apps/api/src/modules/scenarios/scenarios.service.ts b/api/apps/api/src/modules/scenarios/scenarios.service.ts index 91ff9c67f5..3034c2c267 100644 --- a/api/apps/api/src/modules/scenarios/scenarios.service.ts +++ b/api/apps/api/src/modules/scenarios/scenarios.service.ts @@ -1,5 +1,6 @@ import { HttpService, Injectable } from '@nestjs/common'; import { FetchSpecification } from 'nestjs-base-service'; +import * as stream from 'stream'; import { AppInfoDTO } from '@marxan-api/dto/info.dto'; import { AppConfig } from '@marxan-api/utils/config.utils'; @@ -14,6 +15,7 @@ import { CreateScenarioDTO } from './dto/create.scenario.dto'; import { UpdateScenarioDTO } from './dto/update.scenario.dto'; import { UpdateScenarioPlanningUnitLockStatusDto } from './dto/update-scenario-planning-unit-lock-status.dto'; import { SolutionResultCrudService } from './solutions-result/solution-result-crud.service'; +import { CostSurfaceViewService } from './cost-surface-readmodel/cost-surface-view.service'; @Injectable() export class ScenariosService { @@ -28,6 +30,7 @@ export class ScenariosService { private readonly costSurface: CostSurfaceFacade, private readonly httpService: HttpService, private readonly solutionsCrudService: SolutionResultCrudService, + private readonly costSurfaceView: CostSurfaceViewService, ) {} async findAllPaginated(fetchSpecification: FetchSpecification) { @@ -114,6 +117,14 @@ export class ScenariosService { return this.solutionsCrudService.findAll(fetchSpecification); } + async getCostSurfaceCsv( + scenarioId: string, + stream: stream.Writable, + ): Promise { + await this.assertScenario(scenarioId); + await this.costSurfaceView.read(scenarioId, stream); + } + private async assertScenario(scenarioId: string) { await this.crudService.getById(scenarioId); } diff --git a/api/apps/api/test/scenario-cost-surface/scenario-cost-surface.e2e-spec.ts b/api/apps/api/test/scenario-cost-surface/scenario-cost-surface.e2e-spec.ts new file mode 100644 index 0000000000..568492a4fd --- /dev/null +++ b/api/apps/api/test/scenario-cost-surface/scenario-cost-surface.e2e-spec.ts @@ -0,0 +1,33 @@ +import { PromiseType } from 'utility-types'; +import { createWorld } from './world'; + +let world: PromiseType>; + +beforeAll(async () => { + world = await createWorld(); +}); + +describe(`When scenario has PUs with cost and lock status`, () => { + beforeAll(async () => { + await world.GivenScenarioWithPuAndLocks(); + }); + + it(`returns relevant data`, async () => { + const result = await world.WhenGettingMarxanData(); + const [headers, ...costAndStatus] = result.split('\n'); + + expect(headers).toEqual('id\tcost\tstatus'); + expect(costAndStatus).toMatchInlineSnapshot(` + Array [ + "0 200 ", + "1 400 1", + "2 600 2", + "3 800 2", + ] + `); + }); +}); + +afterAll(async () => { + await world?.cleanup(); +}); diff --git a/api/apps/api/test/scenario-cost-surface/world.ts b/api/apps/api/test/scenario-cost-surface/world.ts new file mode 100644 index 0000000000..1543c91126 --- /dev/null +++ b/api/apps/api/test/scenario-cost-surface/world.ts @@ -0,0 +1,123 @@ +import { bootstrapApplication } from '../utils/api-application'; +import { GivenUserIsLoggedIn } from '../steps/given-user-is-logged-in'; +import * as request from 'supertest'; +import { In, Repository } from 'typeorm'; +import { getRepositoryToken } from '@nestjs/typeorm'; +import { + LockStatus, + ScenariosPlanningUnitGeoEntity, + ScenariosPuCostDataGeo, +} from '@marxan/scenarios-planning-unit'; +import { Polygon } from 'geojson'; +import { DbConnections } from '@marxan-api/ormconfig.connections'; +import { + PlanningUnitsGeom, + ShapeType, +} from '@marxan-jobs/planning-unit-geometry'; +import { GivenProjectExists } from '../steps/given-project'; +import { ScenariosTestUtils } from '../utils/scenarios.test.utils'; +import { ScenarioType } from '@marxan-api/modules/scenarios/scenario.api.entity'; +import { v4 } from 'uuid'; + +export const createWorld = async () => { + const app = await bootstrapApplication(); + const jwt = await GivenUserIsLoggedIn(app); + const { cleanup: projectCleanup, projectId } = await GivenProjectExists( + app, + jwt, + ); + const scenarioId = ( + await ScenariosTestUtils.createScenario(app, jwt, { + name: `scenario-name`, + type: ScenarioType.marxan, + projectId, + }) + ).data.id; + const geometries: string[] = []; + const scenariosPuData: string[] = []; + + const puGeometryRepo: Repository = app.get( + getRepositoryToken(PlanningUnitsGeom, DbConnections.geoprocessingDB), + ); + const scenarioPuDataRepo: Repository = app.get( + getRepositoryToken( + ScenariosPlanningUnitGeoEntity, + DbConnections.geoprocessingDB, + ), + ); + const scenarioPuDataCostRepo: Repository = app.get( + getRepositoryToken(ScenariosPuCostDataGeo, DbConnections.geoprocessingDB), + ); + + return { + cleanup: async () => { + await ScenariosTestUtils.deleteScenario(app, jwt, scenarioId); + await projectCleanup(); + await scenarioPuDataCostRepo.delete({ + scenariosPuDataId: In(scenariosPuData), + }); + await puGeometryRepo.delete({ + id: In(geometries), + }); + await scenarioPuDataRepo.delete({ + scenarioId, + }); + await app.close(); + }, + GivenScenarioWithPuAndLocks: async () => { + const polygons: Polygon[] = [1, 2, 3, 4].map((i) => ({ + type: 'Polygon', + coordinates: [ + [ + [0, i], + [i, i], + [i, 0], + [0, 0], + [0, i], + ], + ], + })); + const geoRows = ( + await puGeometryRepo.insert( + polygons.map((poly) => ({ + theGeom: () => + `st_multi(ST_GeomFromGeoJSON('${JSON.stringify(poly)}'))`, + type: ShapeType.Square, + })), + ) + ).identifiers; + + geometries.push(...geoRows.map((geo) => geo.id)); + const scenarioPuData = await scenarioPuDataRepo.save( + geometries.map((id, index) => + scenarioPuDataRepo.create({ + puGeometryId: id, + scenarioId, + planningUnitMarxanId: index, + lockStatus: + index === 0 + ? LockStatus.Unstated + : index === 1 + ? LockStatus.LockedIn + : LockStatus.LockedOut, + }), + ), + ); + scenariosPuData.push(...scenarioPuData.map((spud) => spud.id)); + await scenarioPuDataCostRepo.save( + scenarioPuData.map((spud, index) => ({ + cost: (index + 1) * 200, + scenariosPuDataId: spud.id, + scenariosPlanningUnit: spud, + planningUnitId: v4(), + })), + ); + }, + WhenGettingMarxanData: async () => + ( + await request(app.getHttpServer()) + .get(`/api/v1/scenarios/${scenarioId}/marxan/dat/pu.dat`) + .set('Authorization', `Bearer ${jwt}`) + ).text, + }; +}; diff --git a/api/apps/geoprocessing/src/modules/planning-units/planning-units.module.ts b/api/apps/geoprocessing/src/modules/planning-units/planning-units.module.ts index cb2c6c5bec..380de5e9f1 100644 --- a/api/apps/geoprocessing/src/modules/planning-units/planning-units.module.ts +++ b/api/apps/geoprocessing/src/modules/planning-units/planning-units.module.ts @@ -1,10 +1,10 @@ import { Logger, Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; import { TileModule } from '@marxan-geoprocessing/modules/tile/tile.module'; +import { PlanningUnitsGeom } from '@marxan-jobs/planning-unit-geometry'; import { PlanningUnitsProcessor } from './planning-units.worker'; import { PlanningUnitsController } from './planning-units.controller'; import { ShapefileService } from '../shapefiles/shapefiles.service'; -import { PlanningUnitsGeom } from '@marxan-geoprocessing/modules/planning-units/planning-units.geo.entity'; import { PlanningUnitsService } from './planning-units.service'; import { FileService } from '../files/files.service'; import { WorkerModule } from '../worker'; diff --git a/api/apps/geoprocessing/src/modules/planning-units/planning-units.service.ts b/api/apps/geoprocessing/src/modules/planning-units/planning-units.service.ts index e9b0ae9473..c866b41395 100644 --- a/api/apps/geoprocessing/src/modules/planning-units/planning-units.service.ts +++ b/api/apps/geoprocessing/src/modules/planning-units/planning-units.service.ts @@ -10,8 +10,8 @@ import { ApiProperty } from '@nestjs/swagger'; import { BBox } from 'geojson'; import { Transform } from 'class-transformer'; -import { PlanningUnitsGeom } from '@marxan-geoprocessing/modules/planning-units/planning-units.geo.entity'; import { nominatim2bbox } from '@marxan-geoprocessing/utils/bbox.utils'; +import { PlanningUnitsGeom } from '@marxan-jobs/planning-unit-geometry'; export class tileSpecification extends TileRequest { @ApiProperty() diff --git a/api/apps/geoprocessing/src/modules/surface-cost/adapters/typeorm-cost-surface.ts b/api/apps/geoprocessing/src/modules/surface-cost/adapters/typeorm-cost-surface.ts index 9c1b5f811f..435103c4a8 100644 --- a/api/apps/geoprocessing/src/modules/surface-cost/adapters/typeorm-cost-surface.ts +++ b/api/apps/geoprocessing/src/modules/surface-cost/adapters/typeorm-cost-surface.ts @@ -5,7 +5,7 @@ import { flatten } from 'lodash'; import { CostSurfacePersistencePort } from '../ports/persistence/cost-surface-persistence.port'; import { PlanningUnitCost } from '../ports/planning-unit-cost'; -import { ScenariosPuCostDataGeo } from '../../scenarios/scenarios-pu-cost-data.geo.entity'; +import { ScenariosPuCostDataGeo } from '@marxan/scenarios-planning-unit'; @Injectable() export class TypeormCostSurface implements CostSurfacePersistencePort { diff --git a/api/apps/geoprocessing/src/modules/surface-cost/surface-cost.module.ts b/api/apps/geoprocessing/src/modules/surface-cost/surface-cost.module.ts index 6d0a3cc502..5610e3f22c 100644 --- a/api/apps/geoprocessing/src/modules/surface-cost/surface-cost.module.ts +++ b/api/apps/geoprocessing/src/modules/surface-cost/surface-cost.module.ts @@ -18,9 +18,9 @@ import { ShapefileConverterPort } from './ports/shapefile-converter/shapefile-co import { TypeormCostSurface } from './adapters/typeorm-cost-surface'; import { ShapefileConverter } from './adapters/shapefile-converter'; -import { ScenariosPuCostDataGeo } from '../scenarios/scenarios-pu-cost-data.geo.entity'; import { PuCostExtractor } from './adapters/pu-cost-extractor'; import { AvailablePlanningUnitsRepository } from './adapters/available-planning-units-repository'; +import { ScenariosPuCostDataGeo } from '@marxan/scenarios-planning-unit'; @Module({ imports: [ diff --git a/api/apps/geoprocessing/test/integration/cost-surface/planning-unit-fixtures.ts b/api/apps/geoprocessing/test/integration/cost-surface/planning-unit-fixtures.ts index fa63048729..f572fd9b7e 100644 --- a/api/apps/geoprocessing/test/integration/cost-surface/planning-unit-fixtures.ts +++ b/api/apps/geoprocessing/test/integration/cost-surface/planning-unit-fixtures.ts @@ -3,8 +3,10 @@ import { getRepositoryToken } from '@nestjs/typeorm'; import { Repository } from 'typeorm'; import { v4 } from 'uuid'; -import { ScenariosPuCostDataGeo } from '@marxan-geoprocessing/modules/scenarios/scenarios-pu-cost-data.geo.entity'; -import { ScenariosPlanningUnitGeoEntity } from '@marxan/scenarios-planning-unit'; +import { + ScenariosPlanningUnitGeoEntity, + ScenariosPuCostDataGeo, +} from '@marxan/scenarios-planning-unit'; import { GivenScenarioPuDataExists } from '../../steps/given-scenario-pu-data-exists'; diff --git a/api/apps/geoprocessing/test/integration/planning-unit-inclusion/world.ts b/api/apps/geoprocessing/test/integration/planning-unit-inclusion/world.ts index 3c27cb45c9..be6963c682 100644 --- a/api/apps/geoprocessing/test/integration/planning-unit-inclusion/world.ts +++ b/api/apps/geoprocessing/test/integration/planning-unit-inclusion/world.ts @@ -4,10 +4,6 @@ import { getRepositoryToken } from '@nestjs/typeorm'; import { v4 } from 'uuid'; import { Feature, MultiPolygon, Polygon } from 'geojson'; -import { - PlanningUnitsGeom, - ShapeType, -} from '@marxan-geoprocessing/modules/planning-units/planning-units.geo.entity'; import { LockStatus, ScenariosPlanningUnitGeoEntity, @@ -16,6 +12,10 @@ import { AreaUnitSampleGeometry, AreaUnitSampleGeometryProps, } from '@marxan-geoprocessing/modules/scenario-planning-units-inclusion/__mocks__/include-sample'; +import { + PlanningUnitsGeom, + ShapeType, +} from '@marxan-jobs/planning-unit-geometry'; export const createWorld = async (app: INestApplication) => { const scenarioId = v4(); diff --git a/api/libs/planning-unit-geometry/src/index.ts b/api/libs/planning-unit-geometry/src/index.ts index bae4344cbc..3f06f8a6d8 100644 --- a/api/libs/planning-unit-geometry/src/index.ts +++ b/api/libs/planning-unit-geometry/src/index.ts @@ -1,2 +1,3 @@ export { queueName } from './queue-name'; export { JobInput } from './job-input'; +export { ShapeType, PlanningUnitsGeom } from './planning-units.geo.entity'; diff --git a/api/apps/geoprocessing/src/modules/planning-units/planning-units.geo.entity.ts b/api/libs/planning-unit-geometry/src/planning-units.geo.entity.ts similarity index 93% rename from api/apps/geoprocessing/src/modules/planning-units/planning-units.geo.entity.ts rename to api/libs/planning-unit-geometry/src/planning-units.geo.entity.ts index 23555a1709..fb2881c052 100644 --- a/api/apps/geoprocessing/src/modules/planning-units/planning-units.geo.entity.ts +++ b/api/libs/planning-unit-geometry/src/planning-units.geo.entity.ts @@ -1,6 +1,3 @@ -/** - * @todo move the planningUnitsGeom entity to the api - */ import { ApiProperty } from '@nestjs/swagger'; import { Column, Entity, PrimaryGeneratedColumn } from 'typeorm'; import { Geometry } from 'geojson'; diff --git a/api/libs/scenarios-planning-unit/src/index.ts b/api/libs/scenarios-planning-unit/src/index.ts index e25696aa62..af56ea53a5 100644 --- a/api/libs/scenarios-planning-unit/src/index.ts +++ b/api/libs/scenarios-planning-unit/src/index.ts @@ -1,4 +1,5 @@ export { LockStatus } from './lock-status.enum'; export { ScenariosPlanningUnitGeoEntity } from './scenarios-planning-unit.geo.entity'; export { ScenariosOutputResultsGeoEntity } from './scenarios-output-results.geo.entity'; +export { ScenariosPuCostDataGeo } from './scenarios-pu-cost-data.geo.entity'; export * from './domain'; diff --git a/api/apps/geoprocessing/src/modules/scenarios/scenarios-pu-cost-data.geo.entity.ts b/api/libs/scenarios-planning-unit/src/scenarios-pu-cost-data.geo.entity.ts similarity index 100% rename from api/apps/geoprocessing/src/modules/scenarios/scenarios-pu-cost-data.geo.entity.ts rename to api/libs/scenarios-planning-unit/src/scenarios-pu-cost-data.geo.entity.ts