test(geoprocessing): cost-surface: integration test

Vizzuality · Jun 2, 2021 · fe54135 · fe54135
1 parent 64354e1
commit fe54135
Show file tree

Hide file tree

Showing 18 changed files with 341 additions and 56 deletions.
diff --git a/...ps/geoprocessing/src/modules/surface-cost/adapters/available-planning-units-repository.ts b/...ps/geoprocessing/src/modules/surface-cost/adapters/available-planning-units-repository.ts
@@ -22,7 +22,7 @@ export class AvailablePlanningUnitsRepository
         },
       })
       .then((rows) => ({
-        ids: rows.map((row) => row.id),
+        ids: rows.map((row) => row.puGeometryId),
       }));
   }
 }
diff --git a/api/apps/geoprocessing/src/modules/surface-cost/adapters/pu-cost-extractor.ts b/api/apps/geoprocessing/src/modules/surface-cost/adapters/pu-cost-extractor.ts
@@ -4,12 +4,7 @@ import { FeatureCollection, GeoJSON, Geometry } from 'geojson';
 import { PlanningUnitCost } from '../ports/planning-unit-cost';
 import { PuExtractorPort } from '../ports/pu-extractor/pu-extractor.port';
 
-type Properties = {
-  cost: number;
-  planningUnitId: string;
-};
-
-type MaybeCost = MaybeProperties<Properties>;
+type MaybeCost = MaybeProperties<PlanningUnitCost>;
 
 export class PuCostExtractor implements PuExtractorPort {
   extract(geo: GeoJSON): PlanningUnitCost[] {
@@ -29,7 +24,7 @@ export class PuCostExtractor implements PuExtractorPort {
     }
 
     return puCosts.map((puCost) => ({
-      planningUnitId: puCost.planningUnitId,
+      puId: puCost.puId,
       cost: puCost.cost,
     }));
   }
@@ -38,11 +33,11 @@ export class PuCostExtractor implements PuExtractorPort {
     return geo.type === 'FeatureCollection';
   }
 
-  private hasCostValues(properties: MaybeCost): properties is Properties {
+  private hasCostValues(properties: MaybeCost): properties is PlanningUnitCost {
     return (
       isDefined(properties) &&
       isDefined(properties.cost) &&
-      isDefined(properties.planningUnitId)
+      isDefined(properties.puId)
     );
   }
 }
diff --git a/api/apps/geoprocessing/src/modules/surface-cost/adapters/shapefile-converter.ts b/api/apps/geoprocessing/src/modules/surface-cost/adapters/shapefile-converter.ts
@@ -1,9 +1,11 @@
+import { Injectable } from '@nestjs/common';
 import { GeoJSON } from 'geojson';
 import { ShapefileService } from '../../shapefiles/shapefiles.service';
 
 import { CostSurfaceJobInput } from '../cost-surface-job-input';
 import { ShapefileConverterPort } from '../ports/shapefile-converter/shapefile-converter.port';
 
+@Injectable()
 export class ShapefileConverter implements ShapefileConverterPort {
   constructor(private readonly converter: ShapefileService) {}
 

diff --git a/api/apps/geoprocessing/src/modules/surface-cost/adapters/typeorm-cost-surface.ts b/api/apps/geoprocessing/src/modules/surface-cost/adapters/typeorm-cost-surface.ts
@@ -18,7 +18,7 @@ export class TypeormCostSurface implements CostSurfacePersistencePort {
 
   async save(_: string, values: PlanningUnitCost[]): Promise<void> {
     const pairs = values.map<[string, number]>((pair) => [
-      pair.planningUnitId,
+      pair.puId,
       pair.cost,
     ]);
     await this.costs.query(

diff --git a/api/apps/geoprocessing/src/modules/surface-cost/application/__mocks__/cost.ts b/api/apps/geoprocessing/src/modules/surface-cost/application/__mocks__/cost.ts
@@ -5,5 +5,5 @@ export const getCostByPlanningUnit = (
 ): PlanningUnitCost[] =>
   planningUnitsIds.map((pu) => ({
     cost: 200,
-    planningUnitId: pu,
+    puId: pu,
   }));
diff --git a/api/apps/geoprocessing/src/modules/surface-cost/application/__mocks__/geojson.ts b/api/apps/geoprocessing/src/modules/surface-cost/application/__mocks__/geojson.ts
@@ -8,7 +8,7 @@ export const getGeoJson = (
   features: planningUnitsIds.map((id) => ({
     properties: {
       cost: 200,
-      planningUnitId: id,
+      puId: id,
     },
     type: 'Feature',
     geometry: {

diff --git a/api/apps/geoprocessing/src/modules/surface-cost/application/surface-cost-processor.ts b/api/apps/geoprocessing/src/modules/surface-cost/application/surface-cost-processor.ts
@@ -29,7 +29,7 @@ export class SurfaceCostProcessor
     ).ids;
 
     const { errors } = canPlanningUnitsBeLocked(
-      surfaceCosts.map((cost) => cost.planningUnitId),
+      surfaceCosts.map((cost) => cost.puId),
       scenarioPlanningUnitIds,
     );
     if (errors.length > 0) {

diff --git a/api/apps/geoprocessing/src/modules/surface-cost/application/surface-cost-worker.ts b/api/apps/geoprocessing/src/modules/surface-cost/application/surface-cost-worker.ts
@@ -3,25 +3,23 @@ import { EventBus } from '@nestjs/cqrs';
 import { Job, Worker } from 'bullmq';
 
 import {
-  ApiEvent,
   API_EVENT_KINDS,
+  ApiEvent,
 } from '@marxan-geoprocessing/modules/api-events';
-import {
-  WorkerBuilder,
-  WorkerProcessor,
-} from '@marxan-geoprocessing/modules/worker';
+import { WorkerBuilder } from '@marxan-geoprocessing/modules/worker';
 
 import { CostSurfaceJobInput } from '../cost-surface-job-input';
 
 import { queueName } from './queue-name';
+import { SurfaceCostProcessor } from './surface-cost-processor';
 
 @Injectable()
 export class SurfaceCostWorker {
   #worker: Worker;
 
   constructor(
     private readonly wrapper: WorkerBuilder,
-    private readonly processor: WorkerProcessor<CostSurfaceJobInput, true>,
+    private readonly processor: SurfaceCostProcessor,
     private readonly eventBus: EventBus,
   ) {
     this.#worker = wrapper.build(queueName, processor);

diff --git a/api/apps/geoprocessing/src/modules/surface-cost/ports/planning-unit-cost.ts b/api/apps/geoprocessing/src/modules/surface-cost/ports/planning-unit-cost.ts
@@ -1,4 +1,12 @@
+/**
+ * when considering Shapefile metadata, properties names are limited to 10
+ * characters:
+ *
+ * https://support.esri.com/en/technical-article/000022868#:~:text=This%20is%20a%20known%20limitation,character%20limitation%20for%20field%20names
+ *
+ * Thus, we shouldn't base on longer key names (like "planningUnitId")
+ */
 export interface PlanningUnitCost {
-  planningUnitId: string;
+  puId: string;
   cost: number;
 }
diff --git a/api/apps/geoprocessing/src/modules/surface-cost/surface-cost.module.ts b/api/apps/geoprocessing/src/modules/surface-cost/surface-cost.module.ts
@@ -34,10 +34,7 @@ import { AvailablePlanningUnitsRepository } from './adapters/available-planning-
   ],
   providers: [
     SurfaceCostWorker,
-    {
-      provide: WorkerProcessor,
-      useClass: SurfaceCostProcessor,
-    },
+    SurfaceCostProcessor,
     {
       provide: CostSurfacePersistencePort,
       useClass: TypeormCostSurface,

diff --git a/...integration/cost-surface/cost-surface-job/process-shapefile-with-cost-surface.e2e-spec.ts b/...integration/cost-surface/cost-surface-job/process-shapefile-with-cost-surface.e2e-spec.ts
@@ -0,0 +1,32 @@
+import { SurfaceCostProcessor } from '@marxan-geoprocessing/modules/surface-cost/application/surface-cost-processor';
+import { INestApplication } from '@nestjs/common';
+import { PromiseType } from 'utility-types';
+import { bootstrapApplication, delay } from '../../../utils';
+import { createWorld } from './steps/world';
+
+let app: INestApplication;
+let sut: SurfaceCostProcessor;
+let world: PromiseType<ReturnType<typeof createWorld>>;
+
+beforeAll(async () => {
+  app = await bootstrapApplication();
+  world = await createWorld(app);
+  sut = app.get(SurfaceCostProcessor);
+});
+
+afterAll(async () => {
+  await world.cleanup();
+  await app?.close();
+}, 500 * 1000);
+
+describe(`given scenario has some planning units`, () => {
+  beforeEach(async () => {
+    await world.GivenPuCostDataExists();
+  });
+  it(`updates cost surface`, async () => {
+    await sut.process(world.getShapefileWithCost());
+    await delay(1000);
+    const costs = (await world.GetPuCostsData()).map((pu) => pu.cost);
+    expect(costs.every((cost) => cost === world.newCost)).toBeTruthy();
+  }, 10000);
+});
diff --git a/api/apps/geoprocessing/test/integration/cost-surface/cost-surface-job/steps/world.ts b/api/apps/geoprocessing/test/integration/cost-surface/cost-surface-job/steps/world.ts
@@ -0,0 +1,76 @@
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-ignore
+import { convert } from 'geojson2shp';
+
+import { INestApplication } from '@nestjs/common';
+import { Feature, Polygon } from 'geojson';
+import { Job } from 'bullmq';
+
+import { AppConfig } from '@marxan-geoprocessing/utils/config.utils';
+import { defaultSrid } from '@marxan-geoprocessing/types/spatial-data-format';
+import { PlanningUnitCost } from '@marxan-geoprocessing/modules/surface-cost/ports/planning-unit-cost';
+import { CostSurfaceJobInput } from '@marxan-geoprocessing/modules/surface-cost/cost-surface-job-input';
+
+import { getFixtures } from '../../planning-unit-fixtures';
+
+export const createWorld = async (app: INestApplication) => {
+  const newCost = 199.99;
+  const fixtures = await getFixtures(app);
+  const shapefile = await getShapefileForPlanningUnits(
+    fixtures.planningUnitsIds,
+    newCost,
+  );
+
+  return {
+    newCost,
+    cleanup: async () => {
+      await fixtures.cleanup();
+    },
+    GivenPuCostDataExists: fixtures.GivenPuCostDataExists,
+    GetPuCostsData: () => fixtures.GetPuCostsData(fixtures.scenarioId),
+    getShapefileWithCost: () =>
+      (({
+        data: {
+          scenarioId: fixtures.scenarioId,
+          shapefile,
+        },
+        id: 'test-job',
+      } as unknown) as Job<CostSurfaceJobInput>),
+  };
+};
+
+const getShapefileForPlanningUnits = async (
+  ids: string[],
+  cost: number,
+): Promise<CostSurfaceJobInput['shapefile']> => {
+  const baseDir = AppConfig.get<string>(
+    'storage.sharedFileStorage.localPath',
+  ) as string;
+  const fileName = 'shape-with-cost';
+  const fileFullPath = `${baseDir}/${fileName}.zip`;
+  const features: Feature<Polygon, PlanningUnitCost>[] = ids.map((puId) => ({
+    type: 'Feature',
+    bbox: [0, 0, 0, 0, 0, 0],
+    geometry: { type: 'Polygon', coordinates: [[[0, 0]]] },
+    properties: { cost, puId },
+  }));
+  await convert(features, fileFullPath, options);
+
+  return {
+    filename: fileName,
+    buffer: {} as any,
+    mimetype: 'application/zip',
+    path: fileFullPath,
+    destination: baseDir,
+    fieldname: 'attachment',
+    size: 1,
+    originalname: `${fileName}.zip`,
+    stream: {} as any,
+    encoding: '',
+  };
+};
+
+const options = {
+  layer: 'my-layer',
+  targetCrs: defaultSrid,
+};
diff --git a/...face-repo/cost-surface-update.e2e-spec.ts → ...face-repo/cost-surface-update.e2e-spec.ts b/...face-repo/cost-surface-update.e2e-spec.ts → ...face-repo/cost-surface-update.e2e-spec.ts
@@ -1,16 +1,19 @@
 import { INestApplication } from '@nestjs/common';
-import { TypeormCostSurface } from '../../../src/modules/surface-cost/adapters/typeorm-cost-surface';
-import { bootstrapApplication } from '../../utils/geo-application';
-import { CostSurfaceUpdateWorld, createWorld } from './world';
-import { CostSurfacePersistencePort } from '../../../src/modules/surface-cost/ports/persistence/cost-surface-persistence.port';
+import { PromiseType } from 'utility-types';
+
+import { TypeormCostSurface } from '../../../../src/modules/surface-cost/adapters/typeorm-cost-surface';
+import { CostSurfacePersistencePort } from '../../../../src/modules/surface-cost/ports/persistence/cost-surface-persistence.port';
+
+import { bootstrapApplication } from '../../../utils';
+import { getFixtures } from '../planning-unit-fixtures';
 
 let app: INestApplication;
 let sut: TypeormCostSurface;
-let world: CostSurfaceUpdateWorld;
+let world: PromiseType<ReturnType<typeof getFixtures>>;
 
 beforeAll(async () => {
   app = await bootstrapApplication();
-  world = await createWorld(app);
+  world = await getFixtures(app);
   sut = app.get(CostSurfacePersistencePort);
 });
 
@@ -33,11 +36,11 @@ describe(`when updating some of the costs`, () => {
     await sut.save(world.scenarioId, [
       {
         cost: 9999,
-        planningUnitId: costOf9999Id,
+        puId: costOf9999Id,
       },
       {
         cost: 1,
-        planningUnitId: costOf1Id,
+        puId: costOf1Id,
       },
     ]);
 

diff --git a/...st/integration/cost-surface-repo/world.ts → ...on/cost-surface/planning-unit-fixtures.ts b/...st/integration/cost-surface-repo/world.ts → ...on/cost-surface/planning-unit-fixtures.ts
@@ -3,23 +3,12 @@ import { getRepositoryToken } from '@nestjs/typeorm';
 import { Repository } from 'typeorm';
 import { v4 } from 'uuid';
 
+import { ScenariosPuCostDataGeo } from '@marxan-geoprocessing/modules/scenarios/scenarios-pu-cost-data.geo.entity';
 import { ScenariosPlanningUnitGeoEntity } from '@marxan/scenarios-planning-unit';
-import { GivenScenarioPuDataExists } from '../../steps/given-scenario-pu-data-exists';
-import { ScenariosPuCostDataGeo } from '../../../src/modules/scenarios/scenarios-pu-cost-data.geo.entity';
 
-export interface CostSurfaceUpdateWorld {
-  cleanup: () => Promise<void>;
-  scenarioId: string;
-  planningUnitsIds: string[];
-  GivenPuCostDataExists: () => Promise<string[]>;
-  GetPuCostsData: (
-    scenarioId: string,
-  ) => Promise<{ scenario_id: string; cost: number; pu_id: string }[]>;
-}
+import { GivenScenarioPuDataExists } from '../../steps/given-scenario-pu-data-exists';
 
-export const createWorld = async (
-  app: INestApplication,
-): Promise<CostSurfaceUpdateWorld> => {
+export const getFixtures = async (app: INestApplication) => {
   const scenarioId = v4();
   const puCostRepoToken = getRepositoryToken(ScenariosPuCostDataGeo);
   const puDataRepoToken = getRepositoryToken(ScenariosPlanningUnitGeoEntity);
@@ -37,6 +26,11 @@ export const createWorld = async (
   const puIds = scenarioPuData.rows.map((row) => row.puGeometryId);
 
   return {
+    planningUnitDataRepo: scenarioPuData,
+    planningUnitCostDataRepo: puCostDataRepo,
+    scenarioId,
+    planningUnitsIds: puIds,
+    scenarioPlanningUnitsGeometry: scenarioPuData,
     GetPuCostsData: async (
       scenarioId: string,
     ): Promise<{ scenario_id: string; cost: number; pu_id: string }[]> =>
@@ -57,8 +51,6 @@ export const createWorld = async (
           ),
         )
         .then((rows) => rows.map((row) => row.planningUnitId)),
-    planningUnitsIds: puIds,
-    scenarioId,
     cleanup: async () => {
       await puDataRepo.delete({
         scenarioId,

diff --git a/api/apps/geoprocessing/test/utils/delay.ts b/api/apps/geoprocessing/test/utils/delay.ts
@@ -0,0 +1,2 @@
+export const delay = (ms = 1000) =>
+  new Promise((resolve) => setTimeout(resolve, ms));
diff --git a/api/apps/geoprocessing/test/utils/index.ts b/api/apps/geoprocessing/test/utils/index.ts
@@ -0,0 +1,2 @@
+export { delay } from './delay';
+export { bootstrapApplication } from './geo-application';
diff --git a/api/package.json b/api/package.json
@@ -48,8 +48,8 @@
     "@nestjs/platform-express": "^7.5.1",
     "@nestjs/swagger": "^4.7.9",
     "@nestjs/typeorm": "^7.1.5",
-    "bcrypt": "^5.0.0",
     "@turf/turf": "^5.1.6",
+    "bcrypt": "^5.0.0",
     "bullmq": "^1.15.1",
     "class-transformer": "^0.3.1",
     "class-validator": "^0.12.2",
@@ -61,8 +61,8 @@
     "http-proxy": "^1.18.1",
     "jsonapi-serializer": "^3.6.7",
     "lodash": "^4.17.20",
-    "ms": "^2.1.3",
     "mapshaper": "0.5.57",
+    "ms": "^2.1.3",
     "multer": "^1.4.2",
     "nestjs-base-service": "0.6.0",
     "passport": "^0.4.1",
@@ -74,11 +74,11 @@
     "reflect-metadata": "^0.1.13",
     "rimraf": "^3.0.2",
     "rxjs": "^6.6.3",
+    "swagger-ui-express": "^4.1.6",
     "temp-dir": "^2.0.0",
     "typeorm": "0.2.30",
     "unzipper": "^0.10.11",
-    "uuid": "8.3.2",
-    "swagger-ui-express": "^4.1.6"
+    "uuid": "8.3.2"
   },
   "devDependencies": {
     "@nestjs/cli": "^7.5.1",
@@ -108,6 +108,7 @@
     "eslint": "^7.12.1",
     "eslint-config-prettier": "^6.15.0",
     "eslint-plugin-prettier": "^3.1.4",
+    "geojson2shp": "0.4.0",
     "husky": "^4.3.0",
     "jest": "^26.6.3",
     "lint-staged": "^10.5.2",