diff --git a/api/apps/api/src/modules/authentication/dto/sign-up.dto.ts b/api/apps/api/src/modules/authentication/dto/sign-up.dto.ts index 2449c9f286..cfc55d17c1 100644 --- a/api/apps/api/src/modules/authentication/dto/sign-up.dto.ts +++ b/api/apps/api/src/modules/authentication/dto/sign-up.dto.ts @@ -1,4 +1,4 @@ -import { ApiProperty } from '@nestjs/swagger'; +import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; import { IsDefined, IsEmail, @@ -16,6 +16,7 @@ import { export class SignUpDto { @IsOptional() @IsString() + @ApiPropertyOptional() displayName?: string; @IsEmail() diff --git a/data/bots/.gitignore b/data/bots/.gitignore new file mode 100644 index 0000000000..ebf4281dc0 --- /dev/null +++ b/data/bots/.gitignore @@ -0,0 +1 @@ +!lib diff --git a/data/bots/README.md b/data/bots/README.md index 33db65804e..f4e740cee2 100644 --- a/data/bots/README.md +++ b/data/bots/README.md @@ -4,19 +4,52 @@ This folder contains friendly bots for the Marxan API. They take care of automating things. -How to run them: -1.- Install deno if not already installed: +## Running + +1. Install deno if not already installed + `curl -fsSL https://deno.land/x/install/install.sh | sh` -create a .env file with the next info: + +2. Configure the bot(s) you wish to run: this is done via an `.env` file in the + bot's directory + ``` -API_URL=http://localhost:3030 -USERNAME= -PASSWORD= -POSTGRES_URL= +API_URL= +USERNAME= +PASSWORD= +POSTGRES_URL= ``` -2.- Run demo cases: -`deno run --allow-all bot.ts` +Individual demo bots (`demo-brazil` and `demo-australia`) are currently +expecting the user whose credentials are configured in `.env` to already exist +in the API instance configured, when run individually. + +The `core-demos` bot needs the same `.env` file, but it will first create a new +user with the credentials configured, then run the Brazil and Australia demo +bots using a JWT for the newly created user. + +3. Run bot + +`OPTIC_MIN_LEVEL=Info deno run --allow-read --allow-net --allow-env ./bot.ts` + +For a cleaner output (just errors) set `OPTIC_MIN_LEVEL=Error`; to see debug +output (mostly `inspect` of data from API responses), set +`OPTIC_MIN_LEVEL=Debug`. + +## Developing on bots + +Things are quite in flux right now, but in general + +* `libbot` is where new bot functionality should be added, via classes that take + care of a single domain of the platform. +* `libbot/scenario-status` currently handles waiting for scenario status changes + for all kinds of async operations; it may be advisable to split the + operation-specific waiting code to individual modules if the current module + grows further +* do not forget to run `deno fmt` before committing new code + +## Compiling bots -3.- Compile demo cases: -`deno compile ` \ No newline at end of file +If wanting to run bots somewhere without having to copy over the source tree, +or if it is not possible or practical to install the Deno runtime, bots can +be compiled to a single binary via `deno compile`. diff --git a/data/bots/core-demos/bot.ts b/data/bots/core-demos/bot.ts new file mode 100644 index 0000000000..cbc39b668a --- /dev/null +++ b/data/bots/core-demos/bot.ts @@ -0,0 +1,28 @@ +import { + dirname, + fromFileUrl, + relative, +} from "https://deno.land/std@0.103.0/path/mod.ts"; +import { config } from "https://deno.land/x/dotenv@v2.0.0/mod.ts"; +import { runBot as runBrazilBot } from "./demo-brazil/core.ts"; +import { runBot as runAustraliaBot } from "./demo-australia/core.ts"; +import { Users } from "../lib/libbot/users.ts"; + +const scriptPath = dirname(relative(Deno.cwd(), fromFileUrl(import.meta.url))); + +const { API_URL, USERNAME, PASSWORD, POSTGRES_URL } = config({ + path: scriptPath + "/.env", +}); + +const settings = { + apiUrl: API_URL, + credentials: { + username: USERNAME, + password: PASSWORD, + }, +}; + +await new Users(settings).signUp(); + +await runBrazilBot(settings); +await runAustraliaBot(settings); diff --git a/data/bots/core-demos/demo-australia/bot.ts b/data/bots/core-demos/demo-australia/bot.ts new file mode 100644 index 0000000000..2e8e5857a7 --- /dev/null +++ b/data/bots/core-demos/demo-australia/bot.ts @@ -0,0 +1,23 @@ +import { + dirname, + fromFileUrl, + relative, +} from "https://deno.land/std@0.103.0/path/mod.ts"; +import { config } from "https://deno.land/x/dotenv@v2.0.0/mod.ts"; +import { runBot } from "./core.ts"; + +const scriptPath = dirname(relative(Deno.cwd(), fromFileUrl(import.meta.url))); + +const { API_URL, USERNAME, PASSWORD, POSTGRES_URL } = config({ + path: scriptPath + "/.env", +}); + +const settings = { + apiUrl: API_URL, + credentials: { + username: USERNAME, + password: PASSWORD, + }, +}; + +await runBot(settings); diff --git a/data/bots/core-demos/demo-australia/core.ts b/data/bots/core-demos/demo-australia/core.ts new file mode 100644 index 0000000000..beddda079b --- /dev/null +++ b/data/bots/core-demos/demo-australia/core.ts @@ -0,0 +1,109 @@ +import { + dirname, + fromFileUrl, + relative, +} from "https://deno.land/std@0.103.0/path/mod.ts"; +import { sleep } from "https://deno.land/x/sleep@v1.2.0/mod.ts"; +import { createBot } from "../../lib/libbot/init.ts"; +import { MarxanBotConfig } from "../../lib/libbot/marxan-bot.ts"; +import { SpecificationStatus } from "../../lib/libbot/geo-feature-specifications.ts"; +import { getDemoFeatureSpecificationFromFeatureNamesForProject } from "./lib.ts"; + +const scriptPath = dirname(relative(Deno.cwd(), fromFileUrl(import.meta.url))); + +export const runBot = async (settings: MarxanBotConfig) => { + const bot = await createBot(settings); + + const organization = await bot.organizations.create({ + name: "[demo] Australia - Kimberley " + crypto.randomUUID(), + description: "", + }); + + const planningUnitAreakm2 = 2000; + + const planningAreaId = await bot.planningAreaUploader.uploadFromFile( + `${scriptPath}/kimberley.zip`, + ); + const project = await bot.projects.createInOrganization(organization.id, { + name: "Australia - Kimberley " + crypto.randomUUID(), + description: "", + planningAreaId: planningAreaId, + planningUnitGridShape: "hexagon", + planningUnitAreakm2, + }); + + // We don't expose status of planning unit grid calculations yet so we cannot + // poll for completion of this task, but a reasonable, project-specific wait + // here will do when there is nothing else keeping the API and PostgreSQL busy. + await sleep(30); + + // Scenario creation with the bare minimum; From there we need to be setting + // other traits via patch. + const scenario = await bot.scenarios.createInProject(project.id, { + name: `Kimberley - scenario 01`, + type: "marxan", + description: "Demo scenario", + metadata: bot.metadata.analysisPreview(), + }); + + // get the list of protected areas in the region and use all of them + const paCategories = await bot.protectedAreas + .getIucnCategoriesForPlanningAreaWithId(planningAreaId); + + await bot.scenarios.update(scenario.id, { + wdpaIucnCategories: paCategories, + }); + + await bot.scenarios.update(scenario.id, { + wdpaThreshold: 50, + metadata: bot.metadata.analysisPreview(), + }); + + await bot.scenarioStatus.waitForPlanningAreaProtectedCalculationFor( + project.id, + scenario.id, + "short", + ); + + //Setup features in the project + const wantedFeatures = [ + // "demo_ecoregions_new_class_split", + // "calidris_(erolia)_ferruginea", + // "chlamydosaurus_kingii", + // "erythrura_(chloebia)_gouldiae", + // "haliaeetus_(pontoaetus)_leucogaster", + // "malurus_(malurus)_coronatus", + // "mesembriomys_macrurus", + // "onychogalea_unguifera", + // "pseudechis_australis", + // "wyulda_squamicaudata", + "zyzomys_woodwardi", + ]; + + const featuresForSpecification = + await getDemoFeatureSpecificationFromFeatureNamesForProject( + project.id, + bot, + wantedFeatures, + ); + + await bot.geoFeatureSpecifications.submitForScenario( + scenario.id, + featuresForSpecification, + SpecificationStatus.created, + ); + + await bot.scenarioStatus.waitForFeatureSpecificationCalculationFor( + project.id, + scenario.id, + "some", + ); + + await bot.marxanExecutor.runForScenario(scenario.id); + + await bot.scenarioStatus.waitForMarxanCalculationsFor( + project.id, + scenario.id, + "some", + ); +}; diff --git a/data/bots/demo-australia/kimberley.zip b/data/bots/core-demos/demo-australia/kimberley.zip similarity index 100% rename from data/bots/demo-australia/kimberley.zip rename to data/bots/core-demos/demo-australia/kimberley.zip diff --git a/data/bots/core-demos/demo-australia/lib.ts b/data/bots/core-demos/demo-australia/lib.ts new file mode 100644 index 0000000000..88e5d2c046 --- /dev/null +++ b/data/bots/core-demos/demo-australia/lib.ts @@ -0,0 +1,38 @@ +import { logDebug } from "../../lib/libbot/logger.ts"; +import _ from "https://deno.land/x/lodash@4.17.15-es/lodash.js"; +import { Bot } from "../../lib/libbot/init.ts"; + +export const getDemoFeatureSpecificationFromFeatureNamesForProject = async ( + projectId: string, + bot: Bot, + wantedFeatures: string[], +) => { + const geoFeatureIds = await Promise.all( + wantedFeatures.map(async (f) => + (await bot.geoFeatures.getIdFromQueryStringInProject(projectId, f))[0] + ), + ); + + logDebug( + `geoFeatureIds for inclusion in specification:\n${ + Deno.inspect(geoFeatureIds) + }`, + ); + + const featuresForSpecification = geoFeatureIds.map((i) => ({ + kind: "plain", + featureId: i.id, + marxanSettings: { + prop: 0.3, + fpf: 1, + }, + })); + + logDebug( + `Features for specification:\n${ + Deno.inspect(featuresForSpecification, { depth: 6 }) + }`, + ); + + return featuresForSpecification; +}; diff --git a/data/bots/core-demos/demo-brazil/bot.ts b/data/bots/core-demos/demo-brazil/bot.ts new file mode 100644 index 0000000000..2e8e5857a7 --- /dev/null +++ b/data/bots/core-demos/demo-brazil/bot.ts @@ -0,0 +1,23 @@ +import { + dirname, + fromFileUrl, + relative, +} from "https://deno.land/std@0.103.0/path/mod.ts"; +import { config } from "https://deno.land/x/dotenv@v2.0.0/mod.ts"; +import { runBot } from "./core.ts"; + +const scriptPath = dirname(relative(Deno.cwd(), fromFileUrl(import.meta.url))); + +const { API_URL, USERNAME, PASSWORD, POSTGRES_URL } = config({ + path: scriptPath + "/.env", +}); + +const settings = { + apiUrl: API_URL, + credentials: { + username: USERNAME, + password: PASSWORD, + }, +}; + +await runBot(settings); diff --git a/data/bots/core-demos/demo-brazil/core.ts b/data/bots/core-demos/demo-brazil/core.ts new file mode 100644 index 0000000000..94fcd9f1cb --- /dev/null +++ b/data/bots/core-demos/demo-brazil/core.ts @@ -0,0 +1,110 @@ +import { + dirname, + fromFileUrl, + relative, +} from "https://deno.land/std@0.103.0/path/mod.ts"; +import { sleep } from "https://deno.land/x/sleep@v1.2.0/mod.ts"; +import { createBot } from "../../lib/libbot/init.ts"; +import { MarxanBotConfig } from "../../lib/libbot/marxan-bot.ts"; +import { SpecificationStatus } from "../../lib/libbot/geo-feature-specifications.ts"; +import { getDemoFeatureSpecificationFromFeatureNamesForProject } from "./lib.ts"; + +const scriptPath = dirname(relative(Deno.cwd(), fromFileUrl(import.meta.url))); + +export const runBot = async (settings: MarxanBotConfig) => { + const bot = await createBot(settings); + + const organization = await bot.organizations.create({ + name: "[demo] Brazil " + crypto.randomUUID(), + description: "", + }); + + const planningUnitAreakm2 = 2000; + + const planningAreaId = await bot.planningAreaUploader.uploadFromFile( + `${scriptPath}/test_mata.zip`, + ); + + const project = await bot.projects.createInOrganization(organization.id, { + name: "Brazil " + crypto.randomUUID(), + description: "", + planningAreaId: planningAreaId, + planningUnitGridShape: "hexagon", + planningUnitAreakm2, + }); + + // We don't expose status of planning unit grid calculations yet so we cannot + // poll for completion of this task, but a reasonable, project-specific wait + // here will do when there is nothing else keeping the API and PostgreSQL busy. + await sleep(30); + + // Scenario creation with the bare minimum; From there we need to be setting + // other traits via patch. + const scenario = await bot.scenarios.createInProject(project.id, { + name: `Brazil - scenario 01`, + type: "marxan", + description: "Demo scenario", + metadata: bot.metadata.analysisPreview(), + }); + + // get the list of protected areas in the region and use all of them + const paCategories = await bot.protectedAreas + .getIucnCategoriesForPlanningAreaWithId(planningAreaId); + + await bot.scenarios.update(scenario.id, { + wdpaIucnCategories: paCategories, + }); + + await bot.scenarios.update(scenario.id, { + wdpaThreshold: 50, + metadata: bot.metadata.analysisPreview(), + }); + + await bot.scenarioStatus.waitForPlanningAreaProtectedCalculationFor( + project.id, + scenario.id, + "short", + ); + + //Setup features in the project + const wantedFeatures = [ + // "demo_ecoregions_new_class_split", + // "buteogallus_urubitinga", + // "caluromys_philander", + // "chiroxiphia_caudata", + // "leopardus_pardalis", + // "megarynchus_pitangua", + // "phyllodytes_tuberculosus", + // "priodontes_maximus", + // "proceratophrys_bigibbosa", + // "tapirus_terrestris", + "thalurania_glaucopis", + ]; + + const featuresForSpecification = + await getDemoFeatureSpecificationFromFeatureNamesForProject( + project.id, + bot, + wantedFeatures, + ); + + await bot.geoFeatureSpecifications.submitForScenario( + scenario.id, + featuresForSpecification, + SpecificationStatus.created, + ); + + await bot.scenarioStatus.waitForFeatureSpecificationCalculationFor( + project.id, + scenario.id, + "short", + ); + + await bot.marxanExecutor.runForScenario(scenario.id); + + await bot.scenarioStatus.waitForMarxanCalculationsFor( + project.id, + scenario.id, + "some", + ); +}; diff --git a/data/bots/core-demos/demo-brazil/lib.ts b/data/bots/core-demos/demo-brazil/lib.ts new file mode 100644 index 0000000000..88e5d2c046 --- /dev/null +++ b/data/bots/core-demos/demo-brazil/lib.ts @@ -0,0 +1,38 @@ +import { logDebug } from "../../lib/libbot/logger.ts"; +import _ from "https://deno.land/x/lodash@4.17.15-es/lodash.js"; +import { Bot } from "../../lib/libbot/init.ts"; + +export const getDemoFeatureSpecificationFromFeatureNamesForProject = async ( + projectId: string, + bot: Bot, + wantedFeatures: string[], +) => { + const geoFeatureIds = await Promise.all( + wantedFeatures.map(async (f) => + (await bot.geoFeatures.getIdFromQueryStringInProject(projectId, f))[0] + ), + ); + + logDebug( + `geoFeatureIds for inclusion in specification:\n${ + Deno.inspect(geoFeatureIds) + }`, + ); + + const featuresForSpecification = geoFeatureIds.map((i) => ({ + kind: "plain", + featureId: i.id, + marxanSettings: { + prop: 0.3, + fpf: 1, + }, + })); + + logDebug( + `Features for specification:\n${ + Deno.inspect(featuresForSpecification, { depth: 6 }) + }`, + ); + + return featuresForSpecification; +}; diff --git a/data/bots/demo-brazil/test_mata.zip b/data/bots/core-demos/demo-brazil/test_mata.zip similarity index 100% rename from data/bots/demo-brazil/test_mata.zip rename to data/bots/core-demos/demo-brazil/test_mata.zip diff --git a/data/bots/demo-okavango/bot.ts b/data/bots/core-demos/demo-okavango/bot.ts similarity index 90% rename from data/bots/demo-okavango/bot.ts rename to data/bots/core-demos/demo-okavango/bot.ts index 268a9174b3..11baeaf72d 100644 --- a/data/bots/demo-okavango/bot.ts +++ b/data/bots/core-demos/demo-okavango/bot.ts @@ -64,7 +64,7 @@ console.log(organization); const planningAreaFile = await ( await sendData( API_URL + "/api/v1/projects/planning-area/shapefile", - new Blob([await Deno.readFile(scriptPath + "/corsica.zip")]) + new Blob([await Deno.readFile(scriptPath + "/corsica.zip")]), ) ).json(); @@ -87,8 +87,9 @@ const project = await botClient console.log(project); -await pgClient.connect() -await pgClient.queryArray(`INSERT INTO planning_units_geom (the_geom, type, size) +await pgClient.connect(); +await pgClient.queryArray( + `INSERT INTO planning_units_geom (the_geom, type, size) select st_transform(geom, 4326) as the_geom, 'hexagon' as type, ${planningUnitAreakm2} as size from ( @@ -97,7 +98,8 @@ ${planningUnitAreakm2} as size from ( FROM planning_areas a WHERE project_id = '${project.data.id}' ) grid -ON CONFLICT ON CONSTRAINT planning_units_geom_the_geom_type_key DO NOTHING;`); +ON CONFLICT ON CONSTRAINT planning_units_geom_the_geom_type_key DO NOTHING;`, +); await pgClient.end(); // wait a bit for async job to be picked up and processed @@ -117,14 +119,14 @@ const scenario = await botClient metadata: { scenarioEditingMetadata: { status: { - 'protected-areas': 'draft', - features: 'draft', - analysis: 'draft', + "protected-areas": "draft", + features: "draft", + analysis: "draft", }, - tab: 'analysis', - subtab: 'analysis-preview', - } - } + tab: "analysis", + subtab: "analysis-preview", + }, + }, }) .then((result) => result.data) .catch((e) => { @@ -132,7 +134,7 @@ const scenario = await botClient }); const scenarioTook = Process.hrtime(scenarioStart); -console.log(`Scenario creation done in ${scenarioTook[0]} seconds`); +console.log(`Scenario creation done in ${scenarioTook[0]}ms`); console.log(scenario); @@ -169,7 +171,7 @@ const geoFeatureSpec = await botClient const geoFeatureSpecTook = Process.hrtime(geoFeatureSpecStart); console.log( - `Processing of features for scenario done in ${geoFeatureSpecTook[0]} seconds` + `Processing of features for scenario done in ${geoFeatureSpecTook[0]}ms`, ); -console.log(geoFeatureSpec); \ No newline at end of file +console.log(geoFeatureSpec); diff --git a/data/bots/demo-okavango/planning-area.zip b/data/bots/core-demos/demo-okavango/planning-area.zip similarity index 100% rename from data/bots/demo-okavango/planning-area.zip rename to data/bots/core-demos/demo-okavango/planning-area.zip diff --git a/data/bots/demo-australia/bot.ts b/data/bots/demo-australia/bot.ts deleted file mode 100644 index 3b0ad75a3e..0000000000 --- a/data/bots/demo-australia/bot.ts +++ /dev/null @@ -1,199 +0,0 @@ -import axiod from "https://deno.land/x/axiod@0.22/mod.ts"; -import Process from "https://deno.land/std@0.103.0/node/process.ts"; -import { - dirname, - fromFileUrl, - relative, -} from "https://deno.land/std@0.103.0/path/mod.ts"; -import { config } from "https://deno.land/x/dotenv@v2.0.0/mod.ts"; -import { Client } from "https://deno.land/x/postgres@v0.11.3/mod.ts"; -import { sleep } from "https://deno.land/x/sleep/mod.ts"; - -const scriptPath = dirname(relative(Deno.cwd(), fromFileUrl(import.meta.url))); - -const { API_URL, USERNAME, PASSWORD, POSTGRES_URL } = config({ - path: scriptPath + "/.env", -}); - -const settings = { - apiUrl: API_URL, - credentials: { - username: USERNAME, - password: PASSWORD, - }, -}; - -const jwt = await axiod - .post(`${settings.apiUrl}/auth/sign-in/`, settings.credentials) - .then((result) => result.data.accessToken); - -// const pgClient = new Client(POSTGRES_URL); - -const botClient = axiod.create({ - baseURL: settings.apiUrl + "/api/v1", - headers: { - Authorization: "Bearer " + jwt, - }, -}); - -async function sendData(url: string, data: Blob) { - const formData = new FormData(); - formData.append("file", data, "test_mata.zip"); - - const response = await fetch(url, { - method: "POST", - body: formData, - headers: [["authorization", "Bearer " + jwt]], - }); - - return response; -} - -const organization = await botClient - .post("/organizations", { - name: "Australia - Kimberley organization", - description: "Duis aliquip nostrud sint", - metadata: {}, - }) - .then((result) => result.data) - .catch((e) => { - console.log(e); - }); - -console.log(organization); - -const planningAreaFile = await ( - await sendData( - settings.apiUrl + "/api/v1/projects/planning-area/shapefile", - new Blob([await Deno.readFile(scriptPath + "/kimberley.zip")]) - ) -).json(); - -console.log(planningAreaFile); - -const planningUnitAreakm2 = 50; - -const project = await botClient - .post("/projects", { - name: "Australia - Kimberley project", - organizationId: organization.data.id, - planningUnitGridShape: "hexagon", - planningUnitAreakm2: planningUnitAreakm2, - planningAreaId: planningAreaFile.id, - }) - .then((result) => result.data) - .catch((e) => { - console.log(e); - }); - -console.log(project); - -// wait a bit for async job to be picked up and processed -// @DEBT we should check the actual job status -// await new Promise((r) => setTimeout(r, 30e3)); - -const scenarioStart = Process.hrtime(); - -await sleep(5) - -const paCategories:{data:Array<{id:string, type:string, attributes:object}>} = await botClient.get(`/protected-areas/iucn-categories?filter%5BcustomAreaId%5D=${planningAreaFile.id}`) - .then((result) => result.data) - .catch((e) => { - console.log(e); - }); - -console.log(paCategories); - - -const scenario = await botClient - .post("/scenarios", { - name: `Kimberley scenario`, - type: "marxan", - projectId: project.data.id, - description: "An Australia scenario", - metadata: { - scenarioEditingMetadata: { - status: { - 'protected-areas': 'draft', - features: 'draft', - }, - tab: 'analysis', - subtab: 'analysis-preview', - } - } - }) - .then((result) => result.data) - .catch((e) => { - console.log(e); - }); - await sleep(5) -// get the list of protected areas in the region and use all of them - -await botClient - .patch(`/scenarios/${scenario!.data!.id}`, { - wdpaIucnCategories: paCategories!.data.map((i: {id:string, type:string, attributes:object}): string => i.id), - }).catch((e) => { - console.log(e); - }); - -console.log(scenario); - -await sleep(20) - -await botClient - .patch(`/scenarios/${scenario!.data!.id}`, { - wdpaThreshold: 50, - }).catch((e) => { - console.log(e); - }); - -const scenarioTook = Process.hrtime(scenarioStart); -console.log(`Scenario creation done in ${scenarioTook[0]} seconds`); - -await botClient.get(`/scenarios/${scenario!.data!.id}`) - .then((result) => console.log(result.data)) - .catch((e) => { - console.log(e); - }); - -await sleep(5) - -// Setup features in the project - -const features = await botClient - .get(`/projects/${project.data.id}/features?q=demo`) - .then((result) => result.data) - .catch((e) => { - console.log(e); - }); - -console.log(features); - -const geoFeatureSpecStart = Process.hrtime(); - -const featureRecipe = features!.data.map((x: {id:string, type:string, attributes:object}) => { return { - kind: "plain", - featureId: x.id, - marxanSettings: { - prop: 0.3, - fpf: 1, - }, - }}) -console.log(featureRecipe); -const geoFeatureSpec = await botClient - .post(`/scenarios/${scenario.data.id}/features/specification`, { - status: "draft", - features: featureRecipe - }) - .then((result) => result.data) - .catch((e) => { - console.log(e); - }); - -const geoFeatureSpecTook = Process.hrtime(geoFeatureSpecStart); - -console.log( - `Processing of features for scenario done in ${geoFeatureSpecTook[0]} seconds` -); - -// console.log(geoFeatureSpec); diff --git a/data/bots/demo-brazil/bot.ts b/data/bots/demo-brazil/bot.ts deleted file mode 100644 index 9c10a4fe76..0000000000 --- a/data/bots/demo-brazil/bot.ts +++ /dev/null @@ -1,270 +0,0 @@ -import axiod from "https://deno.land/x/axiod@0.22/mod.ts"; -import Process from "https://deno.land/std@0.103.0/node/process.ts"; -import { - dirname, - fromFileUrl, - relative, -} from "https://deno.land/std@0.103.0/path/mod.ts"; -import { config } from "https://deno.land/x/dotenv@v2.0.0/mod.ts"; -import { Client } from "https://deno.land/x/postgres@v0.11.3/mod.ts"; -import { sleep } from "https://deno.land/x/sleep@v1.2.0/mod.ts"; - -const scriptPath = dirname(relative(Deno.cwd(), fromFileUrl(import.meta.url))); - -const { API_URL, USERNAME, PASSWORD, POSTGRES_URL } = config({ - path: scriptPath + "/.env", -}); - -const settings = { - apiUrl: API_URL, - credentials: { - username: USERNAME, - password: PASSWORD, - }, -}; - -const jwt = await axiod - .post(`${settings.apiUrl}/auth/sign-in/`, settings.credentials) - .then((result) => result.data.accessToken); - -// const pgClient = new Client(POSTGRES_URL); - -const botClient = axiod.create({ - baseURL: settings.apiUrl + "/api/v1", - headers: { - Authorization: "Bearer " + jwt, - }, -}); - -async function sendData(url: string, data: Blob) { - const formData = new FormData(); - formData.append("file", data, "test_mata.zip"); - - const response = await fetch(url, { - method: "POST", - body: formData, - headers: [["authorization", "Bearer " + jwt]], - }); - - return response; -} - -// const MAX_TRYS = 10; TRY_TIMEOUT = 500; -// function toTry() { -// return new Promise((ok, fail) => { -// setTimeout(() => Math.random() < 0.05 ? ok("OK!") : fail("Error"), TRY_TIMEOUT); -// }); -// } -// async function tryNTimes(toTry, count = MAX_TRYS) { -// if (count > 0) { -// const result = await toTry().catch(e => e); -// if (result === "Error") { return await tryNTimes(toTry, count - 1) } -// return result -// } -// return `Tried ${MAX_TRYS} times and failed`; -// } - -// tryNTimes(toTry).then(console.log); - - -async function checkScenarioStatus(id: string) { - return await botClient.get(`/projects​/${id}​/scenarios​/status`, {}) - .then((result) => result.data) - .catch((e) => { - console.log(e); - }); -} - -const organization = await botClient - .post("/organizations", { - name: "Brazil - Atlantic forest organization", - description: "Duis aliquip nostrud sint", - metadata: {}, - }) - .then((result) => result.data) - .catch((e) => { - console.log(e); - }); - -console.log(organization); - -const planningAreaFile = await ( - await sendData( - settings.apiUrl + "/api/v1/projects/planning-area/shapefile", - new Blob([await Deno.readFile(scriptPath + "/test_mata.zip")]) - ) -).json(); - -console.log(planningAreaFile); - -const planningUnitAreakm2 = 50; - -const project = await botClient - .post("/projects", { - name: "Brazil project", - organizationId: organization.data.id, - planningUnitGridShape: "hexagon", - planningUnitAreakm2: planningUnitAreakm2, - planningAreaId: planningAreaFile.id, - }) - .then((result) => result.data) - .catch((e) => { - console.log(e); - }); - -console.log(project); -// wait a bit for async job to be picked up and processed -// @DEBT we should check the actual job status -// await new Promise((r) => setTimeout(r, 30e3)); - -const scenarioStart = Process.hrtime(); - -await sleep(10) - -// Scenario creation with the bare minimum; From there we need to be doin patches to the same scenario -let scenario = await botClient - .post("/scenarios", { - name: `Brazil scenario`, - type: "marxan", - projectId: project.data.id, - description: "A Brazil scenario", - metadata: { - scenarioEditingMetadata: { - status: { - 'protected-areas': 'draft' - }, - tab: 'analysis', - subtab: 'analysis-preview', - } - }, - status: "draft" - }) - .then((result) => result.data) - .catch((e) => { - console.log(e); - }); - - console.log(scenario); - - - console.log(await checkScenarioStatus(project!.data!.id)); - -// get the list of protected areas in the region and use all of them -const paCategories:{data:Array<{id:string, type:string, attributes:object}>} = await botClient.get(`/protected-areas/iucn-categories?filter%5BcustomAreaId%5D=${planningAreaFile.id}`) - .then((result) => result.data) - .catch((e) => { - console.log(e); - }); - -console.log(paCategories); - -await botClient - .patch(`/scenarios/${scenario!.data!.id}`, { - wdpaIucnCategories: paCategories!.data.map((i: {id:string, type:string, attributes:object}): string => i.id), - }).catch((e) => { - console.log(e); - }); - -console.log(scenario); - -await sleep(30) - -await botClient - .patch(`/scenarios/${scenario!.data!.id}`, { - wdpaThreshold: 50, - metadata: { - scenarioEditingMetadata: { - status: { - 'protected-areas': 'draft', - features: 'draft', - analysis: 'draft', - }, - tab: 'analysis', - subtab: 'analysis-preview', - } - } - }).catch((e) => { - console.log(e); - }); - -const scenarioTook = Process.hrtime(scenarioStart); -console.log(`Scenario creation done in ${scenarioTook[0]} seconds`); - -await botClient.get(`/scenarios/${scenario!.data!.id}`) - .then((result) => console.log(result.data)) - .catch((e) => { - console.log(e); - }); - -await sleep(10) - -//Setup features in the project - -// const featureList = [ -// "demo_ecoregions_new_class_split", -// "demo_buteogallus_urubitinga", -// "demo_caluromys_philander", -// "demo_chiroxiphia_caudata", -// "demo_leopardus_pardalis", -// "demo_megarynchus_pitangua", -// "demo_phyllodytes_tuberculosus", -// "demo_tapirus_terrestris", -// "demo_thalurania_glaucopis", -// ] -const features = await botClient - .get(`/projects/${project.data.id}/features?q=demo`) - .then((result) => result.data) - .catch((e) => { - console.log(e); - }); - -console.log(features); - -const geoFeatureSpecStart = Process.hrtime(); - -const featureRecipe = features!.data.map((x: {id:string, type:string, attributes:object}) => { return { - kind: "plain", - featureId: x.id, - marxanSettings: { - prop: 0.3, - fpf: 1, - }, - }}) - console.log(featureRecipe); -const geoFeatureSpec = await botClient - .post(`/scenarios/${scenario.data.id}/features/specification/v2`, { - status: "created", - features: [ - { - kind: "plain", - featureId: features.data[0].id, - marxanSettings: { - prop: 0.3, - fpf: 1, - }, - }, - ], - }) - .then((result) => result.data) - .catch((e) => { - console.log(e); - }); - // metadata: { - // scenarioEditingMetadata: { - // status: { - // 'protected-areas': 'draft', - // features: 'draft', - // analysis: 'draft', - // }, - // tab: 'analysis', - // subtab: 'analysis-preview', - // } - // }, - -const geoFeatureSpecTook = Process.hrtime(geoFeatureSpecStart); - -console.log( - `Processing of features for scenario done in ${geoFeatureSpecTook[0]} seconds` -); - -console.log(geoFeatureSpec); diff --git a/data/bots/lib/libbot/geo-feature-specifications.ts b/data/bots/lib/libbot/geo-feature-specifications.ts new file mode 100644 index 0000000000..e5de872266 --- /dev/null +++ b/data/bots/lib/libbot/geo-feature-specifications.ts @@ -0,0 +1,33 @@ +import { BotHttpClient, getJsonApiDataFromResponse } from "./marxan-bot.ts"; +import { logError } from "./logger.ts"; + +export enum SpecificationStatus { + draft = "draft", + created = "created", +} + +export class GeoFeatureSpecifications { + private baseHttpClient; + + constructor(httpClient: BotHttpClient) { + this.baseHttpClient = httpClient.baseHttpClient; + } + + async submitForScenario( + scenarioId: string, + featuresForSpecification: Record[], + status: SpecificationStatus = SpecificationStatus.draft, + ): Promise { + const specification = { + status, + features: featuresForSpecification, + }; + + return await this.baseHttpClient.post( + `/scenarios/${scenarioId}/features/specification/v2`, + specification, + ) + .then(getJsonApiDataFromResponse) + .catch(logError); + } +} diff --git a/data/bots/lib/libbot/geo-features.ts b/data/bots/lib/libbot/geo-features.ts new file mode 100644 index 0000000000..04f475782b --- /dev/null +++ b/data/bots/lib/libbot/geo-features.ts @@ -0,0 +1,21 @@ +import { BotHttpClient, getJsonApiDataFromResponse } from "./marxan-bot.ts"; +import { logError } from "./logger.ts"; + +export class GeoFeatures { + private baseHttpClient; + + constructor(httpClient: BotHttpClient) { + this.baseHttpClient = httpClient.baseHttpClient; + } + + async getIdFromQueryStringInProject( + projectId: string, + name: string, + ) { + return await this.baseHttpClient.get( + `/projects/${projectId}/features?q=${name}&fields=id`, + ) + .then(getJsonApiDataFromResponse) + .catch(logError); + } +} diff --git a/data/bots/lib/libbot/init.ts b/data/bots/lib/libbot/init.ts new file mode 100644 index 0000000000..a744c6ca44 --- /dev/null +++ b/data/bots/lib/libbot/init.ts @@ -0,0 +1,50 @@ +import { BotHttpClient, MarxanBotConfig } from "./marxan-bot.ts"; +import { Organizations } from "./organizations.ts"; +import { Projects } from "./projects.ts"; +import { Scenarios } from "./scenarios.ts"; +import { ScenarioEditingMetadata } from "./scenario-editing-metadata.ts"; +import { PlanningAreaShapefiles } from "./planning-area-shapefiles.ts"; +import { ProtectedAreas } from "./protected-areas.ts"; +import { GeoFeatureSpecifications } from "./geo-feature-specifications.ts"; +import { GeoFeatures } from "./geo-features.ts"; +import { MarxanCalculations } from "./marxan-calculations.ts"; +import { ScenarioJobStatus } from "./scenario-status.ts"; + +export interface Bot { + organizations: Organizations; + projects: Projects; + scenarios: Scenarios; + scenarioStatus: ScenarioJobStatus; + geoFeatures: GeoFeatures; + geoFeatureSpecifications: GeoFeatureSpecifications; + planningAreaUploader: PlanningAreaShapefiles; + protectedAreas: ProtectedAreas; + marxanExecutor: MarxanCalculations; + metadata: ScenarioEditingMetadata; +} + +export const createBot = async (botConfig: MarxanBotConfig): Promise => { + const httpClient = await BotHttpClient.init({ + apiUrl: botConfig?.apiUrl, + credentials: { + username: botConfig?.credentials.username, + password: botConfig?.credentials.password, + }, + }); + + return { + organizations: new Organizations(httpClient), + projects: new Projects(httpClient), + scenarios: new Scenarios(httpClient), + scenarioStatus: new ScenarioJobStatus(httpClient), + planningAreaUploader: new PlanningAreaShapefiles( + httpClient, + botConfig.apiUrl, + ), + protectedAreas: new ProtectedAreas(httpClient), + geoFeatures: new GeoFeatures(httpClient), + geoFeatureSpecifications: new GeoFeatureSpecifications(httpClient), + marxanExecutor: new MarxanCalculations(httpClient), + metadata: new ScenarioEditingMetadata(), + }; +}; diff --git a/data/bots/lib/libbot/logger.ts b/data/bots/lib/libbot/logger.ts new file mode 100644 index 0000000000..65b9409546 --- /dev/null +++ b/data/bots/lib/libbot/logger.ts @@ -0,0 +1,15 @@ +import { Logger } from "https://deno.land/x/optic/mod.ts"; + +const logger = new Logger(); + +export const logError = (e: unknown) => { + logger.error(JSON.stringify(e, undefined, 2)); +}; + +export const logInfo = (e: unknown) => { + logger.info(JSON.stringify(e, undefined, 2)); +}; + +export const logDebug = (e: unknown) => { + logger.debug(JSON.stringify(e, undefined, 2)); +}; diff --git a/data/bots/lib/libbot/marxan-bot.ts b/data/bots/lib/libbot/marxan-bot.ts new file mode 100644 index 0000000000..d6dcfd0d5b --- /dev/null +++ b/data/bots/lib/libbot/marxan-bot.ts @@ -0,0 +1,42 @@ +import axiod from "https://deno.land/x/axiod@0.22/mod.ts"; +import { IAxiodResponse } from "https://deno.land/x/axiod@0.22/interfaces.ts"; + +export interface MarxanBotConfig { + apiUrl: string; + credentials: { + username: string; + password: string; + }; +} + +const marxanBotBaseSettings = { + baseUrl: "/api/v1", +}; + +export const getJsonApiDataFromResponse = (response: IAxiodResponse) => { + return response?.data?.data; +}; + +export class BotHttpClient { + constructor(config: MarxanBotConfig, jwt: string) { + this.baseHttpClient = axiod.create({ + baseURL: config.apiUrl + marxanBotBaseSettings.baseUrl, + headers: { + Authorization: "Bearer " + jwt, + }, + }); + + this.currentJwt = jwt; + } + + public baseHttpClient; + public currentJwt; + + static async init(config: MarxanBotConfig) { + const jwt = await axiod + .post(`${config.apiUrl}/auth/sign-in/`, config.credentials) + .then((result) => result.data.accessToken); + + return new BotHttpClient(config, jwt); + } +} diff --git a/data/bots/lib/libbot/marxan-calculations.ts b/data/bots/lib/libbot/marxan-calculations.ts new file mode 100644 index 0000000000..5290038c93 --- /dev/null +++ b/data/bots/lib/libbot/marxan-calculations.ts @@ -0,0 +1,16 @@ +import { BotHttpClient, getJsonApiDataFromResponse } from "./marxan-bot.ts"; +import { logError } from "./logger.ts"; + +export class MarxanCalculations { + private baseHttpClient; + + constructor(httpClient: BotHttpClient) { + this.baseHttpClient = httpClient.baseHttpClient; + } + + async runForScenario(scenarioId: string) { + return await this.baseHttpClient.post(`/scenarios/${scenarioId}/marxan`) + .then(getJsonApiDataFromResponse) + .catch(logError); + } +} diff --git a/data/bots/lib/libbot/organizations.ts b/data/bots/lib/libbot/organizations.ts new file mode 100644 index 0000000000..11aea73190 --- /dev/null +++ b/data/bots/lib/libbot/organizations.ts @@ -0,0 +1,35 @@ +import Process from "https://deno.land/std@0.103.0/node/process.ts"; +import { BotHttpClient, getJsonApiDataFromResponse } from "./marxan-bot.ts"; +import { logError, logInfo } from "./logger.ts"; +import { tookMs } from "./util/perf.ts"; + +interface Organization { + name: string; + description: string; + metadata?: Record; +} + +export class Organizations { + private baseHttpClient; + + constructor(httpClient: BotHttpClient) { + this.baseHttpClient = httpClient.baseHttpClient; + } + + async create(organization: Organization) { + const opStart = Process.hrtime(); + + const result = await this.baseHttpClient.post( + "/organizations", + organization, + ) + .then(getJsonApiDataFromResponse) + .catch(logError); + + logInfo( + `Organization was created in ${tookMs(Process.hrtime(opStart))}ms.`, + ); + + return result; + } +} diff --git a/data/bots/lib/libbot/planning-area-shapefiles.ts b/data/bots/lib/libbot/planning-area-shapefiles.ts new file mode 100644 index 0000000000..2f7ef9cef6 --- /dev/null +++ b/data/bots/lib/libbot/planning-area-shapefiles.ts @@ -0,0 +1,46 @@ +import Process from "https://deno.land/std@0.103.0/node/process.ts"; +import { BotHttpClient } from "./marxan-bot.ts"; +import { ShapefileUploader } from "./shapefile-uploader.ts"; +import { logDebug, logError, logInfo } from "./logger.ts"; +import { tookMs } from "./util/perf.ts"; + +interface FileUpload { + url: string; + formField: string; + data: Blob; + fileName: string; + headers: [string, string][]; +} + +export class PlanningAreaShapefiles extends ShapefileUploader { + private url; + + constructor(httpClient: BotHttpClient, urlPrefix: string) { + super(httpClient); + this.url = urlPrefix + "/api/v1/projects/planning-area/shapefile"; + } + + async uploadFromFile(localFilePath: string): Promise { + const opStart = Process.hrtime(); + + const data = new Blob([await Deno.readFile(localFilePath)]); + const planningAreaId: string = await (await this.sendData({ + url: this.url, + formField: "file", + data, + fileName: `${crypto.randomUUID()}.zip`, + headers: [["Authorization", `Bearer ${this.currentJwt}`]], + })) + .json() + .then((data) => data?.id) + .catch(logError); + + logInfo( + `Custom planning area shapefile uploaded in ${ + tookMs(Process.hrtime(opStart)) + }ms.`, + ); + logDebug(`Planning area id: ${planningAreaId}`); + return planningAreaId; + } +} diff --git a/data/bots/lib/libbot/projects.ts b/data/bots/lib/libbot/projects.ts new file mode 100644 index 0000000000..c1619b20dd --- /dev/null +++ b/data/bots/lib/libbot/projects.ts @@ -0,0 +1,64 @@ +import Process from "https://deno.land/std@0.103.0/node/process.ts"; +import { logDebug, logError, logInfo } from "./logger.ts"; +import { BotHttpClient, getJsonApiDataFromResponse } from "./marxan-bot.ts"; +import { tookMs } from "./util/perf.ts"; + +interface Project { + name: string; + description: string; + countryId?: string; + adminAreaLevel1Id?: string; + adminAreaLevel2Id?: string; + planningUnitGridShape: "hexagon" | "square" | "from_shapefile"; + planningUnitAreakm2: number; + planningAreaId?: string; +} + +export class Projects { + private baseHttpClient; + + constructor(httpClient: BotHttpClient) { + this.baseHttpClient = httpClient.baseHttpClient; + } + + async createInOrganization(organizationId: string, project: Project) { + const opStart = Process.hrtime(); + + const result = await this.baseHttpClient.post("/projects", { + ...project, + organizationId, + }) + .then(getJsonApiDataFromResponse) + .catch(logError); + + logInfo(`Project was created in ${tookMs(Process.hrtime(opStart))}ms.`); + logDebug(`Project:\n${Deno.inspect(result)}`); + + return result; + } + + async update(projectId: string, project: Partial) { + const opStart = Process.hrtime(); + + const result = await this.baseHttpClient.patch( + `/projects/${projectId}`, + project, + ) + .then(getJsonApiDataFromResponse) + .catch(logError); + + logInfo(`Project was updated in ${tookMs(Process.hrtime(opStart))}ms.`); + logDebug(`Project:\n${Deno.inspect(result)}`); + + return result; + } + + async checkStatus(id: string) { + return await this.baseHttpClient.get( + `/projects​/${id}​/scenarios​/status`, + {}, + ) + .then(getJsonApiDataFromResponse) + .catch(logError); + } +} diff --git a/data/bots/lib/libbot/protected-areas.ts b/data/bots/lib/libbot/protected-areas.ts new file mode 100644 index 0000000000..43bd2649a9 --- /dev/null +++ b/data/bots/lib/libbot/protected-areas.ts @@ -0,0 +1,53 @@ +import Process from "https://deno.land/std@0.103.0/node/process.ts"; +import { BotHttpClient } from "./marxan-bot.ts"; +import { IUCNCategory } from "./scenarios.ts"; +import { logDebug, logError, logInfo } from "./logger.ts"; +import { tookMs } from "./util/perf.ts"; + +export class ProtectedAreas { + private baseHttpClient; + + constructor(httpClient: BotHttpClient) { + this.baseHttpClient = httpClient.baseHttpClient; + } + + async getIucnCategoriesForPlanningAreaWithId( + protectedAreaId: string, + ): Promise { + const opStart = Process.hrtime(); + + const result = await this.baseHttpClient.get( + `/protected-areas/iucn-categories?filter[customAreaId]=${protectedAreaId}`, + ) + .then((result) => + result?.data.data?.map((i: { id: IUCNCategory }) => i.id) + ) + .catch(logError); + + logInfo( + `Lookup of IUCN categories done in ${tookMs(Process.hrtime(opStart))}ms.`, + ); + logDebug(`IUCN categories within planning area:\n${Deno.inspect(result)}`); + return result; + } + + async getIucnCategoriesForAdminAreaWithId( + adminAreaId: string, + ): Promise { + const opStart = Process.hrtime(); + + const result = await this.baseHttpClient.get( + `/protected-areas/iucn-categories?filter[adminAreaId]=${adminAreaId}`, + ) + .then((result) => + result?.data.data?.map((i: { id: IUCNCategory }) => i.id) + ) + .catch(logError); + + logInfo( + `Lookup of IUCN categories done in ${tookMs(Process.hrtime(opStart))}ms.`, + ); + logDebug(`IUCN categories within admin area:\n${Deno.inspect(result)}`); + return result; + } +} diff --git a/data/bots/lib/libbot/scenario-editing-metadata.ts b/data/bots/lib/libbot/scenario-editing-metadata.ts new file mode 100644 index 0000000000..bc84fad3ab --- /dev/null +++ b/data/bots/lib/libbot/scenario-editing-metadata.ts @@ -0,0 +1,13 @@ +export class ScenarioEditingMetadata { + analysisPreview = () => ({ + scenarioEditingMetadata: { + status: { + "protected-areas": "draft", + features: "draft", + analysis: "draft", + }, + tab: "analysis", + subtab: "analysis-preview", + }, + }); +} diff --git a/data/bots/lib/libbot/scenario-status.ts b/data/bots/lib/libbot/scenario-status.ts new file mode 100644 index 0000000000..5657a3ad39 --- /dev/null +++ b/data/bots/lib/libbot/scenario-status.ts @@ -0,0 +1,217 @@ +import Process from "https://deno.land/std@0.103.0/node/process.ts"; +import { BotHttpClient, getJsonApiDataFromResponse } from "./marxan-bot.ts"; +import { sleep } from "https://deno.land/x/sleep@v1.2.0/mod.ts"; +import _ from "https://deno.land/x/lodash@4.17.15-es/lodash.js"; +import { ms } from "https://deno.land/x/ms@v0.1.0/ms.ts"; +import { logDebug, logError, logInfo } from "./logger.ts"; +import { tookMs } from "./util/perf.ts"; + +const DEFAULT_WATCH_TIMEOUT = 1800; + +type WaitKinds = "short" | "some" | "long"; + +export const WaitForTime: Record = { + short: { + delay: "10s", + interval: "10s", + maxTries: 60, + }, + some: { + delay: "30s", + interval: "30s", + maxTries: 60, + }, + long: { + delay: "60s", + interval: "30s", + maxTries: 120, + }, +}; + +export enum ScenarioJobKinds { + geoFeatureCopy = "geofeatureCopy", + planningAreaProtectedCalculation = "planningAreaProtectedCalculation", + specification = "specification", + marxanRun = "run", +} + +export enum JobStatuses { + running = "running", + done = "done", + failure = "failure", +} + +interface JobStatus { + kind: ScenarioJobKinds; + status: JobStatuses; +} + +interface ScenarioStatus { + id: string; + jobs: JobStatus[]; +} + +interface ProjectStatus { + scenarios: ScenarioStatus[]; +} + +export interface JobSpecification { + jobKind: ScenarioJobKinds; + forProject: string; + forScenario: string; +} + +type msTime = string; + +interface RetryOptions { + delay?: msTime; + interval: msTime; + maxTries: number; +} + +export class ScenarioJobStatus { + private baseHttpClient; + + constructor(httpClient: BotHttpClient) { + this.baseHttpClient = httpClient.baseHttpClient; + } + + async get(job: JobSpecification): Promise { + const projectStatus: ProjectStatus = await this.baseHttpClient.get( + `/projects/${job.forProject}/scenarios/status`, + ) + .then(getJsonApiDataFromResponse) + .then((data) => data.attributes) + .catch(logError); + logDebug(`Project status:\n${Deno.inspect(projectStatus, { depth: 8 })}`); + return projectStatus?.scenarios.find((i) => i.id === job.forScenario)?.jobs + .find((i) => i.kind === job.jobKind)?.status; + } + + async waitFor( + job: JobSpecification, + until: JobStatuses, + retryOptions: RetryOptions, + ): Promise { + logInfo( + `Polling for ${job.jobKind} until status is ${until} for scenario ${job.forScenario}...`, + ); + + if (retryOptions?.delay) { + const delay = ms(retryOptions.delay ?? "0") as number; + logDebug(`Waiting for ${delay / 1e3}s before starting to poll status...`); + await sleep(delay / 1e3); + } + + const interval = ms(retryOptions.interval) as number; + + for (const i of [...Array(retryOptions.maxTries).keys()]) { + logInfo(`Retry ${i} of ${retryOptions.maxTries}...`); + const status = await this.get(job); + if (status === until) { + logInfo(`Current status is ${status}.`); + return true; + } + if(status === JobStatuses.failure) { + logError(`Operation failed.`); + return false; + } + logInfo(`Current status is ${status}: waiting for ${interval / 1e3}s`); + await sleep(interval / 1e3); + } + + return false; + } + + async waitForPlanningAreaProtectedCalculationFor( + projectId: string, + scenarioId: string, + waitForTime: keyof typeof WaitForTime = "short", + ): Promise { + const opStart = Process.hrtime(); + + const waitResult = await this.waitFor( + { + jobKind: ScenarioJobKinds.planningAreaProtectedCalculation, + forProject: projectId, + forScenario: scenarioId, + }, + JobStatuses.done, + WaitForTime[waitForTime], + ); + + const tookSeconds = tookMs(Process.hrtime(opStart)) / 1e3; + + if (waitResult) { + logInfo(`Protected area calculations done in ${tookSeconds}s.`); + } else { + logInfo( + `Waited for ${tookSeconds}s for protected area calculations, but operation is still ongoing.`, + ); + } + + return waitResult; + } + + async waitForFeatureSpecificationCalculationFor( + projectId: string, + scenarioId: string, + waitForTime: keyof typeof WaitForTime = "some", + ): Promise { + const opStart = Process.hrtime(); + + const waitResult = await this.waitFor( + { + jobKind: ScenarioJobKinds.specification, + forProject: projectId, + forScenario: scenarioId, + }, + JobStatuses.done, + WaitForTime[waitForTime], + ); + + const tookSeconds = tookMs(Process.hrtime(opStart)) / 1e3; + + if (waitResult) { + logInfo( + `Geofeature specification calculations done in ${tookSeconds}s.`, + ); + } else { + logInfo( + `Waited for ${tookSeconds}s for geofeature specification calculations, but operation is still ongoing.`, + ); + } + + return waitResult; + } + + async waitForMarxanCalculationsFor( + projectId: string, + scenarioId: string, + waitForTime: keyof typeof WaitForTime = "some", + ): Promise { + const opStart = Process.hrtime(); + + const waitResult = await this.waitFor( + { + jobKind: ScenarioJobKinds.marxanRun, + forProject: projectId, + forScenario: scenarioId, + }, + JobStatuses.done, + WaitForTime[waitForTime], + ); + + const tookSeconds = tookMs(Process.hrtime(opStart)) / 1e3; + + if (waitResult) { + logInfo(`Marxan calculations done in ${tookSeconds}s.`); + } else { + logInfo( + `Waited for ${tookSeconds}s for Marxan calculations, but operation is still ongoing.`, + ); + } + + return waitResult; + } +} diff --git a/data/bots/lib/libbot/scenarios.ts b/data/bots/lib/libbot/scenarios.ts new file mode 100644 index 0000000000..9cd2be7451 --- /dev/null +++ b/data/bots/lib/libbot/scenarios.ts @@ -0,0 +1,79 @@ +import Process from "https://deno.land/std@0.103.0/node/process.ts"; +import { BotHttpClient, getJsonApiDataFromResponse } from "./marxan-bot.ts"; +import { logDebug, logError, logInfo } from "./logger.ts"; +import { tookMs } from "./util/perf.ts"; + +/** + * The kind of Marxan scenario (standard, Marxan with Zones, and possibly other + * kinds in the future). + */ +export enum ScenarioType { + marxan = "marxan", + marxanWithZones = "marxan-with-zones", +} + +export enum IUCNCategory { + Ia = "Ia", + Ib = "Ib", + II = "II", + III = "III", + IV = "IV", + V = "V", + VI = "VI", + NotApplicable = "Not Applicable", + NotAssigned = "Not Assigned", + NotReported = "Not Reported", +} + +interface Scenario { + name: string; + type: "marxan"; + description: string; + wdpaIucnCategories?: IUCNCategory[]; + wdpaThreshold?: number; + boundaryLengthModifier?: number; + // @todo probably not: check + customProtectedAreaIds?: string; + metadata?: { + marxanInputParameterFile?: Record; + scenarioEditingMetadata?: Record; + }; +} + +export class Scenarios { + private baseHttpClient; + + constructor(httpClient: BotHttpClient) { + this.baseHttpClient = httpClient.baseHttpClient; + } + + async createInProject(projectId: string, scenario: Scenario) { + const opStart = Process.hrtime(); + + const result = await this.baseHttpClient.post("/scenarios", { + ...scenario, + projectId, + }) + .then(getJsonApiDataFromResponse) + .catch(logError); + + logInfo(`Scenario was created in ${tookMs(Process.hrtime(opStart))}ms.`); + logDebug(`Scenario:\n${Deno.inspect(result)}`); + return result; + } + + async update(scenarioId: string, scenario: Partial) { + const opStart = Process.hrtime(); + + const result = await this.baseHttpClient.patch( + `/scenarios/${scenarioId}`, + scenario, + ) + .then(getJsonApiDataFromResponse) + .catch(logError); + + logInfo(`Scenario was updated in ${tookMs(Process.hrtime(opStart))}ms.`); + logDebug(`Scenario:\n${Deno.inspect(result)}`); + return result; + } +} diff --git a/data/bots/lib/libbot/shapefile-uploader.ts b/data/bots/lib/libbot/shapefile-uploader.ts new file mode 100644 index 0000000000..98c4884d73 --- /dev/null +++ b/data/bots/lib/libbot/shapefile-uploader.ts @@ -0,0 +1,30 @@ +import { BotHttpClient } from "./marxan-bot.ts"; + +interface FileUpload { + url: string; + formField: string; + data: Blob; + fileName: string; + headers: [string, string][]; +} + +export class ShapefileUploader { + protected currentJwt; + + constructor(httpClient: BotHttpClient) { + this.currentJwt = httpClient.currentJwt; + } + + async sendData(config: FileUpload) { + const formData = new FormData(); + formData.append(config.formField, config.data, config.fileName); + + const response = await fetch(config.url, { + method: "POST", + body: formData, + headers: config.headers, + }); + + return response; + } +} diff --git a/data/bots/lib/libbot/users.ts b/data/bots/lib/libbot/users.ts new file mode 100644 index 0000000000..a650cb0a1b --- /dev/null +++ b/data/bots/lib/libbot/users.ts @@ -0,0 +1,38 @@ +import axiod from "https://deno.land/x/axiod@0.22/mod.ts"; +import Process from "https://deno.land/std@0.103.0/node/process.ts"; +import { getJsonApiDataFromResponse } from "./marxan-bot.ts"; +import { logDebug, logError, logInfo } from "./logger.ts"; +import { tookMs } from "./util/perf.ts"; +import { MarxanBotConfig } from "./marxan-bot.ts"; + +export class Users { + private baseHttpClient; + private credentials; + + constructor(config: MarxanBotConfig) { + this.baseHttpClient = axiod.create({ + baseURL: config.apiUrl, + }); + + this.credentials = config.credentials; + } + + async signUp() { + const opStart = Process.hrtime(); + + const result = await this.baseHttpClient.post( + "/auth/sign-up", + { + email: this.credentials.username, + password: this.credentials.password, + displayName: this.credentials.username, + }, + ) + .then(getJsonApiDataFromResponse) + .catch(logError); + + logInfo(`User was created in ${tookMs(Process.hrtime(opStart))}ms.`); + logDebug(`User:\n${Deno.inspect(result)}`); + return result; + } +} diff --git a/data/bots/lib/libbot/util/perf.ts b/data/bots/lib/libbot/util/perf.ts new file mode 100644 index 0000000000..f8cbdd52f7 --- /dev/null +++ b/data/bots/lib/libbot/util/perf.ts @@ -0,0 +1,7 @@ +/** + * Seconds + nanosecond to milliseconds conversion. + */ +export const tookMs = (hrtime: [number, number]): number => { + const NS_PER_SEC = 1e9; + return (hrtime[0] * NS_PER_SEC + hrtime[1]) / 1e6; +}; diff --git a/data/bots/scenario-hero/bot.ts b/data/bots/scenario-hero/bot.ts index 7b2e595ae4..00c676af91 100644 --- a/data/bots/scenario-hero/bot.ts +++ b/data/bots/scenario-hero/bot.ts @@ -1,87 +1,133 @@ import axiod from "https://deno.land/x/axiod/mod.ts"; -import Process from 'https://deno.land/std@0.103.0/node/process.ts'; +import Process from "https://deno.land/std@0.103.0/node/process.ts"; +import { sleep } from "https://deno.land/x/sleep@v1.2.0/mod.ts"; const settings = { - apiUrl: 'http://localhost:3030', - credentials: { - "username": "aa@example.com", - "password": "aauserpassword" - } + apiUrl: "http://localhost:3030", + credentials: { + "username": "aa@example.com", + "password": "aauserpassword", + }, }; -const jwt = await axiod.post(`${settings.apiUrl}/auth/sign-in/`, settings.credentials) - .then(result => result.data.accessToken); +const jwt = await axiod.post( + `${settings.apiUrl}/auth/sign-in/`, + settings.credentials, +) + .then((result) => result.data.accessToken); const botClient = axiod.create({ - baseURL: settings.apiUrl + '/api/v1', - headers: { - 'Authorization': 'Bearer ' + jwt - } + baseURL: settings.apiUrl + "/api/v1", + headers: { + "Authorization": "Bearer " + jwt, + }, }); -const organization = await botClient.post('/organizations', { - name: "aliquip nulla ut " + crypto.randomUUID(), - description: "Duis aliquip nostrud sint", - metadata: {} -}).then(result => result.data).catch(e => { console.log(e) }); +const organization = await botClient.post("/organizations", { + name: "aliquip nulla ut " + crypto.randomUUID(), + description: "Duis aliquip nostrud sint", + metadata: {}, +}).then((result) => result.data).catch((e) => { + console.log(e); +}); console.log(organization); -const project = await botClient.post('/projects', { - name: 'test project ' + crypto.randomUUID(), - organizationId: organization.data.id, - countryId: 'BWA', - adminAreaLevel1Id: 'BWA.12_1', - adminAreaLevel2Id: 'BWA.12.1_1', - planningUnitGridShape: 'hexagon', - planningUnitAreakm2: 16, -}).then(result => result.data).catch(e => { console.log(e) });; +const project = await botClient.post("/projects", { + name: "test project " + crypto.randomUUID(), + organizationId: organization.data.id, + countryId: "AGO", + // adminAreaLevel1Id: 'BWA.12_1', + // adminAreaLevel2Id: 'BWA.12.1_1', + planningUnitGridShape: "hexagon", + planningUnitAreakm2: 200, +}).then((result) => result.data).catch((e) => { + console.log(e); +}); console.log(project); // wait a bit for async job to be picked up and processed // @DEBT we should check the actual job status -await new Promise(r => setTimeout(r, 30e3)) +await sleep(10); const scenarioStart = Process.hrtime(); -const scenario = await botClient.post('/scenarios', { - name: `test scenario in project ${project.data.attributes.name}`, - type: "marxan", - projectId: project.data.id, - description: "eu et sit", - wdpaIucnCategories: ['Not Applicable'], - wdpaThreshold: 30 -}).then(result => result.data).catch(e => { console.log(e); }); +const scenario = await botClient.post("/scenarios", { + name: `test scenario in project ${project.data.attributes.name}`, + type: "marxan", + projectId: project.data.id, + description: "eu et sit", + // wdpaIucnCategories: ['Not Applicable'], + // wdpaThreshold: 30 +}).then((result) => result.data).catch((e) => { + console.log(e); +}); const scenarioTook = Process.hrtime(scenarioStart); -console.log(`Scenario creation done in ${scenarioTook[0]} seconds`); +console.log(`Scenario creation done in ${scenarioTook[0]}ms`); console.log(scenario); -const pantheraPardusFeature = await botClient.get(`/projects/${project.data.id}/features?q=pantherapardus`) - .then(result => result.data).catch(e => { console.log(e); }); +const demoGiraffaCamelopardalisFeature = await botClient.get( + `/projects/${project.data.id}/features?q=iraffa`, +); +await sleep(10); + +const scenarioStep2 = await botClient + .patch(`/scenarios/${scenario!.data!.id}`, { + wdpaIucnCategories: ["Not Applicable"], + }).then((result) => result.data).catch((e) => { + console.log(e); + }); + +await sleep(10); -console.log(pantheraPardusFeature); +const scenarioStep3 = await botClient + .patch(`/scenarios/${scenario!.data!.id}`, { + wdpaThreshold: 10, + }).then((result) => result.data).catch((e) => { + console.log(e); + }); + +console.log(scenarioStep3); + +const pantheraPardusFeature = await botClient.get( + `/projects/${project.data.id}/features?q=panthera`, +) + .then((result) => result.data).catch((e) => { + console.log(e); + }); + +console.log(demoGiraffaCamelopardalisFeature); const geoFeatureSpecStart = Process.hrtime(); -const geoFeatureSpec = await botClient.post(`/scenarios/${scenario.data.id}/features/specification`, { - status: 'created', +const geoFeatureSpec = await botClient.post( + `/scenarios/${scenario.data.id}/features/specification/v2`, + { + status: "created", features: [ { kind: "plain", - featureId: pantheraPardusFeature.data[0].id, + featureId: demoGiraffaCamelopardalisFeature.data[0].id, marxanSettings: { - prop: 0.3, - fpf: 1 - } + prop: 0.3, + fpf: 1, + }, }, - ] - }).then(result => result.data).catch(e => { console.log(e); }); + ], + }, +).then((result) => result.data).catch((e) => { + console.log(JSON.stringify(e?.response.data?.errors)); +}); + +const geoFeatureSpecTook = Process.hrtime(geoFeatureSpecStart); - const geoFeatureSpecTook = Process.hrtime(geoFeatureSpecStart); +console.log( + `Processing of features for scenario done in ${geoFeatureSpecTook[0]}ms`, +); - console.log(`Processing of features for scenario done in ${geoFeatureSpecTook[0]} seconds`); +console.log(geoFeatureSpec); - console.log(geoFeatureSpec); +await botClient.post(`/scenarios/${scenario.data.id}/marxan`);