feat: download picklists from server and validate

docs/postman/Units API.postman_collection.json

@@ -279,7 +279,7 @@
             },
             {
               "key": "search",
-              "value": "USA"
+              "value": "Netherlands"
             }
           ]
         }
@@ -302,7 +302,7 @@
             },
             {
               "key": "search",
-              "value": "USA"
+              "value": "Netherlands"
             },
             {
               "key": "orgUid",

src/models/labels/labels.stub.json

@@ -12,6 +12,6 @@
     "labelLink": "https://label.link/1",
     "createdAt": "2022-01-18 00:05:45.701 +00:00",
     "updatedAt": "2022-01-18 00:05:45.701 +00:00",
-    "labelType": "stub"
+    "labelType": "Endorsement"
   }
 ]

src/models/projects/projects.stub.json

@@ -200,7 +200,7 @@
     "projectStatus": "Completed",
     "projectStatusDate": "2020-06-07 00:05:45.701 +00:00",
     "unitMetric": "tCO2e",
-    "methodology": "Mitigation of greenhouse gases emissions with reforestation -- Version 1.0",
+    "methodology": "Recovery and utilization of gas from oil fields that would otherwise be flared or vented --- Version 7.0",
     "validationDate": "2021-02-15 00:05:45.701 +00:00",
     "projectTags": "",
     "createdAt": "2015-07-07 00:05:45.701 +00:00",

src/models/units/units.stub.json

@@ -5,7 +5,7 @@
     "vintageYear": 2020,
     "orgUid": "f1c54511-865e-4611-976c-7c3c1f704662",
     "unitOwner": "f1c54511-865e-4611-976c-7c3c1f704662",
-    "countryJurisdictionOfOwner": "USA",
+    "countryJurisdictionOfOwner": "Netherlands",
     "serialNumberBlock": "AXJJFSLGHSHEJ1000-AXJJFSLGHSHEJ1010",
     "serialNumberPattern": "[.*\\D]+([0-9]+)+[-][.*\\D]+([0-9]+)$",
     "unitType": "heard reduction",
@@ -17,7 +17,7 @@
     "marketplaceIdentifier": "AKFEE3",
     "unitRegistryLink": "http://climateWarehouse.com/myRegistry",
     "marketplaceLink": "http://climateWarehouse.com/myMarketplace",
-    "correspondingAdjustmentDeclaration": "Commited",
+    "correspondingAdjustmentDeclaration": "Committed",
     "correspondingAdjustmentStatus": "Not Started",
     "issuanceId": "a6745831-5d5e-45ed-b9fe-fd6aa129df25",
     "createdAt": "2022-01-18 00:05:45.701 +00:00",

src/server.js

@@ -8,6 +8,7 @@ import { Server } from 'socket.io';
 import Debug from 'debug';
 import { connection } from './websocket';
 import { startDataLayerUpdatePolling } from './fullnode';
+import { pullPickListValues } from './utils/picklist-loader';
 
 const debug = Debug('climate-warehouse:server');
 
@@ -47,7 +48,16 @@ function onError(error) {
  * Event listener for HTTP server "listening" event.
  */
 
+const syncPickList = () => {
+  console.log('Syncing PickLists');
+  pullPickListValues();
+};
+
 function onListening() {
+  syncPickList();
+  setInterval(async () => {
+    syncPickList();
+  }, 86400000 /* 1 day */);
   const addr = server.address();
   const bind = typeof addr === 'string' ? 'pipe ' + addr : 'port ' + addr.port;
   debug('Listening on ' + bind);

src/utils/picklist-loader.js

@@ -0,0 +1,17 @@
+import request from 'request-promise';
+import dotenv from 'dotenv';
+dotenv.config();
+
+let downloadedValues = {};
+export const getPicklistValues = () => downloadedValues;
+
+export const pullPickListValues = async () => {
+  const options = {
+    method: 'GET',
+    url: process.env.PICKLIST_URL,
+  };
+
+  downloadedValues = JSON.parse(await request(Object.assign({}, options)));
+
+  //console.log(downloadedValues);
+};

src/utils/validation-utils.js

@@ -0,0 +1,15 @@
+import { getPicklistValues } from './picklist-loader';
+
+export const pickListValidation = (field, name) => (value, helper) => {
+  const pickList = getPicklistValues();
+
+  if (pickList[field].includes(value)) {
+    return value;
+  }
+
+  return helper.message(
+    `${name || field} does not include a valid option ${pickList[field].join(
+      ',',
+    )}`,
+  );
+};

src/validations/labels.validations.js

@@ -1,13 +1,14 @@
 import Joi from 'joi';
 import { labelUnitSchema } from './labelUnit.validations';
+import { pickListValidation } from '../utils/validation-utils';
 
 export const labelSchema = Joi.object({
   // orgUid - derived upon creation
   // warehouseProjectId - derived upon creation
   id: Joi.string().optional(),
   warehouseProjectId: Joi.string().optional(),
   label: Joi.string().required(),
-  labelType: Joi.string().required(),
+  labelType: Joi.string().custom(pickListValidation('labelType')).required(),
   creditingPeriodStartDate: Joi.date().required(),
   creditingPeriodEndDate: Joi.date()
     .min(Joi.ref('creditingPeriodStartDate'))

src/validations/locations.validations.js

@@ -1,10 +1,13 @@
 import Joi from 'joi';
+import { pickListValidation } from '../utils/validation-utils';
 
 export const locationSchema = Joi.object({
   // orgUid - derived upon creation
   // warehouseProjectId - derived upon creation
   id: Joi.string().optional(),
-  country: Joi.string().required(),
+  country: Joi.string()
+    .custom(pickListValidation('countries', 'ProjectLocation Country'))
+    .required(),
   inCountryRegion: Joi.string().required(),
   // Please make 'inCountryRegion' optional.
   geographicIdentifier: Joi.string().required(),

src/validations/projects.validations.js

@@ -9,12 +9,16 @@ import {
   estimationSchema,
 } from '../validations';
 
+import { pickListValidation } from '../utils/validation-utils';
+
 export const baseSchema = {
   // warehouseProjectId - derived upon creation
   // orgUid - derived upon creation
   projectId: Joi.string().required(),
   // optional because if we dont supply it, it assigns to the users own registry
-  registryOfOrigin: Joi.string().optional(),
+  registryOfOrigin: Joi.string()
+    .custom(pickListValidation('registries', 'registryOfOrigin'))
+    .optional(),
   // Need to add 'originProjectId' as a new field. It will be required and STRING type.
   // If current registry is the same as registry of origin, then ID will be the same.
   // If current registry is different from registry of origin, then we will have different IDs.
@@ -23,17 +27,29 @@ export const baseSchema = {
   projectName: Joi.string().required(),
   projectLink: Joi.string().required(),
   projectDeveloper: Joi.string().required(),
-  sector: Joi.string().required(),
-  projectType: Joi.string().required(),
+  sector: Joi.string()
+    .custom(pickListValidation('projectSector', 'sector'))
+    .required(),
+  projectType: Joi.string()
+    .custom(pickListValidation('projectType'))
+    .required(),
   projectTags: Joi.string().optional(),
-  coveredByNDC: Joi.string().required(),
+  coveredByNDC: Joi.string()
+    .custom(pickListValidation('coveredByNDC'))
+    .required(),
   ndcInformation: Joi.string().required(),
   // 'ndcInformation' should be optional, but should carry an additional validation. If 'coveredByNDC' field selects "Inside NDC", then 'ndcInformation' becomes required field.
-  projectStatus: Joi.string().required(),
+  projectStatus: Joi.string()
+    .custom(pickListValidation('projectStatusValues', 'projectStatus'))
+    .required(),
   projectStatusDate: Joi.date().required(),
-  unitMetric: Joi.string().required(),
-  methodology: Joi.string().required(),
-  validationBody: Joi.string().optional(),
+  unitMetric: Joi.string().custom(pickListValidation('unitMetric')).required(),
+  methodology: Joi.string()
+    .custom(pickListValidation('methodology'))
+    .required(),
+  validationBody: Joi.string()
+    .custom(pickListValidation('validationBody'))
+    .optional(),
   validationDate: Joi.string().optional(),
   // should be DATE instead of string.
 

src/validations/ratings.validations.js

@@ -1,10 +1,11 @@
 import Joi from 'joi';
+import { pickListValidation } from '../utils/validation-utils';
 
 export const ratingSchema = Joi.object({
   // orgUid - derived upon creation
   // warehouseProjectId - derived upon creation
   id: Joi.string().optional(),
-  ratingType: Joi.string().required(),
+  ratingType: Joi.string().custom(pickListValidation('ratingType')).required(),
   ratingRangeLowest: Joi.string().required(),
   ratingRangeHighest: Joi.string().required(),
   rating: Joi.string().required(),

src/validations/units.validations.js

@@ -3,6 +3,8 @@ import { transformSerialNumberBlock } from '../utils/helpers';
 import { issuanceSchema } from './issuances.validation';
 import { labelSchema } from './labels.validations';
 
+import { pickListValidation } from '../utils/validation-utils';
+
 const customSerialNumberValidator = (obj, helper) => {
   const { serialNumberBlock, serialNumberPattern } = obj;
 
@@ -32,32 +34,32 @@ const unitsBaseSchema = {
   // orgUid - derived upon unit creation
   projectLocationId: Joi.string().required(),
   unitOwner: Joi.string().required(),
-  countryJurisdictionOfOwner: Joi.string().required(),
-  inCountryJurisdictionOfOwner: Joi.string().required(),
-  //'inCountryJurisdictionOfOwner' should be optional.
+  countryJurisdictionOfOwner: Joi.string()
+    .custom(pickListValidation('countries', 'countryJurisdictionOfOwner'))
+    .required(),
+  inCountryJurisdictionOfOwner: Joi.string().optional(),
   // must be in the form ABC123-XYZ456
   serialNumberBlock: Joi.string().required(),
   serialNumberPattern: Joi.string().required().messages({
     'any.required':
       'serialNumberPattern is required. This pattern must be a regex expression with 2 match groups to match block start and block end. Example: [.*\\D]+([0-9]+)+[-][.*\\D]+([0-9]+)$ that matches ABC1000-ABC1010 TODO: ADD LINK HERE FOR DOCUMENTATION',
   }),
   // match 4 digit year
-  vintageYear: Joi.number().integer().min(1900).max(3000),
-  //'vintageYear' should be required.
-  unitType: Joi.string().valid('heard reduction', 'removal').required(),
+  vintageYear: Joi.number().integer().min(1900).max(3000).required(),
+  unitType: Joi.string().custom(pickListValidation('unitType')).required(),
   marketplace: Joi.string().optional(),
   marketplaceLink: Joi.string().optional(),
   marketplaceIdentifier: Joi.string().optional(),
   unitTags: Joi.string().allow('').optional(),
-  unitStatus: Joi.string().valid('Held', 'For Sale', 'Retired').required(),
+  unitStatus: Joi.string().custom(pickListValidation('unitStatus')).required(),
   unitStatusReason: Joi.string().optional(),
   //'unitStatusReason' should have additional validation based on entry in 'unitStatus'. If user enters "cancelled" or "retired", then 'unitStatusReason' field becomes required.
   unitRegistryLink: Joi.string().required(),
   correspondingAdjustmentDeclaration: Joi.string()
-    .valid('Commited', 'Not Required', 'Unknown')
+    .custom(pickListValidation('correspondingAdjustmentDeclaration'))
     .required(),
   correspondingAdjustmentStatus: Joi.string()
-    .valid('Unknown', 'Not Started', 'Pending')
+    .custom(pickListValidation('correspondingAdjustmentStatus'))
     .required(),
   issuance: issuanceSchema.optional(),
   labels: Joi.array().items(labelSchema).optional(),
@@ -97,6 +99,10 @@ export const unitsSplitSchema = Joi.object({
       Joi.object().keys({
         unitCount: Joi.number().required(),
         unitOwner: Joi.string().optional(),
+        countryJurisdictionOfOwner: Joi.string()
+          .custom(pickListValidation('countries', 'countryJurisdictionOfOwner'))
+          .optional(),
+        inCountryJurisdictionOfOwner: Joi.string().optional(),
       }),
     )
     .min(2)

tests/integration/project.spec.js

@@ -4,13 +4,18 @@ import chai from 'chai';
 const { expect } = chai;
 
 import * as testFixtures from '../test-fixtures';
+import { pullPickListValues } from '../../src/utils/picklist-loader';
 
 import { POLLING_INTERVAL } from '../../src/fullnode';
 const TEST_WAIT_TIME = POLLING_INTERVAL * 2;
 
 describe('Project Resource Integration Tests', function () {
   let homeOrgUid;
 
+  before(async function () {
+    await pullPickListValues();
+  });
+
   beforeEach(async function () {
     await testFixtures.resetStagingTable();
     await testFixtures.createTestHomeOrg();

tests/integration/unit.spec.js

@@ -6,7 +6,7 @@ const { expect } = chai;
 
 import app from '../../src/server';
 import { UnitMirror } from '../../src/models';
-
+import { pullPickListValues } from '../../src/utils/picklist-loader';
 import * as testFixtures from '../test-fixtures';
 
 import { POLLING_INTERVAL } from '../../src/fullnode';
@@ -15,6 +15,10 @@ const TEST_WAIT_TIME = POLLING_INTERVAL * 2;
 describe('Unit Resource Integration Tests', function () {
   let homeOrgUid;
 
+  before(async function () {
+    await pullPickListValues();
+  });
+
   beforeEach(async function () {
     await testFixtures.resetStagingTable();
     await testFixtures.createTestHomeOrg();
@@ -110,12 +114,12 @@ describe('Unit Resource Integration Tests', function () {
     const createdUnitResult = await supertest(app).post('/v1/units').send({
       serialNumberBlock: 'AXJJFSLGHSHEJ9000-AXJJFSLGHSHEJ9010',
       serialNumberPattern: '[.*\\D]+([0-9]+)+[-][.*\\D]+([0-9]+)$',
-      countryJurisdictionOfOwner: 'USA',
+      countryJurisdictionOfOwner: 'Austria',
       unitOwner: 'TEST_OWNER',
-      unitType: 'removal',
+      unitType: 'Reduction - nature',
       unitStatus: 'Held',
       vintageYear: 2020,
-      correspondingAdjustmentDeclaration: 'Commited',
+      correspondingAdjustmentDeclaration: 'Committed',
       correspondingAdjustmentStatus: 'Pending',
       inCountryJurisdictionOfOwner: 'Maryland',
       unitRegistryLink: 'https://test.link',

tests/test-data/new-project.json

@@ -5,15 +5,15 @@
   "projectName": "Sungei Buloh Wetlands Conservation",
   "projectLink": "https://www.nature.com/articles/s41467-021-21560-2",
   "projectDeveloper": "NParks' National Biodiversity Centre, National Parks Board, Ridgeview Residential College",
-  "sector": "Forestry",
-  "projectType": "Urban Forests",
+  "sector": "Transport",
+  "projectType": "Organic Waste Composting",
   "projectTags": "Wetlands, Reforestation, Million trees",
   "coveredByNDC": "Inside NDC",
   "ndcInformation": "The restoration and conservation project directly aligns to the Singaporean NDC goals to capture 1,000,000 tons of carbon by 2050. This project represents an estimated contribution of 27% towards the NDC.",
-  "projectStatus": "In progress",
+  "projectStatus": "Registered",
   "projectStatusDate": "2022-01-31T00:05:45.701Z",
-  "unitMetric": "tCO2",
-  "methodology": "Surface monitoring",
+  "unitMetric": "tCO2e",
+  "methodology": "Recovery and utilization of gas from oil fields that would otherwise be flared or vented --- Version 7.0",
   "validationBody": "SCS Global Services",
   "validationDate": "2021-06-01T17:00:45.701Z",
   "projectLocations": [
@@ -57,7 +57,7 @@
   ],
   "projectRatings": [
     {
-      "ratingType": "CDP Rating",
+      "ratingType": "CCQI",
       "ratingRangeHighest": "A",
       "ratingRangeLowest": "Z",
       "rating": "G",

tests/test-data/new-unit.json

@@ -1,26 +1,26 @@
 {
   "unitOwner": "f1c54511-865e-4611-976c-7c3c1f704662",
-  "countryJurisdictionOfOwner": "USA",
+  "countryJurisdictionOfOwner": "Netherlands",
   "projectLocationId": "lkjw2er1-nj23-1212-532-dsjdx5-k131",
   "inCountryJurisdictionOfOwner": "California",
   "serialNumberBlock": "AXJJFSLGHSHEJ1000-AXJJFSLGHSHEJ1010",
   "serialNumberPattern": "[.*\\D]+([0-9]+)+[-][.*\\D]+([0-9]+)$",
   "vintageYear": 2016,
-  "unitType": "removal",
+  "unitType": "Removal - nature",
   "marketplace": "California Carbon",
   "marketplaceLink": "https://www.californiacarbon.info/",
   "marketplaceIdentifier": "CCMT405",
   "unitTags": "CC&S, Sequestration, carbon bury",
   "unitStatus": "Held",
   "unitStatusReason": "N/A",
   "unitRegistryLink": "https://www.climateactionreserve.org/about-us/california-climate-action-registry/",
-  "correspondingAdjustmentDeclaration": "Commited",
+  "correspondingAdjustmentDeclaration": "Committed",
   "correspondingAdjustmentStatus": "Not Started",
   "labels": [
     {
       "warehouseProjectId": "11954678-f7a5-47d2-94f8-f4f3138a529c",
       "label": "Green-e® Climate COC certification",
-      "labelType": "Chain of custody certification",
+      "labelType": "Certification",
       "creditingPeriodStartDate": "2014-01-10T00:05:45.701Z",
       "creditingPeriodEndDate": "2019-01-09T00:05:45.701Z",
       "validityPeriodStartDate": "2014-02-01T00:05:45.701Z",