Add geometry and zone-config scripts (electricitymaps#85)

* WIP

* executable through ts-node

* Update script

* reintroduce eslint rules

* add eslint rules back

* use current config folder

* use current config

web/generate-zones-config.ts

@@ -0,0 +1,132 @@
+/* This script aggregates the per-zone config files into a single zones.json/exchanges.json
+file to enable easy importing within web/ */
+import * as yaml from 'js-yaml';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+
+const config = {
+  verifyNoUpdates: process.env.VERIFY_NO_UPDATES !== undefined,
+};
+
+const mergeZones = () => {
+  const basePath = path.resolve(__dirname, '../config/zones');
+
+  const zoneFiles = fs.readdirSync(basePath);
+  const filesWithDirectory = zoneFiles.map((file) => `${basePath}/${file}`);
+
+  const UNNECESSARY_ZONE_FIELDS = new Set([
+    'fallbackZoneMixes',
+    'isLowCarbon',
+    'isRenewable',
+    'emissionFactors',
+  ]);
+  const zones = filesWithDirectory.reduce((zones, filepath) => {
+    const zoneConfig: any = yaml.load(fs.readFileSync(filepath, 'utf8'));
+    for (const key in zoneConfig) {
+      if (UNNECESSARY_ZONE_FIELDS.has(key)) {
+        delete zoneConfig[key];
+      }
+    }
+    Object.assign(zones, { [path.parse(filepath).name]: zoneConfig });
+    return zones;
+  }, {});
+
+  return zones;
+};
+
+const mergeExchanges = () => {
+  const basePath = path.resolve(__dirname, '../config/exchanges');
+
+  const exchangeFiles = fs.readdirSync(basePath);
+  const filesWithDirectory = exchangeFiles.map((file) => `${basePath}/${file}`);
+
+  const exchanges = filesWithDirectory.reduce((exchanges, filepath) => {
+    const exchangeKey = path.parse(filepath).name.split('_').join('->');
+    Object.assign(exchanges, {
+      [exchangeKey]: yaml.load(fs.readFileSync(filepath, 'utf8')),
+    });
+    return exchanges;
+  }, {});
+
+  return exchanges;
+};
+
+const mergeRatioParameters = () => {
+  // merge the fallbackZoneMixes, isLowCarbon, isRenewable params into a single object
+  const basePath = path.resolve(__dirname, '../config');
+
+  const defaultParameters: any = yaml.load(
+    fs.readFileSync(`${basePath}/defaults.yaml`, 'utf8')
+  );
+
+  const zoneFiles = fs.readdirSync(`${basePath}/zones`);
+  const filesWithDirectory = zoneFiles.map((file) => `${basePath}/zones/${file}`);
+
+  const ratioParameters: any = {
+    fallbackZoneMixes: {
+      defaults: defaultParameters.fallbackZoneMixes,
+      zoneOverrides: {},
+    },
+    isLowCarbon: {
+      defaults: defaultParameters.isLowCarbon,
+      zoneOverrides: {},
+    },
+    isRenewable: {
+      defaults: defaultParameters.isRenewable,
+      zoneOverrides: {},
+    },
+  };
+
+  for (const filepath of filesWithDirectory) {
+    const zoneConfig: any = yaml.load(fs.readFileSync(filepath, 'utf8'));
+    const zoneKey = path.parse(filepath).name;
+    for (const key in ratioParameters) {
+      if (zoneConfig[key] !== undefined) {
+        ratioParameters[key].zoneOverrides[zoneKey] = zoneConfig[key];
+      }
+    }
+  }
+
+  return ratioParameters;
+};
+
+const writeJSON = (fileName: any, object: any) => {
+  const directory = path.resolve(path.dirname(fileName));
+
+  if (!fs.existsSync(directory)) {
+    fs.mkdirSync(directory, { recursive: true });
+  }
+
+  fs.writeFileSync(fileName, JSON.stringify(object), { encoding: 'utf8' });
+};
+
+const zonesConfig = mergeZones();
+const exchangesConfig = mergeExchanges();
+
+const autogenConfigPath = path.resolve(__dirname, 'config');
+
+if (config.verifyNoUpdates) {
+  const zonesConfigPrevious = JSON.parse(
+    fs.readFileSync(`${autogenConfigPath}/zones.json`, 'utf8')
+  );
+  const exchangesConfigPrevious = JSON.parse(
+    fs.readFileSync(`${autogenConfigPath}/exchanges.json`, 'utf8')
+  );
+  if (JSON.stringify(zonesConfigPrevious) !== JSON.stringify(zonesConfig)) {
+    console.error(
+      'Did not expect any updates to zones.json. Please run "yarn generate-zones-config" to update.'
+    );
+    process.exit(1);
+  }
+  if (JSON.stringify(exchangesConfigPrevious) !== JSON.stringify(exchangesConfig)) {
+    console.error(
+      'Did not expect any updates to exchanges.json. Please run "yarn generate-zones-config" to update.'
+    );
+    process.exit(1);
+  }
+}
+
+writeJSON(`${autogenConfigPath}/zones.json`, zonesConfig);
+writeJSON(`${autogenConfigPath}/exchanges.json`, exchangesConfig);
+
+export { mergeZones, mergeExchanges, mergeRatioParameters };

web/geo/.gitignore

@@ -0,0 +1 @@
+/tmp

web/geo/README.md

@@ -0,0 +1,14 @@
+# Overview
+
+The world.geojson file is a geojson file containing all geographies and metadata for all zones visible on the electricityMap app. The zones are also used in the backend of Electricity Maps.
+
+You can see tutorials and more information on our [wiki](https://github.com/electricityMaps/electricitymaps-contrib/wiki/Edit-world-geometries).
+
+# How to update world.geojson
+
+To update geographies on the app
+
+1. Create manual changes on world.geojson
+2. Run `ts-node update-world.js` or `pnpm update-world` from web folder
+
+This will validate and generate the new world.json which is a compressed version of the world.geojson.

web/geo/files.ts

@@ -0,0 +1,21 @@
+import yaml from 'js-yaml';
+import fs from 'node:fs';
+import path from 'node:path';
+
+const saveZoneYaml = (zoneKey: string, zone: any) => {
+  const zonePath = path.resolve(__dirname, `../../config/zones/${zoneKey}.yaml`);
+  const sortObjectByKey = (object: any) =>
+    Object.keys(object)
+      .sort()
+      .reduce((result, key) => {
+        result[key] = object[key];
+        return result;
+      }, {});
+  fs.writeFile(zonePath, yaml.dump(sortObjectByKey(zone)), (error) => {
+    if (error) {
+      console.error(error);
+    }
+  });
+};
+
+export { saveZoneYaml };

web/geo/generate-aggregates.ts

@@ -0,0 +1,71 @@
+import { Feature, MultiPolygon, Polygon, Properties, union } from '@turf/turf';
+import { ZoneConfig } from './types';
+
+const generateAggregates = (geojson, zones: ZoneConfig) => {
+  const { features } = geojson;
+
+  const countryZonesToCombine = Object.values(zones)
+    .filter((zone) => zone.subZoneNames && zone.subZoneNames.length > 0)
+    .map((zone) => zone.subZoneNames);
+
+  const zonesToFlagAsNotAggregated = new Set(
+    Object.values(zones)
+      .filter((zone) => zone.subZoneNames && zone.subZoneNames.length > 0)
+      .flatMap((zone) => zone.subZoneNames)
+  );
+
+  const unCombinedZones = features.map((feature) => {
+    if (zonesToFlagAsNotAggregated.has(feature.properties.zoneName)) {
+      feature.properties.isAggregatedView = false;
+      feature.properties.isHighestGranularity = true;
+      return feature;
+    }
+    feature.properties.isAggregatedView = true;
+    feature.properties.isHighestGranularity = true;
+    return feature;
+  });
+
+  const combinedZones = countryZonesToCombine
+    .map((country) => {
+      if (country === undefined) {
+        return null;
+      }
+      const combinedCountry: Feature<MultiPolygon | Polygon, Properties> = {
+        type: 'Feature',
+        properties: {
+          isHighestGranularity: false,
+          isAggregatedView: true,
+          isCombined: true,
+        },
+        geometry: { type: 'MultiPolygon', coordinates: [] },
+      };
+      const multiZoneCountry = unCombinedZones.find(
+        (feature) => feature.properties.zoneName === country[0]
+      );
+      for (const element of country) {
+        const zoneToAdd = unCombinedZones.find(
+          (feature) => feature.properties.zoneName === element
+        );
+
+        const combinedCountryPolygon = combinedCountry.geometry as MultiPolygon;
+
+        const unionGeometry = union(combinedCountryPolygon, zoneToAdd.geometry)?.geometry;
+
+        if (unionGeometry) {
+          combinedCountry.geometry = unionGeometry;
+        }
+      }
+
+      if (combinedCountry.properties) {
+        combinedCountry.properties['countryKey'] = multiZoneCountry.properties.countryKey;
+        combinedCountry.properties['zoneName'] = multiZoneCountry.properties.countryKey;
+      }
+
+      return combinedCountry;
+    })
+    .filter((zone) => zone !== null);
+
+  return unCombinedZones.concat(combinedZones);
+};
+
+export { generateAggregates };

web/geo/generate-exchanges-to-exclude.ts

@@ -0,0 +1,57 @@
+import { mergeExchanges } from '../generate-zones-config';
+import { fileExists, getJSON, writeJSON } from './utilities';
+
+const exchangeConfig = mergeExchanges();
+
+const generateExchangesToIgnore = (OUT_PATH, zonesConfig) => {
+  console.info(`Generating new excluded-aggregated-exchanges.json...`);
+  const countryKeysToExclude = new Set(
+    Object.keys(zonesConfig).filter((key) => {
+      if (zonesConfig[key].subZoneNames?.length > 0) {
+        return key;
+      }
+    })
+  );
+
+  //Create a list of the exchange keys that we don't want to display in a country view
+  const unCombinedExchanges = Object.keys(exchangeConfig).filter((key) => {
+    const split = key.split('->');
+    const zoneOne = split[0];
+    const zoneTwo = split[1];
+
+    const subzoneSplitOne = zoneOne.split('-');
+    const subzoneSplitTwo = zoneTwo.split('-');
+    if (
+      (zoneOne.includes('-') && countryKeysToExclude.has(subzoneSplitOne[0])) ||
+      (zoneTwo.includes('-') && countryKeysToExclude.has(subzoneSplitTwo[0]))
+    ) {
+      return key;
+    }
+  });
+
+  //Create a list of the exchange keys that we don't want to display in the zone view
+  const countryExchangesWithSubzones = Object.keys(exchangeConfig).filter((key) => {
+    const split = key.split('->');
+    const zoneOne = split[0];
+    const zoneTwo = split[1];
+    if (
+      (!zoneOne.includes('-') && countryKeysToExclude.has(zoneOne)) ||
+      (!zoneTwo.includes('-') && countryKeysToExclude.has(zoneTwo))
+    ) {
+      return key;
+    }
+  });
+  const exchanges = {
+    exchangesToExcludeCountryView: unCombinedExchanges,
+    exchangesToExcludeZoneView: countryExchangesWithSubzones,
+  };
+  const existingExchanges = fileExists(OUT_PATH) ? getJSON(OUT_PATH) : {};
+  if (JSON.stringify(exchanges) === JSON.stringify(existingExchanges)) {
+    console.info(`No changes to excluded-aggregated-exchanges.json`);
+    return;
+  }
+
+  writeJSON(OUT_PATH, exchanges);
+};
+
+export { generateExchangesToIgnore };

web/geo/generate-topojson.ts

@@ -0,0 +1,72 @@
+import * as turf from '@turf/turf';
+import { topology } from 'topojson-server';
+import { fileExists, getJSON, round, writeJSON } from './utilities';
+
+function getCenter(geojson, zoneName) {
+  const geojsonFeatures = geojson.features.filter(
+    (f) => f.properties.zoneName === zoneName
+  );
+  if (geojsonFeatures.length !== 1) {
+    console.error(
+      `ERROR: Found ${geojsonFeatures.length} features matching zoneName ${zoneName}`
+    );
+    process.exit(1);
+  }
+
+  const longitudes: number[] = [];
+  const latitudes: number[] = [];
+
+  for (const { geometry } of turf.explode(geojsonFeatures[0].geometry).features) {
+    const [longitude, latitude] = geometry.coordinates;
+    longitudes.push(longitude);
+    latitudes.push(latitude);
+  }
+
+  if (longitudes.length === 0 || latitudes.length === 0) {
+    console.error(
+      `ERROR: Found ${longitudes.length} longitudes and ${latitudes} latitudes for zoneName ${zoneName}`
+    );
+    process.exit(1);
+  }
+
+  return [
+    round((Math.min(...longitudes) + Math.max(...longitudes)) / 2, 1),
+    round((Math.min(...latitudes) + Math.max(...latitudes)) / 2, 1),
+  ];
+}
+
+function generateTopojson(fc, { OUT_PATH, verifyNoUpdates }) {
+  const output = OUT_PATH.split('/').pop();
+  console.info(`Generating new ${output}`);
+  const topo = topology({
+    objects: fc,
+  });
+
+  // We do the following to match the specific format needed for visualization
+  const objects: any = topo.objects.objects;
+  const newObjects = {};
+  for (const geo of objects.geometries) {
+    // Precompute center for enable centering on the zone
+    geo.properties.center = getCenter(fc, geo.properties.zoneName);
+
+    newObjects[geo.properties.zoneName] = geo;
+  }
+  topo.objects = newObjects;
+
+  const currentTopo = fileExists(OUT_PATH) ? getJSON(OUT_PATH) : {};
+  if (JSON.stringify(currentTopo) === JSON.stringify(topo)) {
+    console.info(`No changes to ${output}`);
+    return;
+  }
+
+  if (verifyNoUpdates) {
+    console.error(
+      'Did not expect any updates to world.json. Please run "pnpm update-world"'
+    );
+    process.exit(1);
+  }
+
+  writeJSON(OUT_PATH, topo);
+}
+
+export { generateTopojson };