Spatial Aggregations V2 (electricitymaps#4627)

* Mostly aggregate exchanges * filter exchanges based on subzones * improve naming * add function to get all neighbours * remove unused argument * filter exchanges in left panel for zone or country view * tidy unused code * update format * Update electricitymap/contrib/config/__init__.py Co-authored-by: Markus Killendahl <[email protected]> * PR feedback Co-authored-by: Markus Killendahl <[email protected]>
VIKTORVAV99 · Oct 10, 2022 · d38a47b · d38a47b
1 parent 95e4516
commit d38a47b
Show file tree

Hide file tree

Showing 21 changed files with 300 additions and 319 deletions.
diff --git a/config/exchanges.json b/config/exchanges.json
@@ -114,6 +114,10 @@
     },
     "rotation": -140
   },
+  "AT->IT": {
+    "lonlat": [12.344464, 46.741723],
+    "rotation": -140
+  },
   "AT->SI": {
     "lonlat": [15.014142, 46.613582],
     "parsers": {
@@ -169,6 +173,11 @@
     },
     "rotation": -130
   },
+  "AX->SE": {
+    "capacity": [-80, 80],
+    "lonlat": [19.239, 60.063],
+    "rotation": -130
+  },
   "AZ->GE": {
     "lonlat": [45.153, 41.398],
     "parsers": {
@@ -505,6 +514,10 @@
     },
     "rotation": 180
   },
+  "CH->IT": {
+    "lonlat": [9.047334, 46.113596],
+    "rotation": 180
+  },
   "CN->RU-AS": {
     "lonlat": [123.275128, 53.541248],
     "parsers": {
@@ -550,6 +563,10 @@
     },
     "rotation": 135
   },
+  "DE->DK": {
+    "lonlat": [9.3, 54.9],
+    "rotation": 0
+  },
   "DE->DK-DK1": {
     "capacity": [-2500, 2500],
     "lonlat": [9.3, 54.9],
@@ -601,6 +618,11 @@
     },
     "rotation": -10
   },
+  "DE->NO": {
+    "capacity": [-1444, 1444],
+    "lonlat": [7.2, 55.9],
+    "rotation": -10
+  },
   "DE->PL": {
     "lonlat": [14.585163, 52.410625],
     "parsers": {
@@ -618,6 +640,11 @@
     },
     "rotation": 0
   },
+  "DE->SE": {
+    "lonlat": [13.552264, 54.925814],
+    "capacity": [-615, 615],
+    "rotation": 0
+  },
   "DK-BHM->SE-SE4": {
     "capacity": [-60, 60],
     "lonlat": [14.509682, 55.363057],
@@ -663,6 +690,10 @@
     },
     "rotation": 70
   },
+  "DK->SE": {
+    "lonlat": [11.556268, 56.857802],
+    "rotation": 70
+  },
   "DK-DK2->SE-SE4": {
     "capacity": [-1300, 1700],
     "lonlat": [12.704418, 55.952282],
@@ -761,6 +792,10 @@
     },
     "rotation": -30
   },
+  "FI->NO": {
+    "lonlat": [25.35158, 68.862684],
+    "rotation": -30
+  },
   "FI->RU-1": {
     "capacity": [-1460, 320],
     "lonlat": [28.378, 60.878],
@@ -779,6 +814,10 @@
     },
     "rotation": -90
   },
+  "FI->SE": {
+    "lonlat": [23.857, 66.921],
+    "rotation": -90
+  },
   "FI->SE-SE3": {
     "capacity": [-1200, 1200],
     "lonlat": [19.797, 60.628],
@@ -805,6 +844,18 @@
     },
     "rotation": 70
   },
+  "FR->IT": {
+    "lonlat": [6.9, 44.4],
+    "rotation": 70
+  },
+  "FR-COR->IT": {
+    "lonlat": [10.091148, 42.686804],
+    "parsers": {
+      "exchange": "ENTSOE.fetch_exchange",
+      "exchangeForecast": "ENTSOE.fetch_exchange_forecast"
+    },
+    "rotation": 45
+  },
   "FR-COR->IT-CNO": {
     "lonlat": [10.091148, 42.686804],
     "parsers": {
@@ -865,6 +916,10 @@
     },
     "rotation": 50
   },
+  "GB->NO": {
+    "lonlat": [2.44819, 57.266334],
+    "rotation": 50
+  },
   "GE->RU-1": {
     "lonlat": [42.822242, 43.158267],
     "parsers": {
@@ -912,6 +967,10 @@
     },
     "rotation": -90
   },
+  "GR->IT": {
+    "lonlat": [18.759248, 38.902132],
+    "rotation": -90
+  },
   "GR->MK": {
     "lonlat": [22.011736, 41.160374],
     "parsers": {
@@ -1099,6 +1158,10 @@
     },
     "rotation": 90
   },
+  "IT->SI": {
+    "lonlat": [13.596393, 46.105418],
+    "rotation": 90
+  },
   "IT-SIC->IT-SO": {
     "lonlat": [15.65, 38.1],
     "parsers": {
@@ -1116,6 +1179,10 @@
     },
     "rotation": -160
   },
+  "IT->MT": {
+    "lonlat": [14.675346, 36.264287],
+    "rotation": -160
+  },
   "JP-CB->JP-HR": {
     "lonlat": [136.85, 36.35],
     "parsers": {
@@ -1234,6 +1301,10 @@
     },
     "rotation": -90
   },
+  "LT->SE": {
+    "lonlat": [18.847674, 55.910978],
+    "rotation": -90
+  },
   "LV->RU-1": {
     "capacity": [-970, 970],
     "lonlat": [27.7336, 56.87436],
@@ -1445,6 +1516,10 @@
     },
     "rotation": -90
   },
+  "NO->SE": {
+    "lonlat": [14.055, 64.918],
+    "rotation": 100
+  },
   "NO-NO1->SE-SE3": {
     "capacity": [-2095, 2145],
     "lonlat": [12.254138, 61.008235],
@@ -1516,6 +1591,10 @@
     },
     "rotation": 120
   },
+  "NO->RU-1": {
+    "lonlat": [30.1437, 69.52],
+    "rotation": 120
+  },
   "PL->SE-SE4": {
     "capacity": [-600, 600],
     "lonlat": [15.969256, 55.215172],
@@ -1525,6 +1604,11 @@
     },
     "rotation": -20
   },
+  "PL->SE": {
+    "capacity": [-600, 600],
+    "lonlat": [15.969256, 55.215172],
+    "rotation": -20
+  },
   "PL->SK": {
     "lonlat": [20.614102, 49.359467],
     "parsers": {
@@ -1587,16 +1671,16 @@
   },
   "SE-SE1->SE-SE2": {
     "capacity": [-3300, 3300],
-    "lonlat": [17.869, 65.112],
+    "lonlat": [17.869, 65.469],
     "parsers": {
       "exchange": "ENTSOE.fetch_exchange",
       "exchangeForecast": "ENTSOE.fetch_exchange_forecast"
     },
-    "rotation": 180
+    "rotation": 205
   },
   "SE-SE2->SE-SE3": {
     "capacity": [-7300, 7300],
-    "lonlat": [14.804, 61.661],
+    "lonlat": [14.804, 61.42],
     "parsers": {
       "exchange": "ENTSOE.fetch_exchange",
       "exchangeForecast": "ENTSOE.fetch_exchange_forecast"
@@ -1605,7 +1689,7 @@
   },
   "SE-SE3->SE-SE4": {
     "capacity": [-2000, 5400],
-    "lonlat": [14.518, 57.18],
+    "lonlat": [14.518, 57.78],
     "parsers": {
       "exchange": "ENTSOE.fetch_exchange",
       "exchangeForecast": "ENTSOE.fetch_exchange_forecast"

diff --git a/config/zones.json b/config/zones.json
@@ -1167,7 +1167,7 @@
       "production": "ENTSOE.fetch_production",
       "productionPerModeForecast": "ENTSOE.fetch_wind_solar_forecasts"
     },
-    "subZoneNames": ["DK-DK1", "DK-DK2"],
+    "subZoneNames": ["DK-DK1", "DK-DK2", "DK-BHM"],
     "timezone": "Europe/Copenhagen"
   },
   "DK-BHM": {

diff --git a/electricitymap/contrib/config/__init__.py b/electricitymap/contrib/config/__init__.py
@@ -72,6 +72,20 @@ def generate_zone_neighbours(
     ZONES_CONFIG, EXCHANGES_CONFIG
 )
 
+# This object represents all neighbours regardless of granularity
+def generate_all_neighbours(exchanges_config) -> Dict[ZoneKey, List[ZoneKey]]:
+    zone_neighbours = defaultdict(set)
+    for k, v in exchanges_config.items():
+        zone_1, zone_2 = k.split("->")
+        pairs = [(zone_1, zone_2), (zone_2, zone_1)]
+        for zone_name_1, zone_name_2 in pairs:
+            zone_neighbours[zone_name_1].add(zone_name_2)
+    # Sort
+    return {k: sorted(v) for k, v in zone_neighbours.items()}
+
+
+ALL_NEIGHBOURS: Dict[ZoneKey, List[ZoneKey]] = generate_all_neighbours(EXCHANGES_CONFIG)
+
 
 def emission_factors(zone_key: ZoneKey) -> Dict[str, float]:
     override = CO2EQ_PARAMETERS["emissionFactors"]["zoneOverrides"].get(zone_key, {})

diff --git a/web/geo/generate-exchanges-to-exclude.js b/web/geo/generate-exchanges-to-exclude.js
@@ -1,30 +1,47 @@
 const { getJSON, writeJSON, fileExists } = require('./utilities');
 const exchangeConfig = require('../../config/exchanges.json');
 
-const generateExchangesToIgnore = (OUT_PATH, fc) => {
+const generateExchangesToIgnore = (OUT_PATH, zonesConfig) => {
   console.log(`Generating new excluded-aggregated-exchanges.json...`); // eslint-disable-line no-console
-  const fcCombined = {
-    ...fc,
-    features: fc.features.filter((feature) => {
-      try {
-        return feature.properties.isCombined;
-      } catch (e) {
-        console.log('Error: ', e, 'Feature: ', feature); // eslint-disable-line no-console
-      }
-    }),
-  };
 
-  const countryKeysToExclude = fcCombined.features.map((feature) => feature.properties.countryKey);
+  const countryKeysToExclude = Object.keys(zonesConfig).filter((key) => {
+    if (zonesConfig[key].subZoneNames?.length > 0) {
+      return key;
+    }
+  });
 
+  //Create a list of the exchange keys that we don't want to display in a country view
   const unCombinedExchanges = Object.keys(exchangeConfig).filter((key) => {
     const split = key.split('->');
-    const zoneOne = split[0].slice(0, 2);
-    const zoneTwo = split[1].slice(0, 2);
-    if (zoneOne === zoneTwo && countryKeysToExclude.includes(key.slice(0, 2))) {
+    const zoneOne = split[0];
+    const zoneTwo = split[1];
+
+    const subzoneSplitOne = zoneOne.split('-');
+    const subzoneSplitTwo = zoneTwo.split('-');
+    if (
+      (zoneOne.includes('-') && countryKeysToExclude.includes(subzoneSplitOne[0])) ||
+      (zoneTwo.includes('-') && countryKeysToExclude.includes(subzoneSplitTwo[0]))
+    ) {
       return key;
     }
   });
-  const exchanges = { exchangesToExclude: unCombinedExchanges };
+
+  //Create a list of the exchange keys that we don't want to display in the zone view
+  const countryExchangesWithSubzones = Object.keys(exchangeConfig).filter((key) => {
+    const split = key.split('->');
+    const zoneOne = split[0];
+    const zoneTwo = split[1];
+    if (
+      (!zoneOne.includes('-') && countryKeysToExclude.includes(zoneOne)) ||
+      (!zoneTwo.includes('-') && countryKeysToExclude.includes(zoneTwo))
+    ) {
+      return key;
+    }
+  });
+  const exchanges = {
+    exchangesToExcludeCountryView: unCombinedExchanges,
+    exchangesToExcludeZoneView: countryExchangesWithSubzones,
+  };
   const existingExchanges = fileExists(OUT_PATH) ? getJSON(OUT_PATH) : {};
   if (JSON.stringify(exchanges) === JSON.stringify(existingExchanges)) {
     console.log(`No changes to excluded-aggregated-exchanges.json`); // eslint-disable-line no-console

diff --git a/web/geo/update-world.js b/web/geo/update-world.js
@@ -1,24 +1,12 @@
 const path = require('path');
-
 const { validateGeometry } = require('./validate');
-const { validateGeometryV2 } = require('./validate-v2');
 const { getJSON } = require('./utilities');
 const { generateTopojson } = require('./generate-topojson');
 const { generateAggregates } = require('./generate-aggregates');
 const { generateExchangesToIgnore } = require('./generate-exchanges-to-exclude');
 const { getZonesJson } = require('./files');
 
 const config = {
-  WORLD_PATH: path.resolve(__dirname, './world.geojson'),
-  OUT_PATH: path.resolve(__dirname, '../src/world.json'),
-  ERROR_PATH: path.resolve(__dirname, '.'),
-  MIN_AREA_HOLES: 5000000,
-  MAX_CONVEX_DEVIATION: 0.708,
-  MIN_AREA_INTERSECTION: 500000,
-  SLIVER_RATIO: 0.0001, // ratio of length and area to determine if the polygon is a sliver and should be ignored
-  verifyNoUpdates: process.env.VERIFY_NO_UPDATES !== undefined,
-};
-const configV2 = {
   WORLD_PATH: path.resolve(__dirname, './world.geojson'),
   OUT_PATH: path.resolve(__dirname, '../src/world-aggregated.json'),
   ERROR_PATH: path.resolve(__dirname, '.'),
@@ -32,15 +20,11 @@ const configV2 = {
 const EXCHANGE_OUT_PATH = path.resolve(__dirname, '../src/excluded-aggregated-exchanges.json');
 
 const fc = getJSON(config.WORLD_PATH);
-validateGeometry(fc, config);
-generateTopojson(fc, config);
-
-const fcV2 = getJSON(config.WORLD_PATH);
 const zoneConfig = getZonesJson();
-const aggregates = generateAggregates(fcV2, zoneConfig);
+const aggregates = generateAggregates(fc, zoneConfig);
 
-fcV2.features = aggregates;
+fc.features = aggregates;
 
-validateGeometryV2(fcV2, configV2);
-generateTopojson(fcV2, configV2);
-generateExchangesToIgnore(EXCHANGE_OUT_PATH, fcV2);
+validateGeometry(fc, config);
+generateTopojson(fc, config);
+generateExchangesToIgnore(EXCHANGE_OUT_PATH, zoneConfig);