From 8bf4f4e7590182fd5ee1e3696fcf9fdd1737c92d Mon Sep 17 00:00:00 2001 From: Karn Bianco Date: Thu, 12 Jan 2017 12:36:29 +0000 Subject: [PATCH 1/6] Update README with some Windows setup troubleshooting --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 29e297a609..ccd516904b 100644 --- a/README.md +++ b/README.md @@ -155,3 +155,7 @@ Once you're done doing your changes, submit a [pull request](https://help.github - `KeyError: 'ENTSOE_TOKEN'`: in order to request data from the ENTSOE, you need an API key. You can create an account and request your API key by [following this link](https://transparency.entsoe.eu/content/static_content/Static%20content/web%20api/Guide.html). Once you have it, add it to your `secrets.env` file. - No website found at `http://localhost:8000`: This can happen if you're running Docker in a virtual machine. Find out docker's IP using `docker-machine ip default`, and replace `localhost` by your Docker IP when connecting. + +- `ERROR: for mongo Cannot create container for service mongo: C:/ drive is not shared. Please share it in Docker for Windows Settings` Docker for Windows requires setting up shared drives [(instructions here)](https://docs.docker.com/docker-for-windows/#shared-drives) but if after entering your Windows system username and password nothing happens (not even an error message) you might need to **run Docker for Windows as administrator**. + +- `ERROR: "MongoError: failed to connect to server [mongo:27017] on first connect"` When running `docker-compose up` on Windows this can be caused by an invalid mongo volume path in `docker-compose.yml`. You can fix this by replacing `volumes: ['./mongodata/:/data/db']` with `volumes: ['/data/db']` \ No newline at end of file From 576fcd6ea3147b5eebb3395d2beba0bd45adb174 Mon Sep 17 00:00:00 2001 From: Karn Bianco Date: Thu, 12 Jan 2017 12:42:40 +0000 Subject: [PATCH 2/6] Remove unnecessary troubleshooting note from README --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index ccd516904b..2e6cb17f93 100644 --- a/README.md +++ b/README.md @@ -156,6 +156,4 @@ Once you're done doing your changes, submit a [pull request](https://help.github - No website found at `http://localhost:8000`: This can happen if you're running Docker in a virtual machine. Find out docker's IP using `docker-machine ip default`, and replace `localhost` by your Docker IP when connecting. -- `ERROR: for mongo Cannot create container for service mongo: C:/ drive is not shared. Please share it in Docker for Windows Settings` Docker for Windows requires setting up shared drives [(instructions here)](https://docs.docker.com/docker-for-windows/#shared-drives) but if after entering your Windows system username and password nothing happens (not even an error message) you might need to **run Docker for Windows as administrator**. - -- `ERROR: "MongoError: failed to connect to server [mongo:27017] on first connect"` When running `docker-compose up` on Windows this can be caused by an invalid mongo volume path in `docker-compose.yml`. You can fix this by replacing `volumes: ['./mongodata/:/data/db']` with `volumes: ['/data/db']` \ No newline at end of file +- `ERROR: for mongo Cannot create container for service mongo: C:/ drive is not shared. Please share it in Docker for Windows Settings` Docker for Windows requires setting up shared drives [(instructions here)](https://docs.docker.com/docker-for-windows/#shared-drives) but if after entering your Windows system username and password nothing happens (not even an error message) you might need to **run Docker for Windows as administrator**. \ No newline at end of file From 23114786a522aa8dff8e3ad49fabcf0691c50e8e Mon Sep 17 00:00:00 2001 From: Karn Bianco Date: Thu, 2 Feb 2017 17:04:58 +0000 Subject: [PATCH 3/6] Add data parser for Iceland --- README.md | 1 + feeder/feeder.py | 4 ++- feeder/parsers/IS.py | 67 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 feeder/parsers/IS.py diff --git a/README.md b/README.md index 61fd20801f..05be39c458 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ Each country has a GHG mass flow that depends on neighboring countries. In order - Great Britain: [ENTSOE](https://transparency.entsoe.eu/content/static_content/Static%20content/web%20api/Guide.html) - Greece: [ENTSOE](https://transparency.entsoe.eu/content/static_content/Static%20content/web%20api/Guide.html) - Hungary: [ENTSOE](https://transparency.entsoe.eu/content/static_content/Static%20content/web%20api/Guide.html) +- Iceland [LANDSNET](http://amper.landsnet.is/MapData/api/measurements) - Ireland: [ENTSOE](https://transparency.entsoe.eu/content/static_content/Static%20content/web%20api/Guide.html) - Italy: [ENTSOE](https://transparency.entsoe.eu/content/static_content/Static%20content/web%20api/Guide.html) - Latvia: [ENTSOE](https://transparency.entsoe.eu/content/static_content/Static%20content/web%20api/Guide.html) diff --git a/feeder/feeder.py b/feeder/feeder.py index 73fd0064d3..c717926cd8 100644 --- a/feeder/feeder.py +++ b/feeder/feeder.py @@ -8,6 +8,7 @@ from bson.binary import Binary from pymemcache.client.base import Client +from parsers import IS from parsers import FR from parsers import ENTSOE from parsers import weather @@ -105,6 +106,7 @@ 'GR': ENTSOE.fetch_production, 'HU': ENTSOE.fetch_production, 'IE': ENTSOE.fetch_production, + 'IS': IS.fetch_production, 'IT': ENTSOE.fetch_production, 'LT': ENTSOE.fetch_production, 'LU': ENTSOE.fetch_production, @@ -300,7 +302,7 @@ def validate_production(obj, country_code): raise Exception("Data from %s can't be in the future" % country_code) if obj.get('production', {}).get('unknown', None) is None and \ obj.get('production', {}).get('coal', None) is None and \ - country_code not in ['CH', 'NO']: + country_code not in ['CH', 'NO', 'IS']: raise Exception("Coal or unknown production value is required for %s" % (country_code)) for k, v in obj['production'].iteritems(): if v is None: continue diff --git a/feeder/parsers/IS.py b/feeder/parsers/IS.py new file mode 100644 index 0000000000..36e3b6d4a2 --- /dev/null +++ b/feeder/parsers/IS.py @@ -0,0 +1,67 @@ +import requests +from datetime import datetime +from collections import defaultdict +import xml.etree.ElementTree as ET +import json + +STATIONS = { + + # http://www.landsvirkjun.com/company/powerstations/ + 'Bjarnaflag': 'unknown', # Geothermal (no data?) + 'BLANDA_F': 'hydro', + 'BUDAR_O': 'hydro', + 'BURF_F': 'hydro', + 'FLJOTSDA': 'hydro', + 'Hafio': 'wind', # No data? + 'HRAUN_F': 'hydro', + 'IRAFOS_F': 'hydro', + 'KRAFLA_F': 'unknown', # Geothermal + 'LAXA_F': 'hydro', + 'LAXARVAT': 'hydro', + 'laxa': 'hydro', # No data? + 'LJOSIF_F': 'hydro', + 'SIG_F': 'hydro', + 'Steingrimsstod': 'hydro', # No data? + 'SULTAR_F': 'hydro', + 'VATNSHAM': 'hydro', + + # https://en.wikipedia.org/wiki/List_of_power_stations_in_Iceland + 'KOLVID': 'unknown', # Geothermal + 'LAGARF': 'hydro', + 'MJOLKA': 'hydro', + 'REY': 'unknown', # Geothermal + 'X_NESJAV': 'unknown', # Geothermal + 'SVA': 'unknown', # Geothermal +} + +def fetch_production(country_code='IS', session=None): + + # Query Landsnet for latest power production data for Iceland + r = session or requests.session() + url = 'http://amper.landsnet.is/MapData/api/measurements' + response = r.get(url) + json_obj = json.loads(response.text) + + production = defaultdict(float) + + # Calculate production values for each power station + for key, value in STATIONS.items(): + items = [item for item in json_obj if item["substation"] == key and item["MW"] >= 0] + mw = sum(item['MW'] for item in items) + production[value] = production[value] + mw; + + # Get datetime for last update (e.g. 2017-02-02T14:35:00) + totalpowerflow = next(item for item in json_obj if item["key"] == "TOTAL_POWER_FLOW") + datetime_last = datetime.strptime(totalpowerflow["time"], '%Y-%m-%dT%H:%M:%S') + + data = { + 'countryCode': country_code, + 'production': dict(production), + 'datetime': datetime_last, + 'storage': {}, + 'source': 'landsnet.is', + } + return data + +if __name__ == '__main__': + print fetch_production() \ No newline at end of file From f46c14a08854e3534504506bbfb5fa05e0be6712 Mon Sep 17 00:00:00 2001 From: Karn Bianco Date: Fri, 3 Feb 2017 19:28:28 +0000 Subject: [PATCH 4/6] Add support for geothermal energy production --- feeder/parsers/ENTSOE.py | 11 +++++++---- feeder/parsers/IS.py | 12 ++++++------ shared/co2eq_parameters.js | 1 + web/app/countrytable.js | 2 ++ 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/feeder/parsers/ENTSOE.py b/feeder/parsers/ENTSOE.py index 8ce8bff49c..9a3c4f9dac 100644 --- a/feeder/parsers/ENTSOE.py +++ b/feeder/parsers/ENTSOE.py @@ -271,13 +271,15 @@ def get_wind(values): if 'Wind Onshore' in values or 'Wind Offshore' in values: return values.get('Wind Onshore', 0) + values.get('Wind Offshore', 0) +def get_geothermal(values): + if 'Geothermal' in values: + return values.get('Geothermal', 0); + def get_unknown(values): - if 'Geothermal' in values \ - or 'Marine' in values \ + if 'Marine' in values \ or 'Other renewable' in values \ or 'Other' in values: - return values.get('Geothermal', 0) + \ - values.get('Marine', 0) + \ + return values.get('Marine', 0) + \ values.get('Other renewable', 0) + \ values.get('Other', 0) @@ -335,6 +337,7 @@ def fetch_production(country_code, session=None): 'oil': get_oil(production_values), 'solar': production_values.get('Solar', None), 'wind': get_wind(production_values), + 'geothermal': get_geothermal(production_values), 'unknown': get_unknown(production_values) }, 'storage': { diff --git a/feeder/parsers/IS.py b/feeder/parsers/IS.py index 36e3b6d4a2..9a8939d670 100644 --- a/feeder/parsers/IS.py +++ b/feeder/parsers/IS.py @@ -7,7 +7,7 @@ STATIONS = { # http://www.landsvirkjun.com/company/powerstations/ - 'Bjarnaflag': 'unknown', # Geothermal (no data?) + 'Bjarnaflag': 'geothermal', # No data? 'BLANDA_F': 'hydro', 'BUDAR_O': 'hydro', 'BURF_F': 'hydro', @@ -15,7 +15,7 @@ 'Hafio': 'wind', # No data? 'HRAUN_F': 'hydro', 'IRAFOS_F': 'hydro', - 'KRAFLA_F': 'unknown', # Geothermal + 'KRAFLA_F': 'geothermal', 'LAXA_F': 'hydro', 'LAXARVAT': 'hydro', 'laxa': 'hydro', # No data? @@ -26,12 +26,12 @@ 'VATNSHAM': 'hydro', # https://en.wikipedia.org/wiki/List_of_power_stations_in_Iceland - 'KOLVID': 'unknown', # Geothermal + 'KOLVID': 'geothermal', 'LAGARF': 'hydro', 'MJOLKA': 'hydro', - 'REY': 'unknown', # Geothermal - 'X_NESJAV': 'unknown', # Geothermal - 'SVA': 'unknown', # Geothermal + 'REY': 'geothermal', + 'X_NESJAV': 'geothermal', + 'SVA': 'geothermal', } def fetch_production(country_code='IS', session=None): diff --git a/shared/co2eq_parameters.js b/shared/co2eq_parameters.js index e5739b2109..b3b8eb2313 100644 --- a/shared/co2eq_parameters.js +++ b/shared/co2eq_parameters.js @@ -9,6 +9,7 @@ defaultCo2eqFootprint = { 'oil': 650, 'solar': 45, 'wind': 12, + 'geothermal': 24, 'unknown': 700, // assume conventional 'other': 700 // same as 'unknown'. Here for backward compatibility }; // in gCo2eq/kWh diff --git a/web/app/countrytable.js b/web/app/countrytable.js index 8d003a8a41..2bea3894a0 100644 --- a/web/app/countrytable.js +++ b/web/app/countrytable.js @@ -24,6 +24,7 @@ function CountryTable(selector, co2Color) { 'solar': '#f27406', 'hydro': '#2772b2', 'biomass': '#166a57', + 'geothermal': 'yellow', 'nuclear': '#AEB800', 'gas': '#bb2f51', 'coal': '#ac8c35', @@ -42,6 +43,7 @@ function CountryTable(selector, co2Color) { 'solar', 'hydro', 'hydro storage', + 'geothermal', 'biomass', 'nuclear', 'gas', From 825695bcb842326171d033dcd93d455474dc7fdd Mon Sep 17 00:00:00 2001 From: Karn Bianco Date: Sat, 4 Feb 2017 10:54:29 +0000 Subject: [PATCH 5/6] Iceland: set zero values for known unused energy sources --- feeder/feeder.py | 2 +- feeder/parsers/IS.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/feeder/feeder.py b/feeder/feeder.py index c717926cd8..7a54f6955e 100644 --- a/feeder/feeder.py +++ b/feeder/feeder.py @@ -302,7 +302,7 @@ def validate_production(obj, country_code): raise Exception("Data from %s can't be in the future" % country_code) if obj.get('production', {}).get('unknown', None) is None and \ obj.get('production', {}).get('coal', None) is None and \ - country_code not in ['CH', 'NO', 'IS']: + country_code not in ['CH', 'NO']: raise Exception("Coal or unknown production value is required for %s" % (country_code)) for k, v in obj['production'].iteritems(): if v is None: continue diff --git a/feeder/parsers/IS.py b/feeder/parsers/IS.py index 9a8939d670..7d14839270 100644 --- a/feeder/parsers/IS.py +++ b/feeder/parsers/IS.py @@ -42,9 +42,17 @@ def fetch_production(country_code='IS', session=None): response = r.get(url) json_obj = json.loads(response.text) + # Set zero values for energy sources Iceland doesn't use at all + # Iceland does use some wind and gas but we don't have data for those production = defaultdict(float) + production["solar"] = 0; + production["biomass"] = 0; + production["nuclear"] = 0; + production["coal"] = 0; # Calculate production values for each power station + # The Landsnet API includes measurements for non-generating + # stations (e.g. capacitor banks) so ignore any not in our list for key, value in STATIONS.items(): items = [item for item in json_obj if item["substation"] == key and item["MW"] >= 0] mw = sum(item['MW'] for item in items) From 06ec40f12e22621c5cde060d76e678f8c727a674 Mon Sep 17 00:00:00 2001 From: Karn Bianco Date: Sat, 4 Feb 2017 12:55:42 +0000 Subject: [PATCH 6/6] Iceland: Add capacity data and source --- README.md | 1 + web/app/configs/capacities.json | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/README.md b/README.md index 05be39c458..96fddf8165 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,7 @@ Each country has a GHG mass flow that depends on neighboring countries. In order - Great Britain: [ENTSO-E](https://transparency.entsoe.eu/generation/r2/installedGenerationCapacityAggregation/show) - Greece: [ENTSO-E](https://transparency.entsoe.eu/generation/r2/installedGenerationCapacityAggregation/show) - Hungary: [ENTSO-E](https://transparency.entsoe.eu/generation/r2/installedGenerationCapacityAggregation/show) +- Iceland: [Statistics Iceland](http://px.hagstofa.is/pxen/pxweb/en/Atvinnuvegir/Atvinnuvegir__orkumal/IDN02101.px) - Ireland - All production types: [ENTSO-E](https://transparency.entsoe.eu/generation/r2/installedGenerationCapacityAggregation/show) - Wind: [IWEA](http://www.iwea.com/index.cfm/page/windenergyfaqs?#q21) diff --git a/web/app/configs/capacities.json b/web/app/configs/capacities.json index f5d1ae67ca..2438f2aa2e 100644 --- a/web/app/configs/capacities.json +++ b/web/app/configs/capacities.json @@ -171,6 +171,20 @@ "wind": 3025 } }, + "IS": { + "capacity": { + "biomass": 0, + "oil": 117, + "coal": 0, + "hydro": 1986, + "geothermal": 665, + "nuclear": 0, + "gas": 0, + "unknown": 0, + "solar": 0, + "wind": 3 + } + }, "IT": { "capacity": { "hydro": 22382,