From ef8070811ee78b01e4cde15dd9a0a9e1e6f5626d Mon Sep 17 00:00:00 2001 From: Olivier Corradi Date: Tue, 20 Sep 2016 16:51:04 +0200 Subject: [PATCH] Feature/hungary (#68) * Initial commit (see #65) * Added exchanges * Added HU exchanges --- api/static/app/countryconfig.js | 59 +++++++++++++ api/static/app/exchangeconfig.js | 107 ++++++++++++++++++++++++ api/static/app/main.js | 137 +------------------------------ api/static/index.html | 2 + feeder/Dockerfile | 2 + feeder/feeder.py | 2 + feeder/parsers/HU.py | 64 +++++++++++++++ feeder/requirements.txt | 3 + 8 files changed, 241 insertions(+), 135 deletions(-) create mode 100644 api/static/app/countryconfig.js create mode 100644 api/static/app/exchangeconfig.js create mode 100644 feeder/parsers/HU.py diff --git a/api/static/app/countryconfig.js b/api/static/app/countryconfig.js new file mode 100644 index 0000000000..a96c05574e --- /dev/null +++ b/api/static/app/countryconfig.js @@ -0,0 +1,59 @@ +function addCountryConfiguration(countries) { + countries['DE'].data.capacity = { + biomass: 8970, + coal: 28310 + 21140, + gas: 28490, + hydro: 5590, + nuclear: 10790, + oil: 4240, + solar: 39630, + wind: 3740 + 42460, + }; + countries['DK'].data.capacity = { + hydro: 0, + nuclear: 0, + solar: 790, + wind: 5070, + }; + countries['ES'].data.capacity = { + coal: 11482, + gas: 3498 + 27206, + hydro: 17787 + 2106, + nuclear: 7866, + solar: 4672 + 2300, + wind: 23002, + }; + countries['FI'].data.capacity = { + hydro: 3080, + nuclear: 2860, + wind: 1000 + }; + countries['FR'].data.capacity = { + nuclear: 63130, + oil: 6670, + coal: 2930, + hydro: 10326 + 8204 + 4965, + gas: 6121, + wind: 10358, + solar: 6580 + }; + countries['GB'].data.capacity = { + wind: 13500, + nuclear: 9000, + hydro: 1550, + gas: 38000, + solar: 8780 + }; + countries['NO'].data.capacity = { + hydro: 31000, + nuclear: 0, + solar: 0, + wind: 856, + }; + countries['SE'].data.capacity = { + hydro: 16200, + nuclear: 8849, + solar: 79, + wind: 6025 + }; +} diff --git a/api/static/app/exchangeconfig.js b/api/static/app/exchangeconfig.js new file mode 100644 index 0000000000..cc5eecc778 --- /dev/null +++ b/api/static/app/exchangeconfig.js @@ -0,0 +1,107 @@ +var EXCHANGES_CONFIG = [ + // DK + { + countries: ['DE', 'DK'], + lonlat: [9.3, 54.9], + rotation: 0 + }, + { + countries: ['SE', 'DK'], + lonlat: [13, 55.7], + rotation: -100 + }, + { + countries: ['DK', 'NO'], + lonlat: [8.8, 58], + rotation: -25 + }, + // GB + { + countries: ['GB', 'FR'], + lonlat: [0, 50.4], + rotation: 160 + }, + { + countries: ['IE', 'GB'], + lonlat: [-5.7, 53], + rotation: 100 + }, + { + countries: ['NL', 'GB'], + lonlat: [3.3, 52.4], + rotation: -90 + }, + // FR + { + countries: ['FR', 'ES'], + lonlat: [0.3, 42.9], + rotation: -160 + }, + { + countries: ['FR', 'DE'], + lonlat: [5.7, 49.8], + rotation: 50 + }, + { + countries: ['FR', 'CH'], + lonlat: [6.5, 46.7], + rotation: 90 + }, + { + countries: ['FR', 'IT'], + lonlat: [6.5, 44.5], + rotation: 70 + }, + // RO + { + countries: ['RO', 'HU'], + lonlat: [21.8074107, 47.1141229], + rotation: -70 + }, + { + countries: ['RO', 'UA'], + lonlat: [24.821959, 47.768595], + rotation: 0 + }, + { + countries: ['RO', 'MD'], + lonlat: [28.009764, 47.003312], + rotation: 60 + }, + { + countries: ['RO', 'BG'], + lonlat: [25.609385, 43.674878], + rotation: 180 + }, + { + countries: ['RO', 'RS'], + lonlat: [21.469049, 44.947107], + rotation: -140 + }, + // HU + { + countries: ['HU', 'SK'], + lonlat: [19.615617, 48.204006], + rotation: 0 + }, + { + countries: ['HU', 'UA'], + lonlat: [22.526994, 48.240603], + rotation: 35 + }, + { + countries: ['HU', 'RS'], + lonlat: [19.494768, 46.112673], + rotation: 180 + }, + { + countries: ['HU', 'HR'], + lonlat: [17.407365, 45.967775], + rotation: 210 + }, + { + countries: ['HU', 'AT'], + lonlat: [16.605363, 47.444264], + rotation: -60 + } +]; diff --git a/api/static/app/main.js b/api/static/app/main.js index a7ed5b9855..d0741aa5b1 100644 --- a/api/static/app/main.js +++ b/api/static/app/main.js @@ -38,84 +38,6 @@ if (!nobrowsercheck && !isChrome()) { .range(['black', 'orange']) .domain([300, 440]); - var EXCHANGES_CONFIG = [ - { - countries: ['DE', 'DK'], - lonlat: [9.3, 54.9], - rotation: 0 - }, - { - countries: ['SE', 'DK'], - lonlat: [13, 55.7], - rotation: -100 - }, - { - countries: ['GB', 'FR'], - lonlat: [0, 50.4], - rotation: 160 - }, - { - countries: ['DK', 'NO'], - lonlat: [8.8, 58], - rotation: -25 - }, - { - countries: ['IE', 'GB'], - lonlat: [-5.7, 53], - rotation: 100 - }, - { - countries: ['NL', 'GB'], - lonlat: [3.3, 52.4], - rotation: -90 - }, - { - countries: ['FR', 'ES'], - lonlat: [0.3, 42.9], - rotation: -160 - }, - { - countries: ['FR', 'DE'], - lonlat: [5.7, 49.8], - rotation: 50 - }, - { - countries: ['FR', 'CH'], - lonlat: [6.5, 46.7], - rotation: 90 - }, - { - countries: ['FR', 'IT'], - lonlat: [6.5, 44.5], - rotation: 70 - }, - { - countries: ['RO', 'HU'], - lonlat: [21.8074107, 47.1141229], - rotation: -70 - }, - { - countries: ['RO', 'UA'], - lonlat: [24.821959, 47.768595], - rotation: 0 - }, - { - countries: ['RO', 'MD'], - lonlat: [28.009764, 47.003312], - rotation: 60 - }, - { - countries: ['RO', 'BG'], - lonlat: [25.609385, 43.674878], - rotation: 180 - }, - { - countries: ['RO', 'RS'], - lonlat: [21.469049, 44.947107], - rotation: -140 - } - ]; - // Set up objects var countryMap = new CountryMap('.map', co2color); var exchangeLayer = new ExchangeLayer('.map'); @@ -316,63 +238,8 @@ if (!nobrowsercheck && !isChrome()) { countries[countryCode].data.neighborCo2 = {}; }); - countries['DE'].data.capacity = { - biomass: 8970, - coal: 28310 + 21140, - gas: 28490, - hydro: 5590, - nuclear: 10790, - oil: 4240, - solar: 39630, - wind: 3740 + 42460, - }; - countries['DK'].data.capacity = { - hydro: 0, - nuclear: 0, - solar: 790, - wind: 5070, - }; - countries['ES'].data.capacity = { - coal: 11482, - gas: 3498 + 27206, - hydro: 17787 + 2106, - nuclear: 7866, - solar: 4672 + 2300, - wind: 23002, - }; - countries['FI'].data.capacity = { - hydro: 3080, - nuclear: 2860, - wind: 1000 - }; - countries['FR'].data.capacity = { - nuclear: 63130, - oil: 6670, - coal: 2930, - hydro: 10326 + 8204 + 4965, - gas: 6121, - wind: 10358, - solar: 6580 - }; - countries['GB'].data.capacity = { - wind: 13500, - nuclear: 9000, - hydro: 1550, - gas: 38000, - solar: 8780 - }; - countries['NO'].data.capacity = { - hydro: 31000, - nuclear: 0, - solar: 0, - wind: 856, - }; - countries['SE'].data.capacity = { - hydro: 16200, - nuclear: 8849, - solar: 79, - wind: 6025 - }; + // Load country configs + addCountryConfiguration(countries); // Populate with realtime production data d3.keys(production.data).forEach(function(countryCode) { diff --git a/api/static/index.html b/api/static/index.html index 5de8012661..695d15a9df 100644 --- a/api/static/index.html +++ b/api/static/index.html @@ -26,6 +26,8 @@ + + diff --git a/feeder/Dockerfile b/feeder/Dockerfile index f018294cea..3001d5d671 100644 --- a/feeder/Dockerfile +++ b/feeder/Dockerfile @@ -11,6 +11,8 @@ RUN apt-get install -y maven RUN apt-get install -y libgrib-api-dev && pip install numpy==1.10.1 pyproj==1.9.4 RUN git clone https://github.com/cambecc/grib2json && \ cd grib2json && mvn package && tar -xvf target/grib2json-0.8.0-SNAPSHOT.tar.gz +# Install g++ for Cython required by panda +RUN apt-get install -y g++ # Only add requirements to enable cached builds when it is unchanged ADD requirements.txt /home/requirements.txt RUN pip install -r requirements.txt diff --git a/feeder/feeder.py b/feeder/feeder.py index f2c31106f6..4a915ab9fb 100644 --- a/feeder/feeder.py +++ b/feeder/feeder.py @@ -8,6 +8,7 @@ from parsers.FI import fetch_FI from parsers.FR import fetch_FR from parsers.GB import fetch_GB +from parsers.HU import fetch_HU from parsers.LT import fetch_LT from parsers.LV import fetch_LV from parsers.NO import fetch_NO @@ -28,6 +29,7 @@ fetch_FI, fetch_FR, fetch_GB, + fetch_HU, fetch_LT, fetch_LV, fetch_NO, diff --git a/feeder/parsers/HU.py b/feeder/parsers/HU.py new file mode 100644 index 0000000000..7a0ecc2c3e --- /dev/null +++ b/feeder/parsers/HU.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- + +import arrow +import dateutil +import pandas as pd # In order to read excel +import requests + +COUNTRY_CODE = 'HU' +TIME_ZONE = 'Europe/Budapest' + +def fetch_HU(): + + now = arrow.now(TIME_ZONE) + + # Fetch production + url = ('http://rtdwweb.mavir.hu/webuser/ExportChartXlsIntervalServlet?' + + 'fromDateXls=%s&fromTimeXls=T00%%3A00%%3A00&' % now.format('YYYY-MM-DD') + + 'toDateXls=%s&toTimeXls=T00%%3A00%%3A00&' % now.replace(days=+1).format('YYYY-MM-DD') + + 'resoulutionInput=15&unit=min&outputType=XLS&selectedTabId=tab9405&submitXls=') + df_prod = pd.read_excel(url).dropna() # only keep non NaN value + # Remove points in the future + ix = map(lambda x: arrow.get(x, "YYYY.MM.DD HH:mm:ss").replace(tzinfo=dateutil.tz.gettz(TIME_ZONE)) < now, df_prod.dropna()[u'Időpont']) + df_prod = df_prod.iloc[ix].iloc[-1] # Get the last + + # Fetch + url = ('http://rtdwweb.mavir.hu/webuser/ExportChartXlsIntervalServlet?' + + 'fromDateXls=%s&fromTimeXls=T00%%3A00%%3A00&' % now.format('YYYY-MM-DD') + + 'toDateXls=%s&toTimeXls=T00%%3A00%%3A00&' % now.replace(days=+1).format('YYYY-MM-DD') + + 'resoulutionInput=15&unit=min&outputType=XLS&selectedTabId=tab5230&submitXls=') + df_exchange = pd.read_excel(url).dropna() # only keep non NaN value + # Remove points in the future + ix = map(lambda x: arrow.get(x, "YYYY.MM.DD HH:mm:ss").replace(tzinfo=dateutil.tz.gettz(TIME_ZONE)) < now, df_exchange.dropna()[u'Időpont']) + df_exchange = df_exchange.iloc[ix].iloc[-1] # Get the last + + obj = { + 'countryCode': COUNTRY_CODE, + 'datetime': arrow.get(df_prod[u'Időpont'], "YYYY.MM.DD HH:mm:ss").replace( + tzinfo=dateutil.tz.gettz(TIME_ZONE)).datetime + } + obj['consumption'] = { + } + obj['exchange'] = { + 'SK': float(df_exchange[u'HU-SK mérés']), + 'AT': float(df_exchange[u'HU-AT mérés']), + 'HR': float(df_exchange[u'HU-HR mérés']), + 'RO': float(df_exchange[u'HU-RO mérés']), + 'RS': float(df_exchange[u'HU-RS mérés']), + 'UA': float(df_exchange[u'HU-UK mérés']) + } + obj['production'] = { + 'biomass': float(df_prod[u'Biomassza erőművek net.mérés (15p)']) + float(df_prod[u'Szemétégető erőművek net.mérés (15p)']), + 'coal': float(df_prod[u'Barnakőszén-lignit erőművek net.mérés (15p)']) + float(df_prod[u'Feketekőszén erőművek net.mérés (15p)']), + 'gas': float(df_prod[u'Gáz (fosszilis) erőművek net.mérés (15p)']), + 'hydro': float(df_prod[u'Folyóvizes erőmvek net.mérés (15p)']) + float(df_prod[u'Víztározós vízerőművek net.mérés (15p)']), + 'oil': float(df_prod[u'Olaj (fosszilis) erőművek net.mérés (15p)']), + 'nuclear': float(df_prod[u'Nukleáris erőművek net.mérés (15p)']), + 'solar': float(df_prod[u'Naperőművek net.mérés (15p)']), + 'wind': float(df_prod[u'Szárazföldi szélerőművek net.mérés (15p)']) + } + + return obj + +if __name__ == '__main__': + print fetch_HU() diff --git a/feeder/requirements.txt b/feeder/requirements.txt index d9644fa32b..9ee6823cd1 100644 --- a/feeder/requirements.txt +++ b/feeder/requirements.txt @@ -1,6 +1,9 @@ arrow==0.5.0 +Cython==0.23.4 +pandas==0.16.2 pygrib==2.0.1 pymongo==3.2.2 requests==2.10.0 schedule==0.3.2 +xlrd==1.0.0 -e git+https://github.com/gaelenh/python-statsd-client.git#egg=statsd-client