From b43f864e04414f08919393b4b4fe61739777678e Mon Sep 17 00:00:00 2001 From: nikkozzblu Date: Wed, 18 Jan 2017 17:20:04 +0100 Subject: [PATCH 01/21] Fetch price to DB --- .gitignore | 1 + feeder/feeder.py | 24 ++++++++++++++++++++++++ feeder/parsers/FR.py | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+) diff --git a/.gitignore b/.gitignore index 72a93f094d..ba44dd4687 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ grib2json web/public/dist web/node_modules .ipynb_checkpoints/ +*.pyproj diff --git a/feeder/feeder.py b/feeder/feeder.py index ea080b9d46..3ebefa64ec 100644 --- a/feeder/feeder.py +++ b/feeder/feeder.py @@ -224,6 +224,11 @@ # 'SK->UA': ENTSOE.fetch_exchange, } +PRICE_PARSERS = { + 'RTE': FR.fetch_prices, +} + + # Set up stats import statsd statsd.init_statsd({ @@ -238,12 +243,14 @@ col_gfs = db['gfs'] col_production = db['production'] col_exchange = db['exchange'] +col_price = db['price'] # Set up indices col_consumption.create_index([('datetime', -1), ('countryCode', 1)], unique=True) col_gfs.create_index([('refTime', -1), ('targetTime', 1), ('key', 1)], unique=True) col_gfs.create_index([('refTime', -1), ('targetTime', -1), ('key', 1)], unique=True) col_production.create_index([('datetime', -1), ('countryCode', 1)], unique=True) col_exchange.create_index([('datetime', -1), ('sortedCountryCodes', 1)], unique=True) +col_price.create_index([('datetime', -1), ('countryCode', 1)], unique=True) # Set up memcached MEMCACHED_HOST = os.environ.get('MEMCACHED_HOST', None) @@ -343,6 +350,21 @@ def fetch_exchanges(): statsd.increment('fetch_one_exchange_error') logger.exception('Exception while fetching exchange of %s' % k) +def fetch_prices(): + for authority, parser in PRICE_PARSERS.iteritems(): + try: + with statsd.StatsdTimer('fetch_one_price'): + obj = parser(session) + if not obj: continue + # validate_price(obj) + # Database insert + for row in obj: + result = db_upsert(col_price, row, 'countryCode') + if (result.modified_count or result.upserted_id) and cache: cache.delete(MEMCACHED_STATE_KEY) + except: + statsd.increment('fetch_one_consumption_error') + logger.exception('Exception while fetching pricing of %s' % authority) + def db_upsert_forecast(col, obj, database_key): now = arrow.now().datetime query = { @@ -421,11 +443,13 @@ def fetch_weather(): schedule.every(INTERVAL_SECONDS).seconds.do(fetch_consumptions) schedule.every(INTERVAL_SECONDS).seconds.do(fetch_productions) schedule.every(INTERVAL_SECONDS).seconds.do(fetch_exchanges) +schedule.every(INTERVAL_SECONDS).seconds.do(fetch_prices) fetch_weather() fetch_consumptions() fetch_productions() fetch_exchanges() +fetch_prices() while True: schedule.run_pending() diff --git a/feeder/parsers/FR.py b/feeder/parsers/FR.py index dfb1cf7bef..ad20de985f 100644 --- a/feeder/parsers/FR.py +++ b/feeder/parsers/FR.py @@ -62,5 +62,37 @@ def fetch_production(country_code='FR', session=None): return data + +def fetch_prices(session=None,from_date=None,to_date=None): + r = session or requests.session() + formatted_from = from_date or arrow.now(tz='Europe/Paris').format('DD/MM/YYYY') + formatted_to = to_date or arrow.now(tz='Europe/Paris').replace(days=+1).format('DD/MM/YYYY') + url = 'http://www.rte-france.com/getEco2MixXml.php?type=donneesMarche&dateDeb={}&dateFin={}&mode=NORM'.format(formatted_from, formatted_to) + response = r.get(url) + obj = ET.fromstring(response.content) + mixtr = obj[5] + + data=[] + date_str = mixtr.get('date') + date = arrow.get(arrow.get(date_str).datetime, 'Europe/Paris') + for country_item in mixtr.getchildren(): + if country_item.get('granularite') != 'Global': continue + country_code=country_item.get('perimetre') + value = None + for value in country_item.getchildren(): + if(value.text=='ND'): continue + datarow = { + 'countryCode':{}, + 'datetime':{}, + 'price':{}, + 'source': 'rte-france.com' + } + datarow['countryCode'] = country_code + datarow['datetime'] = date.replace(hours=+int(value.attrib['periode'])).datetime + datarow['price'] = float(value.text) + data.append(datarow) + return data + + if __name__ == '__main__': print fetch_production() From 627457aa616dca6d0e32c4db48280f86575b515b Mon Sep 17 00:00:00 2001 From: nikkozzblu Date: Fri, 20 Jan 2017 18:54:26 +0100 Subject: [PATCH 02/21] Price from ENTSOE Update FR for consistency --- .gitignore | 2 ++ feeder/feeder.py | 45 +++++++++++++++++++++++++++++++++------- feeder/parsers/ENTSOE.py | 44 +++++++++++++++++++++++++++++++++++++++ feeder/parsers/FR.py | 30 +++++++++++++-------------- 4 files changed, 98 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index ba44dd4687..22c8cfd8f7 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ web/public/dist web/node_modules .ipynb_checkpoints/ *.pyproj +*.sln +*.suo diff --git a/feeder/feeder.py b/feeder/feeder.py index 3ebefa64ec..d5eab25458 100644 --- a/feeder/feeder.py +++ b/feeder/feeder.py @@ -225,10 +225,40 @@ } PRICE_PARSERS = { - 'RTE': FR.fetch_prices, + + 'FR': FR.fetch_prices, + 'AT': ENTSOE.fetch_prices, + 'BE': ENTSOE.fetch_prices, + 'BG': ENTSOE.fetch_prices, + 'CH': ENTSOE.fetch_prices, + 'CZ': ENTSOE.fetch_prices, + 'DE': ENTSOE.fetch_prices, + 'DK': ENTSOE.fetch_prices, + 'EE': ENTSOE.fetch_prices, + 'ES': ENTSOE.fetch_prices, + 'FI': ENTSOE.fetch_prices, + 'GB': ENTSOE.fetch_prices, + 'GB-NIR': ENTSOE.fetch_prices, + 'GR': ENTSOE.fetch_prices, + 'HU': ENTSOE.fetch_prices, + 'IE': ENTSOE.fetch_prices, + 'IT': ENTSOE.fetch_prices, + 'LT': ENTSOE.fetch_prices, + 'LU': ENTSOE.fetch_prices, + 'LV': ENTSOE.fetch_prices, + 'NL': ENTSOE.fetch_prices, + 'NO': ENTSOE.fetch_prices, + 'PL': ENTSOE.fetch_prices, + 'PT': ENTSOE.fetch_prices, + 'RO': ENTSOE.fetch_prices, + 'RS': ENTSOE.fetch_prices, + 'SE': ENTSOE.fetch_prices, + 'SI': ENTSOE.fetch_prices, + 'SK': ENTSOE.fetch_prices, } + # Set up stats import statsd statsd.init_statsd({ @@ -351,19 +381,18 @@ def fetch_exchanges(): logger.exception('Exception while fetching exchange of %s' % k) def fetch_prices(): - for authority, parser in PRICE_PARSERS.iteritems(): + for country_code, parser in PRICE_PARSERS.iteritems(): try: with statsd.StatsdTimer('fetch_one_price'): - obj = parser(session) + obj = parser(country_code,session) if not obj: continue # validate_price(obj) # Database insert - for row in obj: - result = db_upsert(col_price, row, 'countryCode') - if (result.modified_count or result.upserted_id) and cache: cache.delete(MEMCACHED_STATE_KEY) + result = db_upsert(col_price,obj, 'countryCode') + if (result.modified_count or result.upserted_id) and cache: cache.delete(MEMCACHED_STATE_KEY) except: - statsd.increment('fetch_one_consumption_error') - logger.exception('Exception while fetching pricing of %s' % authority) + statsd.increment('fetch_one_price_error') + logger.exception('Exception while fetching pricing of %s' % country_code) def db_upsert_forecast(col, obj, database_key): now = arrow.now().datetime diff --git a/feeder/parsers/ENTSOE.py b/feeder/parsers/ENTSOE.py index da75053a1f..5c134e00b4 100644 --- a/feeder/parsers/ENTSOE.py +++ b/feeder/parsers/ENTSOE.py @@ -120,6 +120,19 @@ def query_exchange(in_domain, out_domain, session): soup = BeautifulSoup(response.text, 'html.parser') raise Exception('Failed to get exchange. Reason: %s' % soup.find_all('text')[0].contents[0]) +def query_price(domain, session): + params = { + 'documentType': 'A44', + 'in_Domain': domain, + 'out_Domain': domain, + } + response = query_ENTSOE(session, params) + if response.ok: return response.text + else: + # Grab the error if possible + soup = BeautifulSoup(response.text, 'html.parser') + raise Exception('Failed to get price. Reason: %s' % soup.find_all('text')[0].contents[0]) + def datetime_from_position(start, position, resolution): m = re.search('PT(\d+)([M])', resolution) if m: @@ -192,6 +205,21 @@ def parse_exchange(xml_text, is_import, quantities=None, datetimes=None): datetimes.append(datetime) return quantities, datetimes +def parse_prices(xml_text): + if not xml_text: return None + soup = BeautifulSoup(xml_text, 'html.parser') + # Get all points + prices = [] + datetimes = [] + for timeseries in soup.find_all('timeseries'): + resolution = timeseries.find_all('resolution')[0].contents[0] + datetime_start = arrow.get(timeseries.find_all('start')[0].contents[0]) + for entry in timeseries.find_all('point'): + prices.append(float(entry.find_all('price.amount')[0].contents[0])) + position = int(entry.find_all('position')[0].contents[0]) + datetimes.append(datetime_from_position(datetime_start, position, resolution)) + return prices, datetimes + def get_biomass(values): if 'Biomass' in values or 'Fossil Peat' in values or 'Waste' in values: return values.get('Biomass', 0) + \ @@ -330,3 +358,19 @@ def fetch_exchange(country_code1, country_code2, session=None): 'netFlow': netFlow if country_code1[0] == sorted_country_codes else -1 * netFlow, 'source': 'entsoe.eu' } + +def fetch_prices(country_code, session=None): + if not session: session = requests.session() + domain = ENTSOE_DOMAIN_MAPPINGS[country_code] + # Grab consumption + parsed = parse_prices(query_price(domain, session)) + if parsed: + prices, datetimes = parsed + data = { + 'countryCode': country_code, + 'datetime': datetimes[-1].datetime, + 'price': prices[-1], + 'source': 'entsoe.eu' + } + + return data diff --git a/feeder/parsers/FR.py b/feeder/parsers/FR.py index ad20de985f..774310c64a 100644 --- a/feeder/parsers/FR.py +++ b/feeder/parsers/FR.py @@ -62,8 +62,7 @@ def fetch_production(country_code='FR', session=None): return data - -def fetch_prices(session=None,from_date=None,to_date=None): +def fetch_prices(country_code,session=None,from_date=None,to_date=None): r = session or requests.session() formatted_from = from_date or arrow.now(tz='Europe/Paris').format('DD/MM/YYYY') formatted_to = to_date or arrow.now(tz='Europe/Paris').replace(days=+1).format('DD/MM/YYYY') @@ -72,27 +71,28 @@ def fetch_prices(session=None,from_date=None,to_date=None): obj = ET.fromstring(response.content) mixtr = obj[5] - data=[] + prices = [] + datetimes = [] + date_str = mixtr.get('date') date = arrow.get(arrow.get(date_str).datetime, 'Europe/Paris') for country_item in mixtr.getchildren(): if country_item.get('granularite') != 'Global': continue - country_code=country_item.get('perimetre') + country_c=country_item.get('perimetre') + if(country_code!=country_c): continue value = None for value in country_item.getchildren(): if(value.text=='ND'): continue - datarow = { - 'countryCode':{}, - 'datetime':{}, - 'price':{}, - 'source': 'rte-france.com' - } - datarow['countryCode'] = country_code - datarow['datetime'] = date.replace(hours=+int(value.attrib['periode'])).datetime - datarow['price'] = float(value.text) - data.append(datarow) + datetimes.append(date.replace(hours=+int(value.attrib['periode'])).datetime) + prices.append(float(value.text)) + + data = { + 'countryCode': country_code, + 'datetime': datetimes[-1], + 'price': prices[-1], + 'source': 'rte-france.com', + } return data - if __name__ == '__main__': print fetch_production() From fc4430b5a56d4735b6ad7e9397750a3c47d8682d Mon Sep 17 00:00:00 2001 From: nikkozzblu Date: Sat, 21 Jan 2017 16:46:41 +0100 Subject: [PATCH 03/21] Styling and not importing future prices --- feeder/feeder.py | 68 +++++++++++++++++++--------------------- feeder/parsers/ENTSOE.py | 10 +++--- feeder/parsers/FR.py | 18 ++++++----- 3 files changed, 49 insertions(+), 47 deletions(-) diff --git a/feeder/feeder.py b/feeder/feeder.py index d5eab25458..2321a9f9ee 100644 --- a/feeder/feeder.py +++ b/feeder/feeder.py @@ -225,40 +225,37 @@ } PRICE_PARSERS = { - - 'FR': FR.fetch_prices, - 'AT': ENTSOE.fetch_prices, - 'BE': ENTSOE.fetch_prices, - 'BG': ENTSOE.fetch_prices, - 'CH': ENTSOE.fetch_prices, - 'CZ': ENTSOE.fetch_prices, - 'DE': ENTSOE.fetch_prices, - 'DK': ENTSOE.fetch_prices, - 'EE': ENTSOE.fetch_prices, - 'ES': ENTSOE.fetch_prices, - 'FI': ENTSOE.fetch_prices, - 'GB': ENTSOE.fetch_prices, - 'GB-NIR': ENTSOE.fetch_prices, - 'GR': ENTSOE.fetch_prices, - 'HU': ENTSOE.fetch_prices, - 'IE': ENTSOE.fetch_prices, - 'IT': ENTSOE.fetch_prices, - 'LT': ENTSOE.fetch_prices, - 'LU': ENTSOE.fetch_prices, - 'LV': ENTSOE.fetch_prices, - 'NL': ENTSOE.fetch_prices, - 'NO': ENTSOE.fetch_prices, - 'PL': ENTSOE.fetch_prices, - 'PT': ENTSOE.fetch_prices, - 'RO': ENTSOE.fetch_prices, - 'RS': ENTSOE.fetch_prices, - 'SE': ENTSOE.fetch_prices, - 'SI': ENTSOE.fetch_prices, - 'SK': ENTSOE.fetch_prices, + 'AT': ENTSOE.fetch_price, + 'BE': ENTSOE.fetch_price, + 'BG': ENTSOE.fetch_price, + 'CH': ENTSOE.fetch_price, + 'CZ': ENTSOE.fetch_price, + 'DE': ENTSOE.fetch_price, + 'DK': ENTSOE.fetch_price, + 'EE': ENTSOE.fetch_price, + 'ES': ENTSOE.fetch_price, + 'FR': FR.fetch_price, + 'FI': ENTSOE.fetch_price, + 'GB': ENTSOE.fetch_price, + 'GB-NIR': ENTSOE.fetch_price, + 'GR': ENTSOE.fetch_price, + 'HU': ENTSOE.fetch_price, + 'IE': ENTSOE.fetch_price, + 'IT': ENTSOE.fetch_price, + 'LT': ENTSOE.fetch_price, + 'LU': ENTSOE.fetch_price, + 'LV': ENTSOE.fetch_price, + 'NL': ENTSOE.fetch_price, + 'NO': ENTSOE.fetch_price, + 'PL': ENTSOE.fetch_price, + 'PT': ENTSOE.fetch_price, + 'RO': ENTSOE.fetch_price, + 'RS': ENTSOE.fetch_price, + 'SE': ENTSOE.fetch_price, + 'SI': ENTSOE.fetch_price, + 'SK': ENTSOE.fetch_price, } - - # Set up stats import statsd statsd.init_statsd({ @@ -380,13 +377,12 @@ def fetch_exchanges(): statsd.increment('fetch_one_exchange_error') logger.exception('Exception while fetching exchange of %s' % k) -def fetch_prices(): +def fetch_price(): for country_code, parser in PRICE_PARSERS.iteritems(): try: with statsd.StatsdTimer('fetch_one_price'): obj = parser(country_code,session) if not obj: continue - # validate_price(obj) # Database insert result = db_upsert(col_price,obj, 'countryCode') if (result.modified_count or result.upserted_id) and cache: cache.delete(MEMCACHED_STATE_KEY) @@ -472,13 +468,13 @@ def fetch_weather(): schedule.every(INTERVAL_SECONDS).seconds.do(fetch_consumptions) schedule.every(INTERVAL_SECONDS).seconds.do(fetch_productions) schedule.every(INTERVAL_SECONDS).seconds.do(fetch_exchanges) -schedule.every(INTERVAL_SECONDS).seconds.do(fetch_prices) +schedule.every(INTERVAL_SECONDS).seconds.do(fetch_price) fetch_weather() fetch_consumptions() fetch_productions() fetch_exchanges() -fetch_prices() +fetch_price() while True: schedule.run_pending() diff --git a/feeder/parsers/ENTSOE.py b/feeder/parsers/ENTSOE.py index 5c134e00b4..23dc6863d2 100644 --- a/feeder/parsers/ENTSOE.py +++ b/feeder/parsers/ENTSOE.py @@ -205,7 +205,7 @@ def parse_exchange(xml_text, is_import, quantities=None, datetimes=None): datetimes.append(datetime) return quantities, datetimes -def parse_prices(xml_text): +def parse_price(xml_text): if not xml_text: return None soup = BeautifulSoup(xml_text, 'html.parser') # Get all points @@ -215,9 +215,11 @@ def parse_prices(xml_text): resolution = timeseries.find_all('resolution')[0].contents[0] datetime_start = arrow.get(timeseries.find_all('start')[0].contents[0]) for entry in timeseries.find_all('point'): + datetime=datetime_from_position(datetime_start, position, resolution) + if datetime > arrow.now(tz='Europe/Paris'): continue prices.append(float(entry.find_all('price.amount')[0].contents[0])) position = int(entry.find_all('position')[0].contents[0]) - datetimes.append(datetime_from_position(datetime_start, position, resolution)) + datetimes.append(datetime) return prices, datetimes def get_biomass(values): @@ -359,11 +361,11 @@ def fetch_exchange(country_code1, country_code2, session=None): 'source': 'entsoe.eu' } -def fetch_prices(country_code, session=None): +def fetch_price(country_code, session=None): if not session: session = requests.session() domain = ENTSOE_DOMAIN_MAPPINGS[country_code] # Grab consumption - parsed = parse_prices(query_price(domain, session)) + parsed = parse_price(query_price(domain, session)) if parsed: prices, datetimes = parsed data = { diff --git a/feeder/parsers/FR.py b/feeder/parsers/FR.py index 774310c64a..bd85899029 100644 --- a/feeder/parsers/FR.py +++ b/feeder/parsers/FR.py @@ -62,10 +62,12 @@ def fetch_production(country_code='FR', session=None): return data -def fetch_prices(country_code,session=None,from_date=None,to_date=None): +def fetch_price(country_code, session=None, from_date=None, to_date=None): r = session or requests.session() - formatted_from = from_date or arrow.now(tz='Europe/Paris').format('DD/MM/YYYY') - formatted_to = to_date or arrow.now(tz='Europe/Paris').replace(days=+1).format('DD/MM/YYYY') + dt_now=arrow.now(tz='Europe/Paris') + formatted_from = from_date or dt_now.replace(days=-1).format('DD/MM/YYYY') + formatted_to = to_date or dt_now.format('DD/MM/YYYY') + url = 'http://www.rte-france.com/getEco2MixXml.php?type=donneesMarche&dateDeb={}&dateFin={}&mode=NORM'.format(formatted_from, formatted_to) response = r.get(url) obj = ET.fromstring(response.content) @@ -76,14 +78,16 @@ def fetch_prices(country_code,session=None,from_date=None,to_date=None): date_str = mixtr.get('date') date = arrow.get(arrow.get(date_str).datetime, 'Europe/Paris') - for country_item in mixtr.getchildren(): + for country_item in mixtr.getchildren(): if country_item.get('granularite') != 'Global': continue country_c=country_item.get('perimetre') - if(country_code!=country_c): continue + if country_code != country_c: continue value = None for value in country_item.getchildren(): - if(value.text=='ND'): continue - datetimes.append(date.replace(hours=+int(value.attrib['periode'])).datetime) + if value.text == 'ND': continue + datetime=date.replace(hours=+int(value.attrib['periode'])).datetime + if datetime > dt_now: continue + datetimes.append(datetime) prices.append(float(value.text)) data = { From 16b853fc2a68bab0bfcc1bf806493f3a073d2ac9 Mon Sep 17 00:00:00 2001 From: nikkozzblu Date: Sat, 21 Jan 2017 18:50:55 +0100 Subject: [PATCH 04/21] Last point if no date specify --- feeder/parsers/FR.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/feeder/parsers/FR.py b/feeder/parsers/FR.py index bd85899029..12a9f82549 100644 --- a/feeder/parsers/FR.py +++ b/feeder/parsers/FR.py @@ -64,8 +64,8 @@ def fetch_production(country_code='FR', session=None): def fetch_price(country_code, session=None, from_date=None, to_date=None): r = session or requests.session() - dt_now=arrow.now(tz='Europe/Paris') - formatted_from = from_date or dt_now.replace(days=-1).format('DD/MM/YYYY') + dt_now = arrow.now(tz='Europe/Paris') + formatted_from = from_date or dt_now.format('DD/MM/YYYY') formatted_to = to_date or dt_now.format('DD/MM/YYYY') url = 'http://www.rte-france.com/getEco2MixXml.php?type=donneesMarche&dateDeb={}&dateFin={}&mode=NORM'.format(formatted_from, formatted_to) From e1ede1072db23943590ce4864fb0c8d2ce2786f9 Mon Sep 17 00:00:00 2001 From: nikkozzblu Date: Sat, 21 Jan 2017 18:54:48 +0100 Subject: [PATCH 05/21] BUG position --- feeder/parsers/ENTSOE.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feeder/parsers/ENTSOE.py b/feeder/parsers/ENTSOE.py index 23dc6863d2..3df20515fc 100644 --- a/feeder/parsers/ENTSOE.py +++ b/feeder/parsers/ENTSOE.py @@ -215,10 +215,10 @@ def parse_price(xml_text): resolution = timeseries.find_all('resolution')[0].contents[0] datetime_start = arrow.get(timeseries.find_all('start')[0].contents[0]) for entry in timeseries.find_all('point'): + position = int(entry.find_all('position')[0].contents[0]) datetime=datetime_from_position(datetime_start, position, resolution) if datetime > arrow.now(tz='Europe/Paris'): continue prices.append(float(entry.find_all('price.amount')[0].contents[0])) - position = int(entry.find_all('position')[0].contents[0]) datetimes.append(datetime) return prices, datetimes From c9445737eb00e0e06dec3534cdebc88b9d3650ab Mon Sep 17 00:00:00 2001 From: nikkozzblu Date: Sun, 22 Jan 2017 09:19:43 +0100 Subject: [PATCH 06/21] alphaorder --- feeder/feeder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feeder/feeder.py b/feeder/feeder.py index 2321a9f9ee..0147c1e786 100644 --- a/feeder/feeder.py +++ b/feeder/feeder.py @@ -234,8 +234,8 @@ 'DK': ENTSOE.fetch_price, 'EE': ENTSOE.fetch_price, 'ES': ENTSOE.fetch_price, - 'FR': FR.fetch_price, 'FI': ENTSOE.fetch_price, + 'FR': FR.fetch_price, 'GB': ENTSOE.fetch_price, 'GB-NIR': ENTSOE.fetch_price, 'GR': ENTSOE.fetch_price, From df7610505b319a2add08cc7ec6611e882679940d Mon Sep 17 00:00:00 2001 From: nikkozzblu Date: Tue, 24 Jan 2017 16:36:13 +0100 Subject: [PATCH 07/21] Solar speed --- .gitignore | 2 + web/app/solar.js | 191 +++++++++++++++++++++++++++++++---------------- 2 files changed, 127 insertions(+), 66 deletions(-) diff --git a/.gitignore b/.gitignore index 22c8cfd8f7..a4be10a024 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ web/node_modules *.pyproj *.sln *.suo +*.user +/.vs/config diff --git a/web/app/solar.js b/web/app/solar.js index 6f887702fa..860cb3fc85 100644 --- a/web/app/solar.js +++ b/web/app/solar.js @@ -8,20 +8,28 @@ var grib = require('./grib'); var solarCanvas; function bilinearInterpolate(x, y, x1, x2, y1, y2, Q11, Q12, Q21, Q22) { - var R1 = ((x2 - x)/(x2 - x1))*Q11 + ((x - x1)/(x2 - x1))*Q21; - var R2 = ((x2 - x)/(x2 - x1))*Q12 + ((x - x1)/(x2 - x1))*Q22; - return ((y2 - y)/(y2 - y1))*R1 + ((y - y1)/(y2 - y1))*R2; + var R1 = ((x2 - x) / (x2 - x1)) * Q11 + ((x - x1) / (x2 - x1)) * Q21; + var R2 = ((x2 - x) / (x2 - x1)) * Q12 + ((x - x1) / (x2 - x1)) * Q22; + return ((y2 - y) / (y2 - y1)) * R1 + ((y - y1) / (y2 - y1)) * R2; } -exports.isExpired = function(now, grib1, grib2) { +// This is a minified library for image bluring, should be moved to exported Filter in file image_blur, not sure how to properly do this +var mul_table = [512, 512, 456, 512, 328, 456, 335, 512, 405, 328, 271, 456, 388, 335, 292, 512, 454, 405, 364, 328, 298, 271, 496, 456, 420, 388, 360, 335, 312, 292, 273, 512, 482, 454, 428, 405, 383, 364, 345, 328, 312, 298, 284, 271, 259, 496, 475, 456, 437, 420, 404, 388, 374, 360, 347, 335, 323, 312, 302, 292, 282, 273, 265, 512, 497, 482, 468, 454, 441, 428, 417, 405, 394, 383, 373, 364, 354, 345, 337, 328, 320, 312, 305, 298, 291, 284, 278, 271, 265, 259, 507, 496, 485, 475, 465, 456, 446, 437, 428, 420, 412, 404, 396, 388, 381, 374, 367, 360, 354, 347, 341, 335, 329, 323, 318, 312, 307, 302, 297, 292, 287, 282, 278, 273, 269, 265, 261, 512, 505, 497, 489, 482, 475, 468, 461, 454, 447, 441, 435, 428, 422, 417, 411, 405, 399, 394, 389, 383, 378, 373, 368, 364, 359, 354, 350, 345, 341, 337, 332, 328, 324, 320, 316, 312, 309, 305, 301, 298, 294, 291, 287, 284, 281, 278, 274, 271, 268, 265, 262, 259, 257, 507, 501, 496, 491, 485, 480, 475, 470, 465, 460, 456, 451, 446, 442, 437, 433, 428, 424, 420, 416, 412, 408, 404, 400, 396, 392, 388, 385, 381, 377, 374, 370, 367, 363, 360, 357, 354, 350, 347, 344, 341, 338, 335, 332, 329, 326, 323, 320, 318, 315, 312, 310, 307, 304, 302, 299, 297, 294, 292, 289, 287, 285, 282, 280, 278, 275, 273, 271, 269, 267, 265, 263, 261, 259]; var shg_table = [9, 11, 12, 13, 13, 14, 14, 15, 15, 15, 15, 16, 16, 16, 16, 17, 17, 17, 17, 17, 17, 17, 18, 18, 18, 18, 18, 18, 18, 18, 18, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24]; function stackBlurImage(imageID, canvasID, radius, blurAlphaChannel) { var img = document.getElementById(imageID); var w = img.naturalWidth; var h = img.naturalHeight; var canvas = document.getElementById(canvasID); canvas.style.width = w + "px"; canvas.style.height = h + "px"; canvas.width = w; canvas.height = h; var context = canvas.getContext("2d"); context.clearRect(0, 0, w, h); context.drawImage(img, 0, 0); if (isNaN(radius) || radius < 1) return; if (blurAlphaChannel) stackBlurCanvasRGBA(canvasID, 0, 0, w, h, radius); else stackBlurCanvasRGB(canvasID, 0, 0, w, h, radius) } function stackBlurCanvasRGBA(id, top_x, top_y, width, height, radius) { if (isNaN(radius) || radius < 1) return; radius |= 0; var canvas = document.getElementById(id); var context = canvas.getContext("2d"); var imageData; try { try { imageData = context.getImageData(top_x, top_y, width, height) } catch (e) { try { netscape.security.PrivilegeManager.enablePrivilege("UniversalBrowserRead"); imageData = context.getImageData(top_x, top_y, width, height) } catch (e) { alert("Cannot access local image"); throw new Error("unable to access local image data: " + e); return } } } catch (e) { alert("Cannot access image"); throw new Error("unable to access image data: " + e) } var pixels = imageData.data; var x, y, i, p, yp, yi, yw, r_sum, g_sum, b_sum, a_sum, r_out_sum, g_out_sum, b_out_sum, a_out_sum, r_in_sum, g_in_sum, b_in_sum, a_in_sum, pr, pg, pb, pa, rbs; var div = radius + radius + 1; var w4 = width << 2; var widthMinus1 = width - 1; var heightMinus1 = height - 1; var radiusPlus1 = radius + 1; var sumFactor = radiusPlus1 * (radiusPlus1 + 1) / 2; var stackStart = new BlurStack; var stack = stackStart; for (i = 1; i < div; i++) { stack = stack.next = new BlurStack; if (i == radiusPlus1) var stackEnd = stack } stack.next = stackStart; var stackIn = null; var stackOut = null; yw = yi = 0; var mul_sum = mul_table[radius]; var shg_sum = shg_table[radius]; for (y = 0; y < height; y++) { r_in_sum = g_in_sum = b_in_sum = a_in_sum = r_sum = g_sum = b_sum = a_sum = 0; r_out_sum = radiusPlus1 * (pr = pixels[yi]); g_out_sum = radiusPlus1 * (pg = pixels[yi + 1]); b_out_sum = radiusPlus1 * (pb = pixels[yi + 2]); a_out_sum = radiusPlus1 * (pa = pixels[yi + 3]); r_sum += sumFactor * pr; g_sum += sumFactor * pg; b_sum += sumFactor * pb; a_sum += sumFactor * pa; stack = stackStart; for (i = 0; i < radiusPlus1; i++) { stack.r = pr; stack.g = pg; stack.b = pb; stack.a = pa; stack = stack.next } for (i = 1; i < radiusPlus1; i++) { p = yi + ((widthMinus1 < i ? widthMinus1 : i) << 2); r_sum += (stack.r = pr = pixels[p]) * (rbs = radiusPlus1 - i); g_sum += (stack.g = pg = pixels[p + 1]) * rbs; b_sum += (stack.b = pb = pixels[p + 2]) * rbs; a_sum += (stack.a = pa = pixels[p + 3]) * rbs; r_in_sum += pr; g_in_sum += pg; b_in_sum += pb; a_in_sum += pa; stack = stack.next } stackIn = stackStart; stackOut = stackEnd; for (x = 0; x < width; x++) { pixels[yi + 3] = pa = a_sum * mul_sum >> shg_sum; if (pa != 0) { pa = 255 / pa; pixels[yi] = (r_sum * mul_sum >> shg_sum) * pa; pixels[yi + 1] = (g_sum * mul_sum >> shg_sum) * pa; pixels[yi + 2] = (b_sum * mul_sum >> shg_sum) * pa } else { pixels[yi] = pixels[yi + 1] = pixels[yi + 2] = 0 } r_sum -= r_out_sum; g_sum -= g_out_sum; b_sum -= b_out_sum; a_sum -= a_out_sum; r_out_sum -= stackIn.r; g_out_sum -= stackIn.g; b_out_sum -= stackIn.b; a_out_sum -= stackIn.a; p = yw + ((p = x + radius + 1) < widthMinus1 ? p : widthMinus1) << 2; r_in_sum += stackIn.r = pixels[p]; g_in_sum += stackIn.g = pixels[p + 1]; b_in_sum += stackIn.b = pixels[p + 2]; a_in_sum += stackIn.a = pixels[p + 3]; r_sum += r_in_sum; g_sum += g_in_sum; b_sum += b_in_sum; a_sum += a_in_sum; stackIn = stackIn.next; r_out_sum += pr = stackOut.r; g_out_sum += pg = stackOut.g; b_out_sum += pb = stackOut.b; a_out_sum += pa = stackOut.a; r_in_sum -= pr; g_in_sum -= pg; b_in_sum -= pb; a_in_sum -= pa; stackOut = stackOut.next; yi += 4 } yw += width } for (x = 0; x < width; x++) { g_in_sum = b_in_sum = a_in_sum = r_in_sum = g_sum = b_sum = a_sum = r_sum = 0; yi = x << 2; r_out_sum = radiusPlus1 * (pr = pixels[yi]); g_out_sum = radiusPlus1 * (pg = pixels[yi + 1]); b_out_sum = radiusPlus1 * (pb = pixels[yi + 2]); a_out_sum = radiusPlus1 * (pa = pixels[yi + 3]); r_sum += sumFactor * pr; g_sum += sumFactor * pg; b_sum += sumFactor * pb; a_sum += sumFactor * pa; stack = stackStart; for (i = 0; i < radiusPlus1; i++) { stack.r = pr; stack.g = pg; stack.b = pb; stack.a = pa; stack = stack.next } yp = width; for (i = 1; i <= radius; i++) { yi = yp + x << 2; r_sum += (stack.r = pr = pixels[yi]) * (rbs = radiusPlus1 - i); g_sum += (stack.g = pg = pixels[yi + 1]) * rbs; b_sum += (stack.b = pb = pixels[yi + 2]) * rbs; a_sum += (stack.a = pa = pixels[yi + 3]) * rbs; r_in_sum += pr; g_in_sum += pg; b_in_sum += pb; a_in_sum += pa; stack = stack.next; if (i < heightMinus1) { yp += width } } yi = x; stackIn = stackStart; stackOut = stackEnd; for (y = 0; y < height; y++) { p = yi << 2; pixels[p + 3] = pa = a_sum * mul_sum >> shg_sum; if (pa > 0) { pa = 255 / pa; pixels[p] = (r_sum * mul_sum >> shg_sum) * pa; pixels[p + 1] = (g_sum * mul_sum >> shg_sum) * pa; pixels[p + 2] = (b_sum * mul_sum >> shg_sum) * pa } else { pixels[p] = pixels[p + 1] = pixels[p + 2] = 0 } r_sum -= r_out_sum; g_sum -= g_out_sum; b_sum -= b_out_sum; a_sum -= a_out_sum; r_out_sum -= stackIn.r; g_out_sum -= stackIn.g; b_out_sum -= stackIn.b; a_out_sum -= stackIn.a; p = x + ((p = y + radiusPlus1) < heightMinus1 ? p : heightMinus1) * width << 2; r_sum += r_in_sum += stackIn.r = pixels[p]; g_sum += g_in_sum += stackIn.g = pixels[p + 1]; b_sum += b_in_sum += stackIn.b = pixels[p + 2]; a_sum += a_in_sum += stackIn.a = pixels[p + 3]; stackIn = stackIn.next; r_out_sum += pr = stackOut.r; g_out_sum += pg = stackOut.g; b_out_sum += pb = stackOut.b; a_out_sum += pa = stackOut.a; r_in_sum -= pr; g_in_sum -= pg; b_in_sum -= pb; a_in_sum -= pa; stackOut = stackOut.next; yi += width } } context.putImageData(imageData, top_x, top_y) } function stackBlurCanvasRGB(id, top_x, top_y, width, height, radius) { if (isNaN(radius) || radius < 1) return; radius |= 0; var canvas = document.getElementById(id); var context = canvas.getContext("2d"); var imageData; try { try { imageData = context.getImageData(top_x, top_y, width, height) } catch (e) { try { netscape.security.PrivilegeManager.enablePrivilege("UniversalBrowserRead"); imageData = context.getImageData(top_x, top_y, width, height) } catch (e) { alert("Cannot access local image"); throw new Error("unable to access local image data: " + e); return } } } catch (e) { alert("Cannot access image"); throw new Error("unable to access image data: " + e) } var pixels = imageData.data; var x, y, i, p, yp, yi, yw, r_sum, g_sum, b_sum, r_out_sum, g_out_sum, b_out_sum, r_in_sum, g_in_sum, b_in_sum, pr, pg, pb, rbs; var div = radius + radius + 1; var w4 = width << 2; var widthMinus1 = width - 1; var heightMinus1 = height - 1; var radiusPlus1 = radius + 1; var sumFactor = radiusPlus1 * (radiusPlus1 + 1) / 2; var stackStart = new BlurStack; var stack = stackStart; for (i = 1; i < div; i++) { stack = stack.next = new BlurStack; if (i == radiusPlus1) var stackEnd = stack } stack.next = stackStart; var stackIn = null; var stackOut = null; yw = yi = 0; var mul_sum = mul_table[radius]; var shg_sum = shg_table[radius]; for (y = 0; y < height; y++) { r_in_sum = g_in_sum = b_in_sum = r_sum = g_sum = b_sum = 0; r_out_sum = radiusPlus1 * (pr = pixels[yi]); g_out_sum = radiusPlus1 * (pg = pixels[yi + 1]); b_out_sum = radiusPlus1 * (pb = pixels[yi + 2]); r_sum += sumFactor * pr; g_sum += sumFactor * pg; b_sum += sumFactor * pb; stack = stackStart; for (i = 0; i < radiusPlus1; i++) { stack.r = pr; stack.g = pg; stack.b = pb; stack = stack.next } for (i = 1; i < radiusPlus1; i++) { p = yi + ((widthMinus1 < i ? widthMinus1 : i) << 2); r_sum += (stack.r = pr = pixels[p]) * (rbs = radiusPlus1 - i); g_sum += (stack.g = pg = pixels[p + 1]) * rbs; b_sum += (stack.b = pb = pixels[p + 2]) * rbs; r_in_sum += pr; g_in_sum += pg; b_in_sum += pb; stack = stack.next } stackIn = stackStart; stackOut = stackEnd; for (x = 0; x < width; x++) { pixels[yi] = r_sum * mul_sum >> shg_sum; pixels[yi + 1] = g_sum * mul_sum >> shg_sum; pixels[yi + 2] = b_sum * mul_sum >> shg_sum; r_sum -= r_out_sum; g_sum -= g_out_sum; b_sum -= b_out_sum; r_out_sum -= stackIn.r; g_out_sum -= stackIn.g; b_out_sum -= stackIn.b; p = yw + ((p = x + radius + 1) < widthMinus1 ? p : widthMinus1) << 2; r_in_sum += stackIn.r = pixels[p]; g_in_sum += stackIn.g = pixels[p + 1]; b_in_sum += stackIn.b = pixels[p + 2]; r_sum += r_in_sum; g_sum += g_in_sum; b_sum += b_in_sum; stackIn = stackIn.next; r_out_sum += pr = stackOut.r; g_out_sum += pg = stackOut.g; b_out_sum += pb = stackOut.b; r_in_sum -= pr; g_in_sum -= pg; b_in_sum -= pb; stackOut = stackOut.next; yi += 4 } yw += width } for (x = 0; x < width; x++) { g_in_sum = b_in_sum = r_in_sum = g_sum = b_sum = r_sum = 0; yi = x << 2; r_out_sum = radiusPlus1 * (pr = pixels[yi]); g_out_sum = radiusPlus1 * (pg = pixels[yi + 1]); b_out_sum = radiusPlus1 * (pb = pixels[yi + 2]); r_sum += sumFactor * pr; g_sum += sumFactor * pg; b_sum += sumFactor * pb; stack = stackStart; for (i = 0; i < radiusPlus1; i++) { stack.r = pr; stack.g = pg; stack.b = pb; stack = stack.next } yp = width; for (i = 1; i <= radius; i++) { yi = yp + x << 2; r_sum += (stack.r = pr = pixels[yi]) * (rbs = radiusPlus1 - i); g_sum += (stack.g = pg = pixels[yi + 1]) * rbs; b_sum += (stack.b = pb = pixels[yi + 2]) * rbs; r_in_sum += pr; g_in_sum += pg; b_in_sum += pb; stack = stack.next; if (i < heightMinus1) { yp += width } } yi = x; stackIn = stackStart; stackOut = stackEnd; for (y = 0; y < height; y++) { p = yi << 2; pixels[p] = r_sum * mul_sum >> shg_sum; pixels[p + 1] = g_sum * mul_sum >> shg_sum; pixels[p + 2] = b_sum * mul_sum >> shg_sum; r_sum -= r_out_sum; g_sum -= g_out_sum; b_sum -= b_out_sum; r_out_sum -= stackIn.r; g_out_sum -= stackIn.g; b_out_sum -= stackIn.b; p = x + ((p = y + radiusPlus1) < heightMinus1 ? p : heightMinus1) * width << 2; r_sum += r_in_sum += stackIn.r = pixels[p]; g_sum += g_in_sum += stackIn.g = pixels[p + 1]; b_sum += b_in_sum += stackIn.b = pixels[p + 2]; stackIn = stackIn.next; r_out_sum += pr = stackOut.r; g_out_sum += pg = stackOut.g; b_out_sum += pb = stackOut.b; r_in_sum -= pr; g_in_sum -= pg; b_in_sum -= pb; stackOut = stackOut.next; yi += width } } context.putImageData(imageData, top_x, top_y) } function BlurStack() { this.r = 0; this.g = 0; this.b = 0; this.a = 0; this.next = null } + +exports.isExpired = function (now, grib1, grib2) { return grib.getTargetTime(grib2) <= moment(now) || grib.getTargetTime(grib1) > moment(now); } -exports.draw = function(canvasSelector, now, grib1, grib2, solarColor, projection, callback) { - // Interpolates between two solar forecasts +exports.draw = function (canvasSelector, now, grib1, grib2, solarColor, projection, callback) { + + // Control the rendering + var gaussianBlur = true; + var levels = false; + + // Interpolates between two solar forecasts< var t_before = grib.getTargetTime(grib1); var t_after = grib.getTargetTime(grib2); - console.log('#1 solar forecast target', + console.log('#1 solar forecast target', moment(t_before).fromNow(), 'made', moment(grib1.header.refTime).fromNow()); console.log('#2 solar forecast target', @@ -30,14 +38,19 @@ exports.draw = function(canvasSelector, now, grib1, grib2, solarColor, projectio if (moment(now) > moment(t_after)) { return console.error('Error while interpolating solar because current time is out of bounds'); } - var k = (now - t_before)/(t_after - t_before); + + + var k = (now - t_before) / (t_after - t_before); var buckets = d3.range(solarColor.range().length) - .map(function(d) { return []; }); + .map(function (d) { return []; }); var bucketIndex = d3.scaleLinear() .rangeRound(d3.range(buckets.length)) .domain(solarColor.domain()) .clamp(true); - var k = (now - t_before)/(t_after - t_before); + var solarScale = solarColor.domain()[solarColor.domain().length - 1]; + + var k = (now - t_before) / (t_after - t_before); + if (!k || !isFinite(k)) k = 0; var Nx = grib1.header.nx; var Ny = grib1.header.ny; @@ -46,7 +59,9 @@ exports.draw = function(canvasSelector, now, grib1, grib2, solarColor, projectio var dx = grib1.header.dx; var dy = grib1.header.dy; - var alphas = solarColor.range().map(function(d) { + var BLUR_RADIUS = 20; + + var alphas = solarColor.range().map(function (d) { return parseFloat(d .replace('(', '') .replace(')', '') @@ -56,72 +71,116 @@ exports.draw = function(canvasSelector, now, grib1, grib2, solarColor, projectio solarCanvas = d3.select(canvasSelector); var ctx = solarCanvas.node().getContext('2d'); - realW = solarCanvas.node().getBoundingClientRect().width, - realH = solarCanvas.node().getBoundingClientRect().height; + realW = solarCanvas.node().getBoundingClientRect().width, + realH = solarCanvas.node().getBoundingClientRect().height; var w = realW, h = realH; - var scaleX = realW / w, - scaleY = realH / h; - function canvasInvertedProjection(arr) { - return projection.invert([arr[0] * scaleX, arr[1] * scaleY]); - } - var img = ctx.createImageData(w, h); - - var xrange = d3.range(w); - function batchDrawColumns(x, batchsize, callback) { - var batch = d3.range(Math.min(batchsize, xrange[xrange.length - 1] - x)) - .map(function(d) { return d + x; }); - console.log('Drawing solar', x, '/', xrange[xrange.length - 1]); - batch.forEach(function(x) { - d3.range(h).forEach(function(y) { - var lonlat = canvasInvertedProjection([x, y]); - var positions = [ - [Math.floor(lonlat[0] - lo1) / dx, Math.floor(la1 - lonlat[1]) / dy + 1], - [Math.floor(lonlat[0] - lo1) / dx, Math.floor(la1 - lonlat[1]) / dy], - [Math.floor(lonlat[0] - lo1) / dx + 1, Math.floor(la1 - lonlat[1]) / dy + 1], - [Math.floor(lonlat[0] - lo1) / dx + 1, Math.floor(la1 - lonlat[1]) / dy], - ]; - var values = positions.map(function(d) { - var n = d[0] + Nx * d[1]; - return d3.interpolate( - grib1.data[n], - grib2.data[n])(k); - }); - var val = bilinearInterpolate( - (lonlat[0] - lo1) / dx, - (la1 - lonlat[1]) / dy, - positions[0][0], - positions[2][0], - positions[0][1], - positions[1][1], - values[0], - values[1], - values[2], - values[3]); - img.data[((y*(img.width*4)) + (x*4)) + 3] = parseInt(alphas[bucketIndex(val)] * 255); - }); + + // Set our domain + var NE = projection.invert([realW, 0]) + var NW = projection.invert([0, 0]) + var SW = projection.invert([realW, realH]) + var SE = projection.invert([0, realH]) + var S = projection.invert([realW / 2, realH]) + var N = projection.invert([realW / 2, 0]) + N[1] = 80; + + var minLat = Math.ceil(SE[1]); + var maxLat = Math.floor(N[1]); + var minLon = Math.ceil(NW[0]); + var maxLon = Math.floor(NE[0]); + h = maxLat - minLat; + w = maxLon - minLon; + + var dt = new Date().getTime(); + console.log('Draw solar start:' + new Date()) + + // Draw initial image (1px 1deg) from grib + var imgGrib = ctx.createImageData(w, h); + d3.range(minLat, maxLat).forEach(function (y) { + d3.range(minLon, maxLon).forEach(function (x) { + + var i = parseInt(x - lo1) / dx; + var j = parseInt(la1 - y) / dy; + var n = i + Nx * j; + var val = d3.interpolate(grib1.data[n], grib2.data[n])(k); + + var pix_x = x - minLon; + var pix_y = h - (y - minLat); + + if (levels) + imgGrib.data[[((pix_y * (imgGrib.width * 4)) + (pix_x * 4)) + 3]] = parseInt(val / solarScale * 255); + else + imgGrib.data[[((pix_y * (imgGrib.width * 4)) + (pix_x * 4)) + 3]] = parseInt((0.85 - val / solarScale) * 255); }); + }); + solarCanvas.attr('height', h); + solarCanvas.attr('width', w); + + ctx.clearRect(0, 0, h, w); + ctx.putImageData(imgGrib, 0, 0); + + console.log('Extract grib:' + (new Date().getTime() - dt)); + dt = new Date().getTime(); + + // Reproject this image on our projection + var sourceData = ctx.getImageData(0, 0, w, h).data, + target = ctx.createImageData(realW, realH), + targetData = target.data; - if (x + batchsize <= xrange[xrange.length - 1]) { - setTimeout(function() { return batchDrawColumns(x + batchsize, batchsize, callback); }, 25); - } else { - console.log('Done drawing solar'); - callback(null); + for (var y = 0, i = -1; y < realH; ++y) { + for (var x = 0; x < realW; ++x) { + var p = projection.invert([x, y]), lon = p[0], lat = p[1]; + if (lon > maxLon || lon < minLon || lat > maxLat || lat < minLat) { i += 4; continue; } + var q = ((maxLat - lat) / (maxLat - minLat) * h | 0) * w + ((lon - minLon) / (maxLon - minLon) * w | 0) << 2; + i += 3; + q += 2; + targetData[++i] = sourceData[++q]; } } - batchDrawColumns(0, 200, function() { - ctx.clearRect(0, 0, w, h); - ctx.putImageData(img, 0, 0); - callback(null); - }); + ctx.clearRect(0, 0, w, h); + solarCanvas.attr('height', realH); + solarCanvas.attr('width', realW); + ctx.putImageData(target, 1, 1); + console.log('Reroject time:' + (new Date().getTime() - dt)); + + // Apply a gaussian blur on grid cells + if (gaussianBlur) { + dt = new Date().getTime(); + + stackBlurCanvasRGBA(solarCanvas._groups[0][0].id, 0, 0, realW, realH, BLUR_RADIUS); + console.log('Blur time:' + (new Date().getTime() - dt)); + } + + // Apply level of opacity rather than continous scale + if (levels) { + dt = new Date().getTime(); + + var bluredData = ctx.getImageData(0, 0, realW, realH).data, + target = ctx.createImageData(realW, realH), + targetData = target.data; + + var i = 3; + d3.range(realH).forEach(function (y) { + d3.range(realW).forEach(function (x) { + targetData[i] = parseInt(alphas[bucketIndex(bluredData[i] * solarScale / 255)] * 255); + i += 4; + }); + }); + ctx.clearRect(0, 0, realW, realH); + ctx.putImageData(target, 0, 0); + console.log('Level time:' + (new Date().getTime() - dt)); + } + + callback(null); }; -exports.show = function() { +exports.show = function () { solarCanvas.transition().style('opacity', 1); } -exports.hide = function() { - if(solarCanvas) solarCanvas.transition().style('opacity', 0); +exports.hide = function () { + if (solarCanvas) solarCanvas.transition().style('opacity', 0); } From b8d463b2e85da96a0dc5474956f8ff8035e862bc Mon Sep 17 00:00:00 2001 From: nikkozzblu Date: Tue, 24 Jan 2017 18:34:12 +0100 Subject: [PATCH 08/21] index and rounding (fix a 0.5deg shift) --- web/app/solar.js | 2 +- web/views/pages/index.ejs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/web/app/solar.js b/web/app/solar.js index 860cb3fc85..f7ae24607e 100644 --- a/web/app/solar.js +++ b/web/app/solar.js @@ -133,7 +133,7 @@ exports.draw = function (canvasSelector, now, grib1, grib2, solarColor, projecti for (var x = 0; x < realW; ++x) { var p = projection.invert([x, y]), lon = p[0], lat = p[1]; if (lon > maxLon || lon < minLon || lat > maxLat || lat < minLat) { i += 4; continue; } - var q = ((maxLat - lat) / (maxLat - minLat) * h | 0) * w + ((lon - minLon) / (maxLon - minLon) * w | 0) << 2; + var q = Math.round(((maxLat - lat) / (maxLat - minLat) * h | 0)) * w + (Math.round((lon - minLon) / (maxLon - minLon) * w) | 0) << 2; i += 3; q += 2; targetData[++i] = sourceData[++q]; diff --git a/web/views/pages/index.ejs b/web/views/pages/index.ejs index 8cfa212581..641e20a901 100644 --- a/web/views/pages/index.ejs +++ b/web/views/pages/index.ejs @@ -94,7 +94,7 @@ - +
Carbon intensity
(gCO2eq/kWh)
From bc56c682f45932d9f2ce87a0b6ab877b6320b34e Mon Sep 17 00:00:00 2001 From: nikkozzblu Date: Tue, 24 Jan 2017 18:45:26 +0100 Subject: [PATCH 09/21] Styling and warning --- web/app/solar.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/web/app/solar.js b/web/app/solar.js index f7ae24607e..55ccd3496b 100644 --- a/web/app/solar.js +++ b/web/app/solar.js @@ -50,7 +50,10 @@ exports.draw = function (canvasSelector, now, grib1, grib2, solarColor, projecti var solarScale = solarColor.domain()[solarColor.domain().length - 1]; var k = (now - t_before) / (t_after - t_before); - if (!k || !isFinite(k)) k = 0; + if (!k || !isFinite(k)) { + console.warn("The start and end forecast are identical !"); + k = 0; + } var Nx = grib1.header.nx; var Ny = grib1.header.ny; @@ -61,7 +64,7 @@ exports.draw = function (canvasSelector, now, grib1, grib2, solarColor, projecti var BLUR_RADIUS = 20; - var alphas = solarColor.range().map(function (d) { + var alphas = solarColor.range().map(function(d) { return parseFloat(d .replace('(', '') .replace(')', '') @@ -76,7 +79,6 @@ exports.draw = function (canvasSelector, now, grib1, grib2, solarColor, projecti var w = realW, h = realH; - // Set our domain var NE = projection.invert([realW, 0]) var NW = projection.invert([0, 0]) From ce1eec586f851b68bb9cff1727b4a455c1fb94f7 Mon Sep 17 00:00:00 2001 From: nikkozzblu Date: Wed, 25 Jan 2017 16:04:43 +0100 Subject: [PATCH 10/21] Optimize rounding and comments --- web/app/solar.js | 42 ++++++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/web/app/solar.js b/web/app/solar.js index 55ccd3496b..f1d834746d 100644 --- a/web/app/solar.js +++ b/web/app/solar.js @@ -39,15 +39,17 @@ exports.draw = function (canvasSelector, now, grib1, grib2, solarColor, projecti return console.error('Error while interpolating solar because current time is out of bounds'); } - var k = (now - t_before) / (t_after - t_before); - var buckets = d3.range(solarColor.range().length) + + var buckets = d3.range(solarColor.range().length) .map(function (d) { return []; }); - var bucketIndex = d3.scaleLinear() + + var bucketIndex = d3.scaleLinear() .rangeRound(d3.range(buckets.length)) .domain(solarColor.domain()) .clamp(true); - var solarScale = solarColor.domain()[solarColor.domain().length - 1]; + + var solarScale = solarColor.domain()[solarColor.domain().length - 1]; var k = (now - t_before) / (t_after - t_before); if (!k || !isFinite(k)) { @@ -62,6 +64,7 @@ exports.draw = function (canvasSelector, now, grib1, grib2, solarColor, projecti var dx = grib1.header.dx; var dy = grib1.header.dy; + // ! This 20px is quite arbitrary, seems to be around the average cell size on my screen var BLUR_RADIUS = 20; var alphas = solarColor.range().map(function(d) { @@ -86,7 +89,7 @@ exports.draw = function (canvasSelector, now, grib1, grib2, solarColor, projecti var SE = projection.invert([0, realH]) var S = projection.invert([realW / 2, realH]) var N = projection.invert([realW / 2, 0]) - N[1] = 80; + N[1] = 80; //Don't know why North is not correctly picked up, but got empty space on top of the map otherwise var minLat = Math.ceil(SE[1]); var maxLat = Math.floor(N[1]); @@ -96,8 +99,7 @@ exports.draw = function (canvasSelector, now, grib1, grib2, solarColor, projecti w = maxLon - minLon; var dt = new Date().getTime(); - console.log('Draw solar start:' + new Date()) - + // Draw initial image (1px 1deg) from grib var imgGrib = ctx.createImageData(w, h); d3.range(minLat, maxLat).forEach(function (y) { @@ -130,15 +132,26 @@ exports.draw = function (canvasSelector, now, grib1, grib2, solarColor, projecti var sourceData = ctx.getImageData(0, 0, w, h).data, target = ctx.createImageData(realW, realH), targetData = target.data; - + + // From https://bl.ocks.org/mbostock/4329423 + // x and y are the coordinate on the new image + // i is the new image 1D normalized index (R G B Alpha for each pixel) + // q is the 1D normalize index for the source map for (var y = 0, i = -1; y < realH; ++y) { for (var x = 0; x < realW; ++x) { - var p = projection.invert([x, y]), lon = p[0], lat = p[1]; - if (lon > maxLon || lon < minLon || lat > maxLat || lat < minLat) { i += 4; continue; } - var q = Math.round(((maxLat - lat) / (maxLat - minLat) * h | 0)) * w + (Math.round((lon - minLon) / (maxLon - minLon) * w) | 0) << 2; - i += 3; - q += 2; - targetData[++i] = sourceData[++q]; + // We shift the lat/lon so that the truncation result in a rounding + var p = projection.invert([x, y]), lon = p[0]+0.5, lat = p[1]-0.5; + + if (lon > maxLon || lon < minLon || lat > maxLat || lat < minLat) + { i += 4; continue; } + + var q = (((maxLat - lat) / (maxLat - minLat)) * h | 0) * w + (((lon - minLon) / (maxLon - minLon)) * w | 0) << 2; + + // Since we are reading the map pixel by pixel we go to the next Alpha channel + i += 4; + // Shift source index to alpha + q += 3; + targetData[i] = sourceData[q]; } } @@ -176,6 +189,7 @@ exports.draw = function (canvasSelector, now, grib1, grib2, solarColor, projecti console.log('Level time:' + (new Date().getTime() - dt)); } + // (This callback could potentially be done before effects) callback(null); }; From 398492d0237397ef4406ecfbf44a8cab1cd57b14 Mon Sep 17 00:00:00 2001 From: nikkozzblu Date: Wed, 25 Jan 2017 16:41:23 +0100 Subject: [PATCH 11/21] Continuous scale replace levels --- web/app/solar.js | 69 +++++++++++++++++++++--------------------------- 1 file changed, 30 insertions(+), 39 deletions(-) diff --git a/web/app/solar.js b/web/app/solar.js index f1d834746d..d7f1fce72a 100644 --- a/web/app/solar.js +++ b/web/app/solar.js @@ -24,7 +24,13 @@ exports.draw = function (canvasSelector, now, grib1, grib2, solarColor, projecti // Control the rendering var gaussianBlur = true; - var levels = false; + var continuousScale = true; + // ! This 20px is quite arbitrary, seems to be around the average cell size on my screen + var BLUR_RADIUS = 20; + var SOLAR_SCALE = 1360; + var MAX_OPACITY = 0.85; + + var maxOpacityPix = 256*MAX_OPACITY; // Interpolates between two solar forecasts< var t_before = grib.getTargetTime(grib1); @@ -41,22 +47,12 @@ exports.draw = function (canvasSelector, now, grib1, grib2, solarColor, projecti var k = (now - t_before) / (t_after - t_before); - var buckets = d3.range(solarColor.range().length) - .map(function (d) { return []; }); - - var bucketIndex = d3.scaleLinear() - .rangeRound(d3.range(buckets.length)) - .domain(solarColor.domain()) - .clamp(true); - - var solarScale = solarColor.domain()[solarColor.domain().length - 1]; - - var k = (now - t_before) / (t_after - t_before); - if (!k || !isFinite(k)) { + if (!k || !isFinite(k)) { console.warn("The start and end forecast are identical !"); k = 0; } + // Grib constants var Nx = grib1.header.nx; var Ny = grib1.header.ny; var lo1 = grib1.header.lo1; @@ -64,17 +60,6 @@ exports.draw = function (canvasSelector, now, grib1, grib2, solarColor, projecti var dx = grib1.header.dx; var dy = grib1.header.dy; - // ! This 20px is quite arbitrary, seems to be around the average cell size on my screen - var BLUR_RADIUS = 20; - - var alphas = solarColor.range().map(function(d) { - return parseFloat(d - .replace('(', '') - .replace(')', '') - .replace('rgba', '') - .split(', ')[3]) - }); - solarCanvas = d3.select(canvasSelector); var ctx = solarCanvas.node().getContext('2d'); realW = solarCanvas.node().getBoundingClientRect().width, @@ -82,15 +67,17 @@ exports.draw = function (canvasSelector, now, grib1, grib2, solarColor, projecti var w = realW, h = realH; - // Set our domain + // Set our visible domain var NE = projection.invert([realW, 0]) var NW = projection.invert([0, 0]) var SW = projection.invert([realW, realH]) var SE = projection.invert([0, realH]) var S = projection.invert([realW / 2, realH]) var N = projection.invert([realW / 2, 0]) - N[1] = 80; //Don't know why North is not correctly picked up, but got empty space on top of the map otherwise - + //Don't know why North is not correctly picked up, but got empty space on top of the map otherwise + N[1] = 80; + + // Convert to grib points var minLat = Math.ceil(SE[1]); var maxLat = Math.floor(N[1]); var minLon = Math.ceil(NW[0]); @@ -99,7 +86,6 @@ exports.draw = function (canvasSelector, now, grib1, grib2, solarColor, projecti w = maxLon - minLon; var dt = new Date().getTime(); - // Draw initial image (1px 1deg) from grib var imgGrib = ctx.createImageData(w, h); d3.range(minLat, maxLat).forEach(function (y) { @@ -114,17 +100,11 @@ exports.draw = function (canvasSelector, now, grib1, grib2, solarColor, projecti var pix_y = h - (y - minLat); if (levels) - imgGrib.data[[((pix_y * (imgGrib.width * 4)) + (pix_x * 4)) + 3]] = parseInt(val / solarScale * 255); + imgGrib.data[[((pix_y * (imgGrib.width * 4)) + (pix_x * 4)) + 3]] = parseInt(val / SOLAR_SCALE * 255); else - imgGrib.data[[((pix_y * (imgGrib.width * 4)) + (pix_x * 4)) + 3]] = parseInt((0.85 - val / solarScale) * 255); + imgGrib.data[[((pix_y * (imgGrib.width * 4)) + (pix_x * 4)) + 3]] = parseInt((0.85 - val / SOLAR_SCALE) * 255); }); }); - solarCanvas.attr('height', h); - solarCanvas.attr('width', w); - - ctx.clearRect(0, 0, h, w); - ctx.putImageData(imgGrib, 0, 0); - console.log('Extract grib:' + (new Date().getTime() - dt)); dt = new Date().getTime(); @@ -169,7 +149,7 @@ exports.draw = function (canvasSelector, now, grib1, grib2, solarColor, projecti console.log('Blur time:' + (new Date().getTime() - dt)); } - // Apply level of opacity rather than continous scale + // Apply the opacity/colour continous scale if (levels) { dt = new Date().getTime(); @@ -180,13 +160,24 @@ exports.draw = function (canvasSelector, now, grib1, grib2, solarColor, projecti var i = 3; d3.range(realH).forEach(function (y) { d3.range(realW).forEach(function (x) { - targetData[i] = parseInt(alphas[bucketIndex(bluredData[i] * solarScale / 255)] * 255); + // The bluredData correspond to the solar value projected from 0 to 255 hence, 128 is mid-scale + if (bluredData[i] > 128) { + // Gold + targetData[i - 3] = 255; + targetData[i - 2] = 215; + targetData[i - 1] = 0; + targetData[i] = maxOpacityPix * (bluredData[i] / 128 - 1); + } + else { + targetData[i] = maxOpacityPix * (1 - bluredData[i] / 128); + } + i += 4; }); }); ctx.clearRect(0, 0, realW, realH); ctx.putImageData(target, 0, 0); - console.log('Level time:' + (new Date().getTime() - dt)); + console.log('Shading time:' + (new Date().getTime() - dt)); } // (This callback could potentially be done before effects) From 5ae3c8dbbba18d4b4a8137e48c0e41f00c66cdce Mon Sep 17 00:00:00 2001 From: nikkozzblu Date: Wed, 25 Jan 2017 18:45:48 +0100 Subject: [PATCH 12/21] legend, formatting filter --- web/app/solar.js | 289 +++++++++++++++++++++++++++++--------- web/views/pages/index.ejs | 2 +- 2 files changed, 221 insertions(+), 70 deletions(-) diff --git a/web/app/solar.js b/web/app/solar.js index d7f1fce72a..cae5ca40d8 100644 --- a/web/app/solar.js +++ b/web/app/solar.js @@ -7,15 +7,6 @@ var grib = require('./grib'); var solarCanvas; -function bilinearInterpolate(x, y, x1, x2, y1, y2, Q11, Q12, Q21, Q22) { - var R1 = ((x2 - x) / (x2 - x1)) * Q11 + ((x - x1) / (x2 - x1)) * Q21; - var R2 = ((x2 - x) / (x2 - x1)) * Q12 + ((x - x1) / (x2 - x1)) * Q22; - return ((y2 - y) / (y2 - y1)) * R1 + ((y - y1) / (y2 - y1)) * R2; -} - -// This is a minified library for image bluring, should be moved to exported Filter in file image_blur, not sure how to properly do this -var mul_table = [512, 512, 456, 512, 328, 456, 335, 512, 405, 328, 271, 456, 388, 335, 292, 512, 454, 405, 364, 328, 298, 271, 496, 456, 420, 388, 360, 335, 312, 292, 273, 512, 482, 454, 428, 405, 383, 364, 345, 328, 312, 298, 284, 271, 259, 496, 475, 456, 437, 420, 404, 388, 374, 360, 347, 335, 323, 312, 302, 292, 282, 273, 265, 512, 497, 482, 468, 454, 441, 428, 417, 405, 394, 383, 373, 364, 354, 345, 337, 328, 320, 312, 305, 298, 291, 284, 278, 271, 265, 259, 507, 496, 485, 475, 465, 456, 446, 437, 428, 420, 412, 404, 396, 388, 381, 374, 367, 360, 354, 347, 341, 335, 329, 323, 318, 312, 307, 302, 297, 292, 287, 282, 278, 273, 269, 265, 261, 512, 505, 497, 489, 482, 475, 468, 461, 454, 447, 441, 435, 428, 422, 417, 411, 405, 399, 394, 389, 383, 378, 373, 368, 364, 359, 354, 350, 345, 341, 337, 332, 328, 324, 320, 316, 312, 309, 305, 301, 298, 294, 291, 287, 284, 281, 278, 274, 271, 268, 265, 262, 259, 257, 507, 501, 496, 491, 485, 480, 475, 470, 465, 460, 456, 451, 446, 442, 437, 433, 428, 424, 420, 416, 412, 408, 404, 400, 396, 392, 388, 385, 381, 377, 374, 370, 367, 363, 360, 357, 354, 350, 347, 344, 341, 338, 335, 332, 329, 326, 323, 320, 318, 315, 312, 310, 307, 304, 302, 299, 297, 294, 292, 289, 287, 285, 282, 280, 278, 275, 273, 271, 269, 267, 265, 263, 261, 259]; var shg_table = [9, 11, 12, 13, 13, 14, 14, 15, 15, 15, 15, 16, 16, 16, 16, 17, 17, 17, 17, 17, 17, 17, 18, 18, 18, 18, 18, 18, 18, 18, 18, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24]; function stackBlurImage(imageID, canvasID, radius, blurAlphaChannel) { var img = document.getElementById(imageID); var w = img.naturalWidth; var h = img.naturalHeight; var canvas = document.getElementById(canvasID); canvas.style.width = w + "px"; canvas.style.height = h + "px"; canvas.width = w; canvas.height = h; var context = canvas.getContext("2d"); context.clearRect(0, 0, w, h); context.drawImage(img, 0, 0); if (isNaN(radius) || radius < 1) return; if (blurAlphaChannel) stackBlurCanvasRGBA(canvasID, 0, 0, w, h, radius); else stackBlurCanvasRGB(canvasID, 0, 0, w, h, radius) } function stackBlurCanvasRGBA(id, top_x, top_y, width, height, radius) { if (isNaN(radius) || radius < 1) return; radius |= 0; var canvas = document.getElementById(id); var context = canvas.getContext("2d"); var imageData; try { try { imageData = context.getImageData(top_x, top_y, width, height) } catch (e) { try { netscape.security.PrivilegeManager.enablePrivilege("UniversalBrowserRead"); imageData = context.getImageData(top_x, top_y, width, height) } catch (e) { alert("Cannot access local image"); throw new Error("unable to access local image data: " + e); return } } } catch (e) { alert("Cannot access image"); throw new Error("unable to access image data: " + e) } var pixels = imageData.data; var x, y, i, p, yp, yi, yw, r_sum, g_sum, b_sum, a_sum, r_out_sum, g_out_sum, b_out_sum, a_out_sum, r_in_sum, g_in_sum, b_in_sum, a_in_sum, pr, pg, pb, pa, rbs; var div = radius + radius + 1; var w4 = width << 2; var widthMinus1 = width - 1; var heightMinus1 = height - 1; var radiusPlus1 = radius + 1; var sumFactor = radiusPlus1 * (radiusPlus1 + 1) / 2; var stackStart = new BlurStack; var stack = stackStart; for (i = 1; i < div; i++) { stack = stack.next = new BlurStack; if (i == radiusPlus1) var stackEnd = stack } stack.next = stackStart; var stackIn = null; var stackOut = null; yw = yi = 0; var mul_sum = mul_table[radius]; var shg_sum = shg_table[radius]; for (y = 0; y < height; y++) { r_in_sum = g_in_sum = b_in_sum = a_in_sum = r_sum = g_sum = b_sum = a_sum = 0; r_out_sum = radiusPlus1 * (pr = pixels[yi]); g_out_sum = radiusPlus1 * (pg = pixels[yi + 1]); b_out_sum = radiusPlus1 * (pb = pixels[yi + 2]); a_out_sum = radiusPlus1 * (pa = pixels[yi + 3]); r_sum += sumFactor * pr; g_sum += sumFactor * pg; b_sum += sumFactor * pb; a_sum += sumFactor * pa; stack = stackStart; for (i = 0; i < radiusPlus1; i++) { stack.r = pr; stack.g = pg; stack.b = pb; stack.a = pa; stack = stack.next } for (i = 1; i < radiusPlus1; i++) { p = yi + ((widthMinus1 < i ? widthMinus1 : i) << 2); r_sum += (stack.r = pr = pixels[p]) * (rbs = radiusPlus1 - i); g_sum += (stack.g = pg = pixels[p + 1]) * rbs; b_sum += (stack.b = pb = pixels[p + 2]) * rbs; a_sum += (stack.a = pa = pixels[p + 3]) * rbs; r_in_sum += pr; g_in_sum += pg; b_in_sum += pb; a_in_sum += pa; stack = stack.next } stackIn = stackStart; stackOut = stackEnd; for (x = 0; x < width; x++) { pixels[yi + 3] = pa = a_sum * mul_sum >> shg_sum; if (pa != 0) { pa = 255 / pa; pixels[yi] = (r_sum * mul_sum >> shg_sum) * pa; pixels[yi + 1] = (g_sum * mul_sum >> shg_sum) * pa; pixels[yi + 2] = (b_sum * mul_sum >> shg_sum) * pa } else { pixels[yi] = pixels[yi + 1] = pixels[yi + 2] = 0 } r_sum -= r_out_sum; g_sum -= g_out_sum; b_sum -= b_out_sum; a_sum -= a_out_sum; r_out_sum -= stackIn.r; g_out_sum -= stackIn.g; b_out_sum -= stackIn.b; a_out_sum -= stackIn.a; p = yw + ((p = x + radius + 1) < widthMinus1 ? p : widthMinus1) << 2; r_in_sum += stackIn.r = pixels[p]; g_in_sum += stackIn.g = pixels[p + 1]; b_in_sum += stackIn.b = pixels[p + 2]; a_in_sum += stackIn.a = pixels[p + 3]; r_sum += r_in_sum; g_sum += g_in_sum; b_sum += b_in_sum; a_sum += a_in_sum; stackIn = stackIn.next; r_out_sum += pr = stackOut.r; g_out_sum += pg = stackOut.g; b_out_sum += pb = stackOut.b; a_out_sum += pa = stackOut.a; r_in_sum -= pr; g_in_sum -= pg; b_in_sum -= pb; a_in_sum -= pa; stackOut = stackOut.next; yi += 4 } yw += width } for (x = 0; x < width; x++) { g_in_sum = b_in_sum = a_in_sum = r_in_sum = g_sum = b_sum = a_sum = r_sum = 0; yi = x << 2; r_out_sum = radiusPlus1 * (pr = pixels[yi]); g_out_sum = radiusPlus1 * (pg = pixels[yi + 1]); b_out_sum = radiusPlus1 * (pb = pixels[yi + 2]); a_out_sum = radiusPlus1 * (pa = pixels[yi + 3]); r_sum += sumFactor * pr; g_sum += sumFactor * pg; b_sum += sumFactor * pb; a_sum += sumFactor * pa; stack = stackStart; for (i = 0; i < radiusPlus1; i++) { stack.r = pr; stack.g = pg; stack.b = pb; stack.a = pa; stack = stack.next } yp = width; for (i = 1; i <= radius; i++) { yi = yp + x << 2; r_sum += (stack.r = pr = pixels[yi]) * (rbs = radiusPlus1 - i); g_sum += (stack.g = pg = pixels[yi + 1]) * rbs; b_sum += (stack.b = pb = pixels[yi + 2]) * rbs; a_sum += (stack.a = pa = pixels[yi + 3]) * rbs; r_in_sum += pr; g_in_sum += pg; b_in_sum += pb; a_in_sum += pa; stack = stack.next; if (i < heightMinus1) { yp += width } } yi = x; stackIn = stackStart; stackOut = stackEnd; for (y = 0; y < height; y++) { p = yi << 2; pixels[p + 3] = pa = a_sum * mul_sum >> shg_sum; if (pa > 0) { pa = 255 / pa; pixels[p] = (r_sum * mul_sum >> shg_sum) * pa; pixels[p + 1] = (g_sum * mul_sum >> shg_sum) * pa; pixels[p + 2] = (b_sum * mul_sum >> shg_sum) * pa } else { pixels[p] = pixels[p + 1] = pixels[p + 2] = 0 } r_sum -= r_out_sum; g_sum -= g_out_sum; b_sum -= b_out_sum; a_sum -= a_out_sum; r_out_sum -= stackIn.r; g_out_sum -= stackIn.g; b_out_sum -= stackIn.b; a_out_sum -= stackIn.a; p = x + ((p = y + radiusPlus1) < heightMinus1 ? p : heightMinus1) * width << 2; r_sum += r_in_sum += stackIn.r = pixels[p]; g_sum += g_in_sum += stackIn.g = pixels[p + 1]; b_sum += b_in_sum += stackIn.b = pixels[p + 2]; a_sum += a_in_sum += stackIn.a = pixels[p + 3]; stackIn = stackIn.next; r_out_sum += pr = stackOut.r; g_out_sum += pg = stackOut.g; b_out_sum += pb = stackOut.b; a_out_sum += pa = stackOut.a; r_in_sum -= pr; g_in_sum -= pg; b_in_sum -= pb; a_in_sum -= pa; stackOut = stackOut.next; yi += width } } context.putImageData(imageData, top_x, top_y) } function stackBlurCanvasRGB(id, top_x, top_y, width, height, radius) { if (isNaN(radius) || radius < 1) return; radius |= 0; var canvas = document.getElementById(id); var context = canvas.getContext("2d"); var imageData; try { try { imageData = context.getImageData(top_x, top_y, width, height) } catch (e) { try { netscape.security.PrivilegeManager.enablePrivilege("UniversalBrowserRead"); imageData = context.getImageData(top_x, top_y, width, height) } catch (e) { alert("Cannot access local image"); throw new Error("unable to access local image data: " + e); return } } } catch (e) { alert("Cannot access image"); throw new Error("unable to access image data: " + e) } var pixels = imageData.data; var x, y, i, p, yp, yi, yw, r_sum, g_sum, b_sum, r_out_sum, g_out_sum, b_out_sum, r_in_sum, g_in_sum, b_in_sum, pr, pg, pb, rbs; var div = radius + radius + 1; var w4 = width << 2; var widthMinus1 = width - 1; var heightMinus1 = height - 1; var radiusPlus1 = radius + 1; var sumFactor = radiusPlus1 * (radiusPlus1 + 1) / 2; var stackStart = new BlurStack; var stack = stackStart; for (i = 1; i < div; i++) { stack = stack.next = new BlurStack; if (i == radiusPlus1) var stackEnd = stack } stack.next = stackStart; var stackIn = null; var stackOut = null; yw = yi = 0; var mul_sum = mul_table[radius]; var shg_sum = shg_table[radius]; for (y = 0; y < height; y++) { r_in_sum = g_in_sum = b_in_sum = r_sum = g_sum = b_sum = 0; r_out_sum = radiusPlus1 * (pr = pixels[yi]); g_out_sum = radiusPlus1 * (pg = pixels[yi + 1]); b_out_sum = radiusPlus1 * (pb = pixels[yi + 2]); r_sum += sumFactor * pr; g_sum += sumFactor * pg; b_sum += sumFactor * pb; stack = stackStart; for (i = 0; i < radiusPlus1; i++) { stack.r = pr; stack.g = pg; stack.b = pb; stack = stack.next } for (i = 1; i < radiusPlus1; i++) { p = yi + ((widthMinus1 < i ? widthMinus1 : i) << 2); r_sum += (stack.r = pr = pixels[p]) * (rbs = radiusPlus1 - i); g_sum += (stack.g = pg = pixels[p + 1]) * rbs; b_sum += (stack.b = pb = pixels[p + 2]) * rbs; r_in_sum += pr; g_in_sum += pg; b_in_sum += pb; stack = stack.next } stackIn = stackStart; stackOut = stackEnd; for (x = 0; x < width; x++) { pixels[yi] = r_sum * mul_sum >> shg_sum; pixels[yi + 1] = g_sum * mul_sum >> shg_sum; pixels[yi + 2] = b_sum * mul_sum >> shg_sum; r_sum -= r_out_sum; g_sum -= g_out_sum; b_sum -= b_out_sum; r_out_sum -= stackIn.r; g_out_sum -= stackIn.g; b_out_sum -= stackIn.b; p = yw + ((p = x + radius + 1) < widthMinus1 ? p : widthMinus1) << 2; r_in_sum += stackIn.r = pixels[p]; g_in_sum += stackIn.g = pixels[p + 1]; b_in_sum += stackIn.b = pixels[p + 2]; r_sum += r_in_sum; g_sum += g_in_sum; b_sum += b_in_sum; stackIn = stackIn.next; r_out_sum += pr = stackOut.r; g_out_sum += pg = stackOut.g; b_out_sum += pb = stackOut.b; r_in_sum -= pr; g_in_sum -= pg; b_in_sum -= pb; stackOut = stackOut.next; yi += 4 } yw += width } for (x = 0; x < width; x++) { g_in_sum = b_in_sum = r_in_sum = g_sum = b_sum = r_sum = 0; yi = x << 2; r_out_sum = radiusPlus1 * (pr = pixels[yi]); g_out_sum = radiusPlus1 * (pg = pixels[yi + 1]); b_out_sum = radiusPlus1 * (pb = pixels[yi + 2]); r_sum += sumFactor * pr; g_sum += sumFactor * pg; b_sum += sumFactor * pb; stack = stackStart; for (i = 0; i < radiusPlus1; i++) { stack.r = pr; stack.g = pg; stack.b = pb; stack = stack.next } yp = width; for (i = 1; i <= radius; i++) { yi = yp + x << 2; r_sum += (stack.r = pr = pixels[yi]) * (rbs = radiusPlus1 - i); g_sum += (stack.g = pg = pixels[yi + 1]) * rbs; b_sum += (stack.b = pb = pixels[yi + 2]) * rbs; r_in_sum += pr; g_in_sum += pg; b_in_sum += pb; stack = stack.next; if (i < heightMinus1) { yp += width } } yi = x; stackIn = stackStart; stackOut = stackEnd; for (y = 0; y < height; y++) { p = yi << 2; pixels[p] = r_sum * mul_sum >> shg_sum; pixels[p + 1] = g_sum * mul_sum >> shg_sum; pixels[p + 2] = b_sum * mul_sum >> shg_sum; r_sum -= r_out_sum; g_sum -= g_out_sum; b_sum -= b_out_sum; r_out_sum -= stackIn.r; g_out_sum -= stackIn.g; b_out_sum -= stackIn.b; p = x + ((p = y + radiusPlus1) < heightMinus1 ? p : heightMinus1) * width << 2; r_sum += r_in_sum += stackIn.r = pixels[p]; g_sum += g_in_sum += stackIn.g = pixels[p + 1]; b_sum += b_in_sum += stackIn.b = pixels[p + 2]; stackIn = stackIn.next; r_out_sum += pr = stackOut.r; g_out_sum += pg = stackOut.g; b_out_sum += pb = stackOut.b; r_in_sum -= pr; g_in_sum -= pg; b_in_sum -= pb; stackOut = stackOut.next; yi += width } } context.putImageData(imageData, top_x, top_y) } function BlurStack() { this.r = 0; this.g = 0; this.b = 0; this.a = 0; this.next = null } - exports.isExpired = function (now, grib1, grib2) { return grib.getTargetTime(grib2) <= moment(now) || grib.getTargetTime(grib1) > moment(now); } @@ -25,12 +16,12 @@ exports.draw = function (canvasSelector, now, grib1, grib2, solarColor, projecti // Control the rendering var gaussianBlur = true; var continuousScale = true; - // ! This 20px is quite arbitrary, seems to be around the average cell size on my screen + // ! This 20px is quite arbitrary, seems to be around the average cell size on my screen var BLUR_RADIUS = 20; - var SOLAR_SCALE = 1360; + var SOLAR_SCALE = 1360; var MAX_OPACITY = 0.85; - - var maxOpacityPix = 256*MAX_OPACITY; + + var maxOpacityPix = 256 * MAX_OPACITY; // Interpolates between two solar forecasts< var t_before = grib.getTargetTime(grib1); @@ -46,13 +37,8 @@ exports.draw = function (canvasSelector, now, grib1, grib2, solarColor, projecti } var k = (now - t_before) / (t_after - t_before); - - if (!k || !isFinite(k)) { - console.warn("The start and end forecast are identical !"); - k = 0; - } - // Grib constants + // Grib constants var Nx = grib1.header.nx; var Ny = grib1.header.ny; var lo1 = grib1.header.lo1; @@ -75,9 +61,9 @@ exports.draw = function (canvasSelector, now, grib1, grib2, solarColor, projecti var S = projection.invert([realW / 2, realH]) var N = projection.invert([realW / 2, 0]) //Don't know why North is not correctly picked up, but got empty space on top of the map otherwise - N[1] = 80; - - // Convert to grib points + N[1] = 80; + + // Convert to grib points var minLat = Math.ceil(SE[1]); var maxLat = Math.floor(N[1]); var minLon = Math.ceil(NW[0]); @@ -85,7 +71,6 @@ exports.draw = function (canvasSelector, now, grib1, grib2, solarColor, projecti h = maxLat - minLat; w = maxLon - minLon; - var dt = new Date().getTime(); // Draw initial image (1px 1deg) from grib var imgGrib = ctx.createImageData(w, h); d3.range(minLat, maxLat).forEach(function (y) { @@ -99,88 +84,79 @@ exports.draw = function (canvasSelector, now, grib1, grib2, solarColor, projecti var pix_x = x - minLon; var pix_y = h - (y - minLat); - if (levels) + if (continuousScale) imgGrib.data[[((pix_y * (imgGrib.width * 4)) + (pix_x * 4)) + 3]] = parseInt(val / SOLAR_SCALE * 255); else imgGrib.data[[((pix_y * (imgGrib.width * 4)) + (pix_x * 4)) + 3]] = parseInt((0.85 - val / SOLAR_SCALE) * 255); }); }); - console.log('Extract grib:' + (new Date().getTime() - dt)); - dt = new Date().getTime(); // Reproject this image on our projection - var sourceData = ctx.getImageData(0, 0, w, h).data, + var sourceData = imgGrib.data, target = ctx.createImageData(realW, realH), targetData = target.data; - - // From https://bl.ocks.org/mbostock/4329423 - // x and y are the coordinate on the new image - // i is the new image 1D normalized index (R G B Alpha for each pixel) - // q is the 1D normalize index for the source map + + // From https://bl.ocks.org/mbostock/4329423 + // x and y are the coordinate on the new image + // i is the new image 1D normalized index (R G B Alpha for each pixel) + // q is the 1D normalize index for the source map for (var y = 0, i = -1; y < realH; ++y) { for (var x = 0; x < realW; ++x) { - // We shift the lat/lon so that the truncation result in a rounding - var p = projection.invert([x, y]), lon = p[0]+0.5, lat = p[1]-0.5; - - if (lon > maxLon || lon < minLon || lat > maxLat || lat < minLat) - { i += 4; continue; } - - var q = (((maxLat - lat) / (maxLat - minLat)) * h | 0) * w + (((lon - minLon) / (maxLon - minLon)) * w | 0) << 2; - - // Since we are reading the map pixel by pixel we go to the next Alpha channel - i += 4; - // Shift source index to alpha + // We shift the lat/lon so that the truncation result in a rounding + var p = projection.invert([x, y]), lon = p[0] + 0.5, lat = p[1] - 0.5; + + if (lon > maxLon || lon < minLon || lat > maxLat || lat < minLat) + { i += 4; continue; } + + var q = (((maxLat - lat) / (maxLat - minLat)) * h | 0) * w + (((lon - minLon) / (maxLon - minLon)) * w | 0) << 2; + + // Since we are reading the map pixel by pixel we go to the next Alpha channel + i += 4; + // Shift source index to alpha q += 3; targetData[i] = sourceData[q]; } } - ctx.clearRect(0, 0, w, h); - solarCanvas.attr('height', realH); - solarCanvas.attr('width', realW); - ctx.putImageData(target, 1, 1); - console.log('Reroject time:' + (new Date().getTime() - dt)); // Apply a gaussian blur on grid cells if (gaussianBlur) { - dt = new Date().getTime(); - - stackBlurCanvasRGBA(solarCanvas._groups[0][0].id, 0, 0, realW, realH, BLUR_RADIUS); - console.log('Blur time:' + (new Date().getTime() - dt)); + target = stackBlurImageOpacity(target, 0, 0, realW, realH, BLUR_RADIUS); } - // Apply the opacity/colour continous scale - if (levels) { - dt = new Date().getTime(); + // Apply level of opacity rather than continous scale + if (continuousScale) { - var bluredData = ctx.getImageData(0, 0, realW, realH).data, - target = ctx.createImageData(realW, realH), - targetData = target.data; + var bluredData = target.data, + next = ctx.createImageData(realW, realH), + nextData = next.data; var i = 3; d3.range(realH).forEach(function (y) { d3.range(realW).forEach(function (x) { - // The bluredData correspond to the solar value projected from 0 to 255 hence, 128 is mid-scale - if (bluredData[i] > 128) { + // The bluredData correspond to the solar value projected from 0 to 255 hence, 128 is mid-scale + if (bluredData[i] > 128) { // Gold - targetData[i - 3] = 255; - targetData[i - 2] = 215; - targetData[i - 1] = 0; - targetData[i] = maxOpacityPix * (bluredData[i] / 128 - 1); + nextData[i - 3] = 255; + nextData[i - 2] = 215; + nextData[i - 1] = 0; + nextData[i] = maxOpacityPix * (bluredData[i] / 128 - 1); } else { - targetData[i] = maxOpacityPix * (1 - bluredData[i] / 128); + nextData[i] = maxOpacityPix * (1 - bluredData[i] / 128); } i += 4; }); }); - ctx.clearRect(0, 0, realW, realH); - ctx.putImageData(target, 0, 0); - console.log('Shading time:' + (new Date().getTime() - dt)); + target = next; } + solarCanvas.attr('height', realH); + solarCanvas.attr('width', realW); + ctx.clearRect(0, 0, realW, realH); + ctx.putImageData(target, 0, 0); - // (This callback could potentially be done before effects) + // (This callback could potentially be done before effects) callback(null); }; @@ -191,3 +167,178 @@ exports.show = function () { exports.hide = function () { if (solarCanvas) solarCanvas.transition().style('opacity', 0); } + + +var mul_table = [ + 512, 512, 456, 512, 328, 456, 335, 512, 405, 328, 271, 456, 388, 335, 292, 512, + 454, 405, 364, 328, 298, 271, 496, 456, 420, 388, 360, 335, 312, 292, 273, 512, + 482, 454, 428, 405, 383, 364, 345, 328, 312, 298, 284, 271, 259, 496, 475, 456, + 437, 420, 404, 388, 374, 360, 347, 335, 323, 312, 302, 292, 282, 273, 265, 512, + 497, 482, 468, 454, 441, 428, 417, 405, 394, 383, 373, 364, 354, 345, 337, 328, + 320, 312, 305, 298, 291, 284, 278, 271, 265, 259, 507, 496, 485, 475, 465, 456, + 446, 437, 428, 420, 412, 404, 396, 388, 381, 374, 367, 360, 354, 347, 341, 335, + 329, 323, 318, 312, 307, 302, 297, 292, 287, 282, 278, 273, 269, 265, 261, 512, + 505, 497, 489, 482, 475, 468, 461, 454, 447, 441, 435, 428, 422, 417, 411, 405, + 399, 394, 389, 383, 378, 373, 368, 364, 359, 354, 350, 345, 341, 337, 332, 328, + 324, 320, 316, 312, 309, 305, 301, 298, 294, 291, 287, 284, 281, 278, 274, 271, + 268, 265, 262, 259, 257, 507, 501, 496, 491, 485, 480, 475, 470, 465, 460, 456, + 451, 446, 442, 437, 433, 428, 424, 420, 416, 412, 408, 404, 400, 396, 392, 388, + 385, 381, 377, 374, 370, 367, 363, 360, 357, 354, 350, 347, 344, 341, 338, 335, + 332, 329, 326, 323, 320, 318, 315, 312, 310, 307, 304, 302, 299, 297, 294, 292, + 289, 287, 285, 282, 280, 278, 275, 273, 271, 269, 267, 265, 263, 261, 259]; + +var shg_table = [ + 9, 11, 12, 13, 13, 14, 14, 15, 15, 15, 15, 16, 16, 16, 16, 17, + 17, 17, 17, 17, 17, 17, 18, 18, 18, 18, 18, 18, 18, 18, 18, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24]; + +// Derived from http://www.quasimondo.com/StackBlurForCanvas/StackBlur.js +function stackBlurImageOpacity(imageData, top_x, top_y, width, height, radius) { + if (isNaN(radius) || radius < 1) return; + radius |= 0; + + var pixels = imageData.data; + + var x, y, i, p, yp, yi, yw, a_sum, a_out_sum, a_in_sum, pa, rbs; + + var div = radius + radius + 1; + var w4 = width << 2; + var widthMinus1 = width - 1; + var heightMinus1 = height - 1; + var radiusPlus1 = radius + 1; + var sumFactor = radiusPlus1 * (radiusPlus1 + 1) / 2; + + var stackStart = new BlurStack(); + var stack = stackStart; + for (i = 1; i < div; i++) { + stack = stack.next = new BlurStack(); + if (i == radiusPlus1) var stackEnd = stack; + } + stack.next = stackStart; + var stackIn = null; + var stackOut = null; + + yw = yi = 0; + + var mul_sum = mul_table[radius]; + var shg_sum = shg_table[radius]; + + for (y = 0; y < height; y++) { + a_in_sum = a_sum = 0; + + a_out_sum = radiusPlus1 * (pa = pixels[yi + 3]); + a_sum += sumFactor * pa; + stack = stackStart; + + for (i = 0; i < radiusPlus1; i++) { + stack.a = pa; + stack = stack.next; + } + + for (i = 1; i < radiusPlus1; i++) { + p = yi + ((widthMinus1 < i ? widthMinus1 : i) << 2); + a_sum += (stack.a = (pa = pixels[p + 3])) * (rbs = radiusPlus1 - i); + a_in_sum += pa; + stack = stack.next; + } + + + stackIn = stackStart; + stackOut = stackEnd; + for (x = 0; x < width; x++) { + pixels[yi + 3] = pa = (a_sum * mul_sum) >> shg_sum; + + a_sum -= a_out_sum; + a_out_sum -= stackIn.a; + + p = (yw + ((p = x + radius + 1) < widthMinus1 ? p : widthMinus1)) << 2; + a_in_sum += (stackIn.a = pixels[p + 3]); + a_sum += a_in_sum; + + stackIn = stackIn.next; + + a_out_sum += (pa = stackOut.a); + + a_in_sum -= pa; + + stackOut = stackOut.next; + + yi += 4; + } + yw += width; + } + + + for (x = 0; x < width; x++) { + a_in_sum = a_sum = 0; + + yi = x << 2; + a_out_sum = radiusPlus1 * (pa = pixels[yi + 3]); + a_sum += sumFactor * pa; + + stack = stackStart; + + for (i = 0; i < radiusPlus1; i++) { + stack.a = pa; + stack = stack.next; + } + + yp = width; + + for (i = 1; i <= radius; i++) { + yi = (yp + x) << 2; + + a_sum += (stack.a = (pa = pixels[yi + 3])) * rbs; + + a_in_sum += pa; + + stack = stack.next; + + if (i < heightMinus1) { + yp += width; + } + } + + yi = x; + stackIn = stackStart; + stackOut = stackEnd; + for (y = 0; y < height; y++) { + p = yi << 2; + pixels[p + 3] = pa = (a_sum * mul_sum) >> shg_sum; + + a_sum -= a_out_sum; + a_out_sum -= stackIn.a; + + p = (x + (((p = y + radiusPlus1) < heightMinus1 ? p : heightMinus1) * width)) << 2; + a_sum += (a_in_sum += (stackIn.a = pixels[p + 3])); + stackIn = stackIn.next; + a_out_sum += (pa = stackOut.a); + + a_in_sum -= pa; + + stackOut = stackOut.next; + + yi += width; + } + } + return imageData; + +} + +function BlurStack() { + this.a = 0; + this.next = null; +} diff --git a/web/views/pages/index.ejs b/web/views/pages/index.ejs index b8942c6341..9434dbc00c 100644 --- a/web/views/pages/index.ejs +++ b/web/views/pages/index.ejs @@ -94,7 +94,7 @@
- +
Carbon intensity
(gCO2eq/kWh)
From e917d1a67de29bb4baa073c21efebe9bac9c52ef Mon Sep 17 00:00:00 2001 From: nikkozzblu Date: Wed, 25 Jan 2017 18:48:20 +0100 Subject: [PATCH 13/21] forgot those 2 --- web/app/horizontalcolorbar.js | 2 +- web/app/main.js | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/web/app/horizontalcolorbar.js b/web/app/horizontalcolorbar.js index a574933abe..b38df1aa2f 100644 --- a/web/app/horizontalcolorbar.js +++ b/web/app/horizontalcolorbar.js @@ -132,7 +132,7 @@ HorizontalColorbar.prototype.render = function() { // Draw the horizontal axis var axis = d3.axisBottom(this.scale) .tickSizeInner(this.colorbarHeight / 2.0) - .tickPadding(3); + .tickPadding(3).ticks(7); if (this.d3TickFormat) axis.tickFormat(this.d3TickFormat); if (this.d3TickValues) diff --git a/web/app/main.js b/web/app/main.js index 05965e467e..b4eb12a1f2 100644 --- a/web/app/main.js +++ b/web/app/main.js @@ -188,11 +188,10 @@ var co2Colorbar = new HorizontalColorbar('.co2-colorbar', co2color) var windColorbar = new HorizontalColorbar('.wind-colorbar', windColor) .markerColor('black'); var solarColorbarColor = d3.scaleLinear() - .domain([0, minDayDSWRF, maxSolarDSWRF]) - .range(['black', 'black', 'white']) - .clamp(solarColor.clamp()); + .domain([0, 1360/2, 1360]) + .range(['black', 'white', 'gold']) var solarColorbar = new HorizontalColorbar('.solar-colorbar', solarColorbarColor) - .markerColor('red'); + .markerColor('red') var tableDisplayEmissions = countryTable.displayByEmissions(); From f7f1dd93717901c2f8120776f9ffdd8a79c378a7 Mon Sep 17 00:00:00 2001 From: nikkozzblu Date: Mon, 6 Feb 2017 10:49:44 +0100 Subject: [PATCH 14/21] Merge lang from local to updated branch --- docker-compose.yml | 1 + web/app/configs/lang.json | 35 ++++++ web/app/countrytable.js | 3 +- web/app/main.js | 18 ++- web/app/tooltip.js | 6 +- web/locales/en.json | 45 ++++++++ web/locales/fr.json | 45 ++++++++ web/server.js | 13 +++ web/views/pages/index.ejs | 232 +++++++++++++++++++------------------- 9 files changed, 267 insertions(+), 131 deletions(-) create mode 100644 web/app/configs/lang.json create mode 100644 web/locales/en.json create mode 100644 web/locales/fr.json diff --git a/docker-compose.yml b/docker-compose.yml index ef235b937f..cf59d1931c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,6 +19,7 @@ services: - './web/server.js:/home/web/server.js' - './web/views:/home/web/views' - './web/webpack.config.js:/home/web/webpack.config.js' + - './web/locales:/home/web/locales' feeder: build: context: . diff --git a/web/app/configs/lang.json b/web/app/configs/lang.json new file mode 100644 index 0000000000..20b792513b --- /dev/null +++ b/web/app/configs/lang.json @@ -0,0 +1,35 @@ +{ + "en": + { + "wind": "wind", + "solar": "solar", + "hydro": "hydro", + "hydrostorage": "hydro storage", + "biomass": "biomass", + "nuclear": "nuclear", + "gas": "gas", + "coal": "coal", + "oil": "oil", + "unknown": "unknown", + "exportto": "of export to", + "importfrom":"of import from", + "productionimport":"production/import", + "consumption":"consumption" + }, + "fr": { + "wind": "éolien", + "solar": "solaire", + "hydro": "hydro", + "hydrostorage": "stockage hydro", + "biomass": "biomasse", + "nuclear": "nucléaire", + "gas": "gaz", + "coal": "charbon", + "oil": "fioul", + "unknown": "inconnu", + "exportto": "exportée vers", + "importfrom":" importée depuis", + "productionimport":"production/import", + "consumption":"consommation" + } +} diff --git a/web/app/countrytable.js b/web/app/countrytable.js index 4d2adada5e..ab1c75b29d 100644 --- a/web/app/countrytable.js +++ b/web/app/countrytable.js @@ -1,7 +1,7 @@ var d3 = require('d3'); var moment = require('moment'); -function CountryTable(selector, co2Color) { +function CountryTable(selector, co2Color, lang) { var that = this; this.root = d3.select(selector); @@ -86,7 +86,6 @@ CountryTable.prototype.render = function() { return 'translate(0,' + (i * (that.ROW_HEIGHT + that.PADDING_Y)) + ')'; }); gNewRow.append('text') - .text(function(d) { return d.mode }) .attr('transform', 'translate(0, ' + this.TEXT_ADJUST_Y + ')'); gNewRow.append('rect') .attr('class', 'capacity') diff --git a/web/app/main.js b/web/app/main.js index a682256d64..2a2171fbb4 100644 --- a/web/app/main.js +++ b/web/app/main.js @@ -21,6 +21,7 @@ var Wind = require('./wind'); // Configs var capacities = require('json-loader!./configs/capacities.json'); var zones = require('json-loader!./configs/zones.json'); +var lang = require('json-loader!./configs/lang.json'); // Constants var REFRESH_TIME_MINUTES = 5; @@ -33,6 +34,7 @@ var selectedCountryCode; var forceRemoteEndpoint = false; var customDate; var timelineEnabled = false; +var reqLang = 'en'; function isMobile() { return (/android|blackberry|iemobile|ipad|iphone|ipod|opera mini|webos/i).test(navigator.userAgent); @@ -79,6 +81,8 @@ args.forEach(function(arg) { } else if (kv[0] == 'countryCode') { selectedCountryCode = kv[1]; replaceHistoryState('countryCode', selectedCountryCode); + } else if(kv[0] == 'lang'){ + reqLang = kv[1]; } }); @@ -179,7 +183,7 @@ var solarColor = d3.scaleLinear() // Set up objects var countryMap = new CountryMap('.map', co2color); var exchangeLayer = new ExchangeLayer('.map', co2color); -var countryTable = new CountryTable('.country-table', co2color); +var countryTable = new CountryTable('.country-table', co2color, lang[reqLang]); var co2Colorbar = new HorizontalColorbar('.co2-colorbar', co2color) .markerColor('white') @@ -188,18 +192,10 @@ var windColorbar = new HorizontalColorbar('.wind-colorbar', windColor) .markerColor('black'); d3.select('.wind-colorbar').style('display', windEnabled ? 'block': 'none'); var solarColorbarColor = d3.scaleLinear() -<<<<<<< HEAD - .domain([0, 1360/2, 1360]) - .range(['black', 'white', 'gold']) -var solarColorbar = new HorizontalColorbar('.solar-colorbar', solarColorbarColor) - .markerColor('red') -======= .domain([0, 0.5 * maxSolarDSWRF, maxSolarDSWRF]) .range(['black', 'white', 'gold']) var solarColorbar = new HorizontalColorbar('.solar-colorbar', solarColorbarColor) - .markerColor('red'); -d3.select('.solar-colorbar').style('display', solarEnabled ? 'block': 'none'); ->>>>>>> refs/remotes/corradio/master + .markerColor('red') var tableDisplayEmissions = countryTable.displayByEmissions(); @@ -394,7 +390,7 @@ if (isSmallScreen()) { }); // Tooltip setup - Tooltip.setupCountryTable(countryTable, countries, co2Colorbar, co2color); + Tooltip.setupCountryTable(countryTable, countries, co2Colorbar, co2color, lang[reqLang] ); } function dataLoaded(err, state, argSolar, argWind) { diff --git a/web/app/tooltip.js b/web/app/tooltip.js index d290502c13..0869111a12 100644 --- a/web/app/tooltip.js +++ b/web/app/tooltip.js @@ -15,7 +15,7 @@ function getConsumption(country) { } // ** Country table -exports.setupCountryTable = function (countryTable, countries, co2Colorbar, co2color) { +exports.setupCountryTable = function (countryTable, countries, co2Colorbar, co2color,lang) { countryTable .onExchangeMouseOver(function (d, countryCode) { var isExport = d.value < 0; @@ -25,7 +25,7 @@ exports.setupCountryTable = function (countryTable, countries, co2Colorbar, co2c co2Colorbar.currentMarker(co2intensity); var tooltip = d3.select('#countrypanel-exchange-tooltip'); tooltip.style('display', 'inline'); - tooltip.select('#label').text(isExport ? 'export to' : 'import from'); + tooltip.select('#label').text(isExport ? lang['exportto'] : lang['importfrom']); tooltip.select('#country-code').text(d.key); tooltip.select('.emission-rect') .style('background-color', co2intensity ? co2color(co2intensity) : 'gray'); @@ -82,7 +82,7 @@ exports.setupCountryTable = function (countryTable, countries, co2Colorbar, co2c co2Colorbar.currentMarker(co2intensity); var tooltip = d3.select('#countrypanel-production-tooltip'); tooltip.style('display', 'inline'); - tooltip.selectAll('#mode').text(d.mode); + tooltip.selectAll('#mode').text(d.text || d.mode); tooltip.select('.emission-rect') .style('background-color', co2intensity ? co2color(co2intensity) : 'gray'); tooltip.select('.emission-intensity') diff --git a/web/locales/en.json b/web/locales/en.json new file mode 100644 index 0000000000..0301197b7b --- /dev/null +++ b/web/locales/en.json @@ -0,0 +1,45 @@ +{ + "oops": "Oops! We are having trouble reaching the server. We will try again in a few seconds.", + "carbonintensity": "Carbon intensity", + "maintitle": "Live CO2 emissions of the European electricity consumption", + "source": "source", + "electricityprice": "Electricity price (day-ahead)", + "electricityproduction": "Electricity production", + "showemissions": "show emissions", + "bysource": "by source", + "emissions": "Emissions", + "showelectricity": "show electricity", + "youareseeing": "You are currently seeing a ", + "limitedversion": "limited mobile version", + "forthe": "For the", + "fullversion": "full experience", + "visitoncomputer": "visit this page on your computer", + "thisshowsrealtime": "This shows in real-time", + "whereelectricitycomesfrom": "where your electricity comes from", + "and": "and", + "homuchco2": "how much CO2", + "wasemitted": "was emitted to produce it", + "takeinaccount": "We take into account electricity", + "importexport": "imports and exports", + "betweencountries": "between countries", + "thisproject": "This project is", + "opensource": "Open Source", + "see": "see", + "datasources": "data sources", + "tip": "Tip: Click on a country to start exploring", + "windpotential": "Wind power potential", + "solarpotential": "Solar power potential", + "likethisvisu": "Like the visualization?", + "loveyourfeedback": "We would love to hear your feedback", + "foundbugs": "Found bugs or have ideas? Report them", + "here": "here", + "co2signalprefix": "Check out how our", + "co2signalsuffix": "can help your devices and electric vehicle consume electricity at the right time", + "crossborderexport": "Cross-border export", + "carbonintensityexport": "Carbon intensity of export", + "of": "of", + "ofconsumption":"of", + "ofinstalled": "of installed", + "capacity": "capacity", + "back": "back" +} diff --git a/web/locales/fr.json b/web/locales/fr.json new file mode 100644 index 0000000000..60e7da3398 --- /dev/null +++ b/web/locales/fr.json @@ -0,0 +1,45 @@ +{ + "oops": "Oops! Nous rencontrons un problème de connectivité au serveur. Une nouvelle tentative aura lieu dans quelques instants.", + "carbonintensity": "Intensité carbone", + "maintitle": "Emissions de CO2 de la production électrique Européene en temps réel", + "source": "sources", + "electricityprice": "Prix de l'électricité (day-ahead)", + "electricityproduction": "Production électrique", + "showemissions": "montrer les émissions", + "bysource": "par source", + "emissions": "Emissions", + "showelectricity": "montrer la production électrique", + "youareseeing": "Vous voyez actuellement", + "limitedversion": "une version mobile limitée", + "forthe": "Pour une", + "fullversion": "meilleur experience", + "visitoncomputer": "visitez ce site depuis un ordinateur", + "thisshowsrealtime": "Cette carte indique", + "whereelectricitycomesfrom": "de quelles sources provient votre électricité", + "and": "et", + "homuchco2": "quelle quantité de CO2", + "wasemitted": "a été émise pour la produire", + "takeinaccount": "Elle prend en compte les", + "importexport": "imports et exports", + "betweencountries": "entre pays", + "thisproject": "Ceci est un projet", + "opensource": "Open Source", + "see": "voir", + "datasources": "data sources", + "tip": "Tip: Cliquez un pays pour explorer sa production", + "windpotential": "Potentiel éolien", + "solarpotential": "Potentiel solaire", + "likethisvisu": "Vous aimez cette carte intéractive?", + "loveyourfeedback": "Envoyez nous vos commentaires", + "foundbugs": "Un bug ? une idée ? Cliquez", + "here": "ici", + "co2signalprefix": "Découvrez aussi comment", + "co2signalsuffix": "peut aider vos appareils et véhicules électriques à consommer l électricité au meilleur moments", + "crossborderexport": "Export cross-frontalier", + "carbonintensityexport": "Intensité carbone de l export", + "of":"de", + "ofconsumption": "de la consomation", + "ofinstalled": "de la capacité installée en", + "capacity": " ", + "back": "retour" +} diff --git a/web/server.js b/web/server.js index 1538b605bc..b9e6c1dbf5 100644 --- a/web/server.js +++ b/web/server.js @@ -20,6 +20,8 @@ var http = require('http'); var Memcached = require('memcached'); var moment = require('moment'); var MongoClient = require('mongodb').MongoClient; +var i18n = require('i18n'); + //var statsd = require('node-statsd'); // TODO: Remove // Custom modules @@ -41,6 +43,17 @@ app.use(function(req, res, next) { // * Static and templating var STATIC_PATH = process.env['STATIC_PATH'] || (__dirname + '/public'); app.use(express.static(STATIC_PATH, {etag: true, maxAge: isProduction ? '24h': '0'})); +//multi-language +i18n.configure({ + // where to store json files - defaults to './locales' relative to modules directory + locales:['en', 'fr'], + directory: __dirname + '/locales', + defaultLocale: 'en', + queryParameter: 'lang' + // sets a custom cookie name to parse locale settings from - defaults to NULL + //cookie: 'lang', +}); +app.use(i18n.init); app.set('view engine', 'ejs'); var BUNDLE_HASH = !isProduction ? 'dev' : JSON.parse(fs.readFileSync(STATIC_PATH + '/dist/manifest.json')).hash; diff --git a/web/views/pages/index.ejs b/web/views/pages/index.ejs index 5a35d923eb..ddfedc5d35 100644 --- a/web/views/pages/index.ejs +++ b/web/views/pages/index.ejs @@ -18,7 +18,7 @@ - Electricity Map | Live CO2 emissions of the European electricity consumption + Electricity Map | <%= __('maintitle') %> @@ -39,8 +39,8 @@ @@ -91,120 +91,113 @@
-
Oops! We're having trouble reaching the server. We'll try again in a few seconds.
+
<%= __('oops') %>
-
Carbon intensity
(gCO2eq/kWh)
+
<%= __('carbonintensity') %>
(gCO2eq/kWh)
-
-

- Live CO2 emissions of the European electricity consumption -

-
-
- - -
-
- - - -
-
-

- -

-
- - -
-

- You're currently seeing a limited mobile version. For the full experience, visit this page on your computer. -

-

- This shows in real-time where your electricity comes from and how much CO2 was emitted to produce it. -

-

- We take into account electricity imports and exports between countries. -

+
+
+

+ <%= __('maintitle') %> +

- This project is Open Source (see data sources). +

-

- Tip: Click on a country to start exploring ⟶ -

-
-

+
+ -
-
+ +
+

+ <%= __('youareseeing') %> <%= __('limitedversion') %>. <%= __('forthe') %> <%= __('fullversion') %>, <%= __('visitoncomputer') %>. +

+

+ <%= __('thisshowsrealtime') %> <%= __('whereelectricitycomesfrom') %> <%= __('and') %> <%= __('homuchco2') %> <%= __('wasemitted') %>. +

+

+ <%= __('takeinaccount') %> <%= __('importexport') %> <%= __('betweencountries') %>. +

- - - + <%= __('thisproject') %> <%= __('opensource') %> <%= __('see') %> <%= __('datasources') %>).

- - - + <%= __('tip') %> ⟶

+
+

+
+
+
+

+ + + +

+

+ + + +

+
+
+
+
+ <%= __('likethisvisu') %> <%= __('loveyourfeedback') %>!
+ <%= __('foundbugs') %> <%= __('here') %>.
+

+ <%= __('co2signalprefix') %> CO2 Signal <%= __('co2signalsuffix') %>. +

+ +
+
+ + + + + + + Slack + +
-
-
- Like the visualization? We would love to hear your feedback!
- Found bugs or have ideas? Report them here.
-

- Check out how our CO2 Signal can help your devices and electric vehicle consume electricity at the right time. -

-
- -
+
+ +
+
+
 
- - - - - - - Slack - - -
-
- -
-
-
 
- Carbon intensity:
+ <%= __('carbonintensity') %>:
: -
gCO2eq/kWh ( % fossil fuels)
+
gCO2eq/kWh

- Electricity price (day-ahead): €/MWh + <%= __('electricityprice') %>: €/MWh
- Cross-border export:
+ <%= __('crossborderexport') %>:
→ : MW

- Carbon intensity of export:
+ <%= __('carbonintensityexport') %>:
gCO2eq/kWh
- of
- ()
+ <%= __('carbonintensity') %> <%= __('of') %> :
+
gCO2eq/kWh (source: IPCC 2014)

- utilizing of installed capacity
- ()
+ <%= __('of') %>
+ ()

- with a carbon intensity of
-
gCO2eq/kWh (source: IPCC 2014) + <%= __('ofinstalled') %> <%= __('capacity') %>
+ ()
- of
- ()
+ <%= __('carbonintensity') %>
+ :
gCO2eq/kWh

- with a carbon intensity of
- :
gCO2eq/kWh
+ <%= __('of') %>
+ () +
+
+
+
+
+ + + +
From 4a4b7b96aacb1cd40bb6bd2bc7840a1cd6a9e000 Mon Sep 17 00:00:00 2001 From: nikkozzblu Date: Mon, 6 Feb 2017 13:11:27 +0100 Subject: [PATCH 15/21] Fix rejected updates --- web/app/tooltip.js | 2 +- web/locales/en.json | 9 +- web/locales/fr.json | 8 +- web/public/css/styles.css | 2 +- web/views/pages/index.ejs | 199 +++++++++++++++++++------------------- 5 files changed, 111 insertions(+), 109 deletions(-) diff --git a/web/app/tooltip.js b/web/app/tooltip.js index 0869111a12..3816275187 100644 --- a/web/app/tooltip.js +++ b/web/app/tooltip.js @@ -98,7 +98,7 @@ exports.setupCountryTable = function (countryTable, countries, co2Colorbar, co2c var value = d.isStorage ? d.storage : d.production; var domain = d.isStorage ? totalPositive : totalPositive; - var domainName = d.isStorage ? ('electricity is stored using ' + d.mode) : ('electricity comes from ' + d.mode); + var domainName = d.isStorage ? ( lang['electricitystored'] +' ' + (d.text || d.mode)) : ( lang['electricityfrom'] + ' ' + (d.text || d.mode)); var isNull = !isFinite(value) || value == undefined; var productionProportion = !isNull ? Math.round(value / domain * 100) : '?'; diff --git a/web/locales/en.json b/web/locales/en.json index 0301197b7b..2ff6b1c0e5 100644 --- a/web/locales/en.json +++ b/web/locales/en.json @@ -38,8 +38,11 @@ "crossborderexport": "Cross-border export", "carbonintensityexport": "Carbon intensity of export", "of": "of", - "ofconsumption":"of", - "ofinstalled": "of installed", - "capacity": "capacity", + "ofconsumption": "of", + "ofinstalled": "of installed capacity", + "ofelectricity": "of", + "comesfrom": "electricity comes from", + "utilizing": "utilizing", + "withcarbonintensity": "with a carbon intensity of", "back": "back" } diff --git a/web/locales/fr.json b/web/locales/fr.json index 60e7da3398..da8ec56302 100644 --- a/web/locales/fr.json +++ b/web/locales/fr.json @@ -16,7 +16,7 @@ "visitoncomputer": "visitez ce site depuis un ordinateur", "thisshowsrealtime": "Cette carte indique", "whereelectricitycomesfrom": "de quelles sources provient votre électricité", - "and": "et", + "and": "et", "homuchco2": "quelle quantité de CO2", "wasemitted": "a été émise pour la produire", "takeinaccount": "Elle prend en compte les", @@ -39,7 +39,11 @@ "carbonintensityexport": "Intensité carbone de l export", "of":"de", "ofconsumption": "de la consomation", - "ofinstalled": "de la capacité installée en", + "ofinstalled": "de la capacité installée", + "ofelectricity": "de l électricité de", + "comesfrom": "est produite par", "capacity": " ", + "utilizing": "utilisant", + "withcarbonintensity": "avec une intensité carbone de", "back": "retour" } diff --git a/web/public/css/styles.css b/web/public/css/styles.css index 60845ab475..25a0a5e0e0 100644 --- a/web/public/css/styles.css +++ b/web/public/css/styles.css @@ -278,4 +278,4 @@ hr { .time-travel { display: none; -} +} \ No newline at end of file diff --git a/web/views/pages/index.ejs b/web/views/pages/index.ejs index ddfedc5d35..85d032e4f1 100644 --- a/web/views/pages/index.ejs +++ b/web/views/pages/index.ejs @@ -39,8 +39,8 @@ @@ -56,12 +56,10 @@ js.id = id; js.src = "https://platform.twitter.com/widgets.js"; fjs.parentNode.insertBefore(js, fjs); - t._e = []; t.ready = function(f) { t._e.push(f); }; - return t; }(document, "script", "twitter-wjs")); @@ -80,7 +78,6 @@ FB.AppEvents.logPageView('pageview'); <% } %> }; - (function(d, s, id){ var js, fjs = d.getElementsByTagName(s)[0]; if (d.getElementById(id)) {return;} @@ -100,31 +97,41 @@
<%= __('carbonintensity') %>
(gCO2eq/kWh)
-
-
-

- <%= __('maintitle') %> -

-

- -

-
-