From 9a0ee220550fc43b504d00fd50697c94414b33b3 Mon Sep 17 00:00:00 2001 From: Olivier Corradi Date: Sun, 24 Sep 2017 15:30:18 +0200 Subject: [PATCH] Add mockserver --- README.md | 43 ++++++++++++--------- docker-compose.yml | 6 +++ mockserver/Dockerfile | 4 ++ mockserver/public/v3/state | 1 + mockserver/update_state.py | 76 ++++++++++++++++++++++++++++++++++++++ web/app/dataservice.js | 20 ++++++---- web/app/main.js | 7 ++-- 7 files changed, 128 insertions(+), 29 deletions(-) create mode 100644 mockserver/Dockerfile create mode 100644 mockserver/public/v3/state create mode 100644 mockserver/update_state.py diff --git a/README.md b/README.md index 5c207cd0b8..2bb9e4e518 100644 --- a/README.md +++ b/README.md @@ -200,6 +200,25 @@ We use the [Natural Earth Data Cultural Vectors](http://www.naturalearthdata.com ## Contribute Want to help? Join us on slack at [http://slack.tmrow.co](http://slack.tmrow.co). +### Running locally + +To get started, clone or [fork](https://help.github.com/articles/fork-a-repo/) the repository, and install [Docker](https://docs.docker.com/engine/installation/). + +The frontend will need compiling. In order to do this, open a terminal and run +``` +docker-compose run --rm web npm run watch +``` +This will watch over source file changes, and recompile if needed. + +Now that the frontend is compiled, you can run the application by running the following command in a new terminal: +``` +docker-compose up --build +``` + +Head over to [http://localhost:8000/](http://localhost:8000/) and you should see the map! Note that the backend is responsible for calculation carbon emissions, so the map will be empty. + +Once you're done doing your changes, submit a [pull request](https://help.github.com/articles/using-pull-requests/) to get them integrated into the production version. + ### Adding a new country It is very simple to add a new country. The Electricity Map backend runs a list of so-called *parsers* every 5min. Those parsers are responsible to fetch the generation mix for a given country (check out the existing list in the [parsers](https://github.com/corradio/electricitymap/tree/master/parsers) directory, or look at the [work in progress](https://github.com/tmrowco/electricitymap/issues?q=is%3Aissue+is%3Aopen+label%3Aparser)). @@ -236,28 +255,16 @@ Storage values can be both positive (when storing energy) or negative (when the The parser can also return an array of objects if multiple time values can be fetched. The backend will automatically update past values properly. -For more info, check out the [example](https://github.com/corradio/electricitymap/tree/master/parsers/example.py) or browse existing [parsers](https://github.com/corradio/electricitymap/tree/master/parsers). - -### Frontend contributions +Once you're done, add your parser to the [zones.json](https://github.com/corradio/electricitymap/tree/master/config/zones.json) and [exchanges.json](https://github.com/corradio/electricitymap/tree/master/config/exchanges.json) configuration files. -To get started, clone or [fork](https://help.github.com/articles/fork-a-repo/) the repository, and install [Docker](https://docs.docker.com/engine/installation/). - -The frontend will need compiling. In order to do this, open a terminal and run -``` -docker-compose run --rm web npm run watch -``` -This will watch over source file changes, and recompile if needed. +For more info, check out the [example](https://github.com/corradio/electricitymap/tree/master/parsers/example.py) or browse existing [parsers](https://github.com/corradio/electricitymap/tree/master/parsers). -Now that the frontend is compiled, you can run the application (which will use our existing backend to pull data), by running the following command in a new terminal: +### Testing parsers locally +We've added a testing server locally. In order to test your parser, you can run ``` -docker-compose build --build-arg 'ELECTRICITYMAP_PUBLIC_TOKEN=' web -docker-compose up +PYTHONPATH=. python mockserver/update_state.py ``` -where `` should be replaced by a development token (contact the team on Slack in order to get one). - -Head over to [http://localhost:8000/](http://localhost:8000/) and you should see the map! - -Once you're done doing your changes, submit a [pull request](https://help.github.com/articles/using-pull-requests/) to get them integrated into the production version. +from the root directory, replacing `` by the zone identifier of the parser you want to test. This will fetch production and exchanges and assign it a random carbon intensity value. It should appear on the map as you refresh your local browser. ### Troubleshooting diff --git a/docker-compose.yml b/docker-compose.yml index f997df2e4c..e60731ccb9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,16 @@ version: '2' services: + mockserver: + build: mockserver + command: http-server -p 9000 --cors -c-1 + ports: ['9000:9000'] + volumes: ['./mockserver/public:/home/public'] web: build: context: . dockerfile: web/Dockerfile command: npm run server-dev + depends_on: ['mockserver'] environment: - NODE_ENV=development ports: ['8000:8000'] diff --git a/mockserver/Dockerfile b/mockserver/Dockerfile new file mode 100644 index 0000000000..16830dbb13 --- /dev/null +++ b/mockserver/Dockerfile @@ -0,0 +1,4 @@ +FROM node:8.5.0 +WORKDIR /home +EXPOSE 9000 +RUN npm install http-server -g diff --git a/mockserver/public/v3/state b/mockserver/public/v3/state new file mode 100644 index 0000000000..b51126c611 --- /dev/null +++ b/mockserver/public/v3/state @@ -0,0 +1 @@ +{"data": {"exchanges": {"BR-S->UY": {"sortedCountryCodes": "BR-S->UY", "netFlow": 0.2, "datetime": "2017-09-24T10:17:00-03:00", "co2intensity": 475.7995468281372, "source": "ons.org.br"}, "BR-CS->BR-S": {"sortedCountryCodes": "BR-CS->BR-S", "netFlow": 3291.799, "datetime": "2017-09-24T10:17:00-03:00", "co2intensity": null, "source": "ons.org.br"}}, "datetime": "2017-09-24T10:17:00-03:00", "countries": {"UY": {"exchange": {"BR-S": 0.2}}, "BR-S": {"countryCode": "BR-S", "exchange": {"UY": -0.2, "BR-CS": 3291.799}, "storage": {"hydro": null}, "datetime": "2017-09-24T10:17:00-03:00", "source": "ons.org.br", "production": {"nuclear": 0.0, "hydro": 3366.93042, "solar": 0.0, "wind": 576.1216, "unknown": 1304.81372}, "co2intensity": 475.7995468281372}, "BR-CS": {"exchange": {"BR-S": -3291.799}}}}} \ No newline at end of file diff --git a/mockserver/update_state.py b/mockserver/update_state.py new file mode 100644 index 0000000000..8da977f435 --- /dev/null +++ b/mockserver/update_state.py @@ -0,0 +1,76 @@ +# This script should be run from the root directory +import arrow, importlib, json, pprint, sys +from random import random +pp = pprint.PrettyPrinter(indent=2) + +# Read parser import list from config jsons +zones_config = json.load(open('config/zones.json')) +exchanges_config = json.load(open('config/exchanges.json')) + +# Read zone_name from commandline +if not len(sys.argv) > 1: + raise Exception('Missing argument ') +zone_name = sys.argv[1] +zone_config = zones_config[zone_name] + +# Find parsers +production_parser = zone_config['parsers']['production'] +exchange_parser_keys = [] +for k in exchanges_config.keys(): + zones = k.split('->') + if zone_name in zones: exchange_parser_keys.append(k) + +# Import / run production parser +print 'Finding and executing %s production parser %s..' % (zone_name, production_parser) +mod_name, fun_name = production_parser.split('.') +mod = importlib.import_module('parsers.%s' % mod_name) +production = getattr(mod, fun_name)(zone_name) +pp.pprint(production) + +# Import / run exchange parser(s) +exchanges = [] +for k in exchange_parser_keys: + exchange_parser = exchanges_config[k]['parsers']['exchange'] + print 'Finding and executing %s exchange parser %s..' % (k, exchange_parser) + mod_name, fun_name = exchange_parser.split('.') + mod = importlib.import_module('parsers.%s' % mod_name) + sorted_zone_names = sorted(k.split('->')) + exchange = getattr(mod, fun_name)(sorted_zone_names[0], sorted_zone_names[1]) + exchanges.append(exchange) + pp.pprint(exchange) + +# Load and update state +print 'Updating and writing state..' +with open('mockserver/public/v3/state', 'r') as f: + obj = json.load(f)['data'] + obj['countries'][zone_name] = {} + # Update production + obj['countries'][zone_name] = production + production['datetime'] = arrow.get(production['datetime']).isoformat() + # Set random co2 value + production['co2intensity'] = random() * 500 + # Update exchanges + for e in exchanges: + zones = e['sortedCountryCodes'].split('->') + e['datetime'] = arrow.get(e['datetime']).isoformat() + obj['exchanges'][e['sortedCountryCodes']] = e.copy() + origin_zone = zones[0] if e['netFlow'] >= 0 else zones[1] + obj['exchanges'][e['sortedCountryCodes']]['co2intensity'] = \ + obj['countries'].get(origin_zone, {}).get('co2intensity') + for z in zones: + other_zone = zones[(zones.index(z) + 1) % 2] + if not z in obj['countries']: + obj['countries'][z] = {} + if not 'exchange' in obj['countries'][z]: + obj['countries'][z]['exchange'] = {} + obj['countries'][z]['exchange'][other_zone] = e['netFlow'] + print z, other_zone + if z == zones[0]: + obj['countries'][z]['exchange'][other_zone] *= -1 + + # Set state datetime + obj['datetime'] = production['datetime'] + # Save +with open('mockserver/public/v3/state', 'w') as f: + json.dump({'data': obj}, f) +print '..done' diff --git a/web/app/dataservice.js b/web/app/dataservice.js index eb9d0945ef..ae74d388f6 100644 --- a/web/app/dataservice.js +++ b/web/app/dataservice.js @@ -7,14 +7,18 @@ var moment = require('moment'); // API function protectedJsonRequest(endpoint, path, callback) { - var t = new Date().getTime(); - var md = forge.md.sha256.create(); - var s = md.update(ELECTRICITYMAP_PUBLIC_TOKEN + path + t).digest().toHex(); - return d3.json(endpoint + path) - .header('electricitymap-token', Cookies.get('electricitymap-token')) - .header('x-request-timestamp', t) - .header('x-signature', s) - .get(null, callback); + if (isLocalhost) { + return d3.json(endpoint + path, callback); + } else { + var t = new Date().getTime(); + var md = forge.md.sha256.create(); + var s = md.update(ELECTRICITYMAP_PUBLIC_TOKEN + path + t).digest().toHex(); + return d3.json(endpoint + path) + .header('electricitymap-token', Cookies.get('electricitymap-token')) + .header('x-request-timestamp', t) + .header('x-signature', s) + .get(null, callback); + } } // GFS Parameters diff --git a/web/app/main.js b/web/app/main.js index 727fcc8138..fec5155927 100644 --- a/web/app/main.js +++ b/web/app/main.js @@ -53,9 +53,12 @@ function replaceHistoryState(key, value) { history.replaceState(historyState, '', getHistoryStateURL()); } +// Global window variables +isLocalhost = window.location.href.indexOf('electricitymap') == -1; + // Global State var selectedCountryCode; -var useRemoteEndpoint = true; +var useRemoteEndpoint = isLocalhost ? false : true; var customDate; var currentMoment; var colorBlindModeEnabled = false; @@ -102,10 +105,8 @@ function parseQueryString(querystring) { } parseQueryString(location.search); - // Computed State var colorBlindModeEnabled = Cookies.get('colorBlindModeEnabled') == 'true' || false; -var isLocalhost = window.location.href.indexOf('electricitymap') == -1; var isEmbedded = window.top !== window.self; var REMOTE_ENDPOINT = 'https://api.electricitymap.org'; var LOCAL_ENDPOINT = 'http://localhost:9000';