Add mockserver

README.md

@@ -200,6 +200,25 @@ We use the [Natural Earth Data Cultural Vectors](http://www.naturalearthdata.com
 ## Contribute
 Want to help? Join us on slack at [http://slack.tmrow.co](http://slack.tmrow.co).
 
+### Running locally
+
+To get started, clone or [fork](https://help.github.com/articles/fork-a-repo/) the repository, and install [Docker](https://docs.docker.com/engine/installation/).
+
+The frontend will need compiling. In order to do this, open a terminal and run
+```
+docker-compose run --rm web npm run watch
+```
+This will watch over source file changes, and recompile if needed.
+
+Now that the frontend is compiled, you can run the application by running the following command in a new terminal:
+```
+docker-compose up --build
+```
+
+Head over to [http://localhost:8000/](http://localhost:8000/) and you should see the map! Note that the backend is responsible for calculation carbon emissions, so the map will be empty.
+
+Once you're done doing your changes, submit a [pull request](https://help.github.com/articles/using-pull-requests/) to get them integrated into the production version.
+
 ### Adding a new country
 It is very simple to add a new country. The Electricity Map backend runs a list of so-called *parsers* every 5min. Those parsers are responsible to fetch the generation mix for a given country (check out the existing list in the [parsers](https://github.com/corradio/electricitymap/tree/master/parsers) directory, or look at the [work in progress](https://github.com/tmrowco/electricitymap/issues?q=is%3Aissue+is%3Aopen+label%3Aparser)).
 
@@ -236,28 +255,16 @@ Storage values can be both positive (when storing energy) or negative (when the
 
 The parser can also return an array of objects if multiple time values can be fetched. The backend will automatically update past values properly.
 
-For more info, check out the [example](https://github.com/corradio/electricitymap/tree/master/parsers/example.py) or browse existing [parsers](https://github.com/corradio/electricitymap/tree/master/parsers).
-
-### Frontend contributions
+Once you're done, add your parser to the [zones.json](https://github.com/corradio/electricitymap/tree/master/config/zones.json) and [exchanges.json](https://github.com/corradio/electricitymap/tree/master/config/exchanges.json) configuration files.
 
-To get started, clone or [fork](https://help.github.com/articles/fork-a-repo/) the repository, and install [Docker](https://docs.docker.com/engine/installation/).
-
-The frontend will need compiling. In order to do this, open a terminal and run
-```
-docker-compose run --rm web npm run watch
-```
-This will watch over source file changes, and recompile if needed.
+For more info, check out the [example](https://github.com/corradio/electricitymap/tree/master/parsers/example.py) or browse existing [parsers](https://github.com/corradio/electricitymap/tree/master/parsers).
 
-Now that the frontend is compiled, you can run the application (which will use our existing backend to pull data), by running the following command in a new terminal:
+### Testing parsers locally
+We've added a testing server locally. In order to test your parser, you can run
 ```
-docker-compose build --build-arg 'ELECTRICITYMAP_PUBLIC_TOKEN=<token>' web
-docker-compose up
+PYTHONPATH=. python mockserver/update_state.py <zone_name>
 ```
-where `<token>` should be replaced by a development token (contact the team on Slack in order to get one).
-
-Head over to [http://localhost:8000/](http://localhost:8000/) and you should see the map!
-
-Once you're done doing your changes, submit a [pull request](https://help.github.com/articles/using-pull-requests/) to get them integrated into the production version.
+from the root directory, replacing `<zone_name>` by the zone identifier of the parser you want to test. This will fetch production and exchanges and assign it a random carbon intensity value. It should appear on the map as you refresh your local browser.
 
 ### Troubleshooting
 

docker-compose.yml

@@ -1,10 +1,16 @@
 version: '2'
 services:
+  mockserver:
+    build: mockserver
+    command: http-server -p 9000 --cors -c-1
+    ports: ['9000:9000']
+    volumes: ['./mockserver/public:/home/public']
   web:
     build:
       context: .
       dockerfile: web/Dockerfile
     command: npm run server-dev
+    depends_on: ['mockserver']
     environment:
       - NODE_ENV=development
     ports: ['8000:8000']

mockserver/Dockerfile

@@ -0,0 +1,4 @@
+FROM node:8.5.0
+WORKDIR /home
+EXPOSE 9000
+RUN npm install http-server -g

mockserver/public/v3/state

@@ -0,0 +1 @@
+{"data": {"exchanges": {"BR-S->UY": {"sortedCountryCodes": "BR-S->UY", "netFlow": 0.2, "datetime": "2017-09-24T10:17:00-03:00", "co2intensity": 475.7995468281372, "source": "ons.org.br"}, "BR-CS->BR-S": {"sortedCountryCodes": "BR-CS->BR-S", "netFlow": 3291.799, "datetime": "2017-09-24T10:17:00-03:00", "co2intensity": null, "source": "ons.org.br"}}, "datetime": "2017-09-24T10:17:00-03:00", "countries": {"UY": {"exchange": {"BR-S": 0.2}}, "BR-S": {"countryCode": "BR-S", "exchange": {"UY": -0.2, "BR-CS": 3291.799}, "storage": {"hydro": null}, "datetime": "2017-09-24T10:17:00-03:00", "source": "ons.org.br", "production": {"nuclear": 0.0, "hydro": 3366.93042, "solar": 0.0, "wind": 576.1216, "unknown": 1304.81372}, "co2intensity": 475.7995468281372}, "BR-CS": {"exchange": {"BR-S": -3291.799}}}}}

mockserver/update_state.py

@@ -0,0 +1,76 @@
+# This script should be run from the root directory
+import arrow, importlib, json, pprint, sys
+from random import random
+pp = pprint.PrettyPrinter(indent=2)
+
+# Read parser import list from config jsons
+zones_config = json.load(open('config/zones.json'))
+exchanges_config = json.load(open('config/exchanges.json'))
+
+# Read zone_name from commandline
+if not len(sys.argv) > 1:
+    raise Exception('Missing argument <zone_name>')
+zone_name = sys.argv[1]
+zone_config = zones_config[zone_name]
+
+# Find parsers
+production_parser = zone_config['parsers']['production']
+exchange_parser_keys = []
+for k in exchanges_config.keys():
+    zones = k.split('->')
+    if zone_name in zones: exchange_parser_keys.append(k)
+
+# Import / run production parser
+print 'Finding and executing %s production parser %s..' % (zone_name, production_parser)
+mod_name, fun_name = production_parser.split('.')
+mod = importlib.import_module('parsers.%s' % mod_name)
+production = getattr(mod, fun_name)(zone_name)
+pp.pprint(production)
+
+# Import / run exchange parser(s)
+exchanges = []
+for k in exchange_parser_keys:
+    exchange_parser = exchanges_config[k]['parsers']['exchange']
+    print 'Finding and executing %s exchange parser %s..' % (k, exchange_parser)
+    mod_name, fun_name = exchange_parser.split('.')
+    mod = importlib.import_module('parsers.%s' % mod_name)
+    sorted_zone_names = sorted(k.split('->'))
+    exchange = getattr(mod, fun_name)(sorted_zone_names[0], sorted_zone_names[1])
+    exchanges.append(exchange)
+    pp.pprint(exchange)
+
+# Load and update state
+print 'Updating and writing state..'
+with open('mockserver/public/v3/state', 'r') as f:
+    obj = json.load(f)['data']
+    obj['countries'][zone_name] = {}
+    # Update production
+    obj['countries'][zone_name] = production
+    production['datetime'] = arrow.get(production['datetime']).isoformat()
+    # Set random co2 value
+    production['co2intensity'] = random() * 500
+    # Update exchanges
+    for e in exchanges:
+        zones = e['sortedCountryCodes'].split('->')
+        e['datetime'] = arrow.get(e['datetime']).isoformat()
+        obj['exchanges'][e['sortedCountryCodes']] = e.copy()
+        origin_zone = zones[0] if e['netFlow'] >= 0 else zones[1]
+        obj['exchanges'][e['sortedCountryCodes']]['co2intensity'] = \
+            obj['countries'].get(origin_zone, {}).get('co2intensity')
+        for z in zones:
+            other_zone = zones[(zones.index(z) + 1) % 2]
+            if not z in obj['countries']:
+                obj['countries'][z] = {}
+            if not 'exchange' in obj['countries'][z]:
+                obj['countries'][z]['exchange'] = {}
+            obj['countries'][z]['exchange'][other_zone] = e['netFlow']
+            print z, other_zone
+            if z == zones[0]:
+                obj['countries'][z]['exchange'][other_zone] *= -1
+
+    # Set state datetime
+    obj['datetime'] = production['datetime']
+    # Save
+with open('mockserver/public/v3/state', 'w') as f:
+    json.dump({'data': obj}, f)
+print '..done'

web/app/dataservice.js

@@ -7,14 +7,18 @@ var moment = require('moment');
 
 // API
 function protectedJsonRequest(endpoint, path, callback) {
-    var t = new Date().getTime();
-    var md = forge.md.sha256.create();
-    var s = md.update(ELECTRICITYMAP_PUBLIC_TOKEN + path + t).digest().toHex();
-    return d3.json(endpoint + path)
-        .header('electricitymap-token', Cookies.get('electricitymap-token'))
-        .header('x-request-timestamp', t)
-        .header('x-signature', s)
-        .get(null, callback);
+    if (isLocalhost) {
+        return d3.json(endpoint + path, callback);
+    } else {
+        var t = new Date().getTime();
+        var md = forge.md.sha256.create();
+        var s = md.update(ELECTRICITYMAP_PUBLIC_TOKEN + path + t).digest().toHex();
+        return d3.json(endpoint + path)
+            .header('electricitymap-token', Cookies.get('electricitymap-token'))
+            .header('x-request-timestamp', t)
+            .header('x-signature', s)
+            .get(null, callback);
+    }
 }
 
 // GFS Parameters

web/app/main.js

@@ -53,9 +53,12 @@ function replaceHistoryState(key, value) {
     history.replaceState(historyState, '', getHistoryStateURL());
 }
 
+// Global window variables
+isLocalhost = window.location.href.indexOf('electricitymap') == -1;
+
 // Global State
 var selectedCountryCode;
-var useRemoteEndpoint = true;
+var useRemoteEndpoint = isLocalhost ? false : true;
 var customDate;
 var currentMoment;
 var colorBlindModeEnabled = false;
@@ -102,10 +105,8 @@ function parseQueryString(querystring) {
 }
 parseQueryString(location.search);
 
-
 // Computed State
 var colorBlindModeEnabled = Cookies.get('colorBlindModeEnabled') == 'true' || false;
-var isLocalhost = window.location.href.indexOf('electricitymap') == -1;
 var isEmbedded = window.top !== window.self;
 var REMOTE_ENDPOINT = 'https://api.electricitymap.org';
 var LOCAL_ENDPOINT = 'http://localhost:9000';