Fixes electricitymaps#112

docker-compose.yml

@@ -19,6 +19,7 @@ services:
     environment:
       - 'ENTSOE_TOKEN=${ENTSOE_TOKEN}'
       - ENV=development
+      - MEMCACHED_HOST=memcached
       - 'MONGO_URL=mongodb://mongo:27017/electricity'
     volumes:
       - './api/data:/home/data'

feeder/feeder.py

@@ -1,5 +1,6 @@
 import arrow
 import glob
+from pymemcache.client.base import Client
 import pymongo
 import logging, os, schedule, time
 import requests
@@ -41,11 +42,16 @@ def import_country(country_code):
     logging.getLogger('statsd').addHandler(logging.StreamHandler())
 else: logger.addHandler(logging.StreamHandler())
 
-
+# Set up database
 client = pymongo.MongoClient(os.environ.get('MONGO_URL', 'mongodb://localhost:27017'))
 db = client['electricity']
 col = db['realtime']
 
+# Set up memcached
+MEMCACHED_HOST = os.environ.get('MEMCACHED_HOST', None)
+cache = Client((MEMCACHED_HOST, 11211))
+
+# Set up requests
 session = requests.session()
 
 def fetch_countries():
@@ -59,7 +65,9 @@ def fetch_countries():
                     print obj['datetime'], arrow.now()
                     raise Exception("Data from %s can't be in the future" % parser)
                 logging.info('INSERT %s' % obj)
-                try: col.insert_one(obj)
+                try:
+                    col.insert_one(obj)
+                    cache.delete('production')
                 except pymongo.errors.DuplicateKeyError:
                     # (datetime, countryCode) does already exist. Don't raise.
                     # Note: with this design, the oldest record stays.

feeder/requirements.txt

@@ -3,6 +3,7 @@ beautifulsoup4==4.5.1
 Cython==0.23.4
 pandas==0.16.2
 pygrib==2.0.1
+pymemcache==1.4.0
 pymongo==3.2.2
 requests==2.10.0
 schedule==0.3.2

production.yaml

@@ -17,6 +17,7 @@ services:
     environment:
      - 'ENTSOE_TOKEN=${ENTSOE_TOKEN}'
      - ENV=production
+     - MEMCACHED_HOST=memcached
      - MONGO_URL=mongodb://mongo:27017
      - STATSD_HOST=grafana-graphite-statsd.infrastructure_default
     env_file: ./mailgun.env