From 6495538fb0650c5a9b6cac5053af393ce544b47b Mon Sep 17 00:00:00 2001
From: Olivier Corradi <olivier.corradi@gmail.com>
Date: Tue, 8 Nov 2016 12:45:56 +0100
Subject: [PATCH] Fixes #112

---
 docker-compose.yml      |  1 +
 feeder/feeder.py        | 12 ++++++++++--
 feeder/requirements.txt |  1 +
 production.yaml         |  1 +
 4 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 61cbc9dfb1..d8841c09c9 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -19,6 +19,7 @@ services:
     environment:
       - 'ENTSOE_TOKEN=${ENTSOE_TOKEN}'
       - ENV=development
+      - MEMCACHED_HOST=memcached
       - 'MONGO_URL=mongodb://mongo:27017/electricity'
     volumes:
       - './api/data:/home/data'
diff --git a/feeder/feeder.py b/feeder/feeder.py
index 9cff698fe5..a9dc5fc558 100644
--- a/feeder/feeder.py
+++ b/feeder/feeder.py
@@ -1,5 +1,6 @@
 import arrow
 import glob
+from pymemcache.client.base import Client
 import pymongo
 import logging, os, schedule, time
 import requests
@@ -41,11 +42,16 @@ def import_country(country_code):
     logging.getLogger('statsd').addHandler(logging.StreamHandler())
 else: logger.addHandler(logging.StreamHandler())
 
-
+# Set up database
 client = pymongo.MongoClient(os.environ.get('MONGO_URL', 'mongodb://localhost:27017'))
 db = client['electricity']
 col = db['realtime']
 
+# Set up memcached
+MEMCACHED_HOST = os.environ.get('MEMCACHED_HOST', None)
+cache = Client((MEMCACHED_HOST, 11211))
+
+# Set up requests
 session = requests.session()
 
 def fetch_countries():
@@ -59,7 +65,9 @@ def fetch_countries():
                     print obj['datetime'], arrow.now()
                     raise Exception("Data from %s can't be in the future" % parser)
                 logging.info('INSERT %s' % obj)
-                try: col.insert_one(obj)
+                try:
+                    col.insert_one(obj)
+                    cache.delete('production')
                 except pymongo.errors.DuplicateKeyError:
                     # (datetime, countryCode) does already exist. Don't raise.
                     # Note: with this design, the oldest record stays.
diff --git a/feeder/requirements.txt b/feeder/requirements.txt
index 75b08bed1c..6c72734cfb 100644
--- a/feeder/requirements.txt
+++ b/feeder/requirements.txt
@@ -3,6 +3,7 @@ beautifulsoup4==4.5.1
 Cython==0.23.4
 pandas==0.16.2
 pygrib==2.0.1
+pymemcache==1.4.0
 pymongo==3.2.2
 requests==2.10.0
 schedule==0.3.2
diff --git a/production.yaml b/production.yaml
index 7da4da03a6..79fdfe5faa 100644
--- a/production.yaml
+++ b/production.yaml
@@ -17,6 +17,7 @@ services:
     environment:
      - 'ENTSOE_TOKEN=${ENTSOE_TOKEN}'
      - ENV=production
+     - MEMCACHED_HOST=memcached
      - MONGO_URL=mongodb://mongo:27017
      - STATSD_HOST=grafana-graphite-statsd.infrastructure_default
     env_file: ./mailgun.env