Pickle base fix

server/src/app.py

@@ -4,7 +4,7 @@
 from sanic_gzip import Compress
 from threading import Timer
 from datetime import datetime
-from multiprocessing import cpu_count
+from multiprocessing import cpu_count, Process
 
 from services.pinClusterService import PinClusterService
 from services.heatmapService import HeatmapService
@@ -15,6 +15,7 @@
 from services.dataService import DataService
 
 from utils.sanic import add_performance_header
+from utils.picklebase import pb
 from config import config
 
 app = Sanic(__name__)
@@ -166,6 +167,9 @@ async def test_multiple_workers(request):
     if workers == -1:
         workers = max(cpu_count() // 2, 1)
 
+    if pb.enabled:
+        Process(target=pb.populate).start()
+
     app.run(
         port=port,
         host=host,

server/src/services/dataService.py

@@ -19,7 +19,7 @@ def standardFilters(self,
         '''
         Generates filters for dates, request types, and ncs.
         '''
-        if pb.enabled:
+        if pb.available():
             return {
                 'startDate': startDate,
                 'endDate': endDate,
@@ -44,7 +44,7 @@ def comparisonFilters(self,
         '''
         Generates filters for the comparison endpoints.
         '''
-        if pb.enabled:
+        if pb.available():
             return {
                 'startDate': startDate,
                 'endDate': endDate,
@@ -94,7 +94,7 @@ def query(self, fields, filters, table=default_table):
         if not fields or not filters:
             return {'Error': 'fields and filters are required'}
 
-        if pb.enabled:
+        if pb.available():
             return pb.query(table, fields, filters)
 
         fields = (', ').join(fields)

server/src/utils/picklebase/__init__.py

@@ -1,27 +1,35 @@
 import os
 from .query import query as query_pb
 from .populate import populate as populate_pb
+from .data_access import clear_data, set_ready, check_ready
 
 
 class PickleBase(object):
     def __init__(self):
-        self.enabled = False
+        self.enabled = int(os.environ.get('PICKLEBASE', 0)) == 1
+        self.ready = False
+
+    def available(self):
+        if not self.enabled:
+            return False
+        if self.ready:
+            return True
+        self.ready = check_ready()
+        return self.ready
 
     def populate(self):
-        populate_pb()
+        try:
+            clear_data()
+            populate_pb()
+            set_ready()
+            print('PICKLEBASE IS READY')
+        except Exception as e:
+            self.enabled = False
+            print('FAILED TO POPULATE PICKLEBASE')
+            print(e)
 
     def query(self, table, fields, filters):
         return query_pb(table, fields, filters)
 
 
 pb = PickleBase()
-
-
-if int(os.environ.get('PICKLEBASE', 0)) == 1:
-    print('PICKLEBASE ENABLED')
-    try:
-        pb.populate()
-        pb.enabled = True
-    except Exception as e:
-        print('FAILED TO POPULATE PICKLEBASE')
-        print(e)

server/src/utils/picklebase/data_access.py

@@ -6,6 +6,7 @@
 
 TMP_DIR = os.environ.get('TMP_DIR', os.getcwd())
 DATA_DIR = os.path.join(TMP_DIR, 'static/picklebase')
+READY_FILE = os.path.join(DATA_DIR, 'ready')
 
 
 def clear_data():
@@ -58,3 +59,12 @@ def load_meta(table):
     path = meta_path(table)
     with open(path, 'r') as f:
         return json.load(f)
+
+
+def set_ready():
+    with open(READY_FILE, 'w'):
+        pass
+
+
+def check_ready():
+    return os.path.isfile(READY_FILE)

server/src/utils/picklebase/populate.py

@@ -1,6 +1,5 @@
 import os
 from utils.database import db
-from .data_access import clear_data
 from .create_table import create_table
 
 
@@ -52,6 +51,5 @@ def optimize(batch):
 
 
 def populate():
-    clear_data()
     create_map_table()
     create_vis_table()

server/src/utils/picklebase/query.py

@@ -12,6 +12,8 @@ def get_batch_nums(table, startDate, endDate):
 
 
 def query(table, fields, filters):
+    print('QUERYING PICKLEBASE')
+
     startDate = pd.to_datetime(filters['startDate'])
     endDate = pd.to_datetime(filters['endDate'])
     requestTypes = filters['requestTypes']