diff --git a/server/src/app.py b/server/src/app.py index 76a2743c4..fc0d79900 100644 --- a/server/src/app.py +++ b/server/src/app.py @@ -4,7 +4,7 @@ from sanic_gzip import Compress from threading import Timer from datetime import datetime -from multiprocessing import cpu_count +from multiprocessing import cpu_count, Process from services.pinClusterService import PinClusterService from services.heatmapService import HeatmapService @@ -15,6 +15,7 @@ from services.dataService import DataService from utils.sanic import add_performance_header +from utils.picklebase import pb from config import config app = Sanic(__name__) @@ -166,6 +167,9 @@ async def test_multiple_workers(request): if workers == -1: workers = max(cpu_count() // 2, 1) + if pb.enabled: + Process(target=pb.populate).start() + app.run( port=port, host=host, diff --git a/server/src/services/dataService.py b/server/src/services/dataService.py index 17f9c73a5..fe7149570 100644 --- a/server/src/services/dataService.py +++ b/server/src/services/dataService.py @@ -19,7 +19,7 @@ def standardFilters(self, ''' Generates filters for dates, request types, and ncs. ''' - if pb.enabled: + if pb.available(): return { 'startDate': startDate, 'endDate': endDate, @@ -44,7 +44,7 @@ def comparisonFilters(self, ''' Generates filters for the comparison endpoints. ''' - if pb.enabled: + if pb.available(): return { 'startDate': startDate, 'endDate': endDate, @@ -94,7 +94,7 @@ def query(self, fields, filters, table=default_table): if not fields or not filters: return {'Error': 'fields and filters are required'} - if pb.enabled: + if pb.available(): return pb.query(table, fields, filters) fields = (', ').join(fields) diff --git a/server/src/utils/picklebase/__init__.py b/server/src/utils/picklebase/__init__.py index a0aea54f3..cfdd9c32d 100644 --- a/server/src/utils/picklebase/__init__.py +++ b/server/src/utils/picklebase/__init__.py @@ -1,27 +1,35 @@ import os from .query import query as query_pb from .populate import populate as populate_pb +from .data_access import clear_data, set_ready, check_ready class PickleBase(object): def __init__(self): - self.enabled = False + self.enabled = int(os.environ.get('PICKLEBASE', 0)) == 1 + self.ready = False + + def available(self): + if not self.enabled: + return False + if self.ready: + return True + self.ready = check_ready() + return self.ready def populate(self): - populate_pb() + try: + clear_data() + populate_pb() + set_ready() + print('PICKLEBASE IS READY') + except Exception as e: + self.enabled = False + print('FAILED TO POPULATE PICKLEBASE') + print(e) def query(self, table, fields, filters): return query_pb(table, fields, filters) pb = PickleBase() - - -if int(os.environ.get('PICKLEBASE', 0)) == 1: - print('PICKLEBASE ENABLED') - try: - pb.populate() - pb.enabled = True - except Exception as e: - print('FAILED TO POPULATE PICKLEBASE') - print(e) diff --git a/server/src/utils/picklebase/data_access.py b/server/src/utils/picklebase/data_access.py index e9d539ead..b5a0cb8e4 100644 --- a/server/src/utils/picklebase/data_access.py +++ b/server/src/utils/picklebase/data_access.py @@ -6,6 +6,7 @@ TMP_DIR = os.environ.get('TMP_DIR', os.getcwd()) DATA_DIR = os.path.join(TMP_DIR, 'static/picklebase') +READY_FILE = os.path.join(DATA_DIR, 'ready') def clear_data(): @@ -58,3 +59,12 @@ def load_meta(table): path = meta_path(table) with open(path, 'r') as f: return json.load(f) + + +def set_ready(): + with open(READY_FILE, 'w'): + pass + + +def check_ready(): + return os.path.isfile(READY_FILE) diff --git a/server/src/utils/picklebase/populate.py b/server/src/utils/picklebase/populate.py index d631ea5ae..90f1218a9 100644 --- a/server/src/utils/picklebase/populate.py +++ b/server/src/utils/picklebase/populate.py @@ -1,6 +1,5 @@ import os from utils.database import db -from .data_access import clear_data from .create_table import create_table @@ -52,6 +51,5 @@ def optimize(batch): def populate(): - clear_data() create_map_table() create_vis_table() diff --git a/server/src/utils/picklebase/query.py b/server/src/utils/picklebase/query.py index b5b384d19..832c62ec5 100644 --- a/server/src/utils/picklebase/query.py +++ b/server/src/utils/picklebase/query.py @@ -12,6 +12,8 @@ def get_batch_nums(table, startDate, endDate): def query(table, fields, filters): + print('QUERYING PICKLEBASE') + startDate = pd.to_datetime(filters['startDate']) endDate = pd.to_datetime(filters['endDate']) requestTypes = filters['requestTypes']