From 4bccd4c744ef8bfc8b399647ba84e3bff301fb18 Mon Sep 17 00:00:00 2001 From: Jake Mensch Date: Thu, 21 May 2020 15:33:58 -0700 Subject: [PATCH 1/2] populating picklebase in separate process --- server/src/app.py | 6 +++- server/src/services/dataService.py | 6 ++-- server/src/utils/picklebase/__init__.py | 32 ++++++++++++++-------- server/src/utils/picklebase/data_access.py | 10 +++++++ server/src/utils/picklebase/populate.py | 2 -- server/src/utils/picklebase/query.py | 2 ++ 6 files changed, 40 insertions(+), 18 deletions(-) diff --git a/server/src/app.py b/server/src/app.py index 76a2743c4..fc0d79900 100644 --- a/server/src/app.py +++ b/server/src/app.py @@ -4,7 +4,7 @@ from sanic_gzip import Compress from threading import Timer from datetime import datetime -from multiprocessing import cpu_count +from multiprocessing import cpu_count, Process from services.pinClusterService import PinClusterService from services.heatmapService import HeatmapService @@ -15,6 +15,7 @@ from services.dataService import DataService from utils.sanic import add_performance_header +from utils.picklebase import pb from config import config app = Sanic(__name__) @@ -166,6 +167,9 @@ async def test_multiple_workers(request): if workers == -1: workers = max(cpu_count() // 2, 1) + if pb.enabled: + Process(target=pb.populate).start() + app.run( port=port, host=host, diff --git a/server/src/services/dataService.py b/server/src/services/dataService.py index 17f9c73a5..fe7149570 100644 --- a/server/src/services/dataService.py +++ b/server/src/services/dataService.py @@ -19,7 +19,7 @@ def standardFilters(self, ''' Generates filters for dates, request types, and ncs. ''' - if pb.enabled: + if pb.available(): return { 'startDate': startDate, 'endDate': endDate, @@ -44,7 +44,7 @@ def comparisonFilters(self, ''' Generates filters for the comparison endpoints. ''' - if pb.enabled: + if pb.available(): return { 'startDate': startDate, 'endDate': endDate, @@ -94,7 +94,7 @@ def query(self, fields, filters, table=default_table): if not fields or not filters: return {'Error': 'fields and filters are required'} - if pb.enabled: + if pb.available(): return pb.query(table, fields, filters) fields = (', ').join(fields) diff --git a/server/src/utils/picklebase/__init__.py b/server/src/utils/picklebase/__init__.py index a0aea54f3..cfdd9c32d 100644 --- a/server/src/utils/picklebase/__init__.py +++ b/server/src/utils/picklebase/__init__.py @@ -1,27 +1,35 @@ import os from .query import query as query_pb from .populate import populate as populate_pb +from .data_access import clear_data, set_ready, check_ready class PickleBase(object): def __init__(self): - self.enabled = False + self.enabled = int(os.environ.get('PICKLEBASE', 0)) == 1 + self.ready = False + + def available(self): + if not self.enabled: + return False + if self.ready: + return True + self.ready = check_ready() + return self.ready def populate(self): - populate_pb() + try: + clear_data() + populate_pb() + set_ready() + print('PICKLEBASE IS READY') + except Exception as e: + self.enabled = False + print('FAILED TO POPULATE PICKLEBASE') + print(e) def query(self, table, fields, filters): return query_pb(table, fields, filters) pb = PickleBase() - - -if int(os.environ.get('PICKLEBASE', 0)) == 1: - print('PICKLEBASE ENABLED') - try: - pb.populate() - pb.enabled = True - except Exception as e: - print('FAILED TO POPULATE PICKLEBASE') - print(e) diff --git a/server/src/utils/picklebase/data_access.py b/server/src/utils/picklebase/data_access.py index e9d539ead..96fad4c24 100644 --- a/server/src/utils/picklebase/data_access.py +++ b/server/src/utils/picklebase/data_access.py @@ -6,6 +6,7 @@ TMP_DIR = os.environ.get('TMP_DIR', os.getcwd()) DATA_DIR = os.path.join(TMP_DIR, 'static/picklebase') +READY_FILE = os.path.join(DATA_DIR, 'ready') def clear_data(): @@ -58,3 +59,12 @@ def load_meta(table): path = meta_path(table) with open(path, 'r') as f: return json.load(f) + + +def set_ready(): + with open(READY_FILE, 'w') as f: + pass + + +def check_ready(): + return os.path.isfile(READY_FILE) diff --git a/server/src/utils/picklebase/populate.py b/server/src/utils/picklebase/populate.py index d631ea5ae..90f1218a9 100644 --- a/server/src/utils/picklebase/populate.py +++ b/server/src/utils/picklebase/populate.py @@ -1,6 +1,5 @@ import os from utils.database import db -from .data_access import clear_data from .create_table import create_table @@ -52,6 +51,5 @@ def optimize(batch): def populate(): - clear_data() create_map_table() create_vis_table() diff --git a/server/src/utils/picklebase/query.py b/server/src/utils/picklebase/query.py index b5b384d19..832c62ec5 100644 --- a/server/src/utils/picklebase/query.py +++ b/server/src/utils/picklebase/query.py @@ -12,6 +12,8 @@ def get_batch_nums(table, startDate, endDate): def query(table, fields, filters): + print('QUERYING PICKLEBASE') + startDate = pd.to_datetime(filters['startDate']) endDate = pd.to_datetime(filters['endDate']) requestTypes = filters['requestTypes'] From 2fd05b2243d480a4197dea96b63149a83c6b30f3 Mon Sep 17 00:00:00 2001 From: Jake Mensch Date: Thu, 21 May 2020 15:41:14 -0700 Subject: [PATCH 2/2] linting --- server/src/utils/picklebase/data_access.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/utils/picklebase/data_access.py b/server/src/utils/picklebase/data_access.py index 96fad4c24..b5a0cb8e4 100644 --- a/server/src/utils/picklebase/data_access.py +++ b/server/src/utils/picklebase/data_access.py @@ -62,7 +62,7 @@ def load_meta(table): def set_ready(): - with open(READY_FILE, 'w') as f: + with open(READY_FILE, 'w'): pass