diff --git a/.github/workflows/Daily_Update_Backend_Dev.yml b/.github/workflows/Daily_Update_Backend_Dev.yml index 20350bd59..d38c5e516 100644 --- a/.github/workflows/Daily_Update_Backend_Dev.yml +++ b/.github/workflows/Daily_Update_Backend_Dev.yml @@ -14,6 +14,7 @@ jobs: username: ec2-user host: ${{ secrets.AWS_DEV_HOST }} key: ${{ secrets.AWS_DEV_PRIVATE_KEY }} + command_timeout: 30m script: | set -e cd 311-data/server diff --git a/server/.env.example b/server/.env.example index 2877a4ca2..341312cb1 100644 --- a/server/.env.example +++ b/server/.env.example @@ -25,6 +25,7 @@ PICKLEBASE_BATCH_SIZE=400000 # Picklecache PICKLECACHE_ENABLED=0 +PICKLECACHE_TTL_SECONDS=3600 # Socrata SOCRATA_TOKEN= diff --git a/server/api/bin/db_update.py b/server/api/bin/db_update.py index 7b4c03c73..170c4498e 100644 --- a/server/api/bin/db_update.py +++ b/server/api/bin/db_update.py @@ -6,6 +6,10 @@ if __name__ == '__main__': import db import pb + import cache + + if hasattr(cache, 'clean'): + cache.clean() db.requests.update() diff --git a/server/api/src/cache/__init__.py b/server/api/src/cache/__init__.py index f35a04232..9764b58a5 100644 --- a/server/api/src/cache/__init__.py +++ b/server/api/src/cache/__init__.py @@ -3,8 +3,8 @@ if Redis.ENABLED: from .redis import get, set elif Picklecache.ENABLED: - from .picklecache import get, set + from .picklecache import get, set, clean else: from .stub import get, set -__all__ = ['get', 'set'] +__all__ = ['get', 'set', 'clean'] diff --git a/server/api/src/cache/picklecache.py b/server/api/src/cache/picklecache.py index ab5e83d9e..d7eda3bd3 100644 --- a/server/api/src/cache/picklecache.py +++ b/server/api/src/cache/picklecache.py @@ -1,9 +1,12 @@ import os +import time import pickle -from settings import Server +from settings import Server, Picklecache +from utils.log import log, log_heading CACHE_DIR = os.path.join(Server.TMP_DIR, 'picklecache') +TTL_SECONDS = Picklecache.TTL_SECONDS os.makedirs(CACHE_DIR, exist_ok=True) @@ -22,4 +25,22 @@ def set(key, value): with open(path, 'wb') as f: pickle.dump(value, f, protocol=pickle.HIGHEST_PROTOCOL) except Exception as e: - print(e) + log(e) + + +def clean(): + log_heading('cleaning picklecache') + file_names = os.listdir(CACHE_DIR) + + if len(file_names) == 0: + log('picklecache is empty.') + return + + now = time.time() + for file_name in file_names: + path = os.path.join(CACHE_DIR, file_name) + stat = os.stat(path) + age = round(now - stat.st_mtime) + if age > TTL_SECONDS: + log(f'deleting: {file_name} (age {age})') + os.remove(path) diff --git a/server/api/src/settings.py b/server/api/src/settings.py index 70360ebfa..b0e763bf5 100644 --- a/server/api/src/settings.py +++ b/server/api/src/settings.py @@ -39,6 +39,7 @@ class Picklebase: class Picklecache: ENABLED = env('PICKLECACHE_ENABLED', to.BOOL) + TTL_SECONDS = env('PICKLECACHE_TTL_SECONDS', to.INT) class Github: diff --git a/server/api/src/utils/log.py b/server/api/src/utils/log.py index 0a54de8ee..8cf4fee15 100644 --- a/server/api/src/utils/log.py +++ b/server/api/src/utils/log.py @@ -13,6 +13,8 @@ class log_colors: def log(message='', color=None, dedent=False): + message = str(message) + if dedent: message = dedenter(message)