Back dev update fixes

.github/workflows/Daily_Update_Backend_Dev.yml

@@ -14,6 +14,7 @@ jobs:
           username: ec2-user
           host: ${{ secrets.AWS_DEV_HOST }}
           key: ${{ secrets.AWS_DEV_PRIVATE_KEY }}
+          command_timeout: 30m
           script: |
             set -e
             cd 311-data/server

server/.env.example

@@ -25,6 +25,7 @@ PICKLEBASE_BATCH_SIZE=400000
 
 # Picklecache
 PICKLECACHE_ENABLED=0
+PICKLECACHE_TTL_SECONDS=3600
 
 # Socrata
 SOCRATA_TOKEN=

server/api/bin/db_update.py

@@ -6,6 +6,10 @@
 if __name__ == '__main__':
     import db
     import pb
+    import cache
+
+    if hasattr(cache, 'clean'):
+        cache.clean()
 
     db.requests.update()
 

server/api/src/cache/__init__.py

@@ -3,8 +3,8 @@
 if Redis.ENABLED:
     from .redis import get, set
 elif Picklecache.ENABLED:
-    from .picklecache import get, set
+    from .picklecache import get, set, clean
 else:
     from .stub import get, set
 
-__all__ = ['get', 'set']
+__all__ = ['get', 'set', 'clean']

server/api/src/cache/picklecache.py

@@ -1,9 +1,12 @@
 import os
+import time
 import pickle
-from settings import Server
+from settings import Server, Picklecache
+from utils.log import log, log_heading
 
 
 CACHE_DIR = os.path.join(Server.TMP_DIR, 'picklecache')
+TTL_SECONDS = Picklecache.TTL_SECONDS
 os.makedirs(CACHE_DIR, exist_ok=True)
 
 
@@ -22,4 +25,22 @@ def set(key, value):
         with open(path, 'wb') as f:
             pickle.dump(value, f, protocol=pickle.HIGHEST_PROTOCOL)
     except Exception as e:
-        print(e)
+        log(e)
+
+
+def clean():
+    log_heading('cleaning picklecache')
+    file_names = os.listdir(CACHE_DIR)
+
+    if len(file_names) == 0:
+        log('picklecache is empty.')
+        return
+
+    now = time.time()
+    for file_name in file_names:
+        path = os.path.join(CACHE_DIR, file_name)
+        stat = os.stat(path)
+        age = round(now - stat.st_mtime)
+        if age > TTL_SECONDS:
+            log(f'deleting: {file_name} (age {age})')
+            os.remove(path)

server/api/src/settings.py

@@ -39,6 +39,7 @@ class Picklebase:
 
 class Picklecache:
     ENABLED = env('PICKLECACHE_ENABLED', to.BOOL)
+    TTL_SECONDS = env('PICKLECACHE_TTL_SECONDS', to.INT)
 
 
 class Github:

server/api/src/utils/log.py

@@ -13,6 +13,8 @@ class log_colors:
 
 
 def log(message='', color=None, dedent=False):
+    message = str(message)
+
     if dedent:
         message = dedenter(message)