Skip to content

Commit

Permalink
Merge pull request #698 from hackforla/BACK-DevUpdateFixes
Browse files Browse the repository at this point in the history
Back dev update fixes
  • Loading branch information
adamkendis authored Jun 18, 2020
2 parents e85cab5 + 16983df commit fd9a3a9
Show file tree
Hide file tree
Showing 7 changed files with 34 additions and 4 deletions.
1 change: 1 addition & 0 deletions .github/workflows/Daily_Update_Backend_Dev.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ jobs:
username: ec2-user
host: ${{ secrets.AWS_DEV_HOST }}
key: ${{ secrets.AWS_DEV_PRIVATE_KEY }}
command_timeout: 30m
script: |
set -e
cd 311-data/server
Expand Down
1 change: 1 addition & 0 deletions server/.env.example
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ PICKLEBASE_BATCH_SIZE=400000

# Picklecache
PICKLECACHE_ENABLED=0
PICKLECACHE_TTL_SECONDS=3600

# Socrata
SOCRATA_TOKEN=
Expand Down
4 changes: 4 additions & 0 deletions server/api/bin/db_update.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@
if __name__ == '__main__':
import db
import pb
import cache

if hasattr(cache, 'clean'):
cache.clean()

db.requests.update()

Expand Down
4 changes: 2 additions & 2 deletions server/api/src/cache/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
if Redis.ENABLED:
from .redis import get, set
elif Picklecache.ENABLED:
from .picklecache import get, set
from .picklecache import get, set, clean
else:
from .stub import get, set

__all__ = ['get', 'set']
__all__ = ['get', 'set', 'clean']
25 changes: 23 additions & 2 deletions server/api/src/cache/picklecache.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
import os
import time
import pickle
from settings import Server
from settings import Server, Picklecache
from utils.log import log, log_heading


CACHE_DIR = os.path.join(Server.TMP_DIR, 'picklecache')
TTL_SECONDS = Picklecache.TTL_SECONDS
os.makedirs(CACHE_DIR, exist_ok=True)


Expand All @@ -22,4 +25,22 @@ def set(key, value):
with open(path, 'wb') as f:
pickle.dump(value, f, protocol=pickle.HIGHEST_PROTOCOL)
except Exception as e:
print(e)
log(e)


def clean():
log_heading('cleaning picklecache')
file_names = os.listdir(CACHE_DIR)

if len(file_names) == 0:
log('picklecache is empty.')
return

now = time.time()
for file_name in file_names:
path = os.path.join(CACHE_DIR, file_name)
stat = os.stat(path)
age = round(now - stat.st_mtime)
if age > TTL_SECONDS:
log(f'deleting: {file_name} (age {age})')
os.remove(path)
1 change: 1 addition & 0 deletions server/api/src/settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ class Picklebase:

class Picklecache:
ENABLED = env('PICKLECACHE_ENABLED', to.BOOL)
TTL_SECONDS = env('PICKLECACHE_TTL_SECONDS', to.INT)


class Github:
Expand Down
2 changes: 2 additions & 0 deletions server/api/src/utils/log.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ class log_colors:


def log(message='', color=None, dedent=False):
message = str(message)

if dedent:
message = dedenter(message)

Expand Down

0 comments on commit fd9a3a9

Please sign in to comment.