Merge pull request #115 from ropable/master

Refine video_import command, update Kustomize resources

kustomize/base/deployment.yaml

@@ -26,10 +26,10 @@ spec:
           value: "Australia/Perth"
         resources:
           requests:
-            memory: "128Mi"
-            cpu: "10m"
+            memory: "100Mi"
+            cpu: "5m"
           limits:
-            memory: "4096Mi"
+            memory: "2048Mi"
             cpu: "1000m"
         startupProbe:
           tcpSocket:

kustomize/base/deployment_hpa.yaml

@@ -3,8 +3,8 @@ kind: HorizontalPodAutoscaler
 metadata:
   name: penguins-deployment-hpa
 spec:
-  minReplicas: 1
-  maxReplicas: 3
+  minReplicas: 2
+  maxReplicas: 10
   scaleTargetRef:
     apiVersion: apps/v1
     kind: Deployment
@@ -13,5 +13,5 @@ spec:
         name: cpu
         target:
           type: Utilization
-          averageUtilization: 250
+          averageUtilization: 500
       type: Resource

kustomize/overlays/prod/kustomization.yaml

@@ -21,4 +21,4 @@ patches:
   - path: cronjob_import_patch.yaml
 images:
   - name: ghcr.io/dbca-wa/penguins
-    newTag: 1.2.0
+    newTag: 1.2.1

observations/management/commands/video_import.py

@@ -1,6 +1,7 @@
 from datetime import datetime, timedelta
 from django.core.management.base import BaseCommand
 import logging
+import re
 from storages.backends.azure_storage import AzureStorage
 
 from observations.models import Video, Camera
@@ -13,12 +14,15 @@ def handle(self, *args, **options):
         logger = logging.getLogger('observations')
         logger.info('Starting import of encoded videos')
 
+        logger.info('Querying uploaded videos in Azure')
         store = AzureStorage()
         all_blobs = store.list_all()
         video_formats = ('.mp4',)
         video_paths = [blob for blob in all_blobs if blob.endswith(video_formats)]
         count = 0
+        pattern = '^([-\d]+)_([-\d]+)_(.+)$'
 
+        logger.info('Checking for unpublished videos')
         for path in video_paths:
             if Video.objects.filter(file=path).exists():
                 continue
@@ -29,22 +33,22 @@ def handle(self, *args, **options):
             try:
                 video_datetime = datetime.strptime(timestamp, "%Y-%m-%d_%H")
             except:
-                video_datetime = None
+                logger.warning('Unable to parse timestamp from {}'.format(video_filename))
+                continue
 
             # Parse camera name and find matching object
-            camera_key = video_filename[17:-4].split("_")[0]
+            matches = re.findall(pattern, video_filename)[0]  # Returns the tuple of matches.
+            camera = matches[2]  # Examples: North.mp4, t1_south_pole.mp4
+            camera_key = camera.split(".")[0]  # Examples: North, t1_south_pole
             if Camera.objects.filter(camera_key__icontains=camera_key).exists():
                 camera = Camera.objects.filter(camera_key__icontains=camera_key).first()
             else:
                 camera = None
 
-            if not video_datetime:
-                logger.warning('Unable to parse timestamp from {}'.format(video_filename))
             if not camera:
                 logger.warning('No matching camera found, skipping video: {}'.format(video_filename))
 
             if video_datetime and camera:
-                print("Importing {} for camera {}, date {}".format(video_filename, camera, video_datetime))
                 Video.objects.create(
                     date=video_datetime.date(),
                     start_time=video_datetime.time(),
@@ -54,4 +58,4 @@ def handle(self, *args, **options):
                 )
 
                 count += 1
-        logger.info("Task completed, {} videos imported".format(count))
+        logger.info('Import task completed, {} videos imported'.format(count))

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "penguins"
-version = "1.2.0"
+version = "1.2.1"
 description = "Department of Parks and Wildlife Little Penguins Observations application"
 authors = ["Ashley Felton <[email protected]>"]
 license = "Apache-2.0"