Merge pull request #194 from ropable/master

Consolidate models & views, add unit tests, update Kustomize overlays, add Git precommit hook & Sentry config.

.github/workflows/run-tests.yml

@@ -0,0 +1,76 @@
+name: "Run unit tests"
+
+on:
+  # Triggers the workflow on push or pull request events but only for the master branch
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+  workflow_dispatch:
+
+jobs:
+  run_tests:
+    name: Run Django Tests
+    runs-on: ubuntu-latest
+    env:
+      DATABASE_URL: postgis://postgres:postgres@localhost:5432/postgres
+    services:
+      postgres:
+        image: postgis/postgis:13-3.1-alpine
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+        ports:
+          - 5432:5432
+        options: --mount type=tmpfs,destination=/var/lib/postgresql/data --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+    steps:
+      #----------------------------------------------
+      # Install GDAL into the environment
+      #----------------------------------------------
+      - name: Install GDAL
+        id: install-gdal
+        run: |
+          sudo apt-add-repository ppa:ubuntugis/ubuntugis-unstable
+          sudo apt-get update
+          sudo apt-get install gdal-bin libgdal-dev
+      #----------------------------------------------
+      # Checkout repo and set up Python
+      #----------------------------------------------
+      - name: Check out repository
+        uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        id: setup-python
+        with:
+          python-version: '3.10'
+      #----------------------------------------------
+      # Install & configure Poetry
+      #----------------------------------------------
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+      #----------------------------------------------
+      # Load cached venv if cache exists
+      #----------------------------------------------
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v3
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
+      #----------------------------------------------
+      # Install project dependencies if cache does not exist
+      #----------------------------------------------
+      - name: Install project dependencies
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction --no-root
+      #----------------------------------------------
+      # Run unit tests
+      #----------------------------------------------
+      - name: Run tests
+        run: |
+          source .venv/bin/activate
+          python manage.py test

.pre-commit-config.yaml

@@ -0,0 +1,11 @@
+repos:
+- repo: local
+  hooks:
+    - id: trufflehog
+      name: TruffleHog
+      description: Detect secrets in your data.
+      entry: bash -c 'trufflehog git file://. --since-commit HEAD --only-verified --fail --no-update'
+      # For running trufflehog in docker, use the following entry instead:
+      # entry: bash -c 'docker run --rm -v "$(pwd):/workdir" -i --rm trufflesecurity/trufflehog:latest git file:///workdir --since-commit HEAD --only-verified --fail'
+      language: system
+      stages: ["commit", "push"]

Dockerfile

@@ -1,5 +1,6 @@
+# syntax=docker/dockerfile:1
 # Prepare the base environment.
-FROM python:3.10.12-slim-bookworm as builder_base_rt
+FROM python:3.10.13-slim as builder_base_rt
 MAINTAINER [email protected]
 LABEL org.opencontainers.image.source https://github.com/dbca-wa/resource_tracking
 
@@ -18,7 +19,7 @@ COPY poetry.lock pyproject.toml ./
 RUN poetry config virtualenvs.create false \
   && poetry install --no-interaction --no-ansi --only main
 
-# Install a non-root user.
+# Create a non-root user.
 ARG UID=10001
 ARG GID=10001
 RUN groupadd -g "${GID}" appuser \

README.md

@@ -5,6 +5,22 @@ from a mailbox and displays it on a collection of layers provided by
 Geoserver. The application also downloads observation data from
 automatic weather stations.
 
+# Installation
+
+The recommended way to set up this project for development is using
+[Poetry](https://python-poetry.org/docs/) to install and manage a virtual Python
+environment. With Poetry installed, change into the project directory and run:
+
+    poetry install
+
+To run Python commands in the virtualenv, thereafter run them like so:
+
+    poetry run python manage.py
+
+Manage new or updating project dependencies with Poetry also, like so:
+
+    poetry add newpackage==1.0
+
 # Environment variables
 
 This project uses confy to set environment variables (in a `.env` file).
@@ -13,18 +29,40 @@ The following variables are required for the project to run:
     DATABASE_URL="postgis://USER:PASSWORD@HOST:PORT/DATABASE_NAME"
     SECRET_KEY="ThisIsASecretKey"
 
-Variables below may also need to be defined (these are context-dependent):
-
-    DEBUG=True
-    CSRF_COOKIE_SECURE=False
-    SESSION_COOKIE_SECURE=False
-    ALLOWED_DOMAINS=".domain.wa.gov.au,localhost"
-    EMAIL_HOST="email.host"
-    EMAIL_PORT=25
-    EMAIL_USER="[email protected]"
-    EMAIL_PASSWORD="password"
-    TRACPLUS_URL="https://your-trackplus-gateway-url/parameters"
-    DFES_URL="https://dfes-resource-api/endpoint"
-    DFES_USER="username"
-    DFES_PASS="password"
-    KMI_VEHICLE_BASE_URL="https://kmi.base.url/json/endpoint"
+Other environment variables will be required to run the project in production
+(these are context-dependent).
+
+# Running
+
+Use `runserver` to run a local copy of the application:
+
+    poetry run python manage.py runserver 0:8080
+
+Run console commands manually:
+
+    poetry run python manage.py shell_plus
+
+# Unit tests
+
+Run unit tests like so:
+
+    poetry run python manage.py test --keepdb -v2
+
+# Docker image
+
+To build a new Docker image from the `Dockerfile`:
+
+    docker image build -t ghcr.io/dbca-wa/resource_tracking .
+
+# Pre-commit hooks
+
+This project includes the following pre-commit hooks:
+
+- TruffleHog (credential scanning): https://github.com/marketplace/actions/trufflehog-oss
+
+Pre-commit hooks may have additional system dependencies to run. Optionally
+install pre-commit hooks locally like so:
+
+    poetry run pre-commit install
+
+Reference: https://pre-commit.com/

kustomize/README.md

@@ -7,7 +7,6 @@ Declarative management of Kubernetes objects using Kustomize.
 Within an overlay directory, create a `.env` file to contain required secret
 values in the format KEY=value (i.e. `overlays/uat/.env`). Example:
 
-    POSTGRES_PASSWORD=value
     DATABASE_URL=value
     SECRET_KEY=value
 

kustomize/base/deployment.yaml

@@ -3,15 +3,21 @@ kind: Deployment
 metadata:
   name: resourcetracking-deployment
 spec:
+  replicas: 2
   strategy:
     type: RollingUpdate
   template:
     spec:
       containers:
       - name: resourcetracking
+        image: ghcr.io/dbca-wa/resource_tracking
         env:
         - name: ALLOWED_DOMAINS
           value: ".dbca.wa.gov.au"
+        - name: CSRF_COOKIE_SECURE
+          value: "True"
+        - name: SESSION_COOKIE_SECURE
+          value: "True"
         - name: TZ
           value: "Australia/Perth"
         resources:
@@ -39,4 +45,12 @@ spec:
           periodSeconds: 3
           failureThreshold: 3
           timeoutSeconds: 2
+        securityContext:
+          runAsNonRoot: true
+          privileged: false
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+              - ALL
+          readOnlyRootFilesystem: false
       restartPolicy: Always

kustomize/base/kustomization.yaml

@@ -1,4 +1,3 @@
 resources:
 - deployment.yaml
-- persistentvolumeclaim.yaml
-- statefulset.yaml
+- service.yaml

kustomize/base/service.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: resourcetracking-clusterip
+spec:
+  type: ClusterIP
+  ports:
+  - name: wsgi
+    port: 8080
+    protocol: TCP
+    targetPort: 8080

kustomize/overlays/prod/cronjobs/harvest-dfes/kustomization.yaml

@@ -0,0 +1,12 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ../../../../template
+nameSuffix: -harvest-dfes
+patches:
+- target:
+    group: batch
+    version: v1
+    kind: CronJob
+    name: resourcetracking-cronjob
+  path: patch.yaml

kustomize/overlays/prod/cronjobs/harvest-dfes/patch.yaml

@@ -0,0 +1,6 @@
+- op: replace
+  path: /spec/schedule
+  value: "* * * * *"
+- op: replace
+  path: /spec/jobTemplate/spec/template/spec/containers/0/args
+  value: ["manage.py", "harvest_dfes_feed"]

kustomize/overlays/prod/cronjobs/harvest-email-dplus/kustomization.yaml

@@ -0,0 +1,12 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ../../../../template
+nameSuffix: -harvest-dplus
+patches:
+- target:
+    group: batch
+    version: v1
+    kind: CronJob
+    name: resourcetracking-cronjob
+  path: patch.yaml

kustomize/overlays/prod/cronjobs/harvest-email-dplus/patch.yaml

@@ -0,0 +1,6 @@
+- op: replace
+  path: /spec/schedule
+  value: "* * * * *"
+- op: replace
+  path: /spec/jobTemplate/spec/template/spec/containers/0/args
+  value: ["manage.py", "harvest_tracking_email", "--device-type", "dplus"]