diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 95fdba3..cfeaef2 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -83,7 +83,7 @@ jobs: uses: docker/build-push-action@v6 with: context: . - file: Dockerfile + file: deploy/django/Dockerfile tags: ${{ env.REGISTRY }}/${{ github.repository }}/live_data_server:${{ steps.latest_tag.outputs.latest_tag }} push: true @@ -92,6 +92,6 @@ jobs: uses: docker/build-push-action@v6 with: context: . - file: Dockerfile + file: deploy/django/Dockerfile tags: ${{ env.REGISTRY }}/${{ github.repository }}/live_data_server:${{ steps.tag.outputs.tag }} push: true diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml index 6139562..70936cd 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unittest.yml @@ -41,7 +41,7 @@ jobs: - name: Start docker containers run: | - cp ./deploy-config/docker-compose.envlocal.yml docker-compose.yml + cp ./deploy/docker-compose.envlocal.yml docker-compose.yml docker compose up --build -d - name: Sleep, wait for containers to start up diff --git a/.gitignore b/.gitignore index 000bd40..0fb10b8 100644 --- a/.gitignore +++ b/.gitignore @@ -64,3 +64,6 @@ target/ #Ipython Notebook .ipynb_checkpoints + +# Ruff cache +.ruff_cache diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6fb156b..7808194 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.6 + rev: v0.6.1 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 91897e4..0000000 --- a/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -FROM continuumio/miniconda3:23.10.0-1 - -COPY environment.yml . -RUN conda env create - -WORKDIR /var/www/livedata - -COPY docker-entrypoint.sh /usr/bin/ - -COPY src app -RUN mkdir ./static - -RUN chmod +x /usr/bin/docker-entrypoint.sh -ENTRYPOINT ["conda", "run", "--no-capture-output", "-n", "livedata", "/usr/bin/docker-entrypoint.sh"] diff --git a/Makefile b/Makefile index f8e21ff..e21e3f2 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,7 @@ docker/compose/validate: ## validate the version of the docker-compose command. @./scripts/docker-compose_validate.sh $(DOCKER_COMPOSE) docker/compose/local: docker/compose/validate ## compose and start the service locally - \cp ./deploy-config/docker-compose.envlocal.yml docker-compose.yml + \cp ./deploy/docker-compose.envlocal.yml docker-compose.yml $(DOCKER_COMPOSE) up --build .PHONY: clean diff --git a/README.md b/README.md index 855c67f..e11e755 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ Developer documentation at make docker/compose/local ``` - This command will copy `deploy-config/docker-compose.envlocal.yml` into `docker-compose.yml` before composing all the services. + This command will copy `deploy/docker-compose.envlocal.yml` into `docker-compose.yml` before composing all the services. Type `make help` to learn about other macros available as make targets. For instance, `make docker/pruneall` will stop all containers, then remove all containers, images, networks, and volumes. diff --git a/codecov.yml b/codecov.yml index 3b7deba..332f654 100644 --- a/codecov.yml +++ b/codecov.yml @@ -8,3 +8,5 @@ coverage: default: target: 80% threshold: 10% +ignore: + - "src/apps/plots/migrations" diff --git a/deploy/django/Dockerfile b/deploy/django/Dockerfile new file mode 100644 index 0000000..6ec7740 --- /dev/null +++ b/deploy/django/Dockerfile @@ -0,0 +1,31 @@ +FROM continuumio/miniconda3:23.10.0-1 AS base + +### System dependencies and cron job setup +RUN apt-get update -y && \ + # apt upgrade -y && \ + apt-get install -y \ + vim cron + +# Set up cron job to purge expired data once a month +COPY scripts/periodic-purge.sh /var/opt/ +RUN echo "0 0 1 * * /var/opt/periodic-purge.sh >> /var/log/cron.log 2>&1" > /etc/cron.d/root && \ + chmod 0644 /etc/cron.d/root && \ + crontab /etc/cron.d/root && \ + touch /var/log/cron.log + +### Environment setup +FROM base AS build + +COPY environment.yml . +RUN conda env create + +WORKDIR /var/www/livedata +COPY src app +RUN mkdir ./static + +### Final image +FROM build AS final + +COPY deploy/django/docker-entrypoint.sh /usr/bin/ +RUN chmod +x /usr/bin/docker-entrypoint.sh +ENTRYPOINT ["conda", "run", "--no-capture-output", "-n", "livedata", "/usr/bin/docker-entrypoint.sh"] diff --git a/docker-entrypoint.sh b/deploy/django/docker-entrypoint.sh similarity index 87% rename from docker-entrypoint.sh rename to deploy/django/docker-entrypoint.sh index cd6f173..dc72e82 100755 --- a/docker-entrypoint.sh +++ b/deploy/django/docker-entrypoint.sh @@ -1,9 +1,13 @@ #!/bin/sh set -e +# start cron, export root env variables +service cron start +env >>/etc/environment + # wait for database until PGPASSWORD=${DATABASE_PASS} psql -h "${DATABASE_HOST}" -U "${DATABASE_USER}" -d "${DATABASE_NAME}" -c '\q'; do - >&2 echo "Postgres is unavailable - sleeping" + echo >&2 "Postgres is unavailable - sleeping" sleep 1 done diff --git a/deploy-config/docker-compose.envlocal.yml b/deploy/docker-compose.envlocal.yml similarity index 93% rename from deploy-config/docker-compose.envlocal.yml rename to deploy/docker-compose.envlocal.yml index 6b76db9..75f0970 100644 --- a/deploy-config/docker-compose.envlocal.yml +++ b/deploy/docker-compose.envlocal.yml @@ -7,7 +7,7 @@ services: - "443:443" volumes: - web-static:/var/www/livedata/static - - ./deploy-config/nginx/envlocal.conf:/etc/nginx/conf.d/nginx.conf + - ./deploy/nginx/envlocal.conf:/etc/nginx/conf.d/nginx.conf depends_on: django: condition: service_healthy @@ -15,7 +15,7 @@ services: django: build: context: . - dockerfile: Dockerfile + dockerfile: deploy/django/Dockerfile network: host environment: APP_DEBUG: 1 # 0 for False, otherwise will evaluate to True diff --git a/deploy-config/nginx/docker-entrypoint.sh b/deploy/nginx/docker-entrypoint.sh similarity index 100% rename from deploy-config/nginx/docker-entrypoint.sh rename to deploy/nginx/docker-entrypoint.sh diff --git a/deploy-config/nginx/envlocal.conf b/deploy/nginx/envlocal.conf similarity index 100% rename from deploy-config/nginx/envlocal.conf rename to deploy/nginx/envlocal.conf diff --git a/docs/developer/config_for_local_use.rst b/docs/developer/config_for_local_use.rst index 0c5f504..7f39044 100644 --- a/docs/developer/config_for_local_use.rst +++ b/docs/developer/config_for_local_use.rst @@ -54,7 +54,7 @@ After the secrets are set, you can start the server with: make docker/compose/local -This command will copy ``deploy-config/docker-compose.envlocal.yml`` into ``./docker-compose.yml`` before composing all the services. +This command will copy ``deploy/docker-compose.envlocal.yml`` into ``./docker-compose.yml`` before composing all the services. | Run ``make help`` to learn about other macros available as make targets. | For instance, ``make docker/pruneall`` will stop all containers, then remove all containers, images, networks, and volumes. diff --git a/docs/developer/troubleshoot/unresponsive.rst b/docs/developer/troubleshoot/unresponsive.rst index b20bba1..e965827 100644 --- a/docs/developer/troubleshoot/unresponsive.rst +++ b/docs/developer/troubleshoot/unresponsive.rst @@ -24,9 +24,9 @@ The logs indicate a problem with the certificate files. An additional test is to substitute the `nginx.conf file for the testing environment `_ with the -`local environment one `_, +`local environment one `_, which does not contain SSL certificates. Don't forget to change -`the server name `_ +`the server name `_ from `"localhost"` to `"testfixture02-test.ornl.gov"`. Redeploy after this. If the http://testfixture02-test.ornl.gov/admin (notice the `http` instead of `https`) app is served now, then it's a problem of the secure connection. diff --git a/environment.yml b/environment.yml index 0a0f7ea..3b041ea 100644 --- a/environment.yml +++ b/environment.yml @@ -11,7 +11,6 @@ dependencies: - psycopg=3.2 - gunicorn - pytest - - build - versioningit - toml - pre-commit diff --git a/scripts/periodic-purge.sh b/scripts/periodic-purge.sh new file mode 100755 index 0000000..5c5caa5 --- /dev/null +++ b/scripts/periodic-purge.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +. /opt/conda/etc/profile.d/conda.sh +printf "=%.0s" {1..88} +printf "\nStarting periodic purge - $(date)\n" +conda run -n livedata python /var/www/livedata/app/manage.py purge_expired_data +printf "=%.0s" {1..88}; printf "\n" diff --git a/src/apps/plots/admin.py b/src/apps/plots/admin.py index cb6d290..07fab4f 100644 --- a/src/apps/plots/admin.py +++ b/src/apps/plots/admin.py @@ -1,6 +1,7 @@ -from apps.plots.models import DataRun, Instrument, PlotData from django.contrib import admin +from apps.plots.models import DataRun, Instrument, PlotData + class PlotDataAdmin(admin.ModelAdmin): readonly_fields = ("data_run",) diff --git a/src/apps/plots/apps.py b/src/apps/plots/apps.py index 9f62667..dc6b5af 100644 --- a/src/apps/plots/apps.py +++ b/src/apps/plots/apps.py @@ -3,3 +3,4 @@ class PlotsConfig(AppConfig): name = "apps.plots" + default_auto_field = "django.db.models.BigAutoField" diff --git a/src/apps/plots/management/commands/purge_expired_data.py b/src/apps/plots/management/commands/purge_expired_data.py index 982f9d9..e2f1284 100644 --- a/src/apps/plots/management/commands/purge_expired_data.py +++ b/src/apps/plots/management/commands/purge_expired_data.py @@ -1,13 +1,17 @@ -from apps.plots.models import DataRun from django.core.management.base import BaseCommand from django.utils import timezone +from apps.plots.models import DataRun + class Command(BaseCommand): help = "Delete expired runs and related plots" def handle(self, *args, **options): # noqa: ARG002 runs = DataRun.objects.all() + expired_runs = 0 for run in runs: if run.expiration_date < timezone.now(): + expired_runs += 1 run.delete() + self.stdout.write(self.style.SUCCESS(f"Deleted {expired_runs} expired runs")) diff --git a/src/apps/plots/migrations/0003_alter_default_auto_fields.py b/src/apps/plots/migrations/0003_alter_default_auto_fields.py new file mode 100644 index 0000000..c7a6588 --- /dev/null +++ b/src/apps/plots/migrations/0003_alter_default_auto_fields.py @@ -0,0 +1,27 @@ +# Generated by Django 4.2.15 on 2024-08-23 14:47 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("plots", "0002_datarun_expiration_date"), + ] + + operations = [ + migrations.AlterField( + model_name="datarun", + name="id", + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID"), + ), + migrations.AlterField( + model_name="instrument", + name="id", + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID"), + ), + migrations.AlterField( + model_name="plotdata", + name="id", + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID"), + ), + ] diff --git a/src/apps/plots/view_util.py b/src/apps/plots/view_util.py index 4872756..583bbeb 100644 --- a/src/apps/plots/view_util.py +++ b/src/apps/plots/view_util.py @@ -8,11 +8,12 @@ from datetime import datetime from typing import Optional -from apps.plots.models import DataRun, Instrument, PlotData from django.conf import settings from django.http import HttpResponse from django.utils import timezone +from apps.plots.models import DataRun, Instrument, PlotData + def generate_key(instrument, run_id): """ diff --git a/src/apps/plots/views.py b/src/apps/plots/views.py index 146294f..c7be7db 100644 --- a/src/apps/plots/views.py +++ b/src/apps/plots/views.py @@ -6,7 +6,6 @@ import logging from datetime import timedelta -from apps.plots.models import DataRun, Instrument, PlotData from django.conf import settings from django.contrib.auth import authenticate, login from django.http import HttpResponse, HttpResponseNotFound, JsonResponse @@ -15,6 +14,8 @@ from django.views.decorators.cache import cache_page from django.views.decorators.csrf import csrf_exempt +from apps.plots.models import DataRun, Instrument, PlotData + from . import view_util