From 15975c26e17727385f2d236984b097b652ddcf9e Mon Sep 17 00:00:00 2001 From: Remy van der Wereld Date: Sun, 23 Jun 2024 17:15:39 +0200 Subject: [PATCH 01/42] Initial commit --- README.md | 40 +++++++++++++++++++++------------------- bin/setup_credentials.sh | 2 +- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 1d096cd03..af7424694 100644 --- a/README.md +++ b/README.md @@ -53,13 +53,13 @@ docker-compose -f docker-compose.local.yml up To create all necessary credentials run the following command: -``` +```bash bash bin/setup_credentials.sh ``` This will create superuser admin account with the following credentials -``` +```bash email: admin@admin.com password: admin ``` @@ -75,14 +75,15 @@ Set LOCAL_DEVELOPMENT_AUTHENTICATION environment variable to True (default) Run unit tests locally with: -``` -docker compose run --rm zaak-gateway python manage.py test +```bash +docker compose -f docker-compose.local.yml run --rm zaak-gateway python manage.py test ``` To run tests for a specific module, add a path: -``` -docker compose run --rm zaak-gateway python manage.py test apps/cases +```bash +docker compose -f docker-compose.local.yml run --rm zaak-gateway python manage.py test apps/cases + ``` ## API documentation (Swagger) @@ -105,24 +106,25 @@ Create a `.env.local` file, on the root of your project, and override the variab Start your project with the newly created environment variables, like so: -``` -docker compose --env-file .env.local up +```bash +docker compose -f docker-compose.local.yml --env-file .env.local up ``` ## Enabling Keycloak authentication for a locally run zaken-frontend -Set LOCAL_DEVELOPMENT_AUTHENTICATION environment variable to False +Set `LOCAL_DEVELOPMENT_AUTHENTICATION` environment variable to False ## Generating Mock Data -You can generate mock data easily (from the API swagger environment) by executing the /api/v1/generate-mock/ GET request. +You can generate mock data easily (from the API swagger environment) by executing the `/api/v1/generate-mock/` GET request. ## Update fixtures Generate new fixtures json file: -``` -docker-compose run --rm zaak-gateway python manage.py dumpdata --indent 2 -o temp_fixture.json [app_name] +```bash +docker compose -f docker-compose.local.yml run --rm zaak-gateway python manage.py dumpdata --indent 2 -o temp_fixture.json [app_name] + ``` Now manually copy changes you need to the corresponding fixtures file. @@ -131,13 +133,13 @@ Now manually copy changes you need to the corresponding fixtures file. You can add pre-commit hooks for checking and cleaning up your changes: -``` +```bash bash bin/install_pre_commit.sh ``` You can also run the following command to ensure all files adhere to coding conventions: -``` +```bash bash bin/cleanup_pre_commit.sh ``` @@ -167,7 +169,7 @@ Note that the apps and models should be updated whenever applications and models For changes to the model you have to migrate the DB. -```python +```bash python manage.py makemigrations --name python manage.py migrate @@ -193,14 +195,14 @@ Try the online modeler for BPMN-models: https://bpmn.io/. This is a lightweight Clone the bpmn-io Github repo for editing: https://github.com/bpmn-io/bpmn-js-examples. -``` +```bash git clone git@github.com:bpmn-io/bpmn-js-examples.git ``` If you'd like to use Camunda Platform execution related properties, include the camunda-bpmn-moddle dependency which tells the modeler about camunda:XXX extension properties: https://github.com/bpmn-io/bpmn-js-examples/tree/master/properties-panel#camunda-platform Follow next steps: -``` +```bash cd bpmn-js-examples cd properties-panel @@ -265,7 +267,7 @@ So if you think existing cases will get stuck in the model, just create a new ve Example: `housing_corporation` has a new minor version model and the latest version was `5.0.0` but `debrief` has latest version `6.0.0`. Then the new minor version of `housing_corporation` will be `6.1.0`. -``` +```json bpmn_models/default/ ├─ debrief/ │ ├─ 0.1.0/ @@ -280,7 +282,7 @@ bpmn_models/default/ - Add the new version to `WORKFLOW_SPEC_CONFIG` in `settings.py`: -```python +```json "housing_corporation": { "versions": { "5.0.0": {}, diff --git a/bin/setup_credentials.sh b/bin/setup_credentials.sh index 39854b0a7..19530925b 100755 --- a/bin/setup_credentials.sh +++ b/bin/setup_credentials.sh @@ -1,2 +1,2 @@ # Creates a superuser for the zaak-gateway backend -echo "from django.contrib.auth import get_user_model; get_user_model().objects.create_superuser('admin@admin.com', 'admin')" | docker-compose -f docker-compose.local.yml run --rm zaak-gateway python manage.py shell +echo "from django.contrib.auth import get_user_model; get_user_model().objects.create_superuser('admin@admin.com', 'admin')" | docker-compose -f docker-compose.local.yml run -T --rm zaak-gateway python manage.py shell From a4c09c6c35d444af7bedbdbf4727801724d20416 Mon Sep 17 00:00:00 2001 From: Remy van der Wereld Date: Sun, 23 Jun 2024 17:34:14 +0200 Subject: [PATCH 02/42] Update action --- .github/workflows/codeql-analysis.yml | 4 ++-- app/requirements.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index d0c157762..93933d85d 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -35,10 +35,10 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v3.1.2 + uses: actions/setup-python@v5 with: python-version: '3.9.16' - name: Install dependencies diff --git a/app/requirements.txt b/app/requirements.txt index 592e679a1..642cd7e49 100644 --- a/app/requirements.txt +++ b/app/requirements.txt @@ -29,7 +29,7 @@ django-health-check==3.16.4 django-ipware==4.0.2 django-permissions-policy==4.1.0 django-redis==5.0.0 -django-relativedelta==1.1.2 +django-relativedelta==2.0.0 django-solo==1.1.5 django-spaghetti-and-meatballs==0.4.2 django-timezone-field==4.2.1 From 61560d1ec11e183e410443424dc300f6d96c1146 Mon Sep 17 00:00:00 2001 From: Remy van der Wereld Date: Sun, 23 Jun 2024 17:42:44 +0200 Subject: [PATCH 03/42] Updated codeql analyze --- .github/workflows/codeql-analysis.yml | 9 +++++++-- app/requirements.txt | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 93933d85d..7883b419a 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -41,6 +41,11 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.9.16' + + # Update docker base image for a higher python version than 3.9 and remove this line. + - name: Install specific pip version for python 3.9 + run: python -m pip install pip==23.1.2 + - name: Install dependencies run: | python -m pip install --upgrade pip @@ -68,7 +73,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v1 + uses: github/codeql-action/autobuild@v3 # ℹ️ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -82,4 +87,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v3 diff --git a/app/requirements.txt b/app/requirements.txt index 642cd7e49..592e679a1 100644 --- a/app/requirements.txt +++ b/app/requirements.txt @@ -29,7 +29,7 @@ django-health-check==3.16.4 django-ipware==4.0.2 django-permissions-policy==4.1.0 django-redis==5.0.0 -django-relativedelta==2.0.0 +django-relativedelta==1.1.2 django-solo==1.1.5 django-spaghetti-and-meatballs==0.4.2 django-timezone-field==4.2.1 From 082c6c86ea57322811487890b9923ab8f173c9ec Mon Sep 17 00:00:00 2001 From: Remy van der Wereld Date: Sun, 23 Jun 2024 17:47:35 +0200 Subject: [PATCH 04/42] Puin specific version pip --- .github/workflows/codeql-analysis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 7883b419a..433e4e055 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -48,7 +48,6 @@ jobs: - name: Install dependencies run: | - python -m pip install --upgrade pip if [ -f ./app/requirements.txt ]; echo "Installing python dependencies" then pip install -r ./app/requirements.txt; From 86eb87641d7d99e9575cdef2fabb3d3a05537218 Mon Sep 17 00:00:00 2001 From: Remy van der Wereld Date: Sun, 23 Jun 2024 17:58:27 +0200 Subject: [PATCH 05/42] Updated CodeQL action --- .github/workflows/codeql-analysis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 433e4e055..a0c8275a3 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -43,7 +43,7 @@ jobs: python-version: '3.9.16' # Update docker base image for a higher python version than 3.9 and remove this line. - - name: Install specific pip version for python 3.9 + - name: Install specific pip version 23.1.2 for python 3.9 run: python -m pip install pip==23.1.2 - name: Install dependencies @@ -58,7 +58,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} # Override the default behavior so that the action doesn't attempt From 421b208c8a0e648b3da9891dccb9cc4614584340 Mon Sep 17 00:00:00 2001 From: Remy van der Wereld Date: Mon, 24 Jun 2024 16:23:01 +0200 Subject: [PATCH 06/42] Updated dependencies --- .env | 5 ++- app/Dockerfile | 16 ++++++--- app/apps/openzaak/helpers.py | 2 +- app/apps/workflow/admin.py | 7 ++-- app/config/settings.py | 6 +--- app/config/urls.py | 9 +++-- app/requirements.txt | 59 ++++++++++++++++---------------- docker-compose.local.yml | 6 ++-- open-zaak/open-notificaties/.env | 2 +- 9 files changed, 55 insertions(+), 57 deletions(-) diff --git a/.env b/.env index 67619905a..5c262406c 100644 --- a/.env +++ b/.env @@ -9,9 +9,8 @@ DATABASE_USER=zaken DATABASE_PASSWORD=insecure DATABASE_HOST=database DATABASE_PORT=5432 -SENTRY_DSN=https://foo@foo.data.amsterdam.nl/0 -LOCAL_DEVELOPMENT_AUTHENTICATION=False -LOGGING_LEVEL=WARNING # To prevent flooding the logging in local development. Default is DEBUG. +LOCAL_DEVELOPMENT_AUTHENTICATION=True +LOGGING_LEVEL=WARNING # To prevent flooding the logging in local development. Default is DEBUG. WARNING SECRET_KEY_TOP_ZAKEN=SECRET_KEY_TOP_ZAKEN SECRET_KEY_TON_ZAKEN=SECRET_KEY_TON_ZAKEN BELASTING_API_URL=https://api-acc.belastingen.centric.eu/bel/inn/afne/vora/v1/vorderingenidentificatienummer/ diff --git a/app/Dockerfile b/app/Dockerfile index dc109dbbb..bfeb7fe2a 100644 --- a/app/Dockerfile +++ b/app/Dockerfile @@ -1,5 +1,4 @@ -FROM amsterdam/python:3.9-buster -LABEL maintainer="datapunt@amsterdam.nl" +FROM python:3.10-bullseye ENV REQUESTS_CA_BUNDLE /etc/ssl/certs/ca-certificates.crt @@ -10,11 +9,16 @@ RUN chmod 644 /usr/local/share/ca-certificates/adp_rootca.crt \ ENV PYTHONUNBUFFERED 1 -RUN apt-get update && apt-get install -y +# Update and install necessary packages including GDAL +RUN apt-get update && apt-get install -y \ + gdal-bin \ + libgdal-dev \ + graphviz \ + graphviz-dev \ + postgresql-client + RUN pip install --upgrade pip RUN pip install uwsgi - -RUN apt-get install graphviz graphviz-dev -y RUN pip install pygraphviz RUN echo "10.240.5.72 acc.api.data.amsterdam.nl" >> /etc/hosts || echo "Could not write to /etc/hosts" @@ -30,6 +34,8 @@ RUN chmod +x /app/wait-for.sh RUN chmod +x /app/celery.sh RUN chmod +x /app/deploy/docker-entrypoint.sh +# Set the GDAL_LIBRARY_PATH environment variable +ENV GDAL_LIBRARY_PATH /usr/lib/libgdal.so ENTRYPOINT ["/app/deploy/docker-entrypoint.sh"] CMD ["uwsgi", "--ini", "/app/deploy/config.ini"] diff --git a/app/apps/openzaak/helpers.py b/app/apps/openzaak/helpers.py index c49afe008..acac5265c 100644 --- a/app/apps/openzaak/helpers.py +++ b/app/apps/openzaak/helpers.py @@ -16,7 +16,7 @@ from apps.cases.models import CaseDocument from django.conf import settings from django.utils import timezone -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from zgw_consumers.api_models.base import factory from zgw_consumers.api_models.catalogi import ZaakType from zgw_consumers.api_models.documenten import Document diff --git a/app/apps/workflow/admin.py b/app/apps/workflow/admin.py index cb6f7b0b2..5d05c4aa1 100644 --- a/app/apps/workflow/admin.py +++ b/app/apps/workflow/admin.py @@ -1,8 +1,7 @@ -from django.conf.urls import url from django.contrib import admin, messages from django.http import HttpResponseRedirect from django.template.response import TemplateResponse -from django.urls import reverse +from django.urls import re_path, reverse from django.utils.html import format_html, mark_safe from .forms import ResetSubworkflowsForm, UpdateDataForWorkflowsForm @@ -145,12 +144,12 @@ def update_data(self, obj): def get_urls(self): urls = super().get_urls() custom_urls = [ - url( + re_path( r"^(?P.+)/reset-subworkflows/$", self.admin_site.admin_view(self.admin_process_reset_subworkflows), name="reset-subworkflows", ), - url( + re_path( r"^(?P.+)/update-data-for-subworkflow/$", self.admin_site.admin_view(self.admin_update_data_for_workflow), name="update-data-for-subworkflow", diff --git a/app/config/settings.py b/app/config/settings.py index 2461afffa..53c6c959c 100644 --- a/app/config/settings.py +++ b/app/config/settings.py @@ -33,7 +33,7 @@ ZAKEN_CONTAINER_HOST = os.getenv("ZAKEN_CONTAINER_HOST") -ALLOWED_HOSTS = "*" +ALLOWED_HOSTS = ["*"] CORS_ORIGIN_WHITELIST = os.environ.get("CORS_ORIGIN_WHITELIST").split(",") CORS_ORIGIN_ALLOW_ALL = False @@ -339,11 +339,7 @@ def filter_traces(envelope): ) AXES_RESET_ON_SUCCESS = True -AXES_ONLY_USER_FAILURES = ( - True # Enabled because IP from user is always .amsterdam.nl gateway -) AXES_ENABLED = os.getenv("AXES_ENABLED", "True") == "True" -AXES_META_PRECEDENCE_ORDER = ["HTTP_X_FORWARDED_FOR", "REMOTE_ADDR"] # Simple JWT is used for local development authentication only. SIMPLE_JWT = { diff --git a/app/config/urls.py b/app/config/urls.py index f45e6ba07..ca98ebe38 100644 --- a/app/config/urls.py +++ b/app/config/urls.py @@ -36,11 +36,10 @@ from apps.visits.views import VisitViewSet from apps.workflow.views import CaseUserTaskViewSet, GenericCompletedTaskViewSet from django.conf import settings -from django.conf.urls import include, url from django.conf.urls.static import static from django.contrib import admin from django.http import JsonResponse -from django.urls import path +from django.urls import include, path, re_path from django.views.generic import View from drf_spectacular.views import SpectacularAPIView, SpectacularSwaggerView from rest_framework.routers import DefaultRouter @@ -124,9 +123,9 @@ def get(self, request, *args, **kwargs): name="notification-callback", ), path("data-model/", include("django_spaghetti.urls")), - url("health/", include("health_check.urls")), - url("startup/", is_healthy), - url(regex=r"^$", view=MyView.as_view(), name="index"), + path("health/", include("health_check.urls")), + path("startup/", is_healthy), + re_path(r"^$", view=MyView.as_view(), name="index"), ] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) if settings.DEBUG: diff --git a/app/requirements.txt b/app/requirements.txt index 592e679a1..df6946da6 100644 --- a/app/requirements.txt +++ b/app/requirements.txt @@ -1,41 +1,41 @@ amqp<6.0.0 -asgiref==3.4.1 +asgiref==3.8.1 attrs==21.2.0 azure-identity beautifulsoup4==4.10.0 -billiard==3.6.4.0 +billiard==4.2.0 cached-property==1.5.2 -celery==5.2.6 +celery==5.4.0 certifi==2023.5.7 cffi==1.14.6 chardet==4.0.0 click==8.1.2 -click-didyoumean==0.0.3 +click-didyoumean==0.3.1 click-plugins==1.1.1 click-repl==0.2.0 cryptography==41.0.1 -datapunt-keycloak-oidc @ git+https://github.com/petercuret/keycloak_oidc.git@master +datapunt-keycloak-oidc @ git+https://github.com/remyvdwereld/keycloak_oidc_top.git@main debugpy==1.4.1 -Django==3.2.13 -django-axes==5.32.0 -django-celery-beat==2.2.1 -django-celery-results==2.2.0 -django-choices==1.7.2 -django-cors-headers==3.8.0 -django-csp==3.7 -django-extensions==3.1.3 -django-filter==2.4.0 -django-health-check==3.16.4 -django-ipware==4.0.2 -django-permissions-policy==4.1.0 -django-redis==5.0.0 -django-relativedelta==1.1.2 -django-solo==1.1.5 +Django==4.2.13 +django-axes==6.5.0 +django-celery-beat==2.6.0 +django-celery-results==2.5.1 +django-choices==2.0.0 +django-cors-headers==4.4.0 +django-csp==3.8 +django-extensions==3.2.3 +django-filter==24.2 +django-health-check==3.18.3 +django-ipware==7.0.1 +django-permissions-policy==4.20.0 +django-redis==5.4.0 +django-relativedelta==2.0.0 +django-solo==2.2.0 django-spaghetti-and-meatballs==0.4.2 -django-timezone-field==4.2.1 -djangorestframework==3.12.4 -djangorestframework-simplejwt==4.8.0 -drf-spectacular==0.17.2 +django-timezone-field==6.1.0 +djangorestframework==3.15.2 +djangorestframework-simplejwt==5.3.1 +drf-spectacular==0.27.2 drf-writable-nested==0.6.3 spiffworkflow==1.1.0 Faker==8.13.2 @@ -49,8 +49,8 @@ inflection==0.5.1 josepy==1.13.0 jsonschema==4.4.0 kombu<6.0 -model-bakery==1.3.2 -mozilla-django-oidc==1.2.4 +model-bakery==1.18.1 +mozilla-django-oidc==4.0.1 packaging prompt-toolkit==3.0.19 psycopg2==2.9.1 @@ -65,17 +65,16 @@ pytz==2022.1 redis==3.5.3 requests==2.26.0 requests-mock==1.9.3 -sentry-sdk==1.19.1 six==1.16.0 soupsieve==2.2.1 sqlparse==0.4.2 tenacity==8.0.1 text-unidecode==1.3 -typing-extensions==3.10.0.2 +typing-extensions==4.12.2 uritemplate==3.0.1 urllib3==1.26.15 -uWSGI==2.0.19.1 -vine==5.0.0 +uWSGI==2.0.26 +vine==5.1.0 wcwidth==0.2.5 zgw-consumers==0.18.0 zipp==3.5.0 diff --git a/docker-compose.local.yml b/docker-compose.local.yml index 15d8a1aab..d6a61e66e 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -14,7 +14,7 @@ services: - zaak-redis env_file: - .env - command: bash -c "/app/wait-for.sh zaak-redis:6379 && /app/wait-for.sh database:5432 && /app/deploy/docker-entrypoint.development.sh" + command: bash -c '/app/deploy/docker-entrypoint.development.sh' volumes: - ./app:/app stdin_open: true @@ -46,7 +46,7 @@ services: image: ${REGISTRY:-127.0.0.1:5001}/${REPOSITORY:-salmagundi/zaken-backend}:${VERSION:-latest} container_name: zaken_celery_worker hostname: zaken_celery_worker - command: bash -c "/app/wait-for.sh zaak-redis:6379 && /app/wait-for.sh zaak-gateway:8000 -- /app/celery.sh" + command: bash -c '/app/celery.sh' depends_on: - zaak-gateway - database @@ -61,7 +61,7 @@ services: image: ${REGISTRY:-127.0.0.1:5001}/${REPOSITORY:-salmagundi/zaken-backend}:${VERSION:-latest} container_name: zaken_celery_beat hostname: zaken_celery_beat - command: sh -c "/app/wait-for.sh zaak-redis:6379 && /app/wait-for.sh zaak-gateway:8000 -- celery -A config beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler" + command: sh -c 'celery -A config beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler' depends_on: - zaak-gateway - database diff --git a/open-zaak/open-notificaties/.env b/open-zaak/open-notificaties/.env index 82b7c5143..853446a56 100644 --- a/open-zaak/open-notificaties/.env +++ b/open-zaak/open-notificaties/.env @@ -1,6 +1,6 @@ SECRET_KEY=${SECRET_KEY:-7(h1r2hk)8z9+05edulo_3qzymwbo&c24=)qz7+_@3&2sp=u%i} IS_HTTPS=no -ALLOWED_HOSTS=* +ALLOWED_HOSTS=["*"] # SUBPATH=${SUBPATH:-/localhost} DEBUG=yes DJANGO_SETTINGS_MODULE=nrc.conf.docker From aab1197bf7ef181f923ae2ce4c2e7ccb4eee9e1c Mon Sep 17 00:00:00 2001 From: Remy van der Wereld Date: Mon, 24 Jun 2024 16:30:59 +0200 Subject: [PATCH 07/42] Updated bpmn linting --- .github/workflows/bpmn.workflow.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/bpmn.workflow.yml b/.github/workflows/bpmn.workflow.yml index d029b47ba..47cecd798 100644 --- a/.github/workflows/bpmn.workflow.yml +++ b/.github/workflows/bpmn.workflow.yml @@ -11,11 +11,11 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Use Node.js for BPMN Linting - uses: actions/setup-node@v3.1.1 + uses: actions/setup-node@v4 with: - node-version: '12.x' + node-version: '16.x' - name: Install dependencies for BPMN Linting run: npm install -g bpmnlint - name: Linting BPMN models From 2d8a9c92373debf0f4ec5127d948b2dd9fb8d178 Mon Sep 17 00:00:00 2001 From: Remy van der Wereld Date: Mon, 24 Jun 2024 16:44:50 +0200 Subject: [PATCH 08/42] Updated actions --- .github/workflows/e2e-tests.yml | 2 +- .github/workflows/integration-tests.yml | 4 ++-- .github/workflows/main.workflow.yml | 12 ++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index e3cb4e76b..7796b24bc 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -14,7 +14,7 @@ jobs: ################################################### - name: Checkout backend - uses: actions/checkout@v3 + uses: actions/checkout@v4 ################################################### # BACKEND diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 16439fad3..63a386872 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -9,13 +9,13 @@ jobs: # CHECKOUT ################################################### - name: Checkout frontend - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: frontend repository: amsterdam/zaken-frontend - name: Checkout backend - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: backend diff --git a/.github/workflows/main.workflow.yml b/.github/workflows/main.workflow.yml index 8c15f940d..7530f0c77 100644 --- a/.github/workflows/main.workflow.yml +++ b/.github/workflows/main.workflow.yml @@ -9,20 +9,20 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v3.1.2 + uses: actions/setup-python@v5 with: - python-version: 3.9.16 + python-version: 3.10.14 - name: Linting run: bash bin/cleanup_pre_commit.sh - name: Build Docker image - run: docker-compose build -f docker-compose.local.yml + run: docker-compose -f docker-compose.local.yml build - name: Create Docker network run: docker network create zaken_network - name: Create TOP and Zaken Docker network run: docker network create top_and_zaak_backend_bridge - name: Start images - run: docker-compose -f docker-compose.local.yml up -d + run: docker-compose -f docker-compose.local.yml up -d - name: Run Tests - run: docker-compose exec -T zaak-gateway python manage.py test + run: docker-compose -f docker-compose.local.yml exec -T zaak-gateway python manage.py test From 9d9596d2c41d3ddb5f087a6a5b325b6aab670d17 Mon Sep 17 00:00:00 2001 From: Remy van der Wereld Date: Mon, 24 Jun 2024 22:03:25 +0200 Subject: [PATCH 09/42] Updated python version --- .github/workflows/e2e-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 7796b24bc..9fc967ac7 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -31,7 +31,7 @@ jobs: # TEST ################################################### - - run: sudo apt-get install python3.9 + - run: sudo apt-get install python3.10 # Test version - run: python --version From 9c2cac71f97c1279025c880c231222a00c281266 Mon Sep 17 00:00:00 2001 From: Remy van der Wereld Date: Mon, 24 Jun 2024 22:15:23 +0200 Subject: [PATCH 10/42] Added check to migration --- .../0024_alter_default_column_ids_azure.py | 34 ++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/app/apps/cases/migrations/0024_alter_default_column_ids_azure.py b/app/apps/cases/migrations/0024_alter_default_column_ids_azure.py index 10842dfce..e3469c057 100644 --- a/app/apps/cases/migrations/0024_alter_default_column_ids_azure.py +++ b/app/apps/cases/migrations/0024_alter_default_column_ids_azure.py @@ -14,14 +14,40 @@ class Migration(migrations.Migration): sql=""" DO $$ BEGIN - -- cases_casetheme + -- Check if the sequence for cases_casetheme exists IF NOT EXISTS (SELECT FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.relkind = 'S' AND c.relname = 'cases_casetheme_id_seq' AND n.nspname = 'public') THEN CREATE SEQUENCE public.cases_casetheme_id_seq INCREMENT BY 1 START WITH 1 MINVALUE 1 NO MAXVALUE CACHE 1; END IF; + + -- Check if the id column is not an identity column before altering the default + IF NOT EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_name = 'cases_casetheme' + AND column_name = 'id' + AND is_identity = 'YES' + ) THEN + ALTER TABLE cases_casetheme ALTER COLUMN id SET DEFAULT nextval('public.cases_casetheme_id_seq'::regclass); + END IF; END $$; - - ALTER TABLE cases_casetheme ALTER COLUMN id SET DEFAULT nextval('public.cases_casetheme_id_seq'::regclass); - """ + """, + reverse_sql=""" + DO $$ + BEGIN + -- Drop the default for the id column if it exists + IF EXISTS ( + SELECT 1 + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE c.relkind = 'S' + AND c.relname = 'cases_casetheme_id_seq' + AND n.nspname = 'public' + ) THEN + ALTER TABLE cases_casetheme ALTER COLUMN id DROP DEFAULT; + END IF; + END + $$; + """, ) ] From 4331c2b003466eedad1ca224d0b65db841a284f7 Mon Sep 17 00:00:00 2001 From: Remy van der Wereld Date: Mon, 24 Jun 2024 22:28:01 +0200 Subject: [PATCH 11/42] Undo migration --- .../0024_alter_default_column_ids_azure.py | 34 +++---------------- 1 file changed, 4 insertions(+), 30 deletions(-) diff --git a/app/apps/cases/migrations/0024_alter_default_column_ids_azure.py b/app/apps/cases/migrations/0024_alter_default_column_ids_azure.py index e3469c057..10842dfce 100644 --- a/app/apps/cases/migrations/0024_alter_default_column_ids_azure.py +++ b/app/apps/cases/migrations/0024_alter_default_column_ids_azure.py @@ -14,40 +14,14 @@ class Migration(migrations.Migration): sql=""" DO $$ BEGIN - -- Check if the sequence for cases_casetheme exists + -- cases_casetheme IF NOT EXISTS (SELECT FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.relkind = 'S' AND c.relname = 'cases_casetheme_id_seq' AND n.nspname = 'public') THEN CREATE SEQUENCE public.cases_casetheme_id_seq INCREMENT BY 1 START WITH 1 MINVALUE 1 NO MAXVALUE CACHE 1; END IF; - - -- Check if the id column is not an identity column before altering the default - IF NOT EXISTS ( - SELECT 1 - FROM information_schema.columns - WHERE table_name = 'cases_casetheme' - AND column_name = 'id' - AND is_identity = 'YES' - ) THEN - ALTER TABLE cases_casetheme ALTER COLUMN id SET DEFAULT nextval('public.cases_casetheme_id_seq'::regclass); - END IF; - END - $$; - """, - reverse_sql=""" - DO $$ - BEGIN - -- Drop the default for the id column if it exists - IF EXISTS ( - SELECT 1 - FROM pg_class c - JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE c.relkind = 'S' - AND c.relname = 'cases_casetheme_id_seq' - AND n.nspname = 'public' - ) THEN - ALTER TABLE cases_casetheme ALTER COLUMN id DROP DEFAULT; - END IF; END $$; - """, + + ALTER TABLE cases_casetheme ALTER COLUMN id SET DEFAULT nextval('public.cases_casetheme_id_seq'::regclass); + """ ) ] From 2b27a82b12a45e9380ccee1960d28919a54135c0 Mon Sep 17 00:00:00 2001 From: Remy van der Wereld Date: Tue, 25 Jun 2024 09:20:41 +0200 Subject: [PATCH 12/42] Remove migration 24 --- .../0024_alter_default_column_ids_azure.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/app/apps/cases/migrations/0024_alter_default_column_ids_azure.py b/app/apps/cases/migrations/0024_alter_default_column_ids_azure.py index 10842dfce..77177dea7 100644 --- a/app/apps/cases/migrations/0024_alter_default_column_ids_azure.py +++ b/app/apps/cases/migrations/0024_alter_default_column_ids_azure.py @@ -9,19 +9,4 @@ class Migration(migrations.Migration): ("cases", "0023_add_multiple_tags"), ] - operations = [ - migrations.RunSQL( - sql=""" - DO $$ - BEGIN - -- cases_casetheme - IF NOT EXISTS (SELECT FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.relkind = 'S' AND c.relname = 'cases_casetheme_id_seq' AND n.nspname = 'public') THEN - CREATE SEQUENCE public.cases_casetheme_id_seq INCREMENT BY 1 START WITH 1 MINVALUE 1 NO MAXVALUE CACHE 1; - END IF; - END - $$; - - ALTER TABLE cases_casetheme ALTER COLUMN id SET DEFAULT nextval('public.cases_casetheme_id_seq'::regclass); - """ - ) - ] + operations = [] From 2a37e41229219798d18108f76e625f625f3d1eeb Mon Sep 17 00:00:00 2001 From: Remy van der Wereld Date: Tue, 25 Jun 2024 09:57:55 +0200 Subject: [PATCH 13/42] Removed spaghetti --- .../0024_alter_default_column_ids_azure.py | 17 ++++++++++++++++- app/config/settings.py | 19 ------------------- app/config/urls.py | 1 - app/requirements.txt | 1 - 4 files changed, 16 insertions(+), 22 deletions(-) diff --git a/app/apps/cases/migrations/0024_alter_default_column_ids_azure.py b/app/apps/cases/migrations/0024_alter_default_column_ids_azure.py index 77177dea7..10842dfce 100644 --- a/app/apps/cases/migrations/0024_alter_default_column_ids_azure.py +++ b/app/apps/cases/migrations/0024_alter_default_column_ids_azure.py @@ -9,4 +9,19 @@ class Migration(migrations.Migration): ("cases", "0023_add_multiple_tags"), ] - operations = [] + operations = [ + migrations.RunSQL( + sql=""" + DO $$ + BEGIN + -- cases_casetheme + IF NOT EXISTS (SELECT FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.relkind = 'S' AND c.relname = 'cases_casetheme_id_seq' AND n.nspname = 'public') THEN + CREATE SEQUENCE public.cases_casetheme_id_seq INCREMENT BY 1 START WITH 1 MINVALUE 1 NO MAXVALUE CACHE 1; + END IF; + END + $$; + + ALTER TABLE cases_casetheme ALTER COLUMN id SET DEFAULT nextval('public.cases_casetheme_id_seq'::regclass); + """ + ) + ] diff --git a/app/config/settings.py b/app/config/settings.py index 53c6c959c..e8fefaede 100644 --- a/app/config/settings.py +++ b/app/config/settings.py @@ -54,7 +54,6 @@ "drf_spectacular", "django_extensions", "django_filters", - "django_spaghetti", "django_celery_beat", "django_celery_results", "zgw_consumers", @@ -84,24 +83,6 @@ "apps.workflow", ) -# Add apps here to make them appear in the graphing visualisation -SPAGHETTI_SAUCE = { - "apps": [ - "users", - "cases", - "debriefings", - "permits", - "fines", - "addresses", - "visits", - "events", - "summons", - "decisions", - "schedules", - ], - "show_fields": False, -} - DATABASE_HOST = os.getenv("DATABASE_HOST", "database") DATABASE_NAME = os.getenv("DATABASE_NAME", "dev") DATABASE_USER = os.getenv("DATABASE_USER", "dev") diff --git a/app/config/urls.py b/app/config/urls.py index ca98ebe38..770a72f55 100644 --- a/app/config/urls.py +++ b/app/config/urls.py @@ -122,7 +122,6 @@ def get(self, request, *args, **kwargs): ReceiveNotificationView.as_view(), name="notification-callback", ), - path("data-model/", include("django_spaghetti.urls")), path("health/", include("health_check.urls")), path("startup/", is_healthy), re_path(r"^$", view=MyView.as_view(), name="index"), diff --git a/app/requirements.txt b/app/requirements.txt index df6946da6..aba7fcc5f 100644 --- a/app/requirements.txt +++ b/app/requirements.txt @@ -31,7 +31,6 @@ django-permissions-policy==4.20.0 django-redis==5.4.0 django-relativedelta==2.0.0 django-solo==2.2.0 -django-spaghetti-and-meatballs==0.4.2 django-timezone-field==6.1.0 djangorestframework==3.15.2 djangorestframework-simplejwt==5.3.1 From 1a4f27d0955a9b6fbcc497f8aa0937aa368a61b3 Mon Sep 17 00:00:00 2001 From: Remy van der Wereld Date: Tue, 25 Jun 2024 16:24:10 +0200 Subject: [PATCH 14/42] Removed migrations --- .../0024_alter_default_column_ids_azure.py | 17 +-------- .../migrations/0026_update_sequence_pk.py | 20 +---------- .../0002_alter_default_column_ids_azure.py | 17 +-------- .../migrations/0003_update_sequence_pk.py | 20 +---------- .../0005_alter_default_column_ids_azure.py | 17 +-------- .../migrations/0006_update_sequence_pk.py | 20 +---------- .../0009_alter_default_column_ids_azure.py | 22 +----------- .../migrations/0010_update_sequence_pk.py | 35 +------------------ docker-compose.local.yml | 2 +- 9 files changed, 9 insertions(+), 161 deletions(-) diff --git a/app/apps/cases/migrations/0024_alter_default_column_ids_azure.py b/app/apps/cases/migrations/0024_alter_default_column_ids_azure.py index 10842dfce..77177dea7 100644 --- a/app/apps/cases/migrations/0024_alter_default_column_ids_azure.py +++ b/app/apps/cases/migrations/0024_alter_default_column_ids_azure.py @@ -9,19 +9,4 @@ class Migration(migrations.Migration): ("cases", "0023_add_multiple_tags"), ] - operations = [ - migrations.RunSQL( - sql=""" - DO $$ - BEGIN - -- cases_casetheme - IF NOT EXISTS (SELECT FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.relkind = 'S' AND c.relname = 'cases_casetheme_id_seq' AND n.nspname = 'public') THEN - CREATE SEQUENCE public.cases_casetheme_id_seq INCREMENT BY 1 START WITH 1 MINVALUE 1 NO MAXVALUE CACHE 1; - END IF; - END - $$; - - ALTER TABLE cases_casetheme ALTER COLUMN id SET DEFAULT nextval('public.cases_casetheme_id_seq'::regclass); - """ - ) - ] + operations = [] diff --git a/app/apps/cases/migrations/0026_update_sequence_pk.py b/app/apps/cases/migrations/0026_update_sequence_pk.py index cc8b0b239..e527534f5 100644 --- a/app/apps/cases/migrations/0026_update_sequence_pk.py +++ b/app/apps/cases/migrations/0026_update_sequence_pk.py @@ -9,22 +9,4 @@ class Migration(migrations.Migration): ("cases", "0025_remove_unused_tables_azure"), ] - operations = [ - migrations.RunSQL( - sql=""" - DO $$ - DECLARE - max_id BIGINT; - BEGIN - -- Find the maximum id value in the table - SELECT MAX(id) INTO max_id FROM public.cases_casetheme; - - -- If there are no records, set max_id to 0 - IF max_id IS NULL THEN - max_id := 0; - END IF; - EXECUTE FORMAT('ALTER SEQUENCE public.cases_casetheme_id_seq RESTART WITH %s;', max_id + 1); - END $$; - """ - ) - ] + operations = [] diff --git a/app/apps/events/migrations/0002_alter_default_column_ids_azure.py b/app/apps/events/migrations/0002_alter_default_column_ids_azure.py index 0d73b1725..7379dd91d 100644 --- a/app/apps/events/migrations/0002_alter_default_column_ids_azure.py +++ b/app/apps/events/migrations/0002_alter_default_column_ids_azure.py @@ -9,19 +9,4 @@ class Migration(migrations.Migration): ("events", "0001_initial"), ] - operations = [ - migrations.RunSQL( - sql=""" - DO $$ - BEGIN - -- events_caseevent - IF NOT EXISTS (SELECT FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.relkind = 'S' AND c.relname = 'events_caseevent_id_seq' AND n.nspname = 'public') THEN - CREATE SEQUENCE public.events_caseevent_id_seq INCREMENT BY 1 START WITH 1 MINVALUE 1 NO MAXVALUE CACHE 1; - END IF; - END - $$; - - ALTER TABLE events_caseevent ALTER COLUMN id SET DEFAULT nextval('public.events_caseevent_id_seq'::regclass); - """ - ) - ] + operations = [] diff --git a/app/apps/events/migrations/0003_update_sequence_pk.py b/app/apps/events/migrations/0003_update_sequence_pk.py index a3cf25dfd..f19fb8b11 100644 --- a/app/apps/events/migrations/0003_update_sequence_pk.py +++ b/app/apps/events/migrations/0003_update_sequence_pk.py @@ -9,22 +9,4 @@ class Migration(migrations.Migration): ("events", "0002_alter_default_column_ids_azure"), ] - operations = [ - migrations.RunSQL( - sql=""" - DO $$ - DECLARE - max_id BIGINT; - BEGIN - -- Find the maximum id value in the table - SELECT MAX(id) INTO max_id FROM public.events_caseevent; - - -- If there are no records, set max_id to 0 - IF max_id IS NULL THEN - max_id := 0; - END IF; - EXECUTE FORMAT('ALTER SEQUENCE public.events_caseevent_id_seq RESTART WITH %s;', max_id + 1); - END $$; - """ - ) - ] + operations = [] diff --git a/app/apps/schedules/migrations/0005_alter_default_column_ids_azure.py b/app/apps/schedules/migrations/0005_alter_default_column_ids_azure.py index 770023ca2..3c324d8f8 100644 --- a/app/apps/schedules/migrations/0005_alter_default_column_ids_azure.py +++ b/app/apps/schedules/migrations/0005_alter_default_column_ids_azure.py @@ -9,19 +9,4 @@ class Migration(migrations.Migration): ("schedules", "0004_schedule_housing_corporation_combiteam"), ] - operations = [ - migrations.RunSQL( - sql=""" - DO $$ - BEGIN - -- schedules_action - IF NOT EXISTS (SELECT FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.relkind = 'S' AND c.relname = 'schedules_action_id_seq' AND n.nspname = 'public') THEN - CREATE SEQUENCE public.schedules_action_id_seq INCREMENT BY 1 START WITH 1 MINVALUE 1 NO MAXVALUE CACHE 1; - END IF; - END - $$; - - ALTER TABLE schedules_action ALTER COLUMN id SET DEFAULT nextval('public.schedules_action_id_seq'::regclass); - """ - ) - ] + operations = [] diff --git a/app/apps/schedules/migrations/0006_update_sequence_pk.py b/app/apps/schedules/migrations/0006_update_sequence_pk.py index 7174ba1df..cba00f061 100644 --- a/app/apps/schedules/migrations/0006_update_sequence_pk.py +++ b/app/apps/schedules/migrations/0006_update_sequence_pk.py @@ -9,22 +9,4 @@ class Migration(migrations.Migration): ("schedules", "0005_alter_default_column_ids_azure"), ] - operations = [ - migrations.RunSQL( - sql=""" - DO $$ - DECLARE - max_id BIGINT; - BEGIN - -- Find the maximum id value in the table - SELECT MAX(id) INTO max_id FROM public.schedules_action; - - -- If there are no records, set max_id to 0 - IF max_id IS NULL THEN - max_id := 0; - END IF; - EXECUTE FORMAT('ALTER SEQUENCE public.schedules_action_id_seq RESTART WITH %s;', max_id + 1); - END $$; - """ - ) - ] + operations = [] diff --git a/app/apps/workflow/migrations/0009_alter_default_column_ids_azure.py b/app/apps/workflow/migrations/0009_alter_default_column_ids_azure.py index 5c4e31bd1..1ef9805be 100644 --- a/app/apps/workflow/migrations/0009_alter_default_column_ids_azure.py +++ b/app/apps/workflow/migrations/0009_alter_default_column_ids_azure.py @@ -9,24 +9,4 @@ class Migration(migrations.Migration): ("workflow", "0008_workflowoption_enabled_on_case_closed"), ] - operations = [ - migrations.RunSQL( - sql=""" - DO $$ - BEGIN - -- workflow_caseusertask - IF NOT EXISTS (SELECT FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.relkind = 'S' AND c.relname = 'workflow_caseusertask_id_seq' AND n.nspname = 'public') THEN - CREATE SEQUENCE public.workflow_caseusertask_id_seq INCREMENT BY 1 START WITH 1 MINVALUE 1 NO MAXVALUE CACHE 1; - END IF; - -- workflow_caseworkflow - IF NOT EXISTS (SELECT FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.relkind = 'S' AND c.relname = 'workflow_caseworkflow_id_seq' AND n.nspname = 'public') THEN - CREATE SEQUENCE public.workflow_caseworkflow_id_seq INCREMENT BY 1 START WITH 1 MINVALUE 1 NO MAXVALUE CACHE 1; - END IF; - END - $$; - - ALTER TABLE workflow_caseusertask ALTER COLUMN id SET DEFAULT nextval('public.workflow_caseusertask_id_seq'::regclass); - ALTER TABLE workflow_caseworkflow ALTER COLUMN id SET DEFAULT nextval('public.workflow_caseworkflow_id_seq'::regclass); - """ - ) - ] + operations = [] diff --git a/app/apps/workflow/migrations/0010_update_sequence_pk.py b/app/apps/workflow/migrations/0010_update_sequence_pk.py index b949d4e11..54b5b6829 100644 --- a/app/apps/workflow/migrations/0010_update_sequence_pk.py +++ b/app/apps/workflow/migrations/0010_update_sequence_pk.py @@ -9,37 +9,4 @@ class Migration(migrations.Migration): ("workflow", "0009_alter_default_column_ids_azure"), ] - operations = [ - migrations.RunSQL( - sql=""" - DO $$ - DECLARE - max_id BIGINT; - BEGIN - -- workflow_caseusertask - -- Find the maximum id value in the table - SELECT MAX(id) INTO max_id FROM public.workflow_caseusertask; - - -- If there are no records, set max_id to 0 - IF max_id IS NULL THEN - max_id := 0; - END IF; - EXECUTE FORMAT('ALTER SEQUENCE public.workflow_caseusertask_id_seq RESTART WITH %s;', max_id + 1); - END $$; - DO $$ - DECLARE - max_id BIGINT; - BEGIN - -- workflow_caseworkflow - -- Find the maximum id value in the table - SELECT MAX(id) INTO max_id FROM public.workflow_caseworkflow; - - -- If there are no records, set max_id to 0 - IF max_id IS NULL THEN - max_id := 0; - END IF; - EXECUTE FORMAT('ALTER SEQUENCE public.workflow_caseworkflow_id_seq RESTART WITH %s;', max_id + 1); - END $$; - """ - ) - ] + operations = [] diff --git a/docker-compose.local.yml b/docker-compose.local.yml index d6a61e66e..25a922f6b 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -33,7 +33,7 @@ services: database: networks: - zaken_network - image: amsterdam/postgres12 + image: postgis/postgis:14-3.2 shm_size: "512m" ports: - "6409:5432" From b824da1dcc32eaf7b4a3e575caa9b08412ee7bf0 Mon Sep 17 00:00:00 2001 From: Remy van der Wereld Date: Thu, 27 Jun 2024 10:46:44 +0200 Subject: [PATCH 15/42] Updated tests --- app/apps/addresses/mock.py | 34 +++++++++++++++--------- app/apps/addresses/tests/tests_models.py | 30 +++++++++++++-------- 2 files changed, 40 insertions(+), 24 deletions(-) diff --git a/app/apps/addresses/mock.py b/app/apps/addresses/mock.py index db8d79c32..f271716b7 100644 --- a/app/apps/addresses/mock.py +++ b/app/apps/addresses/mock.py @@ -79,21 +79,29 @@ def mock_do_bag_search_id_result_without_links(): } -def mock_get_bag_data_result(): +def mock_get_bag_identificatie_and_stadsdeel_result_without_stadsdeel(): return { - "_stadsdeel": { - "_links": { - "self": { - "href": "https://api.data.amsterdam.nl/gebieden/stadsdeel/03630930000000/" + "_embedded": { + "adresseerbareobjecten": [ + { + "huisnummer": 42, + "identificatie": "123456789" + # No "gebiedenStadsdeelNaam" key } - }, - "_display": "Weesp (S)", - "code": "S", - "naam": "Weesp", - "dataset": "gebieden", - }, + ] + } } -def mock_get_bag_data_result_without_stadsdeel(): - return {} +def mock_get_bag_identificatie_and_stadsdeel_result(): + return { + "_embedded": { + "adresseerbareobjecten": [ + { + "identificatie": "123456789", + "huisnummer": 42, + "gebiedenStadsdeelNaam": "Zuidoost", + } + ] + } + } diff --git a/app/apps/addresses/tests/tests_models.py b/app/apps/addresses/tests/tests_models.py index d85b07282..048a50cba 100644 --- a/app/apps/addresses/tests/tests_models.py +++ b/app/apps/addresses/tests/tests_models.py @@ -3,8 +3,8 @@ from apps.addresses.mock import ( mock_do_bag_search_id_result, mock_do_bag_search_id_result_without_links, - mock_get_bag_data_result, - mock_get_bag_data_result_without_stadsdeel, + mock_get_bag_identificatie_and_stadsdeel_result, + mock_get_bag_identificatie_and_stadsdeel_result_without_stadsdeel, ) from apps.addresses.models import Address, District from django.core import management @@ -22,21 +22,22 @@ def setUp(self): def test_can_create_address(self): """Tests Address object creation without bag mocks""" self.assertEquals(Address.objects.count(), 0) - baker.make(Address) - self.assertEquals(Address.objects.count(), 1) @patch("apps.addresses.models.do_bag_search_by_bag_id") + @patch("apps.addresses.models.do_bag_search_benkagg_by_bag_id") def test_can_create_address_with_bag_result_without_verblijftobject_url( - self, mock_do_bag_search_id, mock_get_bag_data + self, mock_do_bag_search_id, mock_do_bag_search_benkagg_id ): """Tests Address object creation with bag data mocks without verblijftobject url""" mock_do_bag_search_id.return_value = ( mock_do_bag_search_id_result_without_links() ) - mock_get_bag_data.return_value = mock_get_bag_data_result() + mock_do_bag_search_benkagg_id.return_value = ( + mock_get_bag_identificatie_and_stadsdeel_result() + ) self.assertEquals(Address.objects.count(), 0) self.assertEquals(District.objects.count(), 0) @@ -46,13 +47,16 @@ def test_can_create_address_with_bag_result_without_verblijftobject_url( mock_do_bag_search_id.assert_called() @patch("apps.addresses.models.do_bag_search_by_bag_id") + @patch("apps.addresses.models.do_bag_search_benkagg_by_bag_id") def test_can_create_address_with_bag_result_without_verblijftobject_stadsdeel( - self, mock_do_bag_search_id, mock_get_bag_data + self, mock_do_bag_search_id, mock_do_bag_search_benkagg_id ): - """Tests Address object creation with bag data mocks without verblijftobject stadsdeel entry""" + """Tests Address object creation with bag data mocks without stadsdeel entry""" mock_do_bag_search_id.return_value = mock_do_bag_search_id_result() - mock_get_bag_data.return_value = mock_get_bag_data_result_without_stadsdeel() + mock_do_bag_search_benkagg_id.return_value = ( + mock_get_bag_identificatie_and_stadsdeel_result_without_stadsdeel() + ) self.assertEquals(Address.objects.count(), 0) self.assertEquals(District.objects.count(), 0) @@ -62,13 +66,16 @@ def test_can_create_address_with_bag_result_without_verblijftobject_stadsdeel( mock_do_bag_search_id.assert_called() @patch("apps.addresses.models.do_bag_search_by_bag_id") + @patch("apps.addresses.models.do_bag_search_benkagg_by_bag_id") def test_can_create_address_with_bag_result( - self, mock_do_bag_search_id, mock_get_bag_data + self, mock_do_bag_search_id, mock_do_bag_search_benkagg_id ): """Tests Address object creation with bag data mocks""" mock_do_bag_search_id.return_value = mock_do_bag_search_id_result() - mock_get_bag_data.return_value = mock_get_bag_data_result() + mock_do_bag_search_benkagg_id.return_value = ( + mock_get_bag_identificatie_and_stadsdeel_result() + ) self.assertEquals(Address.objects.count(), 0) self.assertEquals(District.objects.count(), 0) @@ -76,6 +83,7 @@ def test_can_create_address_with_bag_result( baker.make(Address) mock_do_bag_search_id.assert_called() + mock_do_bag_search_benkagg_id.assert_called() self.assertEquals(Address.objects.count(), 1) self.assertEquals(District.objects.count(), 1) From 0f3f06ef2a60a14cc46f6dad84c361f8dcb8e99d Mon Sep 17 00:00:00 2001 From: Remy van der Wereld Date: Thu, 27 Jun 2024 13:26:07 +0200 Subject: [PATCH 16/42] Updated tests --- app/apps/addresses/models.py | 4 --- app/apps/addresses/tests/tests_models.py | 36 ++++++------------------ 2 files changed, 9 insertions(+), 31 deletions(-) diff --git a/app/apps/addresses/models.py b/app/apps/addresses/models.py index f62aa20cc..85b5c9c39 100644 --- a/app/apps/addresses/models.py +++ b/app/apps/addresses/models.py @@ -113,7 +113,6 @@ def get_bag_identificatie_and_stadsdeel(self): """ # When moving the import to the beginning of the file, a Django error follows: # ImproperlyConfigured: AUTH_USER_MODEL refers to model 'users.User' that has not been installed. - from utils.exceptions import DistrictNotFoundError response = do_bag_search_benkagg_by_bag_id(self.bag_id) adresseerbareobjecten = response.get("_embedded", {}).get( @@ -133,12 +132,9 @@ def get_bag_identificatie_and_stadsdeel(self): if nummeraanduiding_id: self.nummeraanduiding_id = nummeraanduiding_id - # Add Stadsdeel to address. district_name = found_bag_object.get("gebiedenStadsdeelNaam") if district_name: self.district = District.objects.get_or_create(name=district_name)[0] - else: - raise DistrictNotFoundError(f"API benkagg bag_id: {self.bag_id}") def update_bag_data(self): self.get_bag_address_data() diff --git a/app/apps/addresses/tests/tests_models.py b/app/apps/addresses/tests/tests_models.py index 048a50cba..fc6580ec1 100644 --- a/app/apps/addresses/tests/tests_models.py +++ b/app/apps/addresses/tests/tests_models.py @@ -10,7 +10,6 @@ from django.core import management from django.test import TestCase from model_bakery import baker -from utils.exceptions import DistrictNotFoundError # TODO: Expand this by mocking the BAG API @@ -25,31 +24,10 @@ def test_can_create_address(self): baker.make(Address) self.assertEquals(Address.objects.count(), 1) - @patch("apps.addresses.models.do_bag_search_by_bag_id") @patch("apps.addresses.models.do_bag_search_benkagg_by_bag_id") - def test_can_create_address_with_bag_result_without_verblijftobject_url( - self, mock_do_bag_search_id, mock_do_bag_search_benkagg_id - ): - """Tests Address object creation with bag data mocks without verblijftobject url""" - - mock_do_bag_search_id.return_value = ( - mock_do_bag_search_id_result_without_links() - ) - mock_do_bag_search_benkagg_id.return_value = ( - mock_get_bag_identificatie_and_stadsdeel_result() - ) - - self.assertEquals(Address.objects.count(), 0) - self.assertEquals(District.objects.count(), 0) - - with self.assertRaises(DistrictNotFoundError): - baker.make(Address) - mock_do_bag_search_id.assert_called() - @patch("apps.addresses.models.do_bag_search_by_bag_id") - @patch("apps.addresses.models.do_bag_search_benkagg_by_bag_id") - def test_can_create_address_with_bag_result_without_verblijftobject_stadsdeel( - self, mock_do_bag_search_id, mock_do_bag_search_benkagg_id + def test_can_create_address_with_bag_result_without_stadsdeel( + self, mock_do_bag_search_benkagg_id, mock_do_bag_search_id ): """Tests Address object creation with bag data mocks without stadsdeel entry""" @@ -61,14 +39,18 @@ def test_can_create_address_with_bag_result_without_verblijftobject_stadsdeel( self.assertEquals(Address.objects.count(), 0) self.assertEquals(District.objects.count(), 0) - with self.assertRaises(DistrictNotFoundError): - baker.make(Address) + baker.make(Address) + mock_do_bag_search_id.assert_called() + mock_do_bag_search_benkagg_id.assert_called() + + self.assertEquals(Address.objects.count(), 1) + self.assertEquals(District.objects.count(), 0) @patch("apps.addresses.models.do_bag_search_by_bag_id") @patch("apps.addresses.models.do_bag_search_benkagg_by_bag_id") def test_can_create_address_with_bag_result( - self, mock_do_bag_search_id, mock_do_bag_search_benkagg_id + self, mock_do_bag_search_benkagg_id, mock_do_bag_search_id ): """Tests Address object creation with bag data mocks""" From acfeb70c1a4796122a3320a1c9e36e980996a8ff Mon Sep 17 00:00:00 2001 From: Remy van der Wereld Date: Sat, 29 Jun 2024 22:48:43 +0200 Subject: [PATCH 17/42] Updated urllib --- app/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/requirements.txt b/app/requirements.txt index aba7fcc5f..cac5fbdf5 100644 --- a/app/requirements.txt +++ b/app/requirements.txt @@ -62,7 +62,7 @@ python-crontab==2.5.1 python-dateutil==2.8.2 pytz==2022.1 redis==3.5.3 -requests==2.26.0 +requests==2.32.3 requests-mock==1.9.3 six==1.16.0 soupsieve==2.2.1 @@ -71,7 +71,7 @@ tenacity==8.0.1 text-unidecode==1.3 typing-extensions==4.12.2 uritemplate==3.0.1 -urllib3==1.26.15 +urllib3==2.2.2 uWSGI==2.0.26 vine==5.1.0 wcwidth==0.2.5 From 195b755cfd1a3f9503c6aa01262e570c506f9c03 Mon Sep 17 00:00:00 2001 From: Remy van der Wereld Date: Sat, 29 Jun 2024 22:57:33 +0200 Subject: [PATCH 18/42] Test if server reponse --- .github/workflows/e2e-tests.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 9fc967ac7..ea71f313c 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -39,6 +39,14 @@ jobs: - run: python3 -m pip install -r requirements.txt working-directory: e2e-tests + - name: Check if API server is running + run: | + while ! curl -sSf http://127.0.0.1:8080/api/v1 > /dev/null; do + echo "Waiting for API server to be available..." + sleep 5 + done + shell: bash + - run: API_HOST=http://127.0.0.1:8080/api/v1 python3 -m unittest working-directory: e2e-tests From d5f1abbd69a5ba170555ece3762df936257f1f10 Mon Sep 17 00:00:00 2001 From: Remy van der Wereld Date: Sun, 30 Jun 2024 21:28:52 +0200 Subject: [PATCH 19/42] test Github action --- .github/workflows/e2e-tests.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index ea71f313c..af59c90d3 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -31,23 +31,23 @@ jobs: # TEST ################################################### - - run: sudo apt-get install python3.10 + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: '3.10' - # Test version - - run: python --version + - name: Upgrade pip + run: python3 -m pip install --upgrade pip - - run: python3 -m pip install -r requirements.txt - working-directory: e2e-tests + - name: Test version + run: python --version - - name: Check if API server is running - run: | - while ! curl -sSf http://127.0.0.1:8080/api/v1 > /dev/null; do - echo "Waiting for API server to be available..." - sleep 5 - done - shell: bash + - name: Install requirements + run: python3 -m pip install -r requirements.txt + working-directory: e2e-tests - - run: API_HOST=http://127.0.0.1:8080/api/v1 python3 -m unittest + - name: Run tests + run: API_HOST=http://127.0.0.1:8080/api/v1 python3 -m unittest working-directory: e2e-tests ################################################### @@ -60,8 +60,8 @@ jobs: - if: ${{ failure() }} name: "ON FAILURE: Backend logs" - run: docker logs zaken-backend_zaak-gateway_1 + run: docker logs zaken-backend_zaak-gateway-1 - if: ${{ failure() }} name: "ON FAILURE: Database logs" - run: docker logs zaken-backend_database_1 + run: docker logs zaken-backend_database-1 From 86b612b69846ef81d1e0ec75364ed07ef4e4ecea Mon Sep 17 00:00:00 2001 From: Remy van der Wereld Date: Sun, 30 Jun 2024 21:40:58 +0200 Subject: [PATCH 20/42] Test --- .github/workflows/e2e-tests.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index af59c90d3..8cf3506ae 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -36,9 +36,6 @@ jobs: with: python-version: '3.10' - - name: Upgrade pip - run: python3 -m pip install --upgrade pip - - name: Test version run: python --version @@ -47,7 +44,7 @@ jobs: working-directory: e2e-tests - name: Run tests - run: API_HOST=http://127.0.0.1:8080/api/v1 python3 -m unittest + run: API_HOST=http://localhost:8080/api/v1 python3 -m unittest working-directory: e2e-tests ################################################### From 38ce73cd6b1b3b3fa70e8e56c573540aca429e9a Mon Sep 17 00:00:00 2001 From: Remy van der Wereld Date: Mon, 1 Jul 2024 10:20:26 +0200 Subject: [PATCH 21/42] Updated dependencies --- .github/workflows/e2e-tests.yml | 4 ++-- app/requirements.txt | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 8cf3506ae..57d4ec642 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -57,8 +57,8 @@ jobs: - if: ${{ failure() }} name: "ON FAILURE: Backend logs" - run: docker logs zaken-backend_zaak-gateway-1 + run: docker logs zaken-backend_zaak-gateway_1 - if: ${{ failure() }} name: "ON FAILURE: Database logs" - run: docker logs zaken-backend_database-1 + run: docker logs zaken-backend_database_1 diff --git a/app/requirements.txt b/app/requirements.txt index cac5fbdf5..bc4619823 100644 --- a/app/requirements.txt +++ b/app/requirements.txt @@ -6,7 +6,7 @@ beautifulsoup4==4.10.0 billiard==4.2.0 cached-property==1.5.2 celery==5.4.0 -certifi==2023.5.7 +certifi==2024.6.2 cffi==1.14.6 chardet==4.0.0 click==8.1.2 @@ -42,7 +42,7 @@ freezegun==1.1.0 deepdiff prettyprinter gemma-zds-client==1.0.0 -idna==3.3 +idna==3.7 importlib-metadata==4.8.1 inflection==0.5.1 josepy==1.13.0 @@ -66,7 +66,7 @@ requests==2.32.3 requests-mock==1.9.3 six==1.16.0 soupsieve==2.2.1 -sqlparse==0.4.2 +sqlparse==0.5.0 tenacity==8.0.1 text-unidecode==1.3 typing-extensions==4.12.2 From b2e16fba345bdda1082f5f6df32572e074f3fbfa Mon Sep 17 00:00:00 2001 From: Remy van der Wereld Date: Mon, 1 Jul 2024 10:28:47 +0200 Subject: [PATCH 22/42] Uncommented docker-entrypoint users_user --- app/deploy/docker-entrypoint.development.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/deploy/docker-entrypoint.development.sh b/app/deploy/docker-entrypoint.development.sh index bd2fd497b..774779375 100755 --- a/app/deploy/docker-entrypoint.development.sh +++ b/app/deploy/docker-entrypoint.development.sh @@ -15,7 +15,7 @@ python manage.py collectstatic --no-input chmod -R 777 /static # echo Clear tables -# python manage.py shell -c "from django.db import connection; cursor = connection.cursor(); cursor.execute('drop table if exists "django_migrations" cascade; drop table if exists "django_content_type" cascade; drop table if exists "auth_permission" cascade; drop table if exists "auth_group" cascade; drop table if exists "auth_group_permissions" cascade; drop table if exists "users_user" cascade; drop table if exists "users_user_groups" cascade; drop table if exists "users_user_user_permissions" cascade; drop table if exists "django_admin_log" cascade; drop table if exists "django_session" cascade;'); cursor.close();" +python manage.py shell -c "from django.db import connection; cursor = connection.cursor(); cursor.execute('drop table if exists "django_migrations" cascade; drop table if exists "django_content_type" cascade; drop table if exists "auth_permission" cascade; drop table if exists "auth_group" cascade; drop table if exists "auth_group_permissions" cascade; drop table if exists "users_user" cascade; drop table if exists "users_user_groups" cascade; drop table if exists "users_user_user_permissions" cascade; drop table if exists "django_admin_log" cascade; drop table if exists "django_session" cascade;'); cursor.close();" echo Apply migrations python manage.py migrate --noinput From 7c74f69d3f779a3f99b137449e84b5f3e09b6608 Mon Sep 17 00:00:00 2001 From: Remy van der Wereld Date: Mon, 1 Jul 2024 10:50:06 +0200 Subject: [PATCH 23/42] Test removing volume --- .github/workflows/e2e-tests.yml | 6 +++++- app/deploy/docker-entrypoint.development.sh | 2 +- docker-compose.yml | 3 --- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 57d4ec642..9c900d618 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -23,6 +23,10 @@ jobs: - run: docker network create zaken_network - run: docker network create top_and_zaak_backend_bridge - run: docker-compose -f docker-compose.local.yml up --detach + + + - run: docker volume rm postgresql-data + - run: sleep 30 - run: bash bin/setup_credentials.sh - run: ./e2e-tests/fix_models.sh @@ -44,7 +48,7 @@ jobs: working-directory: e2e-tests - name: Run tests - run: API_HOST=http://localhost:8080/api/v1 python3 -m unittest + run: API_HOST=http://127.0.0.1:8080/api/v1 python3 -m unittest working-directory: e2e-tests ################################################### diff --git a/app/deploy/docker-entrypoint.development.sh b/app/deploy/docker-entrypoint.development.sh index 774779375..bd2fd497b 100755 --- a/app/deploy/docker-entrypoint.development.sh +++ b/app/deploy/docker-entrypoint.development.sh @@ -15,7 +15,7 @@ python manage.py collectstatic --no-input chmod -R 777 /static # echo Clear tables -python manage.py shell -c "from django.db import connection; cursor = connection.cursor(); cursor.execute('drop table if exists "django_migrations" cascade; drop table if exists "django_content_type" cascade; drop table if exists "auth_permission" cascade; drop table if exists "auth_group" cascade; drop table if exists "auth_group_permissions" cascade; drop table if exists "users_user" cascade; drop table if exists "users_user_groups" cascade; drop table if exists "users_user_user_permissions" cascade; drop table if exists "django_admin_log" cascade; drop table if exists "django_session" cascade;'); cursor.close();" +# python manage.py shell -c "from django.db import connection; cursor = connection.cursor(); cursor.execute('drop table if exists "django_migrations" cascade; drop table if exists "django_content_type" cascade; drop table if exists "auth_permission" cascade; drop table if exists "auth_group" cascade; drop table if exists "auth_group_permissions" cascade; drop table if exists "users_user" cascade; drop table if exists "users_user_groups" cascade; drop table if exists "users_user_user_permissions" cascade; drop table if exists "django_admin_log" cascade; drop table if exists "django_session" cascade;'); cursor.close();" echo Apply migrations python manage.py migrate --noinput diff --git a/docker-compose.yml b/docker-compose.yml index 874be84b7..9c9423df3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,8 +9,5 @@ services: - 5678:5678 env_file: - .env - command: bash -c "/app/wait-for.sh zaak-redis:6379 && /app/wait-for.sh database:5432 && /app/deploy/docker-entrypoint.development.sh" - # volumes: - # - ./app:/app stdin_open: true tty: true From bdd0b475d61db8c6027d216ea203407aab9f7d18 Mon Sep 17 00:00:00 2001 From: Remy van der Wereld Date: Mon, 1 Jul 2024 10:55:15 +0200 Subject: [PATCH 24/42] Commented e2e tests --- .github/workflows/e2e-tests.yml | 108 +++++++++++++++----------------- 1 file changed, 52 insertions(+), 56 deletions(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 9c900d618..ecf1db9fd 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -1,68 +1,64 @@ -# action.yml -name: "Run Integration test" -on: - push: - branches: - - develop - pull_request: -jobs: - main: - runs-on: ubuntu-latest - steps: - ################################################### - # CHECKOUT - ################################################### +# # action.yml +# name: "Run Integration test" +# on: +# push: +# branches: +# - develop +# pull_request: +# jobs: +# main: +# runs-on: ubuntu-latest +# steps: +# ################################################### +# # CHECKOUT +# ################################################### - - name: Checkout backend - uses: actions/checkout@v4 +# - name: Checkout backend +# uses: actions/checkout@v4 - ################################################### - # BACKEND - ################################################### +# ################################################### +# # BACKEND +# ################################################### - - run: docker network create zaken_network - - run: docker network create top_and_zaak_backend_bridge - - run: docker-compose -f docker-compose.local.yml up --detach +# - run: docker network create zaken_network +# - run: docker network create top_and_zaak_backend_bridge +# - run: docker-compose -f docker-compose.local.yml up --detach +# - run: sleep 30 +# - run: bash bin/setup_credentials.sh +# - run: ./e2e-tests/fix_models.sh +# ################################################### +# # TEST +# ################################################### - - run: docker volume rm postgresql-data +# - name: Set up Python 3.10 +# uses: actions/setup-python@v5 +# with: +# python-version: '3.10' - - run: sleep 30 - - run: bash bin/setup_credentials.sh - - run: ./e2e-tests/fix_models.sh +# - name: Test version +# run: python --version - ################################################### - # TEST - ################################################### +# - name: Install requirements +# run: python3 -m pip install -r requirements.txt +# working-directory: e2e-tests - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: '3.10' +# - name: Run tests +# run: API_HOST=http://127.0.0.1:8080/api/v1 python3 -m unittest +# working-directory: e2e-tests - - name: Test version - run: python --version +# ################################################### +# # On failure +# ################################################### - - name: Install requirements - run: python3 -m pip install -r requirements.txt - working-directory: e2e-tests +# - if: ${{ failure() }} +# name: "ON FAILURE: Which containers were running while failing?" +# run: docker ps -a - - name: Run tests - run: API_HOST=http://127.0.0.1:8080/api/v1 python3 -m unittest - working-directory: e2e-tests +# - if: ${{ failure() }} +# name: "ON FAILURE: Backend logs" +# run: docker logs zaken-backend_zaak-gateway_1 - ################################################### - # On failure - ################################################### - - - if: ${{ failure() }} - name: "ON FAILURE: Which containers were running while failing?" - run: docker ps -a - - - if: ${{ failure() }} - name: "ON FAILURE: Backend logs" - run: docker logs zaken-backend_zaak-gateway_1 - - - if: ${{ failure() }} - name: "ON FAILURE: Database logs" - run: docker logs zaken-backend_database_1 +# - if: ${{ failure() }} +# name: "ON FAILURE: Database logs" +# run: docker logs zaken-backend_database_1 From 72f153573d4946f59db5189cdaad6b8d29ed03a4 Mon Sep 17 00:00:00 2001 From: Remy van der Wereld Date: Mon, 1 Jul 2024 11:03:06 +0200 Subject: [PATCH 25/42] Tests --- .github/workflows/e2e-tests.yml | 104 ++++++++++---------- .github/workflows/integration-tests.yml | 120 ++++++++++++------------ .github/workflows/main.workflow.yml | 20 ++-- 3 files changed, 122 insertions(+), 122 deletions(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index ecf1db9fd..e9ca1ea45 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -1,64 +1,64 @@ -# # action.yml -# name: "Run Integration test" -# on: -# push: -# branches: -# - develop -# pull_request: -# jobs: -# main: -# runs-on: ubuntu-latest -# steps: -# ################################################### -# # CHECKOUT -# ################################################### +# action.yml +name: "Run e2e-tests" +on: + push: + branches: + - develop + pull_request: +jobs: + main: + runs-on: ubuntu-latest + steps: + ################################################### + # CHECKOUT + ################################################### -# - name: Checkout backend -# uses: actions/checkout@v4 + - name: Checkout backend + uses: actions/checkout@v4 -# ################################################### -# # BACKEND -# ################################################### + ################################################### + # BACKEND + ################################################### -# - run: docker network create zaken_network -# - run: docker network create top_and_zaak_backend_bridge -# - run: docker-compose -f docker-compose.local.yml up --detach -# - run: sleep 30 -# - run: bash bin/setup_credentials.sh -# - run: ./e2e-tests/fix_models.sh + - run: docker network create zaken_network + - run: docker network create top_and_zaak_backend_bridge + - run: docker-compose -f docker-compose.local.yml up --detach + - run: sleep 30 + - run: bash bin/setup_credentials.sh + - run: ./e2e-tests/fix_models.sh -# ################################################### -# # TEST -# ################################################### + ################################################### + # TEST + ################################################### -# - name: Set up Python 3.10 -# uses: actions/setup-python@v5 -# with: -# python-version: '3.10' + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: '3.10' -# - name: Test version -# run: python --version + - name: Test version + run: python --version -# - name: Install requirements -# run: python3 -m pip install -r requirements.txt -# working-directory: e2e-tests + - name: Install requirements + run: python3 -m pip install -r requirements.txt + working-directory: e2e-tests -# - name: Run tests -# run: API_HOST=http://127.0.0.1:8080/api/v1 python3 -m unittest -# working-directory: e2e-tests + - name: Run tests + run: API_HOST=http://127.0.0.1:8080/api/v1 python3 -m unittest + working-directory: e2e-tests -# ################################################### -# # On failure -# ################################################### + ################################################### + # On failure + ################################################### -# - if: ${{ failure() }} -# name: "ON FAILURE: Which containers were running while failing?" -# run: docker ps -a + - if: ${{ failure() }} + name: "ON FAILURE: Which containers were running while failing?" + run: docker ps -a -# - if: ${{ failure() }} -# name: "ON FAILURE: Backend logs" -# run: docker logs zaken-backend_zaak-gateway_1 + - if: ${{ failure() }} + name: "ON FAILURE: Backend logs" + run: docker logs zaken-backend_zaak-gateway_1 -# - if: ${{ failure() }} -# name: "ON FAILURE: Database logs" -# run: docker logs zaken-backend_database_1 + - if: ${{ failure() }} + name: "ON FAILURE: Database logs" + run: docker logs zaken-backend_database_1 diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 63a386872..b44681442 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -1,60 +1,60 @@ -# action.yml -name: 'Start frontend' -on: pull_request -jobs: - main: - runs-on: ubuntu-latest - steps: - ################################################### - # CHECKOUT - ################################################### - - name: Checkout frontend - uses: actions/checkout@v4 - with: - path: frontend - repository: amsterdam/zaken-frontend - - - name: Checkout backend - uses: actions/checkout@v4 - with: - path: backend - - ################################################### - # BACKEND - ################################################### - - run: docker network create zaken_network - working-directory: backend - - - run: docker network create top_and_zaak_backend_bridge - working-directory: backend - - - run: docker-compose -f docker-compose.local.yml up --build --detach - working-directory: backend - - ################################################### - # FRONTEND - ################################################### - - run: npm install - working-directory: frontend - - - run: npm run lint:fix - working-directory: frontend - - - run: npm run start &>/dev/null & - working-directory: frontend - - ################################################### - # On failure - ################################################### - - - if: ${{ failure() }} - name: "ON FAILURE: Which containers were running while failing?" - run: docker ps -a - - - if: ${{ failure() }} - name: "ON FAILURE: Backend logs" - run: docker logs backend_zaak-gateway_1 - - - if: ${{ failure() }} - name: "ON FAILURE: Database logs" - run: docker logs backend_database_1 +# # action.yml +# name: 'Run integration-tests' +# on: pull_request +# jobs: +# main: +# runs-on: ubuntu-latest +# steps: +# ################################################### +# # CHECKOUT +# ################################################### +# - name: Checkout frontend +# uses: actions/checkout@v4 +# with: +# path: frontend +# repository: amsterdam/zaken-frontend + +# - name: Checkout backend +# uses: actions/checkout@v4 +# with: +# path: backend + +# ################################################### +# # BACKEND +# ################################################### +# - run: docker network create zaken_network +# working-directory: backend + +# - run: docker network create top_and_zaak_backend_bridge +# working-directory: backend + +# - run: docker-compose -f docker-compose.local.yml up --build --detach +# working-directory: backend + +# ################################################### +# # FRONTEND +# ################################################### +# - run: npm install +# working-directory: frontend + +# - run: npm run lint:fix +# working-directory: frontend + +# - run: npm run start &>/dev/null & +# working-directory: frontend + +# ################################################### +# # On failure +# ################################################### + +# - if: ${{ failure() }} +# name: "ON FAILURE: Which containers were running while failing?" +# run: docker ps -a + +# - if: ${{ failure() }} +# name: "ON FAILURE: Backend logs" +# run: docker logs backend_zaak-gateway_1 + +# - if: ${{ failure() }} +# name: "ON FAILURE: Database logs" +# run: docker logs backend_database_1 diff --git a/.github/workflows/main.workflow.yml b/.github/workflows/main.workflow.yml index 7530f0c77..b4b6d01a2 100644 --- a/.github/workflows/main.workflow.yml +++ b/.github/workflows/main.workflow.yml @@ -16,13 +16,13 @@ jobs: python-version: 3.10.14 - name: Linting run: bash bin/cleanup_pre_commit.sh - - name: Build Docker image - run: docker-compose -f docker-compose.local.yml build - - name: Create Docker network - run: docker network create zaken_network - - name: Create TOP and Zaken Docker network - run: docker network create top_and_zaak_backend_bridge - - name: Start images - run: docker-compose -f docker-compose.local.yml up -d - - name: Run Tests - run: docker-compose -f docker-compose.local.yml exec -T zaak-gateway python manage.py test + # - name: Build Docker image + # run: docker-compose -f docker-compose.local.yml build + # - name: Create Docker network + # run: docker network create zaken_network + # - name: Create TOP and Zaken Docker network + # run: docker network create top_and_zaak_backend_bridge + # - name: Start images + # run: docker-compose -f docker-compose.local.yml up -d + # - name: Run Tests + # run: docker-compose -f docker-compose.local.yml exec -T zaak-gateway python manage.py test From 79b803fdf6e1cffee3171436748bacfda1a2eaed Mon Sep 17 00:00:00 2001 From: Remy van der Wereld Date: Mon, 1 Jul 2024 11:16:59 +0200 Subject: [PATCH 26/42] Added docker compose tests --- .github/workflows/bpmn.workflow.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/e2e-tests.yml | 2 +- docker-compose.tests.yml | 75 +++++++++++++++++++++++++++ 4 files changed, 78 insertions(+), 3 deletions(-) create mode 100644 docker-compose.tests.yml diff --git a/.github/workflows/bpmn.workflow.yml b/.github/workflows/bpmn.workflow.yml index 47cecd798..0c7fdb352 100644 --- a/.github/workflows/bpmn.workflow.yml +++ b/.github/workflows/bpmn.workflow.yml @@ -1,4 +1,4 @@ -name: 'Validate BPMN' +name: 'BPMN validation' on: push: branches: [ main ] diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index a0c8275a3..45b2f2a8c 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -9,7 +9,7 @@ # the `language` matrix defined below to confirm you have the correct set of # supported CodeQL languages. # -name: "Discovering vulnerabilities using CodeQL" +name: "CodeQL analysis for vulnerabilities " on: push: diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index e9ca1ea45..b3c7bf788 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -22,7 +22,7 @@ jobs: - run: docker network create zaken_network - run: docker network create top_and_zaak_backend_bridge - - run: docker-compose -f docker-compose.local.yml up --detach + - run: docker-compose -f docker-compose.tests.yml up --detach - run: sleep 30 - run: bash bin/setup_credentials.sh - run: ./e2e-tests/fix_models.sh diff --git a/docker-compose.tests.yml b/docker-compose.tests.yml new file mode 100644 index 000000000..12a9dff63 --- /dev/null +++ b/docker-compose.tests.yml @@ -0,0 +1,75 @@ +services: + zaak-gateway: + networks: + - zaken_network + - top_and_zaak_backend_bridge + build: app + hostname: zaak-gateway + image: ${REGISTRY:-127.0.0.1:5001}/${REPOSITORY:-salmagundi/zaken-backend}:${VERSION:-latest} + ports: + - 8080:8000 + - 5678:5678 + depends_on: + - database + - zaak-redis + env_file: + - .env + command: bash -c '/app/deploy/docker-entrypoint.development.sh' + stdin_open: true + tty: true + + zaak-redis: + build: redis + environment: + - REDIS_PASSWORD=password + - REDIS_MAXMEM=100mb + networks: + - zaken_network + ports: + - "6379:6379" + + database: + networks: + - zaken_network + image: postgis/postgis:14-3.2 + shm_size: "512m" + ports: + - "6409:5432" + env_file: + - .env + + zaken_celery_worker: + image: ${REGISTRY:-127.0.0.1:5001}/${REPOSITORY:-salmagundi/zaken-backend}:${VERSION:-latest} + container_name: zaken_celery_worker + hostname: zaken_celery_worker + command: bash -c '/app/celery.sh' + depends_on: + - zaak-gateway + - database + - zaak-redis + env_file: + - .env + networks: + - zaken_network + restart: on-failure + + zaken_celery_beat: + image: ${REGISTRY:-127.0.0.1:5001}/${REPOSITORY:-salmagundi/zaken-backend}:${VERSION:-latest} + container_name: zaken_celery_beat + hostname: zaken_celery_beat + command: sh -c 'celery -A config beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler' + depends_on: + - zaak-gateway + - database + - zaak-redis + env_file: + - .env + networks: + - zaken_network + restart: on-failure + +networks: + zaken_network: + external: true + top_and_zaak_backend_bridge: + external: true From 057b058aa427c626c4e6cb0d5e4f547662d5311d Mon Sep 17 00:00:00 2001 From: Remy van der Wereld Date: Mon, 1 Jul 2024 11:38:01 +0200 Subject: [PATCH 27/42] Removed detach --- .github/workflows/e2e-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index b3c7bf788..11c0ac167 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -22,7 +22,8 @@ jobs: - run: docker network create zaken_network - run: docker network create top_and_zaak_backend_bridge - - run: docker-compose -f docker-compose.tests.yml up --detach + # - run: docker-compose -f docker-compose.tests.yml up --detach + - run: docker-compose -f docker-compose.tests.yml up - run: sleep 30 - run: bash bin/setup_credentials.sh - run: ./e2e-tests/fix_models.sh From ca893ca8660af846bcf6d1cab7ff02d39b2575a4 Mon Sep 17 00:00:00 2001 From: Remy van der Wereld Date: Mon, 1 Jul 2024 12:50:58 +0200 Subject: [PATCH 28/42] Updated docker compose --- docker-compose.local.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.local.yml b/docker-compose.local.yml index 25a922f6b..4d45c5ba7 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -46,7 +46,7 @@ services: image: ${REGISTRY:-127.0.0.1:5001}/${REPOSITORY:-salmagundi/zaken-backend}:${VERSION:-latest} container_name: zaken_celery_worker hostname: zaken_celery_worker - command: bash -c '/app/celery.sh' + command: bash -c "/app/wait-for.sh zaak-gateway:8000 -- /app/celery.sh" depends_on: - zaak-gateway - database @@ -61,7 +61,7 @@ services: image: ${REGISTRY:-127.0.0.1:5001}/${REPOSITORY:-salmagundi/zaken-backend}:${VERSION:-latest} container_name: zaken_celery_beat hostname: zaken_celery_beat - command: sh -c 'celery -A config beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler' + command: sh -c "/app/wait-for.sh zaak-gateway:8000 -- celery -A config beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler" depends_on: - zaak-gateway - database From e78f363c1caa655498bee7fb1fabeffd5b2c3c84 Mon Sep 17 00:00:00 2001 From: Nino Date: Mon, 1 Jul 2024 13:16:45 +0200 Subject: [PATCH 29/42] update waitfor --- .github/workflows/e2e-tests.yml | 4 +- app/wait-for.sh | 140 ++++++++++++++++++++++++++++---- docker-compose.local.yml | 4 +- 3 files changed, 130 insertions(+), 18 deletions(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 11c0ac167..ae7d732e6 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -22,8 +22,8 @@ jobs: - run: docker network create zaken_network - run: docker network create top_and_zaak_backend_bridge - # - run: docker-compose -f docker-compose.tests.yml up --detach - - run: docker-compose -f docker-compose.tests.yml up + - run: docker-compose -f docker-compose.local.yml up --detach + # - run: docker-compose -f docker-compose.tests.yml up - run: sleep 30 - run: bash bin/setup_credentials.sh - run: ./e2e-tests/fix_models.sh diff --git a/app/wait-for.sh b/app/wait-for.sh index c568a2081..ea876dfff 100755 --- a/app/wait-for.sh +++ b/app/wait-for.sh @@ -1,8 +1,34 @@ #!/bin/sh -# Obtained on 07-Oct-2018 from https://github.com/eficode/wait-for +# The MIT License (MIT) +# +# Copyright (c) 2017 Eficode Oy +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +VERSION="2.2.4" + +set -- "$@" -- "$TIMEOUT" "$QUIET" "$PROTOCOL" "$HOST" "$PORT" "$result" TIMEOUT=15 QUIET=0 +# The protocol to make the request with, either "tcp" or "http" +PROTOCOL="tcp" echoerr() { if [ "$QUIET" -ne 1 ]; then printf "%s\n" "$*" 1>&2; fi @@ -12,48 +38,113 @@ usage() { exitcode="$1" cat << USAGE >&2 Usage: - $cmdname host:port [-t timeout] [-- command args] + $0 host:port|url [-t timeout] [-- command args] -q | --quiet Do not output any status messages -t TIMEOUT | --timeout=timeout Timeout in seconds, zero for no timeout + Defaults to 15 seconds + -v | --version Show the version of this tool -- COMMAND ARGS Execute command with args after the test finishes USAGE exit "$exitcode" } wait_for() { - for i in `seq $TIMEOUT` ; do - nc -z "$HOST" "$PORT" > /dev/null 2>&1 + case "$PROTOCOL" in + tcp) + if ! command -v nc >/dev/null; then + echoerr 'nc command is missing!' + exit 1 + fi + ;; + http) + if ! command -v wget >/dev/null; then + echoerr 'wget command is missing!' + exit 1 + fi + ;; + esac + + TIMEOUT_END=$(($(date +%s) + TIMEOUT)) + + while :; do + case "$PROTOCOL" in + tcp) + nc -w 1 -z "$HOST" "$PORT" > /dev/null 2>&1 + ;; + http) + wget --timeout=1 --tries=1 -q "$HOST" -O /dev/null > /dev/null 2>&1 + ;; + *) + echoerr "Unknown protocol '$PROTOCOL'" + exit 1 + ;; + esac result=$? + if [ $result -eq 0 ] ; then - if [ $# -gt 0 ] ; then + if [ $# -gt 7 ] ; then + for result in $(seq $(($# - 7))); do + result=$1 + shift + set -- "$@" "$result" + done + + TIMEOUT=$2 QUIET=$3 PROTOCOL=$4 HOST=$5 PORT=$6 result=$7 + shift 7 exec "$@" fi exit 0 fi + + if [ $TIMEOUT -ne 0 -a $(date +%s) -ge $TIMEOUT_END ]; then + echo "Operation timed out" >&2 + exit 1 + fi + sleep 1 done - echo "Operation timed out" >&2 - exit 1 } -while [ $# -gt 0 ] -do +while :; do case "$1" in + http://*|https://*) + HOST="$1" + PROTOCOL="http" + shift 1 + ;; *:* ) HOST=$(printf "%s\n" "$1"| cut -d : -f 1) PORT=$(printf "%s\n" "$1"| cut -d : -f 2) shift 1 ;; + -v | --version) + echo $VERSION + exit + ;; -q | --quiet) QUIET=1 shift 1 ;; - -t) + -q-*) + QUIET=0 + echoerr "Unknown option: $1" + usage 1 + ;; + -q*) + QUIET=1 + result=$1 + shift 1 + set -- -"${result#-q}" "$@" + ;; + -t | --timeout) TIMEOUT="$2" - if [ "$TIMEOUT" = "" ]; then break; fi shift 2 ;; + -t*) + TIMEOUT="${1#-t}" + shift 1 + ;; --timeout=*) TIMEOUT="${1#*=}" shift 1 @@ -65,16 +156,37 @@ do --help) usage 0 ;; + -*) + QUIET=0 + echoerr "Unknown option: $1" + usage 1 + ;; *) + QUIET=0 echoerr "Unknown argument: $1" usage 1 ;; esac done -if [ "$HOST" = "" -o "$PORT" = "" ]; then - echoerr "Error: you need to provide a host and port to test." - usage 2 +if ! [ "$TIMEOUT" -ge 0 ] 2>/dev/null; then + echoerr "Error: invalid timeout '$TIMEOUT'" + usage 3 fi +case "$PROTOCOL" in + tcp) + if [ "$HOST" = "" ] || [ "$PORT" = "" ]; then + echoerr "Error: you need to provide a host and port to test." + usage 2 + fi + ;; + http) + if [ "$HOST" = "" ]; then + echoerr "Error: you need to provide a host to test." + usage 2 + fi + ;; +esac + wait_for "$@" diff --git a/docker-compose.local.yml b/docker-compose.local.yml index 4d45c5ba7..544fd076e 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -46,7 +46,7 @@ services: image: ${REGISTRY:-127.0.0.1:5001}/${REPOSITORY:-salmagundi/zaken-backend}:${VERSION:-latest} container_name: zaken_celery_worker hostname: zaken_celery_worker - command: bash -c "/app/wait-for.sh zaak-gateway:8000 -- /app/celery.sh" + command: bash -c "/app/wait-for.sh http://zaak-gateway:8000 -- /app/celery.sh" depends_on: - zaak-gateway - database @@ -61,7 +61,7 @@ services: image: ${REGISTRY:-127.0.0.1:5001}/${REPOSITORY:-salmagundi/zaken-backend}:${VERSION:-latest} container_name: zaken_celery_beat hostname: zaken_celery_beat - command: sh -c "/app/wait-for.sh zaak-gateway:8000 -- celery -A config beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler" + command: sh -c "/app/wait-for.sh http://zaak-gateway:8000 -- celery -A config beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler" depends_on: - zaak-gateway - database From f1c30cb2e2fd0d3bd0d865ec86e8b7546aa91d2e Mon Sep 17 00:00:00 2001 From: Remy van der Wereld Date: Mon, 1 Jul 2024 13:42:47 +0200 Subject: [PATCH 30/42] Updated actions --- .github/workflows/integration-tests.yml | 120 ++++++++++++------------ .github/workflows/main.workflow.yml | 22 ++--- app/config/settings.py | 1 + app/wait-for.sh | 24 +---- docker-compose.tests.yml | 75 --------------- 5 files changed, 73 insertions(+), 169 deletions(-) delete mode 100644 docker-compose.tests.yml diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index b44681442..b8dee468d 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -1,60 +1,60 @@ -# # action.yml -# name: 'Run integration-tests' -# on: pull_request -# jobs: -# main: -# runs-on: ubuntu-latest -# steps: -# ################################################### -# # CHECKOUT -# ################################################### -# - name: Checkout frontend -# uses: actions/checkout@v4 -# with: -# path: frontend -# repository: amsterdam/zaken-frontend - -# - name: Checkout backend -# uses: actions/checkout@v4 -# with: -# path: backend - -# ################################################### -# # BACKEND -# ################################################### -# - run: docker network create zaken_network -# working-directory: backend - -# - run: docker network create top_and_zaak_backend_bridge -# working-directory: backend - -# - run: docker-compose -f docker-compose.local.yml up --build --detach -# working-directory: backend - -# ################################################### -# # FRONTEND -# ################################################### -# - run: npm install -# working-directory: frontend - -# - run: npm run lint:fix -# working-directory: frontend - -# - run: npm run start &>/dev/null & -# working-directory: frontend - -# ################################################### -# # On failure -# ################################################### - -# - if: ${{ failure() }} -# name: "ON FAILURE: Which containers were running while failing?" -# run: docker ps -a - -# - if: ${{ failure() }} -# name: "ON FAILURE: Backend logs" -# run: docker logs backend_zaak-gateway_1 - -# - if: ${{ failure() }} -# name: "ON FAILURE: Database logs" -# run: docker logs backend_database_1 +# action.yml +name: 'Run integration-tests' +on: pull_request +jobs: + main: + runs-on: ubuntu-latest + steps: + ################################################### + # CHECKOUT + ################################################### + - name: Checkout frontend + uses: actions/checkout@v4 + with: + path: frontend + repository: amsterdam/zaken-frontend + + - name: Checkout backend + uses: actions/checkout@v4 + with: + path: backend + + ################################################### + # BACKEND + ################################################### + - run: docker network create zaken_network + working-directory: backend + + - run: docker network create top_and_zaak_backend_bridge + working-directory: backend + + - run: docker-compose -f docker-compose.local.yml up --build --detach + working-directory: backend + + ################################################### + # FRONTEND + ################################################### + - run: npm install + working-directory: frontend + + - run: npm run lint:fix + working-directory: frontend + + - run: npm run start &>/dev/null & + working-directory: frontend + + ################################################### + # On failure + ################################################### + + - if: ${{ failure() }} + name: "ON FAILURE: Which containers were running while failing?" + run: docker ps -a + + - if: ${{ failure() }} + name: "ON FAILURE: Backend logs" + run: docker logs backend_zaak-gateway_1 + + - if: ${{ failure() }} + name: "ON FAILURE: Database logs" + run: docker logs backend_database_1 diff --git a/.github/workflows/main.workflow.yml b/.github/workflows/main.workflow.yml index b4b6d01a2..775e7ef0e 100644 --- a/.github/workflows/main.workflow.yml +++ b/.github/workflows/main.workflow.yml @@ -1,4 +1,4 @@ -name: 'Install, lint, test & build' +name: 'Main, lint, build & manage.py test' on: push: branches: [ main ] @@ -16,13 +16,13 @@ jobs: python-version: 3.10.14 - name: Linting run: bash bin/cleanup_pre_commit.sh - # - name: Build Docker image - # run: docker-compose -f docker-compose.local.yml build - # - name: Create Docker network - # run: docker network create zaken_network - # - name: Create TOP and Zaken Docker network - # run: docker network create top_and_zaak_backend_bridge - # - name: Start images - # run: docker-compose -f docker-compose.local.yml up -d - # - name: Run Tests - # run: docker-compose -f docker-compose.local.yml exec -T zaak-gateway python manage.py test + - name: Build Docker image + run: docker-compose -f docker-compose.local.yml build + - name: Create Docker network + run: docker network create zaken_network + - name: Create TOP and Zaken Docker network + run: docker network create top_and_zaak_backend_bridge + - name: Start images + run: docker-compose -f docker-compose.local.yml up -d + - name: Run Tests + run: docker-compose -f docker-compose.local.yml exec -T zaak-gateway python manage.py test diff --git a/app/config/settings.py b/app/config/settings.py index b1e1575ce..9614a3210 100644 --- a/app/config/settings.py +++ b/app/config/settings.py @@ -523,6 +523,7 @@ def get_redis_url(): CELERY_TASK_TRACK_STARTED = True CELERY_TASK_TIME_LIMIT = 30 * 60 CELERY_BROKER_URL = get_redis_url() +CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP = True BROKER_CONNECTION_MAX_RETRIES = None BROKER_CONNECTION_TIMEOUT = 120 BROKER_URL = CELERY_BROKER_URL diff --git a/app/wait-for.sh b/app/wait-for.sh index ea876dfff..5988bd359 100755 --- a/app/wait-for.sh +++ b/app/wait-for.sh @@ -1,26 +1,4 @@ -#!/bin/sh - -# The MIT License (MIT) -# -# Copyright (c) 2017 Eficode Oy -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. +# Copied from https://github.com/eficode/wait-for on 01-07-2024 VERSION="2.2.4" diff --git a/docker-compose.tests.yml b/docker-compose.tests.yml deleted file mode 100644 index 12a9dff63..000000000 --- a/docker-compose.tests.yml +++ /dev/null @@ -1,75 +0,0 @@ -services: - zaak-gateway: - networks: - - zaken_network - - top_and_zaak_backend_bridge - build: app - hostname: zaak-gateway - image: ${REGISTRY:-127.0.0.1:5001}/${REPOSITORY:-salmagundi/zaken-backend}:${VERSION:-latest} - ports: - - 8080:8000 - - 5678:5678 - depends_on: - - database - - zaak-redis - env_file: - - .env - command: bash -c '/app/deploy/docker-entrypoint.development.sh' - stdin_open: true - tty: true - - zaak-redis: - build: redis - environment: - - REDIS_PASSWORD=password - - REDIS_MAXMEM=100mb - networks: - - zaken_network - ports: - - "6379:6379" - - database: - networks: - - zaken_network - image: postgis/postgis:14-3.2 - shm_size: "512m" - ports: - - "6409:5432" - env_file: - - .env - - zaken_celery_worker: - image: ${REGISTRY:-127.0.0.1:5001}/${REPOSITORY:-salmagundi/zaken-backend}:${VERSION:-latest} - container_name: zaken_celery_worker - hostname: zaken_celery_worker - command: bash -c '/app/celery.sh' - depends_on: - - zaak-gateway - - database - - zaak-redis - env_file: - - .env - networks: - - zaken_network - restart: on-failure - - zaken_celery_beat: - image: ${REGISTRY:-127.0.0.1:5001}/${REPOSITORY:-salmagundi/zaken-backend}:${VERSION:-latest} - container_name: zaken_celery_beat - hostname: zaken_celery_beat - command: sh -c 'celery -A config beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler' - depends_on: - - zaak-gateway - - database - - zaak-redis - env_file: - - .env - networks: - - zaken_network - restart: on-failure - -networks: - zaken_network: - external: true - top_and_zaak_backend_bridge: - external: true From d99f7a14d528efcdc627fafdad601576951e6915 Mon Sep 17 00:00:00 2001 From: Nino Date: Mon, 1 Jul 2024 14:22:13 +0200 Subject: [PATCH 31/42] add on failure --- .github/workflows/main.workflow.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/main.workflow.yml b/.github/workflows/main.workflow.yml index 775e7ef0e..7b37320c2 100644 --- a/.github/workflows/main.workflow.yml +++ b/.github/workflows/main.workflow.yml @@ -26,3 +26,15 @@ jobs: run: docker-compose -f docker-compose.local.yml up -d - name: Run Tests run: docker-compose -f docker-compose.local.yml exec -T zaak-gateway python manage.py test + + - if: ${{ failure() }} + name: "ON FAILURE: Which containers were running while failing?" + run: docker ps -a + + - if: ${{ failure() }} + name: "ON FAILURE: Backend logs" + run: docker logs zaken-backend_zaak-gateway_1 + + - if: ${{ failure() }} + name: "ON FAILURE: Database logs" + run: docker logs zaken-backend_database_1 From 63408b0d604e0be7b136145054dca66bea0e7648 Mon Sep 17 00:00:00 2001 From: Nino Date: Mon, 1 Jul 2024 14:42:57 +0200 Subject: [PATCH 32/42] rm detach --- .github/workflows/main.workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.workflow.yml b/.github/workflows/main.workflow.yml index 7b37320c2..b9ec014ed 100644 --- a/.github/workflows/main.workflow.yml +++ b/.github/workflows/main.workflow.yml @@ -23,7 +23,7 @@ jobs: - name: Create TOP and Zaken Docker network run: docker network create top_and_zaak_backend_bridge - name: Start images - run: docker-compose -f docker-compose.local.yml up -d + run: docker-compose -f docker-compose.local.yml up - name: Run Tests run: docker-compose -f docker-compose.local.yml exec -T zaak-gateway python manage.py test From 26a0e454f649cdd17566af32821505077e61e993 Mon Sep 17 00:00:00 2001 From: Nino Date: Mon, 1 Jul 2024 15:06:48 +0200 Subject: [PATCH 33/42] undo --- app/wait-for.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/app/wait-for.sh b/app/wait-for.sh index 5988bd359..1a50ef746 100755 --- a/app/wait-for.sh +++ b/app/wait-for.sh @@ -1,3 +1,4 @@ +#!/bin/sh # Copied from https://github.com/eficode/wait-for on 01-07-2024 VERSION="2.2.4" From 4ae1a5be1692b7bfbb8765ef9654a7dbcd707872 Mon Sep 17 00:00:00 2001 From: Remy van der Wereld Date: Mon, 1 Jul 2024 15:11:43 +0200 Subject: [PATCH 34/42] test multiple actions --- .github/workflows/integration-tests.yml | 120 ++++++++++++------------ 1 file changed, 60 insertions(+), 60 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index b8dee468d..b44681442 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -1,60 +1,60 @@ -# action.yml -name: 'Run integration-tests' -on: pull_request -jobs: - main: - runs-on: ubuntu-latest - steps: - ################################################### - # CHECKOUT - ################################################### - - name: Checkout frontend - uses: actions/checkout@v4 - with: - path: frontend - repository: amsterdam/zaken-frontend - - - name: Checkout backend - uses: actions/checkout@v4 - with: - path: backend - - ################################################### - # BACKEND - ################################################### - - run: docker network create zaken_network - working-directory: backend - - - run: docker network create top_and_zaak_backend_bridge - working-directory: backend - - - run: docker-compose -f docker-compose.local.yml up --build --detach - working-directory: backend - - ################################################### - # FRONTEND - ################################################### - - run: npm install - working-directory: frontend - - - run: npm run lint:fix - working-directory: frontend - - - run: npm run start &>/dev/null & - working-directory: frontend - - ################################################### - # On failure - ################################################### - - - if: ${{ failure() }} - name: "ON FAILURE: Which containers were running while failing?" - run: docker ps -a - - - if: ${{ failure() }} - name: "ON FAILURE: Backend logs" - run: docker logs backend_zaak-gateway_1 - - - if: ${{ failure() }} - name: "ON FAILURE: Database logs" - run: docker logs backend_database_1 +# # action.yml +# name: 'Run integration-tests' +# on: pull_request +# jobs: +# main: +# runs-on: ubuntu-latest +# steps: +# ################################################### +# # CHECKOUT +# ################################################### +# - name: Checkout frontend +# uses: actions/checkout@v4 +# with: +# path: frontend +# repository: amsterdam/zaken-frontend + +# - name: Checkout backend +# uses: actions/checkout@v4 +# with: +# path: backend + +# ################################################### +# # BACKEND +# ################################################### +# - run: docker network create zaken_network +# working-directory: backend + +# - run: docker network create top_and_zaak_backend_bridge +# working-directory: backend + +# - run: docker-compose -f docker-compose.local.yml up --build --detach +# working-directory: backend + +# ################################################### +# # FRONTEND +# ################################################### +# - run: npm install +# working-directory: frontend + +# - run: npm run lint:fix +# working-directory: frontend + +# - run: npm run start &>/dev/null & +# working-directory: frontend + +# ################################################### +# # On failure +# ################################################### + +# - if: ${{ failure() }} +# name: "ON FAILURE: Which containers were running while failing?" +# run: docker ps -a + +# - if: ${{ failure() }} +# name: "ON FAILURE: Backend logs" +# run: docker logs backend_zaak-gateway_1 + +# - if: ${{ failure() }} +# name: "ON FAILURE: Database logs" +# run: docker logs backend_database_1 From 93fdfa15a8f39212b210884c4ae45c31b56506c8 Mon Sep 17 00:00:00 2001 From: Remy van der Wereld Date: Mon, 1 Jul 2024 15:17:27 +0200 Subject: [PATCH 35/42] Test --- .github/workflows/main.workflow.yml | 38 ++++++++++++++--------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/main.workflow.yml b/.github/workflows/main.workflow.yml index b9ec014ed..b4c58d9b3 100644 --- a/.github/workflows/main.workflow.yml +++ b/.github/workflows/main.workflow.yml @@ -16,25 +16,25 @@ jobs: python-version: 3.10.14 - name: Linting run: bash bin/cleanup_pre_commit.sh - - name: Build Docker image - run: docker-compose -f docker-compose.local.yml build - - name: Create Docker network - run: docker network create zaken_network - - name: Create TOP and Zaken Docker network - run: docker network create top_and_zaak_backend_bridge - - name: Start images - run: docker-compose -f docker-compose.local.yml up - - name: Run Tests - run: docker-compose -f docker-compose.local.yml exec -T zaak-gateway python manage.py test + # - name: Build Docker image + # run: docker-compose -f docker-compose.local.yml build + # - name: Create Docker network + # run: docker network create zaken_network + # - name: Create TOP and Zaken Docker network + # run: docker network create top_and_zaak_backend_bridge + # - name: Start images + # run: docker-compose -f docker-compose.local.yml up + # - name: Run Tests + # run: docker-compose -f docker-compose.local.yml exec -T zaak-gateway python manage.py test - - if: ${{ failure() }} - name: "ON FAILURE: Which containers were running while failing?" - run: docker ps -a + # - if: ${{ failure() }} + # name: "ON FAILURE: Which containers were running while failing?" + # run: docker ps -a - - if: ${{ failure() }} - name: "ON FAILURE: Backend logs" - run: docker logs zaken-backend_zaak-gateway_1 + # - if: ${{ failure() }} + # name: "ON FAILURE: Backend logs" + # run: docker logs zaken-backend_zaak-gateway_1 - - if: ${{ failure() }} - name: "ON FAILURE: Database logs" - run: docker logs zaken-backend_database_1 + # - if: ${{ failure() }} + # name: "ON FAILURE: Database logs" + # run: docker logs zaken-backend_database_1 From 92bc5d358cbae309226e931175a445b9cff2a3d8 Mon Sep 17 00:00:00 2001 From: Nino Date: Mon, 1 Jul 2024 15:31:44 +0200 Subject: [PATCH 36/42] Revert "add on failure" This reverts commit d99f7a14d528efcdc627fafdad601576951e6915. --- .github/workflows/integration-tests.yml | 120 ++++++++++++------------ .github/workflows/main.workflow.yml | 32 ++----- app/wait-for.sh | 1 - 3 files changed, 70 insertions(+), 83 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index b44681442..b8dee468d 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -1,60 +1,60 @@ -# # action.yml -# name: 'Run integration-tests' -# on: pull_request -# jobs: -# main: -# runs-on: ubuntu-latest -# steps: -# ################################################### -# # CHECKOUT -# ################################################### -# - name: Checkout frontend -# uses: actions/checkout@v4 -# with: -# path: frontend -# repository: amsterdam/zaken-frontend - -# - name: Checkout backend -# uses: actions/checkout@v4 -# with: -# path: backend - -# ################################################### -# # BACKEND -# ################################################### -# - run: docker network create zaken_network -# working-directory: backend - -# - run: docker network create top_and_zaak_backend_bridge -# working-directory: backend - -# - run: docker-compose -f docker-compose.local.yml up --build --detach -# working-directory: backend - -# ################################################### -# # FRONTEND -# ################################################### -# - run: npm install -# working-directory: frontend - -# - run: npm run lint:fix -# working-directory: frontend - -# - run: npm run start &>/dev/null & -# working-directory: frontend - -# ################################################### -# # On failure -# ################################################### - -# - if: ${{ failure() }} -# name: "ON FAILURE: Which containers were running while failing?" -# run: docker ps -a - -# - if: ${{ failure() }} -# name: "ON FAILURE: Backend logs" -# run: docker logs backend_zaak-gateway_1 - -# - if: ${{ failure() }} -# name: "ON FAILURE: Database logs" -# run: docker logs backend_database_1 +# action.yml +name: 'Run integration-tests' +on: pull_request +jobs: + main: + runs-on: ubuntu-latest + steps: + ################################################### + # CHECKOUT + ################################################### + - name: Checkout frontend + uses: actions/checkout@v4 + with: + path: frontend + repository: amsterdam/zaken-frontend + + - name: Checkout backend + uses: actions/checkout@v4 + with: + path: backend + + ################################################### + # BACKEND + ################################################### + - run: docker network create zaken_network + working-directory: backend + + - run: docker network create top_and_zaak_backend_bridge + working-directory: backend + + - run: docker-compose -f docker-compose.local.yml up --build --detach + working-directory: backend + + ################################################### + # FRONTEND + ################################################### + - run: npm install + working-directory: frontend + + - run: npm run lint:fix + working-directory: frontend + + - run: npm run start &>/dev/null & + working-directory: frontend + + ################################################### + # On failure + ################################################### + + - if: ${{ failure() }} + name: "ON FAILURE: Which containers were running while failing?" + run: docker ps -a + + - if: ${{ failure() }} + name: "ON FAILURE: Backend logs" + run: docker logs backend_zaak-gateway_1 + + - if: ${{ failure() }} + name: "ON FAILURE: Database logs" + run: docker logs backend_database_1 diff --git a/.github/workflows/main.workflow.yml b/.github/workflows/main.workflow.yml index b4c58d9b3..775e7ef0e 100644 --- a/.github/workflows/main.workflow.yml +++ b/.github/workflows/main.workflow.yml @@ -16,25 +16,13 @@ jobs: python-version: 3.10.14 - name: Linting run: bash bin/cleanup_pre_commit.sh - # - name: Build Docker image - # run: docker-compose -f docker-compose.local.yml build - # - name: Create Docker network - # run: docker network create zaken_network - # - name: Create TOP and Zaken Docker network - # run: docker network create top_and_zaak_backend_bridge - # - name: Start images - # run: docker-compose -f docker-compose.local.yml up - # - name: Run Tests - # run: docker-compose -f docker-compose.local.yml exec -T zaak-gateway python manage.py test - - # - if: ${{ failure() }} - # name: "ON FAILURE: Which containers were running while failing?" - # run: docker ps -a - - # - if: ${{ failure() }} - # name: "ON FAILURE: Backend logs" - # run: docker logs zaken-backend_zaak-gateway_1 - - # - if: ${{ failure() }} - # name: "ON FAILURE: Database logs" - # run: docker logs zaken-backend_database_1 + - name: Build Docker image + run: docker-compose -f docker-compose.local.yml build + - name: Create Docker network + run: docker network create zaken_network + - name: Create TOP and Zaken Docker network + run: docker network create top_and_zaak_backend_bridge + - name: Start images + run: docker-compose -f docker-compose.local.yml up -d + - name: Run Tests + run: docker-compose -f docker-compose.local.yml exec -T zaak-gateway python manage.py test diff --git a/app/wait-for.sh b/app/wait-for.sh index 1a50ef746..5988bd359 100755 --- a/app/wait-for.sh +++ b/app/wait-for.sh @@ -1,4 +1,3 @@ -#!/bin/sh # Copied from https://github.com/eficode/wait-for on 01-07-2024 VERSION="2.2.4" From e85f263e9e652c891a0bf1da3563d50365de7ceb Mon Sep 17 00:00:00 2001 From: Nino Date: Tue, 2 Jul 2024 10:16:19 +0200 Subject: [PATCH 37/42] separate entrypoint files --- .../git revert --no-commit f1c30cb2e2fd0d3bd | 1 + app/deploy/docker-entrypoint.celery.sh | 6 ++++++ app/deploy/docker-entrypoint.development.sh | 14 ++++---------- docker-compose.local.yml | 5 ++++- 4 files changed, 15 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/git revert --no-commit f1c30cb2e2fd0d3bd create mode 100755 app/deploy/docker-entrypoint.celery.sh diff --git a/.github/workflows/git revert --no-commit f1c30cb2e2fd0d3bd b/.github/workflows/git revert --no-commit f1c30cb2e2fd0d3bd new file mode 100644 index 000000000..f8b7d3cbf --- /dev/null +++ b/.github/workflows/git revert --no-commit f1c30cb2e2fd0d3bd @@ -0,0 +1 @@ +git revert --no-commit f1c30cb2e2fd0d3bd0d865ec86e8b7546aa91d2e..HEAD diff --git a/app/deploy/docker-entrypoint.celery.sh b/app/deploy/docker-entrypoint.celery.sh new file mode 100755 index 000000000..7be07b30b --- /dev/null +++ b/app/deploy/docker-entrypoint.celery.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -u # crash on missing env variables +set -e # stop on any error +set -x + +exec "$@" diff --git a/app/deploy/docker-entrypoint.development.sh b/app/deploy/docker-entrypoint.development.sh index bd2fd497b..d1f6daec3 100755 --- a/app/deploy/docker-entrypoint.development.sh +++ b/app/deploy/docker-entrypoint.development.sh @@ -1,4 +1,3 @@ -# Same as entrypoint.sh, but with py-auto-reload enabled to allow auto reloading when making changes during development #!/usr/bin/env bash set -u # crash on missing env variables set -e # stop on any error @@ -10,23 +9,18 @@ until PGPASSWORD=$DATABASE_PASSWORD psql -h $DATABASE_HOST -U $DATABASE_USER -c done echo "Postgres is up!" + + echo Collecting static files python manage.py collectstatic --no-input -chmod -R 777 /static - -# echo Clear tables -# python manage.py shell -c "from django.db import connection; cursor = connection.cursor(); cursor.execute('drop table if exists "django_migrations" cascade; drop table if exists "django_content_type" cascade; drop table if exists "auth_permission" cascade; drop table if exists "auth_group" cascade; drop table if exists "auth_group_permissions" cascade; drop table if exists "users_user" cascade; drop table if exists "users_user_groups" cascade; drop table if exists "users_user_user_permissions" cascade; drop table if exists "django_admin_log" cascade; drop table if exists "django_session" cascade;'); cursor.close();" echo Apply migrations python manage.py migrate --noinput echo Axes check python manage.py check +python manage.py axes_reset python manage.py loaddata fixture -# echo Create root user -# python manage.py shell -c "from django.contrib.auth import get_user_model; get_user_model().objects.create_superuser('admin@admin.com', 'admin')" - -# opens up a port for attaching a remote debugging service using debugpy -exec python -m debugpy --listen 0.0.0.0:5678 ./manage.py runserver 0.0.0.0:8000 +exec "$@" diff --git a/docker-compose.local.yml b/docker-compose.local.yml index 544fd076e..244887258 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -14,7 +14,8 @@ services: - zaak-redis env_file: - .env - command: bash -c '/app/deploy/docker-entrypoint.development.sh' + entrypoint: /app/deploy/docker-entrypoint.development.sh + command: bash -c 'uwsgi --ini /app/deploy/config.ini' volumes: - ./app:/app stdin_open: true @@ -46,6 +47,7 @@ services: image: ${REGISTRY:-127.0.0.1:5001}/${REPOSITORY:-salmagundi/zaken-backend}:${VERSION:-latest} container_name: zaken_celery_worker hostname: zaken_celery_worker + entrypoint: /app/deploy/docker-entrypoint.celery.sh command: bash -c "/app/wait-for.sh http://zaak-gateway:8000 -- /app/celery.sh" depends_on: - zaak-gateway @@ -61,6 +63,7 @@ services: image: ${REGISTRY:-127.0.0.1:5001}/${REPOSITORY:-salmagundi/zaken-backend}:${VERSION:-latest} container_name: zaken_celery_beat hostname: zaken_celery_beat + entrypoint: /app/deploy/docker-entrypoint.celery.sh command: sh -c "/app/wait-for.sh http://zaak-gateway:8000 -- celery -A config beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler" depends_on: - zaak-gateway From d4ec599094e02cd683930dca91c4624a1fcd4066 Mon Sep 17 00:00:00 2001 From: Remy van der Wereld Date: Tue, 2 Jul 2024 16:54:57 +0200 Subject: [PATCH 38/42] Action and entrypoint changes --- .github/workflows/codeql-analysis.yml | 7 +--- .github/workflows/main.workflow.yml | 2 +- app/deploy/docker-entrypoint.development.sh | 4 +- bin/setup_credentials.sh | 2 +- docker-compose.local.yml | 44 ++++++++++----------- 5 files changed, 28 insertions(+), 31 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 45b2f2a8c..c3c8d06a4 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -40,14 +40,11 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.9.16' - - # Update docker base image for a higher python version than 3.9 and remove this line. - - name: Install specific pip version 23.1.2 for python 3.9 - run: python -m pip install pip==23.1.2 + python-version: '3.10' - name: Install dependencies run: | + python -m pip install --upgrade pip if [ -f ./app/requirements.txt ]; echo "Installing python dependencies" then pip install -r ./app/requirements.txt; diff --git a/.github/workflows/main.workflow.yml b/.github/workflows/main.workflow.yml index 775e7ef0e..79be4706c 100644 --- a/.github/workflows/main.workflow.yml +++ b/.github/workflows/main.workflow.yml @@ -13,7 +13,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.10.14 + python-version: '3.10' - name: Linting run: bash bin/cleanup_pre_commit.sh - name: Build Docker image diff --git a/app/deploy/docker-entrypoint.development.sh b/app/deploy/docker-entrypoint.development.sh index d1f6daec3..6a3434f1f 100755 --- a/app/deploy/docker-entrypoint.development.sh +++ b/app/deploy/docker-entrypoint.development.sh @@ -5,7 +5,7 @@ set -x until PGPASSWORD=$DATABASE_PASSWORD psql -h $DATABASE_HOST -U $DATABASE_USER -c '\q'; do echo "Postgres is unavailable - sleeping" - sleep 1 + sleep 5 done echo "Postgres is up!" @@ -23,4 +23,4 @@ python manage.py axes_reset python manage.py loaddata fixture -exec "$@" +exec python -m debugpy --listen 0.0.0.0:5678 ./manage.py runserver 0.0.0.0:8000 diff --git a/bin/setup_credentials.sh b/bin/setup_credentials.sh index 19530925b..c3c81ffe6 100755 --- a/bin/setup_credentials.sh +++ b/bin/setup_credentials.sh @@ -1,2 +1,2 @@ # Creates a superuser for the zaak-gateway backend -echo "from django.contrib.auth import get_user_model; get_user_model().objects.create_superuser('admin@admin.com', 'admin')" | docker-compose -f docker-compose.local.yml run -T --rm zaak-gateway python manage.py shell +echo "from django.contrib.auth import get_user_model; get_user_model().objects.create_superuser('admin@admin.com', 'insecure')" | docker-compose -f docker-compose.local.yml run -T --rm zaak-gateway python manage.py shell diff --git a/docker-compose.local.yml b/docker-compose.local.yml index 244887258..086f58d42 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -1,26 +1,4 @@ services: - zaak-gateway: - networks: - - zaken_network - - top_and_zaak_backend_bridge - build: app - hostname: zaak-gateway - image: ${REGISTRY:-127.0.0.1:5001}/${REPOSITORY:-salmagundi/zaken-backend}:${VERSION:-latest} - ports: - - 8080:8000 - - 5678:5678 - depends_on: - - database - - zaak-redis - env_file: - - .env - entrypoint: /app/deploy/docker-entrypoint.development.sh - command: bash -c 'uwsgi --ini /app/deploy/config.ini' - volumes: - - ./app:/app - stdin_open: true - tty: true - zaak-redis: build: redis environment: @@ -43,6 +21,28 @@ services: volumes: - postgresql-data:/var/lib/postgresql/data + zaak-gateway: + networks: + - zaken_network + - top_and_zaak_backend_bridge + build: app + hostname: zaak-gateway + image: ${REGISTRY:-127.0.0.1:5001}/${REPOSITORY:-salmagundi/zaken-backend}:${VERSION:-latest} + ports: + - 8080:8000 + - 5678:5678 + depends_on: + - database + - zaak-redis + env_file: + - .env + entrypoint: /app/deploy/docker-entrypoint.development.sh + command: bash -c 'uwsgi --ini /app/deploy/config.ini' + volumes: + - ./app:/app + stdin_open: true + tty: true + zaken_celery_worker: image: ${REGISTRY:-127.0.0.1:5001}/${REPOSITORY:-salmagundi/zaken-backend}:${VERSION:-latest} container_name: zaken_celery_worker From 75507660c9d0ae079fd383f111676c638e10fdec Mon Sep 17 00:00:00 2001 From: Remy van der Wereld Date: Tue, 2 Jul 2024 21:18:07 +0200 Subject: [PATCH 39/42] Updated migration with default user manager --- README.md | 2 +- app/apps/addresses/models.py | 4 +- app/apps/users/auth.py | 21 ------ .../migrations/0007_remove_user_manager.py | 20 ++++++ app/apps/users/models.py | 3 - app/apps/users/user_manager.py | 65 ++++++++++--------- 6 files changed, 56 insertions(+), 59 deletions(-) create mode 100644 app/apps/users/migrations/0007_remove_user_manager.py diff --git a/README.md b/README.md index af7424694..5a5dd94c3 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ This will create superuser admin account with the following credentials ```bash email: admin@admin.com -password: admin +password: insecure ``` Visit the Admin at http://localhost:8080/admin/ diff --git a/app/apps/addresses/models.py b/app/apps/addresses/models.py index 85b5c9c39..5a9b4d8d5 100644 --- a/app/apps/addresses/models.py +++ b/app/apps/addresses/models.py @@ -109,10 +109,8 @@ def get_bag_identificatie_and_stadsdeel(self): """ Retrieves the identificatie(nummeraanduiding_id) and stadsdeel of an address by bag_id. nummeraanduiding_id is needed for BRP. - stadsdeel is needed for filtering. + Stadsdeel is needed for filtering. """ - # When moving the import to the beginning of the file, a Django error follows: - # ImproperlyConfigured: AUTH_USER_MODEL refers to model 'users.User' that has not been installed. response = do_bag_search_benkagg_by_bag_id(self.bag_id) adresseerbareobjecten = response.get("_embedded", {}).get( diff --git a/app/apps/users/auth.py b/app/apps/users/auth.py index 7c80a0d30..ee7632d73 100644 --- a/app/apps/users/auth.py +++ b/app/apps/users/auth.py @@ -10,27 +10,6 @@ LOGGER = logging.getLogger(__name__) - -# TODO: email lowercase needs testing first -class AppsOIDCAuthenticationBackend(OIDCAuthenticationBackend): - def get_userinfo(self, access_token, id_token, payload): - user_info = super().get_userinfo(access_token, id_token, payload) - - # make sure email is lower case - user_info["email"] = user_info.get("email", "").lower() - - return user_info - - def filter_users_by_claims(self, claims): - """Return all users matching the specified email.""" - email = claims.get("email") - if not email: - return self.UserModel.objects.none() - - users = self.UserModel.objects.filter(email__exact=email) - return users - - if settings.LOCAL_DEVELOPMENT_AUTHENTICATION: AuthenticationBackend = DevelopmentAuthenticationBackend AuthenticationClass = JWTAuthentication diff --git a/app/apps/users/migrations/0007_remove_user_manager.py b/app/apps/users/migrations/0007_remove_user_manager.py new file mode 100644 index 000000000..739b18df0 --- /dev/null +++ b/app/apps/users/migrations/0007_remove_user_manager.py @@ -0,0 +1,20 @@ +# Generated by Django 4.2.13 on 2024-07-02 15:09 + +import django.contrib.auth.models +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("users", "0006_alter_permission_options"), + ] + + operations = [ + migrations.AlterModelManagers( + name="user", + managers=[ + ("objects", django.contrib.auth.models.UserManager()), + ], + ), + ] diff --git a/app/apps/users/models.py b/app/apps/users/models.py index ce4230da9..9477f21ad 100644 --- a/app/apps/users/models.py +++ b/app/apps/users/models.py @@ -1,6 +1,5 @@ import uuid -from apps.users.user_manager import UserManager from django.contrib.auth.models import AbstractUser, Group from django.db import models from django.db.models.base import Model @@ -46,8 +45,6 @@ class Meta: USERNAME_FIELD = "email" REQUIRED_FIELDS = [] - objects = UserManager() - @property def full_name(self): """ diff --git a/app/apps/users/user_manager.py b/app/apps/users/user_manager.py index fa65c08f5..7d2b9faf6 100644 --- a/app/apps/users/user_manager.py +++ b/app/apps/users/user_manager.py @@ -2,34 +2,37 @@ class UserManager(BaseUserManager): - """Define a model manager for User model with no username field.""" - - use_in_migrations = True - - def _create_user(self, email, password, **extra_fields): - """Create and save a User with the given email and password.""" - if not email: - raise ValueError("The given email must be set") - email = self.normalize_email(email) - user = self.model(email=email, **extra_fields) - user.set_password(password) - user.save(using=self._db) - return user - - def create_user(self, email, password=None, **extra_fields): - """Create and save a regular User with the given email and password.""" - extra_fields.setdefault("is_staff", False) - extra_fields.setdefault("is_superuser", False) - return self._create_user(email, password, **extra_fields) - - def create_superuser(self, email, password, **extra_fields): - """Create and save a SuperUser with the given email and password.""" - extra_fields.setdefault("is_staff", True) - extra_fields.setdefault("is_superuser", True) - - if extra_fields.get("is_staff") is not True: - raise ValueError("Superuser must have is_staff=True.") - if extra_fields.get("is_superuser") is not True: - raise ValueError("Superuser must have is_superuser=True.") - - return self._create_user(email, password, **extra_fields) + """ + This model isn't used anymore but can't be deleted because of the reference in 0001_initial.py + If the migrations are applied in production, this can be removed. + """ + + # use_in_migrations = True + + # def _create_user(self, email, password, **extra_fields): + # """Create and save a User with the given email and password.""" + # if not email: + # raise ValueError("The given email must be set") + # email = self.normalize_email(email) + # user = self.model(email=email, **extra_fields) + # user.set_password(password) + # user.save(using=self._db) + # return user + + # def create_user(self, email, password=None, **extra_fields): + # """Create and save a regular User with the given email and password.""" + # extra_fields.setdefault("is_staff", False) + # extra_fields.setdefault("is_superuser", False) + # return self._create_user(email, password, **extra_fields) + + # def create_superuser(self, email, password, **extra_fields): + # """Create and save a SuperUser with the given email and password.""" + # extra_fields.setdefault("is_staff", True) + # extra_fields.setdefault("is_superuser", True) + + # if extra_fields.get("is_staff") is not True: + # raise ValueError("Superuser must have is_staff=True.") + # if extra_fields.get("is_superuser") is not True: + # raise ValueError("Superuser must have is_superuser=True.") + + # return self._create_user(email, password, **extra_fields) From 5279f51f80159521c21421878d7ed9eef2552c36 Mon Sep 17 00:00:00 2001 From: Nino Date: Wed, 3 Jul 2024 09:46:06 +0200 Subject: [PATCH 40/42] fix test --- app/deploy/docker-entrypoint.development.sh | 2 +- docker-compose.local.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/deploy/docker-entrypoint.development.sh b/app/deploy/docker-entrypoint.development.sh index 6a3434f1f..90dee9d53 100755 --- a/app/deploy/docker-entrypoint.development.sh +++ b/app/deploy/docker-entrypoint.development.sh @@ -23,4 +23,4 @@ python manage.py axes_reset python manage.py loaddata fixture -exec python -m debugpy --listen 0.0.0.0:5678 ./manage.py runserver 0.0.0.0:8000 +exec "$@" diff --git a/docker-compose.local.yml b/docker-compose.local.yml index 086f58d42..5b3df1aed 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -37,7 +37,7 @@ services: env_file: - .env entrypoint: /app/deploy/docker-entrypoint.development.sh - command: bash -c 'uwsgi --ini /app/deploy/config.ini' + command: python -m debugpy --listen 0.0.0.0:5678 ./manage.py runserver 0.0.0.0:8000 volumes: - ./app:/app stdin_open: true From 25d2b285994fd45f36521b74c48286536b1c4670 Mon Sep 17 00:00:00 2001 From: Remy van der Wereld Date: Wed, 3 Jul 2024 10:25:41 +0200 Subject: [PATCH 41/42] Added script for adding user groups --- README.md | 6 ++++++ bin/setup_user_groups.sh | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100755 bin/setup_user_groups.sh diff --git a/README.md b/README.md index 5a5dd94c3..5553961bf 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,12 @@ Visit the Admin at http://localhost:8080/admin/ Check the health page to see if all services are up and running: http://localhost:8080/health +To create all necessary user groups run the following command: + +```bash +bash bin/setup_user_groups.sh +``` + ## Running tests Set LOCAL_DEVELOPMENT_AUTHENTICATION environment variable to True (default) diff --git a/bin/setup_user_groups.sh b/bin/setup_user_groups.sh new file mode 100755 index 000000000..01bd136d6 --- /dev/null +++ b/bin/setup_user_groups.sh @@ -0,0 +1,36 @@ +echo "from apps.users.models import User, UserGroup +from django.contrib.auth.models import Permission + +(group, _) = UserGroup.objects.get_or_create(name='PROJECTMEDEWERKER', display_name='Projectmedewerker') +group.permissions.add(Permission.objects.get(name=\"Can access 'Handelsregister' (bedrijfseigenaren van panden, bedrijfsinformatie)\")) +group.permissions.add(Permission.objects.get(name=\"Can access 'BRP' (persoonsgegevens / ingeschreven personen)\")) +group.permissions.add(Permission.objects.get(name=\"Can access 'invorderingscheck'\")) +group.permissions.add(Permission.objects.get(name='Close a Case (by performing the last task)')) +group.permissions.add(Permission.objects.get(name='Create a new Case')) +group.permissions.add(Permission.objects.get(name='Can perform a tasks')) + +(group, _) = UserGroup.objects.get_or_create(name='TOEZICHTHOUDER', display_name='Toezichthouder') +group.permissions.add(Permission.objects.get(name=\"Can access 'BRP' (persoonsgegevens / ingeschreven personen)\")) +group.permissions.add(Permission.objects.get(name='Can perform a tasks')) + +(group, _) = UserGroup.objects.get_or_create(name='PROJECTHANDHAVER', display_name='Projecthandhaver') +group.permissions.add(Permission.objects.get(name=\"Can access 'Handelsregister' (bedrijfseigenaren van panden, bedrijfsinformatie)\")) +group.permissions.add(Permission.objects.get(name=\"Can access 'BRP' (persoonsgegevens / ingeschreven personen)\")) +group.permissions.add(Permission.objects.get(name=\"Can access 'invorderingscheck'\")) +group.permissions.add(Permission.objects.get(name='Can perform a tasks')) + +(group, _) = UserGroup.objects.get_or_create(name='HANDHAVINGSJURIST', display_name='Handhavingsjurist') +group.permissions.add(Permission.objects.get(name=\"Can access 'Handelsregister' (bedrijfseigenaren van panden, bedrijfsinformatie)\")) +group.permissions.add(Permission.objects.get(name=\"Can access 'BRP' (persoonsgegevens / ingeschreven personen)\")) +group.permissions.add(Permission.objects.get(name=\"Can access 'invorderingscheck'\")) +group.permissions.add(Permission.objects.get(name='Can perform a tasks')) + +(group, _) = UserGroup.objects.get_or_create(name='MEDEWERKER GEVOELIGE ZAKEN', display_name='Medewerker Gevoelige Zaken') +group.permissions.add(Permission.objects.get(name='Can read gevoelige dossiers')) + +(group, _) = UserGroup.objects.get_or_create(name='PROJECTMEDEWERKER_DIGITAAL_TOEZICHT', display_name='Projectmedewerker - Digitaal Toezicht') +group.permissions.add(Permission.objects.get(name=\"Create a new 'Digitaal toezicht' Case\")) + +(group, _) = UserGroup.objects.get_or_create(name='TOEZICHTHOUDER_DIGITAAL_TOEZICHT', display_name='Toezichthouder - Digitaal toezicht') +group.permissions.add(Permission.objects.get(name=\"Can read 'Digitaal toezicht'\")) +group.permissions.add(Permission.objects.get(name=\"Create a new 'Digitaal toezicht' Case\"))" | docker-compose -f docker-compose.local.yml run -T --rm zaak-gateway python manage.py shell From 40826cb18e44ba754ddac555fb1a55f9885d035c Mon Sep 17 00:00:00 2001 From: Remy van der Wereld Date: Tue, 23 Jul 2024 15:39:37 +0200 Subject: [PATCH 42/42] Updated test --- README.md | 7 +++++++ app/apps/addresses/tests/tests_models.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5553961bf..770b270b1 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,13 @@ docker compose -f docker-compose.local.yml run --rm zaak-gateway python manage.p ``` +Or a specific test: + +```bash +docker-compose -f docker-compose.local.yml exec -T zaak-gateway python manage.py test apps.addresses.tests.tests_models.AddressModelTest.test_can_create_address_with_bag_result_without_stadsdeel +``` + + ## API documentation (Swagger) You can access the documentation at: diff --git a/app/apps/addresses/tests/tests_models.py b/app/apps/addresses/tests/tests_models.py index fc6580ec1..e29e280a9 100644 --- a/app/apps/addresses/tests/tests_models.py +++ b/app/apps/addresses/tests/tests_models.py @@ -27,7 +27,7 @@ def test_can_create_address(self): @patch("apps.addresses.models.do_bag_search_benkagg_by_bag_id") @patch("apps.addresses.models.do_bag_search_by_bag_id") def test_can_create_address_with_bag_result_without_stadsdeel( - self, mock_do_bag_search_benkagg_id, mock_do_bag_search_id + self, mock_do_bag_search_id, mock_do_bag_search_benkagg_id ): """Tests Address object creation with bag data mocks without stadsdeel entry"""