diff --git a/.env b/.env index 67619905a..5c262406c 100644 --- a/.env +++ b/.env @@ -9,9 +9,8 @@ DATABASE_USER=zaken DATABASE_PASSWORD=insecure DATABASE_HOST=database DATABASE_PORT=5432 -SENTRY_DSN=https://foo@foo.data.amsterdam.nl/0 -LOCAL_DEVELOPMENT_AUTHENTICATION=False -LOGGING_LEVEL=WARNING # To prevent flooding the logging in local development. Default is DEBUG. +LOCAL_DEVELOPMENT_AUTHENTICATION=True +LOGGING_LEVEL=WARNING # To prevent flooding the logging in local development. Default is DEBUG. WARNING SECRET_KEY_TOP_ZAKEN=SECRET_KEY_TOP_ZAKEN SECRET_KEY_TON_ZAKEN=SECRET_KEY_TON_ZAKEN BELASTING_API_URL=https://api-acc.belastingen.centric.eu/bel/inn/afne/vora/v1/vorderingenidentificatienummer/ diff --git a/.github/workflows/bpmn.workflow.yml b/.github/workflows/bpmn.workflow.yml index d029b47ba..0c7fdb352 100644 --- a/.github/workflows/bpmn.workflow.yml +++ b/.github/workflows/bpmn.workflow.yml @@ -1,4 +1,4 @@ -name: 'Validate BPMN' +name: 'BPMN validation' on: push: branches: [ main ] @@ -11,11 +11,11 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Use Node.js for BPMN Linting - uses: actions/setup-node@v3.1.1 + uses: actions/setup-node@v4 with: - node-version: '12.x' + node-version: '16.x' - name: Install dependencies for BPMN Linting run: npm install -g bpmnlint - name: Linting BPMN models diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index d0c157762..c3c8d06a4 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -9,7 +9,7 @@ # the `language` matrix defined below to confirm you have the correct set of # supported CodeQL languages. # -name: "Discovering vulnerabilities using CodeQL" +name: "CodeQL analysis for vulnerabilities " on: push: @@ -35,12 +35,13 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v3.1.2 + uses: actions/setup-python@v5 with: - python-version: '3.9.16' + python-version: '3.10' + - name: Install dependencies run: | python -m pip install --upgrade pip @@ -54,7 +55,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} # Override the default behavior so that the action doesn't attempt @@ -68,7 +69,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v1 + uses: github/codeql-action/autobuild@v3 # ℹī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -82,4 +83,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index e3cb4e76b..ae7d732e6 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -1,5 +1,5 @@ # action.yml -name: "Run Integration test" +name: "Run e2e-tests" on: push: branches: @@ -14,7 +14,7 @@ jobs: ################################################### - name: Checkout backend - uses: actions/checkout@v3 + uses: actions/checkout@v4 ################################################### # BACKEND @@ -23,6 +23,7 @@ jobs: - run: docker network create zaken_network - run: docker network create top_and_zaak_backend_bridge - run: docker-compose -f docker-compose.local.yml up --detach + # - run: docker-compose -f docker-compose.tests.yml up - run: sleep 30 - run: bash bin/setup_credentials.sh - run: ./e2e-tests/fix_models.sh @@ -31,15 +32,20 @@ jobs: # TEST ################################################### - - run: sudo apt-get install python3.9 + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: '3.10' - # Test version - - run: python --version + - name: Test version + run: python --version - - run: python3 -m pip install -r requirements.txt + - name: Install requirements + run: python3 -m pip install -r requirements.txt working-directory: e2e-tests - - run: API_HOST=http://127.0.0.1:8080/api/v1 python3 -m unittest + - name: Run tests + run: API_HOST=http://127.0.0.1:8080/api/v1 python3 -m unittest working-directory: e2e-tests ################################################### diff --git a/.github/workflows/git revert --no-commit f1c30cb2e2fd0d3bd b/.github/workflows/git revert --no-commit f1c30cb2e2fd0d3bd new file mode 100644 index 000000000..f8b7d3cbf --- /dev/null +++ b/.github/workflows/git revert --no-commit f1c30cb2e2fd0d3bd @@ -0,0 +1 @@ +git revert --no-commit f1c30cb2e2fd0d3bd0d865ec86e8b7546aa91d2e..HEAD diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 16439fad3..b8dee468d 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -1,5 +1,5 @@ # action.yml -name: 'Start frontend' +name: 'Run integration-tests' on: pull_request jobs: main: @@ -9,13 +9,13 @@ jobs: # CHECKOUT ################################################### - name: Checkout frontend - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: frontend repository: amsterdam/zaken-frontend - name: Checkout backend - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: backend diff --git a/.github/workflows/main.workflow.yml b/.github/workflows/main.workflow.yml index 8c15f940d..79be4706c 100644 --- a/.github/workflows/main.workflow.yml +++ b/.github/workflows/main.workflow.yml @@ -1,4 +1,4 @@ -name: 'Install, lint, test & build' +name: 'Main, lint, build & manage.py test' on: push: branches: [ main ] @@ -9,20 +9,20 @@ jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v3.1.2 + uses: actions/setup-python@v5 with: - python-version: 3.9.16 + python-version: '3.10' - name: Linting run: bash bin/cleanup_pre_commit.sh - name: Build Docker image - run: docker-compose build -f docker-compose.local.yml + run: docker-compose -f docker-compose.local.yml build - name: Create Docker network run: docker network create zaken_network - name: Create TOP and Zaken Docker network run: docker network create top_and_zaak_backend_bridge - name: Start images - run: docker-compose -f docker-compose.local.yml up -d + run: docker-compose -f docker-compose.local.yml up -d - name: Run Tests - run: docker-compose exec -T zaak-gateway python manage.py test + run: docker-compose -f docker-compose.local.yml exec -T zaak-gateway python manage.py test diff --git a/README.md b/README.md index 1d096cd03..770b270b1 100644 --- a/README.md +++ b/README.md @@ -53,15 +53,15 @@ docker-compose -f docker-compose.local.yml up To create all necessary credentials run the following command: -``` +```bash bash bin/setup_credentials.sh ``` This will create superuser admin account with the following credentials -``` +```bash email: admin@admin.com -password: admin +password: insecure ``` Visit the Admin at http://localhost:8080/admin/ @@ -69,22 +69,36 @@ Visit the Admin at http://localhost:8080/admin/ Check the health page to see if all services are up and running: http://localhost:8080/health +To create all necessary user groups run the following command: + +```bash +bash bin/setup_user_groups.sh +``` + ## Running tests Set LOCAL_DEVELOPMENT_AUTHENTICATION environment variable to True (default) Run unit tests locally with: -``` -docker compose run --rm zaak-gateway python manage.py test +```bash +docker compose -f docker-compose.local.yml run --rm zaak-gateway python manage.py test ``` To run tests for a specific module, add a path: +```bash +docker compose -f docker-compose.local.yml run --rm zaak-gateway python manage.py test apps/cases + ``` -docker compose run --rm zaak-gateway python manage.py test apps/cases + +Or a specific test: + +```bash +docker-compose -f docker-compose.local.yml exec -T zaak-gateway python manage.py test apps.addresses.tests.tests_models.AddressModelTest.test_can_create_address_with_bag_result_without_stadsdeel ``` + ## API documentation (Swagger) You can access the documentation at: @@ -105,24 +119,25 @@ Create a `.env.local` file, on the root of your project, and override the variab Start your project with the newly created environment variables, like so: -``` -docker compose --env-file .env.local up +```bash +docker compose -f docker-compose.local.yml --env-file .env.local up ``` ## Enabling Keycloak authentication for a locally run zaken-frontend -Set LOCAL_DEVELOPMENT_AUTHENTICATION environment variable to False +Set `LOCAL_DEVELOPMENT_AUTHENTICATION` environment variable to False ## Generating Mock Data -You can generate mock data easily (from the API swagger environment) by executing the /api/v1/generate-mock/ GET request. +You can generate mock data easily (from the API swagger environment) by executing the `/api/v1/generate-mock/` GET request. ## Update fixtures Generate new fixtures json file: -``` -docker-compose run --rm zaak-gateway python manage.py dumpdata --indent 2 -o temp_fixture.json [app_name] +```bash +docker compose -f docker-compose.local.yml run --rm zaak-gateway python manage.py dumpdata --indent 2 -o temp_fixture.json [app_name] + ``` Now manually copy changes you need to the corresponding fixtures file. @@ -131,13 +146,13 @@ Now manually copy changes you need to the corresponding fixtures file. You can add pre-commit hooks for checking and cleaning up your changes: -``` +```bash bash bin/install_pre_commit.sh ``` You can also run the following command to ensure all files adhere to coding conventions: -``` +```bash bash bin/cleanup_pre_commit.sh ``` @@ -167,7 +182,7 @@ Note that the apps and models should be updated whenever applications and models For changes to the model you have to migrate the DB. -```python +```bash python manage.py makemigrations --name python manage.py migrate @@ -193,14 +208,14 @@ Try the online modeler for BPMN-models: https://bpmn.io/. This is a lightweight Clone the bpmn-io Github repo for editing: https://github.com/bpmn-io/bpmn-js-examples. -``` +```bash git clone git@github.com:bpmn-io/bpmn-js-examples.git ``` If you'd like to use Camunda Platform execution related properties, include the camunda-bpmn-moddle dependency which tells the modeler about camunda:XXX extension properties: https://github.com/bpmn-io/bpmn-js-examples/tree/master/properties-panel#camunda-platform Follow next steps: -``` +```bash cd bpmn-js-examples cd properties-panel @@ -265,7 +280,7 @@ So if you think existing cases will get stuck in the model, just create a new ve Example: `housing_corporation` has a new minor version model and the latest version was `5.0.0` but `debrief` has latest version `6.0.0`. Then the new minor version of `housing_corporation` will be `6.1.0`. -``` +```json bpmn_models/default/ ├─ debrief/ │ ├─ 0.1.0/ @@ -280,7 +295,7 @@ bpmn_models/default/ - Add the new version to `WORKFLOW_SPEC_CONFIG` in `settings.py`: -```python +```json "housing_corporation": { "versions": { "5.0.0": {}, diff --git a/app/Dockerfile b/app/Dockerfile index e846b4df3..bfeb7fe2a 100644 --- a/app/Dockerfile +++ b/app/Dockerfile @@ -1,5 +1,4 @@ -FROM amsterdam/python:3.9-buster -LABEL maintainer="datapunt@amsterdam.nl" +FROM python:3.10-bullseye ENV REQUESTS_CA_BUNDLE /etc/ssl/certs/ca-certificates.crt @@ -10,11 +9,16 @@ RUN chmod 644 /usr/local/share/ca-certificates/adp_rootca.crt \ ENV PYTHONUNBUFFERED 1 -RUN apt-get update && apt-get install -y -# RUN pip install --upgrade pip -RUN pip install uwsgi +# Update and install necessary packages including GDAL +RUN apt-get update && apt-get install -y \ + gdal-bin \ + libgdal-dev \ + graphviz \ + graphviz-dev \ + postgresql-client -RUN apt-get install graphviz graphviz-dev -y +RUN pip install --upgrade pip +RUN pip install uwsgi RUN pip install pygraphviz RUN echo "10.240.5.72 acc.api.data.amsterdam.nl" >> /etc/hosts || echo "Could not write to /etc/hosts" @@ -30,6 +34,8 @@ RUN chmod +x /app/wait-for.sh RUN chmod +x /app/celery.sh RUN chmod +x /app/deploy/docker-entrypoint.sh +# Set the GDAL_LIBRARY_PATH environment variable +ENV GDAL_LIBRARY_PATH /usr/lib/libgdal.so ENTRYPOINT ["/app/deploy/docker-entrypoint.sh"] CMD ["uwsgi", "--ini", "/app/deploy/config.ini"] diff --git a/app/apps/addresses/mock.py b/app/apps/addresses/mock.py index db8d79c32..f271716b7 100644 --- a/app/apps/addresses/mock.py +++ b/app/apps/addresses/mock.py @@ -79,21 +79,29 @@ def mock_do_bag_search_id_result_without_links(): } -def mock_get_bag_data_result(): +def mock_get_bag_identificatie_and_stadsdeel_result_without_stadsdeel(): return { - "_stadsdeel": { - "_links": { - "self": { - "href": "https://api.data.amsterdam.nl/gebieden/stadsdeel/03630930000000/" + "_embedded": { + "adresseerbareobjecten": [ + { + "huisnummer": 42, + "identificatie": "123456789" + # No "gebiedenStadsdeelNaam" key } - }, - "_display": "Weesp (S)", - "code": "S", - "naam": "Weesp", - "dataset": "gebieden", - }, + ] + } } -def mock_get_bag_data_result_without_stadsdeel(): - return {} +def mock_get_bag_identificatie_and_stadsdeel_result(): + return { + "_embedded": { + "adresseerbareobjecten": [ + { + "identificatie": "123456789", + "huisnummer": 42, + "gebiedenStadsdeelNaam": "Zuidoost", + } + ] + } + } diff --git a/app/apps/addresses/models.py b/app/apps/addresses/models.py index 4da380d31..2c260b9c0 100644 --- a/app/apps/addresses/models.py +++ b/app/apps/addresses/models.py @@ -115,9 +115,6 @@ def get_bag_identificatie_and_stadsdeel(self): If an address has an standplaats (woonboot) instead of verblijfsobject, the coordinates will be calculated by a polygon. """ - # When moving the import to the beginning of the file, a Django error follows: - # ImproperlyConfigured: AUTH_USER_MODEL refers to model 'users.User' that has not been installed. - from utils.exceptions import DistrictNotFoundError is_boat = self.type == "standplaats" response = do_bag_search_benkagg_by_bag_id(self.bag_id, is_boat) @@ -140,13 +137,10 @@ def get_bag_identificatie_and_stadsdeel(self): if nummeraanduiding_id: self.nummeraanduiding_id = nummeraanduiding_id - # Add Stadsdeel to address. district_name = found_bag_object.get("gebiedenStadsdeelNaam") if district_name: self.district = District.objects.get_or_create(name=district_name)[0] - else: - raise DistrictNotFoundError(f"API benkagg bag_id: {self.bag_id}") # Get coordinates for standplaats (woonboot). ligplaats_geometrie = found_bag_object.get("ligplaatsGeometrie") or {} diff --git a/app/apps/addresses/tests/tests_models.py b/app/apps/addresses/tests/tests_models.py index d85b07282..e29e280a9 100644 --- a/app/apps/addresses/tests/tests_models.py +++ b/app/apps/addresses/tests/tests_models.py @@ -3,14 +3,13 @@ from apps.addresses.mock import ( mock_do_bag_search_id_result, mock_do_bag_search_id_result_without_links, - mock_get_bag_data_result, - mock_get_bag_data_result_without_stadsdeel, + mock_get_bag_identificatie_and_stadsdeel_result, + mock_get_bag_identificatie_and_stadsdeel_result_without_stadsdeel, ) from apps.addresses.models import Address, District from django.core import management from django.test import TestCase from model_bakery import baker -from utils.exceptions import DistrictNotFoundError # TODO: Expand this by mocking the BAG API @@ -22,53 +21,43 @@ def setUp(self): def test_can_create_address(self): """Tests Address object creation without bag mocks""" self.assertEquals(Address.objects.count(), 0) - baker.make(Address) - self.assertEquals(Address.objects.count(), 1) + @patch("apps.addresses.models.do_bag_search_benkagg_by_bag_id") @patch("apps.addresses.models.do_bag_search_by_bag_id") - def test_can_create_address_with_bag_result_without_verblijftobject_url( - self, mock_do_bag_search_id, mock_get_bag_data + def test_can_create_address_with_bag_result_without_stadsdeel( + self, mock_do_bag_search_id, mock_do_bag_search_benkagg_id ): - """Tests Address object creation with bag data mocks without verblijftobject url""" + """Tests Address object creation with bag data mocks without stadsdeel entry""" - mock_do_bag_search_id.return_value = ( - mock_do_bag_search_id_result_without_links() + mock_do_bag_search_id.return_value = mock_do_bag_search_id_result() + mock_do_bag_search_benkagg_id.return_value = ( + mock_get_bag_identificatie_and_stadsdeel_result_without_stadsdeel() ) - mock_get_bag_data.return_value = mock_get_bag_data_result() self.assertEquals(Address.objects.count(), 0) self.assertEquals(District.objects.count(), 0) - with self.assertRaises(DistrictNotFoundError): - baker.make(Address) - mock_do_bag_search_id.assert_called() - - @patch("apps.addresses.models.do_bag_search_by_bag_id") - def test_can_create_address_with_bag_result_without_verblijftobject_stadsdeel( - self, mock_do_bag_search_id, mock_get_bag_data - ): - """Tests Address object creation with bag data mocks without verblijftobject stadsdeel entry""" + baker.make(Address) - mock_do_bag_search_id.return_value = mock_do_bag_search_id_result() - mock_get_bag_data.return_value = mock_get_bag_data_result_without_stadsdeel() + mock_do_bag_search_id.assert_called() + mock_do_bag_search_benkagg_id.assert_called() - self.assertEquals(Address.objects.count(), 0) + self.assertEquals(Address.objects.count(), 1) self.assertEquals(District.objects.count(), 0) - with self.assertRaises(DistrictNotFoundError): - baker.make(Address) - mock_do_bag_search_id.assert_called() - @patch("apps.addresses.models.do_bag_search_by_bag_id") + @patch("apps.addresses.models.do_bag_search_benkagg_by_bag_id") def test_can_create_address_with_bag_result( - self, mock_do_bag_search_id, mock_get_bag_data + self, mock_do_bag_search_benkagg_id, mock_do_bag_search_id ): """Tests Address object creation with bag data mocks""" mock_do_bag_search_id.return_value = mock_do_bag_search_id_result() - mock_get_bag_data.return_value = mock_get_bag_data_result() + mock_do_bag_search_benkagg_id.return_value = ( + mock_get_bag_identificatie_and_stadsdeel_result() + ) self.assertEquals(Address.objects.count(), 0) self.assertEquals(District.objects.count(), 0) @@ -76,6 +65,7 @@ def test_can_create_address_with_bag_result( baker.make(Address) mock_do_bag_search_id.assert_called() + mock_do_bag_search_benkagg_id.assert_called() self.assertEquals(Address.objects.count(), 1) self.assertEquals(District.objects.count(), 1) diff --git a/app/apps/cases/migrations/0024_alter_default_column_ids_azure.py b/app/apps/cases/migrations/0024_alter_default_column_ids_azure.py index 10842dfce..77177dea7 100644 --- a/app/apps/cases/migrations/0024_alter_default_column_ids_azure.py +++ b/app/apps/cases/migrations/0024_alter_default_column_ids_azure.py @@ -9,19 +9,4 @@ class Migration(migrations.Migration): ("cases", "0023_add_multiple_tags"), ] - operations = [ - migrations.RunSQL( - sql=""" - DO $$ - BEGIN - -- cases_casetheme - IF NOT EXISTS (SELECT FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.relkind = 'S' AND c.relname = 'cases_casetheme_id_seq' AND n.nspname = 'public') THEN - CREATE SEQUENCE public.cases_casetheme_id_seq INCREMENT BY 1 START WITH 1 MINVALUE 1 NO MAXVALUE CACHE 1; - END IF; - END - $$; - - ALTER TABLE cases_casetheme ALTER COLUMN id SET DEFAULT nextval('public.cases_casetheme_id_seq'::regclass); - """ - ) - ] + operations = [] diff --git a/app/apps/cases/migrations/0026_update_sequence_pk.py b/app/apps/cases/migrations/0026_update_sequence_pk.py index cc8b0b239..e527534f5 100644 --- a/app/apps/cases/migrations/0026_update_sequence_pk.py +++ b/app/apps/cases/migrations/0026_update_sequence_pk.py @@ -9,22 +9,4 @@ class Migration(migrations.Migration): ("cases", "0025_remove_unused_tables_azure"), ] - operations = [ - migrations.RunSQL( - sql=""" - DO $$ - DECLARE - max_id BIGINT; - BEGIN - -- Find the maximum id value in the table - SELECT MAX(id) INTO max_id FROM public.cases_casetheme; - - -- If there are no records, set max_id to 0 - IF max_id IS NULL THEN - max_id := 0; - END IF; - EXECUTE FORMAT('ALTER SEQUENCE public.cases_casetheme_id_seq RESTART WITH %s;', max_id + 1); - END $$; - """ - ) - ] + operations = [] diff --git a/app/apps/events/migrations/0002_alter_default_column_ids_azure.py b/app/apps/events/migrations/0002_alter_default_column_ids_azure.py index 0d73b1725..7379dd91d 100644 --- a/app/apps/events/migrations/0002_alter_default_column_ids_azure.py +++ b/app/apps/events/migrations/0002_alter_default_column_ids_azure.py @@ -9,19 +9,4 @@ class Migration(migrations.Migration): ("events", "0001_initial"), ] - operations = [ - migrations.RunSQL( - sql=""" - DO $$ - BEGIN - -- events_caseevent - IF NOT EXISTS (SELECT FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.relkind = 'S' AND c.relname = 'events_caseevent_id_seq' AND n.nspname = 'public') THEN - CREATE SEQUENCE public.events_caseevent_id_seq INCREMENT BY 1 START WITH 1 MINVALUE 1 NO MAXVALUE CACHE 1; - END IF; - END - $$; - - ALTER TABLE events_caseevent ALTER COLUMN id SET DEFAULT nextval('public.events_caseevent_id_seq'::regclass); - """ - ) - ] + operations = [] diff --git a/app/apps/events/migrations/0003_update_sequence_pk.py b/app/apps/events/migrations/0003_update_sequence_pk.py index a3cf25dfd..f19fb8b11 100644 --- a/app/apps/events/migrations/0003_update_sequence_pk.py +++ b/app/apps/events/migrations/0003_update_sequence_pk.py @@ -9,22 +9,4 @@ class Migration(migrations.Migration): ("events", "0002_alter_default_column_ids_azure"), ] - operations = [ - migrations.RunSQL( - sql=""" - DO $$ - DECLARE - max_id BIGINT; - BEGIN - -- Find the maximum id value in the table - SELECT MAX(id) INTO max_id FROM public.events_caseevent; - - -- If there are no records, set max_id to 0 - IF max_id IS NULL THEN - max_id := 0; - END IF; - EXECUTE FORMAT('ALTER SEQUENCE public.events_caseevent_id_seq RESTART WITH %s;', max_id + 1); - END $$; - """ - ) - ] + operations = [] diff --git a/app/apps/openzaak/helpers.py b/app/apps/openzaak/helpers.py index c49afe008..acac5265c 100644 --- a/app/apps/openzaak/helpers.py +++ b/app/apps/openzaak/helpers.py @@ -16,7 +16,7 @@ from apps.cases.models import CaseDocument from django.conf import settings from django.utils import timezone -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from zgw_consumers.api_models.base import factory from zgw_consumers.api_models.catalogi import ZaakType from zgw_consumers.api_models.documenten import Document diff --git a/app/apps/schedules/migrations/0005_alter_default_column_ids_azure.py b/app/apps/schedules/migrations/0005_alter_default_column_ids_azure.py index 770023ca2..3c324d8f8 100644 --- a/app/apps/schedules/migrations/0005_alter_default_column_ids_azure.py +++ b/app/apps/schedules/migrations/0005_alter_default_column_ids_azure.py @@ -9,19 +9,4 @@ class Migration(migrations.Migration): ("schedules", "0004_schedule_housing_corporation_combiteam"), ] - operations = [ - migrations.RunSQL( - sql=""" - DO $$ - BEGIN - -- schedules_action - IF NOT EXISTS (SELECT FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.relkind = 'S' AND c.relname = 'schedules_action_id_seq' AND n.nspname = 'public') THEN - CREATE SEQUENCE public.schedules_action_id_seq INCREMENT BY 1 START WITH 1 MINVALUE 1 NO MAXVALUE CACHE 1; - END IF; - END - $$; - - ALTER TABLE schedules_action ALTER COLUMN id SET DEFAULT nextval('public.schedules_action_id_seq'::regclass); - """ - ) - ] + operations = [] diff --git a/app/apps/schedules/migrations/0006_update_sequence_pk.py b/app/apps/schedules/migrations/0006_update_sequence_pk.py index 7174ba1df..cba00f061 100644 --- a/app/apps/schedules/migrations/0006_update_sequence_pk.py +++ b/app/apps/schedules/migrations/0006_update_sequence_pk.py @@ -9,22 +9,4 @@ class Migration(migrations.Migration): ("schedules", "0005_alter_default_column_ids_azure"), ] - operations = [ - migrations.RunSQL( - sql=""" - DO $$ - DECLARE - max_id BIGINT; - BEGIN - -- Find the maximum id value in the table - SELECT MAX(id) INTO max_id FROM public.schedules_action; - - -- If there are no records, set max_id to 0 - IF max_id IS NULL THEN - max_id := 0; - END IF; - EXECUTE FORMAT('ALTER SEQUENCE public.schedules_action_id_seq RESTART WITH %s;', max_id + 1); - END $$; - """ - ) - ] + operations = [] diff --git a/app/apps/users/auth.py b/app/apps/users/auth.py index 7c80a0d30..ee7632d73 100644 --- a/app/apps/users/auth.py +++ b/app/apps/users/auth.py @@ -10,27 +10,6 @@ LOGGER = logging.getLogger(__name__) - -# TODO: email lowercase needs testing first -class AppsOIDCAuthenticationBackend(OIDCAuthenticationBackend): - def get_userinfo(self, access_token, id_token, payload): - user_info = super().get_userinfo(access_token, id_token, payload) - - # make sure email is lower case - user_info["email"] = user_info.get("email", "").lower() - - return user_info - - def filter_users_by_claims(self, claims): - """Return all users matching the specified email.""" - email = claims.get("email") - if not email: - return self.UserModel.objects.none() - - users = self.UserModel.objects.filter(email__exact=email) - return users - - if settings.LOCAL_DEVELOPMENT_AUTHENTICATION: AuthenticationBackend = DevelopmentAuthenticationBackend AuthenticationClass = JWTAuthentication diff --git a/app/apps/users/migrations/0007_remove_user_manager.py b/app/apps/users/migrations/0007_remove_user_manager.py new file mode 100644 index 000000000..739b18df0 --- /dev/null +++ b/app/apps/users/migrations/0007_remove_user_manager.py @@ -0,0 +1,20 @@ +# Generated by Django 4.2.13 on 2024-07-02 15:09 + +import django.contrib.auth.models +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("users", "0006_alter_permission_options"), + ] + + operations = [ + migrations.AlterModelManagers( + name="user", + managers=[ + ("objects", django.contrib.auth.models.UserManager()), + ], + ), + ] diff --git a/app/apps/users/models.py b/app/apps/users/models.py index ce4230da9..9477f21ad 100644 --- a/app/apps/users/models.py +++ b/app/apps/users/models.py @@ -1,6 +1,5 @@ import uuid -from apps.users.user_manager import UserManager from django.contrib.auth.models import AbstractUser, Group from django.db import models from django.db.models.base import Model @@ -46,8 +45,6 @@ class Meta: USERNAME_FIELD = "email" REQUIRED_FIELDS = [] - objects = UserManager() - @property def full_name(self): """ diff --git a/app/apps/users/user_manager.py b/app/apps/users/user_manager.py index fa65c08f5..7d2b9faf6 100644 --- a/app/apps/users/user_manager.py +++ b/app/apps/users/user_manager.py @@ -2,34 +2,37 @@ class UserManager(BaseUserManager): - """Define a model manager for User model with no username field.""" - - use_in_migrations = True - - def _create_user(self, email, password, **extra_fields): - """Create and save a User with the given email and password.""" - if not email: - raise ValueError("The given email must be set") - email = self.normalize_email(email) - user = self.model(email=email, **extra_fields) - user.set_password(password) - user.save(using=self._db) - return user - - def create_user(self, email, password=None, **extra_fields): - """Create and save a regular User with the given email and password.""" - extra_fields.setdefault("is_staff", False) - extra_fields.setdefault("is_superuser", False) - return self._create_user(email, password, **extra_fields) - - def create_superuser(self, email, password, **extra_fields): - """Create and save a SuperUser with the given email and password.""" - extra_fields.setdefault("is_staff", True) - extra_fields.setdefault("is_superuser", True) - - if extra_fields.get("is_staff") is not True: - raise ValueError("Superuser must have is_staff=True.") - if extra_fields.get("is_superuser") is not True: - raise ValueError("Superuser must have is_superuser=True.") - - return self._create_user(email, password, **extra_fields) + """ + This model isn't used anymore but can't be deleted because of the reference in 0001_initial.py + If the migrations are applied in production, this can be removed. + """ + + # use_in_migrations = True + + # def _create_user(self, email, password, **extra_fields): + # """Create and save a User with the given email and password.""" + # if not email: + # raise ValueError("The given email must be set") + # email = self.normalize_email(email) + # user = self.model(email=email, **extra_fields) + # user.set_password(password) + # user.save(using=self._db) + # return user + + # def create_user(self, email, password=None, **extra_fields): + # """Create and save a regular User with the given email and password.""" + # extra_fields.setdefault("is_staff", False) + # extra_fields.setdefault("is_superuser", False) + # return self._create_user(email, password, **extra_fields) + + # def create_superuser(self, email, password, **extra_fields): + # """Create and save a SuperUser with the given email and password.""" + # extra_fields.setdefault("is_staff", True) + # extra_fields.setdefault("is_superuser", True) + + # if extra_fields.get("is_staff") is not True: + # raise ValueError("Superuser must have is_staff=True.") + # if extra_fields.get("is_superuser") is not True: + # raise ValueError("Superuser must have is_superuser=True.") + + # return self._create_user(email, password, **extra_fields) diff --git a/app/apps/workflow/admin.py b/app/apps/workflow/admin.py index cb6f7b0b2..5d05c4aa1 100644 --- a/app/apps/workflow/admin.py +++ b/app/apps/workflow/admin.py @@ -1,8 +1,7 @@ -from django.conf.urls import url from django.contrib import admin, messages from django.http import HttpResponseRedirect from django.template.response import TemplateResponse -from django.urls import reverse +from django.urls import re_path, reverse from django.utils.html import format_html, mark_safe from .forms import ResetSubworkflowsForm, UpdateDataForWorkflowsForm @@ -145,12 +144,12 @@ def update_data(self, obj): def get_urls(self): urls = super().get_urls() custom_urls = [ - url( + re_path( r"^(?P.+)/reset-subworkflows/$", self.admin_site.admin_view(self.admin_process_reset_subworkflows), name="reset-subworkflows", ), - url( + re_path( r"^(?P.+)/update-data-for-subworkflow/$", self.admin_site.admin_view(self.admin_update_data_for_workflow), name="update-data-for-subworkflow", diff --git a/app/apps/workflow/migrations/0009_alter_default_column_ids_azure.py b/app/apps/workflow/migrations/0009_alter_default_column_ids_azure.py index 5c4e31bd1..1ef9805be 100644 --- a/app/apps/workflow/migrations/0009_alter_default_column_ids_azure.py +++ b/app/apps/workflow/migrations/0009_alter_default_column_ids_azure.py @@ -9,24 +9,4 @@ class Migration(migrations.Migration): ("workflow", "0008_workflowoption_enabled_on_case_closed"), ] - operations = [ - migrations.RunSQL( - sql=""" - DO $$ - BEGIN - -- workflow_caseusertask - IF NOT EXISTS (SELECT FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.relkind = 'S' AND c.relname = 'workflow_caseusertask_id_seq' AND n.nspname = 'public') THEN - CREATE SEQUENCE public.workflow_caseusertask_id_seq INCREMENT BY 1 START WITH 1 MINVALUE 1 NO MAXVALUE CACHE 1; - END IF; - -- workflow_caseworkflow - IF NOT EXISTS (SELECT FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.relkind = 'S' AND c.relname = 'workflow_caseworkflow_id_seq' AND n.nspname = 'public') THEN - CREATE SEQUENCE public.workflow_caseworkflow_id_seq INCREMENT BY 1 START WITH 1 MINVALUE 1 NO MAXVALUE CACHE 1; - END IF; - END - $$; - - ALTER TABLE workflow_caseusertask ALTER COLUMN id SET DEFAULT nextval('public.workflow_caseusertask_id_seq'::regclass); - ALTER TABLE workflow_caseworkflow ALTER COLUMN id SET DEFAULT nextval('public.workflow_caseworkflow_id_seq'::regclass); - """ - ) - ] + operations = [] diff --git a/app/apps/workflow/migrations/0010_update_sequence_pk.py b/app/apps/workflow/migrations/0010_update_sequence_pk.py index b949d4e11..54b5b6829 100644 --- a/app/apps/workflow/migrations/0010_update_sequence_pk.py +++ b/app/apps/workflow/migrations/0010_update_sequence_pk.py @@ -9,37 +9,4 @@ class Migration(migrations.Migration): ("workflow", "0009_alter_default_column_ids_azure"), ] - operations = [ - migrations.RunSQL( - sql=""" - DO $$ - DECLARE - max_id BIGINT; - BEGIN - -- workflow_caseusertask - -- Find the maximum id value in the table - SELECT MAX(id) INTO max_id FROM public.workflow_caseusertask; - - -- If there are no records, set max_id to 0 - IF max_id IS NULL THEN - max_id := 0; - END IF; - EXECUTE FORMAT('ALTER SEQUENCE public.workflow_caseusertask_id_seq RESTART WITH %s;', max_id + 1); - END $$; - DO $$ - DECLARE - max_id BIGINT; - BEGIN - -- workflow_caseworkflow - -- Find the maximum id value in the table - SELECT MAX(id) INTO max_id FROM public.workflow_caseworkflow; - - -- If there are no records, set max_id to 0 - IF max_id IS NULL THEN - max_id := 0; - END IF; - EXECUTE FORMAT('ALTER SEQUENCE public.workflow_caseworkflow_id_seq RESTART WITH %s;', max_id + 1); - END $$; - """ - ) - ] + operations = [] diff --git a/app/config/settings.py b/app/config/settings.py index ecc1c5f25..b9a4ba199 100644 --- a/app/config/settings.py +++ b/app/config/settings.py @@ -33,7 +33,7 @@ ZAKEN_CONTAINER_HOST = os.getenv("ZAKEN_CONTAINER_HOST") -ALLOWED_HOSTS = "*" +ALLOWED_HOSTS = ["*"] CORS_ORIGIN_WHITELIST = os.environ.get("CORS_ORIGIN_WHITELIST").split(",") CORS_ORIGIN_ALLOW_ALL = False @@ -54,7 +54,6 @@ "drf_spectacular", "django_extensions", "django_filters", - "django_spaghetti", "django_celery_beat", "django_celery_results", "zgw_consumers", @@ -84,24 +83,6 @@ "apps.workflow", ) -# Add apps here to make them appear in the graphing visualisation -SPAGHETTI_SAUCE = { - "apps": [ - "users", - "cases", - "debriefings", - "permits", - "fines", - "addresses", - "visits", - "events", - "summons", - "decisions", - "schedules", - ], - "show_fields": False, -} - DATABASE_HOST = os.getenv("DATABASE_HOST", "database") DATABASE_NAME = os.getenv("DATABASE_NAME", "dev") DATABASE_USER = os.getenv("DATABASE_USER", "dev") @@ -339,11 +320,7 @@ def filter_traces(envelope): ) AXES_RESET_ON_SUCCESS = True -AXES_ONLY_USER_FAILURES = ( - True # Enabled because IP from user is always .amsterdam.nl gateway -) AXES_ENABLED = os.getenv("AXES_ENABLED", "True") == "True" -AXES_META_PRECEDENCE_ORDER = ["HTTP_X_FORWARDED_FOR", "REMOTE_ADDR"] # Simple JWT is used for local development authentication only. SIMPLE_JWT = { @@ -540,6 +517,7 @@ def get_redis_url(): CELERY_TASK_TRACK_STARTED = True CELERY_TASK_TIME_LIMIT = 30 * 60 CELERY_BROKER_URL = get_redis_url() +CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP = True BROKER_CONNECTION_MAX_RETRIES = None BROKER_CONNECTION_TIMEOUT = 120 BROKER_URL = CELERY_BROKER_URL diff --git a/app/config/urls.py b/app/config/urls.py index f45e6ba07..770a72f55 100644 --- a/app/config/urls.py +++ b/app/config/urls.py @@ -36,11 +36,10 @@ from apps.visits.views import VisitViewSet from apps.workflow.views import CaseUserTaskViewSet, GenericCompletedTaskViewSet from django.conf import settings -from django.conf.urls import include, url from django.conf.urls.static import static from django.contrib import admin from django.http import JsonResponse -from django.urls import path +from django.urls import include, path, re_path from django.views.generic import View from drf_spectacular.views import SpectacularAPIView, SpectacularSwaggerView from rest_framework.routers import DefaultRouter @@ -123,10 +122,9 @@ def get(self, request, *args, **kwargs): ReceiveNotificationView.as_view(), name="notification-callback", ), - path("data-model/", include("django_spaghetti.urls")), - url("health/", include("health_check.urls")), - url("startup/", is_healthy), - url(regex=r"^$", view=MyView.as_view(), name="index"), + path("health/", include("health_check.urls")), + path("startup/", is_healthy), + re_path(r"^$", view=MyView.as_view(), name="index"), ] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) if settings.DEBUG: diff --git a/app/deploy/docker-entrypoint.celery.sh b/app/deploy/docker-entrypoint.celery.sh new file mode 100755 index 000000000..7be07b30b --- /dev/null +++ b/app/deploy/docker-entrypoint.celery.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -u # crash on missing env variables +set -e # stop on any error +set -x + +exec "$@" diff --git a/app/deploy/docker-entrypoint.development.sh b/app/deploy/docker-entrypoint.development.sh index bd2fd497b..90dee9d53 100755 --- a/app/deploy/docker-entrypoint.development.sh +++ b/app/deploy/docker-entrypoint.development.sh @@ -1,4 +1,3 @@ -# Same as entrypoint.sh, but with py-auto-reload enabled to allow auto reloading when making changes during development #!/usr/bin/env bash set -u # crash on missing env variables set -e # stop on any error @@ -6,27 +5,22 @@ set -x until PGPASSWORD=$DATABASE_PASSWORD psql -h $DATABASE_HOST -U $DATABASE_USER -c '\q'; do echo "Postgres is unavailable - sleeping" - sleep 1 + sleep 5 done echo "Postgres is up!" + + echo Collecting static files python manage.py collectstatic --no-input -chmod -R 777 /static - -# echo Clear tables -# python manage.py shell -c "from django.db import connection; cursor = connection.cursor(); cursor.execute('drop table if exists "django_migrations" cascade; drop table if exists "django_content_type" cascade; drop table if exists "auth_permission" cascade; drop table if exists "auth_group" cascade; drop table if exists "auth_group_permissions" cascade; drop table if exists "users_user" cascade; drop table if exists "users_user_groups" cascade; drop table if exists "users_user_user_permissions" cascade; drop table if exists "django_admin_log" cascade; drop table if exists "django_session" cascade;'); cursor.close();" echo Apply migrations python manage.py migrate --noinput echo Axes check python manage.py check +python manage.py axes_reset python manage.py loaddata fixture -# echo Create root user -# python manage.py shell -c "from django.contrib.auth import get_user_model; get_user_model().objects.create_superuser('admin@admin.com', 'admin')" - -# opens up a port for attaching a remote debugging service using debugpy -exec python -m debugpy --listen 0.0.0.0:5678 ./manage.py runserver 0.0.0.0:8000 +exec "$@" diff --git a/app/requirements.txt b/app/requirements.txt index 0994d8b31..bb8c43eca 100644 --- a/app/requirements.txt +++ b/app/requirements.txt @@ -1,41 +1,40 @@ amqp<6.0.0 -asgiref==3.4.1 +asgiref==3.8.1 attrs==21.2.0 azure-identity beautifulsoup4==4.10.0 -billiard==3.6.4.0 +billiard==4.2.0 cached-property==1.5.2 -celery==5.2.6 -certifi==2023.5.7 +celery==5.4.0 +certifi==2024.6.2 cffi==1.14.6 chardet==4.0.0 click==8.1.2 -click-didyoumean==0.0.3 +click-didyoumean==0.3.1 click-plugins==1.1.1 click-repl==0.2.0 cryptography==41.0.1 -datapunt-keycloak-oidc @ git+https://github.com/petercuret/keycloak_oidc.git@master +datapunt-keycloak-oidc @ git+https://github.com/remyvdwereld/keycloak_oidc_top.git@main debugpy==1.4.1 -Django==3.2.13 -django-axes==5.32.0 -django-celery-beat==2.2.1 -django-celery-results==2.2.0 -django-choices==1.7.2 -django-cors-headers==3.8.0 -django-csp==3.7 -django-extensions==3.1.3 -django-filter==2.4.0 -django-health-check==3.16.4 -django-ipware==4.0.2 -django-permissions-policy==4.1.0 -django-redis==5.0.0 -django-relativedelta==1.1.2 -django-solo==1.1.5 -django-spaghetti-and-meatballs==0.4.2 -django-timezone-field==4.2.1 -djangorestframework==3.12.4 -djangorestframework-simplejwt==4.8.0 -drf-spectacular==0.17.2 +Django==4.2.13 +django-axes==6.5.0 +django-celery-beat==2.6.0 +django-celery-results==2.5.1 +django-choices==2.0.0 +django-cors-headers==4.4.0 +django-csp==3.8 +django-extensions==3.2.3 +django-filter==24.2 +django-health-check==3.18.3 +django-ipware==7.0.1 +django-permissions-policy==4.20.0 +django-redis==5.4.0 +django-relativedelta==2.0.0 +django-solo==2.2.0 +django-timezone-field==6.1.0 +djangorestframework==3.15.2 +djangorestframework-simplejwt==5.3.1 +drf-spectacular==0.27.2 drf-writable-nested==0.6.3 spiffworkflow==1.1.0 Faker==8.13.2 @@ -43,14 +42,14 @@ freezegun==1.1.0 deepdiff prettyprinter gemma-zds-client==1.0.0 -idna==3.3 +idna==3.7 importlib-metadata==4.8.1 inflection==0.5.1 josepy==1.13.0 jsonschema==4.4.0 kombu<6.0 -model-bakery==1.3.2 -mozilla-django-oidc==1.2.4 +model-bakery==1.18.1 +mozilla-django-oidc==4.0.1 packaging prompt-toolkit==3.0.19 psycopg2==2.9.1 @@ -64,19 +63,18 @@ python-crontab==2.5.1 python-dateutil==2.8.2 pytz==2022.1 redis==3.5.3 -requests==2.26.0 +requests==2.32.3 requests-mock==1.9.3 -sentry-sdk==1.19.1 six==1.16.0 soupsieve==2.2.1 -sqlparse==0.4.2 +sqlparse==0.5.0 tenacity==8.0.1 text-unidecode==1.3 -typing-extensions==3.10.0.2 +typing-extensions==4.12.2 uritemplate==3.0.1 -urllib3==1.26.15 -uWSGI==2.0.19.1 -vine==5.0.0 +urllib3==2.2.2 +uWSGI==2.0.26 +vine==5.1.0 wcwidth==0.2.5 zgw-consumers==0.18.0 zipp==3.5.0 diff --git a/app/wait-for.sh b/app/wait-for.sh index c568a2081..5988bd359 100755 --- a/app/wait-for.sh +++ b/app/wait-for.sh @@ -1,8 +1,12 @@ -#!/bin/sh -# Obtained on 07-Oct-2018 from https://github.com/eficode/wait-for +# Copied from https://github.com/eficode/wait-for on 01-07-2024 +VERSION="2.2.4" + +set -- "$@" -- "$TIMEOUT" "$QUIET" "$PROTOCOL" "$HOST" "$PORT" "$result" TIMEOUT=15 QUIET=0 +# The protocol to make the request with, either "tcp" or "http" +PROTOCOL="tcp" echoerr() { if [ "$QUIET" -ne 1 ]; then printf "%s\n" "$*" 1>&2; fi @@ -12,48 +16,113 @@ usage() { exitcode="$1" cat << USAGE >&2 Usage: - $cmdname host:port [-t timeout] [-- command args] + $0 host:port|url [-t timeout] [-- command args] -q | --quiet Do not output any status messages -t TIMEOUT | --timeout=timeout Timeout in seconds, zero for no timeout + Defaults to 15 seconds + -v | --version Show the version of this tool -- COMMAND ARGS Execute command with args after the test finishes USAGE exit "$exitcode" } wait_for() { - for i in `seq $TIMEOUT` ; do - nc -z "$HOST" "$PORT" > /dev/null 2>&1 + case "$PROTOCOL" in + tcp) + if ! command -v nc >/dev/null; then + echoerr 'nc command is missing!' + exit 1 + fi + ;; + http) + if ! command -v wget >/dev/null; then + echoerr 'wget command is missing!' + exit 1 + fi + ;; + esac + + TIMEOUT_END=$(($(date +%s) + TIMEOUT)) + + while :; do + case "$PROTOCOL" in + tcp) + nc -w 1 -z "$HOST" "$PORT" > /dev/null 2>&1 + ;; + http) + wget --timeout=1 --tries=1 -q "$HOST" -O /dev/null > /dev/null 2>&1 + ;; + *) + echoerr "Unknown protocol '$PROTOCOL'" + exit 1 + ;; + esac result=$? + if [ $result -eq 0 ] ; then - if [ $# -gt 0 ] ; then + if [ $# -gt 7 ] ; then + for result in $(seq $(($# - 7))); do + result=$1 + shift + set -- "$@" "$result" + done + + TIMEOUT=$2 QUIET=$3 PROTOCOL=$4 HOST=$5 PORT=$6 result=$7 + shift 7 exec "$@" fi exit 0 fi + + if [ $TIMEOUT -ne 0 -a $(date +%s) -ge $TIMEOUT_END ]; then + echo "Operation timed out" >&2 + exit 1 + fi + sleep 1 done - echo "Operation timed out" >&2 - exit 1 } -while [ $# -gt 0 ] -do +while :; do case "$1" in + http://*|https://*) + HOST="$1" + PROTOCOL="http" + shift 1 + ;; *:* ) HOST=$(printf "%s\n" "$1"| cut -d : -f 1) PORT=$(printf "%s\n" "$1"| cut -d : -f 2) shift 1 ;; + -v | --version) + echo $VERSION + exit + ;; -q | --quiet) QUIET=1 shift 1 ;; - -t) + -q-*) + QUIET=0 + echoerr "Unknown option: $1" + usage 1 + ;; + -q*) + QUIET=1 + result=$1 + shift 1 + set -- -"${result#-q}" "$@" + ;; + -t | --timeout) TIMEOUT="$2" - if [ "$TIMEOUT" = "" ]; then break; fi shift 2 ;; + -t*) + TIMEOUT="${1#-t}" + shift 1 + ;; --timeout=*) TIMEOUT="${1#*=}" shift 1 @@ -65,16 +134,37 @@ do --help) usage 0 ;; + -*) + QUIET=0 + echoerr "Unknown option: $1" + usage 1 + ;; *) + QUIET=0 echoerr "Unknown argument: $1" usage 1 ;; esac done -if [ "$HOST" = "" -o "$PORT" = "" ]; then - echoerr "Error: you need to provide a host and port to test." - usage 2 +if ! [ "$TIMEOUT" -ge 0 ] 2>/dev/null; then + echoerr "Error: invalid timeout '$TIMEOUT'" + usage 3 fi +case "$PROTOCOL" in + tcp) + if [ "$HOST" = "" ] || [ "$PORT" = "" ]; then + echoerr "Error: you need to provide a host and port to test." + usage 2 + fi + ;; + http) + if [ "$HOST" = "" ]; then + echoerr "Error: you need to provide a host to test." + usage 2 + fi + ;; +esac + wait_for "$@" diff --git a/bin/setup_credentials.sh b/bin/setup_credentials.sh index 39854b0a7..c3c81ffe6 100755 --- a/bin/setup_credentials.sh +++ b/bin/setup_credentials.sh @@ -1,2 +1,2 @@ # Creates a superuser for the zaak-gateway backend -echo "from django.contrib.auth import get_user_model; get_user_model().objects.create_superuser('admin@admin.com', 'admin')" | docker-compose -f docker-compose.local.yml run --rm zaak-gateway python manage.py shell +echo "from django.contrib.auth import get_user_model; get_user_model().objects.create_superuser('admin@admin.com', 'insecure')" | docker-compose -f docker-compose.local.yml run -T --rm zaak-gateway python manage.py shell diff --git a/bin/setup_user_groups.sh b/bin/setup_user_groups.sh new file mode 100755 index 000000000..01bd136d6 --- /dev/null +++ b/bin/setup_user_groups.sh @@ -0,0 +1,36 @@ +echo "from apps.users.models import User, UserGroup +from django.contrib.auth.models import Permission + +(group, _) = UserGroup.objects.get_or_create(name='PROJECTMEDEWERKER', display_name='Projectmedewerker') +group.permissions.add(Permission.objects.get(name=\"Can access 'Handelsregister' (bedrijfseigenaren van panden, bedrijfsinformatie)\")) +group.permissions.add(Permission.objects.get(name=\"Can access 'BRP' (persoonsgegevens / ingeschreven personen)\")) +group.permissions.add(Permission.objects.get(name=\"Can access 'invorderingscheck'\")) +group.permissions.add(Permission.objects.get(name='Close a Case (by performing the last task)')) +group.permissions.add(Permission.objects.get(name='Create a new Case')) +group.permissions.add(Permission.objects.get(name='Can perform a tasks')) + +(group, _) = UserGroup.objects.get_or_create(name='TOEZICHTHOUDER', display_name='Toezichthouder') +group.permissions.add(Permission.objects.get(name=\"Can access 'BRP' (persoonsgegevens / ingeschreven personen)\")) +group.permissions.add(Permission.objects.get(name='Can perform a tasks')) + +(group, _) = UserGroup.objects.get_or_create(name='PROJECTHANDHAVER', display_name='Projecthandhaver') +group.permissions.add(Permission.objects.get(name=\"Can access 'Handelsregister' (bedrijfseigenaren van panden, bedrijfsinformatie)\")) +group.permissions.add(Permission.objects.get(name=\"Can access 'BRP' (persoonsgegevens / ingeschreven personen)\")) +group.permissions.add(Permission.objects.get(name=\"Can access 'invorderingscheck'\")) +group.permissions.add(Permission.objects.get(name='Can perform a tasks')) + +(group, _) = UserGroup.objects.get_or_create(name='HANDHAVINGSJURIST', display_name='Handhavingsjurist') +group.permissions.add(Permission.objects.get(name=\"Can access 'Handelsregister' (bedrijfseigenaren van panden, bedrijfsinformatie)\")) +group.permissions.add(Permission.objects.get(name=\"Can access 'BRP' (persoonsgegevens / ingeschreven personen)\")) +group.permissions.add(Permission.objects.get(name=\"Can access 'invorderingscheck'\")) +group.permissions.add(Permission.objects.get(name='Can perform a tasks')) + +(group, _) = UserGroup.objects.get_or_create(name='MEDEWERKER GEVOELIGE ZAKEN', display_name='Medewerker Gevoelige Zaken') +group.permissions.add(Permission.objects.get(name='Can read gevoelige dossiers')) + +(group, _) = UserGroup.objects.get_or_create(name='PROJECTMEDEWERKER_DIGITAAL_TOEZICHT', display_name='Projectmedewerker - Digitaal Toezicht') +group.permissions.add(Permission.objects.get(name=\"Create a new 'Digitaal toezicht' Case\")) + +(group, _) = UserGroup.objects.get_or_create(name='TOEZICHTHOUDER_DIGITAAL_TOEZICHT', display_name='Toezichthouder - Digitaal toezicht') +group.permissions.add(Permission.objects.get(name=\"Can read 'Digitaal toezicht'\")) +group.permissions.add(Permission.objects.get(name=\"Create a new 'Digitaal toezicht' Case\"))" | docker-compose -f docker-compose.local.yml run -T --rm zaak-gateway python manage.py shell diff --git a/docker-compose.local.yml b/docker-compose.local.yml index 15d8a1aab..5b3df1aed 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -1,25 +1,4 @@ services: - zaak-gateway: - networks: - - zaken_network - - top_and_zaak_backend_bridge - build: app - hostname: zaak-gateway - image: ${REGISTRY:-127.0.0.1:5001}/${REPOSITORY:-salmagundi/zaken-backend}:${VERSION:-latest} - ports: - - 8080:8000 - - 5678:5678 - depends_on: - - database - - zaak-redis - env_file: - - .env - command: bash -c "/app/wait-for.sh zaak-redis:6379 && /app/wait-for.sh database:5432 && /app/deploy/docker-entrypoint.development.sh" - volumes: - - ./app:/app - stdin_open: true - tty: true - zaak-redis: build: redis environment: @@ -33,7 +12,7 @@ services: database: networks: - zaken_network - image: amsterdam/postgres12 + image: postgis/postgis:14-3.2 shm_size: "512m" ports: - "6409:5432" @@ -42,11 +21,34 @@ services: volumes: - postgresql-data:/var/lib/postgresql/data + zaak-gateway: + networks: + - zaken_network + - top_and_zaak_backend_bridge + build: app + hostname: zaak-gateway + image: ${REGISTRY:-127.0.0.1:5001}/${REPOSITORY:-salmagundi/zaken-backend}:${VERSION:-latest} + ports: + - 8080:8000 + - 5678:5678 + depends_on: + - database + - zaak-redis + env_file: + - .env + entrypoint: /app/deploy/docker-entrypoint.development.sh + command: python -m debugpy --listen 0.0.0.0:5678 ./manage.py runserver 0.0.0.0:8000 + volumes: + - ./app:/app + stdin_open: true + tty: true + zaken_celery_worker: image: ${REGISTRY:-127.0.0.1:5001}/${REPOSITORY:-salmagundi/zaken-backend}:${VERSION:-latest} container_name: zaken_celery_worker hostname: zaken_celery_worker - command: bash -c "/app/wait-for.sh zaak-redis:6379 && /app/wait-for.sh zaak-gateway:8000 -- /app/celery.sh" + entrypoint: /app/deploy/docker-entrypoint.celery.sh + command: bash -c "/app/wait-for.sh http://zaak-gateway:8000 -- /app/celery.sh" depends_on: - zaak-gateway - database @@ -61,7 +63,8 @@ services: image: ${REGISTRY:-127.0.0.1:5001}/${REPOSITORY:-salmagundi/zaken-backend}:${VERSION:-latest} container_name: zaken_celery_beat hostname: zaken_celery_beat - command: sh -c "/app/wait-for.sh zaak-redis:6379 && /app/wait-for.sh zaak-gateway:8000 -- celery -A config beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler" + entrypoint: /app/deploy/docker-entrypoint.celery.sh + command: sh -c "/app/wait-for.sh http://zaak-gateway:8000 -- celery -A config beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler" depends_on: - zaak-gateway - database diff --git a/docker-compose.yml b/docker-compose.yml index 874be84b7..9c9423df3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,8 +9,5 @@ services: - 5678:5678 env_file: - .env - command: bash -c "/app/wait-for.sh zaak-redis:6379 && /app/wait-for.sh database:5432 && /app/deploy/docker-entrypoint.development.sh" - # volumes: - # - ./app:/app stdin_open: true tty: true diff --git a/open-zaak/open-notificaties/.env b/open-zaak/open-notificaties/.env index 82b7c5143..853446a56 100644 --- a/open-zaak/open-notificaties/.env +++ b/open-zaak/open-notificaties/.env @@ -1,6 +1,6 @@ SECRET_KEY=${SECRET_KEY:-7(h1r2hk)8z9+05edulo_3qzymwbo&c24=)qz7+_@3&2sp=u%i} IS_HTTPS=no -ALLOWED_HOSTS=* +ALLOWED_HOSTS=["*"] # SUBPATH=${SUBPATH:-/localhost} DEBUG=yes DJANGO_SETTINGS_MODULE=nrc.conf.docker