diff --git a/.env b/.env index e93147cc9..3a453ab8f 100644 --- a/.env +++ b/.env @@ -14,7 +14,7 @@ LOGGING_LEVEL=WARNING # To prevent flooding the logging in local development. De SECRET_KEY_TOP_ZAKEN=SECRET_KEY_TOP_ZAKEN SECRET_KEY_TON_ZAKEN=SECRET_KEY_TON_ZAKEN BELASTING_API_URL=https://api-acc.belastingen.centric.eu/bel/inn/afne/vora/v1/vorderingenidentificatienummer/ -BAG_API_SEARCH_URL=https://api.data.amsterdam.nl/atlas/search/adres/ +BAG_API_PDOK_URL=https://api.pdok.nl/bzk/locatieserver/search/v3_1/free BAG_API_NUMMERAANDUIDING_SEARCH_URL=https://api.data.amsterdam.nl/v1/bag/nummeraanduidingen/ BAG_API_BENKAGG_SEARCH_URL=https://api.data.amsterdam.nl/v1/benkagg/adresseerbareobjecten/ DECOS_JOIN_USERNAME=ZakenTop diff --git a/app/apps/addresses/mock.py b/app/apps/addresses/mock.py index f271716b7..5f08c751b 100644 --- a/app/apps/addresses/mock.py +++ b/app/apps/addresses/mock.py @@ -1,81 +1,58 @@ -def mock_do_bag_search_id_result(): +def mock_do_bag_search_pdok_by_bag_id_result(): return { - "_links": { - "self": { - "href": "https://api.data.amsterdam.nl/atlas/search/adres/?q=1100MOmo%2042&page=1" - }, - "next": {"href": None}, - "prev": {"href": None}, - }, - "count_hits": 1, - "count": 1, - "results": [ - { - "_links": { - "self": { - "href": "https://api.data.amsterdam.nl/bag/v1.1/verblijfsobject/0363010001028805/" - } - }, - "type": "verblijfsobject", - "dataset": "v11_nummeraanduiding", - "adres": "Mockemstraat 42", - "postcode": "1100MO", - "straatnaam": "Mockemstraat", - "straatnaam_no_ws": "Mockemstraat", - "huisnummer": 42, - "toevoeging": "42", - "bag_huisletter": "", - "bag_toevoeging": "", - "woonplaats": "Amsterdam", - "type_adres": "Hoofdadres", - "status": "Naamgeving uitgegeven", - "landelijk_id": "0363200000516944", - "vbo_status": "Verblijfsobject in gebruik", - "adresseerbaar_object_id": "0363010001028805", - "subtype": "verblijfsobject", - "centroid": [6.969577908893136, 52.82184218979086], - "subtype_id": "0363010001028805", - "_display": "Mockemstraat 42", - } - ], - } - - -def mock_do_bag_search_id_result_without_links(): - return { - "_links": { - "self": { - "href": "https://api.data.amsterdam.nl/atlas/search/adres/?q=1100MOmo%2042&page=1" - }, - "next": {"href": None}, - "prev": {"href": None}, - }, - "count_hits": 1, - "count": 1, - "results": [ - { - "type": "verblijfsobject", - "dataset": "v11_nummeraanduiding", - "adres": "Mockemstraat 42", - "postcode": "1100MO", - "straatnaam": "Mockemstraat", - "straatnaam_no_ws": "Mockemstraat", - "huisnummer": 42, - "toevoeging": "42", - "bag_huisletter": "", - "bag_toevoeging": "", - "woonplaats": "Amsterdam", - "type_adres": "Hoofdadres", - "status": "Naamgeving uitgegeven", - "landelijk_id": "03635000650516944", - "vbo_status": "Verblijfsobject in gebruik", - "adresseerbaar_object_id": "03635000650516944", - "subtype": "verblijfsobject", - "centroid": [4.969577908893136, 52.82184218979086], - "subtype_id": "03635000650516944", - "_display": "Mockemstraat 42", - } - ], + "response": { + "numFound": 1, + "start": 0, + "maxScore": 7.2593327, + "numFoundExact": True, + "docs": [ + { + "bron": "BAG", + "woonplaatscode": "3594", + "type": "adres", + "woonplaatsnaam": "Amsterdam", + "wijkcode": "WK0363AF", + "huis_nlt": "1", + "openbareruimtetype": "Weg", + "buurtnaam": "Waterloopleinbuurt", + "gemeentecode": "0363", + "rdf_seealso": "http://bag.basisregistraties.overheid.nl/bag/id/nummeraanduiding/0363200012145295", + "weergavenaam": "Amstel 1, 1011PN Amsterdam", + "suggest": [ + "Amstel 1, 1011PN Amsterdam", + "Amstel 1, 1011 PN Amsterdam", + ], + "adrestype": "hoofdadres", + "straatnaam_verkort": "Amstel", + "id": "adr-9c02454e0f09cd9347aeb11cc03c9fb7", + "gekoppeld_perceel": ["ASD12-P-3514"], + "gemeentenaam": "Amsterdam", + "buurtcode": "BU0363AF09", + "wijknaam": "Nieuwmarkt/Lastage", + "identificatie": "0363010012143319-0363200012145295", + "openbareruimte_id": "0363300000002701", + "waterschapsnaam": "Waterschap Amstel, Gooi en Vecht", + "provinciecode": "PV27", + "postcode": "1011PN", + "provincienaam": "Noord-Holland", + "centroide_ll": "POINT(4.90016547 52.3676456)", + "geometrie_ll": "POINT(4.90016547 52.3676456)", + "nummeraanduiding_id": "0363200012145295", + "waterschapscode": "11", + "adresseerbaarobject_id": "0363010012143319", + "huisnummer": 1, + "provincieafkorting": "NH", + "geometrie_rd": "POINT(121828.874 486751.728)", + "centroide_rd": "POINT(121828.874 486751.728)", + "straatnaam": "Amstel", + "shards": "bag", + "_version_": 1816306460560195585, + "typesortering": 4.0, + "sortering": 1.0, + "shard": "bag", + } + ], + } } @@ -84,7 +61,7 @@ def mock_get_bag_identificatie_and_stadsdeel_result_without_stadsdeel(): "_embedded": { "adresseerbareobjecten": [ { - "huisnummer": 42, + "huisnummer": 1, "identificatie": "123456789" # No "gebiedenStadsdeelNaam" key } @@ -99,8 +76,9 @@ def mock_get_bag_identificatie_and_stadsdeel_result(): "adresseerbareobjecten": [ { "identificatie": "123456789", - "huisnummer": 42, + "huisnummer": 1, "gebiedenStadsdeelNaam": "Zuidoost", + "typeAdresseerbaarObjectOmschrijving": "verblijfsobject", } ] } diff --git a/app/apps/addresses/models.py b/app/apps/addresses/models.py index 2c260b9c0..3da9fce0c 100644 --- a/app/apps/addresses/models.py +++ b/app/apps/addresses/models.py @@ -2,10 +2,9 @@ from django.db import models from utils.api_queries_bag import ( - do_bag_search_benkagg_by_bag_id, - do_bag_search_by_bag_id, + do_bag_search_benkagg_by_id, + do_bag_search_pdok_by_bag_id, ) -from utils.coordinates import convert_polygon_to_latlng logger = logging.getLogger(__name__) @@ -80,44 +79,40 @@ def get_or_create_by_bag_id(bag_id): return Address.objects.get_or_create(bag_id=bag_id)[0] def get_bag_address_data(self): - bag_search_response = do_bag_search_by_bag_id(self.bag_id) - bag_search_results = bag_search_response.get("results", []) - + bag_search_response = do_bag_search_pdok_by_bag_id(self.bag_id) + bag_search_results = bag_search_response.get("response", {}).get("docs", []) if bag_search_results: - # A BAG search will return an array with 1 or more results. - # There could be a "Nevenadres" so check addresses for "Hoofdadres". - - found_address = None - for address in bag_search_results: - if address.get("type_adres") == "Hoofdadres": - found_address = address - break # Found first desired object so break the loop. - - found_bag_data = found_address or bag_search_results[0] - + found_bag_data = bag_search_results[0] self.postal_code = found_bag_data.get("postcode", "") self.street_name = found_bag_data.get("straatnaam", "") self.number = found_bag_data.get("huisnummer", "") - self.suffix_letter = found_bag_data.get("bag_huisletter", "") - self.suffix = found_bag_data.get("bag_toevoeging", "") - # Temporarily property for type. Could be verblijfsobject (huis) or standplaats (woonboot). - self.type = found_bag_data.get("type", "verblijfsobject") - - centroid = found_bag_data.get("centroid", None) + self.suffix_letter = found_bag_data.get("huisletter", "") + self.suffix = found_bag_data.get("huisnummertoevoeging", "") + self.nummeraanduiding_id = found_bag_data.get("nummeraanduiding_id", "") + centroid_string = found_bag_data.get("centroide_ll", None) + centroid = self._parse_centroid(centroid_string) if centroid: self.lng = centroid[0] self.lat = centroid[1] - def get_bag_identificatie_and_stadsdeel(self): + def _parse_centroid(self, centroid): + # Check if the string starts with 'POINT(' and ends with ')' + if centroid.startswith("POINT(") and centroid.endswith(")"): + # Remove the 'POINT(' at the beginning and ')' at the end + coordinates_str = centroid[6:-1] + # Split the string by space to get the individual numbers + coordinates = coordinates_str.split() + # Convert the string numbers to float and return as a list + return [float(coordinates[0]), float(coordinates[1])] + else: + raise ValueError("Input string is not in the correct format.") + + def get_bag_type_and_stadsdeel(self): """ - Retrieves the identificatie(nummeraanduiding_id) and stadsdeel of an address by bag_id. - nummeraanduiding_id is needed for BRP and stadsdeel is used for filtering. - If an address has an standplaats (woonboot) instead of verblijfsobject, the coordinates - will be calculated by a polygon. + Retrieves the stadsdeel and type of address by identificatie(nummeraanduiding_id). """ - is_boat = self.type == "standplaats" - response = do_bag_search_benkagg_by_bag_id(self.bag_id, is_boat) + response = do_bag_search_benkagg_by_id(self.nummeraanduiding_id) adresseerbareobjecten = response.get("_embedded", {}).get( "adresseerbareobjecten", [] @@ -132,29 +127,19 @@ def get_bag_identificatie_and_stadsdeel(self): ), {}, ) - - nummeraanduiding_id = found_bag_object.get("identificatie") - if nummeraanduiding_id: - self.nummeraanduiding_id = nummeraanduiding_id - + # Temporarily property for type. Could be verblijfsobject (huis) or standplaats (woonboot). + # It's not used by now, but could be useful in the future. + self.type = found_bag_object.get("typeAdresseerbaarObjectOmschrijving") district_name = found_bag_object.get("gebiedenStadsdeelNaam") if district_name: self.district = District.objects.get_or_create(name=district_name)[0] - # Get coordinates for standplaats (woonboot). - ligplaats_geometrie = found_bag_object.get("ligplaatsGeometrie") or {} - ligplaats_coordinates = ligplaats_geometrie.get("coordinates") - if ligplaats_coordinates: - (lat, lng) = convert_polygon_to_latlng(ligplaats_coordinates) - self.lng = lng - self.lat = lat - def update_bag_data(self): self.get_bag_address_data() # Prevent a nummeraanduiding_id error while creating a case. try: - self.get_bag_identificatie_and_stadsdeel() + self.get_bag_type_and_stadsdeel() except Exception as e: logger.error( f"Could not retrieve nummeraanduiding_id for bag_id:{self.bag_id}: {e}" diff --git a/app/apps/addresses/tests/tests_models.py b/app/apps/addresses/tests/tests_models.py index e29e280a9..7977f38e6 100644 --- a/app/apps/addresses/tests/tests_models.py +++ b/app/apps/addresses/tests/tests_models.py @@ -1,8 +1,7 @@ from unittest.mock import patch from apps.addresses.mock import ( - mock_do_bag_search_id_result, - mock_do_bag_search_id_result_without_links, + mock_do_bag_search_pdok_by_bag_id_result, mock_get_bag_identificatie_and_stadsdeel_result, mock_get_bag_identificatie_and_stadsdeel_result_without_stadsdeel, ) @@ -24,14 +23,16 @@ def test_can_create_address(self): baker.make(Address) self.assertEquals(Address.objects.count(), 1) - @patch("apps.addresses.models.do_bag_search_benkagg_by_bag_id") - @patch("apps.addresses.models.do_bag_search_by_bag_id") + @patch("apps.addresses.models.do_bag_search_benkagg_by_id") + @patch("apps.addresses.models.do_bag_search_pdok_by_bag_id") def test_can_create_address_with_bag_result_without_stadsdeel( - self, mock_do_bag_search_id, mock_do_bag_search_benkagg_id + self, mock_do_bag_search_pdok_by_bag_id, mock_do_bag_search_benkagg_id ): """Tests Address object creation with bag data mocks without stadsdeel entry""" - mock_do_bag_search_id.return_value = mock_do_bag_search_id_result() + mock_do_bag_search_pdok_by_bag_id.return_value = ( + mock_do_bag_search_pdok_by_bag_id_result() + ) mock_do_bag_search_benkagg_id.return_value = ( mock_get_bag_identificatie_and_stadsdeel_result_without_stadsdeel() ) @@ -41,20 +42,22 @@ def test_can_create_address_with_bag_result_without_stadsdeel( baker.make(Address) - mock_do_bag_search_id.assert_called() + mock_do_bag_search_pdok_by_bag_id.assert_called() mock_do_bag_search_benkagg_id.assert_called() self.assertEquals(Address.objects.count(), 1) self.assertEquals(District.objects.count(), 0) - @patch("apps.addresses.models.do_bag_search_by_bag_id") - @patch("apps.addresses.models.do_bag_search_benkagg_by_bag_id") + @patch("apps.addresses.models.do_bag_search_benkagg_by_id") + @patch("apps.addresses.models.do_bag_search_pdok_by_bag_id") def test_can_create_address_with_bag_result( - self, mock_do_bag_search_benkagg_id, mock_do_bag_search_id + self, mock_do_bag_search_pdok_by_bag_id, mock_do_bag_search_benkagg_id ): """Tests Address object creation with bag data mocks""" - mock_do_bag_search_id.return_value = mock_do_bag_search_id_result() + mock_do_bag_search_pdok_by_bag_id.return_value = ( + mock_do_bag_search_pdok_by_bag_id_result() + ) mock_do_bag_search_benkagg_id.return_value = ( mock_get_bag_identificatie_and_stadsdeel_result() ) @@ -62,10 +65,12 @@ def test_can_create_address_with_bag_result( self.assertEquals(Address.objects.count(), 0) self.assertEquals(District.objects.count(), 0) - baker.make(Address) + address = baker.make(Address) - mock_do_bag_search_id.assert_called() + mock_do_bag_search_pdok_by_bag_id.assert_called() mock_do_bag_search_benkagg_id.assert_called() self.assertEquals(Address.objects.count(), 1) self.assertEquals(District.objects.count(), 1) + + self.assertEquals(address.district.name, "Zuidoost") diff --git a/app/apps/cases/forms.py b/app/apps/cases/forms.py deleted file mode 100644 index f30393936..000000000 --- a/app/apps/cases/forms.py +++ /dev/null @@ -1,88 +0,0 @@ -import json -import logging -from json.decoder import JSONDecodeError - -from apps.cases.models import CaseTheme -from apps.users.models import User -from django import forms - -from .serializers import BWVCaseImportValidSerializer - -logger = logging.getLogger(__name__) - - -class ImportBWVCaseDataForm(forms.Form): - json_data = forms.CharField( - label="json data", - widget=forms.Textarea( - attrs={ - "placeholder": 'format: {"123bwv_id": {"postcode": "1234ab", "field": "value", ...}} --', - "class": "vLargeTextField", - } - ), - required=True, - ) - user = forms.ModelChoiceField( - queryset=User.objects.all(), - label="Kies een gebruiker", - to_field_name="pk", - required=True, - ) - status_name = forms.ChoiceField( - choices=( - ("Huisbezoek", "Huisbezoek"), - ("Hercontrole", "Hercontrole"), - ), - label="Kies een de status van de zaak", - required=True, - ) - theme = forms.ModelChoiceField( - queryset=CaseTheme.objects.all(), - label="Kies een thema", - to_field_name="pk", - required=True, - ) - SUBWORKFLOWS_CHOICES = ( - ("", " - begin taak van het gekozen thema - "), - ("visit", "visit (taak: Bepalen processtap)"), - ("debrief", "debrief (taak: Verwerken debrief)"), - ("summon", "summon (taak: Opstellen concept aanschrijving)"), - ("decision", "decision (taak: Opstellen concept besluit)"), - ) - subworkflow = forms.ChoiceField( - choices=SUBWORKFLOWS_CHOICES, - label="Kies in welke fase deze zaken moeten starten", - required=False, - ) - - def clean_json_data(self): - data = self.cleaned_data["json_data"] - try: - data = data.replace('"NaN"', "null") - data = data.replace('"nan"', "null") - data = data.replace('"NaT"', "null") - - data = json.loads(data) - - data = [ - { - "legacy_bwv_case_id": k, - **v, - } - for k, v in data.items() - ] - logger.info("BWV import cases: raw json count") - logger.info(len(data)) - - def strip_value(value): - value = value.strip() if isinstance(value, str) else value - return value - - data = [dict((k, strip_value(v)) for k, v in d.items()) for d in data] - serializer = BWVCaseImportValidSerializer(data=data, many=True) - serializer.is_valid(raise_exception=True) - return serializer.data - except JSONDecodeError as e: - self.add_error("json_data", f"JSONDecodeError: {e}") - except Exception as e: - self.add_error("json_data", f"Serializer error: {e}") diff --git a/app/apps/cases/serializers/__init__.py b/app/apps/cases/serializers/__init__.py index b43a46cbc..036e05aa2 100644 --- a/app/apps/cases/serializers/__init__.py +++ b/app/apps/cases/serializers/__init__.py @@ -1,3 +1,2 @@ -from .bwv import * # noqa from .case import * # noqa from .main import * # noqa diff --git a/app/apps/cases/templates/import/body.html b/app/apps/cases/templates/import/body.html deleted file mode 100644 index a2ddc6c65..000000000 --- a/app/apps/cases/templates/import/body.html +++ /dev/null @@ -1,107 +0,0 @@ -{% extends "admin/base_site.html" %} -{% load i18n admin_urls static admin_modify %} - -{% block coltype %}colM{% endblock %} - -{% block bodyclass %}change-form{% endblock %} - -{% block extrastyle %}{{ block.super }}{% endblock %} - -{% block content_title %} -

Importeer BWV zaken

-

Invoer

-{% endblock %} -{% block content %} -
- - {% if not commited %} -
- {% csrf_token %} -
-
-
- {{form.json_data}} -
-
-
-
- - {{form.user}} -
-
-
-
- - {{form.status_name}} -
-
-
-
- - {{form.theme}} -
-
-
-
- - {{form.subworkflow}} -
-
-
- -
-
-
-
- - {% if validation_form_valid %} - {% include './result.html' %} -
- {% if used_theme_instances.reasons %} -
-
-
- -
    - {% for option in used_theme_instances.reasons %} -
  • - -
    • - {% endfor %} -
-
-
-
-
- -
    - {% for option in used_theme_instances.projects %} -
  • - -
    • - {% endfor %} -
-
-
-
- {% endif %} -
- - -
-
- {% endif %} - - {% else %} - {% if create_update_results %} - De import is geslaagd. - {% include './result.html' %} - {% else %} - Data ontbreekt - {% endif %} -
Terug naar import formulier - {% endif %} -
- - -{% endblock %} diff --git a/app/apps/cases/templates/import/head.html b/app/apps/cases/templates/import/head.html deleted file mode 100644 index 4ccb28248..000000000 --- a/app/apps/cases/templates/import/head.html +++ /dev/null @@ -1,30 +0,0 @@ - - - - - AZA - Import BWV Zaken - - - - - - - - - - - - - - - {% include './styles.html' %} - diff --git a/app/apps/cases/templates/import/result.html b/app/apps/cases/templates/import/result.html deleted file mode 100644 index 827d2af2c..000000000 --- a/app/apps/cases/templates/import/result.html +++ /dev/null @@ -1,50 +0,0 @@ -{% if validation_form_valid and not commited %} -

Voorlopig resultaat

- {{ original_data|length }} zaken in de json data.
- {{ create_update_results|length }} zaken zullen worden aangemaakt of aangepast. - - {% if create_update_errors %} - {{ create_update_errors|length }} zaken zullen errors genereren, en worden overgeslagen. - - {% endif %} - {% if address_mismatches %} - {{ address_mismatches|length }} zaken hebben we meer over minder dan één adres gevonden. - - {% endif %} - {% if visit_errors %} - {{ visit_errors|length }} zaken zijn niet gevonden in TOP, en hebben dus ook geen bezoek info. - - {% endif %} - {% if missing_themes %} - {{ missing_themes|length }} zaken hebben geen correcte theme data voor deze omgeving - - {% endif %} -{% elif commited %} -

Resultaat

- {{ create_update_results|length }} zaken zijn aangemaakt of aangepast. - -{% endif %} diff --git a/app/apps/cases/templates/import/styles.html b/app/apps/cases/templates/import/styles.html deleted file mode 100644 index d209d87c7..000000000 --- a/app/apps/cases/templates/import/styles.html +++ /dev/null @@ -1,83 +0,0 @@ - diff --git a/app/apps/cases/views/__init__.py b/app/apps/cases/views/__init__.py index 144b8e690..a932cb301 100644 --- a/app/apps/cases/views/__init__.py +++ b/app/apps/cases/views/__init__.py @@ -1,4 +1,3 @@ -from .bwv import * # noqa from .case import * # noqa from .main import * # noqa from .theme import * # noqa diff --git a/app/apps/cases/views/bwv.py b/app/apps/cases/views/bwv.py deleted file mode 100644 index 47a667f8d..000000000 --- a/app/apps/cases/views/bwv.py +++ /dev/null @@ -1,648 +0,0 @@ -import datetime -import logging -import mimetypes -import os -from collections import OrderedDict - -import requests -from apps.addresses.models import HousingCorporation -from apps.cases.forms import ImportBWVCaseDataForm -from apps.cases.models import Case, CaseProject, CaseReason, CaseTheme -from apps.cases.serializers import ( - BWVMeldingenSerializer, - BWVStatusSerializer, - LegacyCaseCreateSerializer, -) -from apps.users.models import User -from apps.visits.models import Visit -from apps.visits.serializers import VisitSerializer -from apps.workflow.models import GenericCompletedTask -from apps.workflow.serializers import GenericCompletedTaskCreateSerializer -from django.conf import settings -from django.contrib.auth.decorators import user_passes_test -from django.contrib.auth.mixins import UserPassesTestMixin -from django.http import HttpResponse -from django.shortcuts import redirect -from django.urls import reverse -from django.views.generic.edit import FormView -from utils.api_queries_bag import do_bag_search_address_exact - -logger = logging.getLogger(__name__) - - -@user_passes_test(lambda u: u.is_superuser) -def download_data(request): - DATABASE_USER = os.environ.get("DATABASE_USER") - DATABASE_PASSWORD = os.environ.get("DATABASE_PASSWORD") - DATABASE_HOST = os.environ.get("DATABASE_HOST") - DATABASE_NAME = os.environ.get("DATABASE_NAME") - - filename = "zaken_db.sql" - - command = f"PGPASSWORD='{DATABASE_PASSWORD}' pg_dump -U {DATABASE_USER} -d {DATABASE_NAME} -h {DATABASE_HOST} > {filename}" - os.system(command) - - fl_path = "/app/" - - fl = open(os.path.join(fl_path, filename), "r") - mime_type, _ = mimetypes.guess_type(fl_path) - response = HttpResponse(fl, content_type=mime_type) - response["Content-Disposition"] = "attachment; filename=%s" % filename - return response - - -class ImportBWVCaseDataView(UserPassesTestMixin, FormView): - form_class = ImportBWVCaseDataForm - template_name = "import/body.html" - - reason_translate = { - "melding": "SIG melding", - "sia_melding": "SIG melding", - "sia": "SIG melding", - "project": "Project", - "digitaal_toezicht": "Digitaal toezicht", - "melding_eigenaar": "Leegstandsmelding eigenaar", - "melding_bi": "BI melding", - "eigen_onderzoek": "Eigen onderzoek", - "corporatie_melding": "Corporatie melding", - "politie_(SBA2.0)": "Politie (SBA 2.0)", - "doorzon": "Doorzon", - "ilprowo": "Ilprowo", - "mma": "MMA", - "leegstandsmelding_eigenaar": "Leegstandsmelding eigenaar", - } - label_translate = { - "HM_DATE_CREATED": "Datum aangemaakt", - "WS_DATE_CREATED": "Datum aangemaakt", - "WS_DATE_MODIFIED": "Datum aangepast", - "HM_DATE_MODIFIED": "Datum aangepast", - "HM_USER_CREATED": "Aangemaakt door", - "WS_USER_CREATED": "Aangemaakt door", - "HM_USER_MODIFIED": "Aangepast door", - "WS_USER_MODIFIED": "Aangepast door", - "HM_SITUATIE_SCHETS": "Situatieschets", - "WS_STA_CD_OMSCHRIJVING": "Stadium naam", - "HM_MELDER_TELNR": "Melder telefoonnummer", - "HB_OPMERKING": "Huisbezoek opmerking", - "HB_HIT": "Huisbezoek hit", - "HB_TOEZ_HDR1_CODE": "Huisbezoek toezichthouder 1", - "HB_TOEZ_HDR2_CODE": "Huisbezoek toezichthouder 2", - "HB_BEVINDING_DATUM": "Huisbezoek datum", - "HB_BEVINDING_TIJD": "Huisbezoek tijd", - "WS_TOELICHTING": "Toelichting", - } - - def translate_key_to_label(self, key): - label = self.label_translate.get(key) - if label: - return label - else: - label = key - if label.find("_", 0, 3) >= 0: - label = label.split("_", 1)[1] - return label.lower().replace("_", " ").capitalize() - - def _add_address(self, data): - address_mismatches = [] - results = [] - for d in data: - bag_result = do_bag_search_address_exact(d).get("results", []) - bag_result = [r for r in bag_result] - - d_clone = dict(d) - if bag_result: - d_clone["bag_id"] = bag_result[0]["adresseerbaar_object_id"] - results.append(d_clone) - else: - address_mismatches.append({"data": d_clone, "address": bag_result}) - - return results, address_mismatches - - def _get_headers(self, auth_header=None): - token = settings.SECRET_KEY_AZA_TOP - headers = { - "Authorization": f"{auth_header}" if auth_header else f"{token}", - "content-type": "application/json", - } - return headers - - def _fetch_visit(self, legacy_bwv_case_id): - url = f"{settings.TOP_API_URL}/cases/{legacy_bwv_case_id}/visits/" - try: - response = requests.get( - url=url, - headers=self._get_headers(), - timeout=5, - ) - response.raise_for_status() - except Exception: - return response.status_code - else: - return response.json() - - def _add_visits(self, data, *args, **kwargs): - errors = [] - if settings.TOP_API_URL and settings.SECRET_KEY_AZA_TOP: - for d in data: - visits = self._fetch_visit(d["legacy_bwv_case_id"]) - - if isinstance(visits, list): - for visit in visits: - visit["authors"] = [ - tm.get("user", {}) for tm in visit.get("team_members", []) - ] - d["visits"] = visits - else: - errors.append( - { - "legacy_bwv_case_id": d["legacy_bwv_case_id"], - "status_code": visits, - } - ) - return data, errors - - def add_housing_corporation(self, data, *args, **kwargs): - for d in data: - existing_corpo = ( - d.get("housing_corporation") - if d.get("housing_corporation") is not None - else "thisisnocorporation" - ) - housing_corporation = HousingCorporation.objects.filter( - bwv_name__icontains=existing_corpo - ).first() - d["housing_corporation"] = ( - housing_corporation.id if housing_corporation else None - ) - return data - - def add_theme(self, data, *args, **kwargs): - theme = CaseTheme.objects.get(id=kwargs.get("theme")) - used_theme_instances = { - "reasons": [], - "projects": [], - } - missing_themes = [] - for d in data: - reason = CaseReason.objects.filter( - name=self.reason_translate.get(d.get("reason")), - theme=theme, - ).first() - project = CaseProject.objects.filter( - name=d["project"], - theme=theme, - ).first() - if reason and project: - d["reason"] = reason.id - d["project"] = project.id - d["theme"] = theme.id - used_theme_instances["reasons"].append(reason) - used_theme_instances["projects"].append(project) - else: - d["theme"] = "not_found" - missing_themes.append( - { - "legacy_bwv_case_id": d["legacy_bwv_case_id"], - "reason": reason, - "reason_found": d["reason"], - "project": project, - "project_found": d["project"], - } - ) - data = [d for d in data if d.get("theme") != "not_found"] - return data, missing_themes, used_theme_instances - - def add_status_name(self, data, *args, **kwargs): - status_name = kwargs.get("status_name") - if status_name: - for d in data: - d["status_name"] = status_name - return data - - def _get_object(self, case_id): - return Case.objects.filter( - legacy_bwv_case_id=case_id, is_legacy_bwv=True - ).first() - - def _create_or_update( - self, data, request, commit, user=None, reasons=[], projects=[], kwargs={} - ): - theme = CaseTheme.objects.get(id=kwargs.get("theme")) - melding = CaseReason.objects.get( - name=settings.DEFAULT_REASON, - theme=theme, - ) - errors = [] - results = [] - context = {"request": request} - for d in data: - d_clone = dict(d) - instance = self._get_object(d.get("legacy_bwv_case_id")) - - d["subworkflow"] = kwargs.get("subworkflow") - if d["reason"] == melding.id: - del d["project"] - - if instance: - serializer = self.get_serializer(instance, data=d, context=context) - else: - serializer = self.get_serializer(data=d, context=context) - - if serializer.is_valid(raise_exception=True): - d_clone.update( - { - "case": d.get("legacy_bwv_case_id"), - "created": False if instance else True, - } - ) - if commit: - if d["reason"] in reasons: - if d["reason"] != melding.id and d["project"] not in projects: - continue - else: - continue - - case = serializer.save() - if user: - case.author = user - case.save() - d_clone["case"] = case.id - - # create visits, no update - for visit in d.get("visits", []): - visit["case"] = case.id - visit["task"] = "-1" - visit_instances = Visit.objects.filter( - case=case, - start_time=visit.get("start_time"), - situation=visit.get("situation"), - observations=visit.get("observations"), - can_next_visit_go_ahead=visit.get( - "can_next_visit_go_ahead" - ), - can_next_visit_go_ahead_description=visit.get( - "can_next_visit_go_ahead_description" - ), - suggest_next_visit=visit.get("suggest_next_visit"), - suggest_next_visit_description=visit.get( - "suggest_next_visit_description" - ), - notes=visit.get("notes"), - ) - - visit_serializer = VisitSerializer(data=visit) - if visit_serializer.is_valid() and not visit_instances: - visit_serializer.save() - else: - logger.info(f"Visit serializer errors, case '{case.id}'") - logger.info(visit_serializer.errors) - - results.append(d_clone) - else: - errors.append( - { - "legacy_bwv_case_id": d.get("legacy_bwv_case_id"), - "errors": serializer.errors, - } - ) - return errors, results - - def create_additional_types(self, result_data, user=None, *args, **kwargs): - errors = [] - for d in result_data: - events = d["meldingen"] + d["geschiedenis"] - case = d["case"] - events = [ - dict( - e, - date_added=datetime.datetime.strptime( - e["date_added"], "%d-%m-%Y %H:%M:%S %z" - ), - case_user_task_id="-1", - author=user.id, - case=d["case"], - ) - for e in events - ] - - events_sorted = sorted(events, key=lambda d: d.get("date_added")) - - without_existing_events = [ - e - for e in events_sorted - if not GenericCompletedTask.objects.filter( - case__id=d.get("case"), - description=e.get("description"), - ) - ] - events_serializer = GenericCompletedTaskCreateSerializer( - data=without_existing_events, - context={"request": self.request}, - many=True, - ) - if events_serializer.is_valid(): - events_instances = events_serializer.save() - for e in events_instances: - e.author = user - try: - date_added = dict( - ( - ee.get("variables", {}).get("bwv_id"), - ee.get("date_added"), - ) - for ee in events - ).get(e.variables.get("bwv_id")) - e.date_added = date_added - except Exception: - pass - e.save() - else: - logger.info( - f"GenericCompletedTaskCreateSerializer errors: case '{case}'" - ) - logger.info(events_serializer.errors) - return errors, result_data - - def _parse_case_data_to_case_serializer(self, data): - map = { - "WV_DATE_CREATED": "start_date", - "WV_MEDEDELINGEN": "description", - "ADS_PSCD": "postcode", - "ADS_HSNR": "huisnummer", - "ADS_HSLT": "huisletter", - "ADS_HSTV": "toev", - "CASE_REASON": "reason", - "WV_BEH_CD_OMSCHRIJVING": "project", - "ADS_WOCO": "housing_corporation", - } - - def to_int(v): - - try: - v = int(v.strip().split(".")[0]) - except Exception: - pass - return v - - transform = { - "huisnummer": to_int, - "toev": to_int, - } - - def clean(value, key): - value = value.strip() if isinstance(value, str) else value - value = value if not transform.get(key) else transform.get(key)(value) - try: - value = datetime.datetime.strptime(value, "%d-%m-%Y").strftime( - "%Y-%m-%d" - ) - except Exception: - pass - return value - - return [ - dict((map.get(k, k), clean(v, map.get(k, k))) for k, v in d.items()) - for d in data - ] - - def get_success_url(self, **kwargs): - return reverse(kwargs.get("url_name")) - - @property - def get_serializer(self): - return LegacyCaseCreateSerializer - - def _add_bwv_meldingen(self, data): - for d in data: - additionals_list_items = d.get("meldingen", {}).get("meldingen", {}) - d["meldingen"] = [] - case = d.get("legacy_bwv_case_id") - if additionals_list_items: - additionals_list = [v for k, v in additionals_list_items.items()] - additionals_serializer = BWVMeldingenSerializer( - data=additionals_list, many=True - ) - if additionals_serializer.is_valid(): - sorted_data = sorted( - additionals_serializer.data, - key=lambda d: d.get("HM_DATE_CREATED"), - ) - - validated_data = [] - for status_variables in sorted_data: - mapped_form_data = OrderedDict( - ( - f"{list(status_variables.keys()).index(k):02}{k}", - { - "label": self.translate_key_to_label(k), - "value": v, - }, - ) - for k, v in status_variables.items() - ) - date_added = ( - f'{status_variables.get("HM_DATE_CREATED")} 00:00:00 +0000' - ) - status_data = OrderedDict( - { - "description": f"BWV Melding: {status_variables.get('HOTLINE_MELDING_ID')}", - "variables": OrderedDict( - { - "mapped_form_data": mapped_form_data, - "bwv_id": status_variables.get( - "HOTLINE_MELDING_ID" - ), - } - ), - "date_added": date_added, - } - ) - - validated_data.append(status_data) - - d["meldingen"] = validated_data - else: - logger.info(f"BWVMeldingenSerializer errors: case '{case}'") - logger.info(additionals_serializer.errors) - - return data - - def _add_bwv_status(self, data): - for d in data: - case = d.get("legacy_bwv_case_id") - bwv_status_items = d.get("geschiedenis", {}).get("history", {}) - d["geschiedenis"] = [] - if bwv_status_items: - - generic_completed_task_list = [v for k, v in bwv_status_items.items()] - - status_serializer = BWVStatusSerializer( - data=generic_completed_task_list, many=True - ) - - if status_serializer.is_valid(): - sorted_data = sorted( - status_serializer.data, key=lambda d: d.get("WS_DATE_CREATED") - ) - - validated_status_data = [] - for status_variables in sorted_data: - mapped_form_data = OrderedDict( - ( - f"{list(status_variables.keys()).index(k):02}{k}", - { - "label": self.translate_key_to_label(k), - "value": v, - }, - ) - for k, v in status_variables.items() - ) - date_added = ( - f'{status_variables.get("WS_DATE_CREATED")} 00:00:00 +0000' - ) - status_data = { - "description": f"BWV Status: {status_variables.get('WS_STA_CD_OMSCHRIJVING')}", - "variables": { - "mapped_form_data": mapped_form_data, - "bwv_id": status_variables.get("STADIUM_ID"), - }, - "date_added": date_added, - } - - validated_status_data.append(status_data) - - d["geschiedenis"] = validated_status_data - else: - logger.info(f"BWVStatusSerializer errors: case '{case}'") - logger.info(status_serializer.errors) - - return data - - def add_parsed_data(self, data, *args, **kwargs): - data, visit_errors = self._add_visits(data, *args, **kwargs) - data, missing_themes, used_theme_instances = self.add_theme( - data, *args, **kwargs - ) - data = self.add_housing_corporation(data, *args, **kwargs) - data = self.add_status_name(data, *args, **kwargs) - data = self._add_bwv_meldingen(data) - data = self._add_bwv_status(data) - return data, visit_errors, missing_themes, used_theme_instances - - def test_func(self): - return self.request.user.is_superuser - - def get(self, request, *args, **kwargs): - context = self.get_context_data(**kwargs) - create_update_results = [] - if request.GET.get("commit"): - data = self.request.session.get("validated_cases_data") - form_data = self.request.session.get("validated_cases_data_form_data") - reasons = [int(id) for id in request.GET.getlist("reason", [])] - projects = [int(id) for id in request.GET.getlist("project", [])] - kwargs.update(form_data) - - user = kwargs.get("user") - if user: - user = User.objects.get(id=user) - if data: - create_update_errors, create_update_results = self._create_or_update( - data, - request, - True, - user, - reasons, - projects, - kwargs, - ) - ( - create_additionals_errors, - create_additionals_results, - ) = self.create_additional_types( - create_update_results, - user, - ) - del self.request.session["validated_cases_data"] - del self.request.session["validated_cases_data_form_data"] - else: - return redirect(reverse("import-bwv-cases")) - context.update( - { - "commited": True, - "create_update_results": create_update_results, - } - ) - return self.render_to_response(context) - - def post(self, request, *args, **kwargs): - form_class = self.get_form_class() - form = self.get_form(form_class) - context = self.get_context_data(**kwargs) - data = ( - original_data - ) = ( - address_mismatches - ) = ( - create_update_errors - ) = create_update_results = visit_errors = missing_themes = [] - used_theme_instances = {} - form_valid = False - - if form.is_valid(): - original_data = form.cleaned_data["json_data"] - form_data = dict( - (k, str(v.id) if hasattr(v, "id") else v) - for k, v in form.cleaned_data.items() - if k - in [ - "user", - "project", - "reason", - "theme", - "status_name", - "subworkflow", - ] - ) - kwargs.update(form_data) - - data = self._parse_case_data_to_case_serializer(original_data) - data, address_mismatches = self._add_address(data) - ( - data, - visit_errors, - missing_themes, - used_theme_instances, - ) = self.add_parsed_data(data, *args, **kwargs) - used_theme_instances["reasons"] = list(set(used_theme_instances["reasons"])) - used_theme_instances["projects"] = list( - set(used_theme_instances["projects"]) - ) - - create_update_errors, create_update_results = self._create_or_update( - data, - request, - False, - kwargs.get("user"), - kwargs=kwargs, - ) - form_valid = True - self.request.session["validated_cases_data"] = create_update_results - self.request.session["validated_cases_data_form_data"] = form_data - else: - logger.info("bwv import errors") - logger.info(form.errors) - - context.update( - { - "validation_form_valid": form_valid, - "data": data, - "original_data": original_data, - "address_mismatches": address_mismatches, - "create_update_errors": create_update_errors, - "create_update_results": create_update_results, - "visit_errors": visit_errors, - "missing_themes": missing_themes, - "used_theme_instances": used_theme_instances, - } - ) - return self.render_to_response(context) diff --git a/app/apps/fines/legacy_const.py b/app/apps/fines/legacy_const.py deleted file mode 100644 index 92915678d..000000000 --- a/app/apps/fines/legacy_const.py +++ /dev/null @@ -1,6 +0,0 @@ -# These are legacy BWV states which possible have fines connected to them -STADIA_WITH_FINES = [ - "Meldplicht beschikking boete", - "Invordering dwangsom", - "Opleggen boete", -] diff --git a/app/apps/health/apps.py b/app/apps/health/apps.py index 20d0182e2..1c29f7647 100644 --- a/app/apps/health/apps.py +++ b/app/apps/health/apps.py @@ -7,8 +7,8 @@ class HealthConfig(AppConfig): def ready(self): from .health_checks import ( # OpenZaakZaken,; OpenZaakZakenAlfresco,; OpenZaakZakenCatalogus, - BAGAtlasServiceCheck, BAGBenkaggNummeraanduidingenServiceCheck, + BAGPdokServiceCheck, Belastingdienst, BRPServiceCheck, CeleryExecuteTask, @@ -18,8 +18,8 @@ def ready(self): Toeristischeverhuur, ) - plugin_dir.register(BAGAtlasServiceCheck) plugin_dir.register(BAGBenkaggNummeraanduidingenServiceCheck) + plugin_dir.register(BAGPdokServiceCheck) plugin_dir.register(BRPServiceCheck) plugin_dir.register(Belastingdienst) plugin_dir.register(CeleryExecuteTask) diff --git a/app/apps/health/health_checks.py b/app/apps/health/health_checks.py index 59ba36427..8e5282db6 100644 --- a/app/apps/health/health_checks.py +++ b/app/apps/health/health_checks.py @@ -8,7 +8,10 @@ from health_check.backends import BaseHealthCheckBackend from health_check.exceptions import ServiceUnavailable from requests.exceptions import HTTPError, SSLError, Timeout -from utils.api_queries_bag import do_bag_search_benkagg_by_bag_id +from utils.api_queries_bag import ( + do_bag_search_benkagg_by_id, + do_bag_search_pdok_by_bag_id, +) from utils.api_queries_toeristische_verhuur import ( get_bag_vakantieverhuur_registrations, get_bsn_vakantieverhuur_registrations, @@ -70,16 +73,6 @@ def identifier(self): return self.__class__.__name__ -class BAGAtlasServiceCheck(APIServiceCheckBackend): - """ - Endpoint for checking the BAG Atlas Service API Endpoint - """ - - critical_service = True - api_url = settings.BAG_API_SEARCH_URL - verbose_name = "BAG Atlas" - - class BRPServiceCheck(APIServiceCheckBackend): """ Endpoint for checking the BRP Service API Endpoint @@ -96,11 +89,12 @@ class BAGBenkaggNummeraanduidingenServiceCheck(BaseHealthCheckBackend): """ critical_service = True - verbose_name = "BAG Benkagg Nummeraanduidingen" + verbose_name = "BAG Benkagg Stadsdelen" def check_status(self): try: - response = do_bag_search_benkagg_by_bag_id(settings.BAG_ID_AMSTEL_1) + IDENTIFICATIE_AMSTEL_1 = "0363200012145295" + response = do_bag_search_benkagg_by_id(IDENTIFICATIE_AMSTEL_1) message = response.get("message") if message: self.add_error(ServiceUnavailable(f"{message}"), message) @@ -120,6 +114,38 @@ def identifier(self): return self.verbose_name +class BAGPdokServiceCheck(BaseHealthCheckBackend): + """ + Endpoint for checking the BAG PDOK API + """ + + critical_service = True + verbose_name = "BAG PDOK API" + + def check_status(self): + try: + bag_search_response = do_bag_search_pdok_by_bag_id(settings.BAG_ID_AMSTEL_1) + bag_search_results = bag_search_response.get("response", {}).get("docs", []) + if bag_search_results: + found_bag_data = bag_search_results[0] + weergavenaam = found_bag_data.get("weergavenaam") + + if not weergavenaam == "Amstel 1, 1011PN Amsterdam": + self.add_error( + ServiceUnavailable(f"No expected results: {weergavenaam}") + ) + + except HTTPError as e: + logger.error(e) + self.add_error(ServiceUnavailable(f"HTTPError {e.response.status_code}.")) + except Exception as e: + logger.error(e) + self.add_error(ServiceUnavailable(f"Failed {e}"), e) + + def identifier(self): + return self.verbose_name + + class CeleryExecuteTask(BaseHealthCheckBackend): def check_status(self): result = debug_task.apply_async(ignore_result=False) diff --git a/app/config/settings.py b/app/config/settings.py index c05f25298..6fbf16846 100644 --- a/app/config/settings.py +++ b/app/config/settings.py @@ -333,14 +333,10 @@ def filter_traces(envelope): } # BAG Atlas -BAG_API_SEARCH_URL = os.getenv( - "BAG_API_SEARCH_URL", "https://api.data.amsterdam.nl/atlas/search/adres/" -) -# BAG Nummeraanduidingen -BAG_API_NUMMERAANDUIDING_SEARCH_URL = os.getenv( - "BAG_API_NUMMERAANDUIDING_SEARCH_URL", - "https://api.data.amsterdam.nl/v1/bag/nummeraanduidingen/", +BAG_API_PDOK_URL = os.getenv( + "BAG_API_PDOK_URL", "https://api.pdok.nl/bzk/locatieserver/search/v3_1/free" ) + # BAG benkagg for nummeraanduidingen and stadsdeel BAG_API_BENKAGG_SEARCH_URL = os.getenv( "BAG_API_BENKAGG_SEARCH_URL", diff --git a/app/config/urls.py b/app/config/urls.py index a34f3a6d2..3b30b01d0 100644 --- a/app/config/urls.py +++ b/app/config/urls.py @@ -9,8 +9,6 @@ CaseViewSet, CitizenReportViewSet, DocumentTypeViewSet, - ImportBWVCaseDataView, - download_data, ) from apps.debriefings.views import DebriefingViewSet from apps.decisions.views import DecisionTypeViewSet, DecisionViewSet @@ -94,12 +92,6 @@ def get(self, request, *args, **kwargs): urlpatterns = [ # Admin environment - path("admin/download_data/", download_data), - path( - "admin/import-bwv-cases", - ImportBWVCaseDataView.as_view(), - name="import-bwv-cases", - ), path("admin/", admin.site.urls), # API Routing path("api/v1/", include(router.urls)), diff --git a/app/utils/api_queries_bag.py b/app/utils/api_queries_bag.py index 687491d5e..a2a972a6e 100644 --- a/app/utils/api_queries_bag.py +++ b/app/utils/api_queries_bag.py @@ -10,17 +10,17 @@ @retry(stop=stop_after_attempt(3), after=after_log(logger, logging.ERROR)) -def do_bag_search_benkagg_by_bag_id(bag_id, is_boat=False): +def do_bag_search_benkagg_by_id(identificatie): """ - Search BAG identificatie (nummeraanduiding_id) and stadsdeel using an adresseertVerblijfsobjectId + Search BAG by identificatie (nummeraanduiding_id). + With benkagg you cannot search by bag_id (adresseerbaarobject_id). + You can use the parameters 'verblijfsobjectIdentificatie', 'ligplaatsIdentificatie' or 'standplaatsIdentificatie' + with a bag_id but then you need to know the type of object first (woonhuis, woonboot of woonwagen). + This is really annoying. """ - identification_type = "adresseertVerblijfsobjectIdentificatie" - if is_boat: - identification_type = "ligplaatsIdentificatie" - address_search = requests.get( settings.BAG_API_BENKAGG_SEARCH_URL, - params={identification_type: bag_id}, + params={"identificatie": identificatie}, headers=headers, timeout=30, ) @@ -28,57 +28,17 @@ def do_bag_search_benkagg_by_bag_id(bag_id, is_boat=False): @retry(stop=stop_after_attempt(3), after=after_log(logger, logging.ERROR)) -def do_bag_search_by_bag_id(bag_id): +def do_bag_search_pdok_by_bag_id(bag_id): """ - Search BAG using a BWV 'landelijk BAG ID' + Search BAG PDOK using a 'adresseerbaarobject_id' """ address_search = requests.get( - settings.BAG_API_SEARCH_URL, params={"q": bag_id}, timeout=5 + settings.BAG_API_PDOK_URL, + params={ + "q": bag_id, + "fq": f"gemeentenaam:(amsterdam) AND (type:adres) AND (adresseerbaarobject_id: {bag_id}) AND (adrestype: hoofdadres)", + }, + timeout=5, ) - return address_search.json() - - -# BWV migration queries -def get_bag_search_query(address): - """ - Constructs a BAG search query using the address data - """ - hsltr = address.get("huisletter", "") or "" - toev = address.get("toev", "") or "" - - query = f"{address.get('postcode')} {address.get('huisnummer')} {hsltr}{toev}" - - return query.strip() - -@retry(stop=stop_after_attempt(3), after=after_log(logger, logging.ERROR)) -def do_bag_search_address(address): - """ - Search BAG using a BWV address - """ - query = get_bag_search_query(address) - address_search = requests.get( - settings.BAG_API_SEARCH_URL, - params={"q": query}, - timeout=30, - ) return address_search.json() - - -def do_bag_search_address_exact(address): - """ - Filter BAG results using bag address search with exact matching on query string fields - """ - result = do_bag_search_address(address) - - result["results"] = [ - r - for r in result["results"] - if r.get("huisnummer") == address.get("huisnummer", "") - and r.get("postcode") == address.get("postcode", "") - and r.get("bag_huisletter") == (address.get("huisletter", "") or "") - and r.get("bag_toevoeging") == str(address.get("toev", "") or "") - ] - result["count_hits"] = len(result["results"]) - result["count"] = len(result["results"]) - return result