Skip to content

Commit

Permalink
Merge pull request #1240 from Amsterdam/feature/replace-atlas-with-pdok
Browse files Browse the repository at this point in the history 
replace atlas with pdok api
  • Loading branch information
@remyvdwereld
remyvdwereld authored Nov 22, 2024
2 parents 9a8aa7b + 1f66991 commit 4cfc6a4
Show file tree
Hide file tree
Showing 18 changed files with 164 additions and 1,236 deletions.
2 changes: 1 addition & 1 deletion .env
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ LOGGING_LEVEL=WARNING # To prevent flooding the logging in local development. De
SECRET_KEY_TOP_ZAKEN=SECRET_KEY_TOP_ZAKEN
SECRET_KEY_TON_ZAKEN=SECRET_KEY_TON_ZAKEN
BELASTING_API_URL=https://api-acc.belastingen.centric.eu/bel/inn/afne/vora/v1/vorderingenidentificatienummer/
BAG_API_SEARCH_URL=https://api.data.amsterdam.nl/atlas/search/adres/
BAG_API_PDOK_URL=https://api.pdok.nl/bzk/locatieserver/search/v3_1/free
BAG_API_NUMMERAANDUIDING_SEARCH_URL=https://api.data.amsterdam.nl/v1/bag/nummeraanduidingen/
BAG_API_BENKAGG_SEARCH_URL=https://api.data.amsterdam.nl/v1/benkagg/adresseerbareobjecten/
DECOS_JOIN_USERNAME=ZakenTop
Expand Down
136 changes: 57 additions & 79 deletions app/apps/addresses/mock.py
View file
Original file line number Diff line number Diff line change
@@ -1,81 +1,58 @@
def mock_do_bag_search_id_result():
def mock_do_bag_search_pdok_by_bag_id_result():
return {
"_links": {
"self": {
"href": "https://api.data.amsterdam.nl/atlas/search/adres/?q=1100MOmo%2042&page=1"
},
"next": {"href": None},
"prev": {"href": None},
},
"count_hits": 1,
"count": 1,
"results": [
{
"_links": {
"self": {
"href": "https://api.data.amsterdam.nl/bag/v1.1/verblijfsobject/0363010001028805/"
}
},
"type": "verblijfsobject",
"dataset": "v11_nummeraanduiding",
"adres": "Mockemstraat 42",
"postcode": "1100MO",
"straatnaam": "Mockemstraat",
"straatnaam_no_ws": "Mockemstraat",
"huisnummer": 42,
"toevoeging": "42",
"bag_huisletter": "",
"bag_toevoeging": "",
"woonplaats": "Amsterdam",
"type_adres": "Hoofdadres",
"status": "Naamgeving uitgegeven",
"landelijk_id": "0363200000516944",
"vbo_status": "Verblijfsobject in gebruik",
"adresseerbaar_object_id": "0363010001028805",
"subtype": "verblijfsobject",
"centroid": [6.969577908893136, 52.82184218979086],
"subtype_id": "0363010001028805",
"_display": "Mockemstraat 42",
}
],
}


def mock_do_bag_search_id_result_without_links():
return {
"_links": {
"self": {
"href": "https://api.data.amsterdam.nl/atlas/search/adres/?q=1100MOmo%2042&page=1"
},
"next": {"href": None},
"prev": {"href": None},
},
"count_hits": 1,
"count": 1,
"results": [
{
"type": "verblijfsobject",
"dataset": "v11_nummeraanduiding",
"adres": "Mockemstraat 42",
"postcode": "1100MO",
"straatnaam": "Mockemstraat",
"straatnaam_no_ws": "Mockemstraat",
"huisnummer": 42,
"toevoeging": "42",
"bag_huisletter": "",
"bag_toevoeging": "",
"woonplaats": "Amsterdam",
"type_adres": "Hoofdadres",
"status": "Naamgeving uitgegeven",
"landelijk_id": "03635000650516944",
"vbo_status": "Verblijfsobject in gebruik",
"adresseerbaar_object_id": "03635000650516944",
"subtype": "verblijfsobject",
"centroid": [4.969577908893136, 52.82184218979086],
"subtype_id": "03635000650516944",
"_display": "Mockemstraat 42",
}
],
"response": {
"numFound": 1,
"start": 0,
"maxScore": 7.2593327,
"numFoundExact": True,
"docs": [
{
"bron": "BAG",
"woonplaatscode": "3594",
"type": "adres",
"woonplaatsnaam": "Amsterdam",
"wijkcode": "WK0363AF",
"huis_nlt": "1",
"openbareruimtetype": "Weg",
"buurtnaam": "Waterloopleinbuurt",
"gemeentecode": "0363",
"rdf_seealso": "http://bag.basisregistraties.overheid.nl/bag/id/nummeraanduiding/0363200012145295",
"weergavenaam": "Amstel 1, 1011PN Amsterdam",
"suggest": [
"Amstel 1, 1011PN Amsterdam",
"Amstel 1, 1011 PN Amsterdam",
],
"adrestype": "hoofdadres",
"straatnaam_verkort": "Amstel",
"id": "adr-9c02454e0f09cd9347aeb11cc03c9fb7",
"gekoppeld_perceel": ["ASD12-P-3514"],
"gemeentenaam": "Amsterdam",
"buurtcode": "BU0363AF09",
"wijknaam": "Nieuwmarkt/Lastage",
"identificatie": "0363010012143319-0363200012145295",
"openbareruimte_id": "0363300000002701",
"waterschapsnaam": "Waterschap Amstel, Gooi en Vecht",
"provinciecode": "PV27",
"postcode": "1011PN",
"provincienaam": "Noord-Holland",
"centroide_ll": "POINT(4.90016547 52.3676456)",
"geometrie_ll": "POINT(4.90016547 52.3676456)",
"nummeraanduiding_id": "0363200012145295",
"waterschapscode": "11",
"adresseerbaarobject_id": "0363010012143319",
"huisnummer": 1,
"provincieafkorting": "NH",
"geometrie_rd": "POINT(121828.874 486751.728)",
"centroide_rd": "POINT(121828.874 486751.728)",
"straatnaam": "Amstel",
"shards": "bag",
"_version_": 1816306460560195585,
"typesortering": 4.0,
"sortering": 1.0,
"shard": "bag",
}
],
}
}


Expand All @@ -84,7 +61,7 @@ def mock_get_bag_identificatie_and_stadsdeel_result_without_stadsdeel():
"_embedded": {
"adresseerbareobjecten": [
{
"huisnummer": 42,
"huisnummer": 1,
"identificatie": "123456789"
# No "gebiedenStadsdeelNaam" key
}
Expand All @@ -99,8 +76,9 @@ def mock_get_bag_identificatie_and_stadsdeel_result():
"adresseerbareobjecten": [
{
"identificatie": "123456789",
"huisnummer": 42,
"huisnummer": 1,
"gebiedenStadsdeelNaam": "Zuidoost",
"typeAdresseerbaarObjectOmschrijving": "verblijfsobject",
}
]
}
Expand Down
73 changes: 29 additions & 44 deletions app/apps/addresses/models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,9 @@

from django.db import models
from utils.api_queries_bag import (
do_bag_search_benkagg_by_bag_id,
do_bag_search_by_bag_id,
do_bag_search_benkagg_by_id,
do_bag_search_pdok_by_bag_id,
)
from utils.coordinates import convert_polygon_to_latlng

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -80,44 +79,40 @@ def get_or_create_by_bag_id(bag_id):
return Address.objects.get_or_create(bag_id=bag_id)[0]

def get_bag_address_data(self):
bag_search_response = do_bag_search_by_bag_id(self.bag_id)
bag_search_results = bag_search_response.get("results", [])

bag_search_response = do_bag_search_pdok_by_bag_id(self.bag_id)
bag_search_results = bag_search_response.get("response", {}).get("docs", [])
if bag_search_results:
# A BAG search will return an array with 1 or more results.
# There could be a "Nevenadres" so check addresses for "Hoofdadres".

found_address = None
for address in bag_search_results:
if address.get("type_adres") == "Hoofdadres":
found_address = address
break # Found first desired object so break the loop.

found_bag_data = found_address or bag_search_results[0]

found_bag_data = bag_search_results[0]
self.postal_code = found_bag_data.get("postcode", "")
self.street_name = found_bag_data.get("straatnaam", "")
self.number = found_bag_data.get("huisnummer", "")
self.suffix_letter = found_bag_data.get("bag_huisletter", "")
self.suffix = found_bag_data.get("bag_toevoeging", "")
# Temporarily property for type. Could be verblijfsobject (huis) or standplaats (woonboot).
self.type = found_bag_data.get("type", "verblijfsobject")

centroid = found_bag_data.get("centroid", None)
self.suffix_letter = found_bag_data.get("huisletter", "")
self.suffix = found_bag_data.get("huisnummertoevoeging", "")
self.nummeraanduiding_id = found_bag_data.get("nummeraanduiding_id", "")
centroid_string = found_bag_data.get("centroide_ll", None)
centroid = self._parse_centroid(centroid_string)
if centroid:
self.lng = centroid[0]
self.lat = centroid[1]

def get_bag_identificatie_and_stadsdeel(self):
def _parse_centroid(self, centroid):
# Check if the string starts with 'POINT(' and ends with ')'
if centroid.startswith("POINT(") and centroid.endswith(")"):
# Remove the 'POINT(' at the beginning and ')' at the end
coordinates_str = centroid[6:-1]
# Split the string by space to get the individual numbers
coordinates = coordinates_str.split()
# Convert the string numbers to float and return as a list
return [float(coordinates[0]), float(coordinates[1])]
else:
raise ValueError("Input string is not in the correct format.")

def get_bag_type_and_stadsdeel(self):
"""
Retrieves the identificatie(nummeraanduiding_id) and stadsdeel of an address by bag_id.
nummeraanduiding_id is needed for BRP and stadsdeel is used for filtering.
If an address has an standplaats (woonboot) instead of verblijfsobject, the coordinates
will be calculated by a polygon.
Retrieves the stadsdeel and type of address by identificatie(nummeraanduiding_id).
"""

is_boat = self.type == "standplaats"
response = do_bag_search_benkagg_by_bag_id(self.bag_id, is_boat)
response = do_bag_search_benkagg_by_id(self.nummeraanduiding_id)

adresseerbareobjecten = response.get("_embedded", {}).get(
"adresseerbareobjecten", []
Expand All @@ -132,29 +127,19 @@ def get_bag_identificatie_and_stadsdeel(self):
),
{},
)

nummeraanduiding_id = found_bag_object.get("identificatie")
if nummeraanduiding_id:
self.nummeraanduiding_id = nummeraanduiding_id

# Temporarily property for type. Could be verblijfsobject (huis) or standplaats (woonboot).
# It's not used by now, but could be useful in the future.
self.type = found_bag_object.get("typeAdresseerbaarObjectOmschrijving")
district_name = found_bag_object.get("gebiedenStadsdeelNaam")

if district_name:
self.district = District.objects.get_or_create(name=district_name)[0]

# Get coordinates for standplaats (woonboot).
ligplaats_geometrie = found_bag_object.get("ligplaatsGeometrie") or {}
ligplaats_coordinates = ligplaats_geometrie.get("coordinates")
if ligplaats_coordinates:
(lat, lng) = convert_polygon_to_latlng(ligplaats_coordinates)
self.lng = lng
self.lat = lat

def update_bag_data(self):
self.get_bag_address_data()
# Prevent a nummeraanduiding_id error while creating a case.
try:
self.get_bag_identificatie_and_stadsdeel()
self.get_bag_type_and_stadsdeel()
except Exception as e:
logger.error(
f"Could not retrieve nummeraanduiding_id for bag_id:{self.bag_id}: {e}"
Expand Down
31 changes: 18 additions & 13 deletions app/apps/addresses/tests/tests_models.py
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
from unittest.mock import patch

from apps.addresses.mock import (
mock_do_bag_search_id_result,
mock_do_bag_search_id_result_without_links,
mock_do_bag_search_pdok_by_bag_id_result,
mock_get_bag_identificatie_and_stadsdeel_result,
mock_get_bag_identificatie_and_stadsdeel_result_without_stadsdeel,
)
Expand All @@ -24,14 +23,16 @@ def test_can_create_address(self):
baker.make(Address)
self.assertEquals(Address.objects.count(), 1)

@patch("apps.addresses.models.do_bag_search_benkagg_by_bag_id")
@patch("apps.addresses.models.do_bag_search_by_bag_id")
@patch("apps.addresses.models.do_bag_search_benkagg_by_id")
@patch("apps.addresses.models.do_bag_search_pdok_by_bag_id")
def test_can_create_address_with_bag_result_without_stadsdeel(
self, mock_do_bag_search_id, mock_do_bag_search_benkagg_id
self, mock_do_bag_search_pdok_by_bag_id, mock_do_bag_search_benkagg_id
):
"""Tests Address object creation with bag data mocks without stadsdeel entry"""

mock_do_bag_search_id.return_value = mock_do_bag_search_id_result()
mock_do_bag_search_pdok_by_bag_id.return_value = (
mock_do_bag_search_pdok_by_bag_id_result()
)
mock_do_bag_search_benkagg_id.return_value = (
mock_get_bag_identificatie_and_stadsdeel_result_without_stadsdeel()
)
Expand All @@ -41,31 +42,35 @@ def test_can_create_address_with_bag_result_without_stadsdeel(

baker.make(Address)

mock_do_bag_search_id.assert_called()
mock_do_bag_search_pdok_by_bag_id.assert_called()
mock_do_bag_search_benkagg_id.assert_called()

self.assertEquals(Address.objects.count(), 1)
self.assertEquals(District.objects.count(), 0)

@patch("apps.addresses.models.do_bag_search_by_bag_id")
@patch("apps.addresses.models.do_bag_search_benkagg_by_bag_id")
@patch("apps.addresses.models.do_bag_search_benkagg_by_id")
@patch("apps.addresses.models.do_bag_search_pdok_by_bag_id")
def test_can_create_address_with_bag_result(
self, mock_do_bag_search_benkagg_id, mock_do_bag_search_id
self, mock_do_bag_search_pdok_by_bag_id, mock_do_bag_search_benkagg_id
):
"""Tests Address object creation with bag data mocks"""

mock_do_bag_search_id.return_value = mock_do_bag_search_id_result()
mock_do_bag_search_pdok_by_bag_id.return_value = (
mock_do_bag_search_pdok_by_bag_id_result()
)
mock_do_bag_search_benkagg_id.return_value = (
mock_get_bag_identificatie_and_stadsdeel_result()
)

self.assertEquals(Address.objects.count(), 0)
self.assertEquals(District.objects.count(), 0)

baker.make(Address)
address = baker.make(Address)

mock_do_bag_search_id.assert_called()
mock_do_bag_search_pdok_by_bag_id.assert_called()
mock_do_bag_search_benkagg_id.assert_called()

self.assertEquals(Address.objects.count(), 1)
self.assertEquals(District.objects.count(), 1)

self.assertEquals(address.district.name, "Zuidoost")
Loading

0 comments on commit 4cfc6a4

Please sign in to comment.