Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

replace atlas with pdok api #1240

Merged
merged 6 commits into from
Nov 22, 2024
Merged

replace atlas with pdok api #1240

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f77abc7
replace atlas with pdok api
NvdLaan Jul 9, 2024
1832ed4
rm
NvdLaan Jul 10, 2024
434f19c
refactor: remove legacy BWV references and update BAG API settings
remyvdwereld Nov 21, 2024
b32283b
test: reorder patch decorators for address model test
remyvdwereld Nov 21, 2024
2356678
fix: update mock data huisnummer and add type description for address…
remyvdwereld Nov 22, 2024
1f66991
Added comments
remyvdwereld Nov 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .env
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ LOGGING_LEVEL=WARNING # To prevent flooding the logging in local development. De
SECRET_KEY_TOP_ZAKEN=SECRET_KEY_TOP_ZAKEN
SECRET_KEY_TON_ZAKEN=SECRET_KEY_TON_ZAKEN
BELASTING_API_URL=https://api-acc.belastingen.centric.eu/bel/inn/afne/vora/v1/vorderingenidentificatienummer/
BAG_API_SEARCH_URL=https://api.data.amsterdam.nl/atlas/search/adres/
BAG_API_PDOK_URL=https://api.pdok.nl/bzk/locatieserver/search/v3_1/free
BAG_API_NUMMERAANDUIDING_SEARCH_URL=https://api.data.amsterdam.nl/v1/bag/nummeraanduidingen/
BAG_API_BENKAGG_SEARCH_URL=https://api.data.amsterdam.nl/v1/benkagg/adresseerbareobjecten/
DECOS_JOIN_USERNAME=ZakenTop
Expand Down
136 changes: 57 additions & 79 deletions app/apps/addresses/mock.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,81 +1,58 @@
def mock_do_bag_search_id_result():
def mock_do_bag_search_pdok_by_bag_id_result():
return {
"_links": {
"self": {
"href": "https://api.data.amsterdam.nl/atlas/search/adres/?q=1100MOmo%2042&page=1"
},
"next": {"href": None},
"prev": {"href": None},
},
"count_hits": 1,
"count": 1,
"results": [
{
"_links": {
"self": {
"href": "https://api.data.amsterdam.nl/bag/v1.1/verblijfsobject/0363010001028805/"
}
},
"type": "verblijfsobject",
"dataset": "v11_nummeraanduiding",
"adres": "Mockemstraat 42",
"postcode": "1100MO",
"straatnaam": "Mockemstraat",
"straatnaam_no_ws": "Mockemstraat",
"huisnummer": 42,
"toevoeging": "42",
"bag_huisletter": "",
"bag_toevoeging": "",
"woonplaats": "Amsterdam",
"type_adres": "Hoofdadres",
"status": "Naamgeving uitgegeven",
"landelijk_id": "0363200000516944",
"vbo_status": "Verblijfsobject in gebruik",
"adresseerbaar_object_id": "0363010001028805",
"subtype": "verblijfsobject",
"centroid": [6.969577908893136, 52.82184218979086],
"subtype_id": "0363010001028805",
"_display": "Mockemstraat 42",
}
],
}


def mock_do_bag_search_id_result_without_links():
return {
"_links": {
"self": {
"href": "https://api.data.amsterdam.nl/atlas/search/adres/?q=1100MOmo%2042&page=1"
},
"next": {"href": None},
"prev": {"href": None},
},
"count_hits": 1,
"count": 1,
"results": [
{
"type": "verblijfsobject",
"dataset": "v11_nummeraanduiding",
"adres": "Mockemstraat 42",
"postcode": "1100MO",
"straatnaam": "Mockemstraat",
"straatnaam_no_ws": "Mockemstraat",
"huisnummer": 42,
"toevoeging": "42",
"bag_huisletter": "",
"bag_toevoeging": "",
"woonplaats": "Amsterdam",
"type_adres": "Hoofdadres",
"status": "Naamgeving uitgegeven",
"landelijk_id": "03635000650516944",
"vbo_status": "Verblijfsobject in gebruik",
"adresseerbaar_object_id": "03635000650516944",
"subtype": "verblijfsobject",
"centroid": [4.969577908893136, 52.82184218979086],
"subtype_id": "03635000650516944",
"_display": "Mockemstraat 42",
}
],
"response": {
"numFound": 1,
"start": 0,
"maxScore": 7.2593327,
"numFoundExact": True,
"docs": [
{
"bron": "BAG",
"woonplaatscode": "3594",
"type": "adres",
"woonplaatsnaam": "Amsterdam",
"wijkcode": "WK0363AF",
"huis_nlt": "1",
"openbareruimtetype": "Weg",
"buurtnaam": "Waterloopleinbuurt",
"gemeentecode": "0363",
"rdf_seealso": "http://bag.basisregistraties.overheid.nl/bag/id/nummeraanduiding/0363200012145295",
"weergavenaam": "Amstel 1, 1011PN Amsterdam",
"suggest": [
"Amstel 1, 1011PN Amsterdam",
"Amstel 1, 1011 PN Amsterdam",
],
"adrestype": "hoofdadres",
"straatnaam_verkort": "Amstel",
"id": "adr-9c02454e0f09cd9347aeb11cc03c9fb7",
"gekoppeld_perceel": ["ASD12-P-3514"],
"gemeentenaam": "Amsterdam",
"buurtcode": "BU0363AF09",
"wijknaam": "Nieuwmarkt/Lastage",
"identificatie": "0363010012143319-0363200012145295",
"openbareruimte_id": "0363300000002701",
"waterschapsnaam": "Waterschap Amstel, Gooi en Vecht",
"provinciecode": "PV27",
"postcode": "1011PN",
"provincienaam": "Noord-Holland",
"centroide_ll": "POINT(4.90016547 52.3676456)",
"geometrie_ll": "POINT(4.90016547 52.3676456)",
"nummeraanduiding_id": "0363200012145295",
"waterschapscode": "11",
"adresseerbaarobject_id": "0363010012143319",
"huisnummer": 1,
"provincieafkorting": "NH",
"geometrie_rd": "POINT(121828.874 486751.728)",
"centroide_rd": "POINT(121828.874 486751.728)",
"straatnaam": "Amstel",
"shards": "bag",
"_version_": 1816306460560195585,
"typesortering": 4.0,
"sortering": 1.0,
"shard": "bag",
}
],
}
}


Expand All @@ -84,7 +61,7 @@ def mock_get_bag_identificatie_and_stadsdeel_result_without_stadsdeel():
"_embedded": {
"adresseerbareobjecten": [
{
"huisnummer": 42,
"huisnummer": 1,
"identificatie": "123456789"
# No "gebiedenStadsdeelNaam" key
}
Expand All @@ -99,8 +76,9 @@ def mock_get_bag_identificatie_and_stadsdeel_result():
"adresseerbareobjecten": [
{
"identificatie": "123456789",
"huisnummer": 42,
"huisnummer": 1,
"gebiedenStadsdeelNaam": "Zuidoost",
"typeAdresseerbaarObjectOmschrijving": "verblijfsobject",
}
]
}
Expand Down
73 changes: 29 additions & 44 deletions app/apps/addresses/models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,9 @@

from django.db import models
from utils.api_queries_bag import (
do_bag_search_benkagg_by_bag_id,
do_bag_search_by_bag_id,
do_bag_search_benkagg_by_id,
do_bag_search_pdok_by_bag_id,
)
from utils.coordinates import convert_polygon_to_latlng

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -80,44 +79,40 @@ def get_or_create_by_bag_id(bag_id):
return Address.objects.get_or_create(bag_id=bag_id)[0]

def get_bag_address_data(self):
bag_search_response = do_bag_search_by_bag_id(self.bag_id)
bag_search_results = bag_search_response.get("results", [])

bag_search_response = do_bag_search_pdok_by_bag_id(self.bag_id)
bag_search_results = bag_search_response.get("response", {}).get("docs", [])
if bag_search_results:
# A BAG search will return an array with 1 or more results.
# There could be a "Nevenadres" so check addresses for "Hoofdadres".

found_address = None
for address in bag_search_results:
if address.get("type_adres") == "Hoofdadres":
found_address = address
break # Found first desired object so break the loop.

found_bag_data = found_address or bag_search_results[0]

found_bag_data = bag_search_results[0]
self.postal_code = found_bag_data.get("postcode", "")
self.street_name = found_bag_data.get("straatnaam", "")
self.number = found_bag_data.get("huisnummer", "")
self.suffix_letter = found_bag_data.get("bag_huisletter", "")
self.suffix = found_bag_data.get("bag_toevoeging", "")
# Temporarily property for type. Could be verblijfsobject (huis) or standplaats (woonboot).
self.type = found_bag_data.get("type", "verblijfsobject")

centroid = found_bag_data.get("centroid", None)
self.suffix_letter = found_bag_data.get("huisletter", "")
self.suffix = found_bag_data.get("huisnummertoevoeging", "")
self.nummeraanduiding_id = found_bag_data.get("nummeraanduiding_id", "")
centroid_string = found_bag_data.get("centroide_ll", None)
centroid = self._parse_centroid(centroid_string)
if centroid:
self.lng = centroid[0]
self.lat = centroid[1]

def get_bag_identificatie_and_stadsdeel(self):
def _parse_centroid(self, centroid):
# Check if the string starts with 'POINT(' and ends with ')'
if centroid.startswith("POINT(") and centroid.endswith(")"):
# Remove the 'POINT(' at the beginning and ')' at the end
coordinates_str = centroid[6:-1]
# Split the string by space to get the individual numbers
coordinates = coordinates_str.split()
# Convert the string numbers to float and return as a list
return [float(coordinates[0]), float(coordinates[1])]
else:
raise ValueError("Input string is not in the correct format.")

def get_bag_type_and_stadsdeel(self):
"""
Retrieves the identificatie(nummeraanduiding_id) and stadsdeel of an address by bag_id.
nummeraanduiding_id is needed for BRP and stadsdeel is used for filtering.
If an address has an standplaats (woonboot) instead of verblijfsobject, the coordinates
will be calculated by a polygon.
Retrieves the stadsdeel and type of address by identificatie(nummeraanduiding_id).
"""

is_boat = self.type == "standplaats"
response = do_bag_search_benkagg_by_bag_id(self.bag_id, is_boat)
response = do_bag_search_benkagg_by_id(self.nummeraanduiding_id)

adresseerbareobjecten = response.get("_embedded", {}).get(
"adresseerbareobjecten", []
Expand All @@ -132,29 +127,19 @@ def get_bag_identificatie_and_stadsdeel(self):
),
{},
)

nummeraanduiding_id = found_bag_object.get("identificatie")
if nummeraanduiding_id:
self.nummeraanduiding_id = nummeraanduiding_id

# Temporarily property for type. Could be verblijfsobject (huis) or standplaats (woonboot).
# It's not used by now, but could be useful in the future.
self.type = found_bag_object.get("typeAdresseerbaarObjectOmschrijving")
district_name = found_bag_object.get("gebiedenStadsdeelNaam")

if district_name:
self.district = District.objects.get_or_create(name=district_name)[0]

# Get coordinates for standplaats (woonboot).
ligplaats_geometrie = found_bag_object.get("ligplaatsGeometrie") or {}
ligplaats_coordinates = ligplaats_geometrie.get("coordinates")
if ligplaats_coordinates:
(lat, lng) = convert_polygon_to_latlng(ligplaats_coordinates)
self.lng = lng
self.lat = lat

def update_bag_data(self):
self.get_bag_address_data()
# Prevent a nummeraanduiding_id error while creating a case.
try:
self.get_bag_identificatie_and_stadsdeel()
self.get_bag_type_and_stadsdeel()
except Exception as e:
logger.error(
f"Could not retrieve nummeraanduiding_id for bag_id:{self.bag_id}: {e}"
Expand Down
31 changes: 18 additions & 13 deletions app/apps/addresses/tests/tests_models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
from unittest.mock import patch

from apps.addresses.mock import (
mock_do_bag_search_id_result,
mock_do_bag_search_id_result_without_links,
mock_do_bag_search_pdok_by_bag_id_result,
mock_get_bag_identificatie_and_stadsdeel_result,
mock_get_bag_identificatie_and_stadsdeel_result_without_stadsdeel,
)
Expand All @@ -24,14 +23,16 @@ def test_can_create_address(self):
baker.make(Address)
self.assertEquals(Address.objects.count(), 1)

@patch("apps.addresses.models.do_bag_search_benkagg_by_bag_id")
@patch("apps.addresses.models.do_bag_search_by_bag_id")
@patch("apps.addresses.models.do_bag_search_benkagg_by_id")
@patch("apps.addresses.models.do_bag_search_pdok_by_bag_id")
def test_can_create_address_with_bag_result_without_stadsdeel(
self, mock_do_bag_search_id, mock_do_bag_search_benkagg_id
self, mock_do_bag_search_pdok_by_bag_id, mock_do_bag_search_benkagg_id
):
"""Tests Address object creation with bag data mocks without stadsdeel entry"""

mock_do_bag_search_id.return_value = mock_do_bag_search_id_result()
mock_do_bag_search_pdok_by_bag_id.return_value = (
mock_do_bag_search_pdok_by_bag_id_result()
)
mock_do_bag_search_benkagg_id.return_value = (
mock_get_bag_identificatie_and_stadsdeel_result_without_stadsdeel()
)
Expand All @@ -41,31 +42,35 @@ def test_can_create_address_with_bag_result_without_stadsdeel(

baker.make(Address)

mock_do_bag_search_id.assert_called()
mock_do_bag_search_pdok_by_bag_id.assert_called()
mock_do_bag_search_benkagg_id.assert_called()

self.assertEquals(Address.objects.count(), 1)
self.assertEquals(District.objects.count(), 0)

@patch("apps.addresses.models.do_bag_search_by_bag_id")
@patch("apps.addresses.models.do_bag_search_benkagg_by_bag_id")
@patch("apps.addresses.models.do_bag_search_benkagg_by_id")
@patch("apps.addresses.models.do_bag_search_pdok_by_bag_id")
def test_can_create_address_with_bag_result(
self, mock_do_bag_search_benkagg_id, mock_do_bag_search_id
self, mock_do_bag_search_pdok_by_bag_id, mock_do_bag_search_benkagg_id
):
"""Tests Address object creation with bag data mocks"""

mock_do_bag_search_id.return_value = mock_do_bag_search_id_result()
mock_do_bag_search_pdok_by_bag_id.return_value = (
mock_do_bag_search_pdok_by_bag_id_result()
)
mock_do_bag_search_benkagg_id.return_value = (
mock_get_bag_identificatie_and_stadsdeel_result()
)

self.assertEquals(Address.objects.count(), 0)
self.assertEquals(District.objects.count(), 0)

baker.make(Address)
address = baker.make(Address)

mock_do_bag_search_id.assert_called()
mock_do_bag_search_pdok_by_bag_id.assert_called()
mock_do_bag_search_benkagg_id.assert_called()

self.assertEquals(Address.objects.count(), 1)
self.assertEquals(District.objects.count(), 1)

self.assertEquals(address.district.name, "Zuidoost")
Loading
Loading