Merge branch 'develop'

requirements.in

@@ -8,7 +8,7 @@ django-two-factor-auth[phonenumbers]
 flake8
 requests
 requests_cache
-git+https://github.com/City-of-Helsinki/[email protected].87#egg=django-munigeo
+git+https://github.com/City-of-Helsinki/[email protected].88#egg=django-munigeo
 pytz
 django-cors-headers
 django-extensions

requirements.txt

@@ -14,27 +14,27 @@ attrs==24.2.0
     #   jsonschema
     #   referencing
     #   requests-cache
-black==24.8.0
+black==24.10.0
     # via -r requirements.in
 bmi-arcgis-restapi==2.4.9
     # via -r requirements.in
-build==1.2.2
+build==1.2.2.post1
     # via pip-tools
 cattrs==24.1.2
     # via requests-cache
 certifi==2024.8.30
     # via
     #   requests
     #   sentry-sdk
-charset-normalizer==3.3.2
+charset-normalizer==3.4.0
     # via requests
 click==8.1.7
     # via
     #   black
     #   pip-tools
-coverage[toml]==7.6.1
+coverage[toml]==7.6.2
     # via pytest-cov
-django==5.1.1
+django==5.1.2
     # via
     #   -r requirements.in
     #   django-cors-headers
@@ -70,7 +70,7 @@ django-mptt==0.16.0
     # via
     #   -r requirements.in
     #   django-munigeo
-django-munigeo @ git+https://github.com/City-of-Helsinki/[email protected].87
+django-munigeo @ git+https://github.com/City-of-Helsinki/[email protected].88
     # via -r requirements.in
 django-otp==1.5.4
     # via django-two-factor-auth
@@ -112,7 +112,7 @@ jedi==0.19.1
     # via -r requirements.in
 jsonschema==4.23.0
     # via drf-spectacular
-jsonschema-specifications==2023.12.1
+jsonschema-specifications==2024.10.1
     # via jsonschema
 libvoikko==4.3
     # via -r requirements.in
@@ -137,7 +137,7 @@ pathspec==0.12.1
     # via black
 pep8-naming==0.14.1
     # via -r requirements.in
-phonenumbers==8.13.45
+phonenumbers==8.13.47
     # via django-two-factor-auth
 pip-tools==7.4.1
     # via -r requirements.in
@@ -155,7 +155,7 @@ pyflakes==3.2.0
     # via flake8
 pypng==0.20220715.0
     # via qrcode
-pyproject-hooks==1.1.0
+pyproject-hooks==1.2.0
     # via
     #   build
     #   pip-tools
@@ -196,7 +196,7 @@ rpds-py==0.20.0
     # via
     #   jsonschema
     #   referencing
-sentry-sdk==2.14.0
+sentry-sdk==2.16.0
     # via -r requirements.in
 six==1.16.0
     # via
@@ -205,7 +205,7 @@ six==1.16.0
     #   url-normalize
 sqlparse==0.5.1
     # via django
-tomli==2.0.1
+tomli==2.0.2
     # via
     #   black
     #   build
@@ -221,7 +221,7 @@ typing-extensions==4.12.2
     #   cattrs
     #   django-modeltranslation
     #   qrcode
-tzdata==2024.1
+tzdata==2024.2
     # via -r requirements.in
 uritemplate==4.1.1
     # via drf-spectacular

services/management/commands/school_district_import/school_district_importer.py

@@ -0,0 +1,170 @@
+import logging
+import re
+from datetime import datetime
+
+from django.contrib.gis.gdal import CoordTransform, DataSource, SpatialReference
+from django.contrib.gis.geos import MultiPolygon
+from munigeo import ocd
+from munigeo.models import (
+    AdministrativeDivision,
+    AdministrativeDivisionGeometry,
+    AdministrativeDivisionType,
+    Municipality,
+)
+
+from services.management.commands.lipas_import import MiniWFS
+
+logger = logging.getLogger(__name__)
+SRID = 3067
+
+
+class SchoolDistrictImporter:
+    WFS_BASE = "https://kartta.hel.fi/ws/geoserver/avoindata/wfs"
+
+    def __init__(self, district_type):
+        self.district_type = district_type
+
+    def import_districts(self, data):
+        """
+        Data is a dictionary containing the following keys:
+        - source_type: The name of the WFS layer to fetch
+        - division_type:  AdministrativeDivisionType type of the division
+        - ocd_id: The key to use in the OCD ID generation
+
+        """
+        wfs = MiniWFS(self.WFS_BASE)
+        municipality = Municipality.objects.get(id="helsinki")
+
+        source_type = data["source_type"]
+        division_type = data["division_type"]
+        ocd_id = data["ocd_id"]
+
+        try:
+            url = wfs.get_feature(type_name=source_type)
+            layer = DataSource(url)[0]
+        except Exception as e:
+            logger.error(f"Error retrieving data for {source_type}: {e}")
+            return
+
+        logger.info(f"Retrieved {len(layer)} {source_type} features.")
+        logger.info("Processing data...")
+
+        division_type_obj, _ = AdministrativeDivisionType.objects.get_or_create(
+            type=division_type
+        )
+
+        for feature in layer:
+            self.import_division(
+                feature,
+                division_type_obj,
+                municipality,
+                ocd_id,
+                source_type,
+            )
+
+    def import_division(
+        self, feature, division_type_obj, municipality, ocd_id, source_type
+    ):
+        origin_id = feature.get("id")
+        if not origin_id:
+            logger.info("Skipping feature without id.")
+            return
+
+        division, _ = AdministrativeDivision.objects.get_or_create(
+            origin_id=origin_id, type=division_type_obj
+        )
+
+        division.municipality = municipality
+        division.parent = municipality.division
+
+        division.ocd_id = ocd.make_id(
+            **{ocd_id: str(origin_id), "parent": municipality.division.ocd_id}
+        )
+
+        service_point_id = str(feature.get("toimipiste_id"))
+
+        if self.district_type == "school":
+            division.service_point_id = service_point_id
+            division.units = [service_point_id]
+
+            if "suomi" in source_type:
+                name = feature.get("nimi_fi")
+                division.name_fi = feature.get("nimi_fi")
+                division.start = self.create_start_date(name)
+                division.end = self.create_end_date(name)
+            if "ruotsi" in source_type:
+                name = feature.get("nimi_se")
+                division.name_sv = feature.get("nimi_se")
+                division.start = self.create_start_date(name)
+                division.end = self.create_end_date(name)
+
+        elif self.district_type == "preschool":
+            units = service_point_id.split(",")
+            division.units = units
+
+            division.name_fi = feature.get("nimi_fi")
+            division.name_sv = feature.get("nimi_se")
+
+            division.extra = {"schoolyear": feature.get("lukuvuosi")}
+
+        division.save()
+
+        self.save_geometry(feature, division)
+
+    def create_start_date(self, name):
+        year = re.split(r"[ -]", name)[-2]
+        return f"{year}-08-01"
+
+    def create_end_date(self, name):
+        year = re.split(r"[ -]", name)[-1]
+        return f"{year }-07-31"
+
+    def save_geometry(self, feature, division):
+        geom = feature.geom
+        if not geom.srid:
+            geom.srid = SRID
+        if geom.srid != SRID:
+            geom.transform(SRID)
+            ct = CoordTransform(SpatialReference(geom.srid), SpatialReference(SRID))
+            geom.transform(ct)
+
+        geom = geom.geos
+        if geom.geom_type == "Polygon":
+            geom = MultiPolygon(geom.buffer(0), srid=geom.srid)
+
+        try:
+            geom_obj = division.geometry
+        except AdministrativeDivisionGeometry.DoesNotExist:
+            geom_obj = AdministrativeDivisionGeometry(division=division)
+
+        geom_obj.boundary = geom
+        geom_obj.save()
+
+    def remove_old_school_year(self, division_type):
+        """
+        During 1.8.-15.12. only the current school year is shown.
+        During 16.12.-31.7. both the current and the next school year are shown.
+
+        The source might be named as "tuleva" but it might still actually be the current school year.
+
+        If today is between 1.8.-15.12 delete the previous year.
+        """
+        division_type_obj = AdministrativeDivisionType.objects.get(type=division_type)
+
+        today = datetime.today()
+
+        last_year = today.year - 1
+        last_year_start_date = f"{last_year}-08-01"
+
+        if datetime(today.year, 8, 1) <= today <= datetime(today.year, 12, 15):
+            if self.district_type == "school":
+                AdministrativeDivision.objects.filter(
+                    type=division_type_obj,
+                    start=last_year_start_date,
+                ).delete()
+
+            if self.district_type == "preschool":
+                AdministrativeDivision.objects.filter(
+                    type=division_type_obj,
+                    extra__schoolyear=f"{last_year}-{last_year + 1}",
+                ).delete()

services/management/commands/update_helsinki_preschool_districts.py

@@ -0,0 +1,54 @@
+from django.core.management.base import BaseCommand
+from munigeo.models import AdministrativeDivision
+
+from services.management.commands.school_district_import.school_district_importer import (
+    SchoolDistrictImporter,
+)
+
+PRESCHOOL_DISTRICT_DATA = [
+    {
+        "source_type": "avoindata:Esiopetusalue_suomi",
+        "division_type": "preschool_education_fi",
+        "ocd_id": "esiopetuksen_oppilaaksiottoalue_fi",
+    },
+    {
+        "source_type": "avoindata:Esiopetusalue_suomi_tuleva",
+        "division_type": "preschool_education_fi",
+        "ocd_id": "esiopetuksen_oppilaaksiottoalue_fi",
+    },
+    {
+        "source_type": "avoindata:Esiopetusalue_ruotsi",
+        "division_type": "preschool_education_sv",
+        "ocd_id": "esiopetuksen_oppilaaksiottoalue_sv",
+    },
+    {
+        "source_type": "avoindata:Esiopetusalue_ruotsi_tuleva",
+        "division_type": "preschool_education_sv",
+        "ocd_id": "esiopetuksen_oppilaaksiottoalue_sv",
+    },
+]
+
+
+class Command(BaseCommand):
+    help = (
+        "Update Helsinki preschool districts. "
+        "Usage: ./manage.py update_helsinki_preschool_districts"
+    )
+
+    def handle(self, *args, **options):
+        division_types = list(
+            {data["division_type"] for data in PRESCHOOL_DISTRICT_DATA}
+        )
+
+        # Remove old divisions before importing new ones to avoid possible duplicates as the source layers may change
+        AdministrativeDivision.objects.filter(
+            type__type__in=division_types, municipality__id="helsinki"
+        ).delete()
+
+        importer = SchoolDistrictImporter(district_type="preschool")
+
+        for data in PRESCHOOL_DISTRICT_DATA:
+            importer.import_districts(data)
+
+        for division_type in division_types:
+            importer.remove_old_school_year(division_type)