f

airflow/dags/update_app.py

@@ -261,7 +261,7 @@ def copy_public_data_zoneurba(**context):
     def copy_public_data_commune_epcis(**context):
         return copy_table_from_dw_to_app(
             from_table="public_for_app.for_app_commune_epcis",
-            to_table="public.public_data_commune_epcis",
+            to_table="public.public_data_commune_epci",
             environment=context["params"]["environment"],
             btree_index_columns=[
                 ["commune_id"],

airflow/dbt_profile.yml

@@ -6,7 +6,7 @@ sparte:
       pass: "{{ env_var('DBT_DB_PASSWORD') }}"
       port: "{{ env_var('DBT_DB_PORT') | as_number }}"
       schema: "{{ env_var('DBT_DB_SCHEMA') }}"
-      threads: 4
+      threads: 1
       type: "postgres"
       user: "{{ env_var('DBT_DB_USER') }}"
   target: dev

airflow/include/sql/sparte/macros/delete_from_this_where_field_not_in.sql

@@ -8,7 +8,7 @@
     {% endif %}
     DELETE FROM {{ this }}
     WHERE {{ this_field }} in (
-        SELECT {{ this_field }} FROM {{ this }} AS foo
+        SELECT DISTINCT {{ this_field }} FROM {{ this }} AS foo
         LEFT JOIN {{ ref(table) }} AS bar
         ON foo.{{ this_field }} = bar.{{ that_field }}
         WHERE {{ that_field }} is null

airflow/include/sql/sparte/models/for_app/for_app_artificialarea.sql

@@ -12,6 +12,6 @@ SELECT
     departement,
     commune_code             AS city,
     surface / 10000          AS surface,
-    ST_TRANSFORM(geom, 4326) AS mpoly
+    {{ make_valid_multipolygon('ST_Transform(geom, 4326)') }} as mpoly
 FROM
     {{ ref('artificial_commune') }}

airflow/include/sql/sparte/models/ocsge/intersected/artificial_commune.sql

@@ -1,7 +1,10 @@
 {{
     config(
         materialized='incremental',
-        post_hook="{{ delete_from_this_where_field_not_in('ocsge_loaded_date', 'occupation_du_sol', 'loaded_date') }}"
+        pre_hook="{{ delete_from_this_where_field_not_in('ocsge_loaded_date', 'occupation_du_sol', 'loaded_date') }}",
+        indexes=[
+            {'columns': ['ocsge_loaded_date'], 'type': 'btree'}
+        ]
     )
 }}
 

airflow/include/sql/sparte/models/ocsge/intersected/difference_commune.sql

@@ -1,7 +1,10 @@
 {{
     config(
         materialized='incremental',
-        post_hook="{{ delete_from_this_where_field_not_in('ocsge_loaded_date', 'difference', 'loaded_date') }}"
+        pre_hook="{{ delete_from_this_where_field_not_in('ocsge_loaded_date', 'difference', 'loaded_date') }}",
+        indexes=[
+            {'columns': ['ocsge_loaded_date'], 'type': 'btree'}
+        ]
     )
 }}
 

airflow/include/sql/sparte/models/ocsge/intersected/occupation_du_sol_commune.sql

@@ -1,7 +1,7 @@
 {{
     config(
         materialized='incremental',
-        post_hook="{{ delete_from_this_where_field_not_in('ocsge_loaded_date', 'occupation_du_sol', 'loaded_date') }}",
+        pre_hook="{{ delete_from_this_where_field_not_in('ocsge_loaded_date', 'occupation_du_sol', 'loaded_date') }}",
         indexes=[
             {'columns': ['departement'], 'type': 'btree'},
             {'columns': ['year'], 'type': 'btree'},

airflow/include/sql/sparte/models/ocsge/intersected/occupation_du_sol_zonage_urbanisme.sql

@@ -10,7 +10,7 @@
             {'columns': ['zonage_gpu_timestamp'], 'type': 'btree'},
             {'columns': ['geom'], 'type': 'gist'}
         ],
-        post_hook=[
+        pre_hook=[
             "{{ delete_from_this_where_field_not_in('ocsge_loaded_date', 'occupation_du_sol', 'loaded_date') }}",
             "{{ delete_from_this_where_field_not_in('zonage_checksum', 'zonage_urbanisme', 'checksum') }}",
         ]

airflow/include/sql/sparte/models/sudocuh/scot.sql

@@ -9,7 +9,7 @@ SELECT
     circonscription_dept        AS circonscription_departement_code,
     circonscription_departement AS circonscription_departement_nom,
     scot.id_scot::text,
-    nom_scot,
+    REPLACE(initcap(nom_scot), 'Scot', 'SCOT') as nom_scot,
     derniere_procedure,
     scot_moderne,
     code_etat_code              AS code_etat,

project/migrations/0097_projectcommune_commune_insee.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.13 on 2024-10-22 08:51
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("project", "0096_alter_historicalproject_land_id_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="projectcommune",
+            name="commune_insee",
+            field=models.CharField(blank=True, max_length=250, null=True, verbose_name="Code INSEE"),
+        ),
+    ]

project/migrations/0098_auto_20241022_1051.py

@@ -0,0 +1,19 @@
+# Generated by Django 4.2.13 on 2024-10-22 08:51
+
+from django.db import migrations
+
+
+def modify_project_commune_from_primary_keys_to_natural_key(apps, schema_editor):
+    ProjectCommune = apps.get_model("project", "ProjectCommune")
+    for project_commune in ProjectCommune.objects.all():
+        commune = project_commune.commune
+        project_commune.commune_insee = commune.insee
+        project_commune.save()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("project", "0097_projectcommune_commune_insee"),
+    ]
+
+    operations = [migrations.RunPython(modify_project_commune_from_primary_keys_to_natural_key)]

project/migrations/0099_auto_20241022_1057.py

@@ -0,0 +1,20 @@
+# Generated by Django 4.2.13 on 2024-10-22 08:57
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("project", "0098_auto_20241022_1051"),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            """
+    ALTER TABLE public.project_projectcommune
+    ALTER COLUMN commune_id TYPE text USING commune_id::text;
+
+
+    """
+        )
+    ]

project/migrations/0100_auto_20241022_1100.py

@@ -0,0 +1,19 @@
+# Generated by Django 4.2.13 on 2024-10-22 09:00
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("project", "0099_auto_20241022_1057"),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            """
+        UPDATE project_projectcommune
+        SET commune_id = commune_insee;
+
+    """
+        )
+    ]

project/migrations/0101_remove_projectcommune_commune_insee.py

@@ -0,0 +1,16 @@
+# Generated by Django 4.2.13 on 2024-10-22 09:02
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("project", "0100_auto_20241022_1100"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="projectcommune",
+            name="commune_insee",
+        ),
+    ]

project/models/project_base.py

@@ -1,7 +1,7 @@
 import collections
 import logging
 from decimal import Decimal
-from typing import Dict, List, Literal
+from typing import Dict, Literal
 
 import pandas as pd
 from django.conf import settings
@@ -96,18 +96,6 @@ class ProjectCommune(models.Model):
     group_name = models.CharField("Nom du groupe", max_length=100, blank=True, null=True)
 
 
-class CityGroup:
-    def __init__(self, name: str):
-        self.name = name
-        self.cities: List[Commune] = list()
-
-    def append(self, project_commune: ProjectCommune) -> None:
-        self.cities.append(project_commune.commune)
-
-    def __str__(self) -> str:
-        return self.name
-
-
 class Project(BaseProject):
     class OcsgeCoverageStatus(models.TextChoices):
         COMPLETE_UNIFORM = "COMPLETE_UNIFORM", "Complet et uniforme"
@@ -404,21 +392,6 @@ def years(self):
     def nb_years_before_2031(self):
         return 2031 - int(self.analyse_end_date)
 
-    _city_group_list = None
-
-    @property
-    def city_group_list(self):
-        if self._city_group_list is None:
-            self._city_group_list = list()
-            qs = ProjectCommune.objects.filter(project=self)
-            qs = qs.select_related("commune")
-            qs = qs.order_by("group_name", "commune__name")
-            for project_commune in qs:
-                if len(self._city_group_list) == 0 or self._city_group_list[-1].name != project_commune.group_name:
-                    self._city_group_list.append(CityGroup(project_commune.group_name))
-                self._city_group_list[-1].append(project_commune)
-        return self._city_group_list
-
     def delete(self):
         self.cover_image.delete(save=False)
         return super().delete()
@@ -940,8 +913,13 @@ def get_available_millesimes(self, commit=False):
         millesimes = set()
 
         departements = self.cities.values_list("departement", flat=True)
+        print(departements)
+
+        departements_qs = Departement.objects.filter(source_id__in=departements)
+
+        print(departements_qs)
 
-        for departement in Departement.objects.filter(id__in=departements):
+        for departement in Departement.objects.filter(source_id__in=departements):
             if departement.ocsge_millesimes:
                 millesimes.update(departement.ocsge_millesimes)
 
@@ -1116,7 +1094,7 @@ def get_neighbors(self):
         return (
             AdminRef.get_class(self.land_type)
             .objects.filter(mpoly__intersects=self.combined_emprise.buffer(0.0001))
-            .exclude(id=int(self.land_id))
+            .exclude(mpoly=self.land.mpoly)
         )
 
     def get_comparison_lands(

public_data/models/administration/Commune.py

@@ -97,7 +97,7 @@ def get_ocsge_millesimes(self) -> set:
         return self.departement.ocsge_millesimes
 
     def get_cities(self):
-        return Commune.objects.filter(id=self.id).all()
+        return Commune.objects.filter(insee=self.insee).all()
 
     def get_official_id(self) -> str:
         return self.insee if self.insee is not None else ""