Skip to content

Commit

Permalink
Merge pull request #648 from MTES-MCT/feat-airflow-insee-cog
Browse files Browse the repository at this point in the history 
Passage des données de consommation au COG 2024
  • Loading branch information
@alexisig
alexisig authored Oct 17, 2024
2 parents c941d0c + 114f1f8 commit 3f195d1
Show file tree
Hide file tree
Showing 9 changed files with 389 additions and 65 deletions.
4 changes: 3 additions & 1 deletion airflow/include/sql/sparte/dbt_project.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,5 +26,7 @@ models:
+schema: gpu
majic:
+schema: majic
dgaln:
sudocuh:
+schema: sudocuh
insee:
+schema: insee
113 changes: 113 additions & 0 deletions airflow/include/sql/sparte/macros/majic/divide_majic.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
{% macro divide_majic(initial_commune_code, final_commune_code, percent) %}
SELECT
'{{ final_commune_code }}' as commune_code,
conso_2009_2010 * {{ percent }} / 100 as conso_2009_2010,
conso_2009_2010_activite * {{ percent }} / 100 as conso_2009_2010_activite,
conso_2009_2010_habitat * {{ percent }} / 100 as conso_2009_2010_habitat,
conso_2009_2010_mixte * {{ percent }} / 100 as conso_2009_2010_mixte,
conso_2009_2010_route * {{ percent }} / 100 as conso_2009_2010_route,
conso_2009_2010_ferroviaire * {{ percent }} / 100 as conso_2009_2010_ferroviaire,
conso_2009_2010_inconnu * {{ percent }} / 100 as conso_2009_2010_inconnu,
conso_2010_2011 * {{ percent }} / 100 as conso_2010_2011,
conso_2010_2011_activite * {{ percent }} / 100 as conso_2010_2011_activite,
conso_2010_2011_habitat * {{ percent }} / 100 as conso_2010_2011_habitat,
conso_2010_2011_mixte * {{ percent }} / 100 as conso_2010_2011_mixte,
conso_2010_2011_route * {{ percent }} / 100 as conso_2010_2011_route,
conso_2010_2011_ferroviaire * {{ percent }} / 100 as conso_2010_2011_ferroviaire,
conso_2010_2011_inconnu * {{ percent }} / 100 as conso_2010_2011_inconnu,
conso_2011_2012 * {{ percent }} / 100 as conso_2011_2012,
conso_2011_2012_activite * {{ percent }} / 100 as conso_2011_2012_activite,
conso_2011_2012_habitat * {{ percent }} / 100 as conso_2011_2012_habitat,
conso_2011_2012_mixte * {{ percent }} / 100 as conso_2011_2012_mixte,
conso_2011_2012_route * {{ percent }} / 100 as conso_2011_2012_route,
conso_2011_2012_ferroviaire * {{ percent }} / 100 as conso_2011_2012_ferroviaire,
conso_2011_2012_inconnu * {{ percent }} / 100 as conso_2011_2012_inconnu,
conso_2012_2013 * {{ percent }} / 100 as conso_2012_2013,
conso_2012_2013_activite * {{ percent }} / 100 as conso_2012_2013_activite,
conso_2012_2013_habitat * {{ percent }} / 100 as conso_2012_2013_habitat,
conso_2012_2013_mixte * {{ percent }} / 100 as conso_2012_2013_mixte,
conso_2012_2013_route * {{ percent }} / 100 as conso_2012_2013_route,
conso_2012_2013_ferroviaire * {{ percent }} / 100 as conso_2012_2013_ferroviaire,
conso_2012_2013_inconnu * {{ percent }} / 100 as conso_2012_2013_inconnu,
conso_2013_2014 * {{ percent }} / 100 as conso_2013_2014,
conso_2013_2014_activite * {{ percent }} / 100 as conso_2013_2014_activite,
conso_2013_2014_habitat * {{ percent }} / 100 as conso_2013_2014_habitat,
conso_2013_2014_mixte * {{ percent }} / 100 as conso_2013_2014_mixte,
conso_2013_2014_route * {{ percent }} / 100 as conso_2013_2014_route,
conso_2013_2014_ferroviaire * {{ percent }} / 100 as conso_2013_2014_ferroviaire,
conso_2013_2014_inconnu * {{ percent }} / 100 as conso_2013_2014_inconnu,
conso_2014_2015 * {{ percent }} / 100 as conso_2014_2015,
conso_2014_2015_activite * {{ percent }} / 100 as conso_2014_2015_activite,
conso_2014_2015_habitat * {{ percent }} / 100 as conso_2014_2015_habitat,
conso_2014_2015_mixte * {{ percent }} / 100 as conso_2014_2015_mixte,
conso_2014_2015_route * {{ percent }} / 100 as conso_2014_2015_route,
conso_2014_2015_ferroviaire * {{ percent }} / 100 as conso_2014_2015_ferroviaire,
conso_2014_2015_inconnu * {{ percent }} / 100 as conso_2014_2015_inconnu,
conso_2015_2016 * {{ percent }} / 100 as conso_2015_2016,
conso_2015_2016_activite * {{ percent }} / 100 as conso_2015_2016_activite,
conso_2015_2016_habitat * {{ percent }} / 100 as conso_2015_2016_habitat,
conso_2015_2016_mixte * {{ percent }} / 100 as conso_2015_2016_mixte,
conso_2015_2016_route * {{ percent }} / 100 as conso_2015_2016_route,
conso_2015_2016_ferroviaire * {{ percent }} / 100 as conso_2015_2016_ferroviaire,
conso_2015_2016_inconnu * {{ percent }} / 100 as conso_2015_2016_inconnu,
conso_2016_2017 * {{ percent }} / 100 as conso_2016_2017,
conso_2016_2017_activite * {{ percent }} / 100 as conso_2016_2017_activite,
conso_2016_2017_habitat * {{ percent }} / 100 as conso_2016_2017_habitat,
conso_2016_2017_mixte * {{ percent }} / 100 as conso_2016_2017_mixte,
conso_2016_2017_route * {{ percent }} / 100 as conso_2016_2017_route,
conso_2016_2017_ferroviaire * {{ percent }} / 100 as conso_2016_2017_ferroviaire,
conso_2016_2017_inconnu * {{ percent }} / 100 as conso_2016_2017_inconnu,
conso_2017_2018 * {{ percent }} / 100 as conso_2017_2018,
conso_2017_2018_activite * {{ percent }} / 100 as conso_2017_2018_activite,
conso_2017_2018_habitat * {{ percent }} / 100 as conso_2017_2018_habitat,
conso_2017_2018_mixte * {{ percent }} / 100 as conso_2017_2018_mixte,
conso_2017_2018_route * {{ percent }} / 100 as conso_2017_2018_route,
conso_2017_2018_ferroviaire * {{ percent }} / 100 as conso_2017_2018_ferroviaire,
conso_2017_2018_inconnu * {{ percent }} / 100 as conso_2017_2018_inconnu,
conso_2018_2019 * {{ percent }} / 100 as conso_2018_2019,
conso_2018_2019_activite * {{ percent }} / 100 as conso_2018_2019_activite,
conso_2018_2019_habitat * {{ percent }} / 100 as conso_2018_2019_habitat,
conso_2018_2019_mixte * {{ percent }} / 100 as conso_2018_2019_mixte,
conso_2018_2019_route * {{ percent }} / 100 as conso_2018_2019_route,
conso_2018_2019_ferroviaire * {{ percent }} / 100 as conso_2018_2019_ferroviaire,
conso_2018_2019_inconnu * {{ percent }} / 100 as conso_2018_2019_inconnu,
conso_2019_2020 * {{ percent }} / 100 as conso_2019_2020,
conso_2019_2020_activite * {{ percent }} / 100 as conso_2019_2020_activite,
conso_2019_2020_habitat * {{ percent }} / 100 as conso_2019_2020_habitat,
conso_2019_2020_mixte * {{ percent }} / 100 as conso_2019_2020_mixte,
conso_2019_2020_route * {{ percent }} / 100 as conso_2019_2020_route,
conso_2019_2020_ferroviaire * {{ percent }} / 100 as conso_2019_2020_ferroviaire,
conso_2019_2020_inconnu * {{ percent }} / 100 as conso_2019_2020_inconnu,
conso_2020_2021 * {{ percent }} / 100 as conso_2020_2021,
conso_2020_2021_activite * {{ percent }} / 100 as conso_2020_2021_activite,
conso_2020_2021_habitat * {{ percent }} / 100 as conso_2020_2021_habitat,
conso_2020_2021_mixte * {{ percent }} / 100 as conso_2020_2021_mixte,
conso_2020_2021_route * {{ percent }} / 100 as conso_2020_2021_route,
conso_2020_2021_ferroviaire * {{ percent }} / 100 as conso_2020_2021_ferroviaire,
conso_2020_2021_inconnu * {{ percent }} / 100 as conso_2020_2021_inconnu,
conso_2021_2022 * {{ percent }} / 100 as conso_2021_2022,
conso_2021_2022_activite * {{ percent }} / 100 as conso_2021_2022_activite,
conso_2021_2022_habitat * {{ percent }} / 100 as conso_2021_2022_habitat,
conso_2021_2022_mixte * {{ percent }} / 100 as conso_2021_2022_mixte,
conso_2021_2022_route * {{ percent }} / 100 as conso_2021_2022_route,
conso_2021_2022_ferroviaire * {{ percent }} / 100 as conso_2021_2022_ferroviaire,
conso_2021_2022_inconnu * {{ percent }} / 100 as conso_2021_2022_inconnu,
conso_2022_2023 * {{ percent }} / 100 as conso_2022_2023,
conso_2022_2023_activite * {{ percent }} / 100 as conso_2022_2023_activite,
conso_2022_2023_habitat * {{ percent }} / 100 as conso_2022_2023_habitat,
conso_2022_2023_mixte * {{ percent }} / 100 as conso_2022_2023_mixte,
conso_2022_2023_route * {{ percent }} / 100 as conso_2022_2023_route,
conso_2022_2023_ferroviaire * {{ percent }} / 100 as conso_2022_2023_ferroviaire,
conso_2022_2023_inconnu * {{ percent }} / 100 as conso_2022_2023_inconnu,
conso_2009_2023 * {{ percent }} / 100 as conso_2009_2023,
conso_2009_2023_activite * {{ percent }} / 100 as conso_2009_2023_activite,
conso_2009_2023_habitat * {{ percent }} / 100 as conso_2009_2023_habitat,
conso_2009_2023_mixte * {{ percent }} / 100 as conso_2009_2023_mixte,
conso_2009_2023_route * {{ percent }} / 100 as conso_2009_2023_route,
conso_2009_2023_ferroviaire * {{ percent }} / 100 as conso_2009_2023_ferroviaire,
conso_2009_2023_inconnu * {{ percent }} / 100 as conso_2009_2023_inconnu
FROM
{{ ref('consommation') }}
WHERE
commune_code = '{{ initial_commune_code }}'
{% endmacro %}
27 changes: 1 addition & 26 deletions airflow/include/sql/sparte/macros/majic.sql → ...include/sql/sparte/macros/majic/majic.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,6 @@

SELECT
idcom as commune_code,
idcomtxt as commune_name,
idreg as region_code,
idregtxt as region_name,
iddep as departement_code,
iddeptxt as departement_name,
epci23 as epci_code,
epci23txt as epci_name,
scot as scot_name,
aav2020 as aire_attraction_ville,
aav2020txt as tranche_aire_attraction_ville,
aav2020_ty as typologie_aire_attraction_ville,
naf09art10 as conso_2009_2010,
art09act10 as conso_2009_2010_activite,
art09hab10 as conso_2009_2010_habitat,
Expand Down Expand Up @@ -119,21 +108,7 @@
art09mix23 as conso_2009_2023_mixte,
art09rou23 as conso_2009_2023_route,
art09fer23 as conso_2009_2023_ferroviaire,
art09inc23 as conso_2009_2023_inconnu,
pop14 as population_2014,
pop20 as population_2020,
pop1420 as population_variation_2014_2020,
men14 as menages_2014,
men20 as menages_2020,
men1420 as menages_variation_2014_2020,
emp14 as emplois_2014,
emp20 as emplois_2020,
emp1420 as emplois_variation_2014_2020,
mepart1420 as nouveaux_menages_et_emplois_par_ha_2014_2020, -- Nombre de ménages + emplois supplémentaire par ha artificialisé (période 2014-2020). Égal à (men1420 + emp1420) / (naf14art20/10000)
menhab1420 as nouveaux_menages_par_ha_consomme_pour_habitat_2014_2020, -- Nombre de ménages par ha artificialisé à destination de l’habitat (période 2014-2020). Égal à men1420 / (art14hab20 / 10000)
artpop1420 as metre_carre_consomme_par_nouveau_habitant_2014_2020, -- Nombre de m² artificialisé par habitant supplémentaire (période 2014-2020). Égal à naf14art20 / pop1420
surfcom202 as surface_commune_metre_carre_2020,
geom
art09inc23 as conso_2009_2023_inconnu
FROM
{{ source('public', source_table_name) }}
{% endmacro %}
119 changes: 119 additions & 0 deletions airflow/include/sql/sparte/macros/majic/merge_majic.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
{% macro merge_majic(final_commune_code, communes_code_to_merge) %}
SELECT
min('{{ final_commune_code }}') as commune_code,
sum(conso_2009_2010) as conso_2009_2010,
sum(conso_2009_2010_activite) as conso_2009_2010_activite,
sum(conso_2009_2010_habitat) as conso_2009_2010_habitat,
sum(conso_2009_2010_mixte) as conso_2009_2010_mixte,
sum(conso_2009_2010_route) as conso_2009_2010_route,
sum(conso_2009_2010_ferroviaire) as conso_2009_2010_ferroviaire,
sum(conso_2009_2010_inconnu) as conso_2009_2010_inconnu,
sum(conso_2010_2011) as conso_2010_2011,
sum(conso_2010_2011_activite) as conso_2010_2011_activite,
sum(conso_2010_2011_habitat) as conso_2010_2011_habitat,
sum(conso_2010_2011_mixte) as conso_2010_2011_mixte,
sum(conso_2010_2011_route) as conso_2010_2011_route,
sum(conso_2010_2011_ferroviaire) as conso_2010_2011_ferroviaire,
sum(conso_2010_2011_inconnu) as conso_2010_2011_inconnu,
sum(conso_2011_2012) as conso_2011_2012,
sum(conso_2011_2012_activite) as conso_2011_2012_activite,
sum(conso_2011_2012_habitat) as conso_2011_2012_habitat,
sum(conso_2011_2012_mixte) as conso_2011_2012_mixte,
sum(conso_2011_2012_route) as conso_2011_2012_route,
sum(conso_2011_2012_ferroviaire) as conso_2011_2012_ferroviaire,
sum(conso_2011_2012_inconnu) as conso_2011_2012_inconnu,
sum(conso_2012_2013) as conso_2012_2013,
sum(conso_2012_2013_activite) as conso_2012_2013_activite,
sum(conso_2012_2013_habitat) as conso_2012_2013_habitat,
sum(conso_2012_2013_mixte) as conso_2012_2013_mixte,
sum(conso_2012_2013_route) as conso_2012_2013_route,
sum(conso_2012_2013_ferroviaire) as conso_2012_2013_ferroviaire,
sum(conso_2012_2013_inconnu) as conso_2012_2013_inconnu,
sum(conso_2013_2014) as conso_2013_2014,
sum(conso_2013_2014_activite) as conso_2013_2014_activite,
sum(conso_2013_2014_habitat) as conso_2013_2014_habitat,
sum(conso_2013_2014_mixte) as conso_2013_2014_mixte,
sum(conso_2013_2014_route) as conso_2013_2014_route,
sum(conso_2013_2014_ferroviaire) as conso_2013_2014_ferroviaire,
sum(conso_2013_2014_inconnu) as conso_2013_2014_inconnu,
sum(conso_2014_2015) as conso_2014_2015,
sum(conso_2014_2015_activite) as conso_2014_2015_activite,
sum(conso_2014_2015_habitat) as conso_2014_2015_habitat,
sum(conso_2014_2015_mixte) as conso_2014_2015_mixte,
sum(conso_2014_2015_route) as conso_2014_2015_route,
sum(conso_2014_2015_ferroviaire) as conso_2014_2015_ferroviaire,
sum(conso_2014_2015_inconnu) as conso_2014_2015_inconnu,
sum(conso_2015_2016) as conso_2015_2016,
sum(conso_2015_2016_activite) as conso_2015_2016_activite,
sum(conso_2015_2016_habitat) as conso_2015_2016_habitat,
sum(conso_2015_2016_mixte) as conso_2015_2016_mixte,
sum(conso_2015_2016_route) as conso_2015_2016_route,
sum(conso_2015_2016_ferroviaire) as conso_2015_2016_ferroviaire,
sum(conso_2015_2016_inconnu) as conso_2015_2016_inconnu,
sum(conso_2016_2017) as conso_2016_2017,
sum(conso_2016_2017_activite) as conso_2016_2017_activite,
sum(conso_2016_2017_habitat) as conso_2016_2017_habitat,
sum(conso_2016_2017_mixte) as conso_2016_2017_mixte,
sum(conso_2016_2017_route) as conso_2016_2017_route,
sum(conso_2016_2017_ferroviaire) as conso_2016_2017_ferroviaire,
sum(conso_2016_2017_inconnu) as conso_2016_2017_inconnu,
sum(conso_2017_2018) as conso_2017_2018,
sum(conso_2017_2018_activite) as conso_2017_2018_activite,
sum(conso_2017_2018_habitat) as conso_2017_2018_habitat,
sum(conso_2017_2018_mixte) as conso_2017_2018_mixte,
sum(conso_2017_2018_route) as conso_2017_2018_route,
sum(conso_2017_2018_ferroviaire) as conso_2017_2018_ferroviaire,
sum(conso_2017_2018_inconnu) as conso_2017_2018_inconnu,
sum(conso_2018_2019) as conso_2018_2019,
sum(conso_2018_2019_activite) as conso_2018_2019_activite,
sum(conso_2018_2019_habitat) as conso_2018_2019_habitat,
sum(conso_2018_2019_mixte) as conso_2018_2019_mixte,
sum(conso_2018_2019_route) as conso_2018_2019_route,
sum(conso_2018_2019_ferroviaire) as conso_2018_2019_ferroviaire,
sum(conso_2018_2019_inconnu) as conso_2018_2019_inconnu,
sum(conso_2019_2020) as conso_2019_2020,
sum(conso_2019_2020_activite) as conso_2019_2020_activite,
sum(conso_2019_2020_habitat) as conso_2019_2020_habitat,
sum(conso_2019_2020_mixte) as conso_2019_2020_mixte,
sum(conso_2019_2020_route) as conso_2019_2020_route,
sum(conso_2019_2020_ferroviaire) as conso_2019_2020_ferroviaire,
sum(conso_2019_2020_inconnu) as conso_2019_2020_inconnu,
sum(conso_2020_2021) as conso_2020_2021,
sum(conso_2020_2021_activite) as conso_2020_2021_activite,
sum(conso_2020_2021_habitat) as conso_2020_2021_habitat,
sum(conso_2020_2021_mixte) as conso_2020_2021_mixte,
sum(conso_2020_2021_route) as conso_2020_2021_route,
sum(conso_2020_2021_ferroviaire) as conso_2020_2021_ferroviaire,
sum(conso_2020_2021_inconnu) as conso_2020_2021_inconnu,
sum(conso_2021_2022) as conso_2021_2022,
sum(conso_2021_2022_activite) as conso_2021_2022_activite,
sum(conso_2021_2022_habitat) as conso_2021_2022_habitat,
sum(conso_2021_2022_mixte) as conso_2021_2022_mixte,
sum(conso_2021_2022_route) as conso_2021_2022_route,
sum(conso_2021_2022_ferroviaire) as conso_2021_2022_ferroviaire,
sum(conso_2021_2022_inconnu) as conso_2021_2022_inconnu,
sum(conso_2022_2023) as conso_2022_2023,
sum(conso_2022_2023_activite) as conso_2022_2023_activite,
sum(conso_2022_2023_habitat) as conso_2022_2023_habitat,
sum(conso_2022_2023_mixte) as conso_2022_2023_mixte,
sum(conso_2022_2023_route) as conso_2022_2023_route,
sum(conso_2022_2023_ferroviaire) as conso_2022_2023_ferroviaire,
sum(conso_2022_2023_inconnu) as conso_2022_2023_inconnu,
sum(conso_2009_2023) as conso_2009_2023,
sum(conso_2009_2023_activite) as conso_2009_2023_activite,
sum(conso_2009_2023_habitat) as conso_2009_2023_habitat,
sum(conso_2009_2023_mixte) as conso_2009_2023_mixte,
sum(conso_2009_2023_route) as conso_2009_2023_route,
sum(conso_2009_2023_ferroviaire) as conso_2009_2023_ferroviaire,
sum(conso_2009_2023_inconnu) as conso_2009_2023_inconnu
FROM
{{ ref('consommation') }}
WHERE
commune_code in (
'{{ final_commune_code }}',
{% for commune_code in communes_code_to_merge %}
'{{ commune_code }}'
{% if not loop.last %},{% endif %}
{% endfor %}
)
{% endmacro %}
19 changes: 19 additions & 0 deletions airflow/include/sql/sparte/models/insee/cog_changes_2024.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
{{ config(materialized='table') }}

SELECT
index,
"MOD" AS type_de_changement,
"TYPECOM_AV" AS type_commune_avant,
"COM_AV" AS code_commune_avant,
"TNCC_AV" AS type_de_nom_en_clair_avant,
"NCC_AV" AS nom_en_clair_avant,
"LIBELLE_AV" AS libelle_avant,
"TYPECOM_AP" AS type_commune_apres,
"COM_AP" AS code_commune_apres,
"TNCC_AP" AS type_de_nom_en_clair_apres,
"NCC_AP" AS nom_en_clair_apres,
"NCCENR_AP" AS nom_en_clair_enrichi_apres,
"LIBELLE_AP" AS libelle_apres,
TO_DATE("DATE_EFF", 'YYYY-MM-DD') AS date_effet
FROM
{{ source('public', 'insee_cog_changes_2024') }}
45 changes: 45 additions & 0 deletions airflow/include/sql/sparte/models/insee/schema.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@

version: 2

type_de_changement: &type_de_changement
values : [
10, # Changement de nom
20, # Création
21, # Rétablissement
30, # Suppression
31, # Fusion simple
32, # Création de commune nouvelle
33, # Fusion association
34, # Transformation de fusion association en fusion simple
35, # Suppression de commune déléguée
41, # Changement de code dû à un changement de département
50, # Changement de code dû à un transfert de chef-lieu
70, # Transformation de commune associée en commune déléguée
71 # 71 is not in the official list but is present in the data
]

type_commune: &type_commune
values : [
"COM", # Commune
"COMA", # Commune associée
"COMD", # Commune déléguée
"ARM" # Arrondissement municipal
]

models:
- name: cog_changes_2024
columns:
- name: type_de_changement
data_tests:
- accepted_values: *type_de_changement
- name: type_commune_avant
data_tests:
- accepted_values: *type_commune
- name: type_commune_apres
data_tests:
- accepted_values: *type_commune

sources:
- name: public
tables:
- name: insee_cog_changes_2024
Loading

0 comments on commit 3f195d1

Please sign in to comment.