metabase: Add gps data

gip-inclusion · Jul 17, 2024 · 772e244 · 772e244
1 parent a773892
commit 772e244
Show file tree

Hide file tree

Showing 2 changed files with 56 additions and 16 deletions.
diff --git a/itou/metabase/management/commands/populate_metabase_emplois.py b/itou/metabase/management/commands/populate_metabase_emplois.py
@@ -500,15 +500,31 @@ def populate_enums(self):
             store_df(df=df, table_name=table_name)
 
     def populate_gps_groups(self):
-        queryset = (
-            FollowUpGroup.objects.all()
-            .select_related("beneficiary")
-            .only("beneficiary__department", "id", "created_at", "created_in_bulk", "updated_at")
-        )
+        # The annotate allows us to reduce the excecution time by constructing a lot less python objects
+        queryset = FollowUpGroup.objects.all().annotate(beneficiary_department=F("beneficiary__department"))
+
         populate_table(gps.GroupsTable, batch_size=100_000, querysets=[queryset])
 
     def populate_gps_memberships(self):
-        queryset = FollowUpGroupMembership.objects.all()
+        queryset = (
+            FollowUpGroupMembership.objects.all()
+            .annotate(
+                companies_departments=ArrayAgg(
+                    "member__companymembership__company__department",
+                    filter=Q(member__companymembership__is_active=True),
+                    distinct=True,
+                    ordering="member__companymembership__company__department",
+                )
+            )
+            .annotate(
+                prescriber_departments=ArrayAgg(
+                    "member__prescribermembership__organization__department",
+                    filter=Q(member__prescribermembership__is_active=True),
+                    distinct=True,
+                    ordering="member__prescribermembership__organization__department",
+                )
+            )
+        )
 
         populate_table(gps.MembershipsTable, batch_size=100_000, querysets=[queryset])
 
@@ -537,11 +553,5 @@ def build_dbt_daily(self):
 
     @timeit
     @monitor(monitor_slug="populate-metabase-emplois")
-    @tenacity.retry(
-        retry=tenacity.retry_if_not_exception_type(RuntimeError),
-        stop=tenacity.stop_after_attempt(3),
-        wait=tenacity.wait_fixed(5),
-        after=log_retry_attempt,
-    )
     def handle(self, mode, **options):
         self.MODE_TO_OPERATION[mode]()
diff --git a/itou/metabase/tables/gps.py b/itou/metabase/tables/gps.py
@@ -34,22 +34,28 @@
         },
         {
             "name": "department",
-            "type": "varchar",
+            "type": "text",
             "comment": "Département du bénéficiaire",
-            "fn": lambda o: o.beneficiary.department,
+            "fn": lambda o: o.beneficiary_department,
         },
     ]
 )
 
-GroupsTable = MetabaseTable(name="gps_membres_v1")
-GroupsTable.add_columns(
+MembershipsTable = MetabaseTable(name="gps_membres_v1")
+MembershipsTable.add_columns(
     [
         {
             "name": "id",
             "type": "integer",
             "comment": "ID de la relation",
             "fn": lambda o: o.pk,
         },
+        {
+            "name": "group_id",
+            "type": "integer",
+            "comment": "ID du group de suivi",
+            "fn": lambda o: o.follow_up_group_id,
+        },
         {
             "name": "created_at",
             "type": "timestamp with time zone",  # which is UTC
@@ -62,6 +68,30 @@
             "comment": "Date de modification",
             "fn": lambda o: o.updated_at,
         },
+        {
+            "name": "ended_at",
+            "type": "timestamp with time zone",  # which is UTC
+            "comment": "Date de desactivation",
+            "fn": lambda o: o.ended_at,
+        },
+        {
+            "name": "is_referent",
+            "type": "boolean",
+            "comment": "Est référent",
+            "fn": lambda o: o.is_referent,
+        },
+        {
+            "name": "user_id",
+            "type": "integer",
+            "comment": "ID du member",
+            "fn": lambda o: o.member_id,
+        },
+        {
+            "name": "org_departments",
+            "type": "text[]",
+            "comment": "Départements de l'organisation",
+            "fn": lambda o: o.companies_departments or o.prescriber_departments,
+        },
         {
             "name": "created_in_bulk",
             "type": "boolean",