Statistiques: Ajout de nouvelles données pour le pilotage

clevercloud/cron.json

@@ -29,7 +29,6 @@
 
   "0 0 * * 1 $ROOT/clevercloud/run_management_command.sh shorten_active_sessions",
   "0 2 * * 1 $ROOT/clevercloud/crons/populate_metabase_matomo.sh",
-  "0 3 * * 1 $ROOT/clevercloud/crons/populate_c1_analyses.sh",
 
   "0 0 1 * * $ROOT/clevercloud/run_management_command.sh delete_old_emails --wet-run",
   "0 0 1 * * $ROOT/clevercloud/run_management_command.sh sync_cities --wet-run",

config/settings/base.py

@@ -420,13 +420,6 @@
 METABASE_USER = os.getenv("METABASE_USER")
 METABASE_PASSWORD = os.getenv("METABASE_PASSWORD")
 
-# use almost the same settings for C1 analyses db as base PG.
-C1_METABASE_HOST = os.getenv("C1_METABASE_HOST")
-C1_METABASE_PORT = os.getenv("C1_METABASE_PORT")
-C1_METABASE_USER = os.getenv("C1_METABASE_USER")
-C1_METABASE_PASSWORD = os.getenv("C1_METABASE_PASSWORD")
-C1_METABASE_DATABASE = os.getenv("C1_METABASE_DATABASE")
-
 # Embedding signed Metabase dashboard
 METABASE_SITE_URL = os.getenv("METABASE_SITE_URL")
 METABASE_SECRET_KEY = os.getenv("METABASE_SECRET_KEY")

config/settings/dev.py

@@ -77,13 +77,6 @@
 METABASE_PASSWORD = os.getenv("METABASE_PASSWORD", os.getenv("PGPASSWORD", "password"))  # noqa: F405
 METABASE_DATABASE = os.getenv("METABASE_DATABASE", os.getenv("PGDATABASE", "metabase"))  # noqa: F405
 
-# use almost the same settings for C1 analyses db as base PG.
-C1_METABASE_HOST = os.getenv("C1_METABASE_HOST", os.getenv("PGHOST", "127.0.0.1"))
-C1_METABASE_PORT = os.getenv("C1_METABASE_PORT", os.getenv("PGPORT", "5432"))
-C1_METABASE_USER = os.getenv("C1_METABASE_USER", os.getenv("PGUSER", "postgres"))
-C1_METABASE_PASSWORD = os.getenv("C1_METABASE_PASSWORD", os.getenv("PGPASSWORD", "password"))
-C1_METABASE_DATABASE = os.getenv("C1_METABASE_DATABASE", os.getenv("PGDATABASE", "metabase"))
-
 FORCE_IC_LOGIN = os.getenv("FORCE_IC_LOGIN", "True") == "True"
 
 AWS_STORAGE_BUCKET_NAME = "dev"

itou/metabase/dataframes.py

@@ -8,7 +8,7 @@
 from psycopg import sql
 from tqdm import tqdm
 
-from itou.metabase.db import DB_CURSOR, create_table, get_cursor, get_new_table_name, rename_table_atomically
+from itou.metabase.db import MetabaseDatabaseCursor, create_table, get_new_table_name, rename_table_atomically
 
 
 PANDA_DATAFRAME_TO_PSQL_TYPES_MAPPING = {
@@ -59,7 +59,7 @@ def store_df(df, table_name, max_attempts=5):
             create_table(new_table_name, columns, reset=True)
             for df_chunk in tqdm(df_chunks):
                 rows = df_chunk.replace({np.nan: None}).to_dict(orient="split")["data"]
-                with get_cursor(DB_CURSOR.C2) as (cursor, conn):
+                with MetabaseDatabaseCursor() as (cursor, conn):
                     with cursor.copy(
                         sql.SQL("COPY {table_name} FROM STDIN WITH (FORMAT BINARY)").format(
                             table_name=sql.Identifier(new_table_name),

itou/metabase/db.py

@@ -3,7 +3,6 @@
 """
 
 import copy
-import enum
 import gc
 import os
 import urllib
@@ -18,21 +17,17 @@
 
 
 class MetabaseDatabaseCursor:
-    def __init__(self, settings_prefix):
-        self.settings_prefix = settings_prefix
+    def __init__(self):
         self.cursor = None
         self.connection = None
 
-    def get_setting(self, name):
-        return getattr(settings, f"{self.settings_prefix}{name}")
-
     def __enter__(self):
         self.connection = psycopg.connect(
-            host=self.get_setting("HOST"),
-            port=self.get_setting("PORT"),
-            dbname=self.get_setting("DATABASE"),
-            user=self.get_setting("USER"),
-            password=self.get_setting("PASSWORD"),
+            host=settings.METABASE_HOST,
+            port=settings.METABASE_PORT,
+            dbname=settings.METABASE_DATABASE,
+            user=settings.METABASE_USER,
+            password=settings.METABASE_PASSWORD,
             keepalives=1,
             keepalives_idle=30,
             keepalives_interval=5,
@@ -48,20 +43,6 @@ def __exit__(self, exc_type, exc_value, exc_traceback):
             self.connection.close()
 
 
-class DB_CURSOR(enum.Enum):
-    C1 = "C1"
-    C2 = "C2"
-
-
-def get_cursor(cursor_name):
-    """Allow to easily monkey patch cursor in tests"""
-    settings_prefix = {
-        DB_CURSOR.C2: "METABASE_",
-        DB_CURSOR.C1: "C1_METABASE_",
-    }[cursor_name]
-    return MetabaseDatabaseCursor(settings_prefix)
-
-
 def get_current_dir():
     return os.path.dirname(os.path.realpath(__file__))
 
@@ -74,7 +55,7 @@ def get_old_table_name(table_name):
     return f"z_old_{table_name}"
 
 
-def rename_table_atomically(from_table_name, to_table_name, cursor_name=DB_CURSOR.C2):
+def rename_table_atomically(from_table_name, to_table_name):
     """
     Rename from_table_name to to_table_name.
     Most of the time, we replace an existing table, so we will first rename
@@ -84,7 +65,7 @@ def rename_table_atomically(from_table_name, to_table_name, cursor_name=DB_CURSO
     are deleted as well, they will be rebuilt by the next run of the airflow DAG `dbt_daily`.
     """
 
-    with get_cursor(cursor_name) as (cur, conn):
+    with MetabaseDatabaseCursor() as (cur, conn):
         # CASCADE will drop airflow staging views (e.g. stg_structures) as well.
         cur.execute(
             sql.SQL("DROP TABLE IF EXISTS {} CASCADE").format(sql.Identifier(get_old_table_name(to_table_name)))
@@ -110,9 +91,9 @@ def rename_table_atomically(from_table_name, to_table_name, cursor_name=DB_CURSO
         conn.commit()
 
 
-def create_table(table_name: str, columns: list[str, str], reset=False, cursor_name=DB_CURSOR.C2):
+def create_table(table_name: str, columns: list[str, str], reset=False):
     """Create table from columns names and types"""
-    with get_cursor(cursor_name) as (cursor, conn):
+    with MetabaseDatabaseCursor() as (cursor, conn):
         if reset:
             cursor.execute(sql.SQL("DROP TABLE IF EXISTS {table_name}").format(table_name=sql.Identifier(table_name)))
         create_table_query = sql.SQL("CREATE TABLE IF NOT EXISTS {table_name} ({fields_with_type})").format(
@@ -152,7 +133,7 @@ def create_unversioned_tables_if_needed():
     The present function creates these unversioned tables without any content, at least now all the requests
     can complete locally and we have a good visibility of how many tables are left to be automated.
     """
-    with get_cursor(DB_CURSOR.C2) as (cur, conn):
+    with MetabaseDatabaseCursor() as (cur, conn):
         create_table_sql_requests = """
             /* TODO @defajait DROP ASAP - use codes_insee_vs_codes_postaux instead */
             CREATE TABLE IF NOT EXISTS "commune_gps" (
@@ -268,7 +249,7 @@ def create_unversioned_tables_if_needed():
         print("Done.")
 
 
-def populate_table(table, batch_size, querysets=None, extra_object=None, cursor_name=DB_CURSOR.C2):
+def populate_table(table, batch_size, querysets=None, extra_object=None):
     """
     About commits: a single final commit freezes the itou-metabase-db temporarily, making
     our GUI unable to connect to the db during this commit.
@@ -309,9 +290,9 @@ def populate_table(table, batch_size, querysets=None, extra_object=None, cursor_
     print(f"Injecting {total_rows} rows with {len(table.columns)} columns into table {table_name}:")
 
     new_table_name = get_new_table_name(table_name)
-    create_table(new_table_name, [(c["name"], c["type"]) for c in table.columns], reset=True, cursor_name=cursor_name)
+    create_table(new_table_name, [(c["name"], c["type"]) for c in table.columns], reset=True)
 
-    with get_cursor(cursor_name) as (cur, conn):
+    with MetabaseDatabaseCursor() as (cur, conn):
 
         def inject_chunk(table_columns, chunk, new_table_name):
             rows = [[c["fn"](row) for c in table_columns] for row in chunk]
@@ -358,4 +339,4 @@ def inject_chunk(table_columns, chunk, new_table_name):
             # Trigger garbage collection to optimize memory use.
             gc.collect()
 
-    rename_table_atomically(new_table_name, table_name, cursor_name=cursor_name)
+    rename_table_atomically(new_table_name, table_name)

itou/metabase/management/commands/metabase_to_csv.py

@@ -18,7 +18,7 @@
 
 from psycopg import sql
 
-from itou.metabase.db import DB_CURSOR, get_cursor
+from itou.metabase.db import MetabaseDatabaseCursor
 from itou.utils.command import BaseCommand
 
 
@@ -30,7 +30,7 @@ def add_arguments(self, parser):
         parser.add_argument("--table_name", action="store", dest="table_name", type=str)
 
     def handle(self, prefix, table_name, **kwargs):
-        with get_cursor(DB_CURSOR.C2) as (cursor, _conn):
+        with MetabaseDatabaseCursor() as (cursor, _conn):
             self.stdout.write(f"exporting {table_name=}")
             cursor.execute(
                 sql.SQL(

itou/metabase/management/commands/populate_metabase_emplois.py

@@ -269,7 +269,7 @@ def populate_job_seekers(self):
         """
         queryset = (
             User.objects.filter(kind=UserKind.JOB_SEEKER)
-            .select_related("jobseeker_profile")
+            .select_related("jobseeker_profile", "created_by")
             .prefetch_related(
                 Prefetch(
                     "eligibility_diagnoses",
@@ -297,7 +297,6 @@ def populate_job_seekers(self):
                     "job_applications",
                     queryset=JobApplication.objects.select_related("to_company"),
                 ),
-                "created_by",
             )
             .annotate(
                 eligibility_diagnoses_count=Count("eligibility_diagnoses", distinct=True),
@@ -327,6 +326,7 @@ def populate_job_applications(self):
             .only(
                 "pk",
                 "created_at",
+                "processed_at",
                 "hiring_start_at",
                 "origin",
                 "sender_kind",

itou/metabase/management/commands/populate_metabase_matomo.py

@@ -13,7 +13,7 @@
 from psycopg import sql
 from sentry_sdk.crons import monitor
 
-from itou.metabase.db import DB_CURSOR, create_table, get_cursor
+from itou.metabase.db import MetabaseDatabaseCursor, create_table
 from itou.utils import constants
 from itou.utils.command import BaseCommand
 
@@ -101,7 +101,7 @@ def update_table_at_date(table_name, column_names, at, rows):
         table_name,
         [(col_name, "varchar") for col_name in column_names],
     )
-    with get_cursor(DB_CURSOR.C2) as (cursor, conn):
+    with MetabaseDatabaseCursor() as (cursor, conn):
         cursor.execute(
             sql.SQL("""DELETE FROM {table_name} WHERE "Date" = {value}""").format(
                 table_name=sql.Identifier(table_name),