#2035 - script de migration des subs inclusion connect -> pro connect

back/package.json

@@ -45,6 +45,7 @@
     "trigger-update-establishments-from-sirene": "ts-node src/scripts/triggerUpdateEstablishmentsFromSireneApiScript.ts",
     "trigger-update-all-establishments-scores": "ts-node src/scripts/triggerUpdateAllEstablishmentsScores.ts",
     "trigger-update-rome-data": "ts-node src/scripts/triggerUpdateRomeData.ts",
+    "trigger-import-pro-connect-external-ids": "ts-node src/scripts/triggerImportProConnectExternalIds",
     "typecheck": "tsc --noEmit",
     "kysely-codegen": "kysely-codegen --dialect postgres",
     "update-agencies-from-PE-referential": "ts-node src/scripts/updateAllPEAgenciesFromPeAgencyReferential.ts",
@@ -101,6 +102,7 @@
     "@types/jsonwebtoken": "^9.0.2",
     "@types/multer": "^1.4.7",
     "@types/node": "^20.15.0",
+    "@types/papaparse": "^5.3.7",
     "@types/pg": "^8.10.2",
     "@types/pg-format": "^1.0.2",
     "@types/pino": "^7.0.4",
@@ -115,6 +117,7 @@
     "jest": "^29.5.0",
     "libphonenumber-js": "^1.11.1",
     "openapi-types": "^12.1.3",
+    "papaparse": "^5.3.2",
     "supertest": "^6.2.2",
     "ts-node-dev": "^2.0.0",
     "ts-prune": "^0.10.3",

back/src/config/pg/kysely/kyselyUtils.ts

@@ -83,3 +83,32 @@ export type KyselyDb = Kysely<Database>;
 
 export const falsyToNull = <T>(value: T | Falsy): T | null =>
   value ? value : null;
+
+//https://github.com/kysely-org/kysely/issues/839
+//https://old.kyse.link/?p=s&i=C0yoagEodj9vv4AxE3TH
+export function values<R extends Record<string, unknown>, A extends string>(
+  records: R[],
+  alias: A,
+) {
+  // Assume there's at least one record and all records
+  // have the same keys.
+  const keys = Object.keys(records[0]);
+
+  // Transform the records into a list of lists such as
+  // ($1, $2, $3), ($4, $5, $6)
+  const values = sql.join(
+    records.map((r) => sql`(${sql.join(keys.map((k) => r[k]))})`),
+  );
+
+  // Create the alias `v(id, v1, v2)` that specifies the table alias
+  // AND a name for each column.
+  const wrappedAlias = sql.ref(alias);
+  const wrappedColumns = sql.join(keys.map(sql.ref));
+  const aliasSql = sql`${wrappedAlias}(${wrappedColumns})`;
+
+  // Finally create a single `AliasedRawBuilder` instance of the
+  // whole thing. Note that we need to explicitly specify
+  // the alias type using `.as<A>` because we are using a
+  // raw sql snippet as the alias.
+  return sql<R>`(values ${values})`.as<A>(aliasSql);
+}

back/src/scripts/triggerImportProConnectExternalIds.ts

@@ -0,0 +1,134 @@
+import { resolve } from "node:path";
+import { readFile } from "fs/promises";
+import Papa from "papaparse";
+import { z } from "zod";
+import { AppConfig } from "../config/bootstrap/appConfig";
+import { createGetPgPoolFn } from "../config/bootstrap/createGateways";
+import { makeKyselyDb, values } from "../config/pg/kysely/kyselyUtils";
+import { createLogger } from "../utils/logger";
+import { handleCRONScript } from "./handleCRONScript";
+
+const logger = createLogger(__filename);
+
+const config = AppConfig.createFromEnv();
+
+const executeUsecase = async (): Promise<{
+  inCSV: number;
+  updated: number;
+  withoutProConnectIdBeforeUpdate: number;
+  withoutProConnectIdAfterUpdate?: {
+    id: string;
+    email: string;
+    inclusion_connect_sub: string | null;
+  }[];
+}> => {
+  const filename = "import_CSV_proconnect.csv"; // CSV file to save in immersion-facile root project folder
+  const path = `../${filename}`;
+  const rawFile = await readFile(resolve(path), { encoding: "utf8" });
+
+  logger.info({ message: `START - Parsing CSV on path ${path}.` });
+  const csv = Papa.parse(rawFile, {
+    header: true,
+    skipEmptyLines: true,
+  });
+  logger.info({ message: `DONE - Parsing CSV on path ${path}.` });
+
+  const csvSchema: z.Schema<
+    {
+      inclusionConnectSub: string;
+      proConnectSub: string;
+    }[]
+  > = z.array(
+    z.object({
+      inclusionConnectSub: z.string().uuid(),
+      proConnectSub: z.string().uuid(),
+    }),
+  );
+
+  const csvData = csv.data;
+  logger.info({ message: `START - Schema parse CSV data : ${csvData.length}` });
+  const csvValues = csvSchema.parse(csvData);
+  logger.info({ message: `DONE - Schema parsed values : ${csvValues.length}` });
+
+  const pool = createGetPgPoolFn(config)();
+  const db = makeKyselyDb(pool);
+
+  const getUserToUpdateQueryBuilder = db
+    .selectFrom("users")
+    .select(["id", "email", "inclusion_connect_sub"])
+    .where("inclusion_connect_sub", "is not", null)
+    .where("pro_connect_sub", "is", null);
+
+  logger.info({ message: "START - Get users without ProConnect sub" });
+  const withoutProConnectIdBeforeUpdate =
+    await getUserToUpdateQueryBuilder.execute();
+  logger.info({
+    message: `DONE - users without ProConnect sub : ${withoutProConnectIdBeforeUpdate.length}`,
+  });
+
+  if (csvValues.length === 0)
+    return {
+      inCSV: csvValues.length,
+      updated: 0,
+      withoutProConnectIdBeforeUpdate: withoutProConnectIdBeforeUpdate.length,
+    };
+
+  logger.info({ message: "START - Update users" });
+  const updatedUserIds = await db
+    .updateTable("users")
+    .from(values(csvValues, "mapping"))
+    .set((eb) => ({
+      pro_connect_sub: eb.ref("mapping.proConnectSub"),
+    }))
+    .whereRef("mapping.inclusionConnectSub", "=", "inclusion_connect_sub")
+    .returning("id")
+    .execute();
+  logger.info({ message: `DONE - ${updatedUserIds.length} Updated users` });
+
+  logger.info({ message: "START - Get users without ProConnect sub" });
+  const userWithIcExternalAndWithoutPcExternalId =
+    await getUserToUpdateQueryBuilder.execute();
+  logger.info({
+    message: `DONE - users without ProConnect sub : ${userWithIcExternalAndWithoutPcExternalId.length}`,
+  });
+
+  return {
+    inCSV: csvValues.length,
+    withoutProConnectIdBeforeUpdate: withoutProConnectIdBeforeUpdate.length,
+    updated: updatedUserIds.length,
+    withoutProConnectIdAfterUpdate: userWithIcExternalAndWithoutPcExternalId,
+  };
+};
+
+/* eslint-disable @typescript-eslint/no-floating-promises */
+handleCRONScript(
+  "importProConnectExternalIds",
+  config,
+  executeUsecase,
+  ({
+    inCSV,
+    updated,
+    withoutProConnectIdBeforeUpdate,
+    withoutProConnectIdAfterUpdate,
+  }) => {
+    return [
+      `Number of users without Pro Connect external Ids to update : ${withoutProConnectIdBeforeUpdate}`,
+      `Number of external Ids mapping in CSV : ${inCSV}`,
+      `Number of users updated with Pro Connect external Ids: ${updated}`,
+      ...(withoutProConnectIdAfterUpdate
+        ? [
+            `Number of users that still not have Pro Connect external Id details : ${withoutProConnectIdAfterUpdate.length}`,
+            `Details : 
+            ${[
+              "   USER ID       |     EMAIL      |       IC_EXTERNAL_ID",
+              ...withoutProConnectIdAfterUpdate.map(
+                ({ id, email, inclusion_connect_sub }) =>
+                  `- ${id} - ${email} - ${inclusion_connect_sub}`,
+              ),
+            ].join("\n    ")}`,
+          ]
+        : []),
+    ].join("\n");
+  },
+  logger,
+);