diff --git a/back/package.json b/back/package.json index a4af1971a7..9bf49fef2b 100644 --- a/back/package.json +++ b/back/package.json @@ -45,6 +45,7 @@ "trigger-update-establishments-from-sirene": "ts-node src/scripts/triggerUpdateEstablishmentsFromSireneApiScript.ts", "trigger-update-all-establishments-scores": "ts-node src/scripts/triggerUpdateAllEstablishmentsScores.ts", "trigger-update-rome-data": "ts-node src/scripts/triggerUpdateRomeData.ts", + "trigger-import-pro-connect-external-ids": "ts-node src/scripts/triggerImportProConnectExternalIds", "typecheck": "tsc --noEmit", "kysely-codegen": "kysely-codegen --dialect postgres", "update-agencies-from-PE-referential": "ts-node src/scripts/updateAllPEAgenciesFromPeAgencyReferential.ts", @@ -101,6 +102,7 @@ "@types/jsonwebtoken": "^9.0.2", "@types/multer": "^1.4.7", "@types/node": "^20.15.0", + "@types/papaparse": "^5.3.7", "@types/pg": "^8.10.2", "@types/pg-format": "^1.0.2", "@types/pino": "^7.0.4", @@ -115,6 +117,7 @@ "jest": "^29.5.0", "libphonenumber-js": "^1.11.1", "openapi-types": "^12.1.3", + "papaparse": "^5.3.2", "supertest": "^6.2.2", "ts-node-dev": "^2.0.0", "ts-prune": "^0.10.3", diff --git a/back/src/config/pg/kysely/kyselyUtils.ts b/back/src/config/pg/kysely/kyselyUtils.ts index 38ff2bcdd4..e6389a6518 100644 --- a/back/src/config/pg/kysely/kyselyUtils.ts +++ b/back/src/config/pg/kysely/kyselyUtils.ts @@ -83,3 +83,32 @@ export type KyselyDb = Kysely; export const falsyToNull = (value: T | Falsy): T | null => value ? value : null; + +//https://github.com/kysely-org/kysely/issues/839 +//https://old.kyse.link/?p=s&i=C0yoagEodj9vv4AxE3TH +export function values, A extends string>( + records: R[], + alias: A, +) { + // Assume there's at least one record and all records + // have the same keys. + const keys = Object.keys(records[0]); + + // Transform the records into a list of lists such as + // ($1, $2, $3), ($4, $5, $6) + const values = sql.join( + records.map((r) => sql`(${sql.join(keys.map((k) => r[k]))})`), + ); + + // Create the alias `v(id, v1, v2)` that specifies the table alias + // AND a name for each column. + const wrappedAlias = sql.ref(alias); + const wrappedColumns = sql.join(keys.map(sql.ref)); + const aliasSql = sql`${wrappedAlias}(${wrappedColumns})`; + + // Finally create a single `AliasedRawBuilder` instance of the + // whole thing. Note that we need to explicitly specify + // the alias type using `.as` because we are using a + // raw sql snippet as the alias. + return sql`(values ${values})`.as(aliasSql); +} diff --git a/back/src/scripts/triggerImportProConnectExternalIds.ts b/back/src/scripts/triggerImportProConnectExternalIds.ts new file mode 100644 index 0000000000..1177fcd735 --- /dev/null +++ b/back/src/scripts/triggerImportProConnectExternalIds.ts @@ -0,0 +1,134 @@ +import { resolve } from "node:path"; +import { readFile } from "fs/promises"; +import Papa from "papaparse"; +import { z } from "zod"; +import { AppConfig } from "../config/bootstrap/appConfig"; +import { createGetPgPoolFn } from "../config/bootstrap/createGateways"; +import { makeKyselyDb, values } from "../config/pg/kysely/kyselyUtils"; +import { createLogger } from "../utils/logger"; +import { handleCRONScript } from "./handleCRONScript"; + +const logger = createLogger(__filename); + +const config = AppConfig.createFromEnv(); + +const executeUsecase = async (): Promise<{ + inCSV: number; + updated: number; + withoutProConnectIdBeforeUpdate: number; + withoutProConnectIdAfterUpdate?: { + id: string; + email: string; + inclusion_connect_sub: string | null; + }[]; +}> => { + const filename = "import_CSV_proconnect.csv"; // CSV file to save in immersion-facile root project folder + const path = `../${filename}`; + const rawFile = await readFile(resolve(path), { encoding: "utf8" }); + + logger.info({ message: `START - Parsing CSV on path ${path}.` }); + const csv = Papa.parse(rawFile, { + header: true, + skipEmptyLines: true, + }); + logger.info({ message: `DONE - Parsing CSV on path ${path}.` }); + + const csvSchema: z.Schema< + { + inclusionConnectSub: string; + proConnectSub: string; + }[] + > = z.array( + z.object({ + inclusionConnectSub: z.string().uuid(), + proConnectSub: z.string().uuid(), + }), + ); + + const csvData = csv.data; + logger.info({ message: `START - Schema parse CSV data : ${csvData.length}` }); + const csvValues = csvSchema.parse(csvData); + logger.info({ message: `DONE - Schema parsed values : ${csvValues.length}` }); + + const pool = createGetPgPoolFn(config)(); + const db = makeKyselyDb(pool); + + const getUserToUpdateQueryBuilder = db + .selectFrom("users") + .select(["id", "email", "inclusion_connect_sub"]) + .where("inclusion_connect_sub", "is not", null) + .where("pro_connect_sub", "is", null); + + logger.info({ message: "START - Get users without ProConnect sub" }); + const withoutProConnectIdBeforeUpdate = + await getUserToUpdateQueryBuilder.execute(); + logger.info({ + message: `DONE - users without ProConnect sub : ${withoutProConnectIdBeforeUpdate.length}`, + }); + + if (csvValues.length === 0) + return { + inCSV: csvValues.length, + updated: 0, + withoutProConnectIdBeforeUpdate: withoutProConnectIdBeforeUpdate.length, + }; + + logger.info({ message: "START - Update users" }); + const updatedUserIds = await db + .updateTable("users") + .from(values(csvValues, "mapping")) + .set((eb) => ({ + pro_connect_sub: eb.ref("mapping.proConnectSub"), + })) + .whereRef("mapping.inclusionConnectSub", "=", "inclusion_connect_sub") + .returning("id") + .execute(); + logger.info({ message: `DONE - ${updatedUserIds.length} Updated users` }); + + logger.info({ message: "START - Get users without ProConnect sub" }); + const userWithIcExternalAndWithoutPcExternalId = + await getUserToUpdateQueryBuilder.execute(); + logger.info({ + message: `DONE - users without ProConnect sub : ${userWithIcExternalAndWithoutPcExternalId.length}`, + }); + + return { + inCSV: csvValues.length, + withoutProConnectIdBeforeUpdate: withoutProConnectIdBeforeUpdate.length, + updated: updatedUserIds.length, + withoutProConnectIdAfterUpdate: userWithIcExternalAndWithoutPcExternalId, + }; +}; + +/* eslint-disable @typescript-eslint/no-floating-promises */ +handleCRONScript( + "importProConnectExternalIds", + config, + executeUsecase, + ({ + inCSV, + updated, + withoutProConnectIdBeforeUpdate, + withoutProConnectIdAfterUpdate, + }) => { + return [ + `Number of users without Pro Connect external Ids to update : ${withoutProConnectIdBeforeUpdate}`, + `Number of external Ids mapping in CSV : ${inCSV}`, + `Number of users updated with Pro Connect external Ids: ${updated}`, + ...(withoutProConnectIdAfterUpdate + ? [ + `Number of users that still not have Pro Connect external Id details : ${withoutProConnectIdAfterUpdate.length}`, + `Details : + ${[ + " USER ID | EMAIL | IC_EXTERNAL_ID", + ...withoutProConnectIdAfterUpdate.map( + ({ id, email, inclusion_connect_sub }) => + `- ${id} - ${email} - ${inclusion_connect_sub}`, + ), + ].join("\n ")}`, + ] + : []), + ].join("\n"); + }, + logger, +); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6bdc566e8e..011303bf73 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -195,6 +195,9 @@ importers: '@types/node': specifier: ^20.15.0 version: 20.15.0 + '@types/papaparse': + specifier: ^5.3.7 + version: 5.3.7 '@types/pg': specifier: ^8.10.2 version: 8.10.2 @@ -237,6 +240,9 @@ importers: openapi-types: specifier: ^12.1.3 version: 12.1.3 + papaparse: + specifier: ^5.3.2 + version: 5.3.2 supertest: specifier: ^6.2.2 version: 6.3.3 @@ -14606,6 +14612,7 @@ snapshots: '@types/node@22.7.9': dependencies: undici-types: 6.19.8 + optional: true '@types/normalize-package-data@2.4.1': {} @@ -14613,7 +14620,7 @@ snapshots: '@types/papaparse@5.3.7': dependencies: - '@types/node': 20.15.0 + '@types/node': 20.17.0 '@types/parse-json@4.0.0': {} @@ -17752,7 +17759,7 @@ snapshots: jest-worker@27.5.1: dependencies: - '@types/node': 22.7.9 + '@types/node': 20.17.0 merge-stream: 2.0.0 supports-color: 8.1.1