fix: add linting command to db package (#392)

oaknational · Nov 27, 2024 · d2177d5 · d2177d5
1 parent 4cb2e45
commit d2177d5
Show file tree

Hide file tree

Showing 6 changed files with 67 additions and 29 deletions.
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -32,6 +32,9 @@ jobs:
       - name: Install deps (with cache)
         run: pnpm install
 
+      - name: Prisma generate (no engine)
+        run: pnpm turbo db-generate:no-engine --cache-dir=".turbo"
+
       - name: Build, lint and type-check
         run: pnpm turbo lint type-check --cache-dir=".turbo"
 

diff --git a/packages/db/package.json b/packages/db/package.json
@@ -50,11 +50,13 @@
     "docker-psql": " docker exec -it oak-ai-beta psql -U oai",
     "docker-reset": "pnpm run docker-clear && pnpm run docker-run",
     "docker-run": "docker run --name oak-ai-beta -p 8432:5432 -d oai-pg",
+    "lint": "eslint . --ext .ts,.tsx",
     "schema-format": "pnpm with-env prisma format",
     "script-add-metadata-to-lesson-summaries": "ts-node --compiler-options {\\\"module\\\":\\\"CommonJS\\\"} ./scripts/processing/add_metadata_to_lesson_summaries.ts",
     "script-batch-process-clean-questions": "ts-node --compiler-options {\\\"module\\\":\\\"CommonJS\\\"} ./scripts/processing/batch_process_clean_questions.ts",
     "script-batch-update-clerk-users": "ts-node --compiler-options {\\\"module\\\":\\\"CommonJS\\\"} ./scripts/processing/update_clerk_users.ts",
     "validate-data": "ts-node --compiler-options {\\\"module\\\":\\\"CommonJS\\\"} ./scripts/import-export/validate_data.ts",
+    "type-check": "tsc --noEmit",
     "with-env": "dotenv -e ../../.env --"
   },
   "prettier": "@oakai/prettier-config",

diff --git a/packages/db/scripts/import-export/validate_data.ts b/packages/db/scripts/import-export/validate_data.ts
@@ -3,6 +3,7 @@
  * This script validates the data in the CSV files against the constraints defined in the Prisma schema.
  *
  **/
+import { aiLogger } from "@oakai/logger";
 import { Prisma } from "@prisma/client";
 import csvParser from "csv-parser";
 import dotenv from "dotenv";
@@ -11,18 +12,21 @@ import * as path from "path";
 
 import { prisma } from "../..";
 
+const logger = aiLogger("db");
+
 const dataDir = path.join(__dirname, "data");
 
 dotenv.config();
 
 // Helper function to log messages
 const log = (message: string) => {
-  console.log(`[LOG] ${new Date().toISOString()}: ${message}`);
+  logger.info(`[LOG] ${new Date().toISOString()}: ${message}`);
 };
 
 // Helper function to get the Prisma model metadata
-async function getModelConstraints() {
+function getModelConstraints() {
   log("Inferring model constraints from Prisma schema...");
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
   const modelConstraints: Record<string, any> = {};
 
   const models = Prisma.dmmf.datamodel.models;
@@ -84,14 +88,17 @@ const validateCSV = (
 ) => {
   return new Promise<void>((resolve, reject) => {
     const errors: string[] = [];
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
     const foreignKeyCheck: Record<string, Set<any>> = {};
 
     log(`Validating CSV file: ${filePath}`);
     fs.createReadStream(filePath)
       .pipe(csvParser())
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
       .on("data", (row: any) => {
         // Check non-nullable fields
         nonNullable.forEach((col) => {
+          // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
           if (!row[col]) {
             errors.push(
               `NULL value found in non-nullable column '${col}' in row ${JSON.stringify(
@@ -101,21 +108,24 @@ const validateCSV = (
           }
         });
 
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
         for (const [fk, refTable] of Object.entries(foreignKeys)) {
           if (!foreignKeyCheck[fk]) {
             foreignKeyCheck[fk] = new Set();
           }
+          // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
           if (row[fk]) {
+            // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
             foreignKeyCheck[fk]!.add(row[fk]);
           }
         }
       })
       .on("end", async () => {
         log(`Completed reading CSV file: ${filePath}`);
 
-        // Validate foriegn keys in CSV
+        // Validate foreign keys in CSV
         for (const [fk, refTable] of Object.entries(foreignKeys)) {
-          const ids: string[] = Array.from(foreignKeyCheck[fk] || []);
+          const ids: string[] = Array.from(foreignKeyCheck[fk] ?? []);
 
           function handleTable(table: string) {
             if (table === "key_stage") {
@@ -136,6 +146,7 @@ const validateCSV = (
               const refIds = new Set<string>();
               fs.createReadStream(refFilePath)
                 .pipe(csvParser())
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
                 .on("data", (row: any) => {
                   refIds.add(row.id);
                 })
@@ -168,12 +179,12 @@ const validateCSV = (
 
 const main = async () => {
   try {
-    const modelConstraints = await getModelConstraints();
+    const modelConstraints = getModelConstraints();
 
     for (const [table, constraints] of Object.entries(modelConstraints)) {
       const filePath = path.join(dataDir, `${table}.csv`);
       if (!fs.existsSync(filePath)) {
-        console.error(`CSV file for table '${table}' does not exist.`);
+        logger.error(`CSV file for table '${table}' does not exist.`);
         process.exit(1);
       }
 
@@ -185,17 +196,17 @@ const main = async () => {
         );
         log(`Validation passed for table '${table}'`);
       } catch (errors) {
-        console.error(`Validation failed for table '${table}':\n`, errors);
+        logger.error(`Validation failed for table '${table}':\n`, errors);
         process.exit(1);
       }
     }
 
     log("All tables validated successfully.");
   } catch (e) {
-    console.error(e);
+    logger.error(e);
     process.exit(1);
   } finally {
-    console.log("Done");
+    logger.info("Done");
     await prisma.$disconnect();
   }
 };
@@ -205,7 +216,7 @@ main()
     await prisma.$disconnect();
   })
   .catch(async (e) => {
-    console.error(e);
+    logger.error(e);
     await prisma.$disconnect();
     process.exit(1);
   });

diff --git a/packages/db/scripts/processing/batch_process_clean_questions.ts b/packages/db/scripts/processing/batch_process_clean_questions.ts
@@ -1,6 +1,11 @@
+import { aiLogger } from "@oakai/logger";
+
 import { prisma } from "../../";
 
-async function getQuestionsAndWriteBatch(skip: number) {
+const logger = aiLogger("db");
+
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+async function getQuestionsAndWriteBatch(_skip: number) {
   const questions = await prisma.quizQuestion.findMany({
     include: {
       answers: {
@@ -32,7 +37,7 @@ async function getQuestionsAndWriteBatch(skip: number) {
       },
     },
   });
-  console.log("questions", questions);
+  logger.info("questions", questions);
   await prisma.questionsForJudgement.createMany({
     data: questions
       .filter(
@@ -68,14 +73,14 @@ async function removeQuestionsContainingWords(string: string) {
   const questionsToRemove = questions.filter((question) =>
     question?.quizQuestion?.question?.includes(string),
   );
-  prisma.questionsForJudgement.deleteMany({
+  await prisma.questionsForJudgement.deleteMany({
     where: {
       id: {
         in: questionsToRemove.map((question) => question.id),
       },
     },
   });
-  console.log("Removed number of questions", questionsToRemove.length);
+  logger.info("Removed number of questions", questionsToRemove.length);
 }
 
 const main = async () => {
@@ -88,10 +93,10 @@ const main = async () => {
     await getQuestionsAndWriteBatch(0);
     await removeQuestionsContainingWords("**not**");
   } catch (e) {
-    console.error(e);
+    logger.error(e);
     process.exit(1);
   } finally {
-    console.log("Done");
+    logger.info("Done");
     await prisma.$disconnect();
   }
 };
@@ -101,7 +106,7 @@ main()
     await prisma.$disconnect();
   })
   .catch(async (e) => {
-    console.error(e);
+    logger.error(e);
     await prisma.$disconnect();
     process.exit(1);
   });

diff --git a/packages/db/scripts/processing/update_clerk_users.ts b/packages/db/scripts/processing/update_clerk_users.ts
@@ -1,5 +1,23 @@
+import { aiLogger } from "@oakai/logger";
+import { z } from "zod";
+
 import { prisma } from "../../";
 
+const logger = aiLogger("db");
+
+const ClerkUserSchema = z.object({
+  id: z.string(),
+  created_at: z.string(),
+  updated_at: z.string(),
+  email_addresses: z.array(
+    z.object({
+      id: z.string(),
+      email_address: z.string().email(),
+    }),
+  ),
+  primary_email_address_id: z.string(),
+});
+
 /**
  * This script deletes all records from the `users` table, then re-imports them all
  * from clerk
@@ -20,9 +38,12 @@ async function getUsersBatch({
       },
     },
   );
-  const data = await res.json();
 
-  return data.map((user: any) => {
+  const ClerkResponseSchema = z.array(ClerkUserSchema);
+
+  const validatedData = ClerkResponseSchema.parse(await res.json());
+
+  return validatedData.map((user) => {
     const email: string = user.email_addresses.find(
       (email: { id: string }) => email.id === user.primary_email_address_id,
     )!.email_address;
@@ -46,11 +67,8 @@ async function updateAllUsers() {
   let offset = 0;
   let resultsCount = 0;
 
-  let i = 0;
-
+  // eslint-disable-next-line no-constant-condition
   while (true) {
-    i++;
-
     const data = await getUsersBatch({
       limit,
       offset,
@@ -60,7 +78,7 @@ async function updateAllUsers() {
       break;
     }
 
-    console.log(`Processing batch of ${data.length} users`);
+    logger.info(`Processing batch of ${data.length} users`);
 
     await prisma.user.createMany({
       data,
@@ -70,18 +88,18 @@ async function updateAllUsers() {
     offset += limit;
   }
 
-  console.log(`Created/updated ${resultsCount} user records`);
+  logger.info(`Created/updated ${resultsCount} user records`);
 }
 
 const main = async () => {
   try {
     await prisma.user.deleteMany();
     await updateAllUsers();
   } catch (e) {
-    console.error(e);
+    logger.error(e);
     process.exit(1);
   } finally {
-    console.log("Done");
+    logger.info("Done");
     await prisma.$disconnect();
   }
 };
@@ -91,7 +109,7 @@ main()
     await prisma.$disconnect();
   })
   .catch(async (e) => {
-    console.error(e);
+    logger.error(e);
     await prisma.$disconnect();
     process.exit(1);
   });

diff --git a/turbo.json b/turbo.json
@@ -67,7 +67,6 @@
       "cache": false
     },
     "type-check": {
-      "dependsOn": ["^db-generate"],
       "cache": false
     },
     "test": {