Add contact summary as a field to the training set

techmatters · Nov 4, 2024 · 1848b92 · 1848b92
1 parent 5970c60
commit 1848b92
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 4 deletions.
diff --git a/hrm-domain/scheduled-tasks/generate-ai-training-set/hrmdbAccess.ts b/hrm-domain/scheduled-tasks/generate-ai-training-set/hrmdbAccess.ts
@@ -21,10 +21,11 @@ import { db, pgp } from '@tech-matters/hrm-core/connection-pool';
 
 const HIGH_WATER_MARK = 1000;
 
-const SELECT_CATEGORIES_AND_TRANSCRIPTS_SQL = `
+const SELECT_CATEGORIES_SUMMARY_AND_TRANSCRIPTS_SQL = `
   SELECT
     c."id" AS "contactId",
     c."rawJson"->'categories' AS "categories",
+    c."rawJson"->'caseInformation'->>'callSummary' AS "summary",
     cm."storeTypeSpecificData",
     cm."storeTypeSpecificData"->'location'->>'bucket' AS "transcriptBucket",
     cm."storeTypeSpecificData"->'location'->>'key' AS "transcriptKey"
@@ -33,6 +34,7 @@ const SELECT_CATEGORIES_AND_TRANSCRIPTS_SQL = `
   WHERE 
   c."accountSid" = $<accountSid> AND 
   (SELECT COUNT(*) FROM jsonb_object_keys(COALESCE(c."rawJson"->'categories', '{}'::jsonb))) > 0 AND 
+  COALESCE(c."rawJson"->'caseInformation'->>'callSummary', '') <> '' AND
   cm."storeType" = 'S3' AND 
   cm."storeTypeSpecificData"->>'type' = 'transcript' AND
   cm."storeTypeSpecificData"->>'location' IS NOT NULL
@@ -42,6 +44,7 @@ export type TrainingSetContact = {
   contactId: string;
   accountSid: HrmAccountId;
   categories: Record<string, string[]>;
+  summary: string;
   transcriptKey: string;
   transcriptBucket: string;
 };
@@ -50,7 +53,7 @@ export const streamTrainingSetContacts = async (
   accountSid: HrmAccountId,
 ): Promise<ReadableStream> => {
   const qs = new QueryStream(
-    pgp.as.format(SELECT_CATEGORIES_AND_TRANSCRIPTS_SQL, { accountSid }),
+    pgp.as.format(SELECT_CATEGORIES_SUMMARY_AND_TRANSCRIPTS_SQL, { accountSid }),
     [],
     { highWaterMark: HIGH_WATER_MARK },
   );

diff --git a/hrm-domain/scheduled-tasks/generate-ai-training-set/index.ts b/hrm-domain/scheduled-tasks/generate-ai-training-set/index.ts
@@ -69,7 +69,7 @@ export const generate = async (
   });
 
   for (const { accountSid, shortCode } of accountSidMappings) {
-    // Query the DB for contacts and start streaming records with their ID, categories and transcript location
+    // Query the DB for contacts and start streaming records with their ID, categories, contact summary and transcript location
     const contactStream = await streamTrainingSetContacts(accountSid);
     console.log(`Streaming contacts for ${shortCode}...`);
 

diff --git a/hrm-domain/scheduled-tasks/generate-ai-training-set/trainingSetDocument.ts b/hrm-domain/scheduled-tasks/generate-ai-training-set/trainingSetDocument.ts
@@ -21,15 +21,17 @@ import { getS3Object } from '@tech-matters/s3-client';
 export type TrainingSetDocument = {
   contactId: string;
   categories: Record<string, string[]>;
+  summary: string;
   messages: ExportTranscript['messages'];
 };
 
 const trainingSetDocument = (
-  { contactId, categories }: TrainingSetContact,
+  { contactId, categories, summary }: TrainingSetContact,
   { messages }: ExportTranscript,
 ): TrainingSetDocument => ({
   contactId,
   categories,
+  summary,
   messages,
 });