Read csv config from file (#10)

Signed-off-by: Sean Sundberg <[email protected]>

config/KYCDataValidationQuestions.csv

@@ -0,0 +1,21 @@
+ID,Question,PoCScope,Company,Prompt,Expected Answer,watsonx Response
+1,What is Name and trading name of the organization?,,,,,
+2,What is the registered address of the company?,X,BP P.L.C,from below text find the registered address of the company #?,"1 St James's Square, London, SW1Y 4PD","1 St James's Square, London, SW1Y 4PD"
+3,What is the business/trading address of the company?,,,,,
+4,What is identification number of the organization?,X,BP P.L.C,from below text find  identification number of the organization #? ,102498,102498
+5,Who are the key controllers and authorized signatories?,,,,,
+6,Names all the active directors of the company.,X,BP P.L.C,"from below text, find the names of active directors of the company # in sequence ?","LUND, Helge BLANC, Amanda Jayne DALEY, Pamela","LUND, Helge BLANC, Amanda Jayne DALEY, Pamela"
+7,"What is the status of the organization ex; active, dissolved?",X,BP P.L.C,"from below text, what is the status of the organization # ?",Active,Active
+8,What is the year of incorporation?,X,BP P.L.C,"from below text, What is the year of incorporation of #?",1909,1909
+9,Who are the shareholders of the company along with the percentage of ownership?,,,,,
+10,Who is the ultimate owner of the company?,,,,,
+11,Who are the key controllers and authorized signatories?,,,,,
+12,What is the industry type/SIC/NICS code of the company?,,,,,
+13,What are the products utilized by the company?,,,,,
+14,What is/are operation location/s or jurisdiction/s?,,,,,
+15,Number of employees of the firm,,,,,
+16,Name of the subsidiary of the company,,,,,
+17,What is the Legal entity Type of the organization ex; publicly traded/limited liability etc.,,,,,
+18,What is the turnover or revenue of the organization?,,,,,
+19,Certificate/licence issued by the government.,,,,,
+20,Whats is the next date of confirmation statement?,X,BP P.L.C,"from below text, find the next date of confirmation statement for company #?",30/06/24,30/06/24

src/services/data-extraction/data-extraction.csv.ts

@@ -1,56 +1,79 @@
 import {BehaviorSubject, Observable} from "rxjs";
+import {promises} from 'fs';
+import {join, resolve} from 'path';
 
 import {DataExtractionApi} from "./data-extraction.api";
 import {DataExtractionQuestionModel, DataExtractionResultModel} from "../../models";
 import {first, parseCsv} from "../../utils";
 
-const csvFile: string = `ID,Question,PoCScope,Expected Answer,watsonx Response
-1,What is Name and trading name of the organization?,,,
-2,What is the registered address of the company?,X,"1 St James's Square, London, SW1Y 4PD",
-3,What is the business/trading address of the company?,,,
-4,What is identification number of the organization?,X,102498,
-5,Who are the key controllers and authorized signatories?,,,
-6,Names all the active directors of the company.,X,"LUND, Helge BLANC, Amanda Jayne DALEY, Pamela",
-7,"What is the status of the organization ex; active, dissolved?",X,Active,
-8,What is the year of incorporation?,X,1909,
-9,Who are the shareholders of the company along with the percentage of ownership?,,,
-10,Who is the ultimate owner of the company?,,,
-11,What is the nature of business of the company?,,,
-12,What is the industry type/SIC/NICS code of the company?,,,
-13,What are the products utilized by the company?,,,
-14,What is/are operation location/s or jurisdiction/s?,,,
-15,Number of employees of the firm,,,
-16,Name of the subsidiary of the company,,,
-17,What is the Legal entity Type of the organization ex; publicly traded/limited liability etc.,,,
-18,What is the turnover or revenue of the organization?,,,
-19,Certificate/licence issued by the government.,,,
-20,Whats is the next date of confirmation statement?,X,30/06/24,`
+const csvFile: string = `ID,Question,PoCScope,Company,Prompt,Expected Answer,watsonx Response
+1,What is Name and trading name of the organization?,,,,,
+2,What is the registered address of the company?,X,BP P.L.C,from below text find the registered address of the company #?,"1 St James's Square, London, SW1Y 4PD","1 St James's Square, London, SW1Y 4PD"
+3,What is the business/trading address of the company?,,,,,
+4,What is identification number of the organization?,X,BP P.L.C,from below text find  identification number of the organization #? ,102498,102498
+5,Who are the key controllers and authorized signatories?,,,,,
+6,Names all the active directors of the company.,X,BP P.L.C,"from below text, find the names of active directors of the company # in sequence ?","LUND, Helge BLANC, Amanda Jayne DALEY, Pamela","LUND, Helge BLANC, Amanda Jayne DALEY, Pamela"
+7,"What is the status of the organization ex; active, dissolved?",X,BP P.L.C,"from below text, what is the status of the organization # ?",Active,Active
+8,What is the year of incorporation?,X,BP P.L.C,"from below text, What is the year of incorporation of #?",1909,1909
+9,Who are the shareholders of the company along with the percentage of ownership?,,,,,
+10,Who is the ultimate owner of the company?,,,,,
+11,Who are the key controllers and authorized signatories?,,,,,
+12,What is the industry type/SIC/NICS code of the company?,,,,,
+13,What are the products utilized by the company?,,,,,
+14,What is/are operation location/s or jurisdiction/s?,,,,,
+15,Number of employees of the firm,,,,,
+16,Name of the subsidiary of the company,,,,,
+17,What is the Legal entity Type of the organization ex; publicly traded/limited liability etc.,,,,,
+18,What is the turnover or revenue of the organization?,,,,,
+19,Certificate/licence issued by the government.,,,,,
+20,Whats is the next date of confirmation statement?,X,BP P.L.C,"from below text, find the next date of confirmation statement for company #?",30/06/24,30/06/24`
 
 export interface DataExtractionConfig extends DataExtractionQuestionModel {
     expectedResponse: string;
     prompt: string;
 }
 
-const data: DataExtractionConfig[] = csvFile
-    .split('\n')
-    .map(parseCsv)
-    .map(values => ({
-        id: '' + values[0],
-        question: '' + values[1],
-        prompt: '',
-        inScope: values[2] === 'X',
-        expectedResponse: '' + values[3]
-    }))
-    .filter(val => val.id !== 'ID')
+let data: Promise<DataExtractionConfig[]>;
 
 export abstract class DataExtractionCsv<A> extends DataExtractionApi {
 
-    getCsvData(): DataExtractionConfig[] {
-        return data;
+    async getCsvData(): Promise<DataExtractionConfig[]> {
+        if (data) {
+            return data
+        }
+
+        const curPath = resolve(__dirname)
+
+        return data = new Promise<string>(
+            resolve => {
+                const filepath = join(curPath, '../../../..', 'config/KYCDataValidationQuestions.csv')
+
+                promises.readFile(filepath)
+                    .then(buf => {
+                        resolve(buf.toString())
+                    })
+                    .catch(err => {
+                        resolve(csvFile);
+                    });
+            })
+            .then((fileContent: string) => {
+                return fileContent
+                    .split('\n')
+                    .map(parseCsv)
+                    .map(values => ({
+                        id: '' + values[0],
+                        question: '' + values[1],
+                        inScope: values[2] === 'X',
+                        prompt: values[4],
+                        expectedResponse: '' + values[5]
+                    }))
+                    .filter(val => val.id !== 'ID');
+            })
+
     }
 
     async listQuestions(): Promise<DataExtractionQuestionModel[]> {
-        return this.getCsvData()
+        return (await this.getCsvData())
             .map(val => ({id: val.id, question: val.question, inScope: val.inScope}));
     }
 
@@ -64,16 +87,18 @@ export abstract class DataExtractionCsv<A> extends DataExtractionApi {
         return Promise.all(questions.map(extractDataForQuestion.bind(this)))
     }
 
-    emptyDataExtractionResults(questions: Array<{id: string}>): DataExtractionResultModel[] {
+    async emptyDataExtractionResults(questions: Array<{id: string}>): Promise<DataExtractionResultModel[]> {
         const ids = questions.map(q => q.id);
 
-        return this.getCsvData()
+        return (await this.getCsvData())
             .filter(val => ids.includes(val.id))
             .map(val => Object.assign({}, val, {watsonxResponse: ''}))
     }
 
     extractDataObservable(customer: string, questions: Array<{id: string}>): Observable<DataExtractionResultModel[]> {
-        const subject: BehaviorSubject<DataExtractionResultModel[]> = new BehaviorSubject(this.emptyDataExtractionResults(questions));
+        const subject: BehaviorSubject<DataExtractionResultModel[]> = new BehaviorSubject([]);
+
+        this.emptyDataExtractionResults(questions).then(result => subject.next(result))
 
         this.getBackends().then((auth: A) => {
             questions

src/services/data-extraction/data-extraction.impl.ts

@@ -69,7 +69,7 @@ export class DataExtractionImpl extends DataExtractionCsv<WatsonBackends> implem
     }
 
     async extractDataForQuestionInternal(customer: string, question: {id: string}, backends: WatsonBackends): Promise<DataExtractionResultModel> {
-        const config = first(this.getCsvData().filter(val => val.id === question.id))
+        const config = first((await this.getCsvData()).filter(val => val.id === question.id))
 
         if (!config) {
             throw new Error('Unable to find question: ' + question.id)

src/services/data-extraction/data-extraction.mock.ts

@@ -7,8 +7,10 @@ import {delay, first} from "../../utils";
 export class DataExtractionMock extends DataExtractionCsv<{}> implements DataExtractionApi {
 
     async extractDataForQuestionInternal(_: string, question: {id: string}, auth: {}): Promise<DataExtractionResultModel> {
+        const data = await this.getCsvData();
+
         return delay(500, () => {
-            const config: DataExtractionConfig | undefined = first(this.getCsvData().filter(val => val.id === question.id))
+            const config: DataExtractionConfig | undefined = first(data.filter(val => val.id === question.id))
 
             if (!config) {
                 throw new Error('Error finding question: ' + question.id)