Added throttling of all VertexAI Calls so its faster.

lib/p-throttle.js

@@ -0,0 +1,114 @@
+export class AbortError extends Error {
+	constructor() {
+		super('Throttled function aborted');
+		this.name = 'AbortError';
+	}
+}
+
+export default function pThrottle({limit, interval, strict, onDelay}) {
+	if (!Number.isFinite(limit)) {
+		throw new TypeError('Expected `limit` to be a finite number');
+	}
+
+	if (!Number.isFinite(interval)) {
+		throw new TypeError('Expected `interval` to be a finite number');
+	}
+
+	const queue = new Map();
+
+	let currentTick = 0;
+	let activeCount = 0;
+
+	function windowedDelay() {
+		const now = Date.now();
+
+		if ((now - currentTick) > interval) {
+			activeCount = 1;
+			currentTick = now;
+			return 0;
+		}
+
+		if (activeCount < limit) {
+			activeCount++;
+		} else {
+			currentTick += interval;
+			activeCount = 1;
+		}
+
+		return currentTick - now;
+	}
+
+	const strictTicks = [];
+
+	function strictDelay() {
+		const now = Date.now();
+
+		// Clear the queue if there's a significant delay since the last execution
+		if (strictTicks.length > 0 && now - strictTicks.at(-1) > interval) {
+			strictTicks.length = 0;
+		}
+
+		// If the queue is not full, add the current time and execute immediately
+		if (strictTicks.length < limit) {
+			strictTicks.push(now);
+			return 0;
+		}
+
+		// Calculate the next execution time based on the first item in the queue
+		const nextExecutionTime = strictTicks[0] + interval;
+
+		// Shift the queue and add the new execution time
+		strictTicks.shift();
+		strictTicks.push(nextExecutionTime);
+
+		// Calculate the delay for the current execution
+		return Math.max(0, nextExecutionTime - now);
+	}
+
+	const getDelay = strict ? strictDelay : windowedDelay;
+
+	return function_ => {
+		const throttled = function (...arguments_) {
+			if (!throttled.isEnabled) {
+				return (async () => function_.apply(this, arguments_))();
+			}
+
+			let timeoutId;
+			return new Promise((resolve, reject) => {
+				const execute = () => {
+					resolve(function_.apply(this, arguments_));
+					queue.delete(timeoutId);
+				};
+
+				const delay = getDelay();
+				if (delay > 0) {
+					timeoutId = setTimeout(execute, delay);
+					queue.set(timeoutId, reject);
+					onDelay?.(...arguments_);
+				} else {
+					execute();
+				}
+			});
+		};
+
+		throttled.abort = () => {
+			for (const timeout of queue.keys()) {
+				clearTimeout(timeout);
+				queue.get(timeout)(new AbortError());
+			}
+
+			queue.clear();
+			strictTicks.splice(0, strictTicks.length);
+		};
+
+		throttled.isEnabled = true;
+
+		Object.defineProperty(throttled, 'queueSize', {
+			get() {
+				return queue.size;
+			},
+		});
+
+		return throttled;
+	};
+}

package.json

@@ -13,6 +13,7 @@
   "author": "[email protected]",
   "license": "Apache-2.0",
   "dependencies": {
+    "bottleneck": "^2.19.5",
     "dotenv": "^16.3.1",
     "jquery": "^3.7.1",
     "jquery-ui": "1.14.0"

src/common.js

@@ -85,6 +85,11 @@ export const summarization_configValues = [
   ["Vertex AI Location", "us-central1"],
   ["Gemini Model ID", "gemini-1.5-flash-001"],
   ["Instructions", "Summarize the text."],
+  ["summarization_quality", "TRUE"],
+  ["summarization_helpfulness", "TRUE"],
+  ["summarization_verbosity", "FALSE"],
+  ["groundedness", "FALSE"],
+  ["fulfillment", "FALSE"],
   ["Batch Size (1-10)", "4"], // BatchSize
   ["Time between Batches in Seconds (1-10)", "2"],
 ];
@@ -102,6 +107,40 @@ export const summarization_TableHeader = [
   ],
 ];
 
+// Eval Maps
+
+export const mapSummaryQualityScore = new Map();
+mapSummaryQualityScore.set(1, "1-Very Bad");
+mapSummaryQualityScore.set(2, "2-Bad");
+mapSummaryQualityScore.set(3, "3-OK");
+mapSummaryQualityScore.set(4, "4-Good");
+mapSummaryQualityScore.set(5, "5-Very Good");
+
+export const mapSummaryHelpfulnessScore = new Map();
+mapSummaryHelpfulnessScore.set(1, "1-Unhelpful");
+mapSummaryHelpfulnessScore.set(2, "2-Somewhat Unhelpful");
+mapSummaryHelpfulnessScore.set(3, "3-Neutral");
+mapSummaryHelpfulnessScore.set(4, "4-Somewhat Helpful");
+mapSummaryHelpfulnessScore.set(5, "5-Helpful");
+
+export const mapSummaryVerbosityScore = new Map();
+mapSummaryVerbosityScore.set(-2, "-2-Terse");
+mapSummaryVerbosityScore.set(-1, "-1-Somewhat Terse");
+mapSummaryVerbosityScore.set(0, "0-Optimal");
+mapSummaryVerbosityScore.set(1, "1-Somewhat Verbose");
+mapSummaryVerbosityScore.set(2, "2-Verbose");
+
+export const mapTextgenGroundednessScore = new Map();
+mapTextgenGroundednessScore.set(0, "0-Ungrounded");
+mapTextgenGroundednessScore.set(1, "1-Grounded");
+
+export const mapTextgenFulfillmentScore = new Map();
+mapTextgenFulfillmentScore.set(1, "1-No fulfillment");
+mapTextgenFulfillmentScore.set(2, "2-Poor fulfillment");
+mapTextgenFulfillmentScore.set(3, "3-Some fulfillment");
+mapTextgenFulfillmentScore.set(4, "4-Good fulfillment");
+mapTextgenFulfillmentScore.set(5, "5-Complete fulfillmentl");
+
 export class VertexAIError extends Error {
   constructor(id, message = "Processing Error") {
     super(message);
@@ -145,3 +184,4 @@ export class ResourceNotFoundError extends Error {
     this.statusCode = 404; // Optional: HTTP status code for API errors
   }
 }
+

src/excel/addin.js

@@ -29,7 +29,7 @@ Office.onReady((info) => {
       },
       "cancelSearchTests",
       async function () {
-        await excelSearchRunner.stopSearchTests();
+        await excelSearchRunner.cancelProcessing();
       },
     );
     syntheticQuestionAnswerRunner = new SyntheticQARunner();
@@ -44,7 +44,7 @@ Office.onReady((info) => {
       },
       "cancelGenerateQAData",
       async function () {
-        await syntheticQuestionAnswerRunner.stopSyntheticData();
+        await syntheticQuestionAnswerRunner.cancelProcessing();
       },
     );
 
@@ -60,7 +60,7 @@ Office.onReady((info) => {
       },
       "cancelSummarizationData",
       async function () {
-        await summarizationRunner.stopSummarizationData();
+        await summarizationRunner.cancelProcessing();
       },
     );
   }

src/excel/excel_search_runner.js

@@ -9,6 +9,10 @@ import { getColumn } from "./excel_common.js";
 export class ExcelSearchRunner extends TaskRunner {
   constructor() {
     super();
+    this.throttled_process_summary = this.throttle((a, b, c, d, e) =>
+      this.processSummary(a, b, c, d, e),
+    );
+    this.searchTaskPromiseSet = new Set();
   }
 
   async getSearchConfig() {
@@ -131,28 +135,39 @@ export class ExcelSearchRunner extends TaskRunner {
     });
   }
 
-  async stopSearchTests() {
-    // Set the cancel signal flag
-    this.cancelPressed = true;
-    appendLog("Cancel Requested. Stopping SearchTests execution...");
+  async getResultFromVertexAI(rowNum, config) {
+    var query = this.queryColumn.values;
+    return await callVertexAISearch(rowNum, query[rowNum][0], config);
   }
 
-  async getResultFromExternalAPI(rowNum, config) {
-    var query = this.queryColumn.values;
-    return callVertexAISearch(rowNum, query[rowNum][0], config);
+  async waitForTaskstoFinish() {
+    await Promise.allSettled(this.searchTaskPromiseSet.values());
+  }
+
+  async cancelAllTasks() {
+    this.throttled_process_summary.abort();
+     appendLog(`Cancel Requested for Search Tasks`);
   }
 
   async processRow(response_json, context, config, rowNum) {
+    let numCalls = 1;
     if (response_json.hasOwnProperty("summary")) {
-      await this.processSummary(
+      // process the summary using throttling since it makes an external call
+      const processSummaryPromise = this.throttled_process_summary(
         context,
         config,
         rowNum,
         response_json,
         this.expectedSummaryColumn.values,
-      );
-      appendLog(`testCaseID: ${rowNum} Processed Summary.`);
+      ).then(async (callsSoFar) => {
+        appendLog(`testCaseID: ${rowNum} Processed Search Summary.`);
+      });
+
+      this.searchTaskPromiseSet.add(processSummaryPromise);
+      // wait for processRow to finish
+      await Promise.resolve(processSummaryPromise);
     }
+
     // Check the documents references
     if (response_json.hasOwnProperty("results")) {
       this.checkDocumentLinks(
@@ -162,9 +177,10 @@ export class ExcelSearchRunner extends TaskRunner {
         this.expected_link_1_Column.values,
         this.expected_link_2_Column.values,
       );
-
       appendLog(`testCaseID: ${rowNum} Processed Doc Links.`);
     }
+
+    return numCalls;
   }
 
   async processSummary(context, config, rowNum, result, expectedSummary) {
@@ -173,19 +189,19 @@ export class ExcelSearchRunner extends TaskRunner {
       const actualSummarycell = this.actualSummaryColumn.getRange().getCell(rowNum, 0);
       actualSummarycell.clear(Excel.ClearApplyTo.formats);
       actualSummarycell.values = [[result.summary.summaryText]];
-      // match summaries only if they are not null or empty
+
+      // match summaries only if they are not null or not empty
       if (expectedSummary[rowNum][0] !== null && expectedSummary[rowNum][0] !== "") {
         const score_cell = this.summaryScoreColumn.getRange().getCell(rowNum, 0);
         score_cell.clear(Excel.ClearApplyTo.formats);
 
-        // Catch any errors here and report it in the cell. We don't want failures here to stop processing.
-
         const response = await calculateSimilarityUsingPalm2(
           rowNum,
           result.summary.summaryText,
           expectedSummary[rowNum][0],
           config,
         );
+
         const score = response.output;
 
         if (score.trim() === "same") {
@@ -196,12 +212,13 @@ export class ExcelSearchRunner extends TaskRunner {
           actualSummarycell.format.fill.color = "#FFCCCB";
         }
       }
+      // Catch any errors here and report it in the cell. We don't want failures here to stop processing.
     } catch (err) {
       appendError(`testCaseID: ${rowNum} Error getting Similarity. Error: ${err.message} `, err);
-      /*  // put the error in the cell.
-        score_cell.values = [["Failed. Error: " + err.message]];
-        score_cell.format.fill.color = "#FFCCCB";
-        actualSummarycell.format.fill.color = "#FFCCCB"; */
+      // put the error in the cell.
+      score_cell.values = [["Failed. Error: " + err.message]];
+      score_cell.format.fill.color = "#FFCCCB";
+      actualSummarycell.format.fill.color = "#FFCCCB";
     } finally {
       //await context.sync();
     }