diff --git a/genkit-tools/cli/src/commands/eval-flow.ts b/genkit-tools/cli/src/commands/eval-flow.ts index cbfb3fb2d..3245fca41 100644 --- a/genkit-tools/cli/src/commands/eval-flow.ts +++ b/genkit-tools/cli/src/commands/eval-flow.ts @@ -322,7 +322,7 @@ async function fetchDataSet( testCaseId: randomUUID(), input: inputs[0], output: outputs[0], - context: contexts, + context: JSON.parse(contexts[0]) as string[], reference: references?.at(i), traceIds, }; diff --git a/js/testapps/cat-eval/src/index.ts b/js/testapps/cat-eval/src/index.ts index 5ac4f9ccd..b3a6784aa 100644 --- a/js/testapps/cat-eval/src/index.ts +++ b/js/testapps/cat-eval/src/index.ts @@ -19,7 +19,7 @@ import { devLocalVectorstore } from '@genkit-ai/dev-local-vectorstore'; import { dotprompt } from '@genkit-ai/dotprompt'; import { genkitEval, GenkitMetric } from '@genkit-ai/evaluator'; import { firebase } from '@genkit-ai/firebase'; -import { geminiPro, googleAI } from '@genkit-ai/googleai'; +import { gemini15Pro, googleAI } from '@genkit-ai/googleai'; import { textEmbeddingGecko, vertexAI } from '@genkit-ai/vertexai'; // Turn off safety checks for evaluation so that the LLM as an evaluator can @@ -51,7 +51,7 @@ configureGenkit({ firebase(), googleAI(), genkitEval({ - judge: geminiPro, + judge: gemini15Pro, judgeConfig: PERMISSIVE_SAFETY_SETTINGS, metrics: [GenkitMetric.MALICIOUSNESS], embedder: textEmbeddingGecko,