Merge pull request #104 from Scale3-Labs/rohit/S3EN-2282-remove-redun…

…dant-fields

Rohit/s3 en 2282 remove redundant fields

app/(protected)/project/[project_id]/evaluations/[test_id]/page.tsx

@@ -52,7 +52,7 @@ export default function Page() {
     },
   });
 
-  const [score, setScore] = useState<number>(testData?.test?.min || -1);
+  const [score, setScore] = useState<number>(testData?.test?.min ?? -1);
   const [scorePercent, setScorePercent] = useState<number>(0);
   const [color, setColor] = useState<string>("red");
   const [span, setSpan] = useState<any>(null);
@@ -179,7 +179,7 @@ export default function Page() {
       }
       const result = await response.json();
       const sc =
-        result.evaluations.length > 0 ? result.evaluations[0].ltUserScore : -1;
+        result.evaluations.length > 0 ? result.evaluations[0].ltUserScore ?? -1 : -1;
       onScoreSelected(sc);
       return result;
     },
@@ -233,6 +233,7 @@ export default function Page() {
           body: JSON.stringify({
             id: evaluationsData.evaluations[0].id,
             ltUserScore: score,
+            testId
           }),
         });
         queryClient.invalidateQueries({
@@ -249,12 +250,7 @@ export default function Page() {
             projectId: projectId,
             spanId: span.span_id,
             traceId: span.trace_id,
-            spanStartTime: span?.start_time
-              ? new Date(correctTimestampFormat(span.start_time))
-              : new Date(),
             ltUserScore: score,
-            model: model,
-            prompt: systemPrompt,
             testId: testId,
           }),
         });

app/api/dataset/download/route.ts

@@ -60,8 +60,6 @@ export async function GET(req: NextRequest) {
     const timestamp = new Date().toISOString().slice(0, 19).replace(/[-:]/g, '');
     const filename = `${datasetName}_${timestamp}.csv`;
 
-    console.log(`CSV file '${filename}' `);
-
     return new NextResponse(csv, {
       headers: {
         'Content-Type': 'text/csv',

app/api/evaluation/route.ts

@@ -17,9 +17,6 @@ export async function POST(req: NextRequest) {
     const {
       traceId,
       spanId,
-      model,
-      spanStartTime,
-      prompt,
       userScore,
       userId
     } = data;
@@ -41,14 +38,11 @@ export async function POST(req: NextRequest) {
 
     const evaluation = await prisma.evaluation.create({
       data: {
-        spanStartTime: convertToDateTime64(spanStartTime),
         spanId,
         traceId,
         projectId,
-        model,
         userId,
-        userScore,
-        prompt,
+        userScore
       },
     });
     return NextResponse.json({
@@ -92,10 +86,7 @@ export async function POST(req: NextRequest) {
       traceId,
       spanId,
       projectId,
-      model,
       ltUserScore,
-      spanStartTime,
-      prompt,
       testId,
     } = data;
 
@@ -134,14 +125,11 @@ export async function POST(req: NextRequest) {
 
     const evaluation = await prisma.evaluation.create({
       data: {
-        spanStartTime,
         spanId,
         traceId,
         ltUserId: user.id,
         projectId,
-        model,
         ltUserScore,
-        prompt,
         testId,
       },
     });
@@ -170,12 +158,11 @@ export async function GET(req: NextRequest) {
     }
 
     const spanId = req.nextUrl.searchParams.get("spanId") as string;
-    const prompt = req.nextUrl.searchParams.get("prompt") as string;
 
-    if (!projectId && !spanId && !prompt) {
+    if (!projectId && !spanId) {
       return NextResponse.json(
         {
-          message: "Please provide a projectId or spanId or prompt",
+          message: "Please provide a projectId or spanId",
         },
         { status: 400 }
       );
@@ -199,24 +186,6 @@ export async function GET(req: NextRequest) {
       });
     }
 
-    if (prompt) {
-      const evaluations = await prisma.evaluation.findMany({
-        where: {
-          prompt,
-        },
-      });
-
-      if (!evaluations) {
-        return NextResponse.json({
-          evaluations: [],
-        });
-      }
-
-      return NextResponse.json({
-        evaluations: [evaluations],
-      });
-    }
-
     // check if this user has access to this project
     if(session) {
       const email = session?.user?.email as string;
@@ -264,7 +233,6 @@ export async function GET(req: NextRequest) {
       evaluations,
     });
   } catch (error) {
-    console.error("hereeeee", error);
     return NextResponse.json(
       {
         message: "Internal server error",
@@ -351,15 +319,15 @@ export async function PUT(req: NextRequest) {
     }
 
     const data = await req.json();
-    const { id, ltUserScore } = data;
-
+    const { id, ltUserScore, testId } = data;
     const evaluation = await prisma.evaluation.update({
       where: {
         id,
       },
       data: {
         ltUserId: user.id,
         ltUserScore,
+        testId
       },
     });
 

app/api/metrics/accuracy/route.ts

@@ -1,5 +1,6 @@
 import { authOptions } from "@/lib/auth/options";
 import prisma from "@/lib/prisma";
+import { TraceService } from "@/lib/services/trace_service";
 import { Evaluation } from "@prisma/client";
 import { getServerSession } from "next-auth";
 import { redirect } from "next/navigation";
@@ -13,7 +14,12 @@ export async function GET(req: NextRequest) {
 
   const projectId = req.nextUrl.searchParams.get("projectId") as string;
   const testId = req.nextUrl.searchParams.get("testId") as string;
-  const byModel = req.nextUrl.searchParams.get("by_model") as string;
+  let lastNDays = Number(req.nextUrl.searchParams.get("lastNDays"));
+  let overallAccuracy = 0
+
+  if(Number.isNaN(lastNDays) || lastNDays < 0){
+    lastNDays = 7;
+  }
 
   if (!projectId) {
     return NextResponse.json(
@@ -59,76 +65,66 @@ export async function GET(req: NextRequest) {
   }
 
   let evaluations: Evaluation[] = [];
-  let allEvaluations: Evaluation[] = [];
-  let average: number = 0;
-
-  // get evalutaion for the last 7 days
+  const traceService = new TraceService();
+    //Fetch last 7 days of spanIds from clickhouse
+  const spans =  await traceService.GetSpansInProject(
+    projectId,
+    lastNDays
+  );
+
+  // get evalutaion for the lastNDays
+  // and all evaluations where score is 1 or -1
   evaluations = await prisma.evaluation.findMany({
     where: {
       projectId,
       testId,
-      spanStartTime: {
-        gte: new Date(new Date().getTime() - 7 * 24 * 60 * 60 * 1000),
-      },
-    },
-  });
-
-  // get average evaluation for all evaluations
-  // get all evaluations where score is 1 or -1
-  allEvaluations = await prisma.evaluation.findMany({
-    where: {
-      projectId,
-      testId,
+      spanId: { in: [...spans.map((span) => span.span_id)]},
       ltUserScore: {
         in: [1, -1],
-      },
+      }
     },
   });
-
-  const totalPositive = allEvaluations.reduce((acc, evaluation) => {
-    if (evaluation.ltUserScore === 1) {
-      return acc + 1;
-    }
-    return acc;
-  }, 0);
-
-  const totalNegative = allEvaluations.reduce((acc, evaluation) => {
-    if (evaluation.ltUserScore === -1) {
-      return acc + 1;
-    }
-    return acc;
-  }, 0);
-
-  // calculate average
-  average = (totalPositive / (totalPositive + totalNegative)) * 100;
-
-  if (byModel === "true") {
-    const evaluationsByModel = evaluations.reduce(
-      (acc: any, evaluation: Evaluation) => {
-        if (!evaluation.model) {
-          return acc;
-        }
-        if (acc[evaluation.model]) {
-          acc[evaluation.model].push(evaluation);
-        } else {
-          acc[evaluation.model] = [evaluation];
-        }
-        return acc;
-      },
-      {}
-    );
-
-    return NextResponse.json({
-      evaluations: evaluationsByModel,
-    });
-  }
-
   if (!evaluations) {
-    return NextResponse.json({ evalutions: [], average: 0 }, { status: 200 });
+    return NextResponse.json({ accuracyPerDay: [], overallAccuracy: null }, { status: 200 });
   }
+  const evalsByDate: Record<string, Evaluation[]> = {};
+  evaluations.forEach((evaluation, index) => {
+    const span = spans[index];
+    const date = span.start_time.split("T")[0];
+    if(evalsByDate[date]){
+      evalsByDate[date].push(evaluation);
+    } else {
+      evalsByDate[date] = [evaluation];
+    }
+  })
+  let totalPositive = 0;
+  let totalNegative = 0;
+
+  const accuracyPerDay = Object.entries(evalsByDate).map(([date, scores]) => {
+    let totalPositivePerDay = 0;
+    let totalNegativePerDay = 0;
+
+    scores.forEach((score) => {
+      if (score.ltUserScore === 1) {
+        totalPositivePerDay += 1;
+        totalPositive += 1;
+      } else {
+        totalNegativePerDay+= 1;
+        totalNegative += 1;
+      }
+    });
+    const accuracy = (totalPositive / (totalPositive + totalNegative)) * 100;
+    return {
+      date,
+      accuracy,
+    };
+  });
+  accuracyPerDay.sort((a, b) => new Date(a.date).getTime() - new Date(b.date).getTime());
+  // calculate average
+  overallAccuracy = (totalPositive / (totalPositive + totalNegative)) * 100;
 
   return NextResponse.json({
-    evaluations,
-    average,
+    overallAccuracy,
+    accuracyPerDay
   });
 }

components/charts/eval-chart.tsx

@@ -31,53 +31,8 @@ export function EvalChart({
   if (fetchAccuracy.isLoading || !fetchAccuracy.data) {
     return <LargeChartLoading />;
   } else {
-    // aggregate accuracy per day and return the data
-    const evaluations = fetchAccuracy?.data?.evaluations;
-    const accuracyPerDay = evaluations.reduce((acc: any, evaluation: any) => {
-      const date = evaluation.spanStartTime.split("T")[0];
-      if (acc[date]) {
-        acc[date].push(evaluation.score);
-      } else {
-        acc[date] = [evaluation.score];
-      }
-      return acc;
-    }, {});
-
-    // calculate accuracy by dividing the sum of scores that are only 1s by the number of evaluations times 100
-    const data = Object.keys(accuracyPerDay).map((date) => {
-      const scores = accuracyPerDay[date];
-
-      // count +1s and -1s
-      const totalPositive = scores.reduce((acc: number, score: number) => {
-        if (score === 1) {
-          return acc + 1;
-        }
-        return acc;
-      }, 0);
-
-      const totalNegative = scores.reduce((acc: number, score: number) => {
-        if (score === -1) {
-          return acc + 1;
-        }
-        return acc;
-      }, 0);
-
-      // calculate accuracy
-      const accuracy = (totalPositive / (totalPositive + totalNegative)) * 100;
-
-      return {
-        date,
-        "Evaluated Accuracy(%)": accuracy,
-      };
-    });
-
-    // sort the data by date
-    data.sort((a: any, b: any) => {
-      return (new Date(a.date) as any) - (new Date(b.date) as any);
-    });
-
-    // calculate the overall accuracy
-    const overallAccuracy = fetchAccuracy?.data?.average;
+    const data: Array<Record<string, number>> = fetchAccuracy?.data?.accuracyPerDay;
+    const overallAccuracy: number = fetchAccuracy?.data?.overallAccuracy;
 
     return (
       <>
@@ -93,7 +48,7 @@ export function EvalChart({
           </p>
           <AreaChart
             className="mt-2 h-72"
-            data={data}
+            data={data.map((dat) => ({date: dat.date, "Evaluated Accuracy(%)": dat.accuracy}))}
             index="date"
             categories={["Evaluated Accuracy(%)"]}
             colors={["purple"]}