Add Turbo script for creating development prompts

packages/aila/src/core/prompt/builders/AilaLessonPromptBuilder.ts

@@ -6,7 +6,11 @@ import { prisma as globalPrisma } from "@oakai/db";
 
 import { DEFAULT_RAG_LESSON_PLANS } from "../../../constants";
 import { tryWithErrorReporting } from "../../../helpers/errorReporting";
-import { LooseLessonPlan } from "../../../protocol/schema";
+import { LLMResponseJsonSchema } from "../../../protocol/jsonPatchProtocol";
+import {
+  LessonPlanJsonSchema,
+  LooseLessonPlan,
+} from "../../../protocol/schema";
 import { findAmericanisms } from "../../../utils/language/findAmericanisms";
 import { compressedLessonPlanForRag } from "../../../utils/lessonPlan/compressedLessonPlanForRag";
 import { fetchLessonPlan } from "../../../utils/lessonPlan/fetchLessonPlan";
@@ -86,6 +90,8 @@ export class AilaLessonPromptBuilder extends AilaPromptBuilder {
       baseLessonPlan: baseLessonPlan
         ? compressedLessonPlanForRag(baseLessonPlan)
         : undefined,
+      lessonPlanJsonSchema: JSON.stringify(LessonPlanJsonSchema),
+      llmResponseJsonSchema: JSON.stringify(LLMResponseJsonSchema),
     };
 
     return template(args);

packages/aila/src/features/generation/AilaGeneration.ts

@@ -1,14 +1,11 @@
+import { prisma } from "@oakai/db";
 import { kv } from "@vercel/kv";
 import { getEncoding } from "js-tiktoken";
 
 import { AilaServices } from "../../core";
 import { AilaChat } from "../../core/chat";
 import { AilaGenerationStatus } from "./types";
 
-// While we migrate to versioned prompts, we need to retain the fallback prompt ID
-// #TODO Remove this fallback prompt ID once we are all set up to use versioned prompts
-const fallbackPromptID = "clnnbmzso0000vgtj13dydvs7";
-
 export class AilaGeneration {
   private _aila: AilaServices;
   private _id: string;
@@ -22,7 +19,7 @@ export class AilaGeneration {
   private _completionTokens: number = 0;
   private _totalTokens: number = 0;
   private _systemPrompt: string = "";
-  private _promptId: string = fallbackPromptID;
+  private _promptId: string | null = null;
 
   constructor({
     aila,
@@ -45,7 +42,9 @@ export class AilaGeneration {
     this._startedAt = new Date();
     this._systemPrompt = systemPrompt;
     this._aila = aila;
-    this._promptId = promptId ?? fallbackPromptID;
+    if (promptId) {
+      this._promptId = promptId;
+    }
   }
 
   async complete({
@@ -74,10 +73,8 @@ export class AilaGeneration {
   }
 
   public async setupPromptId(): Promise<string | null> {
-    // This is the prompt ID for the default prompt.
-    // We should have a fallback prompt ID in case the prompt is not available in KV.
     if (!this._promptId) {
-      this._promptId = (await this.fetchPromptId()) ?? fallbackPromptID;
+      this._promptId = await this.fetchPromptId();
     }
     return this._promptId;
   }
@@ -154,9 +151,17 @@ export class AilaGeneration {
 
     // This key is defined in the setupPrompts script in core
     const variantSlug = `${responseMode}-${basedOn ? "basedOn" : "notBasedOn"}-${useRag ? "rag" : "noRag"}`;
-    const kvKey = `prompt:${appSlug}:${promptSlug}:${variantSlug}`;
-    console.log("Fetching prompt ID from KV", kvKey);
-    const promptId = await kv.get<string>(kvKey);
-    return promptId;
+
+    const prompt = await prisma.prompt.findFirst({
+      where: {
+        variant: variantSlug,
+        appId: appSlug,
+        slug: promptSlug,
+      },
+    });
+    if (!prompt) {
+      throw new Error(`Prompt not found - have you run pnpm prompts:dev?`);
+    }
+    return prompt?.id;
   }
 }

packages/core/package.json

@@ -23,6 +23,7 @@
     "process:lessons": "pnpm doppler:run:dev -- ts-node --compiler-options {\\\"module\\\":\\\"CommonJS\\\"} ./src/scripts/processLessons.ts",
     "process:lessons:dev": "pnpm doppler:run:dev -- pnpm process:lessons",
     "prompts": "ts-node --compiler-options {\\\"module\\\":\\\"CommonJS\\\"} ./src/scripts/setupPrompts.ts",
+    "prompts:dev": "pnpm doppler:run:dev -- ts-node --compiler-options {\\\"module\\\":\\\"CommonJS\\\"} ./src/scripts/setupPrompts.ts",
     "seed": "ts-node --compiler-options {\\\"module\\\":\\\"CommonJS\\\"} ./src/scripts/seedLessons.ts",
     "seed:dev": "pnpm doppler:run:dev -- pnpm seed",
     "seed:prd": "pnpm doppler:run:prd -- pnpm seed",

packages/core/src/models/promptVariants.tsx

@@ -1,4 +1,5 @@
 import { PrismaClientWithAccelerate } from "@oakai/db";
+import { kv } from "@vercel/kv";
 import dedent from "ts-dedent";
 import { Md5 } from "ts-md5";
 
@@ -24,25 +25,26 @@ export class PromptVariants {
     this.variant = foundVariant;
   }
 
-  async setCurrent() {
-    const template = this.format();
-    const hash = Md5.hashStr(template);
+  async setCurrent(variant: string, raw = false) {
+    const template = this.format(raw);
+    const { slug } = this.definition;
+    const hash = `${slug}-${variant}-${Md5.hashStr(template)}`;
+    console.log("*** setCurrent ***");
+    console.log("Hash", hash);
+    console.log("Slug", slug);
+    console.log("Variant", variant);
+    console.log("-----");
     const existing = await this.prisma.prompt.findFirst({
       where: {
-        hash,
+        AND: [{ hash }, { slug }, { variant }],
       },
     });
     if (existing) {
       return;
     }
-    const {
-      name,
-      slug,
-      appId: appSlug,
-      inputSchema,
-      outputSchema,
-    } = this.definition;
+    const { name, appId: appSlug, inputSchema, outputSchema } = this.definition;
 
+    console.log("Connecting to app", appSlug);
     const app = await this.prisma.app.findFirstOrThrow({
       where: { slug: appSlug },
     });
@@ -52,7 +54,6 @@ export class PromptVariants {
     }[];
     const maxVersion = maxVersionRows?.[0]?.max_version ?? 0;
     const version = maxVersion + 1;
-    const variant = "main"; // TODO enable support for more than one variant
     const created = await this.prisma.prompt.create({
       data: {
         hash,
@@ -68,9 +69,17 @@ export class PromptVariants {
         variant,
         identifier: `${slug}-${variant}-${version}`,
         version,
-        gitSha: process.env.CACHED_COMMIT_REF ?? null, // Netlify-specific environment variable for the latest git commit
+        gitSha:
+          process.env.VERCEL_GIT_COMMIT_SHA ??
+          process.env.CACHED_COMMIT_REF ??
+          null, // Vercel- and Netlify-specific environment variable for the latest git commit
       },
     });
+
+    // Store the prompt version ID in KV
+    const kvKey = `prompt:${appSlug}:${slug}:${variant}`;
+    await kv.set(kvKey, created.id);
+
     // Mark previous prompts as historic
     await this.prisma.prompt.updateMany({
       data: {
@@ -90,24 +99,29 @@ export class PromptVariants {
     });
   }
 
-  format() {
+  format(raw = false) {
     const { parts } = this.variant;
     const { body, context, output, task } = parts;
+
+    if (raw) {
+      return body;
+    }
+
     return dedent`CONTEXT 
     ${context}
-
+  
     PROMPT INJECTION
     ${promptInjection}
-
+  
     TASK
     ${task}
-
+  
     INSTRUCTIONS
     ${body}
-
+  
     OUTPUT
     ${output}
-
+  
     ERROR HANDLING
     ${errorHandling}`;
   }

packages/core/src/prompts/lesson-assistant/index.test.ts

@@ -0,0 +1,72 @@
+import { LLMResponseJsonSchema } from "../../../../aila/src/protocol/jsonPatchProtocol";
+import { LessonPlanJsonSchema } from "../../../../aila/src/protocol/schema";
+import { template as newTemplate, TemplateProps } from "./index";
+import { template as oldTemplate } from "./old";
+
+// Whilst refactoring the Aila template so that we can store versioning
+// information, we want to make sure that the prompt generated by the new
+// template is the same as the prompt generated by the old template.
+describe("Lesson Assistant Template", () => {
+  const testCases: [string, TemplateProps][] = [
+    [
+      "interactive with RAG and baseLessonPlan",
+      {
+        subject: "Mathematics",
+        keyStage: "KS3",
+        topic: "Algebra",
+        relevantLessonPlans: "Some relevant plans",
+        currentLessonPlan: "Current plan",
+        summaries: "Some summaries",
+        lessonTitle: "Introduction to Equations",
+        responseMode: "interactive",
+        baseLessonPlan: "Base lesson plan",
+        useRag: true,
+        americanisms: [{ word: "color", replacement: "colour" }],
+        lessonPlanJsonSchema: JSON.stringify(LessonPlanJsonSchema),
+        llmResponseJsonSchema: JSON.stringify(LLMResponseJsonSchema),
+      },
+    ],
+    [
+      "generate without RAG or baseLessonPlan",
+      {
+        subject: "English",
+        keyStage: "KS2",
+        topic: "Poetry",
+        relevantLessonPlans: "Some relevant plans",
+        currentLessonPlan: "Current plan",
+        summaries: "Some summaries",
+        lessonTitle: "Rhyme and Rhythm",
+        responseMode: "generate",
+        useRag: false,
+        lessonPlanJsonSchema: JSON.stringify(LessonPlanJsonSchema),
+        llmResponseJsonSchema: JSON.stringify(LLMResponseJsonSchema),
+      },
+    ],
+    [
+      "interactive without RAG but with baseLessonPlan",
+      {
+        subject: "Science",
+        keyStage: "KS4",
+        topic: "Chemistry",
+        relevantLessonPlans: "Some relevant plans",
+        currentLessonPlan: "Current plan",
+        summaries: "Some summaries",
+        lessonTitle: "Chemical Bonding",
+        responseMode: "interactive",
+        baseLessonPlan: "Base lesson plan",
+        useRag: false,
+        americanisms: [],
+        lessonPlanJsonSchema: JSON.stringify(LessonPlanJsonSchema),
+        llmResponseJsonSchema: JSON.stringify(LLMResponseJsonSchema),
+      },
+    ],
+  ];
+
+  test.each(testCases)("%s", (_, props) => {
+    const newResult = newTemplate(props);
+    const oldResult = oldTemplate(props);
+
+    // Compare the results
+    expect(newResult).toBe(oldResult);
+  });
+});