From 4e5e1f22a4ea6262114033e45ffe4817b483f379 Mon Sep 17 00:00:00 2001 From: Stef Lewandowski Date: Tue, 29 Oct 2024 12:43:18 +0000 Subject: [PATCH] feat: selectively include Americanisms, RAG, analytics when instantiating Aila (#287) --- apps/nextjs/src/app/api/chat/chatHandler.ts | 13 ++ packages/aila/src/core/Aila.ts | 33 +++- packages/aila/src/core/AilaFeatureFactory.ts | 18 +- packages/aila/src/core/AilaServices.ts | 6 +- packages/aila/src/core/lesson/AilaLesson.ts | 5 +- .../builders/AilaLessonPromptBuilder.ts | 24 +-- packages/aila/src/core/types.ts | 7 + .../features/americanisms/AilaAmericanisms.ts | 69 ++++++++ .../americanisms/NullAilaAmericanisms.ts | 7 + .../american-british-english-translator.d.ts | 0 .../aila/src/features/americanisms/index.ts | 16 ++ .../src/features/analytics/AilaAnalytics.ts | 2 +- .../analytics/adapters/AnalyticsAdapter.ts | 2 +- .../adapters/PosthogAnalyticsAdapter.ts | 2 +- .../categorisers/AilaCategorisation.ts | 160 ++++++++++++++---- packages/aila/src/features/rag/AilaRag.ts | 5 +- packages/aila/src/features/rag/NullAilaRag.ts | 7 + packages/aila/src/features/rag/index.ts | 9 +- .../src/utils/language/findAmericanisms.ts | 75 -------- .../lessonPlan/fetchLessonPlanContentById.ts | 7 +- .../moderation/moderationErrorHandling.ts | 2 +- packages/eslint-config-custom/index.js | 2 +- packages/logger/index.ts | 1 + 23 files changed, 308 insertions(+), 164 deletions(-) create mode 100644 packages/aila/src/features/americanisms/AilaAmericanisms.ts create mode 100644 packages/aila/src/features/americanisms/NullAilaAmericanisms.ts rename packages/aila/src/{utils/language => features/americanisms}/american-british-english-translator.d.ts (100%) create mode 100644 packages/aila/src/features/americanisms/index.ts create mode 100644 packages/aila/src/features/rag/NullAilaRag.ts delete mode 100644 packages/aila/src/utils/language/findAmericanisms.ts diff --git a/apps/nextjs/src/app/api/chat/chatHandler.ts b/apps/nextjs/src/app/api/chat/chatHandler.ts index 3283a993d..907246e72 100644 --- a/apps/nextjs/src/app/api/chat/chatHandler.ts +++ b/apps/nextjs/src/app/api/chat/chatHandler.ts @@ -3,8 +3,15 @@ import type { AilaInitializationOptions, AilaOptions, AilaPublicChatOptions, + AilaServices, Message, } from "@oakai/aila"; +import { AilaAmericanisms } from "@oakai/aila/src/features/americanisms/AilaAmericanisms"; +import { + DatadogAnalyticsAdapter, + PosthogAnalyticsAdapter, +} from "@oakai/aila/src/features/analytics"; +import { AilaRag } from "@oakai/aila/src/features/rag/AilaRag"; import { LooseLessonPlan } from "@oakai/aila/src/protocol/schema"; import { TracingSpan, @@ -176,6 +183,12 @@ export async function handleChatPostRequest( services: { chatLlmService: llmService, moderationAiClient, + ragService: (aila: AilaServices) => new AilaRag({ aila }), + americanismsService: () => new AilaAmericanisms(), + analyticsAdapters: (aila: AilaServices) => [ + new PosthogAnalyticsAdapter(aila), + new DatadogAnalyticsAdapter(aila), + ], }, lessonPlan: lessonPlan ?? {}, }; diff --git a/packages/aila/src/core/Aila.ts b/packages/aila/src/core/Aila.ts index b47aabd39..41de60f53 100644 --- a/packages/aila/src/core/Aila.ts +++ b/packages/aila/src/core/Aila.ts @@ -1,4 +1,4 @@ -import type { PrismaClientWithAccelerate} from "@oakai/db"; +import type { PrismaClientWithAccelerate } from "@oakai/db"; import { prisma as globalPrisma } from "@oakai/db"; import { aiLogger } from "@oakai/logger"; @@ -7,7 +7,11 @@ import { DEFAULT_TEMPERATURE, DEFAULT_RAG_LESSON_PLANS, } from "../constants"; +import type { AilaAmericanismsFeature } from "../features/americanisms"; +import { NullAilaAmericanisms } from "../features/americanisms/NullAilaAmericanisms"; import { AilaCategorisation } from "../features/categorisation"; +import type { AilaRagFeature } from "../features/rag"; +import { NullAilaRag } from "../features/rag/NullAilaRag"; import type { AilaSnapshotStore } from "../features/snapshotStore"; import type { AilaAnalyticsFeature, @@ -51,9 +55,11 @@ export class Aila implements AilaServices { private _persistence: AilaPersistenceFeature[] = []; private _threatDetection?: AilaThreatDetectionFeature; private _prisma: PrismaClientWithAccelerate; + private _rag: AilaRagFeature; private _plugins: AilaPlugin[]; private _userId!: string | undefined; private _chatId!: string; + private _americanisms: AilaAmericanismsFeature; constructor(options: AilaInitializationOptions) { this._userId = options.chat.userId; @@ -79,13 +85,14 @@ export class Aila implements AilaServices { options.services?.chatCategoriser ?? new AilaCategorisation({ aila: this, - prisma: this._prisma, - chatId: this._chatId, - userId: this._userId, }), }); - this._analytics = AilaFeatureFactory.createAnalytics(this, this._options); + this._analytics = AilaFeatureFactory.createAnalytics( + this, + this._options, + options.services?.analyticsAdapters?.(this), + ); this._moderation = AilaFeatureFactory.createModeration( this, this._options, @@ -104,6 +111,10 @@ export class Aila implements AilaServices { this, this._options, ); + this._rag = options.services?.ragService?.(this) ?? new NullAilaRag(); + this._americanisms = + options.services?.americanismsService?.(this) ?? + new NullAilaAmericanisms(); if (this._analytics) { this._analytics.initialiseAnalyticsContext(); @@ -212,6 +223,18 @@ export class Aila implements AilaServices { return this._chatLlmService; } + public get rag() { + return this._rag; + } + + public get americanisms() { + return this._americanisms; + } + + public get prisma() { + return this._prisma; + } + // Check methods public checkUserIdPresentIfPersisting() { if (!this._chat.userId && this._options.usePersistence) { diff --git a/packages/aila/src/core/AilaFeatureFactory.ts b/packages/aila/src/core/AilaFeatureFactory.ts index 3730830ab..e25d20069 100644 --- a/packages/aila/src/core/AilaFeatureFactory.ts +++ b/packages/aila/src/core/AilaFeatureFactory.ts @@ -1,16 +1,10 @@ // AilaFeatureFactory.ts -import { - DatadogAnalyticsAdapter, - PosthogAnalyticsAdapter, -} from "../features/analytics"; +import type { AnalyticsAdapter } from "../features/analytics"; import { AilaAnalytics } from "../features/analytics/AilaAnalytics"; import { SentryErrorReporter } from "../features/errorReporting/reporters/SentryErrorReporter"; import { AilaModeration } from "../features/moderation"; -import type { - OpenAILike} from "../features/moderation/moderators/OpenAiModerator"; -import { - OpenAiModerator, -} from "../features/moderation/moderators/OpenAiModerator"; +import type { OpenAILike } from "../features/moderation/moderators/OpenAiModerator"; +import { OpenAiModerator } from "../features/moderation/moderators/OpenAiModerator"; import { AilaPrismaPersistence } from "../features/persistence/adaptors/prisma"; import { AilaSnapshotStore } from "../features/snapshotStore"; import { AilaThreatDetection } from "../features/threatDetection"; @@ -28,14 +22,12 @@ export class AilaFeatureFactory { static createAnalytics( aila: AilaServices, options: AilaOptions, + adapters: AnalyticsAdapter[] = [], ): AilaAnalyticsFeature | undefined { if (options.useAnalytics) { return new AilaAnalytics({ aila, - adapters: [ - new PosthogAnalyticsAdapter(aila), - new DatadogAnalyticsAdapter(aila), - ], + adapters, }); } return undefined; diff --git a/packages/aila/src/core/AilaServices.ts b/packages/aila/src/core/AilaServices.ts index 2b0eb1968..816a4b09e 100644 --- a/packages/aila/src/core/AilaServices.ts +++ b/packages/aila/src/core/AilaServices.ts @@ -1,5 +1,7 @@ +import type { AilaAmericanismsFeature } from "../features/americanisms"; import type { AilaAnalytics } from "../features/analytics/AilaAnalytics"; import type { AilaErrorReporter } from "../features/errorReporting"; +import type { AilaRagFeature } from "../features/rag"; import type { AilaSnapshotStore } from "../features/snapshotStore"; import type { AilaAnalyticsFeature, @@ -14,8 +16,8 @@ import type { LooseLessonPlan, } from "../protocol/schema"; import type { Message } from "./chat"; -import type { AilaOptionsWithDefaultFallbackValues } from "./index"; import type { AilaPlugin } from "./plugins"; +import type { AilaOptionsWithDefaultFallbackValues } from "./types"; // This provides a set of interfaces between the Aila core and the features that use it. // We can then mock these out in tests without needing to instantiate the entire Aila object. @@ -64,4 +66,6 @@ export interface AilaServices { readonly persistence?: AilaPersistenceFeature[]; readonly moderation?: AilaModerationFeature; readonly plugins: AilaPlugin[]; + readonly rag: AilaRagFeature; + readonly americanisms: AilaAmericanismsFeature; } diff --git a/packages/aila/src/core/lesson/AilaLesson.ts b/packages/aila/src/core/lesson/AilaLesson.ts index 9810cdc8f..e2b5003e5 100644 --- a/packages/aila/src/core/lesson/AilaLesson.ts +++ b/packages/aila/src/core/lesson/AilaLesson.ts @@ -3,8 +3,7 @@ import { deepClone } from "fast-json-patch"; import { AilaCategorisation } from "../../features/categorisation/categorisers/AilaCategorisation"; import type { AilaCategorisationFeature } from "../../features/types"; -import type { - PatchDocument} from "../../protocol/jsonPatchProtocol"; +import type { PatchDocument } from "../../protocol/jsonPatchProtocol"; import { applyLessonPlanPatch, extractPatches, @@ -39,8 +38,6 @@ export class AilaLesson implements AilaLessonService { categoriser ?? new AilaCategorisation({ aila, - userId: aila.userId, - chatId: aila.chatId, }); } diff --git a/packages/aila/src/core/prompt/builders/AilaLessonPromptBuilder.ts b/packages/aila/src/core/prompt/builders/AilaLessonPromptBuilder.ts index 621c34613..b9afc0cc5 100644 --- a/packages/aila/src/core/prompt/builders/AilaLessonPromptBuilder.ts +++ b/packages/aila/src/core/prompt/builders/AilaLessonPromptBuilder.ts @@ -1,27 +1,17 @@ -import type { - TemplateProps} from "@oakai/core/src/prompts/lesson-assistant"; -import { - template, -} from "@oakai/core/src/prompts/lesson-assistant"; +import type { TemplateProps } from "@oakai/core/src/prompts/lesson-assistant"; +import { template } from "@oakai/core/src/prompts/lesson-assistant"; import { prisma as globalPrisma } from "@oakai/db"; import { aiLogger } from "@oakai/logger"; import { DEFAULT_RAG_LESSON_PLANS } from "../../../constants"; import { tryWithErrorReporting } from "../../../helpers/errorReporting"; import { LLMResponseJsonSchema } from "../../../protocol/jsonPatchProtocol"; -import type { - LooseLessonPlan} from "../../../protocol/schema"; -import { - LessonPlanJsonSchema -} from "../../../protocol/schema"; -import { findAmericanisms } from "../../../utils/language/findAmericanisms"; +import type { LooseLessonPlan } from "../../../protocol/schema"; +import { LessonPlanJsonSchema } from "../../../protocol/schema"; import { compressedLessonPlanForRag } from "../../../utils/lessonPlan/compressedLessonPlanForRag"; import { fetchLessonPlan } from "../../../utils/lessonPlan/fetchLessonPlan"; -import type { - RagLessonPlan} from "../../../utils/rag/fetchRagContent"; -import { - fetchRagContent -} from "../../../utils/rag/fetchRagContent"; +import type { RagLessonPlan } from "../../../utils/rag/fetchRagContent"; +import { fetchRagContent } from "../../../utils/rag/fetchRagContent"; import type { AilaServices } from "../../AilaServices"; import { AilaPromptBuilder } from "../AilaPromptBuilder"; @@ -117,7 +107,7 @@ export class AilaLessonPromptBuilder extends AilaPromptBuilder { summaries: "None", responseMode: this._aila?.options.mode ?? "interactive", useRag: this._aila?.options.useRag ?? true, - americanisms: findAmericanisms(lessonPlan), + americanisms: this._aila.americanisms.findAmericanisms(lessonPlan), baseLessonPlan: baseLessonPlan ? compressedLessonPlanForRag(baseLessonPlan) : undefined, diff --git a/packages/aila/src/core/types.ts b/packages/aila/src/core/types.ts index 118382537..a98b8fa2a 100644 --- a/packages/aila/src/core/types.ts +++ b/packages/aila/src/core/types.ts @@ -1,8 +1,11 @@ import type { PrismaClientWithAccelerate } from "@oakai/db"; +import type { AilaAmericanismsFeature } from "../features/americanisms"; +import type { AnalyticsAdapter } from "../features/analytics"; import type { AilaModerator } from "../features/moderation/moderators"; import type { OpenAILike } from "../features/moderation/moderators/OpenAiModerator"; import type { AilaPersistence } from "../features/persistence"; +import type { AilaRagFeature } from "../features/rag"; import type { AilaThreatDetector } from "../features/threatDetection"; import type { AilaAnalyticsFeature, @@ -12,6 +15,7 @@ import type { AilaThreatDetectionFeature, } from "../features/types"; import type { LooseLessonPlan } from "../protocol/schema"; +import type { AilaServices } from "./AilaServices"; import type { Message } from "./chat"; import type { LLMService } from "./llm/LLMService"; import type { AilaPlugin } from "./plugins/types"; @@ -73,5 +77,8 @@ export type AilaInitializationOptions = { chatCategoriser?: AilaCategorisationFeature; chatLlmService?: LLMService; moderationAiClient?: OpenAILike; + ragService?: (aila: AilaServices) => AilaRagFeature; + americanismsService?: (aila: AilaServices) => AilaAmericanismsFeature; + analyticsAdapters?: (aila: AilaServices) => AnalyticsAdapter[]; }; }; diff --git a/packages/aila/src/features/americanisms/AilaAmericanisms.ts b/packages/aila/src/features/americanisms/AilaAmericanisms.ts new file mode 100644 index 000000000..b73444075 --- /dev/null +++ b/packages/aila/src/features/americanisms/AilaAmericanisms.ts @@ -0,0 +1,69 @@ +// eslint-disable-next-line @typescript-eslint/triple-slash-reference +/// +import { textify } from "@oakai/core/src/models/lessonPlans"; +import translator from "american-british-english-translator"; + +import type { AilaAmericanismsFeature } from "."; +import type { + AmericanismIssue, + AmericanismIssueBySection, +} from "../../features/americanisms"; +import type { LessonPlanKeys, LooseLessonPlan } from "../../protocol/schema"; + +export type TranslationResult = Record< + string, + Array<{ [phrase: string]: { issue: string; details: string } }> +>; + +export class AilaAmericanisms implements AilaAmericanismsFeature { + private findSectionAmericanisms( + section: LessonPlanKeys, + lessonPlan: LooseLessonPlan, + ): AmericanismIssueBySection | undefined { + const filterOutPhrases = new Set([ + "practice", + "practices", + "gas", + "gases", + "period", + "periods", + "fall", + "falls", + ]); + + const sectionContent = lessonPlan[section]; + if (!sectionContent) return; + + const sectionText = textify(sectionContent); + const sectionAmericanismScan: TranslationResult = translator.translate( + sectionText, + { american: true }, + ); + + const issues: AmericanismIssue[] = []; + Object.values(sectionAmericanismScan).forEach((lineIssues) => { + lineIssues.forEach((lineIssue) => { + const [phrase, issueDefinition] = Object.entries(lineIssue)[0] ?? []; + if (phrase && issueDefinition && !filterOutPhrases.has(phrase)) { + if (!issues.some((issue) => issue.phrase === phrase)) { + issues.push({ phrase, ...issueDefinition }); + } + } + }); + }); + + return { section, issues }; + } + + public findAmericanisms(lessonPlan: LooseLessonPlan) { + return Object.keys(lessonPlan).flatMap((section) => { + const sectionIssues = this.findSectionAmericanisms( + section as LessonPlanKeys, + lessonPlan, + ); + return sectionIssues && sectionIssues.issues.length > 0 + ? [sectionIssues] + : []; + }); + } +} diff --git a/packages/aila/src/features/americanisms/NullAilaAmericanisms.ts b/packages/aila/src/features/americanisms/NullAilaAmericanisms.ts new file mode 100644 index 000000000..7b5ce8e9a --- /dev/null +++ b/packages/aila/src/features/americanisms/NullAilaAmericanisms.ts @@ -0,0 +1,7 @@ +import type { AilaAmericanismsFeature } from "."; + +export class NullAilaAmericanisms implements AilaAmericanismsFeature { + public findAmericanisms() { + return []; + } +} diff --git a/packages/aila/src/utils/language/american-british-english-translator.d.ts b/packages/aila/src/features/americanisms/american-british-english-translator.d.ts similarity index 100% rename from packages/aila/src/utils/language/american-british-english-translator.d.ts rename to packages/aila/src/features/americanisms/american-british-english-translator.d.ts diff --git a/packages/aila/src/features/americanisms/index.ts b/packages/aila/src/features/americanisms/index.ts new file mode 100644 index 000000000..41004974f --- /dev/null +++ b/packages/aila/src/features/americanisms/index.ts @@ -0,0 +1,16 @@ +import type { LooseLessonPlan } from "../../protocol/schema"; + +export type AmericanismIssueBySection = { + section: string; + issues: AmericanismIssue[]; +}; + +export type AmericanismIssue = { + phrase: string; + issue?: string; + details?: string; +}; + +export interface AilaAmericanismsFeature { + findAmericanisms(lessonPlan: LooseLessonPlan): AmericanismIssueBySection[]; +} diff --git a/packages/aila/src/features/analytics/AilaAnalytics.ts b/packages/aila/src/features/analytics/AilaAnalytics.ts index 778813432..0a28776eb 100644 --- a/packages/aila/src/features/analytics/AilaAnalytics.ts +++ b/packages/aila/src/features/analytics/AilaAnalytics.ts @@ -1,4 +1,4 @@ -import type { AilaServices } from "../../core"; +import type { AilaServices } from "../../core/AilaServices"; import type { AnalyticsAdapter } from "./adapters/AnalyticsAdapter"; export class AilaAnalytics { diff --git a/packages/aila/src/features/analytics/adapters/AnalyticsAdapter.ts b/packages/aila/src/features/analytics/adapters/AnalyticsAdapter.ts index 6b8b677bd..c99612c6b 100644 --- a/packages/aila/src/features/analytics/adapters/AnalyticsAdapter.ts +++ b/packages/aila/src/features/analytics/adapters/AnalyticsAdapter.ts @@ -1,4 +1,4 @@ -import type { AilaServices } from "../../../core"; +import type { AilaServices } from "../../../core/AilaServices"; export abstract class AnalyticsAdapter { protected _aila: AilaServices; diff --git a/packages/aila/src/features/analytics/adapters/PosthogAnalyticsAdapter.ts b/packages/aila/src/features/analytics/adapters/PosthogAnalyticsAdapter.ts index 57a632b20..4b0edb998 100644 --- a/packages/aila/src/features/analytics/adapters/PosthogAnalyticsAdapter.ts +++ b/packages/aila/src/features/analytics/adapters/PosthogAnalyticsAdapter.ts @@ -2,7 +2,7 @@ import { getEncoding } from "js-tiktoken"; import { PostHog } from "posthog-node"; import invariant from "tiny-invariant"; -import type { AilaServices } from "../../../core"; +import type { AilaServices } from "../../../core/AilaServices"; import { reportCompletionAnalyticsEvent } from "../../../lib/openai/OpenAICompletionWithLogging"; import { AnalyticsAdapter } from "./AnalyticsAdapter"; diff --git a/packages/aila/src/features/categorisation/categorisers/AilaCategorisation.ts b/packages/aila/src/features/categorisation/categorisers/AilaCategorisation.ts index 869bcc3ac..4ff724c72 100644 --- a/packages/aila/src/features/categorisation/categorisers/AilaCategorisation.ts +++ b/packages/aila/src/features/categorisation/categorisers/AilaCategorisation.ts @@ -1,33 +1,23 @@ -import { RAG } from "@oakai/core/src/rag"; -import { - type PrismaClientWithAccelerate, - prisma as globalPrisma, -} from "@oakai/db"; +import { CategoriseKeyStageAndSubjectResponse } from "@oakai/core/src/rag"; +import { keyStages, subjects } from "@oakai/core/src/utils/subjects"; +import { aiLogger } from "@oakai/logger"; +import type { ChatCompletionMessageParam } from "openai/resources"; +import { Md5 } from "ts-md5"; -import type { AilaServices, Message } from "../../../core"; +import { DEFAULT_CATEGORISE_MODEL } from "../../../constants"; +import type { AilaServices } from "../../../core/AilaServices"; +import type { Message } from "../../../core/chat"; +import type { OpenAICompletionWithLoggingOptions } from "../../../lib/openai/OpenAICompletionWithLogging"; +import { OpenAICompletionWithLogging } from "../../../lib/openai/OpenAICompletionWithLogging"; import type { LooseLessonPlan } from "../../../protocol/schema"; import type { AilaCategorisationFeature } from "../../types"; +const log = aiLogger("aila:categorisation"); + export class AilaCategorisation implements AilaCategorisationFeature { private _aila: AilaServices; - private _prisma: PrismaClientWithAccelerate; - private _chatId: string; - private _userId: string | undefined; - constructor({ - aila, - prisma, - chatId, - userId, - }: { - aila: AilaServices; - prisma?: PrismaClientWithAccelerate; - chatId: string; - userId?: string; - }) { + constructor({ aila }: { aila: AilaServices }) { this._aila = aila; - this._prisma = prisma ?? globalPrisma; - this._chatId = chatId; - this._userId = userId; } public async categorise( messages: Message[], @@ -39,25 +29,123 @@ export class AilaCategorisation implements AilaCategorisationFeature { .filter((i) => i) .join(" "); - const result = await this.fetchCategorisedInput( - categorisationInput, - this._prisma, - ); + const result = await this.fetchCategorisedInput(categorisationInput); return result; } + private async categoriseKeyStageAndSubject( + input: string, + chatMeta: OpenAICompletionWithLoggingOptions, + ) { + log.info("Categorise input", JSON.stringify(input)); + //# TODO Duplicated for now until we refactor the RAG class + const systemMessage = `You are a classifier which can help me categorise the intent of the user's input for an application which helps a teacher build a lesson plan their students in a UK school. +You accept a string as input and return an object with the keys keyStage, subject, title and topic. + +USER INPUT +The user will likely be starting to make a new lesson plan and this may be their first interaction with the system. So it's likely that the text will include introductory information, such as "Hello, I would like you to make a lesson about {title} for {key stage} students in {subject}." or "I need a lesson plan for {title} for {subject} students in {key stage}." The user may also include a topic for the lesson plan, such as "I need a lesson plan for {title} for {subject} students in {key stage} about {topic}." +The input will be highly variable, so you should be able to handle a wide range of inputs, and extract the relevant information from the input. + +KEY STAGE SLUGS +The following are the key stages you can use to categorise the input. Each of these is slug which we use in the database to refer to them: +${keyStages.join("\n")} + +SUBJECT SLUGS +The following are the subjects you can use to categorise the input. Each of these is a slug which we use in the database to refer to them: +${subjects.join("\n")} + +LESSON TITLES +The title of the lesson plan is the title of the lesson plan that the user wants to create. This could be anything, but it will likely be a short phrase or sentence that describes the topic of the lesson plan. +Do not include "Lesson about" or "…Lesson" in the title. The title should be the standalone main topic of the lesson plan and not mention the word Lesson. It will be used as the title of the lesson plan in our database and displayed to the user in an overview document. + +RETURNED OBJECT +The object you return should have the following shape: +{ + reasoning: string, // Why you have chosen to categorise the input in the way that you have + keyStage: string, // The slug of the key stage that the input is relevant to + subject: string, // The slug of the subject that the input is relevant to + title: string, // The title of the lesson plan + topic: string // The topic of the lesson plan +} + +GUESSING AN APPROPRIATE KEY STAGE, SUBJECT OR TOPIC WHEN NOT SPECIFIED +Where not specified by the user, you should attempt to come up with a reasonable title, key stage, subject and topic based on the input from the user. +For instance, "Plate tectonics" is obviously something covered in Geography and based on your knowledge of the UK education system I'm sure you know that this is often taught in Key Stage 2. +Imagine that you are a teacher who is trying to categorise the input. +You should use your knowledge of the UK education system to make an educated guess about the key stage and subject that the input is relevant to. + +EXAMPLE ALIASES + +Often, teachers will use shorthand to refer to a key stage or subject. For example, "KS3" is often used to refer to "Key Stage 3". You should be able to handle these aliases and return the correct slug for the key stage or subject. +The teacher might also say "Year 10". You should be able to handle this and return the correct slug for the key stage based on the teaching years that are part of the Key Stages in the UK National Curriculum. +For subjects, you should also be able to handle the plural form of the subject. For example, "Maths" should be categorised as "maths" and "Mathematics" should be categorised as "maths". +"PSHE" is often used to refer to "Personal, Social, Health and Economic education" and maps to "psed" in our database. +"PE" is often used to refer to "Physical Education". +"DT" is often used to refer to "Design and Technology". +"RSHE" is often used as a synonym for "PSHE" and "PSED" and maps to "rshe-pshe" in our database. +You should be able to handle any of these aliases and return the correct slug for the subject. +For computing we have both "GCSE" and "non-GCSE". By default, assume that the input is relevant to the GCSE computing curriculum. If the input is relevant to the non-GCSE computing curriculum, the user will specify this in the input. + +PROVIDING REASONING +When key stage, subject and topic are not provided by the user, it may be helpful to write out your reasoning for why you think the input relates to a particular key stage, subject or topic. Start with this reasoning in your response and write out why you think that the input is relevant to the key stage, subject and topic that you have chosen. This will help us to understand your thought process and improve the system in the future. + +BRITISH ENGLISH +The audience for your categorisation is teachers in the UK, so you should use British English when responding to the user. For example, use "Maths" instead of "Math" and "Key Stage 3" instead of "Grade 3". +If the user has provided a title or topic that is in American English, you should still respond with the British English equivalent. For example, if the user has provided the subject "Math" you should respond with "Maths". Or if the user has provided the title "Globalization" you should respond with "Globalisation". + +RESPONDING TO THE USER +All keys are optional but always prefer sending back a keyStage or subject if you are able to take a good guess at what would be appropriate values. If you are *REALLY* not able to determine the key stage or subject, you can return null for these values, but only do so as a last resort! If you are not able to determine the title or topic, you can return null for these values. +Always respond with a valid JSON document. If you are not able to respond for some reason, respond with another valid JSON document with the keys set to null and an "error" key with the value specifying the reason for the error. +Never respond with slugs that are not in the list of slugs provided above. If you are not able to categorise the input, return null for the key stage and subject. This is very important! We use the slugs to categorise the input in our database, so if you return a slug that is not in the list above, we will not be able to categorise the input correctly. +Do not respond with any other output than the object described above. If you do, the system will not be able to understand your response and will not be able to categorise the input correctly. +Thank you and happy classifying!`; + + const promptVersion = Md5.hashStr(systemMessage); + const messages: ChatCompletionMessageParam[] = [ + { role: "system", content: systemMessage }, + { role: "user", content: input }, + ]; + + // #TODO This is the only place where we use this old OpenAICompletionWithLogging + // We should be using methods on the Aila instance instead + const { completion } = await OpenAICompletionWithLogging( + { + ...chatMeta, + promptVersion, + prompt: "categorise_key_stage_and_subject", + }, + { + model: DEFAULT_CATEGORISE_MODEL, + stream: false, + messages, + response_format: { type: "json_object" }, + }, + ); + + try { + const content = completion.choices?.[0]?.message.content; + if (!content) return { error: "No content in response" }; + + const parsedResponse = CategoriseKeyStageAndSubjectResponse.parse( + JSON.parse(content), + ); + log.info("Categorisation results", parsedResponse); + return parsedResponse; + } catch (e) { + return { error: "Error parsing response" }; + } + } + private async fetchCategorisedInput( input: string, - prisma: PrismaClientWithAccelerate, ): Promise { - const rag = new RAG(prisma, { - chatId: this._chatId, - userId: this._userId, - }); - const parsedCategorisation = await rag.categoriseKeyStageAndSubject(input, { - chatId: this._chatId, - userId: this._userId, - }); + const parsedCategorisation = await this.categoriseKeyStageAndSubject( + input, + { + chatId: this._aila.chatId, + userId: this._aila.userId, + }, + ); const { keyStage, subject, title, topic } = parsedCategorisation; const plan: LooseLessonPlan = { keyStage: keyStage ?? undefined, diff --git a/packages/aila/src/features/rag/AilaRag.ts b/packages/aila/src/features/rag/AilaRag.ts index d2b0de9e8..f1ba55851 100644 --- a/packages/aila/src/features/rag/AilaRag.ts +++ b/packages/aila/src/features/rag/AilaRag.ts @@ -3,14 +3,15 @@ import type { PrismaClientWithAccelerate } from "@oakai/db"; import { prisma as globalPrisma } from "@oakai/db"; import { aiLogger } from "@oakai/logger"; -import type { AilaServices } from "../../core"; +import type { AilaRagFeature } from "."; +import type { AilaServices } from "../../core/AilaServices"; import { tryWithErrorReporting } from "../../helpers/errorReporting"; import type { LooseLessonPlan } from "../../protocol/schema"; import { minifyLessonPlanForRelevantLessons } from "../../utils/lessonPlan/minifyLessonPlanForRelevantLessons"; const log = aiLogger("aila:rag"); -export class AilaRag { +export class AilaRag implements AilaRagFeature { private _aila: AilaServices; private _rag: RAG; private _prisma: PrismaClientWithAccelerate; diff --git a/packages/aila/src/features/rag/NullAilaRag.ts b/packages/aila/src/features/rag/NullAilaRag.ts new file mode 100644 index 000000000..8d8606ea5 --- /dev/null +++ b/packages/aila/src/features/rag/NullAilaRag.ts @@ -0,0 +1,7 @@ +import type { AilaRagFeature } from "."; + +export class NullAilaRag implements AilaRagFeature { + public async fetchRagContent() { + return ""; + } +} diff --git a/packages/aila/src/features/rag/index.ts b/packages/aila/src/features/rag/index.ts index 0f6542d26..71cffadc6 100644 --- a/packages/aila/src/features/rag/index.ts +++ b/packages/aila/src/features/rag/index.ts @@ -1 +1,8 @@ -export { AilaRag } from "./AilaRag"; +import type { LooseLessonPlan } from "../../protocol/schema"; + +export interface AilaRagFeature { + fetchRagContent(params: { + numberOfLessonPlansInRag?: number; + lessonPlan?: LooseLessonPlan; + }): Promise; +} diff --git a/packages/aila/src/utils/language/findAmericanisms.ts b/packages/aila/src/utils/language/findAmericanisms.ts deleted file mode 100644 index 78694616b..000000000 --- a/packages/aila/src/utils/language/findAmericanisms.ts +++ /dev/null @@ -1,75 +0,0 @@ -// eslint-disable-next-line @typescript-eslint/triple-slash-reference -/// -import { textify } from "@oakai/core/src/models/lessonPlans"; -import translator from "american-british-english-translator"; - -import type { LessonPlanKeys, LooseLessonPlan } from "../../protocol/schema"; - -export type AmericanismIssueBySection = { - section: string; - issues: AmericanismIssue[]; -}; - -export type AmericanismIssue = { - phrase: string; - issue?: string; - details?: string; -}; - -export type TranslationResult = Record< - string, - Array<{ [phrase: string]: { issue: string; details: string } }> ->; - -const filterOutPhrases = new Set([ - "practice", - "practices", - "gas", - "gases", - "period", - "periods", - "fall", - "falls", -]); - -export function findSectionAmericanisms( - section: LessonPlanKeys, - lessonPlan: LooseLessonPlan, -): AmericanismIssueBySection | undefined { - const sectionContent = lessonPlan[section]; - if (!sectionContent) return; - - const sectionText = textify(sectionContent); - const sectionAmericanismScan: TranslationResult = translator.translate( - sectionText, - { american: true }, - ); - - const issues: AmericanismIssue[] = []; - Object.values(sectionAmericanismScan).forEach((lineIssues) => { - lineIssues.forEach((lineIssue) => { - const [phrase, issueDefinition] = Object.entries(lineIssue)[0] ?? []; - if (phrase && issueDefinition && !filterOutPhrases.has(phrase)) { - if (!issues.some((issue) => issue.phrase === phrase)) { - issues.push({ phrase, ...issueDefinition }); - } - } - }); - }); - - return { section, issues }; -} - -export function findAmericanisms( - lessonPlan: LooseLessonPlan, -): AmericanismIssueBySection[] { - return Object.keys(lessonPlan).flatMap((section) => { - const sectionIssues = findSectionAmericanisms( - section as LessonPlanKeys, - lessonPlan, - ); - return sectionIssues && sectionIssues.issues.length > 0 - ? [sectionIssues] - : []; - }); -} diff --git a/packages/aila/src/utils/lessonPlan/fetchLessonPlanContentById.ts b/packages/aila/src/utils/lessonPlan/fetchLessonPlanContentById.ts index 11f44d50c..6496d98ca 100644 --- a/packages/aila/src/utils/lessonPlan/fetchLessonPlanContentById.ts +++ b/packages/aila/src/utils/lessonPlan/fetchLessonPlanContentById.ts @@ -1,11 +1,8 @@ import type { PrismaClientWithAccelerate } from "@oakai/db"; import { tryWithErrorReporting } from "../../helpers/errorReporting"; -import type { - LooseLessonPlan} from "../../protocol/schema"; -import { - LessonPlanSchemaWhilstStreaming -} from "../../protocol/schema"; +import type { LooseLessonPlan } from "../../protocol/schema"; +import { LessonPlanSchemaWhilstStreaming } from "../../protocol/schema"; export async function fetchLessonPlanContentById( id: string, diff --git a/packages/aila/src/utils/moderation/moderationErrorHandling.ts b/packages/aila/src/utils/moderation/moderationErrorHandling.ts index 2a8428c89..67feb2307 100644 --- a/packages/aila/src/utils/moderation/moderationErrorHandling.ts +++ b/packages/aila/src/utils/moderation/moderationErrorHandling.ts @@ -1,4 +1,4 @@ -import { SafetyViolations as defaultSafetyViolations } from "@oakai/core"; +import { SafetyViolations as defaultSafetyViolations } from "@oakai/core/src/models/safetyViolations"; import { UserBannedError } from "@oakai/core/src/models/safetyViolations"; import type { PrismaClientWithAccelerate } from "@oakai/db"; diff --git a/packages/eslint-config-custom/index.js b/packages/eslint-config-custom/index.js index 51e3a4511..76d707d25 100644 --- a/packages/eslint-config-custom/index.js +++ b/packages/eslint-config-custom/index.js @@ -29,7 +29,7 @@ module.exports = { "no-inner-declarations": "warn", "@typescript-eslint/no-unsafe-enum-comparison": "warn", "@typescript-eslint/no-unnecessary-type-assertion": "warn", - "@typescript-eslint/consistent-type-imports": "error", + "@typescript-eslint/consistent-type-imports": "warn", "@typescript-eslint/comma-dangle": "off", "@typescript-eslint/no-unused-vars": "warn", "@typescript-eslint/require-await": "warn", diff --git a/packages/logger/index.ts b/packages/logger/index.ts index d0eadd815..0b7b57563 100644 --- a/packages/logger/index.ts +++ b/packages/logger/index.ts @@ -16,6 +16,7 @@ type ChildKey = | "admin" | "aila" | "aila:analytics" + | "aila:categorisation" | "aila:errors" | "aila:lesson" | "aila:llm"