Merge main

apps/nextjs/src/app/api/chat/chatHandler.ts

@@ -3,8 +3,15 @@ import type {
   AilaInitializationOptions,
   AilaOptions,
   AilaPublicChatOptions,
+  AilaServices,
   Message,
 } from "@oakai/aila";
+import { AilaAmericanisms } from "@oakai/aila/src/features/americanisms/AilaAmericanisms";
+import {
+  DatadogAnalyticsAdapter,
+  PosthogAnalyticsAdapter,
+} from "@oakai/aila/src/features/analytics";
+import { AilaRag } from "@oakai/aila/src/features/rag/AilaRag";
 import { LooseLessonPlan } from "@oakai/aila/src/protocol/schema";
 import {
   type TracingSpan,
@@ -177,6 +184,12 @@ export async function handleChatPostRequest(
             services: {
               chatLlmService: llmService,
               moderationAiClient,
+              ragService: (aila: AilaServices) => new AilaRag({ aila }),
+              americanismsService: () => new AilaAmericanisms(),
+              analyticsAdapters: (aila: AilaServices) => [
+                new PosthogAnalyticsAdapter(aila),
+                new DatadogAnalyticsAdapter(aila),
+              ],
             },
             lessonPlan: lessonPlan ?? {},
           };

packages/aila/src/core/Aila.ts

@@ -1,12 +1,17 @@
 import type { PrismaClientWithAccelerate } from "@oakai/db";
 import { prisma as globalPrisma } from "@oakai/db/client";
+import { aiLogger } from "@oakai/logger";
 
 import {
   DEFAULT_MODEL,
   DEFAULT_TEMPERATURE,
   DEFAULT_RAG_LESSON_PLANS,
 } from "../constants";
+import type { AilaAmericanismsFeature } from "../features/americanisms";
+import { NullAilaAmericanisms } from "../features/americanisms/NullAilaAmericanisms";
 import { AilaCategorisation } from "../features/categorisation";
+import type { AilaRagFeature } from "../features/rag";
+import { NullAilaRag } from "../features/rag/NullAilaRag";
 import type { AilaSnapshotStore } from "../features/snapshotStore";
 import type {
   AilaAnalyticsFeature,
@@ -50,9 +55,11 @@ export class Aila implements AilaServices {
   private _persistence: AilaPersistenceFeature[] = [];
   private _threatDetection?: AilaThreatDetectionFeature;
   private _prisma: PrismaClientWithAccelerate;
+  private _rag: AilaRagFeature;
   private _plugins: AilaPlugin[];
   private _userId!: string | undefined;
   private _chatId!: string;
+  private _americanisms: AilaAmericanismsFeature;
 
   constructor(options: AilaInitializationOptions) {
     this._userId = options.chat.userId;
@@ -78,13 +85,14 @@ export class Aila implements AilaServices {
         options.services?.chatCategoriser ??
         new AilaCategorisation({
           aila: this,
-          prisma: this._prisma,
-          chatId: this._chatId,
-          userId: this._userId,
         }),
     });
 
-    this._analytics = AilaFeatureFactory.createAnalytics(this, this._options);
+    this._analytics = AilaFeatureFactory.createAnalytics(
+      this,
+      this._options,
+      options.services?.analyticsAdapters?.(this),
+    );
     this._moderation = AilaFeatureFactory.createModeration(
       this,
       this._options,
@@ -103,6 +111,10 @@ export class Aila implements AilaServices {
       this,
       this._options,
     );
+    this._rag = options.services?.ragService?.(this) ?? new NullAilaRag();
+    this._americanisms =
+      options.services?.americanismsService?.(this) ??
+      new NullAilaAmericanisms();
 
     if (this._analytics) {
       this._analytics.initialiseAnalyticsContext();
@@ -211,6 +223,18 @@ export class Aila implements AilaServices {
     return this._chatLlmService;
   }
 
+  public get rag() {
+    return this._rag;
+  }
+
+  public get americanisms() {
+    return this._americanisms;
+  }
+
+  public get prisma() {
+    return this._prisma;
+  }
+
   // Check methods
   public checkUserIdPresentIfPersisting() {
     if (!this._chat.userId && this._options.usePersistence) {

packages/aila/src/core/AilaFeatureFactory.ts

@@ -1,16 +1,10 @@
 // AilaFeatureFactory.ts
-import {
-  DatadogAnalyticsAdapter,
-  PosthogAnalyticsAdapter,
-} from "../features/analytics";
+import type { AnalyticsAdapter } from "../features/analytics";
 import { AilaAnalytics } from "../features/analytics/AilaAnalytics";
 import { SentryErrorReporter } from "../features/errorReporting/reporters/SentryErrorReporter";
 import { AilaModeration } from "../features/moderation";
-import type {
-  OpenAILike} from "../features/moderation/moderators/OpenAiModerator";
-import {
-  OpenAiModerator,
-} from "../features/moderation/moderators/OpenAiModerator";
+import type { OpenAILike } from "../features/moderation/moderators/OpenAiModerator";
+import { OpenAiModerator } from "../features/moderation/moderators/OpenAiModerator";
 import { AilaPrismaPersistence } from "../features/persistence/adaptors/prisma";
 import { AilaSnapshotStore } from "../features/snapshotStore";
 import { AilaThreatDetection } from "../features/threatDetection";
@@ -28,14 +22,12 @@ export class AilaFeatureFactory {
   static createAnalytics(
     aila: AilaServices,
     options: AilaOptions,
+    adapters: AnalyticsAdapter[] = [],
   ): AilaAnalyticsFeature | undefined {
     if (options.useAnalytics) {
       return new AilaAnalytics({
         aila,
-        adapters: [
-          new PosthogAnalyticsAdapter(aila),
-          new DatadogAnalyticsAdapter(aila),
-        ],
+        adapters,
       });
     }
     return undefined;

packages/aila/src/core/AilaServices.ts

@@ -1,5 +1,7 @@
+import type { AilaAmericanismsFeature } from "../features/americanisms";
 import type { AilaAnalytics } from "../features/analytics/AilaAnalytics";
 import type { AilaErrorReporter } from "../features/errorReporting";
+import type { AilaRagFeature } from "../features/rag";
 import type { AilaSnapshotStore } from "../features/snapshotStore";
 import type {
   AilaAnalyticsFeature,
@@ -14,8 +16,8 @@ import type {
   LooseLessonPlan,
 } from "../protocol/schema";
 import type { Message } from "./chat";
-import type { AilaOptionsWithDefaultFallbackValues } from "./index";
 import type { AilaPlugin } from "./plugins";
+import type { AilaOptionsWithDefaultFallbackValues } from "./types";
 
 // This provides a set of interfaces between the Aila core and the features that use it.
 // We can then mock these out in tests without needing to instantiate the entire Aila object.
@@ -64,4 +66,6 @@ export interface AilaServices {
   readonly persistence?: AilaPersistenceFeature[];
   readonly moderation?: AilaModerationFeature;
   readonly plugins: AilaPlugin[];
+  readonly rag: AilaRagFeature;
+  readonly americanisms: AilaAmericanismsFeature;
 }

packages/aila/src/core/lesson/AilaLesson.ts

@@ -3,8 +3,7 @@ import { deepClone } from "fast-json-patch";
 
 import { AilaCategorisation } from "../../features/categorisation/categorisers/AilaCategorisation";
 import type { AilaCategorisationFeature } from "../../features/types";
-import type {
-  PatchDocument} from "../../protocol/jsonPatchProtocol";
+import type { PatchDocument } from "../../protocol/jsonPatchProtocol";
 import {
   applyLessonPlanPatch,
   extractPatches,
@@ -39,8 +38,6 @@ export class AilaLesson implements AilaLessonService {
       categoriser ??
       new AilaCategorisation({
         aila,
-        userId: aila.userId,
-        chatId: aila.chatId,
       });
   }
 

packages/aila/src/core/prompt/builders/AilaLessonPromptBuilder.ts

@@ -1,27 +1,17 @@
-import type {
-  TemplateProps} from "@oakai/core/src/prompts/lesson-assistant";
-import {
-  template,
-} from "@oakai/core/src/prompts/lesson-assistant";
+import type { TemplateProps } from "@oakai/core/src/prompts/lesson-assistant";
+import { template } from "@oakai/core/src/prompts/lesson-assistant";
 import { prisma as globalPrisma } from "@oakai/db";
 import { aiLogger } from "@oakai/logger";
 
 import { DEFAULT_RAG_LESSON_PLANS } from "../../../constants";
 import { tryWithErrorReporting } from "../../../helpers/errorReporting";
 import { LLMResponseJsonSchema } from "../../../protocol/jsonPatchProtocol";
-import type {
-  LooseLessonPlan} from "../../../protocol/schema";
-import {
-  LessonPlanJsonSchema
-} from "../../../protocol/schema";
-import { findAmericanisms } from "../../../utils/language/findAmericanisms";
+import type { LooseLessonPlan } from "../../../protocol/schema";
+import { LessonPlanJsonSchema } from "../../../protocol/schema";
 import { compressedLessonPlanForRag } from "../../../utils/lessonPlan/compressedLessonPlanForRag";
 import { fetchLessonPlan } from "../../../utils/lessonPlan/fetchLessonPlan";
-import type {
-  RagLessonPlan} from "../../../utils/rag/fetchRagContent";
-import {
-  fetchRagContent
-} from "../../../utils/rag/fetchRagContent";
+import type { RagLessonPlan } from "../../../utils/rag/fetchRagContent";
+import { fetchRagContent } from "../../../utils/rag/fetchRagContent";
 import type { AilaServices } from "../../AilaServices";
 import { AilaPromptBuilder } from "../AilaPromptBuilder";
 
@@ -117,7 +107,7 @@ export class AilaLessonPromptBuilder extends AilaPromptBuilder {
       summaries: "None",
       responseMode: this._aila?.options.mode ?? "interactive",
       useRag: this._aila?.options.useRag ?? true,
-      americanisms: findAmericanisms(lessonPlan),
+      americanisms: this._aila.americanisms.findAmericanisms(lessonPlan),
       baseLessonPlan: baseLessonPlan
         ? compressedLessonPlanForRag(baseLessonPlan)
         : undefined,

packages/aila/src/core/types.ts

@@ -1,8 +1,11 @@
 import type { PrismaClientWithAccelerate } from "@oakai/db";
 
+import type { AilaAmericanismsFeature } from "../features/americanisms";
+import type { AnalyticsAdapter } from "../features/analytics";
 import type { AilaModerator } from "../features/moderation/moderators";
 import type { OpenAILike } from "../features/moderation/moderators/OpenAiModerator";
 import type { AilaPersistence } from "../features/persistence";
+import type { AilaRagFeature } from "../features/rag";
 import type { AilaThreatDetector } from "../features/threatDetection";
 import type {
   AilaAnalyticsFeature,
@@ -12,6 +15,7 @@ import type {
   AilaThreatDetectionFeature,
 } from "../features/types";
 import type { LooseLessonPlan } from "../protocol/schema";
+import type { AilaServices } from "./AilaServices";
 import type { Message } from "./chat";
 import type { LLMService } from "./llm/LLMService";
 import type { AilaPlugin } from "./plugins/types";
@@ -73,5 +77,8 @@ export type AilaInitializationOptions = {
     chatCategoriser?: AilaCategorisationFeature;
     chatLlmService?: LLMService;
     moderationAiClient?: OpenAILike;
+    ragService?: (aila: AilaServices) => AilaRagFeature;
+    americanismsService?: (aila: AilaServices) => AilaAmericanismsFeature;
+    analyticsAdapters?: (aila: AilaServices) => AnalyticsAdapter[];
   };
 };

packages/aila/src/features/americanisms/AilaAmericanisms.ts

@@ -0,0 +1,69 @@
+// eslint-disable-next-line @typescript-eslint/triple-slash-reference
+/// <reference path="./american-british-english-translator.d.ts" />
+import { textify } from "@oakai/core/src/models/lessonPlans";
+import translator from "american-british-english-translator";
+
+import type { AilaAmericanismsFeature } from ".";
+import type {
+  AmericanismIssue,
+  AmericanismIssueBySection,
+} from "../../features/americanisms";
+import type { LessonPlanKeys, LooseLessonPlan } from "../../protocol/schema";
+
+export type TranslationResult = Record<
+  string,
+  Array<{ [phrase: string]: { issue: string; details: string } }>
+>;
+
+export class AilaAmericanisms implements AilaAmericanismsFeature {
+  private findSectionAmericanisms(
+    section: LessonPlanKeys,
+    lessonPlan: LooseLessonPlan,
+  ): AmericanismIssueBySection | undefined {
+    const filterOutPhrases = new Set([
+      "practice",
+      "practices",
+      "gas",
+      "gases",
+      "period",
+      "periods",
+      "fall",
+      "falls",
+    ]);
+
+    const sectionContent = lessonPlan[section];
+    if (!sectionContent) return;
+
+    const sectionText = textify(sectionContent);
+    const sectionAmericanismScan: TranslationResult = translator.translate(
+      sectionText,
+      { american: true },
+    );
+
+    const issues: AmericanismIssue[] = [];
+    Object.values(sectionAmericanismScan).forEach((lineIssues) => {
+      lineIssues.forEach((lineIssue) => {
+        const [phrase, issueDefinition] = Object.entries(lineIssue)[0] ?? [];
+        if (phrase && issueDefinition && !filterOutPhrases.has(phrase)) {
+          if (!issues.some((issue) => issue.phrase === phrase)) {
+            issues.push({ phrase, ...issueDefinition });
+          }
+        }
+      });
+    });
+
+    return { section, issues };
+  }
+
+  public findAmericanisms(lessonPlan: LooseLessonPlan) {
+    return Object.keys(lessonPlan).flatMap((section) => {
+      const sectionIssues = this.findSectionAmericanisms(
+        section as LessonPlanKeys,
+        lessonPlan,
+      );
+      return sectionIssues && sectionIssues.issues.length > 0
+        ? [sectionIssues]
+        : [];
+    });
+  }
+}

packages/aila/src/features/americanisms/NullAilaAmericanisms.ts

@@ -0,0 +1,7 @@
+import type { AilaAmericanismsFeature } from ".";
+
+export class NullAilaAmericanisms implements AilaAmericanismsFeature {
+  public findAmericanisms() {
+    return [];
+  }
+}

packages/aila/src/features/americanisms/index.ts

@@ -0,0 +1,16 @@
+import type { LooseLessonPlan } from "../../protocol/schema";
+
+export type AmericanismIssueBySection = {
+  section: string;
+  issues: AmericanismIssue[];
+};
+
+export type AmericanismIssue = {
+  phrase: string;
+  issue?: string;
+  details?: string;
+};
+
+export interface AilaAmericanismsFeature {
+  findAmericanisms(lessonPlan: LooseLessonPlan): AmericanismIssueBySection[];
+}

packages/aila/src/features/analytics/AilaAnalytics.ts

@@ -1,4 +1,4 @@
-import type { AilaServices } from "../../core";
+import type { AilaServices } from "../../core/AilaServices";
 import type { AnalyticsAdapter } from "./adapters/AnalyticsAdapter";
 
 export class AilaAnalytics {

packages/aila/src/features/analytics/adapters/AnalyticsAdapter.ts

@@ -1,4 +1,4 @@
-import type { AilaServices } from "../../../core";
+import type { AilaServices } from "../../../core/AilaServices";
 
 export abstract class AnalyticsAdapter {
   protected _aila: AilaServices;

packages/aila/src/features/analytics/adapters/PosthogAnalyticsAdapter.ts

@@ -2,7 +2,7 @@ import { getEncoding } from "js-tiktoken";
 import { PostHog } from "posthog-node";
 import invariant from "tiny-invariant";
 
-import type { AilaServices } from "../../../core";
+import type { AilaServices } from "../../../core/AilaServices";
 import { reportCompletionAnalyticsEvent } from "../../../lib/openai/OpenAICompletionWithLogging";
 import { AnalyticsAdapter } from "./AnalyticsAdapter";