Update command results to be the result cards of command executions

packages/ai-bot/helpers.ts

@@ -8,20 +8,22 @@ import type {
   MatrixEvent as DiscreteMatrixEvent,
   CardFragmentContent,
   CommandEvent,
-  CommandResultEvent,
-  ReactionEvent,
   Tool,
   SkillsConfigEvent,
+  CommandResultEvent,
 } from 'https://cardstack.com/base/matrix-event';
 import { MatrixEvent, type IRoomEvent } from 'matrix-js-sdk';
 import { ChatCompletionMessageToolCall } from 'openai/resources/chat/completions';
 import * as Sentry from '@sentry/node';
 import { logger } from '@cardstack/runtime-common';
+import {
+  APP_BOXEL_COMMAND_RESULT_EVENT_TYPE,
+  APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE,
+} from '../runtime-common/matrix-constants';
 import {
   APP_BOXEL_CARDFRAGMENT_MSGTYPE,
   APP_BOXEL_MESSAGE_MSGTYPE,
   APP_BOXEL_COMMAND_MSGTYPE,
-  APP_BOXEL_COMMAND_RESULT_MSGTYPE,
   APP_BOXEL_ROOM_SKILLS_EVENT_TYPE,
 } from '@cardstack/runtime-common/matrix-constants';
 
@@ -140,6 +142,16 @@ export function constructHistory(
       }
     }
     let event = { ...rawEvent } as DiscreteMatrixEvent;
+    if (
+      event.type === APP_BOXEL_COMMAND_RESULT_EVENT_TYPE &&
+      event.content.msgtype == APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE
+    ) {
+      let { cardEventId } = event.content.data;
+      event.content.data.card = serializedCardFromFragments(
+        cardEventId,
+        cardFragments,
+      );
+    }
     if (event.type !== 'm.room.message') {
       continue;
     }
@@ -358,60 +370,20 @@ export function getToolChoice(
   return 'auto';
 }
 
-export function isCommandResultEvent(
-  event: DiscreteMatrixEvent,
-): event is CommandResultEvent {
-  return (
-    event.type === 'm.room.message' &&
-    typeof event.content === 'object' &&
-    event.content.msgtype === APP_BOXEL_COMMAND_RESULT_MSGTYPE
-  );
-}
-
-export function isReactionEvent(
-  event: DiscreteMatrixEvent,
-): event is ReactionEvent {
-  return (
-    event.type === 'm.reaction' &&
-    event.content['m.relates_to'].rel_type === 'm.annotation'
-  );
-}
-
-function getReactionStatus(
-  commandEvent: DiscreteMatrixEvent,
-  history: DiscreteMatrixEvent[],
-) {
-  let maybeReactionEvent = history.find((e) => {
-    if (
-      isReactionEvent(e) &&
-      e.content['m.relates_to']?.event_id === commandEvent.event_id
-    ) {
-      return true;
-    }
-    return false;
-  });
-  return maybeReactionEvent && isReactionEvent(maybeReactionEvent)
-    ? maybeReactionEvent.content['m.relates_to'].key
-    : undefined;
-}
-
 function getCommandResult(
   commandEvent: CommandEvent,
   history: DiscreteMatrixEvent[],
 ) {
-  let maybeCommandResultEvent = history.find((e) => {
+  let commandResultEvent = history.find((e) => {
     if (
       isCommandResultEvent(e) &&
       e.content['m.relates_to']?.event_id === commandEvent.event_id
     ) {
       return true;
     }
     return false;
-  });
-  return maybeCommandResultEvent &&
-    isCommandResultEvent(maybeCommandResultEvent)
-    ? maybeCommandResultEvent.content.result
-    : undefined;
+  }) as CommandResultEvent | undefined;
+  return commandResultEvent;
 }
 
 function toToolCall(event: CommandEvent): ChatCompletionMessageToolCall {
@@ -429,21 +401,26 @@ function toPromptMessageWithToolResult(
   event: CommandEvent,
   history: DiscreteMatrixEvent[],
 ): OpenAIPromptMessage {
-  let commandResult = getCommandResult(event as CommandEvent, history);
+  let commandResult = getCommandResult(event, history);
+  let content = 'pending';
   if (commandResult) {
-    return {
-      role: 'tool',
-      content: commandResult,
-      tool_call_id: event.content.data.toolCall.id,
-    };
-  } else {
-    let reactionStatus = getReactionStatus(event, history);
-    return {
-      role: 'tool',
-      content: reactionStatus ?? 'pending',
-      tool_call_id: event.content.data.toolCall.id,
-    };
+    let status = commandResult.content['m.relates_to']?.key;
+    if (
+      commandResult.content.msgtype ===
+      APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE
+    ) {
+      content = `Command ${status}, with result card: ${JSON.stringify(
+        commandResult.content.data.card,
+      )}.\n`;
+    } else {
+      content = `Command ${status}.\n`;
+    }
   }
+  return {
+    role: 'tool',
+    content,
+    tool_call_id: event.content.data.toolCall.id,
+  };
 }
 
 export function getModifyPrompt(
@@ -570,24 +547,13 @@ export function cleanContent(content: string) {
   return content.trim();
 }
 
-export const isCommandReactionEvent = (event?: MatrixEvent) => {
-  if (event === undefined) {
-    return false;
-  }
-  let content = event.getContent();
-  return (
-    event.getType() === 'm.reaction' &&
-    content['m.relates_to']?.rel_type === 'm.annotation'
-  );
-};
-
-export const isCommandReactionStatusApplied = (event?: MatrixEvent) => {
+export const isCommandResultStatusApplied = (event?: MatrixEvent) => {
   if (event === undefined) {
     return false;
   }
-  let content = event.getContent();
   return (
-    isCommandReactionEvent(event) && content['m.relates_to']?.key === 'applied'
+    isCommandResultEvent(event.event as DiscreteMatrixEvent) &&
+    event.getContent()['m.relates_to']?.key === 'applied'
   );
 };
 
@@ -603,3 +569,15 @@ export function isCommandEvent(
     typeof event.content.data.toolCall === 'object'
   );
 }
+
+export function isCommandResultEvent(
+  event?: DiscreteMatrixEvent,
+): event is CommandResultEvent {
+  if (event === undefined) {
+    return false;
+  }
+  return (
+    event.type === APP_BOXEL_COMMAND_RESULT_EVENT_TYPE &&
+    event.content['m.relates_to']?.rel_type === 'm.annotation'
+  );
+}

packages/ai-bot/lib/set-title.ts

@@ -1,18 +1,23 @@
-import { type MatrixEvent, type IEventRelation } from 'matrix-js-sdk';
+import {
+  type MatrixEvent,
+  type IEventRelation,
+  IRoomEvent,
+} from 'matrix-js-sdk';
 import OpenAI from 'openai';
 import {
   type OpenAIPromptMessage,
-  isCommandReactionStatusApplied,
+  isCommandResultStatusApplied,
   attachedCardsToMessage,
   isCommandEvent,
   getRelevantCards,
 } from '../helpers';
 import { MatrixClient } from './matrix';
 import type { MatrixEvent as DiscreteMatrixEvent } from 'https://cardstack.com/base/matrix-event';
+import { ChatCompletionMessageParam } from 'openai/resources';
 
 const SET_TITLE_SYSTEM_MESSAGE = `You are a chat titling system, you must read the conversation and return a suggested title of no more than six words.
-Do NOT say talk or discussion or discussing or chat or chatting, this is implied by the context. 
-The user can optionally apply 'patchCard' by sending data about fields to update. 
+Do NOT say talk or discussion or discussing or chat or chatting, this is implied by the context.
+The user can optionally apply 'patchCard' by sending data about fields to update.
 Explain the general actions and user intent. If 'patchCard' was used, express the title in an active sentence. Do NOT use the word "patch" in the title.`;
 
 export async function setTitle(
@@ -39,7 +44,7 @@ export async function setTitle(
   let result = await openai.chat.completions.create(
     {
       model: 'gpt-4o',
-      messages: startOfConversation,
+      messages: startOfConversation as ChatCompletionMessageParam[],
       stream: false,
     },
     {
@@ -120,15 +125,15 @@ export const getLatestCommandApplyMessage = (
   return [];
 };
 
-export const roomTitleAlreadySet = (rawEventLog: DiscreteMatrixEvent[]) => {
+export const roomTitleAlreadySet = (rawEventLog: IRoomEvent[]) => {
   return (
     rawEventLog.filter((event) => event.type === 'm.room.name').length > 1 ??
     false
   );
 };
 
 const userAlreadyHasSentNMessages = (
-  rawEventLog: DiscreteMatrixEvent[],
+  rawEventLog: IRoomEvent[],
   botUserId: string,
   n = 5,
 ) => {
@@ -140,12 +145,12 @@ const userAlreadyHasSentNMessages = (
 };
 
 export function shouldSetRoomTitle(
-  rawEventLog: DiscreteMatrixEvent[],
+  rawEventLog: IRoomEvent[],
   aiBotUserId: string,
   event?: MatrixEvent,
 ) {
   return (
-    (isCommandReactionStatusApplied(event) ||
+    (isCommandResultStatusApplied(event) ||
       userAlreadyHasSentNMessages(rawEventLog, aiBotUserId)) &&
     !roomTitleAlreadySet(rawEventLog)
   );

packages/ai-bot/main.ts

@@ -11,7 +11,7 @@ import { logger, aiBotUsername } from '@cardstack/runtime-common';
 import {
   type PromptParts,
   constructHistory,
-  isCommandReactionStatusApplied,
+  isCommandResultStatusApplied,
   getPromptParts,
   extractCardFragmentsFromEvents,
 } from './helpers';
@@ -30,6 +30,8 @@ import * as Sentry from '@sentry/node';
 
 import { getAvailableCredits, saveUsageCost } from './lib/ai-billing';
 import { PgAdapter } from '@cardstack/postgres';
+import { ChatCompletionMessageParam } from 'openai/resources';
+import { OpenAIError } from 'openai/error';
 
 let log = logger('ai-bot');
 
@@ -69,12 +71,12 @@ class Assistant {
     if (prompt.tools.length === 0) {
       return this.openai.beta.chat.completions.stream({
         model: prompt.model,
-        messages: prompt.messages,
+        messages: prompt.messages as ChatCompletionMessageParam[],
       });
     } else {
       return this.openai.beta.chat.completions.stream({
         model: prompt.model,
-        messages: prompt.messages,
+        messages: prompt.messages as ChatCompletionMessageParam[],
         tools: prompt.tools,
         tool_choice: prompt.toolChoice,
       });
@@ -250,7 +252,7 @@ Common issues are:
           finalContent = await runner.finalContent();
           await responder.finalize(finalContent);
         } catch (error) {
-          await responder.onError(error);
+          await responder.onError(error as OpenAIError);
         } finally {
           if (generationId) {
             assistant.trackAiUsageCost(senderMatrixUserId, generationId);
@@ -278,7 +280,7 @@ Common issues are:
     if (!room) {
       return;
     }
-    if (!isCommandReactionStatusApplied(event)) {
+    if (!isCommandResultStatusApplied(event)) {
       return;
     }
     log.info(

packages/ai-bot/package.json

@@ -18,6 +18,7 @@
   },
   "devDependencies": {
     "@sinonjs/fake-timers": "^11.2.2",
+    "@types/qunit": "^2.19.12",
     "@types/sinonjs__fake-timers": "^8.1.5",
     "qunit": "^2.18.0"
   },