Merge branch 'main' into cs-7952-command-button-state-to-indicate-whe…

…n-command-is-being

packages/ai-bot/helpers.ts

@@ -19,18 +19,17 @@ import { MatrixEvent, type IRoomEvent } from 'matrix-js-sdk';
 import { ChatCompletionMessageToolCall } from 'openai/resources/chat/completions';
 import * as Sentry from '@sentry/node';
 import { logger } from '@cardstack/runtime-common';
-import {
-  APP_BOXEL_COMMAND_REQUESTS_KEY,
-  APP_BOXEL_COMMAND_DEFINITIONS_MSGTYPE,
-  APP_BOXEL_COMMAND_RESULT_EVENT_TYPE,
-  APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE,
-} from '../runtime-common/matrix-constants';
 import {
   APP_BOXEL_CARDFRAGMENT_MSGTYPE,
   APP_BOXEL_MESSAGE_MSGTYPE,
   APP_BOXEL_ROOM_SKILLS_EVENT_TYPE,
   DEFAULT_LLM,
   APP_BOXEL_ACTIVE_LLM,
+  APP_BOXEL_COMMAND_REQUESTS_KEY,
+  APP_BOXEL_COMMAND_DEFINITIONS_MSGTYPE,
+  APP_BOXEL_COMMAND_RESULT_EVENT_TYPE,
+  APP_BOXEL_COMMAND_RESULT_REL_TYPE,
+  APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE,
 } from '@cardstack/runtime-common/matrix-constants';
 
 let log = logger('ai-bot');
@@ -653,7 +652,7 @@ export async function getModifyPrompt(
       continue;
     }
     if (isCommandResultEvent(event)) {
-      continue;
+      continue; // we'll include these with the tool calls
     }
     if (
       'isStreamingFinished' in event.content &&
@@ -662,40 +661,39 @@ export async function getModifyPrompt(
       continue;
     }
     let body = event.content.body;
-    if (body) {
-      if (event.sender === aiBotUserId) {
-        let toolCalls = toToolCalls(event);
-        let historicalMessage: OpenAIPromptMessage = {
-          role: 'assistant',
-          content: body,
-        };
-        if (toolCalls.length) {
-          historicalMessage.tool_calls = toolCalls;
-        }
-        historicalMessages.push(historicalMessage);
-        if (toolCalls.length) {
-          toPromptMessageWithToolResults(event, history).forEach((message) =>
-            historicalMessages.push(message),
-          );
-        }
-      } else {
-        if (
-          event.content.msgtype === APP_BOXEL_MESSAGE_MSGTYPE &&
-          event.content.data?.context?.openCardIds
-        ) {
-          body = `User message: ${body}
+    if (event.sender === aiBotUserId) {
+      let toolCalls = toToolCalls(event);
+      let historicalMessage: OpenAIPromptMessage = {
+        role: 'assistant',
+        content: body,
+      };
+      if (toolCalls.length) {
+        historicalMessage.tool_calls = toolCalls;
+      }
+      historicalMessages.push(historicalMessage);
+      if (toolCalls.length) {
+        toPromptMessageWithToolResults(event, history).forEach((message) =>
+          historicalMessages.push(message),
+        );
+      }
+    }
+    if (body && event.sender !== aiBotUserId) {
+      if (
+        event.content.msgtype === APP_BOXEL_MESSAGE_MSGTYPE &&
+        event.content.data?.context?.openCardIds
+      ) {
+        body = `User message: ${body}
           Context: the user has the following cards open: ${JSON.stringify(
             event.content.data.context.openCardIds,
           )}`;
-        } else {
-          body = `User message: ${body}
+      } else {
+        body = `User message: ${body}
           Context: the user has no open cards.`;
-        }
-        historicalMessages.push({
-          role: 'user',
-          content: body,
-        });
       }
+      historicalMessages.push({
+        role: 'user',
+        content: body,
+      });
     }
   }
 
@@ -794,30 +792,7 @@ export function isCommandResultEvent(
   }
   return (
     event.type === APP_BOXEL_COMMAND_RESULT_EVENT_TYPE &&
-    event.content['m.relates_to']?.rel_type === 'm.annotation'
+    event.content['m.relates_to']?.rel_type ===
+      APP_BOXEL_COMMAND_RESULT_REL_TYPE
   );
 }
-
-export function eventRequiresResponse(event: MatrixEvent) {
-  // If it's a message, we should respond unless it's a card fragment
-  if (event.getType() === 'm.room.message') {
-    if (
-      event.getContent().msgtype === APP_BOXEL_CARDFRAGMENT_MSGTYPE ||
-      event.getContent().msgtype === APP_BOXEL_COMMAND_DEFINITIONS_MSGTYPE
-    ) {
-      return false;
-    }
-    return true;
-  }
-
-  // If it's a command result with output, we should respond
-  if (
-    event.getType() === APP_BOXEL_COMMAND_RESULT_EVENT_TYPE &&
-    event.getContent().msgtype === APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE
-  ) {
-    return true;
-  }
-
-  // If it's a different type, or a command result without output, we should not respond
-  return false;
-}

packages/ai-bot/lib/responder.ts

@@ -12,11 +12,47 @@ import { CommandRequest } from '@cardstack/runtime-common/commands';
 import { thinkingMessage } from '../constants';
 import type OpenAI from 'openai';
 import type { ChatCompletionSnapshot } from 'openai/lib/ChatCompletionStream';
+import {
+  APP_BOXEL_CARDFRAGMENT_MSGTYPE,
+  APP_BOXEL_COMMAND_DEFINITIONS_MSGTYPE,
+  APP_BOXEL_COMMAND_RESULT_EVENT_TYPE,
+  APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE,
+} from '@cardstack/runtime-common/matrix-constants';
 
 let log = logger('ai-bot');
 
 export class Responder {
-  // internally has a debounced function that will send the matrix messages
+  static eventMayTriggerResponse(event: DiscreteMatrixEvent) {
+    // If it's a message, we should respond unless it's a card fragment
+    if (event.getType() === 'm.room.message') {
+      if (
+        event.getContent().msgtype === APP_BOXEL_CARDFRAGMENT_MSGTYPE ||
+        event.getContent().msgtype === APP_BOXEL_COMMAND_DEFINITIONS_MSGTYPE
+      ) {
+        return false;
+      }
+      return true;
+    }
+
+    // If it's a command result with output, we might respond
+    if (
+      event.getType() === APP_BOXEL_COMMAND_RESULT_EVENT_TYPE &&
+      event.getContent().msgtype ===
+        APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE
+    ) {
+      return true;
+    }
+
+    // If it's a different type, or a command result without output, we should not respond
+    return false;
+  }
+
+  static eventWillDefinitelyTriggerResponse(event: DiscreteMatrixEvent) {
+    return (
+      this.eventMayTriggerResponse(event) &&
+      event.getType() !== APP_BOXEL_COMMAND_RESULT_EVENT_TYPE
+    );
+  }
 
   constructor(
     readonly client: MatrixClient,
@@ -25,6 +61,7 @@ export class Responder {
 
   messagePromises: Promise<ISendEventResponse | void>[] = [];
 
+  initialMessageSent = false;
   responseEventId: string | undefined;
   latestContent = '';
   toolCalls: ChatCompletionSnapshot.Choice.Message.ToolCall[] = [];
@@ -58,15 +95,18 @@ export class Responder {
     await messagePromise;
   };
 
-  async initialize() {
-    let initialMessage = await sendMessageEvent(
-      this.client,
-      this.roomId,
-      thinkingMessage,
-      undefined,
-      { isStreamingFinished: false },
-    );
-    this.responseEventId = initialMessage.event_id;
+  async ensureThinkingMessageSent() {
+    if (!this.initialMessageSent) {
+      let initialMessage = await sendMessageEvent(
+        this.client,
+        this.roomId,
+        thinkingMessage,
+        undefined,
+        { isStreamingFinished: false },
+      );
+      this.responseEventId = initialMessage.event_id;
+      this.initialMessageSent = true;
+    }
   }
 
   async onChunk(

packages/ai-bot/main.ts

@@ -9,7 +9,6 @@ import {
   isCommandResultStatusApplied,
   getPromptParts,
   extractCardFragmentsFromEvents,
-  eventRequiresResponse,
 } from './helpers';
 
 import {
@@ -188,13 +187,15 @@ Common issues are:
         if (toStartOfTimeline) {
           return; // don't print paginated results
         }
-        if (!eventRequiresResponse(event)) {
-          return; // only print messages
-        }
 
         if (senderMatrixUserId === aiBotUserId) {
           return;
         }
+
+        if (!Responder.eventMayTriggerResponse(event)) {
+          return; // early exit for events that will not trigger a response
+        }
+
         log.info(
           '(%s) (Room: "%s" %s) (Message: %s %s)',
           event.getType(),
@@ -205,7 +206,10 @@ Common issues are:
         );
 
         const responder = new Responder(client, room.roomId);
-        await responder.initialize();
+
+        if (Responder.eventWillDefinitelyTriggerResponse(event)) {
+          await responder.ensureThinkingMessageSent();
+        }
 
         let promptParts: PromptParts;
         let initial = await client.roomInitialSync(room!.roomId, 1000);
@@ -216,6 +220,7 @@ Common issues are:
           if (!promptParts.shouldRespond) {
             return;
           }
+          await responder.ensureThinkingMessageSent();
         } catch (e) {
           log.error(e);
           await responder.onError(

packages/ai-bot/tests/chat-titling-test.ts

@@ -4,6 +4,7 @@ import type { MatrixEvent as DiscreteMatrixEvent } from 'https://cardstack.com/b
 import {
   APP_BOXEL_COMMAND_REQUESTS_KEY,
   APP_BOXEL_COMMAND_RESULT_EVENT_TYPE,
+  APP_BOXEL_COMMAND_RESULT_REL_TYPE,
   APP_BOXEL_COMMAND_RESULT_WITH_NO_OUTPUT_MSGTYPE,
   APP_BOXEL_MESSAGE_MSGTYPE,
 } from '@cardstack/runtime-common/matrix-constants';
@@ -416,7 +417,7 @@ module('shouldSetRoomTitle', () => {
         'm.relates_to': {
           event_id: '1',
           key: 'applied',
-          rel_type: 'm.annotation',
+          rel_type: APP_BOXEL_COMMAND_RESULT_REL_TYPE,
         },
         msgtype: APP_BOXEL_COMMAND_RESULT_WITH_NO_OUTPUT_MSGTYPE,
       },

packages/ai-bot/tests/prompt-construction-test.ts

@@ -15,6 +15,7 @@ import {
   APP_BOXEL_MESSAGE_MSGTYPE,
   APP_BOXEL_COMMAND_RESULT_EVENT_TYPE,
   APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE,
+  APP_BOXEL_COMMAND_RESULT_REL_TYPE,
   DEFAULT_LLM,
   APP_BOXEL_COMMAND_REQUESTS_KEY,
 } from '@cardstack/runtime-common/matrix-constants';
@@ -1678,7 +1679,7 @@ test('Return host result of tool call back to open ai', async () => {
       content: {
         'm.relates_to': {
           event_id: 'command-event-id-1',
-          rel_type: 'm.annotation',
+          rel_type: APP_BOXEL_COMMAND_RESULT_REL_TYPE,
           key: 'applied',
         },
         msgtype: APP_BOXEL_COMMAND_RESULT_WITH_OUTPUT_MSGTYPE,

packages/ai-bot/tests/resources/chats/connect-tool-calls-to-results.json

@@ -420,7 +420,7 @@
       "m.relates_to": {
         "event_id": "$VyCYG0DAuudwMGN8hStuSVfXrjMVZu2ZcJgIs9sIwLE",
         "key": "applied",
-        "rel_type": "m.annotation"
+        "rel_type": "app.boxel.commandAnnotation"
       },
       "commandRequestId": "call_or0qFTa8y9Ef8ZVyRDNR2P1T",
       "data": "{\"cardEventId\":\"$dKXYGWZLmS42Yib6sDHwP1surO2PiGCr5t2XHFIGFmc\"}"

packages/ai-bot/tests/resources/chats/required-tools-multiple-messages.json

@@ -355,7 +355,7 @@
       "m.relates_to": {
         "event_id": "$pZ5t1hj93dVwVcrh60-XFNeSL4ztErK6J0wAYr9hI_0",
         "key": "applied",
-        "rel_type": "m.annotation"
+        "rel_type": "app.boxel.commandAnnotation"
       }
     },
     "origin_server_ts": 1734900014044,
@@ -674,7 +674,7 @@
       "m.relates_to": {
         "event_id": "$BQ3wE7TxtE38oid1xynRFTglU2_2P5jJ8k01uXrHj1U",
         "key": "applied",
-        "rel_type": "m.annotation"
+        "rel_type": "app.boxel.commandAnnotation"
       }
     },
     "origin_server_ts": 1734900100494,

packages/ai-bot/tests/resources/chats/two-tool-calls-one-result.json

@@ -428,7 +428,7 @@
       "m.relates_to": {
         "event_id": "$VyCYG0DAuudwMGN8hStuSVfXrjMVZu2ZcJgIs9sIwLE",
         "key": "applied",
-        "rel_type": "m.annotation"
+        "rel_type": "app.boxel.commandAnnotation"
       },
       "commandRequestId": "call_or0qFTa8y9Ef8ZVyRDNR2P1T",
       "data": "{\"cardEventId\":\"$dKXYGWZLmS42Yib6sDHwP1surO2PiGCr5t2XHFIGFmc\"}"

packages/ai-bot/tests/resources/chats/two-tool-calls-two-results.json

@@ -428,7 +428,7 @@
       "m.relates_to": {
         "event_id": "$VyCYG0DAuudwMGN8hStuSVfXrjMVZu2ZcJgIs9sIwLE",
         "key": "applied",
-        "rel_type": "m.annotation"
+        "rel_type": "app.boxel.commandAnnotation"
       },
       "commandRequestId": "call_or0qFTa8y9Ef8ZVyRDNR2P1T",
       "data": "{\"cardEventId\":\"$dKXYGWZLmS42Yib6sDHwP1surO2PiGCr5t2XHFIGFmc\"}"
@@ -469,7 +469,7 @@
       "m.relates_to": {
         "event_id": "$VyCYG0DAuudwMGN8hStuSVfXrjMVZu2ZcJgIs9sIwLE",
         "key": "applied",
-        "rel_type": "m.annotation"
+        "rel_type": "app.boxel.commandAnnotation"
       },
       "commandRequestId": "call_or1e5Ta8y9Ef8ZVyRDNRabcd",
       "data": "{\"cardEventId\":\"$aBcYGWZLmS42Yib6sDHwP1surO2PiGCr5t2XHFIGF12\"}"