feat: approval flow, detecting when input is needed

crates/goose-server/src/routes/reply.rs

@@ -234,8 +234,12 @@ async fn stream_message(
                         }
                     }
                     MessageContent::Text(text) => {
-                        for line in text.text.lines() {
-                            tx.send(ProtocolFormatter::format_text(&format!("{}\\n", line))).await?;
+                            for line in text.text.lines() {
+                            // Append a literal '\n' to each line
+
+                            // Manually escape backslashes for transmission
+                            let escaped_line = line.replace("\\", "\\\\");
+                            tx.send(ProtocolFormatter::format_text(&format!("{}\\n", escaped_line))).await?;
                         }
                     }
                     MessageContent::Image(_) => {
@@ -314,9 +318,71 @@ async fn handler(
     Ok(SseResponse::new(stream))
 }
 
+
+
+#[derive(Debug, Deserialize)]
+struct AskRequest {
+    prompt: String,
+}
+
+#[derive(Debug, serde::Serialize)]
+struct AskResponse {
+    response: String,
+}
+
+
+// simple ask an AI for a response, non streaming
+async fn ask_handler(
+    State(state): State<AppState>,
+    Json(request): Json<AskRequest>,
+) -> Result<Json<AskResponse>, StatusCode> {
+
+    let provider = factory::get_provider(state.provider_config)
+        .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
+
+    let agent = Agent::new(provider);
+
+    // Create a single message for the prompt
+    let messages = vec![Message::user().with_text(request.prompt)];
+
+    // Get response from agent
+    let mut response_text = String::new();
+    let mut stream = match agent.reply(&messages).await {
+        Ok(stream) => stream,
+        Err(e) => {
+            tracing::error!("Failed to start reply stream: {}", e);
+            return Err(StatusCode::INTERNAL_SERVER_ERROR);
+        }
+    };
+
+    while let Some(response) = stream.next().await {
+        match response {
+            Ok(message) => {
+                if message.role == Role::Assistant {
+                    for content in message.content {
+                        if let MessageContent::Text(text) = content {
+                            response_text.push_str(&text.text);
+                            response_text.push('\n');
+                        }
+                    }
+                }
+            }
+            Err(e) => {
+                tracing::error!("Error processing message: {}", e);
+                return Err(StatusCode::INTERNAL_SERVER_ERROR);
+            }
+        }
+    }
+
+    Ok(Json(AskResponse {
+        response: response_text.trim().to_string(),
+    }))
+}
+
 // Configure routes for this module
 pub fn routes(state: AppState) -> Router {
     Router::new()
         .route("/reply", post(handler))
+        .route("/ask", post(ask_handler))
         .with_state(state)
 }

ui/desktop/package.json

@@ -9,7 +9,7 @@
     "package": "electron-forge package",
     "make": "electron-forge make",
     "publish": "electron-forge publish",
-    "test-e2e": "electron-forge start > /tmp/out.txt & ELECTRON_PID=$! && sleep 8 && if grep -q 'renderer: ChatWindow loaded' /tmp/out.txt; then echo 'process is running'; pkill -f electron; else echo 'not starting correctly'; cat /tmp/out.txt; pkill -f electron; exit 1; fi",
+    "test-e2e": "electron-forge start > /tmp/out.txt & ELECTRON_PID=$! && sleep 12 && if grep -q 'renderer: ChatWindow loaded' /tmp/out.txt; then echo 'process is running'; pkill -f electron; else echo 'not starting correctly'; cat /tmp/out.txt; pkill -f electron; exit 1; fi",
     "sign-macos": "cd ./out/Goose-darwin-arm64 && codesign --deep --force --verify --sign \"Developer ID Application: Michael Neale (W2L75AE9HQ)\" Goose.app && ditto -c -k --sequesterRsrc --keepParent Goose.app Goose.zip"
   },
   "devDependencies": {

ui/desktop/src/ChatWindow.tsx

@@ -67,7 +67,9 @@ function ChatContent({
 }) {
   const chat = chats.find((c: Chat) => c.id === selectedChatId);
 
-  window.electron.logInfo('chats' + JSON.stringify(chats, null, 2));
+  //window.electron.logInfo('chats' + JSON.stringify(chats, null, 2));
+
+  const [messageMetadata, setMessageMetadata] = useState<Record<string, string[]>>({});
 
   const {
     messages,
@@ -92,8 +94,22 @@ function ChatContent({
         setStatus('Receiving response...');
       }
     },
-    onFinish: (message, options) => {
+    onFinish: async (message, options) => {
       setStatus('Goose is ready');
+
+
+
+
+      const promptTemplates = [
+        "Take a look at this content, if this looks like it could be asking for a confirmation, return QUESTION. If it looks like it is a list of options or plans to choose from, return OPTIONS, otherwise return READY. \n ### Message Content:\n" + message.content,
+        "If the content is clearly a list of distinct options or plans of action to choose from, and not just a list of things, but clearly a list of things to choose one from from, take into account the Message Content alone, try to format it in a json array, like this JSON array of objects of the form optionTitle:string, optionDescription:string (markdown).\n If is not a list of options or plans to choose from, then return empty list.\n ### Message Content:\n" + message.content,
+        "If the content is a request for input from the user, taking into account the Message Content alone, try to format it in as JSON object that fields in it of the form: fieldName:{type: string, number, email or date, title:string, description:optional markdown, required: true or false}:\n If it does not match, then return empty object.\n ### Message Content:\n" + message.content,
+      ];
+
+      const fetchResponses = await askAi(promptTemplates);
+      setMessageMetadata(prev => ({ ...prev, [message.id]: fetchResponses }));
+
+      console.log('All responses:', fetchResponses);
     },
   });
 
@@ -113,6 +129,10 @@ function ChatContent({
     }
   }, [initialQuery]);
 
+  if (error) {
+    console.log('Error:', error);
+  }
+
   return (
     <div className="chat-content flex flex-col w-screen h-screen bg-window-gradient items-center justify-center p-[10px]">
       <div className="flex w-screen">
@@ -158,7 +178,11 @@ function ChatContent({
                 {message.role === 'user' ? (
                   <UserMessage message={message} />
                 ) : (
-                  <GooseMessage message={message} />
+                  <GooseMessage 
+                    metadata={messageMetadata[message.id]} 
+                    message={message} 
+                    onInputChange={handleInputChange}
+                  />
                 )}
               </div>
             ))}
@@ -170,7 +194,7 @@ function ChatContent({
             {error && (
               <div className="flex items-center justify-center p-4">
                 <div className="text-red-500 bg-red-100 p-3 rounded-lg">
-                  {error.message || 'An error occurred while processing your request'}
+                  {error.message|| 'An error occurred while processing your request'}
                 </div>
               </div>
             )}
@@ -193,7 +217,7 @@ export default function ChatWindow() {
   // Get initial query and history from URL parameters
   const searchParams = new URLSearchParams(window.location.search);
   const initialQuery = searchParams.get('initialQuery');
-  window.electron.logInfo('initialQuery: ' + initialQuery);
+  //window.electron.logInfo('initialQuery: ' + initialQuery);
   const historyParam = searchParams.get('history');
   const initialHistory = historyParam
     ? JSON.parse(decodeURIComponent(historyParam))
@@ -256,4 +280,35 @@ export default function ChatWindow() {
       </div>
     </div>
   );
+
+
+
 }
+
+/**
+ * Utillity to ask the LLM any question to clarify without wider context.
+ */
+async function askAi(promptTemplates: string[]) {
+  console.log('askAi called...');
+  const responses = await Promise.all(promptTemplates.map(async (template) => {
+    const response = await fetch(getApiUrl('/ask'), {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({ prompt: template })
+    });
+
+    if (!response.ok) {
+      throw new Error('Failed to get response');
+    }
+
+    const data = await response.json();
+    console.log('ask Response:', data.response);
+
+    return data.response;
+  }));
+
+  return responses;
+}
+

ui/desktop/src/components/GooseMessage.tsx

@@ -1,13 +1,86 @@
-import React from 'react'
-import ToolInvocation from './ToolInvocation'
-import ReactMarkdown from 'react-markdown'
+import React, { useState } from 'react';
+import ToolInvocation from './ToolInvocation';
+import ReactMarkdown from 'react-markdown';
+import { GPSIcon } from './ui/icons';
 
+interface GooseMessageProps {
+  message: any;
+  metadata: any;
+  onInputChange?: (value: string) => void;
+}
 
-export default function GooseMessage({ message }) {
+export default function GooseMessage({ message, metadata, onInputChange }: GooseMessageProps) {
+  console.log("GooseMessage", metadata);
+
+  let isReady = false;
+  let isQuestion = false;
+  let isOptions = false;
+  let options = [];
+
+  if (metadata) {
+    isReady = metadata[0] === "READY";
+    isQuestion = metadata[0] === "QUESTION";
+    isOptions = metadata[0] === "OPTIONS";
+
+    if (isOptions && metadata[1]) {
+      try {
+        let optionsData = metadata[1];
+
+        // Remove ```json block if present
+        if (optionsData.startsWith('```json')) {
+          optionsData = optionsData.replace(/```json/g, '').replace(/```/g, '');
+        }
+
+        // Parse the options JSON
+        options = JSON.parse(optionsData);
+
+        // Validate the structure of each option
+        options = options.filter(
+          (opt) =>
+            typeof opt.optionTitle === 'string' &&
+            typeof opt.optionDescription === 'string'
+        );
+      } catch (err) {
+        console.error("Failed to parse options data:", err);
+        options = [];
+      }
+    }
+  }
+
+  const [selectedOption, setSelectedOption] = useState(null);
+
+  const handleOptionClick = (index) => {
+    setSelectedOption(index);
+  };
+
+  const handleAccept = () => {
+    if (onInputChange) {
+      onInputChange({ target: { value: "accept" } } as React.ChangeEvent<HTMLInputElement>);
+    }
+  };
+
+  const handleCancel = () => {
+    if (onInputChange) {
+      onInputChange({ target: { value: "No thanks" } } as React.ChangeEvent<HTMLInputElement>);
+    }
+  };
+
+  const handleSubmit = () => {
+    if (selectedOption !== null && onInputChange) {
+      onInputChange({ target: { value: options[selectedOption].optionTitle } } as React.ChangeEvent<HTMLInputElement>);
+    }
+  };
+
+  if (isQuestion || isOptions) {    
+    window.electron.showNotification({
+      title: 'Goose has a question for you',
+      body: `please check with goose to approve the plan of action`,
+    });
+  }
 
   return (
     <div className="flex mb-4">
-      <div className="bg-goose-bubble w-full text-black rounded-2xl p-4">
+      <div className="bg-white w-full text-black rounded-2xl p-4 shadow-md">
         {message.toolInvocations ? (
           <div className="space-y-4">
             {message.toolInvocations.map((toolInvocation) => (
@@ -18,9 +91,60 @@ export default function GooseMessage({ message }) {
             ))}
           </div>
         ) : (
-          <ReactMarkdown className="prose">{message.content}</ReactMarkdown>
+          <>
+            {(!isOptions || options.length == 0) && (
+              <ReactMarkdown className="prose">{message.content}</ReactMarkdown>
+            )}
+            {isQuestion && (
+              <div className="mt-4 bg-gray-100 p-4 rounded-lg shadow-lg">
+                <div className="flex space-x-4">
+                  <button
+                    onClick={handleAccept}
+                    className="flex items-center gap-2 bg-blue-500 text-white px-4 py-2 rounded-md hover:bg-blue-600 transition"
+                  >
+                    <GPSIcon size={14} />
+                    Accept Plan
+                  </button>
+                  <button
+                    onClick={handleCancel}
+                    className="flex items-center gap-2 bg-red-500 text-white px-4 py-2 rounded-md hover:bg-red-600 transition"
+                  >
+                    <GPSIcon size={14} />
+                    Cancel
+                  </button>
+                </div>
+              </div>
+            )}
+            {isOptions && options.length > 0 && (
+              <div className="mt-4 space-y-4">
+                {options.map((opt, index) => (
+                  <div
+                    key={index}
+                    onClick={() => handleOptionClick(index)}
+                    className={`p-4 rounded-lg shadow-md cursor-pointer ${
+                      selectedOption === index
+                        ? 'bg-blue-100 border border-blue-500'
+                        : 'bg-gray-100'
+                    }`}
+                  >
+                    <h3 className="font-semibold text-lg">{opt.optionTitle}</h3>
+                    <ReactMarkdown className="prose">
+                      {opt.optionDescription}
+                    </ReactMarkdown>
+                  </div>
+                ))}
+                <button
+                  onClick={handleSubmit}
+                  className="flex items-center gap-2 bg-blue-500 text-white px-4 py-2 rounded-md hover:bg-blue-600 transition mt-4"
+                >
+                  <GPSIcon size={14} />
+                  Submit
+                </button>
+              </div>
+            )}
+          </>
         )}
       </div>
     </div>
-  )
-};
+  );
+}

ui/desktop/src/main.ts

@@ -1,6 +1,6 @@
 import 'dotenv/config';
 import { loadZshEnv } from './utils/loadEnv';
-import { app, BrowserWindow, Tray, Menu, globalShortcut, ipcMain } from 'electron';
+import { app, BrowserWindow, Tray, Menu, globalShortcut, ipcMain, Notification } from 'electron';
 import path from 'node:path';
 import { start as startGoosed } from './goosed';
 import started from "electron-squirrel-startup";
@@ -218,7 +218,11 @@ app.whenReady().then(async () => {
   });
 
 
-
+  ipcMain.on('notify', (event, data) => {
+    console.log("NOTIFY", data);
+    new Notification({ title: data.title, body: data.body }).show();
+  });
+
   ipcMain.on('logInfo', (_, info) => {
     log.info("from renderer:", info);
   });

ui/desktop/src/preload.js

@@ -21,6 +21,8 @@ contextBridge.exposeInMainWorld('electron', {
   createChatWindow: (query) => ipcRenderer.send('create-chat-window', query),
   resizeWindow: (width, height) => ipcRenderer.send('resize-window', { windowId, width, height }),
   getWindowId: () => windowId,
-  logInfo: (txt) => ipcRenderer.send('logInfo', txt),
+  logInfo: (txt) => ipcRenderer.send('logInfo', txt),  
+  showNotification: (data) => ipcRenderer.send('notify', data),
+
 })