use tags for browsing agent too

agenthub/browsing_agent/browsing_agent.py

@@ -59,7 +59,7 @@ def get_system_message(goal: str, action_space: str) -> str:
 Here is another example with chain of thought of a valid action when providing a concise answer to user:
 "
 In order to accomplish my goal I need to send the information asked back to the user. This page list the information of HP Inkjet Fax Machine, which is the product identified in the objective. Its price is $279.49. I will send a message back to user with the answer.
-```send_msg_to_user("$279.49")```
+<execute_browse>send_msg_to_user("$279.49")</execute_browse>
 "
 """
 
@@ -77,7 +77,7 @@ def get_prompt(error_prefix: str, cur_axtree_txt: str, prev_action_str: str) ->
 Here is an example with chain of thought of a valid action when clicking on a button:
 "
 In order to accomplish my goal I need to click on the button with bid 12
-```click("12")```
+<execute_browse>click("12")</execute_browse>
 "
 """.strip()
     if USE_CONCISE_ANSWER:
@@ -115,7 +115,7 @@ def __init__(
             strict=False,  # less strict on the parsing of the actions
             multiaction=True,  # enable to agent to take multiple actions at once
         )
-
+        self.previous_axtree_txt = ''
         self.reset()
 
     def reset(self) -> None:
@@ -179,6 +179,9 @@ def step(self, state: State) -> Action:
                 if self.error_accumulator > 5:
                     return MessageAction('Too many errors encountered. Task failed.')
             cur_axtree_txt = last_obs.axtree_txt
+            if not cur_axtree_txt:
+                cur_axtree_txt = self.previous_axtree_txt
+            self.previous_axtree_txt = cur_axtree_txt
             if cur_axtree_txt.startswith('AX Error:'):
                 return MessageAction(
                     f'Error encountered when browsing. {cur_axtree_txt}'
@@ -202,6 +205,6 @@ def step(self, state: State) -> Action:
         response = self.llm.completion(
             messages=[message.model_dump() for message in messages],
             temperature=0.0,
-            stop=[')```', ')\n```'],
+            stop=['</execute_browse>'],
         )
         return self.response_parser.parse(response)

agenthub/browsing_agent/response_parser.py

@@ -1,4 +1,5 @@
 import ast
+import re
 
 from opendevin.controller.action_parser import ActionParser, ResponseParser
 from opendevin.core.logger import opendevin_logger as logger
@@ -12,8 +13,8 @@ class BrowsingResponseParser(ResponseParser):
     def __init__(self):
         # Need to pay attention to the item order in self.action_parsers
         super().__init__()
-        self.action_parsers = [BrowsingActionParserMessage()]
-        self.default_parser = BrowsingActionParserBrowseInteractive()
+        self.action_parsers = [BrowsingActionParserBrowseInteractive()]
+        self.default_parser = BrowsingActionParserMessage()
 
     def parse(self, response: str) -> Action:
         action_str = self.parse_response(response)
@@ -24,8 +25,10 @@ def parse_response(self, response) -> str:
         if action_str is None:
             return ''
         action_str = action_str.strip()
-        if not action_str.endswith('```'):
-            action_str = action_str + ')```'
+        start_tag = '<execute_browse>'
+        end_tag = '</execute_browse>'
+        if start_tag in action_str and end_tag not in action_str:
+            action_str += end_tag
         logger.info(action_str)
         return action_str
 
@@ -47,7 +50,7 @@ def __init__(
         pass
 
     def check_condition(self, action_str: str) -> bool:
-        return '```' not in action_str
+        return True
 
     def parse(self, action_str: str) -> Action:
         msg = f'send_msg_to_user("""{action_str}""")'
@@ -69,20 +72,26 @@ def __init__(
         pass
 
     def check_condition(self, action_str: str) -> bool:
-        return True
+        self.action_str = re.search(
+            r'<execute_browse>(.*?)</execute_browse>', action_str, re.DOTALL
+        )
+        return self.action_str is not None
 
     def parse(self, action_str: str) -> Action:
-        thought = action_str.split('```')[0].strip()
-        action_str = action_str.split('```')[1].strip()
+        assert (
+            self.action_str is not None
+        ), 'self.action_str should not be None when parse is called'
+        action_cmd = self.action_str.group(1).strip()
+        thought = action_str.replace(self.action_str.group(0), '').strip()
         msg_content = ''
-        for sub_action in action_str.split('\n'):
+        for sub_action in action_cmd.split('\n'):
             if 'send_msg_to_user(' in sub_action:
                 tree = ast.parse(sub_action)
                 args = tree.body[0].value.args  # type: ignore
                 msg_content = args[0].value
 
         return BrowseInteractiveAction(
-            browser_actions=action_str,
+            browser_actions=action_cmd,
             thought=thought,
             browsergym_send_msg_to_user=msg_content,
         )

opendevin/runtime/browser/utils.py

@@ -18,7 +18,7 @@ async def browse(
 
     if isinstance(action, BrowseURLAction):
         # legacy BrowseURLAction
-        asked_url = action.url
+        asked_url = action.url.strip()
         if not asked_url.startswith('http'):
             asked_url = os.path.abspath(os.curdir) + action.url
         action_str = f'goto("{asked_url}")'
@@ -31,6 +31,8 @@ async def browse(
         raise ValueError(f'Invalid action type: {action.action}')
 
     try:
+        if 'summarize' in action_str:
+            raise ValueError('Summarize is an invalid action')
         # obs provided by BrowserGym:
         # https://github.com/ServiceNow/BrowserGym/blob/main/browsergym/core/src/browsergym/core/env.py#L521
         # https://github.com/ServiceNow/BrowserGym/blob/418421abdc5da4d77dc71d3b82a9e5e931be0c4f/browsergym/core/src/browsergym/core/env.py#L521