llamaindex_tool_calling.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyN4lxULoUE4+hAKtinlcFsW"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "\n",
        "%pip install llama-index-readers-file --quiet\n",
        "%pip install llama-index-embeddings-openai --quiet\n",
        "%pip install llama-index-agent-openai --quiet\n",
        "%pip install llama-index-llms-openai --quiet"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Csx3fLJ682eU",
        "outputId": "34f38b29-ecae-4004-ec10-c82685582bdb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m15.4/15.4 MB\u001b[0m \u001b[31m42.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m290.4/290.4 kB\u001b[0m \u001b[31m21.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m4.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m141.9/141.9 kB\u001b[0m \u001b[31m12.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m320.7/320.7 kB\u001b[0m \u001b[31m21.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m42.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m3.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.3/49.3 kB\u001b[0m \u001b[31m3.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import nest_asyncio\n",
        "nest_asyncio.apply()\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qDUNfot1-U3o",
        "outputId": "901ef3c8-bdb8-4fa3-b38c-74fdf7035fa1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yahU4iNr8mRB"
      },
      "outputs": [],
      "source": [
        "\n",
        "\n",
        "import os\n",
        "os.environ[\"OPENAI_API_KEY\"] = ''"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from llama_index.core.tools import FunctionTool\n",
        "\n",
        "def add(x: int, y: int) -> int:\n",
        "    \"\"\"Adds two integers together.\"\"\"\n",
        "    return x + y\n",
        "\n",
        "def mystery(x: int, y: int) -> int:\n",
        "    \"\"\"Mystery function that operates on top of two numbers.\"\"\"\n",
        "    return (x + y) * (x + y)\n",
        "\n",
        "\n",
        "add_tool = FunctionTool.from_defaults(fn=add)\n",
        "mystery_tool = FunctionTool.from_defaults(fn=mystery)"
      ],
      "metadata": {
        "id": "rvkGO_5E8saj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from llama_index.llms.openai import OpenAI\n",
        "\n",
        "llm = OpenAI(model=\"gpt-3.5-turbo\")\n",
        "response = llm.predict_and_call(\n",
        "    [add_tool, mystery_tool],\n",
        "    \"Tell me the output of the mystery function on 2 and 9\",\n",
        "    verbose=True\n",
        ")\n",
        "print(str(response))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K3MiTOZ28_nv",
        "outputId": "0b6a699f-c162-47a6-e7ea-47ff591a3de5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "=== Calling Function ===\n",
            "Calling function: mystery with args: {\"x\": 2, \"y\": 9}\n",
            "=== Function Output ===\n",
            "121\n",
            "121\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from llama_index.core import SimpleDirectoryReader\n",
        "# load documents\n",
        "documents = SimpleDirectoryReader(input_files=[\"/content/drive/MyDrive/Colab Notebooks/Building Agentic RAG with Llamaindex/metagpt.pdf\"]).load_data()"
      ],
      "metadata": {
        "id": "RFiYNQN2_7UE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from llama_index.core.node_parser import SentenceSplitter\n",
        "splitter = SentenceSplitter(chunk_size=1024)\n",
        "nodes = splitter.get_nodes_from_documents(documents)"
      ],
      "metadata": {
        "id": "hLSqIg-ACt2-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(nodes[0].get_content(metadata_mode=\"all\"))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_Idb5mMGC0BR",
        "outputId": "8f553549-384a-4a9b-9df4-bf9d2b17842b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "page_label: 1\n",
            "file_name: metagpt.pdf\n",
            "file_path: /content/drive/MyDrive/Colab Notebooks/Building Agentic RAG with Llamaindex/metagpt.pdf\n",
            "file_type: application/pdf\n",
            "file_size: 16911937\n",
            "creation_date: 2024-05-31\n",
            "last_modified_date: 2024-05-31\n",
            "\n",
            "Preprint\n",
            "METAGPT: M ETA PROGRAMMING FOR A\n",
            "MULTI -AGENT COLLABORATIVE FRAMEWORK\n",
            "Sirui Hong1∗, Mingchen Zhuge2∗, Jonathan Chen1, Xiawu Zheng3, Yuheng Cheng4,\n",
            "Ceyao Zhang4,Jinlin Wang1,Zili Wang ,Steven Ka Shing Yau5,Zijuan Lin4,\n",
            "Liyang Zhou6,Chenyu Ran1,Lingfeng Xiao1,7,Chenglin Wu1†,J¨urgen Schmidhuber2,8\n",
            "1DeepWisdom,2AI Initiative, King Abdullah University of Science and Technology,\n",
            "3Xiamen University,4The Chinese University of Hong Kong, Shenzhen,\n",
            "5Nanjing University,6University of Pennsylvania,\n",
            "7University of California, Berkeley,8The Swiss AI Lab IDSIA/USI/SUPSI\n",
            "ABSTRACT\n",
            "Remarkable progress has been made on automated problem solving through so-\n",
            "cieties of agents based on large language models (LLMs). Existing LLM-based\n",
            "multi-agent systems can already solve simple dialogue tasks. Solutions to more\n",
            "complex tasks, however, are complicated through logic inconsistencies due to\n",
            "cascading hallucinations caused by naively chaining LLMs. Here we introduce\n",
            "MetaGPT, an innovative meta-programming framework incorporating efficient\n",
            "human workflows into LLM-based multi-agent collaborations. MetaGPT en-\n",
            "codes Standardized Operating Procedures (SOPs) into prompt sequences for more\n",
            "streamlined workflows, thus allowing agents with human-like domain expertise\n",
            "to verify intermediate results and reduce errors. MetaGPT utilizes an assembly\n",
            "line paradigm to assign diverse roles to various agents, efficiently breaking down\n",
            "complex tasks into subtasks involving many agents working together. On col-\n",
            "laborative software engineering benchmarks, MetaGPT generates more coherent\n",
            "solutions than previous chat-based multi-agent systems. Our project can be found\n",
            "at https://github.com/geekan/MetaGPT.\n",
            "1 I NTRODUCTION\n",
            "Autonomous agents utilizing Large Language Models (LLMs) offer promising opportunities to en-\n",
            "hance and replicate human workflows. In real-world applications, however, existing systems (Park\n",
            "et al., 2023; Zhuge et al., 2023; Cai et al., 2023; Wang et al., 2023c; Li et al., 2023; Du et al., 2023;\n",
            "Liang et al., 2023; Hao et al., 2023) tend to oversimplify the complexities. They struggle to achieve\n",
            "effective, coherent, and accurate problem-solving processes, particularly when there is a need for\n",
            "meaningful collaborative interaction (Chen et al., 2024; Zhang et al., 2023; Dong et al., 2023; Zhou\n",
            "et al., 2023; Qian et al., 2023).\n",
            "Through extensive collaborative practice, humans have developed widely accepted Standardized\n",
            "Operating Procedures (SOPs) across various domains (Belbin, 2012; Manifesto, 2001; DeMarco &\n",
            "Lister, 2013). These SOPs play a critical role in supporting task decomposition and effective coor-\n",
            "dination. Furthermore, SOPs outline the responsibilities of each team member, while establishing\n",
            "standards for intermediate outputs. Well-defined SOPs improve the consistent and accurate exe-\n",
            "cution of tasks that align with defined roles and quality standards (Belbin, 2012; Manifesto, 2001;\n",
            "DeMarco & Lister, 2013; Wooldridge & Jennings, 1998). For instance, in a software company,\n",
            "Product Managers analyze competition and user needs to create Product Requirements Documents\n",
            "(PRDs) using a standardized structure, to guide the developmental process.\n",
            "Inspired by such ideas, we design a promising GPT -based Meta -Programming framework called\n",
            "MetaGPT that significantly benefits from SOPs. Unlike other works (Li et al., 2023; Qian et al.,\n",
            "2023), MetaGPT requires agents to generate structured outputs, such as high-quality requirements\n",
            "∗These authors contributed equally to this work.\n",
            "†Chenglin Wu (alexanderwu@fuzhi.ai) is the corresponding author, affiliated with DeepWisdom.\n",
            "1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from llama_index.core import VectorStoreIndex\n",
        "\n",
        "vector_index = VectorStoreIndex(nodes)\n",
        "query_engine = vector_index.as_query_engine(similarity_top_k=2)"
      ],
      "metadata": {
        "id": "bh_L2vpCDcAX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from llama_index.core.vector_stores import MetadataFilters\n",
        "\n",
        "query_engine = vector_index.as_query_engine(\n",
        "    similarity_top_k=2,\n",
        "    filters=MetadataFilters.from_dicts(\n",
        "        [\n",
        "            {\"key\": \"page_label\", \"value\": \"2\"}\n",
        "        ]\n",
        "    )\n",
        ")\n",
        "\n",
        "response = query_engine.query(\n",
        "    \"What are some high-level results of MetaGPT?\",\n",
        ")"
      ],
      "metadata": {
        "id": "yUvMuqUzDtPJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(str(response))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "A34PT_JjEUST",
        "outputId": "3d474d94-f55c-41bf-a407-2b6ba75dc3b7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "MetaGPT achieves a new state-of-the-art in code generation benchmarks with 85.9% and 87.7% in Pass@1, surpassing other popular frameworks like AutoGPT, LangChain, AgentVerse, and ChatDev. Additionally, MetaGPT demonstrates robustness and efficiency by achieving a 100% task completion rate in experimental evaluations, highlighting its effectiveness in handling higher levels of software complexity and offering extensive functionality.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for n in response.source_nodes:\n",
        "    print(n.metadata)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zGTXbbKpEaLm",
        "outputId": "ced8c8ce-c994-4389-c47c-190c91f0f8d0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'page_label': '2', 'file_name': 'metagpt.pdf', 'file_path': '/content/drive/MyDrive/Colab Notebooks/Building Agentic RAG with Llamaindex/metagpt.pdf', 'file_type': 'application/pdf', 'file_size': 16911937, 'creation_date': '2024-05-31', 'last_modified_date': '2024-05-31'}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from typing import List\n",
        "from llama_index.core.vector_stores import FilterCondition\n",
        "\n",
        "\n",
        "def vector_query(\n",
        "    query: str,\n",
        "    page_numbers: List[str]\n",
        ") -> str:\n",
        "    \"\"\"Perform a vector search over an index.\n",
        "\n",
        "    query (str): the string query to be embedded.\n",
        "    page_numbers (List[str]): Filter by set of pages. Leave BLANK if we want to perform a vector search\n",
        "        over all pages. Otherwise, filter by the set of specified pages.\n",
        "\n",
        "    \"\"\"\n",
        "\n",
        "    metadata_dicts = [\n",
        "        {\"key\": \"page_label\", \"value\": p} for p in page_numbers\n",
        "    ]\n",
        "\n",
        "    query_engine = vector_index.as_query_engine(\n",
        "        similarity_top_k=2,\n",
        "        filters=MetadataFilters.from_dicts(\n",
        "            metadata_dicts,\n",
        "            condition=FilterCondition.OR\n",
        "        )\n",
        "    )\n",
        "    response = query_engine.query(query)\n",
        "    return response\n",
        "\n",
        "\n",
        "vector_query_tool = FunctionTool.from_defaults(\n",
        "    name=\"vector_tool\",\n",
        "    fn=vector_query\n",
        ")"
      ],
      "metadata": {
        "id": "MHOfSOIU3yXL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "llm = OpenAI(model=\"gpt-4o\", temperature=0)\n",
        "response = llm.predict_and_call(\n",
        "    [vector_query_tool],\n",
        "    \"What are the high-level results of MetaGPT as described on page 2?\",\n",
        "    verbose=True\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oNFWWJlf6QSF",
        "outputId": "6cea7be3-b3ab-4a76-c3ce-3f9946b6ef71"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "=== Calling Function ===\n",
            "Calling function: vector_tool with args: {\"query\": \"high-level results of MetaGPT\", \"page_numbers\": [\"2\"]}\n",
            "=== Function Output ===\n",
            "MetaGPT achieves a new state-of-the-art (SoTA) in code generation benchmarks with 85.9% and 87.7% in Pass@1. It stands out in handling higher levels of software complexity and offering extensive functionality. In experimental evaluations, MetaGPT achieves a 100% task completion rate, demonstrating robustness and efficiency in terms of time and token costs.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for n in response.source_nodes:\n",
        "    print(n.metadata)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oi9__Pd36yhG",
        "outputId": "653c5897-0e05-4228-cc72-90a0b443cc40"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'page_label': '2', 'file_name': 'metagpt.pdf', 'file_path': '/content/drive/MyDrive/Colab Notebooks/Building Agentic RAG with Llamaindex/metagpt.pdf', 'file_type': 'application/pdf', 'file_size': 16911937, 'creation_date': '2024-05-31', 'last_modified_date': '2024-05-31'}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from llama_index.core import SummaryIndex\n",
        "from llama_index.core.tools import QueryEngineTool\n",
        "\n",
        "summary_index = SummaryIndex(nodes)\n",
        "summary_query_engine = summary_index.as_query_engine(\n",
        "    response_mode=\"tree_summarize\",\n",
        "    use_async=True,\n",
        ")\n",
        "summary_tool = QueryEngineTool.from_defaults(\n",
        "    name=\"summary_tool\",\n",
        "    query_engine=summary_query_engine,\n",
        "    description=(\n",
        "        \"Useful if you want to get a summary of MetaGPT\"\n",
        "    ),\n",
        ")"
      ],
      "metadata": {
        "id": "deNIE5YU9oYN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "response = llm.predict_and_call(\n",
        "    [vector_query_tool, summary_tool],\n",
        "    \"What are the MetaGPT comparisons with ChatDev described on page 8?\",\n",
        "    verbose=True\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vBCQpFiT-CbY",
        "outputId": "e1e2bbe7-8eba-4b24-839b-63734a05b8ef"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "=== Calling Function ===\n",
            "Calling function: vector_tool with args: {\"query\": \"MetaGPT comparisons with ChatDev\", \"page_numbers\": [\"8\"]}\n",
            "=== Function Output ===\n",
            "MetaGPT outperforms ChatDev on the SoftwareDev dataset in various metrics. For instance, MetaGPT achieves a higher score in executability, takes less time for execution, uses fewer tokens for code generation, produces more lines of code per file, and has lower human revision costs compared to ChatDev. These results demonstrate the advantages of MetaGPT over ChatDev in software development tasks.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for n in response.source_nodes:\n",
        "    print(n.metadata)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-CXfXMaj-F4x",
        "outputId": "d85973b7-5359-4dfe-8d6c-d062d9cfdc7a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'page_label': '8', 'file_name': 'metagpt.pdf', 'file_path': '/content/drive/MyDrive/Colab Notebooks/Building Agentic RAG with Llamaindex/metagpt.pdf', 'file_type': 'application/pdf', 'file_size': 16911937, 'creation_date': '2024-05-31', 'last_modified_date': '2024-05-31'}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "response = llm.predict_and_call(\n",
        "    [vector_query_tool, summary_tool],\n",
        "    \"What is a summary of the paper?\",\n",
        "    verbose=True\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WAVqpJ4d_8QP",
        "outputId": "fbce5eea-ff74-4b26-cada-220059875d8b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "=== Calling Function ===\n",
            "Calling function: summary_tool with args: {\"input\": \"MetaGPT\"}\n",
            "=== Function Output ===\n",
            "MetaGPT is an innovative meta-programming framework that incorporates efficient human workflows into Large Language Model (LLM)-based multi-agent collaborations. It encodes Standardized Operating Procedures (SOPs) into prompt sequences for streamlined workflows, allowing agents with human-like domain expertise to verify intermediate results and reduce errors. MetaGPT utilizes an assembly line paradigm to assign diverse roles to various agents, efficiently breaking down complex tasks into subtasks involving many agents working together. It has shown significant improvements in code generation quality and task completion rates compared to other frameworks in various benchmarks. Additionally, MetaGPT is a framework that utilizes natural language programming to generate software applications based on provided requirements, involving multiple agents such as Product Managers, Architects, Engineers, and QA Engineers to streamline the software development process.\n"
          ]
        }
      ]
    }
  ]
}