Advanced_RAG_TinyLlama_LLM_LlamaIndex.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "L4",
      "authorship_tag": "ABX9TyMHqgNHR3fwoFzFJiiv5/2m",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "ce2a847b9f4746b5ae7f34b5ccf1013b": {
          "model_module": "@jupyter-widgets/output",
          "model_name": "OutputModel",
          "model_module_version": "1.0.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/output",
            "_model_module_version": "1.0.0",
            "_model_name": "OutputModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/output",
            "_view_module_version": "1.0.0",
            "_view_name": "OutputView",
            "layout": "IPY_MODEL_6894670075f64c1bb7bf06ea45c05e2c",
            "msg_id": "",
            "outputs": [
              {
                "output_type": "display_data",
                "data": {
                  "text/plain": "\u001b[38;2;106;0;255m⠧\u001b[0m ✨ You're running DeepEval's latest \u001b[38;2;106;0;255mAnswer Relevancy Metric\u001b[0m! \u001b[38;2;55;65;81m(using Llama-3 8B, strict=False, async_mode=True)...\u001b[0m\n",
                  "text/html": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #6a00ff; text-decoration-color: #6a00ff\">⠧</span> ✨ You're running DeepEval's latest <span style=\"color: #6a00ff; text-decoration-color: #6a00ff\">Answer Relevancy Metric</span>! <span style=\"color: #374151; text-decoration-color: #374151\">(using Llama-3 8B, strict=False, async_mode=True)...</span>\n</pre>\n"
                },
                "metadata": {}
              }
            ]
          }
        },
        "6894670075f64c1bb7bf06ea45c05e2c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8908414258b44c65bdedc19e818831c7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b41efc3d9a0a4ed495b7c417a20279a7",
              "IPY_MODEL_a9739a5cc3fb434c8cd8018786c602ac",
              "IPY_MODEL_dc30dc8f96564f7e8d426f2703c9b1a7"
            ],
            "layout": "IPY_MODEL_4a5c416e6a514942a87c2ce4e9c6d59a"
          }
        },
        "b41efc3d9a0a4ed495b7c417a20279a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_01f81eb5fe4f466691fda4cec63d9454",
            "placeholder": "​",
            "style": "IPY_MODEL_2443165a104d49aeb7abccc4346e71c8",
            "value": "generation_config.json: 100%"
          }
        },
        "a9739a5cc3fb434c8cd8018786c602ac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ef7ff31cada24bbfaa30f831fcff06e5",
            "max": 124,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8315f039e7f04ed68535b7420dd4cea2",
            "value": 124
          }
        },
        "dc30dc8f96564f7e8d426f2703c9b1a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_26b5933cb8864c2d86f164fd3baa054c",
            "placeholder": "​",
            "style": "IPY_MODEL_46b3f392c007488fa6ff34aec2403bc8",
            "value": " 124/124 [00:00&lt;00:00, 9.95kB/s]"
          }
        },
        "4a5c416e6a514942a87c2ce4e9c6d59a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "01f81eb5fe4f466691fda4cec63d9454": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2443165a104d49aeb7abccc4346e71c8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ef7ff31cada24bbfaa30f831fcff06e5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8315f039e7f04ed68535b7420dd4cea2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "26b5933cb8864c2d86f164fd3baa054c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "46b3f392c007488fa6ff34aec2403bc8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4627e7e7c9e24a93bd227cd5c375d7ca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_4a83f8d2c8034a7594cff0770b3c6854",
              "IPY_MODEL_4c0ff1085220493cbfdc256a94be703b",
              "IPY_MODEL_b6f6974c47e64485bada093b573ee698"
            ],
            "layout": "IPY_MODEL_9f3b5829b2b8443a9c1b8fdfc9d49015"
          }
        },
        "4a83f8d2c8034a7594cff0770b3c6854": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c3e4a0efc99d423cad10b973a395ba00",
            "placeholder": "​",
            "style": "IPY_MODEL_7254f8933da14a02beb158a42344a4cf",
            "value": "config.json: 100%"
          }
        },
        "4c0ff1085220493cbfdc256a94be703b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_694deae3e53f498c97e07ac94c7cbd1f",
            "max": 683,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8e6dfc36ecfa4f16a01d7a99f20d0fde",
            "value": 683
          }
        },
        "b6f6974c47e64485bada093b573ee698": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_920cf71ffd244db08431175876aa4b10",
            "placeholder": "​",
            "style": "IPY_MODEL_1e41ec305a0e4c3b9354ca54a8260fd2",
            "value": " 683/683 [00:00&lt;00:00, 60.3kB/s]"
          }
        },
        "9f3b5829b2b8443a9c1b8fdfc9d49015": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c3e4a0efc99d423cad10b973a395ba00": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7254f8933da14a02beb158a42344a4cf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "694deae3e53f498c97e07ac94c7cbd1f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8e6dfc36ecfa4f16a01d7a99f20d0fde": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "920cf71ffd244db08431175876aa4b10": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1e41ec305a0e4c3b9354ca54a8260fd2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "90d19082412a4f6f8edc17660e484ebc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d857657ac84a42758e94e46b3abd0fc5",
              "IPY_MODEL_8683c1d42f0b4dacb8d3362930731ff3",
              "IPY_MODEL_7525450a3ec04920bdcaa629048bc013"
            ],
            "layout": "IPY_MODEL_193ee8d6ce61444f8cce1ba395315814"
          }
        },
        "d857657ac84a42758e94e46b3abd0fc5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fd6f1412fe73448d908b6855a9c4da87",
            "placeholder": "​",
            "style": "IPY_MODEL_150d4be236a246888ecd97430deeaa9d",
            "value": "model.safetensors: 100%"
          }
        },
        "8683c1d42f0b4dacb8d3362930731ff3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_87597966a6ce4c2f94c5ac9011481981",
            "max": 328489328,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_dc6cb78c1de04ce9accd2a5ff57ec7e3",
            "value": 328489328
          }
        },
        "7525450a3ec04920bdcaa629048bc013": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2a042deab41f4ac0bd444d26b709a3c5",
            "placeholder": "​",
            "style": "IPY_MODEL_71541ce0dfb547cdb4b586ff6f11ed64",
            "value": " 328M/328M [00:22&lt;00:00, 15.5MB/s]"
          }
        },
        "193ee8d6ce61444f8cce1ba395315814": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fd6f1412fe73448d908b6855a9c4da87": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "150d4be236a246888ecd97430deeaa9d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "87597966a6ce4c2f94c5ac9011481981": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dc6cb78c1de04ce9accd2a5ff57ec7e3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2a042deab41f4ac0bd444d26b709a3c5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "71541ce0dfb547cdb4b586ff6f11ed64": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "61bf7d7752cd4652819495bc64bd7dbd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b3fea1b6c34a47eca24e3fb34b360e0d",
              "IPY_MODEL_426c8700c12a48a2abbea1812c0d46c6",
              "IPY_MODEL_e8bdeb0f71cd44dc8181b45d12204b33"
            ],
            "layout": "IPY_MODEL_9a052c85527d49aea4af76875ec90313"
          }
        },
        "b3fea1b6c34a47eca24e3fb34b360e0d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e493952729614ce590c84b1bc5addd97",
            "placeholder": "​",
            "style": "IPY_MODEL_a551d06ebcaf4947a892174e6b0da8cb",
            "value": "tokenizer_config.json: 100%"
          }
        },
        "426c8700c12a48a2abbea1812c0d46c6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7318c59b50284cf0b5c9fe3e5bac0b59",
            "max": 1196,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_31e0db9d757f434682978bda54962e31",
            "value": 1196
          }
        },
        "e8bdeb0f71cd44dc8181b45d12204b33": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_588d3eef77a04c38a5ccd8a112adf1b5",
            "placeholder": "​",
            "style": "IPY_MODEL_7e6d17c36db84f45af0804218fea8792",
            "value": " 1.20k/1.20k [00:00&lt;00:00, 103kB/s]"
          }
        },
        "9a052c85527d49aea4af76875ec90313": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e493952729614ce590c84b1bc5addd97": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a551d06ebcaf4947a892174e6b0da8cb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7318c59b50284cf0b5c9fe3e5bac0b59": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "31e0db9d757f434682978bda54962e31": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "588d3eef77a04c38a5ccd8a112adf1b5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7e6d17c36db84f45af0804218fea8792": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "896352cdb9204a0e9cbb2bcb3ad38d9a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_679832cea9204ecea20e505f9706fe6d",
              "IPY_MODEL_0c942f2eac3e42fb81abd932db095588",
              "IPY_MODEL_ad700f35bbc34fc5a9909248dd2576e4"
            ],
            "layout": "IPY_MODEL_b5d2cd90bb3d42a99928fdfcf4fd34f2"
          }
        },
        "679832cea9204ecea20e505f9706fe6d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ae1c3923a86e4091bf6d7f8f7b2d8abf",
            "placeholder": "​",
            "style": "IPY_MODEL_af3eecf2ecd74498ae9c1f2e2d3ea81a",
            "value": "vocab.json: 100%"
          }
        },
        "0c942f2eac3e42fb81abd932db095588": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_82da283b58fc488f9961544b5e640bb5",
            "max": 798293,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c72c422c83864364befa7c7bdd3f7b46",
            "value": 798293
          }
        },
        "ad700f35bbc34fc5a9909248dd2576e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f36e4b6c80b44b9cb6d50e0380650b9d",
            "placeholder": "​",
            "style": "IPY_MODEL_0b703d2412284fe5b775a43bc50e11fc",
            "value": " 798k/798k [00:00&lt;00:00, 808kB/s]"
          }
        },
        "b5d2cd90bb3d42a99928fdfcf4fd34f2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ae1c3923a86e4091bf6d7f8f7b2d8abf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "af3eecf2ecd74498ae9c1f2e2d3ea81a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "82da283b58fc488f9961544b5e640bb5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c72c422c83864364befa7c7bdd3f7b46": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f36e4b6c80b44b9cb6d50e0380650b9d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0b703d2412284fe5b775a43bc50e11fc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b4cda185eb704afc8a8635fcdc3d8942": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1468fb3a8af8493cb24f441f878dce80",
              "IPY_MODEL_e916d2cd57bd47b9944085462f5004fd",
              "IPY_MODEL_0fc1d0e29eef4366a28e93df58b2cb93"
            ],
            "layout": "IPY_MODEL_929042bac59d4236a24f0200e9ba0334"
          }
        },
        "1468fb3a8af8493cb24f441f878dce80": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_abb249e28d41422f8176cda6a11c960d",
            "placeholder": "​",
            "style": "IPY_MODEL_b82182c42e904af9b8f159f819e71108",
            "value": "merges.txt: 100%"
          }
        },
        "e916d2cd57bd47b9944085462f5004fd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fa6746f9330d4eb7bc1a7be5de28a407",
            "max": 456356,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_fd1b1d710233401f94da4e53d7d6e280",
            "value": 456356
          }
        },
        "0fc1d0e29eef4366a28e93df58b2cb93": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_718cf6768d72448aa64f96f7213133e0",
            "placeholder": "​",
            "style": "IPY_MODEL_4ecfd953822e406f936162417bbf277a",
            "value": " 456k/456k [00:00&lt;00:00, 35.0MB/s]"
          }
        },
        "929042bac59d4236a24f0200e9ba0334": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "abb249e28d41422f8176cda6a11c960d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b82182c42e904af9b8f159f819e71108": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fa6746f9330d4eb7bc1a7be5de28a407": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fd1b1d710233401f94da4e53d7d6e280": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "718cf6768d72448aa64f96f7213133e0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4ecfd953822e406f936162417bbf277a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2bc113068b8147ce8551f811f54dfad4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_aa5bb7aea6294a2e8152b69f06e8e671",
              "IPY_MODEL_b05c7f02195f47bdbd028452b651d29e",
              "IPY_MODEL_18a6b92a09ea4cbfb2e76073cad13bb2"
            ],
            "layout": "IPY_MODEL_030553a861e24a71bee5de6350d22a43"
          }
        },
        "aa5bb7aea6294a2e8152b69f06e8e671": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6713257f8092406fb81bc053d01d20a0",
            "placeholder": "​",
            "style": "IPY_MODEL_ae3b11c8bc9c4a1ca71d6b96606df465",
            "value": "tokenizer.json: 100%"
          }
        },
        "b05c7f02195f47bdbd028452b651d29e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e3655f35e4604b56a51f6e419634062e",
            "max": 1355876,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_188eaf5a194548169e9f107c7fc7b40a",
            "value": 1355876
          }
        },
        "18a6b92a09ea4cbfb2e76073cad13bb2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4bb0d8f6bbb143efb65197eb42930213",
            "placeholder": "​",
            "style": "IPY_MODEL_02561d427fd040a38e72ebc7c650b6e1",
            "value": " 1.36M/1.36M [00:01&lt;00:00, 1.10MB/s]"
          }
        },
        "030553a861e24a71bee5de6350d22a43": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6713257f8092406fb81bc053d01d20a0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ae3b11c8bc9c4a1ca71d6b96606df465": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e3655f35e4604b56a51f6e419634062e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "188eaf5a194548169e9f107c7fc7b40a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "4bb0d8f6bbb143efb65197eb42930213": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "02561d427fd040a38e72ebc7c650b6e1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0a7518b8a7a34278be38400b19e61e96": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b129bf9396104e07ba80510dd05a45af",
              "IPY_MODEL_b9cab8990c06489aaf9f9c8636f5379a",
              "IPY_MODEL_f26edde5a7494475b79b00baa3a97543"
            ],
            "layout": "IPY_MODEL_a18c03831c0643a59e520c4bd9bcc3fa"
          }
        },
        "b129bf9396104e07ba80510dd05a45af": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a455c67ce22348c2b3e65cdcbf0b22ff",
            "placeholder": "​",
            "style": "IPY_MODEL_81ce34875ca843d6b7477733582d3a33",
            "value": "special_tokens_map.json: 100%"
          }
        },
        "b9cab8990c06489aaf9f9c8636f5379a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b3d88339c8674dac8121f2a7bdb3082d",
            "max": 772,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e95e1c27ad024080be1c17d9a3c0805f",
            "value": 772
          }
        },
        "f26edde5a7494475b79b00baa3a97543": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b49db84cc80049c5b3b4331c46c9e919",
            "placeholder": "​",
            "style": "IPY_MODEL_cd020d888f664242afa86834d52febe9",
            "value": " 772/772 [00:00&lt;00:00, 58.3kB/s]"
          }
        },
        "a18c03831c0643a59e520c4bd9bcc3fa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a455c67ce22348c2b3e65cdcbf0b22ff": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "81ce34875ca843d6b7477733582d3a33": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b3d88339c8674dac8121f2a7bdb3082d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e95e1c27ad024080be1c17d9a3c0805f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b49db84cc80049c5b3b4331c46c9e919": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cd020d888f664242afa86834d52febe9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "413b2d8487a54b3ebee63c1298b7754c": {
          "model_module": "@jupyter-widgets/output",
          "model_name": "OutputModel",
          "model_module_version": "1.0.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/output",
            "_model_module_version": "1.0.0",
            "_model_name": "OutputModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/output",
            "_view_module_version": "1.0.0",
            "_view_name": "OutputView",
            "layout": "IPY_MODEL_6fea62aad9cc40c9acffa02f950dcba5",
            "msg_id": "",
            "outputs": [
              {
                "output_type": "display_data",
                "data": {
                  "text/plain": "\u001b[38;2;106;0;255m⠏\u001b[0m ✨ You're running DeepEval's latest \u001b[38;2;106;0;255mFaithfulness Metric\u001b[0m! \u001b[38;2;55;65;81m(using gpt-3.5-turbo, strict=False, async_mode=True)...\u001b[0m\n",
                  "text/html": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #6a00ff; text-decoration-color: #6a00ff\">⠏</span> ✨ You're running DeepEval's latest <span style=\"color: #6a00ff; text-decoration-color: #6a00ff\">Faithfulness Metric</span>! <span style=\"color: #374151; text-decoration-color: #374151\">(using gpt-3.5-turbo, strict=False, async_mode=True)...</span>\n</pre>\n"
                },
                "metadata": {}
              }
            ]
          }
        },
        "6fea62aad9cc40c9acffa02f950dcba5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8f5eca32bcee46e587c385757274ba0e": {
          "model_module": "@jupyter-widgets/output",
          "model_name": "OutputModel",
          "model_module_version": "1.0.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/output",
            "_model_module_version": "1.0.0",
            "_model_name": "OutputModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/output",
            "_view_module_version": "1.0.0",
            "_view_name": "OutputView",
            "layout": "IPY_MODEL_53ad47d669d94024b4eaa94868ace06c",
            "msg_id": "",
            "outputs": [
              {
                "output_type": "display_data",
                "data": {
                  "text/plain": "\u001b[38;2;106;0;255m⠦\u001b[0m ✨ You're running DeepEval's latest \u001b[38;2;106;0;255mFaithfulness Metric\u001b[0m! \u001b[38;2;55;65;81m(using gpt-4o, strict=False, async_mode=True)...\u001b[0m\n",
                  "text/html": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #6a00ff; text-decoration-color: #6a00ff\">⠦</span> ✨ You're running DeepEval's latest <span style=\"color: #6a00ff; text-decoration-color: #6a00ff\">Faithfulness Metric</span>! <span style=\"color: #374151; text-decoration-color: #374151\">(using gpt-4o, strict=False, async_mode=True)...</span>\n</pre>\n"
                },
                "metadata": {}
              }
            ]
          }
        },
        "53ad47d669d94024b4eaa94868ace06c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c8a846cfc54b4a3cac8f2acb32357acd": {
          "model_module": "@jupyter-widgets/output",
          "model_name": "OutputModel",
          "model_module_version": "1.0.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/output",
            "_model_module_version": "1.0.0",
            "_model_name": "OutputModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/output",
            "_view_module_version": "1.0.0",
            "_view_name": "OutputView",
            "layout": "IPY_MODEL_55c9721fbd434e0a9fac1d14cf4d42e4",
            "msg_id": "",
            "outputs": [
              {
                "output_type": "display_data",
                "data": {
                  "text/plain": "✨ You're running DeepEval's latest \u001b[38;2;106;0;255mAnswer Relevancy Metric\u001b[0m! \u001b[38;2;55;65;81m(using TinyLlama 1B, strict=False, async_mode=True)...\u001b[0m\n",
                  "text/html": "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">✨ You're running DeepEval's latest <span style=\"color: #6a00ff; text-decoration-color: #6a00ff\">Answer Relevancy Metric</span>! <span style=\"color: #374151; text-decoration-color: #374151\">(using TinyLlama 1B, strict=False, async_mode=True)...</span>\n</pre>\n"
                },
                "metadata": {}
              }
            ]
          }
        },
        "55c9721fbd434e0a9fac1d14cf4d42e4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/kwschultz/rag-local-llms/blob/main/Advanced_RAG_TinyLlama_LLM_LlamaIndex.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Advanced RAG Challenge\n",
        "### Kasey Schultz, PhD - 09/21/2024\n",
        "\n",
        "To run this notebook:\n",
        "- Run the `pip install ...` code block\n",
        "- Enable GPU on Colab, on the top right panel showing \"RAM\" and \"Disk\", click the down arrow, select T4 GPU and restart the runtime.\n",
        "- Create a directory called './rag_documents/', and [upload this PDF](https://research.ark-invest.com/hubfs/1_Download_Files_ETF_Website/Investment_Case/Investment%20Case%20For%20Disruptive%20Innovation.pdf?__hstc=6077420.c65cff1b18e88e8ac51483b62d058ce9.1610918026083.1628712241411.1628775645389.291&__hssc=6077420.3.1628775645389&__hsfp=814967395.pdf)\n",
        "---\n",
        "\n",
        "# Approach for Advanced RAG\n",
        "\n",
        "## Document Ingestion\n",
        "\n",
        "- Using Llama Index document loader to extract text from PDF file.\n",
        "\n",
        "- Using Llama Index's IngestionPipeline class to combine chunking and text embedding\n",
        "\n",
        "- With more time I would explore using Llama-Index's LlamaParse API that is suited to PDFs, this would likely boost performance.\n",
        "\n",
        "### PDF Parsing\n",
        "\n",
        "- I experimented with the pdfplumber package to programmatically parse the PDF file structure, analyze code blocks, font sizes and whitespace to partition the text into chunks - it did not perform very well on this PDF file without extensive tweaking.\n",
        "\n",
        "### --> Basic Document Ingestion Pipeline\n",
        "\n",
        "#### Chunking\n",
        "- __Chunking__ using the naive `TokenTextSplitter` from LangChain. This chops the full document text into chunks of uniform size, regardless of text content.\n",
        "  - Chunk size is a fixed 300 tokens with 10% chunk overlap to catch breaks in semantic meaning.\n",
        "\n",
        "#### Text Embedder\n",
        "- Embedder used: `\"sentence-transformers/all-MiniLM-L6-v2\"`, embed dimension 384, good performance for small and efficient SentenceTransformer.\n",
        "\n",
        "\n",
        "### --> Advanced  Document Ingestion Pipeline\n",
        "\n",
        "- __Improved chunking__ strategy using `RecursiveCharacterTextSplitter` from LangChain. This is a smarter chunking strategy that divides the text based on new lines, spaces, etc. that bound individual paragraphs and sentences. Chunk sizes vary.\n",
        "  - If time, I want to experiment with `StatisticalChunker` from the `semantic-chunkers` package. This method uses a text embedding and similarity scores to partition the text into semantically similar chunks - to better divide separate topics and entities. There are multiple methods, the \"gradient\" method is where I would start.\n",
        "\n",
        "---\n",
        "## --> Embedding Models\n",
        "\n",
        "### Basic text embedding\n",
        "- For  I am choosing the versatile SenteneceTransformer MiniLM (`\"sentence-transformers/all-MiniLM-L6-v2\"`). It has good performance for it's size (384 dim embedding) and is a good baseline.\n",
        "\n",
        "### Improved text embedding\n",
        "- For advanced embedding I am choosing a larger open source text embedder `\"BAAI/bge-large-en-v1.5\"`. This embedder is focused on RAG, has a larger 1024 token embedding space for more fine grained contextual understanding.\n",
        "\n",
        "### If more time\n",
        "- I would experiment with vision-language model embeddings that preserve the relationship between image/layout and text information within PDFs containing images and graphs. One significant limitation of RAG on PDFs using only text embeddings is that the process of extracting text from the documents discards a significant amount of information.\n",
        "- I would start with the ColPali model and ColBERT embeddings. The vector store and retrieval methods would need to be enhanced to support multi-vectors (tensors).\n",
        "---\n",
        "##  --> Information Retrieval\n",
        "\n",
        "### Vector Store for Document Chunks\n",
        "Using ChromaDB for local vector store, and similarity score to find top k=5 embedded document chunks.\n",
        "- Since number of documents/chunks is small, saving the ChromaDB locally for quick caching and pipeline re-running without recomputing.\n",
        "\n",
        "#### Basic RAG LLM\n",
        "\n",
        "- For Basic RAG I am using a pre-trained HuggingFace model locally. I tried Mistral API and OpenAI API but got rate limited or needed an account upgrade that was not immediate.\n",
        "  - Model: \"TinyLlama/TinyLlama-1.1B-Chat-v1.0\", good performance for a smaller LLM. Allows for quicker iteration and experimentation on the whole pipeline\n",
        "\n",
        "#### Advanced RAG LLM\n",
        "\n",
        "- For advanced RAG this can be upgraded to a medium size LLM with better comprehension, and that has been fine tuned for question answering or RAG (e.g. Mixtral mixtral-8x7b or Llama3.1 8b).\n",
        "- Even the 8-bit quantized version of these models takes a very long time to load on Google Colab. Leaving this for future follow up.\n",
        "\n",
        "## -->  Retrieval Reranking and Answer Synthesis\n",
        "\n",
        "### Basic RAG\n",
        "- Implementing a simple Response Synthesizer from Llama Index\n",
        "  - Using Compact Response mode, top k chunks are aggregated for the context injected into the query prompt. LLM reasons over all chunks for final answer.\n",
        "\n",
        "### Advanced RAG\n",
        "I implemented a more complex Response Synthesizer\n",
        "- Use Re-ranking model to have a more fine-grained look at the relevant chunks\n",
        "  - model = \"sentence-transformers/msmarco-distilroberta-base-v2\"\n",
        "- Use Refinement prompting to allow an LLM to revise the answer by sequentially reviewing the reranked retrieved chunks\n",
        "\n",
        "#### If more time\n",
        "- Can employ hybrid RAG, extracting metadata from the chunks and applying metadata filtering (e.g. keyword extraction and filtering)\n",
        "- Can try FlagReranker('BAAI/bge-reranker-v2-m3'), fine tuned for RAG Re-Ranking\n",
        "  \n",
        "\n",
        "---\n",
        "## --> Performance Evaluation\n",
        "\n",
        "### Benchmark evaluation\n",
        "With no provided answers, and no human subject matter experts to annotate, I used GPT 4o to provide answers to the questions and serve as \"pseudo ground truth\" for evaluation. The GPT 4o prompt for multi-modal question answering from the uploaded PDF document:\n",
        "```\n",
        "Using only the information provided in the pages of that PDF document, please answer the following question. Make sure your response is succinct and contains no more than 40 words. Question: {<question>}\n",
        "```\n",
        "\n",
        "### RAG Evaluation Metrics\n",
        "\n",
        "I will use Llama Index's implementation of DeepEval to programmatically compute RAG evaluation metrics for retriever and reranker performance, as well as hallucination detection. These functions come from `deepeval.metrics`.\n",
        "\n",
        "#### --> Retriever Metrics\n",
        "Retriever is evaluated on the top-k contexts retrieved for the RAG prompt.\n",
        "\n",
        "1. ContextualPrecision\n",
        "  - Requires 'expected_output', I would substitute the GPT-4o responses in leiu of proper ground truth\n",
        "2. ContextualRecall\n",
        "  - Requires 'expected_output'\n",
        "3. ContextualRelevancy\n",
        "  - Independent of \"correct answers\"\n",
        "\n",
        "#### --> Answer Synthesis Metrics\n",
        "\n",
        "1. AnswerRelevancyMetric\n",
        "  1. Can also evaluate the change in answer relevancy using the top 1 chunk vs. using the top k chunks.\n",
        "2. FaithfulnessMetric\n",
        "  1. Hallucinationn detection, evaluating groundedness of responses.\n",
        "\n",
        "#### --> Overall System Metric\n",
        "\n",
        "1. Correctness\n",
        "  1. Using GPT 4o responses as \"correct\" responses (compared to what the simpler LLM generates).\n",
        "2. Can combine all individual metrics for an aggregated score. Simple sum of scores, or the mean to characterize the pipline as a whole (and compare Basic RAG to Advanced).\n",
        "\n",
        "#### --> Timing Metrics\n",
        "\n",
        "- It is important to measure how long each step in the RAG pipeline takes, and the overall time.\n",
        "\n",
        "#### Augmenting the quesion/answer evaluation dataset\n",
        "\n",
        "- With more time, I could use Llama Index to auto-generate potential quesion and answer pairs from the content to augment the evaluation test data.\n",
        "\n",
        "## Evaluation ~~Results~~ Errors with Basic and Advanced RAG\n",
        "\n",
        "- I was not able to complete all the evaluation metrics for Basic RAG or Advanced RAG\n",
        "  - Using the DeepEval standard GPT4 API call for the LLM judge gave me rate limits and 404s even with a paid account.\n",
        "  - I also got errors using the Llama Index wrapper around the DeepEval methods.\n",
        "- I dug into defining a local LLM for judging, and that implementation eventually led to errors in calling the model for the evaluation metrics\n",
        "- With more time I would fix the errors, and complete the batch evalutions and compare the Basic and Advanced implementations\n",
        "\n",
        "<br>\n",
        "\n",
        "- I moved the errored out evaluation calls to the end of the notebook for reference.\n",
        "\n",
        "\n",
        "---"
      ],
      "metadata": {
        "id": "A06DiIXo-tY6"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "---"
      ],
      "metadata": {
        "id": "uSep2VA3Nok6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install --upgrade sentence-transformers semantic-chunkers semantic-router transformers langchain langchain-community \\\n",
        "datasets nltk chromadb llama-index bitsandbytes==0.43.3 accelerate pdfplumber langchain-text-splitters langchain_experimental \\\n",
        "flash-attn faiss-cpu mistralai FlagEmbedding peft langchain_mistralai deepeval lm-format-enforcer llama-index-embeddings-huggingface \\\n",
        "llama-index-embeddings-instructor llama-index-vector-stores-chroma llama-index-embeddings-mistralai llama-index-llms-mistralai \\\n",
        "llama-index-llms-huggingface llama-index-llms-langchain"
      ],
      "metadata": {
        "id": "90s3DMe8oThR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# ----- API KEYS -----\n",
        "import os\n",
        "from google.colab import userdata\n",
        "from huggingface_hub import login\n",
        "\n",
        "mistral_api_key = userdata.get('MISTRAL_API_KEY')\n",
        "hf_token = userdata.get('HF_TOKEN')\n",
        "openai_api_key = userdata.get('OPENAI_API_KEY')\n",
        "\n",
        "login(token = hf_token)\n",
        "\n",
        "os.environ['OPENAI_API_KEY'] = openai_api_key\n",
        "os.environ['HF_TOKEN'] = hf_token\n",
        "os.environ['MISTRAL_API_KEY'] = mistral_api_key\n",
        "os.environ['TRANSFORMERS_CACHE'] = './hf_cache'\n",
        "os.environ['HF_CACHE'] = './hf_cache'\n",
        "\n",
        "os.mkdir('./rag_documents')\n",
        "os.mkdir('./hf_cache')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-PiDNYcQ85mn",
        "outputId": "52d10436-dde9-44af-bf47-81feaac5c69d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The token has not been saved to the git credentials helper. Pass `add_to_git_credential=True` in this function directly or `--add-to-git-credential` if using via `huggingface-cli` if you want to set the git credential as well.\n",
            "Token is valid (permission: read).\n",
            "Your token has been saved to /root/.cache/huggingface/token\n",
            "Login successful\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import textwrap\n",
        "from tqdm import tqdm\n",
        "import transformers\n",
        "from langchain import hub\n",
        "from sentence_transformers import SentenceTransformer\n",
        "from transformers import AutoModel, AutoTokenizer, AutoModelForCausalLM\n",
        "from langchain.embeddings import HuggingFaceEmbeddings\n",
        "from langchain_community.document_loaders import PDFPlumberLoader\n",
        "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "from langchain_text_splitters import CharacterTextSplitter, SentenceTransformersTokenTextSplitter\n",
        "from langchain.docstore.document import Document as LangchainDocument\n",
        "from transformers import BitsAndBytesConfig\n",
        "from FlagEmbedding import FlagReranker\n",
        "from llama_index.core import Document\n",
        "from llama_index.embeddings.huggingface import HuggingFaceEmbedding\n",
        "from llama_index.core.node_parser import SemanticSplitterNodeParser, TokenTextSplitter\n",
        "from llama_index.llms.huggingface import HuggingFaceLLM\n",
        "from llama_index.core.ingestion import IngestionPipeline, IngestionCache\n",
        "from llama_index.embeddings.instructor import InstructorEmbedding\n",
        "from llama_index.core.node_parser import LangchainNodeParser\n",
        "from llama_index.core import SimpleDirectoryReader, Settings, VectorStoreIndex\n",
        "from langchain_experimental.text_splitter import SemanticChunker"
      ],
      "metadata": {
        "id": "uAUrYq3_-28Y"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "device"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5n7fcme5-3Fy",
        "outputId": "486e5f03-da3a-48ab-88dc-a4513f586e9c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "device(type='cuda')"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Embedding Models\n"
      ],
      "metadata": {
        "id": "5Pvkcez8j6nf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# # ----- Basic RAG --------\n",
        "basic_embedder_name = \"sentence-transformers/all-MiniLM-L6-v2\"  # HuggingFace\n",
        "# # ----- Advanced RAG --------\n",
        "advanced_embedder_name = \"BAAI/bge-large-en-v1.5\"  # HuggingFace"
      ],
      "metadata": {
        "id": "EPBTQnjAZbtg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## PDF Parsing and Document Chunking\n"
      ],
      "metadata": {
        "id": "5KSg8c01j4rJ"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Basic Chunking by n-tokens"
      ],
      "metadata": {
        "id": "9P3jtqKyk20k"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def get_doc_pages(doc_path):\n",
        "    doc_loader = PDFPlumberLoader(doc_path)\n",
        "    pages = doc_loader.load()\n",
        "    return pages\n",
        "\n",
        "def get_pdf_text(doc_path):\n",
        "    doc_pages = get_doc_pages(doc_path)\n",
        "    page_text = [p.page_content for p in doc_pages]\n",
        "    full_text = \"\\n\\n\".join(page_text)\n",
        "    return doc_pages, page_text, full_text\n",
        "\n",
        "def basic_chunking(full_text: str, tokenizer_name: str) -> list[str]:\n",
        "    \"\"\"\n",
        "    Split documents into chunks of size `chunk_size` characters and return a\n",
        "    list of chunks (str). Chunk size determined by SentenceTransformer text\n",
        "    embedder context window size.\n",
        "    \"\"\"\n",
        "    text_splitter = TokenTextSplitter(chunk_size=300, chunk_overlap=0)\n",
        "    return text_splitter.split_text(full_text)\n",
        "\n",
        "\n",
        "arkk_pages, arkk_page_text, arkk_full_text = get_pdf_text(\"./rag_documents/Investment_Case_For_Disruptive_Innovation.pdf\")\n",
        "basic_chunks_tokensplit_fulltext = basic_chunking(arkk_full_text, basic_embedder_name)"
      ],
      "metadata": {
        "id": "ZyBwFsZL1-Fv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(basic_chunks_tokensplit_fulltext)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cnGqk4W4vpql",
        "outputId": "ccaf9c0c-6efb-41e5-ae21-277a085057f5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "31"
            ]
          },
          "metadata": {},
          "execution_count": 49
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "textwrap.wrap(basic_chunks_tokensplit_fulltext[0], width=120)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xqOYsvU4vpof",
        "outputId": "fcfd783c-3621-4983-a697-837b6fbef789"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['1 • Why Invest In Disruptive Innovation? As of June 30, 2024 Sources: ARK Investment Management LLC, 2024. Forecasts are',\n",
              " 'inherently limited and cannot be relied upon. For informational purposes only and should not be considered investment',\n",
              " 'advice or a recommendation to buy, sell, or hold any particular security. Past performance is not indicative of future',\n",
              " 'results.   • 2 DISCLOSURE Risks of Investing in Innovation Please note: Companies that ARK believes are capitalizing on',\n",
              " 'disruptive innovation and developing technologies to displace older technologies or create new markets may not in fact',\n",
              " 'do so. ARK aims to educate investors and seeks to size the potential investment opportunity, noting that risks and',\n",
              " 'uncertainties may impact our projections and research models. Investors should use the content presented for',\n",
              " 'informational purposes only, and be aware of market risk, disruptive innovation risk, regulatory risk, and risks related',\n",
              " 'to certain innovation areas. Please read risk disclosure carefully. RISK OF INVESTING IN INNOVATION RAPID PACE OF CHANGE',\n",
              " 'REGULATORY HURDLES EXPOSURE ACROSS SECTORS AND MARKET CAP DISRUPTIVE POLITICAL OR LEGAL PRESSURE INNOVATION UNCERTAINTY',\n",
              " 'AND UNKNOWNS COMPETITIVE LANDSCAPE à Aim for a cross-sector understanding of technology à Aim to understand the',\n",
              " 'regulatory, market, sector, and combine top-down and bottom-up research.']"
            ]
          },
          "metadata": {},
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "textwrap.wrap(basic_chunks_tokensplit_fulltext[1], width=120)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LVjEE4iEvpl8",
        "outputId": "6869f4fd-e68e-4263-da00-13ed53d71211"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['and company risks. (See Disclosure Page) Sources: ARK Investment Management LLC, 2023.   3 Public Blockchains Upon',\n",
              " 'large-scale adoption, we believe all money and contracts likely will migrate onto Public Blockchains that enableand',\n",
              " 'Multiomic verifydigital scarcityand proof of ownership. The financial ecosystem is likely to reconfigure to accommodate',\n",
              " 'the rise of Sequencing Cryptocurrencies and Smart Contracts. These technologies increase transparency, reduce the',\n",
              " 'influence of capital and The cost to gather, sequence, and understand regulatory controls, and collapse contract',\n",
              " 'execution costs. In digital biological data is falling precipitously. such a world, Digital Wallets would become',\n",
              " 'increasingly Multiomic Technologies provide research Five Innovation necessary as more assets become money-like, and',\n",
              " 'corporations scientists, therapeutic organizations and health and consumers adapt to the new financial infrastructure.',\n",
              " 'platforms with unprecedented access to DNA, Corporate structures themselves may be called into question. RNA, protein,',\n",
              " 'and digital health data. Cancer care Platforms Are should transform with pan-cancer blood tests. Artificial Intelligence',\n",
              " 'Multiomic data should feed into novel Precision Therapies using emerging gene editing techniques Converging And',\n",
              " 'Computational systems and software that evolve with data can that target and cure rare diseases and chronic solve',\n",
              " 'intractable problems, automate knowledge work, and conditions. Multiomics shouldunlock entirely new accelerate',\n",
              " 'technology’s integration into every economic sector. We Programmable Biology capabilities, including']"
            ]
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "textwrap.wrap(basic_chunks_tokensplit_fulltext[2], width=120)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hsFuo8QCaLYK",
        "outputId": "ef0eb9fb-221e-4959-8d98-a9dcc7eb3b3f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['technology ’ s integration into every economic sector. we programmable biology capabilities, including the defining this',\n",
              " 'believe the adoption of neural networks should prove more design and synthesis of novel biological momentous than the',\n",
              " 'introduction of the internet. at scale these constructs with applications across industries, systems will require',\n",
              " 'unprecedented computational resources, and particularly agriculture and food production. technological era ai - specific',\n",
              " 'compute hardware should dominate the next gen cloud datacenters that train and operate ai models. the potential for end',\n",
              " \"- users is clear : a constellation of ai - driven intelligent devices that pervade people's lives, changing the way that\",\n",
              " 'they spend, work, and play. the adoption of artificial intelligence should transform every sector, impact every',\n",
              " 'business, and catalyze every innovation platform. energy storage robotics declining costs of advanced battery technology',\n",
              " 'should cause catalyzed by artificial intelligence, adaptive robots can an explosion in form factors, enabling autonomous',\n",
              " 'mobility operate alongside humans and navigate legacy systems that collapse the cost of getting people and things from',\n",
              " 'infrastructure, changing the way products are made and place to place. electric drivetrain cost declines should unlock',\n",
              " 'sold. 3d printing should contribute to the digitization of micro - mobility and aerial systems, including flying taxis,',\n",
              " 'manufacturing, increasing not only the performance and enabling business models that transform the landscape of cities']"
            ]
          },
          "metadata": {},
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "63XZNm5U8ouR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Basic RAG with Llama-Index, ChromaDB, TinyLlama (LLM)"
      ],
      "metadata": {
        "id": "U3xZ7ZUUYfS7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Must create doc_directory and upload to Colab\n",
        "reader = SimpleDirectoryReader('./rag_documents/')\n",
        "ark_documents = reader.load_data()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PhGes6excaKI",
        "outputId": "c2638d69-e3d9-4aaa-f104-237705d5e232"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:pypdf._reader:Ignoring wrong pointing object 26 0 (offset 0)\n",
            "WARNING:pypdf._reader:Ignoring wrong pointing object 125 0 (offset 0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "---"
      ],
      "metadata": {
        "id": "OPrNbotYix3L"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## LLM and Embedder"
      ],
      "metadata": {
        "id": "dB7hGB00n5VV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# from transformers import AutoConfig\n",
        "\n",
        "quantization_config = BitsAndBytesConfig(\n",
        "    load_in_8bit=True,\n",
        ")\n",
        "\n",
        "\n",
        "hf_model_name = \"TinyLlama/TinyLlama-1.1B-Chat-v1.0\"\n",
        "context_window = 2048\n",
        "llm_model = HuggingFaceLLM(\n",
        "    model_name=hf_model_name,\n",
        "    tokenizer_name=hf_model_name,\n",
        "    context_window=context_window,\n",
        "    max_new_tokens=156,\n",
        "    model_kwargs={\"quantization_config\": quantization_config},\n",
        "    generate_kwargs={\"top_k\": 50},\n",
        "    device_map=\"cuda\",\n",
        ")\n",
        "\n",
        "# hf_model_name = 'mistralai/Mixtral-8x7B-v0.1'\n",
        "# context_window = 4096\n",
        "# llm_model = HuggingFaceLLM(\n",
        "#     model_name=hf_model_name,\n",
        "#     tokenizer_name=hf_model_name,\n",
        "#     context_window=context_window,\n",
        "#     max_new_tokens=256,\n",
        "#     model_kwargs={\"quantization_config\": quantization_config},\n",
        "#     generate_kwargs={\"temperature\": 0.7, \"top_k\": 50, \"top_p\": 0.95},\n",
        "#     device_map=\"cuda\",\n",
        "# )\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(\n",
        "    hf_model_name, cache_dir='./hf_cache'\n",
        ")\n",
        "\n",
        "embed_model = HuggingFaceEmbedding(model_name=basic_embedder_name, cache_folder='./hf_cache')\n",
        "\n",
        "# Llama index settings from model\n",
        "Settings.tokenizer = tokenizer\n",
        "Settings.llm = llm_model\n",
        "Settings.embed_model = embed_model\n",
        "Settings.context_window = context_window\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104,
          "referenced_widgets": [
            "8908414258b44c65bdedc19e818831c7",
            "b41efc3d9a0a4ed495b7c417a20279a7",
            "a9739a5cc3fb434c8cd8018786c602ac",
            "dc30dc8f96564f7e8d426f2703c9b1a7",
            "4a5c416e6a514942a87c2ce4e9c6d59a",
            "01f81eb5fe4f466691fda4cec63d9454",
            "2443165a104d49aeb7abccc4346e71c8",
            "ef7ff31cada24bbfaa30f831fcff06e5",
            "8315f039e7f04ed68535b7420dd4cea2",
            "26b5933cb8864c2d86f164fd3baa054c",
            "46b3f392c007488fa6ff34aec2403bc8"
          ]
        },
        "id": "yugBqQNv0ZB2",
        "outputId": "e94ddc87-7e78-4c76-87a9-61126da5ad51"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "generation_config.json:   0%|          | 0.00/124 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "8908414258b44c65bdedc19e818831c7"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "y0WC8AzDD_3X"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Llama Index Ingestion Pipeline"
      ],
      "metadata": {
        "id": "AiyvMRZUFCrp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# from llama_index.llms.mistralai import MistralAI\n",
        "# from llama_index.embeddings.mistralai import MistralAIEmbedding\n",
        "\n",
        "# Wrap the LangChain parser\n",
        "basic_langchain_parser = LangchainNodeParser(\n",
        "    TokenTextSplitter(chunk_size=300, chunk_overlap=30)\n",
        ")\n",
        "\n",
        "Settings.chunk_size = 300\n",
        "Settings.chunk_overlap = 30\n",
        "\n",
        "# create the pipeline with transformations\n",
        "basic_ingestion_pipeline = IngestionPipeline(\n",
        "    transformations=[\n",
        "        basic_langchain_parser,\n",
        "        embed_model\n",
        "    ]\n",
        ")\n",
        "\n",
        "# run the pipeline\n",
        "basic_nodes = basic_ingestion_pipeline.run(documents=ark_documents)"
      ],
      "metadata": {
        "id": "DxgSr3hcFgKW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(basic_nodes)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "R4l06YcUT10h",
        "outputId": "96974b16-3fff-4aa8-e570-5f217aa55872"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "63"
            ]
          },
          "metadata": {},
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Llama Index and ChromaDB Vector Store"
      ],
      "metadata": {
        "id": "PhsnCItVLghn"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import chromadb\n",
        "from llama_index.core import VectorStoreIndex, SimpleDirectoryReader\n",
        "from llama_index.vector_stores.chroma import ChromaVectorStore\n",
        "from llama_index.core import StorageContext\n",
        "from llama_index.core import PromptTemplate\n",
        "from llama_index.core import VectorStoreIndex, SimpleDirectoryReader\n",
        "from llama_index.core import Settings\n",
        "from llama_index.core.indices.vector_store.retrievers.retriever import VectorIndexRetriever\n",
        "from llama_index.core import VectorStoreIndex, get_response_synthesizer\n",
        "from llama_index.core.retrievers import VectorIndexRetriever\n",
        "from llama_index.core.query_engine import RetrieverQueryEngine\n",
        "from llama_index.core.postprocessor import SimilarityPostprocessor\n",
        "from llama_index.core.response_synthesizers import ResponseMode\n",
        "from llama_index.core.prompts.base import PromptTemplate\n",
        "from llama_index.core.prompts.prompt_type import PromptType"
      ],
      "metadata": {
        "id": "ICjn4ekVO37D"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# initialize client, setting path to save data\n",
        "db = chromadb.PersistentClient(path=\"./chroma_db_basic_minilm\")\n",
        "\n",
        "# create collection\n",
        "chroma_collection = db.get_or_create_collection(\"quickstart\")\n",
        "\n",
        "# assign chroma as the vector_store to the context\n",
        "vector_store = ChromaVectorStore(chroma_collection=chroma_collection)\n",
        "storage_context = StorageContext.from_defaults(vector_store=vector_store)\n",
        "\n",
        "# create your index\n",
        "basic_index = VectorStoreIndex(basic_nodes, storage_context=storage_context, embed_model=embed_model)\n"
      ],
      "metadata": {
        "id": "deuQu7qXLf7h"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "LPEHvkv2znUx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Llama Index Retriever (Basic), with Prompt Engineering"
      ],
      "metadata": {
        "id": "H1ITVBK1N85n"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Retriever function\n",
        "basic_retriever = VectorIndexRetriever(\n",
        "    index=basic_index,\n",
        "    similarity_top_k=5,\n",
        ")\n",
        "\n",
        "# Craft the prompt template\n",
        "DEFAULT_TEXT_QA_PROMPT_TMPL = (\n",
        "    \"Context information is below.\\n\"\n",
        "    \"---------------------\\n\"\n",
        "    \"{context_str}\\n\"\n",
        "    \"---------------------\\n\"\n",
        "    \"Given the context information and not prior knowledge, \"\n",
        "    \"answer the query. In your answer, do not begin by saying 'Based on the given context information', just give the answer directly. \\n\"\n",
        "    \"Query: {query_str}\\n\"\n",
        "    \"Answer: \"\n",
        ")\n",
        "DEFAULT_TEXT_QA_PROMPT = PromptTemplate(\n",
        "    DEFAULT_TEXT_QA_PROMPT_TMPL, prompt_type=PromptType.QUESTION_ANSWER\n",
        ")\n",
        "\n",
        "\n",
        "# configure response synthesizer, using the LLM,\n",
        "# Compact response concatenates all retrieved relevant chunks\n",
        "# Text QA template taken from LangChain, optimized for Llama3\n",
        "basic_response_synthesizer = get_response_synthesizer(\n",
        "    llm = llm_model,\n",
        "    response_mode=ResponseMode.COMPACT,\n",
        "    text_qa_template=DEFAULT_TEXT_QA_PROMPT\n",
        ")\n",
        "\n",
        "# assemble query engine\n",
        "basic_query_engine = RetrieverQueryEngine(\n",
        "    retriever=basic_retriever,\n",
        "    response_synthesizer=basic_response_synthesizer,\n",
        ")\n"
      ],
      "metadata": {
        "id": "W-Ll_FQXN8DM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "ENQthqbUj48M"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Questions for RAG"
      ],
      "metadata": {
        "id": "7a9JO89sAbX0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "questions = \"\"\"\n",
        "Q01. What is the core objective of investing in disruptive innovation according to ARK?\n",
        "\n",
        "Q02. What are the significant risks associated with investing in innovation as highlighted by ARK?\n",
        "\n",
        "Q03. Can you list the converging innovation platforms identified by ARK?\n",
        "\n",
        "Q04. How does ARK describe the impact of Artificial Intelligence on technology’s integration into economic sectors?\n",
        "\n",
        "Q05. What transformative potential does Multiomic Sequencing hold according to ARK?\n",
        "\n",
        "Q06. What are the implications of declining battery technology costs as outlined by ARK?\n",
        "\n",
        "Q07. How is the field of Robotics anticipated to evolve with the advancements in AI?\n",
        "\n",
        "Q08. What does the ARK’s Convergence Scoring Framework illustrate about innovation platforms?\n",
        "\n",
        "Q09. How do neural networks serve as a catalyst for other technologies?\n",
        "\n",
        "Q10. What unique view does ARK have towards Autonomous Mobility and its market potential?\n",
        "\n",
        "Q11. How do AI Chatbots contribute to the development of robotaxis?\n",
        "\n",
        "Q12. What are breakthroughs in DNA Sequencing, particularly with neural networks?\n",
        "\n",
        "Q13. How does the application of AI language models in robotics enhance general task completion rates?\n",
        "\n",
        "Q14. In what ways are battery advances critical to the future of intelligent devices and augmented reality?\n",
        "\n",
        "Q15. How do reusable rockets contribute to global connectivity?\n",
        "\n",
        "Q16. What economic implications do disruptive innovations have according to ARK?\n",
        "\n",
        "Q17. What are the top 10 holdings of ARK Innovation ETF (ARKK)?\n",
        "\n",
        "Q18. What thematic strategies do ARK ETFs focus on?\n",
        "\n",
        "Q19. What is ARK's strategy for capturing the benefits of disruptive innovation in its investment approach?\n",
        "\n",
        "Q20. How does ARK ensure its investment strategies align with reality of disruptive innovation trends?\n",
        "\"\"\"\n",
        "questions = [q[5:] for q in questions.splitlines() if q]"
      ],
      "metadata": {
        "id": "Q4IFEBC8_dJk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Answers from GPT4o (pseudo ground truth)"
      ],
      "metadata": {
        "id": "bt11tAiFCgZ4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "gpt_answers = \"\"\"\n",
        "A01. The core objective of investing in disruptive innovation, according to ARK, is to access long-term growth by investing in companies at the forefront of technological advancements, transforming industries and creating new markets​.\n",
        "\n",
        "A02. ARK highlights significant risks in investing in innovation, including market risk, regulatory hurdles, competitive pressures, political or legal challenges, and uncertainties surrounding the success of disruptive technologies in creating new markets.\n",
        "\n",
        "A03. The converging innovation platforms identified by ARK are public blockchains, multiomic sequencing, artificial intelligence, energy storage, and robotics​.\n",
        "\n",
        "A04. RK describes Artificial Intelligence as accelerating technology integration across all economic sectors by automating knowledge work, solving complex problems, and transforming businesses through AI-driven intelligent devices, which impact how people spend, work, and play​.\n",
        "\n",
        "A05. According to ARK, Multiomic Sequencing holds transformative potential by enabling precision therapies, advancing gene editing, and unlocking programmable biology, which could revolutionize healthcare and industries like agriculture and food production​.\n",
        "\n",
        "A06. ARK outlines that declining battery technology costs will drive the expansion of autonomous mobility, enable micro-mobility and aerial systems, reduce transportation costs, and transform energy systems by shifting from liquid fuel to electricity.\n",
        "\n",
        "A07. ARK anticipates that advancements in AI will enable adaptive robots to operate alongside humans, navigate legacy infrastructure, enhance manufacturing through 3D printing, and reduce production costs with AI-guided robots.\n",
        "\n",
        "A08. ARK’s Convergence Scoring Framework illustrates that innovation platforms, such as AI, robotics, and blockchain, are converging, generating a significant technological wave with the potential to transform industries and drive economic growth​.\n",
        "\n",
        "A09. Neural networks act as a catalyst by automating knowledge work, solving complex problems, and accelerating technology integration across sectors, driving the development of AI-driven devices that transform industries and catalyze other innovation platforms.\n",
        "\n",
        "A10. ARK views Autonomous Mobility as transformative, driven by declining battery costs, enabling new forms of transportation like flying taxis, reducing transport costs, and potentially reshaping city landscapes by decreasing the need for individual car ownership\n",
        "\n",
        "A11. AI chatbots, like ChatGPT, improve conversational AI, which helps autonomous systems like robotaxis interpret complex instructions and interact with users. This advances the human-machine interface, making robotaxi operations more efficient and user-friendly​\n",
        "\n",
        "A12. Breakthroughs in DNA sequencing, powered by neural networks, provide unprecedented access to digital biological data, enabling precision therapies. These advancements accelerate gene editing and programmable biology, unlocking new possibilities in healthcare and industries like agriculture​.\n",
        "\n",
        "A13. AI language models in robotics enhance task completion rates by improving robots' ability to understand and execute complex instructions, enabling more adaptive and precise interactions in various environments​\n",
        "\n",
        "A14. Battery advances are critical as they reduce costs, enabling more efficient intelligent devices and augmented reality systems by supporting longer usage, greater mobility, and integration into everyday tasks and environments​.\n",
        "\n",
        "A15. Reusable rockets reduce the cost of launching satellite constellations, enabling uninterruptible global connectivity by facilitating the deployment of infrastructure needed for consistent communication and data services across the world​\n",
        "\n",
        "A16. According to ARK, disruptive innovations drive economic growth by enhancing productivity, lowering costs, and creating new markets, ultimately contributing to significant long-term gains in real GDP and consumer surplus.\n",
        "\n",
        "A17. The top 10 holdings of ARK Innovation ETF (ARKK) are Tesla, Roku, Coinbase, Roblox, Block, CRISPR Therapeutics, Robinhood, UiPath, Palantir, and Shopify, collectively representing 63% of the portfolio.\n",
        "\n",
        "A18.ARK ETFs focus on thematic strategies such as innovation, next-generation internet, autonomous technology and robotics, genomic revolution, fintech innovation, Israel innovative technology, 3D printing, and space exploration and innovation.\n",
        "\n",
        "A19. ARK's strategy for capturing disruptive innovation involves combining top-down and bottom-up research to identify innovative companies, focusing on multi-cap exposure across sectors, and aiming for the best risk-reward opportunities from innovation-driven themes.\n",
        "\n",
        "A20. ARK ensures alignment with disruptive innovation trends by conducting rigorous research, combining top-down and bottom-up analysis, and continually assessing both market and sector risks, as well as regulatory and competitive landscapes​.\n",
        "\"\"\"\n",
        "gpt_answers = [a[5:] for a in gpt_answers.splitlines() if a]"
      ],
      "metadata": {
        "id": "d5saEbqCCfn6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "aGD7c6wjEJzs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Test one RAG question"
      ],
      "metadata": {
        "id": "KKAxKs8gcQJ4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(questions[0])\n",
        "print()\n",
        "# response = retrieval_chain.invoke({\"input\": questions[0]}) # for LangChain\n",
        "# answer = response[\"answer\"]\n",
        "\n",
        "response = basic_query_engine.query(questions[0])  #Llama Index\n",
        "print(str(response))\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "h6tId1s6_ja7",
        "outputId": "b4239c85-759f-41dc-9210-54ca38d5cbd8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "What is the core objective of investing in disruptive innovation according to ARK?\n",
            "\n",
            "\n",
            "ARK aims to educate investors and seeks to size the potential investment opportunity, noting that risks and uncertainties may impact our projections and research models. Investors should use the content presented for informational purposes only, and be aware of market risk, disruptive innovation risk, regulatory risk, and risks related to certain innovation areas. Please read risk disclosure carefully.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "textwrap.wrap(response.response, width=100)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kxla4C4FIpGE",
        "outputId": "95296b23-bbb4-4591-e1cf-da185abf6a0c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[' ARK aims to educate investors and seeks to size the potential investment opportunity, noting that',\n",
              " 'risks and uncertainties may impact our projections and research models. Investors should use the',\n",
              " 'content presented for informational purposes only, and be aware of market risk, disruptive',\n",
              " 'innovation risk, regulatory risk, and risks related to certain innovation areas. Please read risk',\n",
              " 'disclosure carefully.']"
            ]
          },
          "metadata": {},
          "execution_count": 63
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "kq2Ax3OwIpDN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Call all questions and store answers"
      ],
      "metadata": {
        "id": "-hYCrwSCcg_m"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "basic_rag_answers = []\n",
        "basic_rag_responses = []\n",
        "for q in tqdm(questions):\n",
        "    # response = retrieval_chain.invoke({\"input\": q}) # LangChain\n",
        "    # answer = response[\"answer\"]\n",
        "    response = basic_query_engine.query(q)  # LlamaIndex\n",
        "    basic_rag_answers.append(response.response)\n",
        "    basic_rag_responses.append(response)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ruB0YPEWcOkN",
        "outputId": "5bd1de51-a7f9-4d4e-ed45-1904ea84a6b9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 20/20 [05:24<00:00, 16.24s/it]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "with open('basic_rag_answers.txt', 'w') as f:\n",
        "    for i, answer in enumerate(basic_rag_answers):\n",
        "        f.write(f'A{i+1:02}. {answer}\\n---------------\\n\\n')"
      ],
      "metadata": {
        "id": "TTQTtld0cOh4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "basic_rag_response_contexts = []\n",
        "for response in basic_rag_responses:\n",
        "    basic_rag_response_contexts.append([n.text for n in response.source_nodes])"
      ],
      "metadata": {
        "id": "BQ6axZvNcOdM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "basic_rag_evaluation_data = []\n",
        "for i in range(len(questions)):\n",
        "  eval_data = {\n",
        "      \"input\": questions[i],\n",
        "      \"actual_output\": basic_rag_answers[i],\n",
        "      \"expected_output\": gpt_answers[i],\n",
        "      \"context\": basic_rag_response_contexts[i],\n",
        "      'response': basic_rag_responses[i]\n",
        "  }\n",
        "  basic_rag_evaluation_data.append(eval_data)\n"
      ],
      "metadata": {
        "id": "jm_65xx_kJmR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pickle\n",
        "with open('basic_rag_evaluation_data.pkl', 'wb') as f:\n",
        "    pickle.dump(basic_rag_evaluation_data, f)"
      ],
      "metadata": {
        "id": "39UvQgVioWJ8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pickle\n",
        "with open('basic_rag_evaluation_data.pkl', 'rb') as f:\n",
        "    basic_rag_evaluation_data = pickle.load(f)"
      ],
      "metadata": {
        "id": "PNWQL69kkJgS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# unpack saved data\n",
        "basic_rag_answers = []\n",
        "basic_rag_response_contexts = []\n",
        "basic_rag_responses = []\n",
        "\n",
        "for i in range(len(questions)):\n",
        "  eval_data = basic_rag_evaluation_data[i]\n",
        "  basic_rag_answers.append(eval_data['actual_output'])\n",
        "  basic_rag_response_contexts.append(eval_data['context'])\n",
        "  basic_rag_responses.append(eval_data['response'])\n"
      ],
      "metadata": {
        "id": "ipq9C8DO6rY7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Evaluations\n",
        "\n"
      ],
      "metadata": {
        "id": "4b9ElTTiDp1A"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# from llama_index.llms.openai import OpenAI\n",
        "# gpt = OpenAI(temperature=0, model=\"gpt-3.5-turbo\")\n",
        "\n",
        "# # hit rate limiit with OpenAI (default evaluation method for DeepEval)"
      ],
      "metadata": {
        "id": "83hwDIvdqVah"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Trying to use Llama Index wrapper for Eval methods\n",
        "\n",
        "import nest_asyncio\n",
        "nest_asyncio.apply()\n",
        "\n",
        "# Faithfulness\n",
        "from llama_index.core.evaluation import (\n",
        "    FaithfulnessEvaluator,\n",
        "    RelevancyEvaluator,\n",
        "    CorrectnessEvaluator,\n",
        "    SemanticSimilarityEvaluator,\n",
        "    ContextRelevancyEvaluator,\n",
        "    AnswerRelevancyEvaluator,\n",
        ")\n",
        "\n",
        "faithfulness = FaithfulnessEvaluator(llm=llm_model)\n",
        "relevancy = RelevancyEvaluator(llm=llm_model)\n",
        "correctness = CorrectnessEvaluator(llm=llm_model)\n",
        "semantic_similarity = SemanticSimilarityEvaluator()\n",
        "context_relevancy = ContextRelevancyEvaluator()\n",
        "answer_relevancy = AnswerRelevancyEvaluator(llm=llm_model)"
      ],
      "metadata": {
        "id": "XzcCVC2FqfsE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "faithfulness_eval = faithfulness.evaluate_response(response=basic_rag_responses[0], query=questions[0])\n",
        "print(f\"\\nFaithfulness score for Q01. with Basic RAG: {faithfulness_eval.score}\")\n",
        "print(f\"\\nFaithfulness reason for Q01. with Basic RAG: {faithfulness_eval.reason}\")\n",
        "print('n', faithfulness_eval)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 40
        },
        "id": "yaqW59DC8dJY",
        "outputId": "3b3d55c1-cf1d-4bc1-cb02-63c0a6d9bfcd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[38;2;106;0;255m⠼\u001b[0m ✨ You're running DeepEval's latest \u001b[38;2;106;0;255mFaithfulness Metric\u001b[0m! \u001b[38;2;55;65;81m(using gpt-3.5-turbo, strict=False, async_mode=True)...\u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #6a00ff; text-decoration-color: #6a00ff\">⠼</span> ✨ You're running DeepEval's latest <span style=\"color: #6a00ff; text-decoration-color: #6a00ff\">Faithfulness Metric</span>! <span style=\"color: #374151; text-decoration-color: #374151\">(using gpt-3.5-turbo, strict=False, async_mode=True)...</span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "relevancy.evaluate_response(response=basic_rag_responses[0], query=questions[0], reference=gpt_answers[0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c1tyWDN189Po",
        "outputId": "323fc557-b006-4aeb-c024-db2004e8e029"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "EvaluationResult(query='What is the core objective of investing in disruptive innovation according to ARK?', contexts=['•Risks of Investing in InnovationPlease note: Companies that ARK believes are capitalizing on disruptive innovation and developing technologies to displace older technologies or create new markets may not in fact do so. ARK aims to educate investors and seeks to size the potential investment opportunity, noting that risks and uncertainties may impact our projections and research models. Investors should use the content presented for informational purposes only, and be aware of market risk, disruptive innovation risk, regulatory risk, and risks related to certain innovation areas. Please read risk disclosure carefully.\\nDISRUPTIVE INNOVATIONRAPID PACE OF CHANGE\\nUNCERTAINTY AND UNKNOWNS EXPOSURE ACROSS SECTORS AND MARKET CAPRISK OF INVESTING', 'IN INNOVATIONREGULATORY HURDLES\\nCOMPETITIVE LANDSCAPEPOLITICAL OR LEGAL PRESSURE\\nSources: ARK Investment Management LLC, 2023.à   Aim to understand the regulatory, market, sector,        and company risks. (See Disclosure Page)à   Aim for a cross-sector understanding of technology   and combine top-down and bottom-up research.2DISCLOSURE', '3\\nFive Innovation Platforms Are Converging And Defining This Technological Era\\nSources: ARK Investment Management LLC, 2024. This ARK analysis is based on a range of underlying data from external sources, which may be provided upon request. Forecasts are inherently limited and cannot be relied upon. For informational purposes only and should not be considered investment advice or a recommendation to buy, sell, or hold any particular security. Past performance is not indicative of future results.Public BlockchainsUpon large-scale adoption, we believe all money and contracts likely will migrate onto Public Blockchains that enableand verifydigital scarcityand proof of ownership. The financial ecosystem is likely to reconfigure to accommodate the rise of Cryptocurrencies and Smart Contracts. These technologies increase', 'generation infrastructure towards the edge of the network. RoboticsCatalyzed by artificial intelligence, Adaptive Robots can operate alongside humans and navigate legacy infrastructure, changing the way products are made and sold. 3D Printing should contribute to the digitization of manufacturing, increasing not only the performance and precision of end-use parts but also the resilience of supply chains. Meanwhile, the world’s fastest robots, Reusable Rockets, should continue to reduce the cost of launching satellite constellations and enable uninterruptible connectivity. A nascent innovation platform, robotics could collapse the cost of distance with hypersonic travel, the cost of manufacturing complexity with 3D printers, and the cost of production with AI-guided robots. WHY INVEST IN DISRUPTIVE INNOVATION?', 'definitions of task quality. The sources used are Dell’Acqua et al. 2023 and GitHub 2022. Forecasts are inherently limited and cannot be relied upon. For informational purposes only and should not be considered investment advice or a recommendation to buy, sell, or hold any particular security. Past performance is not indicative of future results.Without CopilotWith CopilotProductivity of Developers On Coding Tasks Using Github Copilot in 20232.2xWHY INVEST IN DISRUPTIVE INNOVATION?'], response='\\nARK aims to educate investors and seeks to size the potential investment opportunity, noting that risks and uncertainties may impact our projections and research models. Investors should use the content presented for informational purposes only, and be aware of market risk, disruptive innovation risk, regulatory risk, and risks related to certain innovation areas. Please read risk disclosure carefully.', passing=False, feedback='\\nDisruptive innovation is a term used to describe technological advancements that disrupt existing markets, industries, and business models. ARK aims to educate investors and seeks to size the potential investment opportunity, noting that risks and uncertainties may impact our projections and research models. Investors should use the content presented for informational purposes only, and be aware of market risk, disruptive innovation risk, regulatory risk, and risks related to certain innovation areas. Please read risk disclosure carefully.\\n\\nARK Investment Management LLC, 2023.\\n\\nARK aims to educate investors and seeks to size the potential investment', score=0.0, pairwise_source=None, invalid_result=False, invalid_reason=None)"
            ]
          },
          "metadata": {},
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "SEHWxqziHLCf"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Evaluation Wrapper Errors with Llama Index\n",
        "- I was not able to complete all the evaluation metrics for Basic RAG or Advanced RAG\n",
        "  - Using the standard GPT4 API call for the LLM judge gave me rate limits and 404s even with a paid account\n",
        "  - I dug into defining a local LLM for judging, and that implementation eventually led to errors in calling the model for the evaluation metrics\n",
        "- With more time I would fix the errors, and complete the batch evalutions and compare the Basic and Advanced implementations\n",
        "\n",
        "<br>\n",
        "\n",
        "- I moved the errored out evaluation calls to the end of the notebook for reference.\n"
      ],
      "metadata": {
        "id": "aeQ762OsHK_n"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "---"
      ],
      "metadata": {
        "id": "FMH0zmE6PKcp"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Advanced RAG"
      ],
      "metadata": {
        "id": "PiglmBnlGbFx"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "## Advanced RAG\n",
        "from llama_index.core.postprocessor import SentenceTransformerRerank\n",
        "\n",
        "adv_embed_model = HuggingFaceEmbedding(model_name=advanced_embedder_name, cache_folder='./hf_cache')\n",
        "\n",
        "# Llama index settings from model\n",
        "Settings.tokenizer = tokenizer\n",
        "Settings.llm = llm_model\n",
        "Settings.embed_model = adv_embed_model\n",
        "Settings.context_window = context_window\n",
        "\n",
        "# Wrap the LangChain parser\n",
        "advanced_langchain_parser = LangchainNodeParser(\n",
        "    RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=0)\n",
        ")\n",
        "Settings.chunk_size = 500\n",
        "Settings.chunk_overlap = 00\n",
        "\n",
        "# create the pipeline with transformations\n",
        "advanced_ingestion_pipeline = IngestionPipeline(\n",
        "    transformations=[\n",
        "        advanced_langchain_parser,\n",
        "        adv_embed_model\n",
        "    ]\n",
        ")\n",
        "# run the pipeline\n",
        "advanced_nodes = advanced_ingestion_pipeline.run(documents=ark_documents)\n",
        "\n",
        "# initialize client, setting path to save data\n",
        "adv_db = chromadb.PersistentClient(path=\"./chroma_db_advanced\")\n",
        "# create collection\n",
        "adv_chroma_collection = adv_db.get_or_create_collection(\"quickstart\")\n",
        "# assign chroma as the vector_store to the context\n",
        "adv_vector_store = ChromaVectorStore(chroma_collection=adv_chroma_collection)\n",
        "adv_storage_context = StorageContext.from_defaults(vector_store=adv_vector_store)\n",
        "# create your index\n",
        "advanced_index = VectorStoreIndex(advanced_nodes, storage_context=adv_storage_context, embed_model=adv_embed_model)\n",
        "\n",
        "advanced_retriever = VectorIndexRetriever(\n",
        "    index=advanced_index,\n",
        "    similarity_top_k=8,\n",
        ")\n",
        "\n",
        "# Craft the prompt template\n",
        "# QA Template\n",
        "DEFAULT_TEXT_QA_PROMPT_TMPL = (\n",
        "    \"Context information is below.\\n\"\n",
        "    \"---------------------\\n\"\n",
        "    \"{context_str}\\n\"\n",
        "    \"---------------------\\n\"\n",
        "    \"Given the context information and not prior knowledge, \"\n",
        "    \"answer the query. In your answer, do not begin by saying 'Based on the given context information', just give the answer directly. \\n\"\n",
        "    \"Query: {query_str}\\n\"\n",
        "    \"Answer: \"\n",
        ")\n",
        "DEFAULT_TEXT_QA_PROMPT = PromptTemplate(\n",
        "    DEFAULT_TEXT_QA_PROMPT_TMPL, prompt_type=PromptType.QUESTION_ANSWER\n",
        ")\n",
        "# Refinement template\n",
        "DEFAULT_REFINE_PROMPT_TMPL = (\n",
        "    \"The original query is as follows: {query_str}\\n\"\n",
        "    \"We have provided an existing answer: {existing_answer}\\n\"\n",
        "    \"We have the opportunity to refine the existing answer \"\n",
        "    \"(only if needed) with some more context below.\\n\"\n",
        "    \"------------\\n\"\n",
        "    \"{context_msg}\\n\"\n",
        "    \"------------\\n\"\n",
        "    \"Given the new context, refine the original answer to better \"\n",
        "    \"answer the query. \"\n",
        "    \"If the context isn't useful, return the original answer.\\n\"\n",
        "    \"Refined Answer: \"\n",
        ")\n",
        "DEFAULT_REFINE_PROMPT = PromptTemplate(\n",
        "    DEFAULT_REFINE_PROMPT_TMPL, prompt_type=PromptType.REFINE\n",
        ")\n",
        "\n",
        "# configure response synthesizer, using the LLM,\n",
        "# Compact response concatenates all retrieved relevant chunks\n",
        "# Text QA template taken from LangChain, optimized for Llama3\n",
        "advanced_response_synthesizer = get_response_synthesizer(\n",
        "    llm = llm_model,\n",
        "    response_mode=ResponseMode.REFINE,\n",
        "    text_qa_template=DEFAULT_TEXT_QA_PROMPT,\n",
        "    refine_template=DEFAULT_REFINE_PROMPT\n",
        ")"
      ],
      "metadata": {
        "id": "81WECqGAD_-4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# assemble advanced query engine\n",
        "advanced_query_engine = RetrieverQueryEngine(\n",
        "    retriever=advanced_retriever,\n",
        "    response_synthesizer=advanced_response_synthesizer,\n",
        "    # Pretrained ReRanker\n",
        "    node_postprocessors = [\n",
        "        SentenceTransformerRerank(\n",
        "            model=\"sentence-transformers/msmarco-distilroberta-base-v2\", top_n=3\n",
        "        )\n",
        "    ]\n",
        "    # Another ReRanker option, fine tuned for RAG Re-Ranking\n",
        "    # reranker = FlagReranker('BAAI/bge-reranker-v2-m3', use_fp16=True)\n",
        ")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 330,
          "referenced_widgets": [
            "4627e7e7c9e24a93bd227cd5c375d7ca",
            "4a83f8d2c8034a7594cff0770b3c6854",
            "4c0ff1085220493cbfdc256a94be703b",
            "b6f6974c47e64485bada093b573ee698",
            "9f3b5829b2b8443a9c1b8fdfc9d49015",
            "c3e4a0efc99d423cad10b973a395ba00",
            "7254f8933da14a02beb158a42344a4cf",
            "694deae3e53f498c97e07ac94c7cbd1f",
            "8e6dfc36ecfa4f16a01d7a99f20d0fde",
            "920cf71ffd244db08431175876aa4b10",
            "1e41ec305a0e4c3b9354ca54a8260fd2",
            "90d19082412a4f6f8edc17660e484ebc",
            "d857657ac84a42758e94e46b3abd0fc5",
            "8683c1d42f0b4dacb8d3362930731ff3",
            "7525450a3ec04920bdcaa629048bc013",
            "193ee8d6ce61444f8cce1ba395315814",
            "fd6f1412fe73448d908b6855a9c4da87",
            "150d4be236a246888ecd97430deeaa9d",
            "87597966a6ce4c2f94c5ac9011481981",
            "dc6cb78c1de04ce9accd2a5ff57ec7e3",
            "2a042deab41f4ac0bd444d26b709a3c5",
            "71541ce0dfb547cdb4b586ff6f11ed64",
            "61bf7d7752cd4652819495bc64bd7dbd",
            "b3fea1b6c34a47eca24e3fb34b360e0d",
            "426c8700c12a48a2abbea1812c0d46c6",
            "e8bdeb0f71cd44dc8181b45d12204b33",
            "9a052c85527d49aea4af76875ec90313",
            "e493952729614ce590c84b1bc5addd97",
            "a551d06ebcaf4947a892174e6b0da8cb",
            "7318c59b50284cf0b5c9fe3e5bac0b59",
            "31e0db9d757f434682978bda54962e31",
            "588d3eef77a04c38a5ccd8a112adf1b5",
            "7e6d17c36db84f45af0804218fea8792",
            "896352cdb9204a0e9cbb2bcb3ad38d9a",
            "679832cea9204ecea20e505f9706fe6d",
            "0c942f2eac3e42fb81abd932db095588",
            "ad700f35bbc34fc5a9909248dd2576e4",
            "b5d2cd90bb3d42a99928fdfcf4fd34f2",
            "ae1c3923a86e4091bf6d7f8f7b2d8abf",
            "af3eecf2ecd74498ae9c1f2e2d3ea81a",
            "82da283b58fc488f9961544b5e640bb5",
            "c72c422c83864364befa7c7bdd3f7b46",
            "f36e4b6c80b44b9cb6d50e0380650b9d",
            "0b703d2412284fe5b775a43bc50e11fc",
            "b4cda185eb704afc8a8635fcdc3d8942",
            "1468fb3a8af8493cb24f441f878dce80",
            "e916d2cd57bd47b9944085462f5004fd",
            "0fc1d0e29eef4366a28e93df58b2cb93",
            "929042bac59d4236a24f0200e9ba0334",
            "abb249e28d41422f8176cda6a11c960d",
            "b82182c42e904af9b8f159f819e71108",
            "fa6746f9330d4eb7bc1a7be5de28a407",
            "fd1b1d710233401f94da4e53d7d6e280",
            "718cf6768d72448aa64f96f7213133e0",
            "4ecfd953822e406f936162417bbf277a",
            "2bc113068b8147ce8551f811f54dfad4",
            "aa5bb7aea6294a2e8152b69f06e8e671",
            "b05c7f02195f47bdbd028452b651d29e",
            "18a6b92a09ea4cbfb2e76073cad13bb2",
            "030553a861e24a71bee5de6350d22a43",
            "6713257f8092406fb81bc053d01d20a0",
            "ae3b11c8bc9c4a1ca71d6b96606df465",
            "e3655f35e4604b56a51f6e419634062e",
            "188eaf5a194548169e9f107c7fc7b40a",
            "4bb0d8f6bbb143efb65197eb42930213",
            "02561d427fd040a38e72ebc7c650b6e1",
            "0a7518b8a7a34278be38400b19e61e96",
            "b129bf9396104e07ba80510dd05a45af",
            "b9cab8990c06489aaf9f9c8636f5379a",
            "f26edde5a7494475b79b00baa3a97543",
            "a18c03831c0643a59e520c4bd9bcc3fa",
            "a455c67ce22348c2b3e65cdcbf0b22ff",
            "81ce34875ca843d6b7477733582d3a33",
            "b3d88339c8674dac8121f2a7bdb3082d",
            "e95e1c27ad024080be1c17d9a3c0805f",
            "b49db84cc80049c5b3b4331c46c9e919",
            "cd020d888f664242afa86834d52febe9"
          ]
        },
        "id": "FlElFejM-2yi",
        "outputId": "6c9b0b5a-2130-42be-e256-0c16a5815319"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/683 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "4627e7e7c9e24a93bd227cd5c375d7ca"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model.safetensors:   0%|          | 0.00/328M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "90d19082412a4f6f8edc17660e484ebc"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at sentence-transformers/msmarco-distilroberta-base-v2 and are newly initialized: ['classifier.dense.bias', 'classifier.dense.weight', 'classifier.out_proj.bias', 'classifier.out_proj.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/1.20k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "61bf7d7752cd4652819495bc64bd7dbd"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "vocab.json:   0%|          | 0.00/798k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "896352cdb9204a0e9cbb2bcb3ad38d9a"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b4cda185eb704afc8a8635fcdc3d8942"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer.json:   0%|          | 0.00/1.36M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "2bc113068b8147ce8551f811f54dfad4"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "special_tokens_map.json:   0%|          | 0.00/772 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0a7518b8a7a34278be38400b19e61e96"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(questions[0])\n",
        "print()\n",
        "response = advanced_query_engine.query(questions[0])  #Llama Index\n",
        "textwrap.wrap(response.response, width=100)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TIvMMVPXRKR1",
        "outputId": "995f663b-9084-43e7-accd-4bbea25acb85"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "What is the core objective of investing in disruptive innovation according to ARK?\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[' ARK seeks to capture disruptive innovation by investing in companies that are at the forefront of',\n",
              " 'technological advancements that are disrupting established industries. This strategy aims to',\n",
              " 'generate long-term capital appreciation while also providing exposure to companies that are at the',\n",
              " \"forefront of innovation. ARK's investment approach is based on a fundamental analysis of the\",\n",
              " 'underlying business models and technologies, as well as a rigorous screening process to identify',\n",
              " \"companies that are poised for growth and potential disruption. ARK's investment team is composed of\",\n",
              " 'experienced investment professionals with a deep understanding of the technology and business',\n",
              " 'landscape. They are committed to identifying and investing in companies that are at the forefront of']"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "advanced_rag_answers = []\n",
        "advanced_rag_responses = []\n",
        "for q in tqdm(questions):\n",
        "    response = advanced_query_engine.query(q)\n",
        "    advanced_rag_responses.append(response)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SNDPc76ARKPi",
        "outputId": "6fd91ac7-a740-4372-a1dd-b5cdd7364cb2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " 15%|█▌        | 3/20 [01:56<11:37, 41.06s/it]"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "BNsa1FIh-2wP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "MN3UkzoFRdOW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "GwvgYvPsRdHE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "----"
      ],
      "metadata": {
        "id": "aw7YEOAHPshW"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Evaluation Experiments with DeepEval and Llama Index"
      ],
      "metadata": {
        "id": "WqPR70w9PTPG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "faithfulness_de = DeepEvalFaithfulnessEvaluator(model=\"gpt-3.5-turbo\")\n",
        "evaluation_result = faithfulness_de.evaluate_response(\n",
        "    query=questions[0], response=basic_rag_responses[0]\n",
        ")\n",
        "print(evaluation_result)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 518,
          "referenced_widgets": [
            "413b2d8487a54b3ebee63c1298b7754c",
            "6fea62aad9cc40c9acffa02f950dcba5"
          ]
        },
        "id": "yQ679mPH_GNa",
        "outputId": "4eaae84a-6f53-4e54-c923-447d78fecb5f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Output()"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "413b2d8487a54b3ebee63c1298b7754c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "ERROR:root:OpenAI rate limit exceeded. Retrying: 1 time(s)...\n",
            "ERROR:root:OpenAI rate limit exceeded. Retrying: 1 time(s)...\n",
            "ERROR:root:OpenAI rate limit exceeded. Retrying: 2 time(s)...\n",
            "ERROR:root:OpenAI rate limit exceeded. Retrying: 2 time(s)...\n",
            "ERROR:root:OpenAI rate limit exceeded. Retrying: 3 time(s)...\n",
            "ERROR:root:OpenAI rate limit exceeded. Retrying: 3 time(s)...\n",
            "ERROR:root:OpenAI rate limit exceeded. Retrying: 4 time(s)...\n",
            "ERROR:root:OpenAI rate limit exceeded. Retrying: 4 time(s)...\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-39-9dbded6d21b1>\u001b[0m in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mfaithfulness_de\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDeepEvalFaithfulnessEvaluator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"gpt-3.5-turbo\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m evaluation_result = faithfulness_de.evaluate_response(\n\u001b[0m\u001b[1;32m      3\u001b[0m     \u001b[0mquery\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mquestions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbasic_rag_responses\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m )\n\u001b[1;32m      5\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mevaluation_result\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/llama_index/core/evaluation/base.py\u001b[0m in \u001b[0;36mevaluate_response\u001b[0;34m(self, query, response, **kwargs)\u001b[0m\n\u001b[1;32m    104\u001b[0m             \u001b[0mcontexts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_content\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mnode\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msource_nodes\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    105\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 106\u001b[0;31m         return self.evaluate(\n\u001b[0m\u001b[1;32m    107\u001b[0m             \u001b[0mquery\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mquery\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mresponse_str\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontexts\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcontexts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    108\u001b[0m         )\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/llama_index/core/evaluation/base.py\u001b[0m in \u001b[0;36mevaluate\u001b[0;34m(self, query, response, contexts, **kwargs)\u001b[0m\n\u001b[1;32m     62\u001b[0m         \u001b[0mtake\u001b[0m \u001b[0;32min\u001b[0m \u001b[0madditional\u001b[0m \u001b[0marguments\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m         \"\"\"\n\u001b[0;32m---> 64\u001b[0;31m         return asyncio_run(\n\u001b[0m\u001b[1;32m     65\u001b[0m             self.aevaluate(\n\u001b[1;32m     66\u001b[0m                 \u001b[0mquery\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mquery\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/llama_index/core/async_utils.py\u001b[0m in \u001b[0;36masyncio_run\u001b[0;34m(coro)\u001b[0m\n\u001b[1;32m     31\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m         \u001b[0;31m# If we're here, there's an existing loop but it's not running\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 33\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mloop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_until_complete\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcoro\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     34\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mRuntimeError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/nest_asyncio.py\u001b[0m in \u001b[0;36mrun_until_complete\u001b[0;34m(self, future)\u001b[0m\n\u001b[1;32m     90\u001b[0m                 \u001b[0mf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_log_destroy_pending\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m             \u001b[0;32mwhile\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdone\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 92\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_once\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     93\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stopping\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m                     \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/nest_asyncio.py\u001b[0m in \u001b[0;36m_run_once\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    113\u001b[0m                 scheduled[0]._when - self.time(), 0), 86400) if scheduled\n\u001b[1;32m    114\u001b[0m             else None)\n\u001b[0;32m--> 115\u001b[0;31m         \u001b[0mevent_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_selector\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mselect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    116\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_process_events\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mevent_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    117\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.10/selectors.py\u001b[0m in \u001b[0;36mselect\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    467\u001b[0m             \u001b[0mready\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    468\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 469\u001b[0;31m                 \u001b[0mfd_event_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_selector\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpoll\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_ev\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    470\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mInterruptedError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    471\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mready\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "faithfulness_de = DeepEvalFaithfulnessEvaluator()\n",
        "evaluation_result = faithfulness_de.evaluate_response(\n",
        "    query=questions[0], response=basic_rag_responses[0]\n",
        ")\n",
        "print(evaluation_result)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 397,
          "referenced_widgets": [
            "8f5eca32bcee46e587c385757274ba0e",
            "53ad47d669d94024b4eaa94868ace06c"
          ]
        },
        "id": "Apkrp-Tb_GQN",
        "outputId": "c7e10b5d-20cb-4e81-8644-6ead8ecb5039"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Output()"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "8f5eca32bcee46e587c385757274ba0e"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "error",
          "ename": "NotFoundError",
          "evalue": "Error code: 404 - {'error': {'message': 'The model `gpt-4o` does not exist or you do not have access to it.', 'type': 'invalid_request_error', 'param': None, 'code': 'model_not_found'}}",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNotFoundError\u001b[0m                             Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-37-7f933add4ff9>\u001b[0m in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mfaithfulness_de\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDeepEvalFaithfulnessEvaluator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m evaluation_result = faithfulness_de.evaluate_response(\n\u001b[0m\u001b[1;32m      3\u001b[0m     \u001b[0mquery\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mquestions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbasic_rag_responses\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m )\n\u001b[1;32m      5\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mevaluation_result\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/llama_index/core/evaluation/base.py\u001b[0m in \u001b[0;36mevaluate_response\u001b[0;34m(self, query, response, **kwargs)\u001b[0m\n\u001b[1;32m    104\u001b[0m             \u001b[0mcontexts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_content\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mnode\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msource_nodes\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    105\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 106\u001b[0;31m         return self.evaluate(\n\u001b[0m\u001b[1;32m    107\u001b[0m             \u001b[0mquery\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mquery\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mresponse_str\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontexts\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcontexts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    108\u001b[0m         )\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/llama_index/core/evaluation/base.py\u001b[0m in \u001b[0;36mevaluate\u001b[0;34m(self, query, response, contexts, **kwargs)\u001b[0m\n\u001b[1;32m     62\u001b[0m         \u001b[0mtake\u001b[0m \u001b[0;32min\u001b[0m \u001b[0madditional\u001b[0m \u001b[0marguments\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m         \"\"\"\n\u001b[0;32m---> 64\u001b[0;31m         return asyncio_run(\n\u001b[0m\u001b[1;32m     65\u001b[0m             self.aevaluate(\n\u001b[1;32m     66\u001b[0m                 \u001b[0mquery\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mquery\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/llama_index/core/async_utils.py\u001b[0m in \u001b[0;36masyncio_run\u001b[0;34m(coro)\u001b[0m\n\u001b[1;32m     31\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m         \u001b[0;31m# If we're here, there's an existing loop but it's not running\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 33\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mloop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_until_complete\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcoro\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     34\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mRuntimeError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/nest_asyncio.py\u001b[0m in \u001b[0;36mrun_until_complete\u001b[0;34m(self, future)\u001b[0m\n\u001b[1;32m     96\u001b[0m                 raise RuntimeError(\n\u001b[1;32m     97\u001b[0m                     'Event loop stopped before Future completed.')\n\u001b[0;32m---> 98\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     99\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_run_once\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.10/asyncio/futures.py\u001b[0m in \u001b[0;36mresult\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    199\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__log_traceback\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    200\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_exception\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 201\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_exception\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_exception_tb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    202\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_result\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    203\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.10/asyncio/tasks.py\u001b[0m in \u001b[0;36m__step\u001b[0;34m(***failed resolving arguments***)\u001b[0m\n\u001b[1;32m    232\u001b[0m                 \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcoro\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    233\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 234\u001b[0;31m                 \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcoro\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mthrow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    235\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    236\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_must_cancel\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/deepeval/integrations/llama_index/evaluators.py\u001b[0m in \u001b[0;36maevaluate\u001b[0;34m(***failed resolving arguments***)\u001b[0m\n\u001b[1;32m    109\u001b[0m             \u001b[0mmodel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    110\u001b[0m         )\n\u001b[0;32m--> 111\u001b[0;31m         \u001b[0;32mawait\u001b[0m \u001b[0mmetric\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0ma_measure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_case\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    112\u001b[0m         return EvaluationResult(\n\u001b[1;32m    113\u001b[0m             \u001b[0mquery\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mquery\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/deepeval/metrics/faithfulness/faithfulness.py\u001b[0m in \u001b[0;36ma_measure\u001b[0;34m(self, test_case, _show_indicator)\u001b[0m\n\u001b[1;32m     87\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0masync_mode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_show_indicator\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0m_show_indicator\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     88\u001b[0m         ):\n\u001b[0;32m---> 89\u001b[0;31m             self.truths, self.claims = await asyncio.gather(\n\u001b[0m\u001b[1;32m     90\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_a_generate_truths\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_case\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mretrieval_context\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_a_generate_claims\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_case\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mactual_output\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.10/asyncio/tasks.py\u001b[0m in \u001b[0;36m__wakeup\u001b[0;34m(self, future)\u001b[0m\n\u001b[1;32m    302\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__wakeup\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    303\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 304\u001b[0;31m             \u001b[0mfuture\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    305\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mBaseException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    306\u001b[0m             \u001b[0;31m# This may also be a cancellation.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.10/asyncio/tasks.py\u001b[0m in \u001b[0;36m__step\u001b[0;34m(***failed resolving arguments***)\u001b[0m\n\u001b[1;32m    230\u001b[0m                 \u001b[0;31m# We use the `send` method directly, because coroutines\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    231\u001b[0m                 \u001b[0;31m# don't have `__iter__` and `__next__` methods.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 232\u001b[0;31m                 \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcoro\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    233\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    234\u001b[0m                 \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcoro\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mthrow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/deepeval/metrics/faithfulness/faithfulness.py\u001b[0m in \u001b[0;36m_a_generate_claims\u001b[0;34m(self, actual_output)\u001b[0m\n\u001b[1;32m    262\u001b[0m         \u001b[0mprompt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mFaithfulnessTemplate\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgenerate_claims\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mactual_output\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    263\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0musing_native_model\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 264\u001b[0;31m             \u001b[0mres\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcost\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mawait\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0ma_generate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprompt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    265\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluation_cost\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mcost\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    266\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrimAndLoadJson\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mres\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tenacity/asyncio/__init__.py\u001b[0m in \u001b[0;36masync_wrapped\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    187\u001b[0m             \u001b[0mcopy\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    188\u001b[0m             \u001b[0masync_wrapped\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatistics\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatistics\u001b[0m  \u001b[0;31m# type: ignore[attr-defined]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 189\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0;32mawait\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    190\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    191\u001b[0m         \u001b[0;31m# Preserve attributes\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tenacity/asyncio/__init__.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, fn, *args, **kwargs)\u001b[0m\n\u001b[1;32m    109\u001b[0m         \u001b[0mretry_state\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mRetryCallState\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mretry_object\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    110\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 111\u001b[0;31m             \u001b[0mdo\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mawait\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mretry_state\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mretry_state\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    112\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mDoAttempt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    113\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tenacity/asyncio/__init__.py\u001b[0m in \u001b[0;36miter\u001b[0;34m(self, retry_state)\u001b[0m\n\u001b[1;32m    151\u001b[0m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    152\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0maction\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miter_state\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mactions\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 153\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mawait\u001b[0m \u001b[0maction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mretry_state\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    154\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    155\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tenacity/_utils.py\u001b[0m in \u001b[0;36minner\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     97\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     98\u001b[0m     \u001b[0;32masync\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0minner\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mtyping\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mAny\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mtyping\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mAny\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mtyping\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mAny\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 99\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mcall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    100\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    101\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0minner\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tenacity/__init__.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(rs)\u001b[0m\n\u001b[1;32m    396\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_post_retry_check_actions\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretry_state\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m\"RetryCallState\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    397\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miter_state\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_explicit_retry\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miter_state\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mretry_run_result\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 398\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_add_action_func\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mrs\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mrs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutcome\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    399\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    400\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.10/concurrent/futures/_base.py\u001b[0m in \u001b[0;36mresult\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    449\u001b[0m                     \u001b[0;32mraise\u001b[0m \u001b[0mCancelledError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    450\u001b[0m                 \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_state\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mFINISHED\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 451\u001b[0;31m                     \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__get_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    452\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    453\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_condition\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.10/concurrent/futures/_base.py\u001b[0m in \u001b[0;36m__get_result\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    401\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_exception\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    402\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 403\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_exception\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    404\u001b[0m             \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    405\u001b[0m                 \u001b[0;31m# Break a reference cycle with the exception in self._exception\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tenacity/asyncio/__init__.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, fn, *args, **kwargs)\u001b[0m\n\u001b[1;32m    112\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mDoAttempt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    113\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 114\u001b[0;31m                     \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mawait\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    115\u001b[0m                 \u001b[0;32mexcept\u001b[0m \u001b[0mBaseException\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# noqa: B902\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    116\u001b[0m                     \u001b[0mretry_state\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_exception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexc_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore[arg-type]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/deepeval/models/gpt_model.py\u001b[0m in \u001b[0;36ma_generate\u001b[0;34m(self, prompt)\u001b[0m\n\u001b[1;32m    180\u001b[0m         \u001b[0mchat_model\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    181\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mget_openai_callback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mcb\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 182\u001b[0;31m             \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mawait\u001b[0m \u001b[0mchat_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mainvoke\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprompt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    183\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mres\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcb\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtotal_cost\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    184\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/langchain_core/language_models/chat_models.py\u001b[0m in \u001b[0;36mainvoke\u001b[0;34m(self, input, config, stop, **kwargs)\u001b[0m\n\u001b[1;32m    303\u001b[0m     ) -> BaseMessage:\n\u001b[1;32m    304\u001b[0m         \u001b[0mconfig\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mensure_config\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 305\u001b[0;31m         llm_result = await self.agenerate_prompt(\n\u001b[0m\u001b[1;32m    306\u001b[0m             \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_convert_input\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    307\u001b[0m             \u001b[0mstop\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstop\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/langchain_core/language_models/chat_models.py\u001b[0m in \u001b[0;36magenerate_prompt\u001b[0;34m(self, prompts, stop, callbacks, **kwargs)\u001b[0m\n\u001b[1;32m    792\u001b[0m     ) -> LLMResult:\n\u001b[1;32m    793\u001b[0m         \u001b[0mprompt_messages\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_messages\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mp\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mprompts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 794\u001b[0;31m         return await self.agenerate(\n\u001b[0m\u001b[1;32m    795\u001b[0m             \u001b[0mprompt_messages\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstop\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    796\u001b[0m         )\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/langchain_core/language_models/chat_models.py\u001b[0m in \u001b[0;36magenerate\u001b[0;34m(self, messages, stop, callbacks, tags, metadata, run_name, run_id, **kwargs)\u001b[0m\n\u001b[1;32m    752\u001b[0m                     ]\n\u001b[1;32m    753\u001b[0m                 )\n\u001b[0;32m--> 754\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mexceptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    755\u001b[0m         flattened_outputs = [\n\u001b[1;32m    756\u001b[0m             \u001b[0mLLMResult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgenerations\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mres\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgenerations\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mllm_output\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mres\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mllm_output\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore[list-item, union-attr]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.10/asyncio/tasks.py\u001b[0m in \u001b[0;36m__step\u001b[0;34m(***failed resolving arguments***)\u001b[0m\n\u001b[1;32m    230\u001b[0m                 \u001b[0;31m# We use the `send` method directly, because coroutines\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    231\u001b[0m                 \u001b[0;31m# don't have `__iter__` and `__next__` methods.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 232\u001b[0;31m                 \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcoro\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    233\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    234\u001b[0m                 \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcoro\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mthrow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/langchain_core/language_models/chat_models.py\u001b[0m in \u001b[0;36m_agenerate_with_cache\u001b[0;34m(self, messages, stop, run_manager, **kwargs)\u001b[0m\n\u001b[1;32m    928\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    929\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0minspect\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msignature\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_agenerate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"run_manager\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 930\u001b[0;31m                 result = await self._agenerate(\n\u001b[0m\u001b[1;32m    931\u001b[0m                     \u001b[0mmessages\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstop\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrun_manager\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrun_manager\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    932\u001b[0m                 )\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/langchain_openai/chat_models/base.py\u001b[0m in \u001b[0;36m_agenerate\u001b[0;34m(self, messages, stop, run_manager, **kwargs)\u001b[0m\n\u001b[1;32m    825\u001b[0m             \u001b[0mgeneration_info\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m\"headers\"\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mdict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mraw_response\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mheaders\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    826\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 827\u001b[0;31m             \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mawait\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync_client\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mpayload\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    828\u001b[0m         return await run_in_executor(\n\u001b[1;32m    829\u001b[0m             \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_create_chat_result\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgeneration_info\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/openai/resources/chat/completions.py\u001b[0m in \u001b[0;36mcreate\u001b[0;34m(self, messages, model, frequency_penalty, function_call, functions, logit_bias, logprobs, max_completion_tokens, max_tokens, n, parallel_tool_calls, presence_penalty, response_format, seed, service_tier, stop, stream, stream_options, temperature, tool_choice, tools, top_logprobs, top_p, user, extra_headers, extra_query, extra_body, timeout)\u001b[0m\n\u001b[1;32m   1410\u001b[0m     ) -> ChatCompletion | AsyncStream[ChatCompletionChunk]:\n\u001b[1;32m   1411\u001b[0m         \u001b[0mvalidate_response_format\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresponse_format\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1412\u001b[0;31m         return await self._post(\n\u001b[0m\u001b[1;32m   1413\u001b[0m             \u001b[0;34m\"/chat/completions\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1414\u001b[0m             body=await async_maybe_transform(\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/openai/_base_client.py\u001b[0m in \u001b[0;36mpost\u001b[0;34m(self, path, cast_to, body, files, options, stream, stream_cls)\u001b[0m\n\u001b[1;32m   1827\u001b[0m             \u001b[0mmethod\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"post\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjson_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbody\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfiles\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mawait\u001b[0m \u001b[0masync_to_httpx_files\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfiles\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1828\u001b[0m         )\n\u001b[0;32m-> 1829\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0;32mawait\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcast_to\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstream\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstream\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstream_cls\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstream_cls\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1830\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1831\u001b[0m     async def patch(\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/openai/_base_client.py\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(self, cast_to, options, stream, stream_cls, remaining_retries)\u001b[0m\n\u001b[1;32m   1521\u001b[0m             \u001b[0mretries_taken\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1522\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1523\u001b[0;31m         return await self._request(\n\u001b[0m\u001b[1;32m   1524\u001b[0m             \u001b[0mcast_to\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcast_to\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1525\u001b[0m             \u001b[0moptions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/openai/_base_client.py\u001b[0m in \u001b[0;36m_request\u001b[0;34m(self, cast_to, options, stream, stream_cls, retries_taken)\u001b[0m\n\u001b[1;32m   1622\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1623\u001b[0m             \u001b[0mlog\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdebug\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Re-raising status error\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1624\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_status_error_from_response\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresponse\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1625\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1626\u001b[0m         return await self._process_response(\n",
            "\u001b[0;31mNotFoundError\u001b[0m: Error code: 404 - {'error': {'message': 'The model `gpt-4o` does not exist or you do not have access to it.', 'type': 'invalid_request_error', 'param': None, 'code': 'model_not_found'}}"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from deepeval.integrations.llama_index import (\n",
        "    DeepEvalAnswerRelevancyEvaluator,\n",
        "    DeepEvalFaithfulnessEvaluator,\n",
        "    DeepEvalContextualRelevancyEvaluator,\n",
        "    DeepEvalSummarizationEvaluator,\n",
        "    DeepEvalBiasEvaluator,\n",
        "    DeepEvalToxicityEvaluator,\n",
        ")"
      ],
      "metadata": {
        "id": "rDwqRr7d8dGy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "ans_relevancy_de = DeepEvalAnswerRelevancyEvaluator(model=llm_de)\n",
        "ans_relevancy_de.evaluate(query=questions[0], response=basic_rag_responses[0], contexts=basic_rag_response_contexts[0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 445,
          "referenced_widgets": [
            "c8a846cfc54b4a3cac8f2acb32357acd",
            "55c9721fbd434e0a9fac1d14cf4d42e4"
          ]
        },
        "id": "FpnRIps88dEO",
        "outputId": "83b8386b-948b-471e-b511-f65d7d4a2a42"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Output()"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "c8a846cfc54b4a3cac8f2acb32357acd"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "error",
          "ename": "AttributeError",
          "evalue": "'list' object has no attribute 'find'",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/deepeval/metrics/answer_relevancy/answer_relevancy.py\u001b[0m in \u001b[0;36m_a_generate_statements\u001b[0;34m(self, actual_output)\u001b[0m\n\u001b[1;32m    233\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 234\u001b[0;31m                 res: Statements = await self.model.a_generate(\n\u001b[0m\u001b[1;32m    235\u001b[0m                     \u001b[0mprompt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mschema\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mStatements\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mTypeError\u001b[0m: CustomTinyLlama3.a_generate() got an unexpected keyword argument 'schema'",
            "\nDuring handling of the above exception, another exception occurred:\n",
            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-47-bfb2ca42c6a0>\u001b[0m in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mans_relevancy_de\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDeepEvalAnswerRelevancyEvaluator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mllm_de\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mans_relevancy_de\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mquery\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mquestions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbasic_rag_responses\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontexts\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbasic_rag_response_contexts\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/llama_index/core/evaluation/base.py\u001b[0m in \u001b[0;36mevaluate\u001b[0;34m(self, query, response, contexts, **kwargs)\u001b[0m\n\u001b[1;32m     62\u001b[0m         \u001b[0mtake\u001b[0m \u001b[0;32min\u001b[0m \u001b[0madditional\u001b[0m \u001b[0marguments\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m         \"\"\"\n\u001b[0;32m---> 64\u001b[0;31m         return asyncio_run(\n\u001b[0m\u001b[1;32m     65\u001b[0m             self.aevaluate(\n\u001b[1;32m     66\u001b[0m                 \u001b[0mquery\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mquery\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/llama_index/core/async_utils.py\u001b[0m in \u001b[0;36masyncio_run\u001b[0;34m(coro)\u001b[0m\n\u001b[1;32m     31\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m         \u001b[0;31m# If we're here, there's an existing loop but it's not running\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 33\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mloop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_until_complete\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcoro\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     34\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mRuntimeError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/nest_asyncio.py\u001b[0m in \u001b[0;36mrun_until_complete\u001b[0;34m(self, future)\u001b[0m\n\u001b[1;32m     96\u001b[0m                 raise RuntimeError(\n\u001b[1;32m     97\u001b[0m                     'Event loop stopped before Future completed.')\n\u001b[0;32m---> 98\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     99\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_run_once\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.10/asyncio/futures.py\u001b[0m in \u001b[0;36mresult\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    199\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__log_traceback\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    200\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_exception\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 201\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_exception\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_exception_tb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    202\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_result\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    203\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.10/asyncio/tasks.py\u001b[0m in \u001b[0;36m__step\u001b[0;34m(***failed resolving arguments***)\u001b[0m\n\u001b[1;32m    230\u001b[0m                 \u001b[0;31m# We use the `send` method directly, because coroutines\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    231\u001b[0m                 \u001b[0;31m# don't have `__iter__` and `__next__` methods.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 232\u001b[0;31m                 \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcoro\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    233\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    234\u001b[0m                 \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcoro\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mthrow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/deepeval/integrations/llama_index/evaluators.py\u001b[0m in \u001b[0;36maevaluate\u001b[0;34m(***failed resolving arguments***)\u001b[0m\n\u001b[1;32m     57\u001b[0m             \u001b[0mmodel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     58\u001b[0m         )\n\u001b[0;32m---> 59\u001b[0;31m         \u001b[0;32mawait\u001b[0m \u001b[0mmetric\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0ma_measure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_case\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     60\u001b[0m         return EvaluationResult(\n\u001b[1;32m     61\u001b[0m             \u001b[0mquery\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mquery\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/deepeval/metrics/answer_relevancy/answer_relevancy.py\u001b[0m in \u001b[0;36ma_measure\u001b[0;34m(self, test_case, _show_indicator)\u001b[0m\n\u001b[1;32m     87\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0masync_mode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_show_indicator\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0m_show_indicator\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     88\u001b[0m         ):\n\u001b[0;32m---> 89\u001b[0;31m             self.statements: List[str] = await self._a_generate_statements(\n\u001b[0m\u001b[1;32m     90\u001b[0m                 \u001b[0mtest_case\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mactual_output\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m             )\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/deepeval/metrics/answer_relevancy/answer_relevancy.py\u001b[0m in \u001b[0;36m_a_generate_statements\u001b[0;34m(self, actual_output)\u001b[0m\n\u001b[1;32m    238\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    239\u001b[0m                 \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mawait\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0ma_generate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprompt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 240\u001b[0;31m                 \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrimAndLoadJson\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mres\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    241\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"statements\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    242\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/deepeval/metrics/utils.py\u001b[0m in \u001b[0;36mtrimAndLoadJson\u001b[0;34m(input_string, metric)\u001b[0m\n\u001b[1;32m    227\u001b[0m     \u001b[0minput_string\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetric\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mOptional\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mBaseMetric\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    228\u001b[0m ) -> Any:\n\u001b[0;32m--> 229\u001b[0;31m     \u001b[0mstart\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput_string\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"{\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    230\u001b[0m     \u001b[0mend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput_string\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrfind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"}\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    231\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mAttributeError\u001b[0m: 'list' object has no attribute 'find'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from tqdm.asyncio import tqdm_asyncio\n",
        "from llama_index.core.evaluation import BatchEvalRunner\n",
        "runner_no_llm = BatchEvalRunner(\n",
        "    {\"semantic_similarity\": semantic_similarity, \"context_relevancy\": context_relevancy},\n",
        "    workers=8,\n",
        ")"
      ],
      "metadata": {
        "id": "z5N-O5d6qfqg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "runner.evaluate_responses(\n",
        "    queries = questions[:1],\n",
        "    responses = basic_rag_responses[:1],\n",
        "    references = gpt_answers[:1],\n",
        "    contexts = basic_rag_response_contexts[1:]\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "dgaGdK4EqfmE",
        "outputId": "654d17e7-6901-4d53-ed92-ffaf7317995e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "TypeError",
          "evalue": "llama_index.core.evaluation.faithfulness.FaithfulnessEvaluator.aevaluate() got multiple values for keyword argument 'contexts'",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-25-1b9539b3eb4e>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mrunner\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluate_responses\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mqueries\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mquestions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponses\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbasic_rag_responses\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreferences\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgpt_answers\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontexts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbasic_rag_response_contexts\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/llama_index/core/evaluation/batch_runner.py\u001b[0m in \u001b[0;36mevaluate_responses\u001b[0;34m(self, queries, responses, **eval_kwargs_lists)\u001b[0m\n\u001b[1;32m    382\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    383\u001b[0m         \"\"\"\n\u001b[0;32m--> 384\u001b[0;31m         return asyncio_run(\n\u001b[0m\u001b[1;32m    385\u001b[0m             self.aevaluate_responses(\n\u001b[1;32m    386\u001b[0m                 \u001b[0mqueries\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mqueries\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/llama_index/core/async_utils.py\u001b[0m in \u001b[0;36masyncio_run\u001b[0;34m(coro)\u001b[0m\n\u001b[1;32m     31\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m         \u001b[0;31m# If we're here, there's an existing loop but it's not running\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 33\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mloop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_until_complete\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcoro\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     34\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mRuntimeError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/nest_asyncio.py\u001b[0m in \u001b[0;36mrun_until_complete\u001b[0;34m(self, future)\u001b[0m\n\u001b[1;32m     96\u001b[0m                 raise RuntimeError(\n\u001b[1;32m     97\u001b[0m                     'Event loop stopped before Future completed.')\n\u001b[0;32m---> 98\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     99\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_run_once\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.10/asyncio/futures.py\u001b[0m in \u001b[0;36mresult\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    199\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__log_traceback\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    200\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_exception\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 201\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_exception\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_exception_tb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    202\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_result\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    203\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.10/asyncio/tasks.py\u001b[0m in \u001b[0;36m__step\u001b[0;34m(***failed resolving arguments***)\u001b[0m\n\u001b[1;32m    232\u001b[0m                 \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcoro\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    233\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 234\u001b[0;31m                 \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcoro\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mthrow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    235\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    236\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_must_cancel\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/llama_index/core/evaluation/batch_runner.py\u001b[0m in \u001b[0;36maevaluate_responses\u001b[0;34m(self, queries, responses, **eval_kwargs_lists)\u001b[0m\n\u001b[1;32m    312\u001b[0m                     )\n\u001b[1;32m    313\u001b[0m                 )\n\u001b[0;32m--> 314\u001b[0;31m         \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mawait\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masyncio_mod\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgather\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0meval_jobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    315\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    316\u001b[0m         \u001b[0;31m# Format results\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.10/asyncio/tasks.py\u001b[0m in \u001b[0;36m__wakeup\u001b[0;34m(self, future)\u001b[0m\n\u001b[1;32m    302\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__wakeup\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfuture\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    303\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 304\u001b[0;31m             \u001b[0mfuture\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    305\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mBaseException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    306\u001b[0m             \u001b[0;31m# This may also be a cancellation.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.10/asyncio/tasks.py\u001b[0m in \u001b[0;36m__step\u001b[0;34m(***failed resolving arguments***)\u001b[0m\n\u001b[1;32m    230\u001b[0m                 \u001b[0;31m# We use the `send` method directly, because coroutines\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    231\u001b[0m                 \u001b[0;31m# don't have `__iter__` and `__next__` methods.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 232\u001b[0;31m                 \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcoro\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    233\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    234\u001b[0m                 \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcoro\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mthrow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tenacity/asyncio/__init__.py\u001b[0m in \u001b[0;36masync_wrapped\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    187\u001b[0m             \u001b[0mcopy\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    188\u001b[0m             \u001b[0masync_wrapped\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatistics\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatistics\u001b[0m  \u001b[0;31m# type: ignore[attr-defined]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 189\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0;32mawait\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    190\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    191\u001b[0m         \u001b[0;31m# Preserve attributes\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tenacity/asyncio/__init__.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, fn, *args, **kwargs)\u001b[0m\n\u001b[1;32m    109\u001b[0m         \u001b[0mretry_state\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mRetryCallState\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mretry_object\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    110\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 111\u001b[0;31m             \u001b[0mdo\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mawait\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mretry_state\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mretry_state\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    112\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mDoAttempt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    113\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tenacity/asyncio/__init__.py\u001b[0m in \u001b[0;36miter\u001b[0;34m(self, retry_state)\u001b[0m\n\u001b[1;32m    151\u001b[0m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    152\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0maction\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miter_state\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mactions\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 153\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mawait\u001b[0m \u001b[0maction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mretry_state\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    154\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    155\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tenacity/_utils.py\u001b[0m in \u001b[0;36minner\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     97\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     98\u001b[0m     \u001b[0;32masync\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0minner\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mtyping\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mAny\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mtyping\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mAny\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mtyping\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mAny\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 99\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mcall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    100\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    101\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0minner\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tenacity/__init__.py\u001b[0m in \u001b[0;36mexc_check\u001b[0;34m(rs)\u001b[0m\n\u001b[1;32m    416\u001b[0m                 \u001b[0mretry_exc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mretry_error_cls\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfut\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    417\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreraise\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 418\u001b[0;31m                     \u001b[0;32mraise\u001b[0m \u001b[0mretry_exc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreraise\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    419\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mretry_exc\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mfut\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    420\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tenacity/__init__.py\u001b[0m in \u001b[0;36mreraise\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    183\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mreraise\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mNoReturn\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    184\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlast_attempt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfailed\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 185\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlast_attempt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    186\u001b[0m         \u001b[0;32mraise\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    187\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.10/concurrent/futures/_base.py\u001b[0m in \u001b[0;36mresult\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    449\u001b[0m                     \u001b[0;32mraise\u001b[0m \u001b[0mCancelledError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    450\u001b[0m                 \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_state\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mFINISHED\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 451\u001b[0;31m                     \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__get_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    452\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    453\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_condition\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.10/concurrent/futures/_base.py\u001b[0m in \u001b[0;36m__get_result\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    401\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_exception\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    402\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 403\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_exception\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    404\u001b[0m             \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    405\u001b[0m                 \u001b[0;31m# Break a reference cycle with the exception in self._exception\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/tenacity/asyncio/__init__.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, fn, *args, **kwargs)\u001b[0m\n\u001b[1;32m    112\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mDoAttempt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    113\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 114\u001b[0;31m                     \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mawait\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    115\u001b[0m                 \u001b[0;32mexcept\u001b[0m \u001b[0mBaseException\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# noqa: B902\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    116\u001b[0m                     \u001b[0mretry_state\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_exception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexc_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore[arg-type]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/llama_index/core/evaluation/batch_runner.py\u001b[0m in \u001b[0;36meval_response_worker\u001b[0;34m(semaphore, evaluator, evaluator_name, query, response, eval_kwargs)\u001b[0m\n\u001b[1;32m     27\u001b[0m         return (\n\u001b[1;32m     28\u001b[0m             \u001b[0mevaluator_name\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 29\u001b[0;31m             await evaluator.aevaluate_response(\n\u001b[0m\u001b[1;32m     30\u001b[0m                 \u001b[0mquery\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mquery\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mresponse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0meval_kwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     31\u001b[0m             ),\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/llama_index/core/evaluation/base.py\u001b[0m in \u001b[0;36maevaluate_response\u001b[0;34m(self, query, response, **kwargs)\u001b[0m\n\u001b[1;32m    125\u001b[0m             \u001b[0mcontexts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_content\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mnode\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msource_nodes\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    126\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 127\u001b[0;31m         return await self.aevaluate(\n\u001b[0m\u001b[1;32m    128\u001b[0m             \u001b[0mquery\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mquery\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mresponse_str\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcontexts\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcontexts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    129\u001b[0m         )\n",
            "\u001b[0;31mTypeError\u001b[0m: llama_index.core.evaluation.faithfulness.FaithfulnessEvaluator.aevaluate() got multiple values for keyword argument 'contexts'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from deepeval.models import DeepEvalBaseLLM\n",
        "\n",
        "class CustomTinyLlama(DeepEvalBaseLLM):\n",
        "    def __init__(self, hf_model, hf_tokenizer):\n",
        "        self.model = hf_model\n",
        "        self.tokenizer = hf_tokenizer\n",
        "\n",
        "    def load_model(self):\n",
        "        return self.model\n",
        "\n",
        "    def generate(self, prompt: str) -> str:\n",
        "        model = self.load_model()\n",
        "\n",
        "        pipeline = transformers.pipeline(\n",
        "            \"text-generation\",\n",
        "            model=model,\n",
        "            tokenizer=self.tokenizer,\n",
        "            use_cache=True,\n",
        "            device_map=\"auto\",\n",
        "            max_length=2500,\n",
        "            do_sample=True,\n",
        "            top_k=5,\n",
        "            num_return_sequences=1,\n",
        "            eos_token_id=self.tokenizer.eos_token_id,\n",
        "            pad_token_id=self.tokenizer.eos_token_id,\n",
        "        )\n",
        "\n",
        "        return pipeline(prompt)\n",
        "\n",
        "    async def a_generate(self, prompt: str) -> str:\n",
        "        return self.generate(prompt)\n",
        "\n",
        "    def get_model_name(self):\n",
        "        return \"TinyLlama 1B\"\n",
        "\n",
        "llm_de = CustomTinyLlama3(\n",
        "    AutoModelForCausalLM.from_pretrained(\n",
        "            hf_model_name,\n",
        "            device_map=\"auto\",\n",
        "        ),\n",
        "    AutoTokenizer.from_pretrained(hf_model_name),\n",
        ")"
      ],
      "metadata": {
        "id": "GfJssdVwqfi7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "from deepeval.integrations.llamaindex import DeepEvalFaithfulnessEvaluator\n",
        "evaluator = DeepEvalFaithfulnessEvaluator()\n",
        "evaluation_result = evaluator.evaluate_response(\n",
        "    query=questions[0], response=basic_rag_responses[0]\n",
        ")\n",
        "print(evaluation_result)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 384
        },
        "id": "cqc1-8-sqffT",
        "outputId": "1d7410c3-7c00-42da-d90d-df689f8d3df3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "ModuleNotFoundError",
          "evalue": "No module named 'deepeval.integrations.llamaindex'",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-43-29995e72c639>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mdeepeval\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mintegrations\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mllamaindex\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mDeepEvalFaithfulnessEvaluator\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mevaluator\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDeepEvalFaithfulnessEvaluator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m evaluation_result = evaluator.evaluate_response(\n\u001b[1;32m      4\u001b[0m     \u001b[0mquery\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mquestions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbasic_rag_responses\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m )\n",
            "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'deepeval.integrations.llamaindex'",
            "",
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0;32m\nNOTE: If your import is failing due to a missing package, you can\nmanually install dependencies using either !pip or !apt.\n\nTo view examples of installing some common dependencies, click the\n\"Open Examples\" button below.\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n"
          ],
          "errorDetails": {
            "actions": [
              {
                "action": "open_url",
                "actionText": "Open Examples",
                "url": "/notebooks/snippets/importing_libraries.ipynb"
              }
            ]
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from deepeval import assert_test\n",
        "from deepeval.test_case import LLMTestCase\n",
        "from deepeval.metrics import AnswerRelevancyMetric\n",
        "\n",
        "def compute_answer_relevancy(\n",
        "    question: str,\n",
        "    response: str,\n",
        "    retrieved_context,\n",
        "    threshold=0.5,\n",
        "    local_llm_evaluator=None,\n",
        "    ):\n",
        "    answer_relevancy_metric = AnswerRelevancyMetric(\n",
        "        threshold=threshold, model=local_llm_evaluator\n",
        "    )\n",
        "    test_case = LLMTestCase(\n",
        "        input=question,\n",
        "        actual_output=response,\n",
        "        retrieval_context=retrieved_context\n",
        "    )\n",
        "    # assert_test(test_case, [answer_relevancy_metric])\n",
        "    return answer_relevancy_metric.measure(test_case)\n"
      ],
      "metadata": {
        "id": "1kD7uIWocOY-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(llm_de.generate(\"What is aquaman's favorite food?\"))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QiGAFgzmfX8U",
        "outputId": "a678e86a-7433-4c10-ba7f-276712a8c704"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[{'generated_text': \"What is aquaman's favorite food? Aquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish. He loves to eat fish and other seafood dishes, and is known to be particularly fond of seafood from the ocean, which is fitting for a character who is a superhero and a king of the underwater kingdom of Atlantis.\\nWhat is Aquaman's favorite food?\\nAquaman's favorite food is seafood, especially fish.\"}]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "question=\"What if these shoes don't fit?\"\n",
        "response=\"We offer a 30-day full refund at no extra cost.\",\n",
        "retrieved_context=[\"All customers are eligible for a 30 day full refund at no extra cost.\"]\n",
        "\n",
        "relevancy_metric = compute_answer_relevancy(\n",
        "    question, response, retrieved_context, local_llm_evaluator=llm_de\n",
        ")\n",
        "print(relevancy_metric.score, relevancy_metric.reason)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 425,
          "referenced_widgets": [
            "ce2a847b9f4746b5ae7f34b5ccf1013b",
            "6894670075f64c1bb7bf06ea45c05e2c"
          ]
        },
        "id": "6kESYRRhCckm",
        "outputId": "9802e08c-98c5-4ee4-beaf-22046d971001"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Output()"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ce2a847b9f4746b5ae7f34b5ccf1013b"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Event loop is already running. Applying nest_asyncio patch to allow async execution...\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Event loop is already running. Applying nest_asyncio patch to allow async execution...\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"></pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "error",
          "ename": "AttributeError",
          "evalue": "'list' object has no attribute 'find'",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/deepeval/metrics/answer_relevancy/answer_relevancy.py\u001b[0m in \u001b[0;36m_a_generate_statements\u001b[0;34m(self, actual_output)\u001b[0m\n\u001b[1;32m    233\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 234\u001b[0;31m                 res: Statements = await self.model.a_generate(\n\u001b[0m\u001b[1;32m    235\u001b[0m                     \u001b[0mprompt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mschema\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mStatements\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mTypeError\u001b[0m: CustomLlama3_8B.a_generate() got an unexpected keyword argument 'schema'",
            "\nDuring handling of the above exception, another exception occurred:\n",
            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-18-1f7074f87f73>\u001b[0m in \u001b[0;36m<cell line: 5>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mretrieved_context\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"All customers are eligible for a 30 day full refund at no extra cost.\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m relevancy_metric = compute_answer_relevancy(\n\u001b[0m\u001b[1;32m      6\u001b[0m     \u001b[0mquestion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretrieved_context\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlocal_llm_evaluator\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mllama3_evaluator_llm\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m )\n",
            "\u001b[0;32m<ipython-input-16-c1b518887cf5>\u001b[0m in \u001b[0;36mcompute_answer_relevancy\u001b[0;34m(question, response, retrieved_context, threshold, local_llm_evaluator)\u001b[0m\n\u001b[1;32m     19\u001b[0m     )\n\u001b[1;32m     20\u001b[0m     \u001b[0;31m# assert_test(test_case, [answer_relevancy_metric])\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0manswer_relevancy_metric\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmeasure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_case\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/deepeval/metrics/answer_relevancy/answer_relevancy.py\u001b[0m in \u001b[0;36mmeasure\u001b[0;34m(self, test_case, _show_indicator)\u001b[0m\n\u001b[1;32m     52\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync_mode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m                 \u001b[0mloop\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_or_create_event_loop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 54\u001b[0;31m                 loop.run_until_complete(\n\u001b[0m\u001b[1;32m     55\u001b[0m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0ma_measure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_case\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_show_indicator\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     56\u001b[0m                 )\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/nest_asyncio.py\u001b[0m in \u001b[0;36mrun_until_complete\u001b[0;34m(self, future)\u001b[0m\n\u001b[1;32m     96\u001b[0m                 raise RuntimeError(\n\u001b[1;32m     97\u001b[0m                     'Event loop stopped before Future completed.')\n\u001b[0;32m---> 98\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     99\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_run_once\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.10/asyncio/futures.py\u001b[0m in \u001b[0;36mresult\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    199\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__log_traceback\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    200\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_exception\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 201\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_exception\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_exception_tb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    202\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_result\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    203\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/lib/python3.10/asyncio/tasks.py\u001b[0m in \u001b[0;36m__step\u001b[0;34m(***failed resolving arguments***)\u001b[0m\n\u001b[1;32m    230\u001b[0m                 \u001b[0;31m# We use the `send` method directly, because coroutines\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    231\u001b[0m                 \u001b[0;31m# don't have `__iter__` and `__next__` methods.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 232\u001b[0;31m                 \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcoro\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    233\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    234\u001b[0m                 \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcoro\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mthrow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/deepeval/metrics/answer_relevancy/answer_relevancy.py\u001b[0m in \u001b[0;36ma_measure\u001b[0;34m(self, test_case, _show_indicator)\u001b[0m\n\u001b[1;32m     87\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0masync_mode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_show_indicator\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0m_show_indicator\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     88\u001b[0m         ):\n\u001b[0;32m---> 89\u001b[0;31m             self.statements: List[str] = await self._a_generate_statements(\n\u001b[0m\u001b[1;32m     90\u001b[0m                 \u001b[0mtest_case\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mactual_output\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m             )\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/deepeval/metrics/answer_relevancy/answer_relevancy.py\u001b[0m in \u001b[0;36m_a_generate_statements\u001b[0;34m(self, actual_output)\u001b[0m\n\u001b[1;32m    238\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    239\u001b[0m                 \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mawait\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0ma_generate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprompt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 240\u001b[0;31m                 \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrimAndLoadJson\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mres\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    241\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"statements\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    242\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/deepeval/metrics/utils.py\u001b[0m in \u001b[0;36mtrimAndLoadJson\u001b[0;34m(input_string, metric)\u001b[0m\n\u001b[1;32m    227\u001b[0m     \u001b[0minput_string\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetric\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mOptional\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mBaseMetric\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    228\u001b[0m ) -> Any:\n\u001b[0;32m--> 229\u001b[0;31m     \u001b[0mstart\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput_string\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"{\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    230\u001b[0m     \u001b[0mend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput_string\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrfind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"}\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    231\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mAttributeError\u001b[0m: 'list' object has no attribute 'find'"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "---"
      ],
      "metadata": {
        "id": "h1Nl5l3APcJZ"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "---"
      ],
      "metadata": {
        "id": "XLy_Vj5TPeHv"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# ----- End of Report -----"
      ],
      "metadata": {
        "id": "MJy4Ke9QPfV_"
      }
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "j86zVv6J1qvM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "_VJlibwW12O9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "aWLysBzXDiOw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "k85vdezpCca8"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "9VB5NOMLStmC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "lDLOf8OjStWF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "---\n",
        "## Unused Code Below, Experimentation"
      ],
      "metadata": {
        "id": "zowShybGvoG0"
      }
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "m3_eMIJj_je5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def basic_chunking_docs(doc_list: list[LangchainDocument], tokenizer_name: str) -> list[str]:\n",
        "    \"\"\"\n",
        "    Split documents into chunks of size `chunk_size` characters and return a\n",
        "    list of chunks (str). Chunk size determined by SentenceTransformer text\n",
        "    embedder context window size.  Return Document objects\n",
        "    \"\"\"\n",
        "    text_splitter = SentenceTransformersTokenTextSplitter(\n",
        "        model_name=basic_embedder_name,\n",
        "        chunk_overlap=0,\n",
        "    )\n",
        "    return text_splitter.split_documents(doc_list)\n",
        "\n",
        "\n",
        "def advanced_chunking(\n",
        "    doc_list: list[str],\n",
        "    huggingface_embedder_name: str\n",
        "    ) -> list[LangchainDocument]:\n",
        "    \"\"\"\n",
        "    Split document pages into chunks using semantic similarity.\n",
        "    Using the higher dimension of the advanced text embedder, and\n",
        "    using the embedding gradient chuck boundary method since all data is in\n",
        "    the financial domain and may have highly correlated embedding similarity.\n",
        "\n",
        "    Applying Chunking on each page separately to preserve that semantic boundary.\n",
        "    \"\"\"\n",
        "    text_splitter = SemanticChunker(\n",
        "        HuggingFaceEmbeddings(model_name=advanced_embedder_name),\n",
        "        breakpoint_threshold_type=\"gradient\"\n",
        "    )\n",
        "    return text_splitter.create_documents(doc_list)\n"
      ],
      "metadata": {
        "id": "0Em46PVpcOWP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "f3HmNkmuvZC0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# from semantic_router.encoders import HuggingFaceEncoder\n",
        "# from semantic_chunkers import StatisticalChunker\n",
        "# from semantic_router.encoders import HuggingFaceEncoder\n",
        "# from langchain_community.document_loaders import TextLoader\n",
        "# from langchain_mistralai.chat_models import ChatMistralAI\n",
        "# from langchain_mistralai.embeddings import MistralAIEmbeddings\n",
        "# from langchain_community.vectorstores import FAISS\n",
        "# from langchain_community.docstore.in_memory import InMemoryDocstore\n",
        "# from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
        "# from langchain.chains.combine_documents import create_stuff_documents_chain\n",
        "# from langchain_core.prompts import ChatPromptTemplate\n",
        "# from langchain.chains import create_retrieval_chain\n",
        "\n",
        "## BASIC RAG IMPLEMENTATION - LangChain\n",
        "\n",
        "# # Load data\n",
        "arkk_pages, arkk_page_text, arkk_full_text = get_pdf_text(\"Investment_Case_For_Disruptive_Innovation.pdf\")\n",
        "# # Split text into chunks\n",
        "doc_chunks = basic_chunking_docs(arkk_pages, basic_embedder_name)\n",
        "\n",
        "# # Define the embedding model\n",
        "embedding_function = HuggingFaceEmbeddings(model_name=basic_embedder_name)\n",
        "\n",
        "# # Create the vector store\n",
        "vector_db = FAISS.from_documents(\n",
        "    doc_chunks,\n",
        "    embedding_function,\n",
        "    docstore= InMemoryDocstore(),\n",
        ")\n",
        "\n",
        "# Define a retriever interface\n",
        "retriever = vector_db.as_retriever(\n",
        "    search_kwargs={\"k\": 3},\n",
        "    search_type = 'similarity'\n",
        ")\n",
        "\n",
        "# Define LLM\n",
        "llm = ChatMistralAI(mistral_api_key=mistral_api_key)\n",
        "\n",
        "# Define prompt template\n",
        "prompt = hub.pull(\"rlm/rag-prompt-mistral\")\n",
        "\"\"\" Prompt used\n",
        "<s> [INST] You are an assistant for question-answering tasks. Use the following pieces\n",
        "of retrieved context to answer the question. If you don't know the answer, just say\n",
        "that you don't know. Use three sentences maximum and keep the answer concise. [/INST]\n",
        " </s> \\n[INST] Question: {question} \\nContext: {context} \\nAnswer: [/INST]\"\n",
        "\"\"\"\n",
        "\n",
        "# Create a retrieval chain to answer questions\n",
        "document_chain = create_stuff_documents_chain(llm, prompt)\n",
        "retrieval_chain = create_retrieval_chain(retriever, document_chain)\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "## Download Llama3 instruct 8b for local LLM eval\n",
        "# from transformers import BitsAndBytesConfig\n",
        "# from transformers import AutoModelForCausalLM, AutoTokenizer\n",
        "\n",
        "\n",
        "# quantization_config = BitsAndBytesConfig(\n",
        "#     load_in_4bit=True,\n",
        "#     bnb_4bit_compute_dtype=torch.float16,\n",
        "#     bnb_4bit_quant_type=\"nf4\",\n",
        "#     bnb_4bit_use_double_quant=True,\n",
        "# )\n",
        "\n",
        "# model_4bit = AutoModelForCausalLM.from_pretrained(\n",
        "#     \"NousResearch/Meta-Llama-3.1-8B-Instruct\",\n",
        "#     device_map=\"auto\",\n",
        "#     quantization_config=quantization_config,\n",
        "# )\n",
        "# llama_tokenizer = AutoTokenizer.from_pretrained(\n",
        "#     \"NousResearch/Meta-Llama-3.1-8B-Instruct\"\n",
        "# )"
      ],
      "metadata": {
        "id": "w4VP_bi4vY_b"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# ### Llama3 prompt\n",
        "# langchain_prompt = hub.pull(\"rlm/rag-prompt-llama3\")\n",
        "# ## NEED To implement the partial varibles by name for query\n",
        "# \"\"\"\n",
        "# \"<|begin_of_text|><|start_header_id|>system<|end_header_id|> You are an assistant for question-answering tasks.\n",
        "# Use the following pieces of retrieved context to answer the question. If you don't know the answer, just say that\n",
        "# you don't know. Use three sentences maximum and keep the answer concise <|eot_id|><|start_header_id|>user<|end_header_id|>\n",
        "# \\nQuestion: {question} \\nContext: {context} \\nAnswer: <|eot_id|><|start_header_id|>assistant<|end_header_id|>\\n\n",
        "# \"\"\"\n",
        "\n",
        "\n",
        "# # RAG Prompt\n",
        "# langchain_prompt = hub.pull(\"rlm/rag-prompt\")\n",
        "# \"\"\"\n",
        "# You are an assistant for question-answering tasks. Use the following pieces of\n",
        "# retrieved context to answer the question. If you don't know the answer, just say\n",
        "# that you don't know. Use three sentences maximum and keep the answer concise.\n",
        "\n",
        "# Question: {question}\n",
        "\n",
        "# Context: {context}\n",
        "\n",
        "# Answer:\n",
        "# \"\"\"\n",
        "\n",
        "# ## Mistral/Mixtral prompt\n",
        "# langchain_prompt_prompt = hub.pull(\"rlm/rag-prompt-mistral\")\n",
        "# \"\"\"\n",
        "# <s> [INST] You are an assistant for question-answering tasks. Use the following pieces\n",
        "#  of retrieved context to answer the question. If you don't know the answer, just say\n",
        "#  that you don't know. Use three sentences maximum and keep the answer concise. [/INST] </s>\n",
        "\n",
        "# [INST] Question: {question}\n",
        "\n",
        "# Context: {context}\n",
        "\n",
        "# Answer: [/INST]\n",
        "# \"\"\"\n"
      ],
      "metadata": {
        "id": "ZMUEkD9MvZ5V"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "FFY0ZxqgUAJS"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}