_semantic-search-demo.ipynb

{
  "metadata": {
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3 (ipykernel)",
      "language": "python"
    },
    "language_info": {
      "name": "python",
      "version": "3.10.4",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "colab": {
      "provenance": []
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "0f72bebe7ec14d168f593229b7d1d9ae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ac15519b665f4df2b0fc3cd79aeb64cf",
              "IPY_MODEL_7ea96b5bac674036badf5436a77b2d2b",
              "IPY_MODEL_65b33cb5a74f4b508efe23f08be03bee"
            ],
            "layout": "IPY_MODEL_56f4a3ad1e5b413ab62d581856b64b2e"
          }
        },
        "ac15519b665f4df2b0fc3cd79aeb64cf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9c17c3435697426cbab5d87b9c695e7f",
            "placeholder": "​",
            "style": "IPY_MODEL_35d9d4ae190d447a875713bd58634e8f",
            "value": "  0%"
          }
        },
        "7ea96b5bac674036badf5436a77b2d2b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "danger",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0f0c373ac96843be9495a27d931182c8",
            "max": 80,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6dd08beb89a44000bea3d538da009ac1",
            "value": 0
          }
        },
        "65b33cb5a74f4b508efe23f08be03bee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6e8b1b531dd747c18ad18b3ec99047e4",
            "placeholder": "​",
            "style": "IPY_MODEL_ce6b56aa66404b069de916e9105d9a65",
            "value": " 0/80 [00:00&lt;?, ?it/s]"
          }
        },
        "56f4a3ad1e5b413ab62d581856b64b2e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9c17c3435697426cbab5d87b9c695e7f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "35d9d4ae190d447a875713bd58634e8f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0f0c373ac96843be9495a27d931182c8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6dd08beb89a44000bea3d538da009ac1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6e8b1b531dd747c18ad18b3ec99047e4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ce6b56aa66404b069de916e9105d9a65": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "22395efbc8b944afadf133e4e32f55fd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_db372f0e0b5e45f7900a13f160f825f5",
              "IPY_MODEL_2ca7a69844f5436d8b6fb3bc8608f013",
              "IPY_MODEL_40914ce6801341b5b898e16d0c7150ea"
            ],
            "layout": "IPY_MODEL_d94b6f6b777348599aaadb4c093db40a"
          }
        },
        "db372f0e0b5e45f7900a13f160f825f5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fab27e0c7c59413a9912629ae7c96b1e",
            "placeholder": "​",
            "style": "IPY_MODEL_b3f3ece736ef43da9cfad0e5a6bcc249",
            "value": "100%"
          }
        },
        "2ca7a69844f5436d8b6fb3bc8608f013": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6a655a09dac54d4697abd00450f64c08",
            "max": 80,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_edf670dd90c5428cb9ca6bbdff2bf78a",
            "value": 80
          }
        },
        "40914ce6801341b5b898e16d0c7150ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e60ec350d38742e6b4d2aeb13f259e1d",
            "placeholder": "​",
            "style": "IPY_MODEL_887302ccad3c41e0bafe6aff358c0a08",
            "value": " 80/80 [04:00&lt;00:00,  3.00s/it]"
          }
        },
        "d94b6f6b777348599aaadb4c093db40a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fab27e0c7c59413a9912629ae7c96b1e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b3f3ece736ef43da9cfad0e5a6bcc249": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6a655a09dac54d4697abd00450f64c08": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "edf670dd90c5428cb9ca6bbdff2bf78a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e60ec350d38742e6b4d2aeb13f259e1d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "887302ccad3c41e0bafe6aff358c0a08": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat_minor": 5,
  "nbformat": 4,
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Welcome to Colab Enterprise <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
        "\n",
        "Connect to a Runtime and begin!"
      ],
      "metadata": {
        "id": "7aZWXpbf0Eph"
      },
      "id": "7aZWXpbf0Eph"
    },
    {
      "cell_type": "markdown",
      "source": [
        "<div class=\"markdown-google-sans\">\n",
        "\n",
        "## **Getting started**\n",
        "</div>\n",
        "\n",
        "Set up Vertexai"
      ],
      "metadata": {
        "id": "OQ_SlL0K1Cw6"
      },
      "id": "OQ_SlL0K1Cw6"
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "IBPUfL6J-2Ui"
      },
      "id": "IBPUfL6J-2Ui"
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install google-cloud-aiplatform"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4edmFM2mAV0e",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1696361188779,
          "user_tz": 240,
          "elapsed": 7512,
          "user": {
            "displayName": "",
            "userId": ""
          }
        },
        "outputId": "196f4d0c-03b4-4e7e-95d8-46faf93dc8d0"
      },
      "id": "4edmFM2mAV0e",
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: google-cloud-aiplatform in /usr/local/lib/python3.10/dist-packages (1.32.0)\n",
            "Requirement already satisfied: google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.32.0 in /usr/local/lib/python3.10/dist-packages (from google-cloud-aiplatform) (2.11.1)\n",
            "Requirement already satisfied: proto-plus<2.0.0dev,>=1.22.0 in /usr/local/lib/python3.10/dist-packages (from google-cloud-aiplatform) (1.22.3)\n",
            "Requirement already satisfied: protobuf!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev,>=3.19.5 in /usr/local/lib/python3.10/dist-packages (from google-cloud-aiplatform) (3.20.3)\n",
            "Requirement already satisfied: packaging>=14.3 in /usr/local/lib/python3.10/dist-packages (from google-cloud-aiplatform) (23.1)\n",
            "Requirement already satisfied: google-cloud-storage<3.0.0dev,>=1.32.0 in /usr/local/lib/python3.10/dist-packages (from google-cloud-aiplatform) (2.8.0)\n",
            "Requirement already satisfied: google-cloud-bigquery<4.0.0dev,>=1.15.0 in /usr/local/lib/python3.10/dist-packages (from google-cloud-aiplatform) (3.10.0)\n",
            "Requirement already satisfied: google-cloud-resource-manager<3.0.0dev,>=1.3.3 in /usr/local/lib/python3.10/dist-packages (from google-cloud-aiplatform) (1.10.3)\n",
            "Requirement already satisfied: shapely<2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-cloud-aiplatform) (1.8.5.post1)\n",
            "Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in /usr/local/lib/python3.10/dist-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.32.0->google-cloud-aiplatform) (1.60.0)\n",
            "Requirement already satisfied: google-auth<3.0.dev0,>=2.14.1 in /usr/local/lib/python3.10/dist-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.32.0->google-cloud-aiplatform) (2.17.3)\n",
            "Requirement already satisfied: requests<3.0.0.dev0,>=2.18.0 in /usr/local/lib/python3.10/dist-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.32.0->google-cloud-aiplatform) (2.31.0)\n",
            "Requirement already satisfied: grpcio<2.0dev,>=1.33.2 in /usr/local/lib/python3.10/dist-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.32.0->google-cloud-aiplatform) (1.57.0)\n",
            "Requirement already satisfied: grpcio-status<2.0.dev0,>=1.33.2 in /usr/local/lib/python3.10/dist-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.32.0->google-cloud-aiplatform) (1.48.2)\n",
            "Requirement already satisfied: google-cloud-core<3.0.0dev,>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from google-cloud-bigquery<4.0.0dev,>=1.15.0->google-cloud-aiplatform) (2.3.3)\n",
            "Requirement already satisfied: google-resumable-media<3.0dev,>=0.6.0 in /usr/local/lib/python3.10/dist-packages (from google-cloud-bigquery<4.0.0dev,>=1.15.0->google-cloud-aiplatform) (2.6.0)\n",
            "Requirement already satisfied: python-dateutil<3.0dev,>=2.7.2 in /usr/local/lib/python3.10/dist-packages (from google-cloud-bigquery<4.0.0dev,>=1.15.0->google-cloud-aiplatform) (2.8.2)\n",
            "Requirement already satisfied: grpc-google-iam-v1<1.0.0dev,>=0.12.4 in /usr/local/lib/python3.10/dist-packages (from google-cloud-resource-manager<3.0.0dev,>=1.3.3->google-cloud-aiplatform) (0.12.6)\n",
            "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.32.0->google-cloud-aiplatform) (5.3.1)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.32.0->google-cloud-aiplatform) (0.3.0)\n",
            "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.32.0->google-cloud-aiplatform) (1.16.0)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth<3.0.dev0,>=2.14.1->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.32.0->google-cloud-aiplatform) (4.9)\n",
            "Requirement already satisfied: google-crc32c<2.0dev,>=1.0 in /usr/local/lib/python3.10/dist-packages (from google-resumable-media<3.0dev,>=0.6.0->google-cloud-bigquery<4.0.0dev,>=1.15.0->google-cloud-aiplatform) (1.5.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0.dev0,>=2.18.0->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.32.0->google-cloud-aiplatform) (3.2.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0.dev0,>=2.18.0->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.32.0->google-cloud-aiplatform) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0.dev0,>=2.18.0->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.32.0->google-cloud-aiplatform) (2.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0.dev0,>=2.18.0->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.32.0->google-cloud-aiplatform) (2023.7.22)\n",
            "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3.0.dev0,>=2.14.1->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<3.0.0dev,>=1.32.0->google-cloud-aiplatform) (0.5.0)\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0m"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "! gcloud auth login"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e3tiAmSLDWml",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1696362012545,
          "user_tz": 240,
          "elapsed": 42284,
          "user": {
            "displayName": "",
            "userId": ""
          }
        },
        "outputId": "6da0b774-6d0c-4e97-e3b5-24fcbeaa0efd"
      },
      "id": "e3tiAmSLDWml",
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "You are running on a Google Compute Engine virtual machine.\n",
            "It is recommended that you use service accounts for authentication.\n",
            "\n",
            "You can run:\n",
            "\n",
            "  $ gcloud config set account `ACCOUNT`\n",
            "\n",
            "to switch accounts if necessary.\n",
            "\n",
            "Your credentials may be visible to others with access to this\n",
            "virtual machine. Are you sure you want to authenticate with\n",
            "your personal account?\n",
            "\n",
            "Do you want to continue (Y/n)?  y\n",
            "\n",
            "Go to the following link in your browser:\n",
            "\n",
            "    https://accounts.google.com/o/oauth2/auth?response_type=code&client_id=32555940559.apps.googleusercontent.com&redirect_uri=https%3A%2F%2Fsdk.cloud.google.com%2Fauthcode.html&scope=openid+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fuserinfo.email+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fcloud-platform+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fappengine.admin+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fsqlservice.login+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fcompute+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Faccounts.reauth&state=9pekC5eQpP2gFG1aYpLQrgsn1PvLKt&prompt=consent&access_type=offline&code_challenge=2Qw4_tEvXq3VTFJNlLzryS9qNDkgpTozoxpdldVDYPQ&code_challenge_method=S256\n",
            "\n",
            "Enter authorization code: 4/0AfJohXlENTris_3NmtNBNlwIzmEjO2zGmg_WGpgYuSGYnbPE3rUZhd1c2okgWBSxfHMJnw\n",
            "\n",
            "You are now logged in as [chrisge@google.com].\n",
            "Your current project is [cd-demo-01].  You can change this setting by running:\n",
            "  $ gcloud config set project PROJECT_ID\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import auth\n",
        "\n",
        "auth.authenticate_user()"
      ],
      "metadata": {
        "id": "1SXseKJZDq7v",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1696362110121,
          "user_tz": 240,
          "elapsed": 591,
          "user": {
            "displayName": "",
            "userId": ""
          }
        }
      },
      "id": "1SXseKJZDq7v",
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "PROJECT_ID = 'cd-demo-01'\n",
        "REGION = 'us-central1'\n",
        "import vertexai\n",
        "vertexai.init(project=PROJECT_ID,\n",
        "              location=REGION)"
      ],
      "metadata": {
        "trusted": true,
        "id": "35856bf2-aa5e-436b-977a-9e5725b1a595",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1696362180785,
          "user_tz": 240,
          "elapsed": 40903,
          "user": {
            "displayName": "",
            "userId": ""
          }
        }
      },
      "execution_count": 6,
      "outputs": [],
      "id": "35856bf2-aa5e-436b-977a-9e5725b1a595"
    },
    {
      "cell_type": "code",
      "source": [
        "from concurrent.futures import ThreadPoolExecutor\n",
        "from tqdm.auto import tqdm\n",
        "import math\n",
        "import functools\n",
        "import time\n",
        "\n",
        "def encode_texts_to_embeddings(sentences):\n",
        "    model = TextEmbeddingModel.from_pretrained(\"textembedding-gecko@001\")\n",
        "    try:\n",
        "        embeddings = model.get_embeddings(sentences)\n",
        "        return [embedding.values for embedding in embeddings]\n",
        "    except Exception:\n",
        "        return [None for _ in range(len(sentences))]\n",
        "\n",
        "def encode_text_to_embedding_batched(sentences, api_calls_per_second = 0.33, batch_size = 5):\n",
        "    # Generates batches and calls embedding API\n",
        "\n",
        "    embeddings_list = []\n",
        "\n",
        "    # Prepare the batches using a generator\n",
        "    batches = generate_batches(sentences, batch_size)\n",
        "\n",
        "    seconds_per_job = 1 / api_calls_per_second\n",
        "\n",
        "    with ThreadPoolExecutor() as executor:\n",
        "        futures = []\n",
        "        for batch in tqdm(\n",
        "            batches, total = math.ceil(len(sentences) / batch_size), position=0\n",
        "        ):\n",
        "            futures.append(\n",
        "                executor.submit(functools.partial(encode_texts_to_embeddings), batch)\n",
        "            )\n",
        "            time.sleep(seconds_per_job)\n",
        "\n",
        "        for future in futures:\n",
        "            embeddings_list.extend(future.result())\n",
        "\n",
        "    is_successful = [\n",
        "        embedding is not None for sentence, embedding in zip(sentences, embeddings_list)\n",
        "    ]\n",
        "    embeddings_list_successful = np.squeeze(\n",
        "        np.stack([embedding for embedding in embeddings_list if embedding is not None])\n",
        "    )\n",
        "    return embeddings_list_successful"
      ],
      "metadata": {
        "id": "nAxo0TaoEP-s",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1696363861788,
          "user_tz": 240,
          "elapsed": 186,
          "user": {
            "displayName": "",
            "userId": ""
          }
        }
      },
      "id": "nAxo0TaoEP-s",
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Generate text embeddings"
      ],
      "metadata": {
        "id": "sgVmjm51EqnQ"
      },
      "id": "sgVmjm51EqnQ"
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "so_df = pd.DataFrame()\n",
        "so_df = pd.read_csv('/content/deploy-build-stackoverflow.csv')"
      ],
      "metadata": {
        "id": "3tuhHLm7JRAn"
      },
      "id": "3tuhHLm7JRAn",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from vertexai.language_models import TextEmbeddingModel\n",
        "\n",
        "\n",
        "model = TextEmbeddingModel.from_pretrained(\n",
        "    \"textembedding-gecko@001\")\n",
        "import time\n",
        "import numpy as np\n",
        "\n",
        "def generate_batches(sentences, batch_size = 5):\n",
        "    for i in range(0, len(sentences), batch_size):\n",
        "        yield sentences[i : i + batch_size]\n",
        "\n",
        "# so_questions = so_df[0:200].input_text.tolist()\n",
        "# batches = generate_batches(sentences = so_questions)\n",
        "\n",
        "so_questions = so_df.input_text.tolist()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 416,
          "referenced_widgets": [
            "0f72bebe7ec14d168f593229b7d1d9ae",
            "ac15519b665f4df2b0fc3cd79aeb64cf",
            "7ea96b5bac674036badf5436a77b2d2b",
            "65b33cb5a74f4b508efe23f08be03bee",
            "56f4a3ad1e5b413ab62d581856b64b2e",
            "9c17c3435697426cbab5d87b9c695e7f",
            "35d9d4ae190d447a875713bd58634e8f",
            "0f0c373ac96843be9495a27d931182c8",
            "6dd08beb89a44000bea3d538da009ac1",
            "6e8b1b531dd747c18ad18b3ec99047e4",
            "ce6b56aa66404b069de916e9105d9a65"
          ]
        },
        "id": "YIhYSHygEeO0",
        "executionInfo": {
          "status": "error",
          "timestamp": 1696363692374,
          "user_tz": 240,
          "elapsed": 184,
          "user": {
            "displayName": "",
            "userId": ""
          }
        },
        "outputId": "3e42e7f4-379d-4630-b5cc-4ff943224651"
      },
      "id": "YIhYSHygEeO0",
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/80 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "0f72bebe7ec14d168f593229b7d1d9ae"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-16-7c97812d1076>\u001b[0m in \u001b[0;36m<cell line: 17>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0mso_questions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mso_df\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minput_text\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtolist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m question_embeddings = encode_text_to_embedding_batched(\n\u001b[0m\u001b[1;32m     18\u001b[0m                             \u001b[0msentences\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mso_questions\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m                             \u001b[0mapi_calls_per_second\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m20\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0;36m60\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-7-bc0df2dd5325>\u001b[0m in \u001b[0;36mencode_text_to_embedding_batched\u001b[0;34m(sentences, api_calls_per_second, batch_size)\u001b[0m\n\u001b[1;32m     15\u001b[0m         ):\n\u001b[1;32m     16\u001b[0m             futures.append(\n\u001b[0;32m---> 17\u001b[0;31m                 \u001b[0mexecutor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msubmit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunctools\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpartial\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mencode_texts_to_embeddings\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     18\u001b[0m             )\n\u001b[1;32m     19\u001b[0m             \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseconds_per_job\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'functools' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### OPTIONAL: Gen embeddings"
      ],
      "metadata": {
        "id": "vw2fFMVMP-Im"
      },
      "id": "vw2fFMVMP-Im"
    },
    {
      "cell_type": "code",
      "source": [
        "question_embeddings = encode_text_to_embedding_batched(\n",
        "                            sentences=so_questions,\n",
        "                            api_calls_per_second = 20/60,\n",
        "                            batch_size = 5)\n",
        "\n",
        "question_embeddings"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 288,
          "referenced_widgets": [
            "22395efbc8b944afadf133e4e32f55fd",
            "db372f0e0b5e45f7900a13f160f825f5",
            "2ca7a69844f5436d8b6fb3bc8608f013",
            "40914ce6801341b5b898e16d0c7150ea",
            "d94b6f6b777348599aaadb4c093db40a",
            "fab27e0c7c59413a9912629ae7c96b1e",
            "b3f3ece736ef43da9cfad0e5a6bcc249",
            "6a655a09dac54d4697abd00450f64c08",
            "edf670dd90c5428cb9ca6bbdff2bf78a",
            "e60ec350d38742e6b4d2aeb13f259e1d",
            "887302ccad3c41e0bafe6aff358c0a08"
          ]
        },
        "id": "xkluEfg4KNWl",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1696364107831,
          "user_tz": 240,
          "elapsed": 240562,
          "user": {
            "displayName": "",
            "userId": ""
          }
        },
        "outputId": "155cd502-fab5-42e3-b639-1f692a855e7f"
      },
      "id": "xkluEfg4KNWl",
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/80 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "22395efbc8b944afadf133e4e32f55fd"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[-0.02668177, -0.02851841,  0.01165711, ..., -0.01442359,\n",
              "        -0.0043021 , -0.06496598],\n",
              "       [-0.04665147, -0.04166554,  0.03035616, ..., -0.02276639,\n",
              "        -0.02370534, -0.03365194],\n",
              "       [-0.00835284, -0.0044221 ,  0.02535915, ..., -0.0284955 ,\n",
              "        -0.03309781, -0.053542  ],\n",
              "       ...,\n",
              "       [-0.00694331, -0.00619193,  0.03283683, ..., -0.0260459 ,\n",
              "        -0.01802497, -0.0144918 ],\n",
              "       [-0.01039375, -0.0048393 ,  0.04200063, ..., -0.0192529 ,\n",
              "        -0.01571974, -0.04594716],\n",
              "       [-0.01133255, -0.02716326,  0.01895321, ..., -0.02744324,\n",
              "        -0.02382765, -0.05456362]])"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pickle\n",
        "with open('/content/questions_embadddings_data.pkl', 'wb') as f:\n",
        "  pickle.dump(question_embeddings, f)"
      ],
      "metadata": {
        "id": "TysXAY7TK0ga",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1696364280924,
          "user_tz": 240,
          "elapsed": 135,
          "user": {
            "displayName": "",
            "userId": ""
          }
        }
      },
      "id": "TysXAY7TK0ga",
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Gen search data"
      ],
      "metadata": {
        "id": "Q6Y7tRd5RIFz"
      },
      "id": "Q6Y7tRd5RIFz"
    },
    {
      "cell_type": "code",
      "source": [
        "# print(\"Shape: \" + str(so_df.shape))\n",
        "# print(so_df)\n",
        "\n",
        "so_df['embeddings'] = question_embeddings.tolist()\n",
        "\n",
        "\n",
        "so_df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "l0n4A7JIMX8s",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1696365835004,
          "user_tz": 240,
          "elapsed": 126,
          "user": {
            "displayName": "",
            "userId": ""
          }
        },
        "outputId": "cf4cdabf-e3fd-4ed5-dd43-938122c9f818"
      },
      "id": "l0n4A7JIMX8s",
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                            input_text  \\\n",
              "0    What IAM role does serviceaccount need for exp...   \n",
              "1    Share (parameterized) cloudbuild.yaml between ...   \n",
              "2    Google Cloud Build Generating cloud_run_revisi...   \n",
              "3    GCP deploy cloud function failed<p>I was tryin...   \n",
              "4    Google Cloud Build show no logs<p>I have a Goo...   \n",
              "..                                                 ...   \n",
              "395  Google Cloud Build how to send context with di...   \n",
              "396  Why Google Cloud Build creates intermediate co...   \n",
              "397  How to configure Serverless VPC Access from Cl...   \n",
              "398  How to pass authenticated state from the cloud...   \n",
              "399  Running Cloud Build trigger via GCP Console re...   \n",
              "\n",
              "                                           output_text  \\\n",
              "0    <p>I needed to activate &quot;Cloud Build API&...   \n",
              "1    <p>I contacted Google Cloud support and they t...   \n",
              "2    <p>The log entry shown corresponds to a <code>...   \n",
              "3    <p>We were using this command to generate the ...   \n",
              "4    <p>I do not have all the details, so trying to...   \n",
              "..                                                 ...   \n",
              "395  <p>Posting it as an answer to give it more vis...   \n",
              "396  <p>Cloud Build uses Docker to execute builds. ...   \n",
              "397  <p>You are mixing 2 concepts:</p>\\n<ul>\\n<li>G...   \n",
              "398  <p>You need to explicitly <a href=\"https://clo...   \n",
              "399  <p>You are correct with your guess and this is...   \n",
              "\n",
              "                                            embeddings  \n",
              "0    [-0.026681765913963318, -0.02851841412484646, ...  \n",
              "1    [-0.04665147140622139, -0.04166553542017937, 0...  \n",
              "2    [-0.008352839387953281, -0.0044220974668860435...  \n",
              "3    [-0.023527665063738823, -0.0423089899122715, 0...  \n",
              "4    [-0.025556528940796852, -0.0016413381090387702...  \n",
              "..                                                 ...  \n",
              "395  [-0.04455234482884407, -0.013983490876853466, ...  \n",
              "396  [-0.00949002057313919, -0.017768824473023415, ...  \n",
              "397  [-0.006943307816982269, -0.006191926542669535,...  \n",
              "398  [-0.010393748059868813, -0.004839301574975252,...  \n",
              "399  [-0.011332545429468155, -0.02716326154768467, ...  \n",
              "\n",
              "[400 rows x 3 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-f97ea8d0-33f6-405c-8f59-cc37e28f5df3\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>input_text</th>\n",
              "      <th>output_text</th>\n",
              "      <th>embeddings</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>What IAM role does serviceaccount need for exp...</td>\n",
              "      <td>&lt;p&gt;I needed to activate &amp;quot;Cloud Build API&amp;...</td>\n",
              "      <td>[-0.026681765913963318, -0.02851841412484646, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Share (parameterized) cloudbuild.yaml between ...</td>\n",
              "      <td>&lt;p&gt;I contacted Google Cloud support and they t...</td>\n",
              "      <td>[-0.04665147140622139, -0.04166553542017937, 0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Google Cloud Build Generating cloud_run_revisi...</td>\n",
              "      <td>&lt;p&gt;The log entry shown corresponds to a &lt;code&gt;...</td>\n",
              "      <td>[-0.008352839387953281, -0.0044220974668860435...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>GCP deploy cloud function failed&lt;p&gt;I was tryin...</td>\n",
              "      <td>&lt;p&gt;We were using this command to generate the ...</td>\n",
              "      <td>[-0.023527665063738823, -0.0423089899122715, 0...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Google Cloud Build show no logs&lt;p&gt;I have a Goo...</td>\n",
              "      <td>&lt;p&gt;I do not have all the details, so trying to...</td>\n",
              "      <td>[-0.025556528940796852, -0.0016413381090387702...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>395</th>\n",
              "      <td>Google Cloud Build how to send context with di...</td>\n",
              "      <td>&lt;p&gt;Posting it as an answer to give it more vis...</td>\n",
              "      <td>[-0.04455234482884407, -0.013983490876853466, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>396</th>\n",
              "      <td>Why Google Cloud Build creates intermediate co...</td>\n",
              "      <td>&lt;p&gt;Cloud Build uses Docker to execute builds. ...</td>\n",
              "      <td>[-0.00949002057313919, -0.017768824473023415, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>397</th>\n",
              "      <td>How to configure Serverless VPC Access from Cl...</td>\n",
              "      <td>&lt;p&gt;You are mixing 2 concepts:&lt;/p&gt;\\n&lt;ul&gt;\\n&lt;li&gt;G...</td>\n",
              "      <td>[-0.006943307816982269, -0.006191926542669535,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>398</th>\n",
              "      <td>How to pass authenticated state from the cloud...</td>\n",
              "      <td>&lt;p&gt;You need to explicitly &lt;a href=\"https://clo...</td>\n",
              "      <td>[-0.010393748059868813, -0.004839301574975252,...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>399</th>\n",
              "      <td>Running Cloud Build trigger via GCP Console re...</td>\n",
              "      <td>&lt;p&gt;You are correct with your guess and this is...</td>\n",
              "      <td>[-0.011332545429468155, -0.02716326154768467, ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>400 rows × 3 columns</p>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "      \n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f97ea8d0-33f6-405c-8f59-cc37e28f5df3')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "      \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-f97ea8d0-33f6-405c-8f59-cc37e28f5df3 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-f97ea8d0-33f6-405c-8f59-cc37e28f5df3');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "  \n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "###  Demo Setup\n"
      ],
      "metadata": {
        "id": "l8_ltCe_ScFC"
      },
      "id": "l8_ltCe_ScFC"
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "from sklearn.metrics.pairwise import cosine_similarity\n",
        "\n",
        "from vertexai.language_models import TextEmbeddingModel\n",
        "embedding_model = TextEmbeddingModel.from_pretrained(\n",
        "    \"textembedding-gecko@001\")\n",
        "\n",
        "from vertexai.language_models import TextGenerationModel\n",
        "generation_model = TextGenerationModel.from_pretrained(\n",
        "    \"text-bison@001\")\n",
        "from IPython.display import Markdown, display\n",
        "t_value = 0.2"
      ],
      "metadata": {
        "id": "ymZ_9xFmS8d7",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1696366959663,
          "user_tz": 240,
          "elapsed": 313,
          "user": {
            "displayName": "",
            "userId": ""
          }
        }
      },
      "id": "ymZ_9xFmS8d7",
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Demo"
      ],
      "metadata": {
        "id": "-uLiIKyuVBaI"
      },
      "id": "-uLiIKyuVBaI"
    },
    {
      "cell_type": "code",
      "source": [
        "query = [' Cloud build shows no logs']"
      ],
      "metadata": {
        "id": "VSigRSWiU7_i",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1696367154230,
          "user_tz": 240,
          "elapsed": 176,
          "user": {
            "displayName": "",
            "userId": ""
          }
        }
      },
      "id": "VSigRSWiU7_i",
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "query_embedding = embedding_model.get_embeddings(query)[0].values\n",
        "cos_sim_array = cosine_similarity([query_embedding],\n",
        "                                  list(so_df.embeddings.values))\n",
        "index_doc = np.argmax(cos_sim_array)\n",
        "# context = so_df.input_text[index_doc] + \\\n",
        "# \"\\n Answer: \" + so_df.output_text[index_doc]\n",
        "# context = \"Answer is \"+ so_df.output_text[index_doc]\n",
        "\n",
        "prompt = f\"\"\"Here is the context: {context}\n",
        "             Using the relevant information from the context,\n",
        "             provide an answer to the query: {query}.\"\n",
        "             If the context doesn't provide \\\n",
        "             any relevant information, answer with\n",
        "             [I couldn't find a good match in the \\\n",
        "             document database for your query]. \\\n",
        "             Make the answer more general to \\\n",
        "             all the related questions.\n",
        "\n",
        "\n",
        "             \"\"\"\n",
        "response = generation_model.predict(prompt = prompt,\n",
        "                                    temperature = t_value,\n",
        "                                    max_output_tokens = 1024)\n",
        "display(Markdown(response.text))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 81
        },
        "id": "tUgvX66HV8kV",
        "executionInfo": {
          "status": "ok",
          "timestamp": 1696367554878,
          "user_tz": 240,
          "elapsed": 1288,
          "user": {
            "displayName": "",
            "userId": ""
          }
        },
        "outputId": "ee66f2c5-c3f9-43ab-8bc4-d509b5ab84d2"
      },
      "id": "tUgvX66HV8kV",
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Markdown object>"
            ],
            "text/markdown": "The logs from the quickstart.sh file are not shown in the Cloud Build console because the script is not executed during the build phase. To see the output of the script, you should run the container (either locally on your computer, or via Cloud Shell). If you want to execute a script during the build phase of the container, you should use the RUN command."
          },
          "metadata": {}
        }
      ]
    }
  ]
}