From 12eb7788bf6fb254a90f6ef3cb9db19bee84cdc7 Mon Sep 17 00:00:00 2001 From: Naveen Kumar M Date: Tue, 16 Jul 2024 20:16:58 +0530 Subject: [PATCH 1/7] Add Parameter for detect intent --- .../evaluation_tool__autoeval__colab.ipynb | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb b/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb index 97a8f30b..dd9b8a5f 100644 --- a/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb +++ b/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb @@ -25,10 +25,12 @@ "execution_count": null, "metadata": { "collapsed": true, - "id": "0U8xQwhKrOUq" + "id": "0U8xQwhKrOUq", + "cellView": "form" }, "outputs": [], "source": [ + "# @markdown `install packages`\n", "!pip install dfcx-scrapi --quiet\n", "!pip install rouge-score --quiet\n", "\n", @@ -265,6 +267,7 @@ "\n", "GLOBAL_SCOPE = [\"https://spreadsheets.google.com/feeds\"]\n", "\n", + "PARAMETERS = \"parameters\"\n", "CONVERSATION_ID = \"conversation_id\"\n", "TURN_INDEX = \"turn_index\"\n", "QUERY = \"query\"\n", @@ -277,7 +280,7 @@ "AGENT_URI = \"projects/{project_id}/locations/{location}/agents/{agent_id}\"\n", "\n", "INPUT_SCHEMA_REQUIRED_COLUMNS = [\n", - " CONVERSATION_ID, TURN_INDEX, QUERY, REFERENCE, EXPECTED_URI\n", + " PARAMETERS,CONVERSATION_ID, TURN_INDEX, QUERY, REFERENCE, EXPECTED_URI\n", "]\n", "\n", "_EXECUTION_SEQUENCE_KEY = \"DataStore Execution Sequence\"\n", @@ -774,6 +777,7 @@ " query_param_mapping = {}\n", "\n", " if parameters:\n", + " parameters = json.loads(parameters)\n", " query_param_mapping[\"parameters\"] = parameters\n", "\n", " if populate_data_store_connection_signals:\n", @@ -792,7 +796,7 @@ "\n", " @retry_api_call([i**2 for i in range(MAX_RETRIES)])\n", " def scrape_detect_intent(\n", - " self, query: str, session_id: str | None = None\n", + " self, query: str, session_id: str | None = None, parameters: str | None = None\n", " ) -> VertexConversationResponse:\n", " if session_id is None:\n", " session_id = self.sessions.build_session_id(self.agent_id)\n", @@ -802,6 +806,7 @@ " text=query,\n", " language_code=self.language_code,\n", " populate_data_store_connection_signals=True,\n", + " parameters=parameters,\n", " )\n", " return VertexConversationResponse.from_query_result(response._pb)\n", "\n", @@ -2141,6 +2146,7 @@ "outputs": [], "source": [ "# test agent on a single query\n", + "parameter_str = '{\"channel\" : \"chat\"}'\n", "response = scraper.scrape_detect_intent(query=\"who is the ceo?\")\n", "print(json.dumps(dataclasses.asdict(response), indent=4))" ] @@ -2154,6 +2160,7 @@ "## Data Loading\n", "\n", "The queryset must be in a tabular format that has to contain the following columns:\n", + "- `parameters` _(parameter which needs to be passed as part of detect intent for agent to route to correct flow/page)_\n", "- `conversation_id` _(unique identifier of a conversation, which must be the same for each row that are part of the same conversation)_\n", "- `turn_index` _(index of the query - expected answer pair within a conversation)_\n", "- `query` _(the input question)_\n", @@ -2166,12 +2173,12 @@ "\n", "An example for the queryset can be seen in this table:\n", "\n", - "| conversation_id | turn_index | query | expected_answer | expected_uri |\n", - "| --- | --- | --- | --- | --- |\n", - "| 0 | 1 | What is the capital of France? | Capital of France is Paris. | exampleurl.com/france |\n", - "| 0 | 2 | How many people live there? | 2.1 million people live in Paris. | exampleurl.com/paris |\n", - "| 1 | 1 | What is the color of the sky? | It is blue. | exampleurl.com/common |\n", - "| 2 | 1 | How many legs does an octopus have? | It has 8 limbs. | exampleurl.com/octopus |\n", + "| parameters | conversation_id | turn_index | query | expected_answer | expected_uri |\n", + "| --- | --- | --- | --- | --- | --- |\n", + "| {\"channel\" : \"chat\"} | 0 | 1 | What is the capital of France? | Capital of France is Paris. | exampleurl.com/france |\n", + "| {\"channel\" : \"chat\"} | 0 | 2 | How many people live there? | 2.1 million people live in Paris. | exampleurl.com/paris |\n", + "| {\"channel\" : \"chat\"} | 1 | 1 | What is the color of the sky? | It is blue. | exampleurl.com/common |\n", + "| {\"channel\" : \"chat\"} | 2 | 1 | How many legs does an octopus have? | It has 8 limbs. | exampleurl.com/octopus |\n", "\n", "---\n", "\n", @@ -2192,10 +2199,12 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "qwhOvOSOmnJ4" + "id": "qwhOvOSOmnJ4", + "cellView": "form" }, "outputs": [], "source": [ + "# @markdown `run this cell to load the queryset manually from this cell`\n", "sample_df = pd.DataFrame(columns=INPUT_SCHEMA_REQUIRED_COLUMNS)\n", "\n", "sample_df.loc[0] = [\"0\", 1 ,\"Who are you?\", \"I am an assistant\", \"www.google.com\"]\n", @@ -2508,16 +2517,7 @@ "metadata": { "colab": { "private_outputs": true, - "provenance": [ - { - "file_id": "17WDmf3DsZGg1ZGwnr40sMXMQtyfxb4ms", - "timestamp": 1713942777165 - }, - { - "file_id": "1b769OFNM8gH56ZzvWUfw4MpGUc-pSF-g", - "timestamp": 1708939513950 - } - ], + "provenance": [], "toc_visible": true }, "kernelspec": { From 57524985759c63d6b8b1e00ce4247c1494e982b2 Mon Sep 17 00:00:00 2001 From: Naveen Kumar M Date: Fri, 19 Jul 2024 15:05:32 +0530 Subject: [PATCH 2/7] Update evaluation_tool__autoeval__colab.ipynb --- .../evaluation_tool__autoeval__colab.ipynb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb b/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb index 7dbbedf7..bba02e91 100644 --- a/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb +++ b/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb @@ -2282,12 +2282,12 @@ "\n", "An example for the queryset can be seen in this table:\n", "\n", - "| parameters | conversation_id | turn_index | query | expected_answer | expected_uri |\n", + "| parameters | conversation_id | turn_index | query | expected_answer | expected_uri | user_metadata \n", "| --- | --- | --- | --- | --- | --- |\n", - "| {\"channel\" : \"chat\"} | 0 | 1 | What is the capital of France? | Capital of France is Paris. | exampleurl.com/france |\n", - "| {\"channel\" : \"chat\"} | 0 | 2 | How many people live there? | 2.1 million people live in Paris. | exampleurl.com/paris |\n", - "| {\"channel\" : \"chat\"} | 1 | 1 | What is the color of the sky? | It is blue. | exampleurl.com/common |\n", - "| {\"channel\" : \"chat\"} | 2 | 1 | How many legs does an octopus have? | It has 8 limbs. | exampleurl.com/octopus |\n", + "| {\"channel\" : \"chat\"} | 0 | 1 | What is the capital of France? | Capital of France is Paris. | exampleurl.com/france | None |\n", + "| {\"channel\" : \"chat\"} | 0 | 2 | How many people live there? | 2.1 million people live in Paris. | exampleurl.com/paris | None |\n", + "| {\"channel\" : \"chat\"} | 1 | 1 | What is the color of the sky? | It is blue. | exampleurl.com/common | None |\n", + "| {\"channel\" : \"chat\"} | 2 | 1 | How many legs does an octopus have? | It has 8 limbs. | exampleurl.com/octopus | None |\n", "\n", "---\n", "\n", From 9afc1c48410ab5c9b85db90def44cd2f9998c92c Mon Sep 17 00:00:00 2001 From: Naveen Kumar M Date: Fri, 19 Jul 2024 15:06:06 +0530 Subject: [PATCH 3/7] Update evaluation_tool__autoeval__colab.ipynb --- .../evaluation_tool__autoeval__colab.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb b/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb index bba02e91..af18aca1 100644 --- a/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb +++ b/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb @@ -2283,7 +2283,7 @@ "An example for the queryset can be seen in this table:\n", "\n", "| parameters | conversation_id | turn_index | query | expected_answer | expected_uri | user_metadata \n", - "| --- | --- | --- | --- | --- | --- |\n", + "| --- | --- | --- | --- | --- | --- | --- |\n", "| {\"channel\" : \"chat\"} | 0 | 1 | What is the capital of France? | Capital of France is Paris. | exampleurl.com/france | None |\n", "| {\"channel\" : \"chat\"} | 0 | 2 | How many people live there? | 2.1 million people live in Paris. | exampleurl.com/paris | None |\n", "| {\"channel\" : \"chat\"} | 1 | 1 | What is the color of the sky? | It is blue. | exampleurl.com/common | None |\n", From 858ce7738dd0aa70d1d66eb1c47688d6062f2946 Mon Sep 17 00:00:00 2001 From: Naveen Kumar M Date: Fri, 19 Jul 2024 15:07:07 +0530 Subject: [PATCH 4/7] Update evaluation_tool__autoeval__colab.ipynb --- .../evaluation_tool__autoeval__colab.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb b/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb index af18aca1..a9f0197c 100644 --- a/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb +++ b/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb @@ -2282,7 +2282,7 @@ "\n", "An example for the queryset can be seen in this table:\n", "\n", - "| parameters | conversation_id | turn_index | query | expected_answer | expected_uri | user_metadata \n", + "| parameters | conversation_id | turn_index | query | expected_answer | expected_uri | user_metadata |\n", "| --- | --- | --- | --- | --- | --- | --- |\n", "| {\"channel\" : \"chat\"} | 0 | 1 | What is the capital of France? | Capital of France is Paris. | exampleurl.com/france | None |\n", "| {\"channel\" : \"chat\"} | 0 | 2 | How many people live there? | 2.1 million people live in Paris. | exampleurl.com/paris | None |\n", From c555f915650e2ae99b6703749191a64a341b74f2 Mon Sep 17 00:00:00 2001 From: Naveen Kumar M Date: Fri, 19 Jul 2024 15:12:03 +0530 Subject: [PATCH 5/7] Update evaluation_tool__autoeval__colab.ipynb --- .../evaluation_tool__autoeval__colab.ipynb | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb b/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb index a9f0197c..503c0d78 100644 --- a/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb +++ b/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb @@ -2275,6 +2275,7 @@ "- `query` _(the input question)_\n", "- `expected_answer` _(the ideal or ground truth answer)_\n", "- `expected_uri` _(the webpage url or more generally the uri that contains the answer to `query`)_.\n", + "- `user_metadata` _(Dict of CX Session endUserMetadata to set in the conversation.)_.\n", "\n", "In addition to the required columns the RougeL metric can also use the following optional column:\n", "\n", From a542477e1c737aa57a54cce9f6d7cedddb195911 Mon Sep 17 00:00:00 2001 From: Naveen Kumar M Date: Fri, 19 Jul 2024 15:27:53 +0530 Subject: [PATCH 6/7] Update evaluation_tool__autoeval__colab.ipynb --- .../evaluation_tool__autoeval__colab.ipynb | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb b/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb index 503c0d78..bdf0ba96 100644 --- a/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb +++ b/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb @@ -2315,6 +2315,7 @@ "outputs": [], "source": [ "# @markdown `run this cell to load data manually`\n", + "\n", "sample_df = pd.DataFrame(columns=INPUT_SCHEMA_REQUIRED_COLUMNS)\n", "\n", "sample_df.loc[0] = [\"0\", 1 ,\"Who are you?\", \"I am an assistant\", \"www.google.com\", None]\n", From 1c3899e8f848f6494e58ca4520fec503a11e9293 Mon Sep 17 00:00:00 2001 From: Naveen Kumar M Date: Fri, 19 Jul 2024 15:48:31 +0530 Subject: [PATCH 7/7] Update evaluation_tool__autoeval__colab.ipynb --- .../evaluation_tool__autoeval__colab.ipynb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb b/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb index bdf0ba96..5dd7921d 100644 --- a/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb +++ b/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb @@ -2318,9 +2318,9 @@ "\n", "sample_df = pd.DataFrame(columns=INPUT_SCHEMA_REQUIRED_COLUMNS)\n", "\n", - "sample_df.loc[0] = [\"0\", 1 ,\"Who are you?\", \"I am an assistant\", \"www.google.com\", None]\n", - "sample_df.loc[1] = [\"1\", 1 ,\"Which is the cheapest plan?\", \"Basic plan\", \"www.google.com\", None]\n", - "sample_df.loc[2] = [\"1\", 2, \"How much?\", \"The Basic plan costs 20$/month\", \"www.google.com\", None]\n", + "sample_df.loc[0] = ['{\"channel\" : \"chat\"}', \"0\", 1 ,\"Who are you?\", \"I am an assistant\", \"www.google.com\", None]\n", + "sample_df.loc[1] = ['{\"channel\" : \"chat\"}', \"1\", 1 ,\"Which is the cheapest plan?\", \"Basic plan\", \"www.google.com\", None]\n", + "sample_df.loc[2] = ['{\"channel\" : \"chat\"}', \"1\", 2, \"How much?\", \"The Basic plan costs 20$/month\", \"www.google.com\", None]\n", "queryset = sample_df" ] },