diff --git a/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb b/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb index 3dffd5e5..5dd7921d 100644 --- a/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb +++ b/examples/vertex_ai_conversation/evaluation_tool__autoeval__colab.ipynb @@ -272,6 +272,7 @@ "\n", "GLOBAL_SCOPE = [\"https://spreadsheets.google.com/feeds\"]\n", "\n", + "PARAMETERS = \"parameters\"\n", "CONVERSATION_ID = \"conversation_id\"\n", "TURN_INDEX = \"turn_index\"\n", "QUERY = \"query\"\n", @@ -285,7 +286,7 @@ "AGENT_URI = \"projects/{project_id}/locations/{location}/agents/{agent_id}\"\n", "\n", "INPUT_SCHEMA_REQUIRED_COLUMNS = [\n", - " CONVERSATION_ID, TURN_INDEX, QUERY, REFERENCE, EXPECTED_URI, USER_METADATA\n", + " PARAMETERS,CONVERSATION_ID, TURN_INDEX, QUERY, REFERENCE, EXPECTED_URI, USER_METADATA\n", "]\n", "\n", "_EXECUTION_SEQUENCE_KEY = \"DataStore Execution Sequence\"\n", @@ -783,6 +784,7 @@ " query_param_mapping = {}\n", "\n", " if parameters:\n", + " parameters = json.loads(parameters)\n", " query_param_mapping[\"parameters\"] = parameters\n", "\n", " if end_user_metadata:\n", @@ -807,6 +809,7 @@ " self,\n", " query: str,\n", " session_id: str | None = None,\n", + " parameters: str | None = None,\n", " user_metadata: str | None = None,\n", " ) -> VertexConversationResponse:\n", " if session_id is None:\n", @@ -825,6 +828,7 @@ " language_code=self.language_code,\n", " end_user_metadata=user_metadata,\n", " populate_data_store_connection_signals=True,\n", + " parameters=parameters,\n", " )\n", " return VertexConversationResponse.from_query_result(response._pb)\n", "\n", @@ -2251,6 +2255,7 @@ "outputs": [], "source": [ "# test agent on a single query\n", + "parameter_str = '{\"channel\" : \"chat\"}'\n", "response = scraper.scrape_detect_intent(query=\"who is the ceo?\")\n", "print(json.dumps(dataclasses.asdict(response), indent=4))" ] @@ -2264,11 +2269,13 @@ "## Data Loading\n", "\n", "The queryset must be in a tabular format that has to contain the following columns:\n", + "- `parameters` _(parameter which needs to be passed as part of detect intent for agent to route to correct flow/page)_\n", "- `conversation_id` _(unique identifier of a conversation, which must be the same for each row that are part of the same conversation)_\n", "- `turn_index` _(index of the query - expected answer pair within a conversation)_\n", "- `query` _(the input question)_\n", "- `expected_answer` _(the ideal or ground truth answer)_\n", "- `expected_uri` _(the webpage url or more generally the uri that contains the answer to `query`)_.\n", + "- `user_metadata` _(Dict of CX Session endUserMetadata to set in the conversation.)_.\n", "\n", "In addition to the required columns the RougeL metric can also use the following optional column:\n", "\n", @@ -2276,12 +2283,12 @@ "\n", "An example for the queryset can be seen in this table:\n", "\n", - "| conversation_id | turn_index | query | expected_answer | expected_uri |\n", - "| --- | --- | --- | --- | --- |\n", - "| 0 | 1 | What is the capital of France? | Capital of France is Paris. | exampleurl.com/france |\n", - "| 0 | 2 | How many people live there? | 2.1 million people live in Paris. | exampleurl.com/paris |\n", - "| 1 | 1 | What is the color of the sky? | It is blue. | exampleurl.com/common |\n", - "| 2 | 1 | How many legs does an octopus have? | It has 8 limbs. | exampleurl.com/octopus |\n", + "| parameters | conversation_id | turn_index | query | expected_answer | expected_uri | user_metadata |\n", + "| --- | --- | --- | --- | --- | --- | --- |\n", + "| {\"channel\" : \"chat\"} | 0 | 1 | What is the capital of France? | Capital of France is Paris. | exampleurl.com/france | None |\n", + "| {\"channel\" : \"chat\"} | 0 | 2 | How many people live there? | 2.1 million people live in Paris. | exampleurl.com/paris | None |\n", + "| {\"channel\" : \"chat\"} | 1 | 1 | What is the color of the sky? | It is blue. | exampleurl.com/common | None |\n", + "| {\"channel\" : \"chat\"} | 2 | 1 | How many legs does an octopus have? | It has 8 limbs. | exampleurl.com/octopus | None |\n", "\n", "---\n", "\n", @@ -2311,9 +2318,9 @@ "\n", "sample_df = pd.DataFrame(columns=INPUT_SCHEMA_REQUIRED_COLUMNS)\n", "\n", - "sample_df.loc[0] = [\"0\", 1 ,\"Who are you?\", \"I am an assistant\", \"www.google.com\", None]\n", - "sample_df.loc[1] = [\"1\", 1 ,\"Which is the cheapest plan?\", \"Basic plan\", \"www.google.com\", None]\n", - "sample_df.loc[2] = [\"1\", 2, \"How much?\", \"The Basic plan costs 20$/month\", \"www.google.com\", None]\n", + "sample_df.loc[0] = ['{\"channel\" : \"chat\"}', \"0\", 1 ,\"Who are you?\", \"I am an assistant\", \"www.google.com\", None]\n", + "sample_df.loc[1] = ['{\"channel\" : \"chat\"}', \"1\", 1 ,\"Which is the cheapest plan?\", \"Basic plan\", \"www.google.com\", None]\n", + "sample_df.loc[2] = ['{\"channel\" : \"chat\"}', \"1\", 2, \"How much?\", \"The Basic plan costs 20$/month\", \"www.google.com\", None]\n", "queryset = sample_df" ] },