From 60cadc96070e97df67c9611b23ab08bbb1b3e453 Mon Sep 17 00:00:00 2001 From: Ankit Singhal Date: Thu, 15 Feb 2024 10:39:18 -0800 Subject: [PATCH 1/4] Adding tests to capture groundedness with expected values --- .../evaluate/e2etests/test_evaluate_e2e.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/sdk/ai/azure-ai-generative/tests/evaluate/e2etests/test_evaluate_e2e.py b/sdk/ai/azure-ai-generative/tests/evaluate/e2etests/test_evaluate_e2e.py index 7a8c06ee5771..46eeef058b53 100644 --- a/sdk/ai/azure-ai-generative/tests/evaluate/e2etests/test_evaluate_e2e.py +++ b/sdk/ai/azure-ai-generative/tests/evaluate/e2etests/test_evaluate_e2e.py @@ -22,10 +22,14 @@ class TestEvaluate(AzureRecordedTestCase): def test_evaluate_built_in_metrics(self, e2e_openai_api_base, e2e_openai_api_key, e2e_openai_completion_deployment_name, tmpdir): test_data = [ - {"question": "How do you create a run?", "context": "AML API only", - "answer": "To create a run using the Azure Machine Learning API, you first need to create an Experiment. Once you have an experiment, you can create a Run object that is associated with that experiment. Here is some Python code that demonstrates this process:\n\n```\nfrom azureml.core import Experiment, Run\nfrom azureml.core.workspace import Workspace\n\n# Define workspace and experiment\nws = Workspace.from_config()\nexp = Experiment(workspace=ws, name='my_experiment')\n\n# Create a new run\nrun = exp.start_logging()\n```\n\nIn this code, the `from_config()` method reads the configuration file that you created when you set up your Azure Machine Learning workspace. The `Experiment` constructor creates an Experiment object that is associated with your workspace, and the `start_logging()` method creates a new Run object that is associated with the Experiment. Now you can use the `run` object to log metrics, upload files, and track other information related to your machine learning experiment."}, - {"question": "How do you log a model?", "context": "Logging can be done using any OSS Sdk", - "answer": "There are a few ways to log models in Azure Machine Learning. \n\nOne way is to use the `register_model()` method of the `Run` object. The `register_model()` method logs a model file in the Azure Machine Learning service workspace and makes it available for deployment. Here's an example:\n\n```python\nfrom azureml.core import Model\n\nmodel_path = '.\/outputs\/my_model.pkl'\nmodel = Model.register(workspace=ws, model_path=model_path, model_name='my_model')\n```\n\nThis code registers the model file located at `model_path` to the Azure Machine Learning service workspace with the name `my_model`. \n\nAnother way to log a model is to save it as an output of a `Run`. If your model generation code is part of a script or Jupyter notebook that runs as an Azure Machine Learning experiment, you can save the model file as an output of the `Run` object. Here's an example:\n\n```python\nfrom sklearn.linear_model import LogisticRegression\nfrom azureml.core.run import Run\n\n# Initialize a run object\nrun = Run.get_context()\n\n# Train your model\nX_train, y_train = ...\nclf = LogisticRegression().fit(X_train, y_train)\n\n# Save the model to the Run object's outputs directory\nmodel_path = 'outputs\/model.pkl'\njoblib.dump(value=clf, filename=model_path)\n\n# Log the model as a run artifact\nrun.upload_file(name=model_path, path_or_stream=model_path)\n```\n\nIn this code, `Run.get_context()` retrieves the current run context object, which you can use to track metadata and metrics for the run. After training your model, you can use `joblib.dump()` to save the model to a file, and then log the file as an artifact of the run using `run.upload_file()`."}, + {"context": "Some are reported as not having been wanted at all.", + "question": "", + "answer": "All are reported as being completely and fully wanted." + }, + {"question": "How do you log a model?", + "context": "There are a few ways to log models in Azure Machine Learning. \n\nOne way is to use the `register_model()` method of the `Run` object. The `register_model()` method logs a model file in the Azure Machine Learning service workspace and makes it available for deployment. Here's an example:\n\n```python\nfrom azureml.core import Model\n\nmodel_path = '.\/outputs\/my_model.pkl'\nmodel = Model.register(workspace=ws, model_path=model_path, model_name='my_model')\n```\n\nThis code registers the model file located at `model_path` to the Azure Machine Learning service workspace with the name `my_model`. \n\nAnother way to log a model is to save it as an output of a `Run`. If your model generation code is part of a script or Jupyter notebook that runs as an Azure Machine Learning experiment, you can save the model file as an output of the `Run` object. Here's an example:\n\n```python\nfrom sklearn.linear_model import LogisticRegression\nfrom azureml.core.run import Run\n\n# Initialize a run object\nrun = Run.get_context()\n\n# Train your model\nX_train, y_train = ...\nclf = LogisticRegression().fit(X_train, y_train)\n\n# Save the model to the Run object's outputs directory\nmodel_path = 'outputs\/model.pkl'\njoblib.dump(value=clf, filename=model_path)\n\n# Log the model as a run artifact\nrun.upload_file(name=model_path, path_or_stream=model_path)\n```\n\nIn this code, `Run.get_context()` retrieves the current run context object, which you can use to track metadata and metrics for the run. After training your model, you can use `joblib.dump()` to save the model to a file, and then log the file as an artifact of the run using `run.upload_file()`.", + "answer": "There are a few ways to log models in Azure Machine Learning. \n\nOne way is to use the `register_model()` method of the `Run` object. The `register_model()` method logs a model file in the Azure Machine Learning service workspace and makes it available for deployment. Here's an example:\n\n```python\nfrom azureml.core import Model\n\nmodel_path = '.\/outputs\/my_model.pkl'\nmodel = Model.register(workspace=ws, model_path=model_path, model_name='my_model')\n```\n\nThis code registers the model file located at `model_path` to the Azure Machine Learning service workspace with the name `my_model`. \n\nAnother way to log a model is to save it as an output of a `Run`. If your model generation code is part of a script or Jupyter notebook that runs as an Azure Machine Learning experiment, you can save the model file as an output of the `Run` object. Here's an example:\n\n```python\nfrom sklearn.linear_model import LogisticRegression\nfrom azureml.core.run import Run\n\n# Initialize a run object\nrun = Run.get_context()\n\n# Train your model\nX_train, y_train = ...\nclf = LogisticRegression().fit(X_train, y_train)\n\n# Save the model to the Run object's outputs directory\nmodel_path = 'outputs\/model.pkl'\njoblib.dump(value=clf, filename=model_path)\n\n# Log the model as a run artifact\nrun.upload_file(name=model_path, path_or_stream=model_path)\n```\n\nIn this code, `Run.get_context()` retrieves the current run context object, which you can use to track metadata and metrics for the run. After training your model, you can use `joblib.dump()` to save the model to a file, and then log the file as an artifact of the run using `run.upload_file()`." + }, ] with tmpdir.as_cwd(): @@ -44,8 +48,8 @@ def test_evaluate_built_in_metrics(self, e2e_openai_api_base, e2e_openai_api_key "deployment_id": e2e_openai_completion_deployment_name, }, data_mapping={ - "questions": "question", - "contexts": "context", + "question": "question", + "context": "context", "y_pred": "answer", "y_test": "truth", }, @@ -57,6 +61,8 @@ def test_evaluate_built_in_metrics(self, e2e_openai_api_base, e2e_openai_api_key assert "gpt_groundedness" in metrics_summary.keys() assert metrics_summary.get("gpt_groundedness") == np.nanmean(tabular_result["gpt_groundedness"]) + assert tabular_result["gpt_groundedness"][0] in [1,2] + assert tabular_result["gpt_groundedness"][1] in [5, 4] def test_duplicate_metrics_name(self, e2e_openai_api_base, e2e_openai_api_key, e2e_openai_completion_deployment_name, tmpdir): From 6f4a111e6c3a109d6df9218a0927630367c2f0ad Mon Sep 17 00:00:00 2001 From: Ankit Singhal <30610298+singankit@users.noreply.github.com> Date: Wed, 27 Mar 2024 17:05:33 -0700 Subject: [PATCH 2/4] Update CHANGELOG.md --- sdk/ai/azure-ai-generative/CHANGELOG.md | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/sdk/ai/azure-ai-generative/CHANGELOG.md b/sdk/ai/azure-ai-generative/CHANGELOG.md index 4c6e0fa3afef..14ebe626f9dc 100644 --- a/sdk/ai/azure-ai-generative/CHANGELOG.md +++ b/sdk/ai/azure-ai-generative/CHANGELOG.md @@ -1,14 +1,9 @@ # Release History -## 1.0.0b8 (Unreleased) - -### Features Added - -### Breaking Changes - -### Bugs Fixed +## 1.0.0b8 (2024-03-27) ### Other Changes +- Bug fixes ## 1.0.0b7 (2024-02-07) From 25e0185573473a44e6fa4aaa42db91dfad26e150 Mon Sep 17 00:00:00 2001 From: Ankit Singhal <30610298+singankit@users.noreply.github.com> Date: Wed, 27 Mar 2024 17:06:13 -0700 Subject: [PATCH 3/4] Update CHANGELOG.md --- sdk/ai/azure-ai-resources/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/ai/azure-ai-resources/CHANGELOG.md b/sdk/ai/azure-ai-resources/CHANGELOG.md index 40bd4dcdb77c..dfd86c501e77 100644 --- a/sdk/ai/azure-ai-resources/CHANGELOG.md +++ b/sdk/ai/azure-ai-resources/CHANGELOG.md @@ -1,6 +1,6 @@ # Release History -## 1.0.0b8 (Unreleased) +## 1.0.0b8 (2024-03-27) ### Features Added - Connections LIST operation now supports returning data connections via new optional flag: include_data_connections. From a8ac8f68e175161787b14abf09e8cdb005d6e86e Mon Sep 17 00:00:00 2001 From: Ankit Singhal <30610298+singankit@users.noreply.github.com> Date: Wed, 27 Mar 2024 18:16:11 -0700 Subject: [PATCH 4/4] Update CHANGELOG.md --- sdk/ai/azure-ai-resources/CHANGELOG.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sdk/ai/azure-ai-resources/CHANGELOG.md b/sdk/ai/azure-ai-resources/CHANGELOG.md index dfd86c501e77..0e570624a5d9 100644 --- a/sdk/ai/azure-ai-resources/CHANGELOG.md +++ b/sdk/ai/azure-ai-resources/CHANGELOG.md @@ -6,12 +6,11 @@ - Connections LIST operation now supports returning data connections via new optional flag: include_data_connections. - Connections have read-only support for 3 new connection types: gen 2, data lake, and azure blob. -### Breaking Changes - ### Bugs Fixed - Connections docstrings now discuss scope field. ### Other Changes +- Bug fixes ## 1.0.0b7 (2024-02-07)