From 409165c1b88234fd02560b83cc162c31819b2ff1 Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Mon, 6 Nov 2023 13:04:31 -0800 Subject: [PATCH 1/5] Pin openai below 1.0 --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 10e5bbd511..abcdea1987 100644 --- a/tox.ini +++ b/tox.ini @@ -344,7 +344,7 @@ deps = framework_tornado: pycurl framework_tornado-tornadolatest: tornado framework_tornado-tornadomaster: https://github.com/tornadoweb/tornado/archive/master.zip - mlmodel_openai: openai[datalib] + mlmodel_openai: openai[datalib]<1.0 mlmodel_openai: protobuf logger_loguru-logurulatest: loguru logger_loguru-loguru06: loguru<0.7 @@ -497,4 +497,4 @@ source = directory = ${TOX_ENV_DIR-.}/htmlcov [coverage:xml] -output = ${TOX_ENV_DIR-.}/coverage.xml \ No newline at end of file +output = ${TOX_ENV_DIR-.}/coverage.xml From d5ad6d065c11bf87d1431e6a06a7742a25cf1046 Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Mon, 6 Nov 2023 13:07:15 -0800 Subject: [PATCH 2/5] Fixup --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index abcdea1987..eb6b801f4d 100644 --- a/tox.ini +++ b/tox.ini @@ -497,4 +497,4 @@ source = directory = ${TOX_ENV_DIR-.}/htmlcov [coverage:xml] -output = ${TOX_ENV_DIR-.}/coverage.xml +output = ${TOX_ENV_DIR-.}/coverage.xml \ No newline at end of file From 167d4fdd4b881cfebdaeb6013f4cb0d3c7a8467b Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Mon, 6 Nov 2023 11:25:04 -0800 Subject: [PATCH 3/5] Add ingest_source to events --- newrelic/hooks/mlmodel_openai.py | 5 +++++ tests/mlmodel_openai/test_chat_completion.py | 8 ++++++++ tests/mlmodel_openai/test_embeddings.py | 1 + 3 files changed, 14 insertions(+) diff --git a/newrelic/hooks/mlmodel_openai.py b/newrelic/hooks/mlmodel_openai.py index 336b35ecc9..5e764e0545 100644 --- a/newrelic/hooks/mlmodel_openai.py +++ b/newrelic/hooks/mlmodel_openai.py @@ -119,6 +119,7 @@ def wrap_embedding_create(wrapped, instance, args, kwargs): response_headers, "x-ratelimit-remaining-requests", True ), "vendor": "openAI", + "ingest_source": "Python", } transaction.record_ml_event("LlmEmbedding", embedding_dict) @@ -208,6 +209,7 @@ def wrap_chat_completion_create(wrapped, instance, args, kwargs): response_headers, "x-ratelimit-remaining-requests", True ), "vendor": "openAI", + "ingest_source": "Python", "response.number_of_messages": len(messages) + len(choices), } @@ -275,6 +277,7 @@ def create_chat_completion_message_event( "sequence": index, "response.model": response_model, "vendor": "openAI", + "ingest_source": "Python", } transaction.record_ml_event("LlmChatCompletionMessage", chat_completion_message_dict) @@ -353,6 +356,7 @@ async def wrap_embedding_acreate(wrapped, instance, args, kwargs): response_headers, "x-ratelimit-remaining-requests", True ), "vendor": "openAI", + "ingest_source": "Python", } transaction.record_ml_event("LlmEmbedding", embedding_dict) @@ -449,6 +453,7 @@ async def wrap_chat_completion_acreate(wrapped, instance, args, kwargs): response_headers, "x-ratelimit-remaining-requests", True ), "vendor": "openAI", + "ingest_source": "Python", } transaction.record_ml_event("LlmChatCompletionSummary", chat_completion_summary_dict) diff --git a/tests/mlmodel_openai/test_chat_completion.py b/tests/mlmodel_openai/test_chat_completion.py index fa320beea7..6f3762a826 100644 --- a/tests/mlmodel_openai/test_chat_completion.py +++ b/tests/mlmodel_openai/test_chat_completion.py @@ -64,6 +64,7 @@ "response.headers.ratelimitRemainingTokens": 39940, "response.headers.ratelimitRemainingRequests": 199, "vendor": "openAI", + "ingest_source": "Python", "response.number_of_messages": 3, }, ), @@ -83,6 +84,7 @@ "sequence": 0, "response.model": "gpt-3.5-turbo-0613", "vendor": "openAI", + "ingest_source": "Python", }, ), ( @@ -101,6 +103,7 @@ "sequence": 1, "response.model": "gpt-3.5-turbo-0613", "vendor": "openAI", + "ingest_source": "Python", }, ), ( @@ -119,6 +122,7 @@ "sequence": 2, "response.model": "gpt-3.5-turbo-0613", "vendor": "openAI", + "ingest_source": "Python", }, ), ] @@ -175,6 +179,7 @@ def test_openai_chat_completion_sync_in_txn_with_convo_id(set_trace_info): "response.headers.ratelimitRemainingTokens": 39940, "response.headers.ratelimitRemainingRequests": 199, "vendor": "openAI", + "ingest_source": "Python", "response.number_of_messages": 3, }, ), @@ -194,6 +199,7 @@ def test_openai_chat_completion_sync_in_txn_with_convo_id(set_trace_info): "sequence": 0, "response.model": "gpt-3.5-turbo-0613", "vendor": "openAI", + "ingest_source": "Python", }, ), ( @@ -212,6 +218,7 @@ def test_openai_chat_completion_sync_in_txn_with_convo_id(set_trace_info): "sequence": 1, "response.model": "gpt-3.5-turbo-0613", "vendor": "openAI", + "ingest_source": "Python", }, ), ( @@ -230,6 +237,7 @@ def test_openai_chat_completion_sync_in_txn_with_convo_id(set_trace_info): "sequence": 2, "response.model": "gpt-3.5-turbo-0613", "vendor": "openAI", + "ingest_source": "Python", }, ), ] diff --git a/tests/mlmodel_openai/test_embeddings.py b/tests/mlmodel_openai/test_embeddings.py index 256150a5c2..180052b0de 100644 --- a/tests/mlmodel_openai/test_embeddings.py +++ b/tests/mlmodel_openai/test_embeddings.py @@ -55,6 +55,7 @@ "response.headers.ratelimitRemainingTokens": 149994, "response.headers.ratelimitRemainingRequests": 197, "vendor": "openAI", + "ingest_source": "Python", }, ), ] From 9037d3f1929d270431adebc3350ff59a9ea3c343 Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Mon, 6 Nov 2023 11:25:38 -0800 Subject: [PATCH 4/5] Remove duplicate test file --- tests/mlmodel_openai/test_error.py | 58 ------------------------------ 1 file changed, 58 deletions(-) delete mode 100644 tests/mlmodel_openai/test_error.py diff --git a/tests/mlmodel_openai/test_error.py b/tests/mlmodel_openai/test_error.py deleted file mode 100644 index e8f5c31518..0000000000 --- a/tests/mlmodel_openai/test_error.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2010 New Relic, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import openai -import pytest - -from newrelic.api.background_task import background_task - -enabled_ml_settings = { - "machine_learning.enabled": True, - "machine_learning.inference_events_value.enabled": True, - "ml_insights_events.enabled": True, -} - -_test_openai_chat_completion_messages = ( - {"role": "system", "content": "You are a scientist."}, - {"role": "user", "content": "What is 212 degrees Fahrenheit converted to Celsius?"}, -) - - -@background_task() -def test_invalid_request_error_model_does_not_exist(): - with pytest.raises(openai.InvalidRequestError): - openai.ChatCompletion.create( - model="does-not-exist", - messages=( - {"role": "system", "content": "Model does not exist."}, - {"role": "user", "content": "What is 212 degrees Fahrenheit converted to Celsius?"}, - ), - temperature=0.7, - max_tokens=100, - ) - - -@background_task() -def test_authentication_error_invalid_api_key(monkeypatch): - monkeypatch.setattr(openai, "api_key", "InvalidKey") - with pytest.raises(openai.error.AuthenticationError): - openai.ChatCompletion.create( - model="gpt-3.5-turbo", - messages=( - {"role": "system", "content": "Invalid API key."}, - {"role": "user", "content": "What is 212 degrees Fahrenheit converted to Celsius?"}, - ), - temperature=0.7, - max_tokens=100, - ) From ec0db86608b70edd11d643013597208d1de2c7dc Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Mon, 6 Nov 2023 13:56:25 -0800 Subject: [PATCH 5/5] Handle 0.32.0.post1 version in tests (#963) --- tests/framework_starlette/test_application.py | 30 +++++++++++-------- tests/framework_starlette/test_bg_tasks.py | 5 ++-- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/tests/framework_starlette/test_application.py b/tests/framework_starlette/test_application.py index 7d36d66ccc..bd89bb9a9c 100644 --- a/tests/framework_starlette/test_application.py +++ b/tests/framework_starlette/test_application.py @@ -17,13 +17,21 @@ import pytest import starlette from testing_support.fixtures import override_ignore_status_codes +from testing_support.validators.validate_code_level_metrics import ( + validate_code_level_metrics, +) +from testing_support.validators.validate_transaction_errors import ( + validate_transaction_errors, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) from newrelic.common.object_names import callable_name -from testing_support.validators.validate_code_level_metrics import validate_code_level_metrics -from testing_support.validators.validate_transaction_errors import validate_transaction_errors -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics +from newrelic.common.package_version_utils import get_package_version_tuple + +starlette_version = get_package_version_tuple("starlette")[:3] -starlette_version = tuple(int(x) for x in starlette.__version__.split(".")) @pytest.fixture(scope="session") def target_application(): @@ -78,6 +86,7 @@ def test_application_non_async(target_application, app_name): response = app.get("/non_async") assert response.status == 200 + # Starting in Starlette v0.20.1, the ExceptionMiddleware class # has been moved to the starlette.middleware.exceptions from # starlette.exceptions @@ -96,8 +105,10 @@ def test_application_non_async(target_application, app_name): ), ) + @pytest.mark.parametrize( - "app_name, transaction_name", middleware_test, + "app_name, transaction_name", + middleware_test, ) def test_application_nonexistent_route(target_application, app_name, transaction_name): @validate_transaction_metrics( @@ -117,10 +128,6 @@ def _test(): def test_exception_in_middleware(target_application, app_name): app = target_application[app_name] - from starlette import __version__ as version - - starlette_version = tuple(int(v) for v in version.split(".")) - # Starlette >=0.15 and <0.17 raises an exception group instead of reraising the ValueError # This only occurs on Python versions >=3.8 if sys.version_info[0:2] > (3, 7) and starlette_version >= (0, 15, 0) and starlette_version < (0, 17, 0): @@ -272,9 +279,8 @@ def _test(): ), ) -@pytest.mark.parametrize( - "app_name,scoped_metrics", middleware_test_exception -) + +@pytest.mark.parametrize("app_name,scoped_metrics", middleware_test_exception) def test_starlette_http_exception(target_application, app_name, scoped_metrics): @validate_transaction_errors(errors=["starlette.exceptions:HTTPException"]) @validate_transaction_metrics( diff --git a/tests/framework_starlette/test_bg_tasks.py b/tests/framework_starlette/test_bg_tasks.py index 5e30fe32e5..9ad8fe61be 100644 --- a/tests/framework_starlette/test_bg_tasks.py +++ b/tests/framework_starlette/test_bg_tasks.py @@ -15,7 +15,6 @@ import sys import pytest -from starlette import __version__ from testing_support.validators.validate_transaction_count import ( validate_transaction_count, ) @@ -23,7 +22,9 @@ validate_transaction_metrics, ) -starlette_version = tuple(int(x) for x in __version__.split(".")) +from newrelic.common.package_version_utils import get_package_version_tuple + +starlette_version = get_package_version_tuple("starlette")[:3] try: from starlette.middleware import Middleware # noqa: F401