diff --git a/newrelic/api/ml_model.py b/newrelic/api/ml_model.py index d01042b359..1951f91312 100644 --- a/newrelic/api/ml_model.py +++ b/newrelic/api/ml_model.py @@ -81,4 +81,4 @@ def record_llm_feedback_event( } feedback_message_event.update(metadata) - transaction.record_ml_event("LlmFeedbackMessage", feedback_message_event) + transaction.record_custom_event("LlmFeedbackMessage", feedback_message_event) diff --git a/newrelic/hooks/external_botocore.py b/newrelic/hooks/external_botocore.py index c075f0874d..8fd9c05e09 100644 --- a/newrelic/hooks/external_botocore.py +++ b/newrelic/hooks/external_botocore.py @@ -119,7 +119,7 @@ def create_chat_completion_message_event( "vendor": "bedrock", "ingest_source": "Python", } - transaction.record_ml_event("LlmChatCompletionMessage", chat_completion_message_dict) + transaction.record_custom_event("LlmChatCompletionMessage", chat_completion_message_dict) def extract_bedrock_titan_text_model(request_body, response_body=None): @@ -376,7 +376,7 @@ def handle_embedding_event( } ) - transaction.record_ml_event("LlmEmbedding", embedding_dict) + transaction.record_custom_event("LlmEmbedding", embedding_dict) def handle_chat_completion_event( @@ -413,7 +413,7 @@ def handle_chat_completion_event( } ) - transaction.record_ml_event("LlmChatCompletionSummary", chat_completion_summary_dict) + transaction.record_custom_event("LlmChatCompletionSummary", chat_completion_summary_dict) create_chat_completion_message_event( transaction=transaction, diff --git a/newrelic/hooks/mlmodel_openai.py b/newrelic/hooks/mlmodel_openai.py index a51d8aae87..a53f2fc623 100644 --- a/newrelic/hooks/mlmodel_openai.py +++ b/newrelic/hooks/mlmodel_openai.py @@ -122,7 +122,8 @@ def wrap_embedding_create(wrapped, instance, args, kwargs): "ingest_source": "Python", } - transaction.record_ml_event("LlmEmbedding", embedding_dict) + transaction.record_custom_event("LlmEmbedding", embedding_dict) + return response @@ -213,7 +214,8 @@ def wrap_chat_completion_create(wrapped, instance, args, kwargs): "response.number_of_messages": len(messages) + len(choices), } - transaction.record_ml_event("LlmChatCompletionSummary", chat_completion_summary_dict) + transaction.record_custom_event("LlmChatCompletionSummary", chat_completion_summary_dict) + message_list = list(messages) if choices: message_list.extend([choices[0].message]) @@ -287,7 +289,9 @@ def create_chat_completion_message_event( "vendor": "openAI", "ingest_source": "Python", } - transaction.record_ml_event("LlmChatCompletionMessage", chat_completion_message_dict) + + transaction.record_custom_event("LlmChatCompletionMessage", chat_completion_message_dict) + return (conversation_id, request_id, message_ids) @@ -368,7 +372,8 @@ async def wrap_embedding_acreate(wrapped, instance, args, kwargs): "ingest_source": "Python", } - transaction.record_ml_event("LlmEmbedding", embedding_dict) + transaction.record_custom_event("LlmEmbedding", embedding_dict) + return response @@ -465,7 +470,8 @@ async def wrap_chat_completion_acreate(wrapped, instance, args, kwargs): "ingest_source": "Python", } - transaction.record_ml_event("LlmChatCompletionSummary", chat_completion_summary_dict) + transaction.record_custom_event("LlmChatCompletionSummary", chat_completion_summary_dict) + message_list = list(messages) if choices: message_list.extend([choices[0].message]) diff --git a/tests/agent_features/test_record_llm_feedback_event.py b/tests/agent_features/test_record_llm_feedback_event.py index 59921ff400..c9489c050e 100644 --- a/tests/agent_features/test_record_llm_feedback_event.py +++ b/tests/agent_features/test_record_llm_feedback_event.py @@ -12,10 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from testing_support.fixtures import reset_core_stats_engine -from testing_support.validators.validate_ml_event_count import validate_ml_event_count -from testing_support.validators.validate_ml_events import validate_ml_events - +from testing_support.fixtures import reset_core_stats_engine, validate_custom_event_count +from testing_support.validators.validate_custom_events import validate_custom_events from newrelic.api.background_task import background_task from newrelic.api.ml_model import record_llm_feedback_event @@ -38,8 +36,8 @@ def test_record_llm_feedback_event_all_args_supplied(): }, ), ] - - @validate_ml_events(llm_feedback_all_args_recorded_events) + + @validate_custom_events(llm_feedback_all_args_recorded_events) @background_task() def _test(): record_llm_feedback_event( @@ -73,7 +71,7 @@ def test_record_llm_feedback_event_required_args_supplied(): ), ] - @validate_ml_events(llm_feedback_required_args_recorded_events) + @validate_custom_events(llm_feedback_required_args_recorded_events) @background_task() def _test(): record_llm_feedback_event(message_id="message_id", rating="Good") @@ -82,7 +80,7 @@ def _test(): @reset_core_stats_engine() -@validate_ml_event_count(count=0) +@validate_custom_event_count(count=0) def test_record_llm_feedback_event_outside_txn(): record_llm_feedback_event( rating="Good", diff --git a/tests/external_botocore/conftest.py b/tests/external_botocore/conftest.py index 6dbf20ef42..c992726b3e 100644 --- a/tests/external_botocore/conftest.py +++ b/tests/external_botocore/conftest.py @@ -42,7 +42,7 @@ "transaction_tracer.stack_trace_threshold": 0.0, "debug.log_data_collector_payloads": True, "debug.record_transaction_failure": True, - "ml_insights_events.enabled": True, + "custom_insights_events.max_attribute_value": 4096 } collector_agent_registration = collector_agent_registration_fixture( app_name="Python Agent Test (external_botocore)", diff --git a/tests/external_botocore/test_bedrock_chat_completion.py b/tests/external_botocore/test_bedrock_chat_completion.py index 4f32a92ac6..18578a887f 100644 --- a/tests/external_botocore/test_bedrock_chat_completion.py +++ b/tests/external_botocore/test_bedrock_chat_completion.py @@ -28,12 +28,12 @@ dt_enabled, override_application_settings, reset_core_stats_engine, + validate_custom_event_count, ) from testing_support.validators.validate_error_trace_attributes import ( validate_error_trace_attributes, ) -from testing_support.validators.validate_ml_event_count import validate_ml_event_count -from testing_support.validators.validate_ml_events import validate_ml_events +from testing_support.validators.validate_custom_events import validate_custom_events from testing_support.validators.validate_transaction_metrics import ( validate_transaction_metrics, ) @@ -106,9 +106,9 @@ def expected_client_error(model_id): # not working with claude @reset_core_stats_engine() def test_bedrock_chat_completion_in_txn_with_convo_id(set_trace_info, exercise_model, expected_events): - @validate_ml_events(expected_events) + @validate_custom_events(expected_events) # One summary event, one user message, and one response message from the assistant - @validate_ml_event_count(count=3) + @validate_custom_event_count(count=3) @validate_transaction_metrics( name="test_bedrock_chat_completion_in_txn_with_convo_id", custom_metrics=[ @@ -128,9 +128,9 @@ def _test(): # not working with claude @reset_core_stats_engine() def test_bedrock_chat_completion_in_txn_no_convo_id(set_trace_info, exercise_model, expected_events_no_convo_id): - @validate_ml_events(expected_events_no_convo_id) + @validate_custom_events(expected_events_no_convo_id) # One summary event, one user message, and one response message from the assistant - @validate_ml_event_count(count=3) + @validate_custom_event_count(count=3) @validate_transaction_metrics( name="test_bedrock_chat_completion_in_txn_no_convo_id", custom_metrics=[ @@ -147,19 +147,19 @@ def _test(): @reset_core_stats_engine() -@validate_ml_event_count(count=0) +@validate_custom_event_count(count=0) def test_bedrock_chat_completion_outside_txn(set_trace_info, exercise_model): set_trace_info() add_custom_attribute("conversation_id", "my-awesome-id") exercise_model(prompt=_test_bedrock_chat_completion_prompt, temperature=0.7, max_tokens=100) -disabled_ml_settings = {"machine_learning.enabled": False, "ml_insights_events.enabled": False} +disabled_custom_insights_settings = {"custom_insights_events.enabled": False} -@override_application_settings(disabled_ml_settings) +@override_application_settings(disabled_custom_insights_settings) @reset_core_stats_engine() -@validate_ml_event_count(count=0) +@validate_custom_event_count(count=0) @validate_transaction_metrics( name="test_bedrock_chat_completion_disabled_settings", custom_metrics=[ diff --git a/tests/external_botocore/test_bedrock_embeddings.py b/tests/external_botocore/test_bedrock_embeddings.py index db985ee467..788e4ec867 100644 --- a/tests/external_botocore/test_bedrock_embeddings.py +++ b/tests/external_botocore/test_bedrock_embeddings.py @@ -27,12 +27,12 @@ dt_enabled, override_application_settings, reset_core_stats_engine, + validate_custom_event_count ) from testing_support.validators.validate_error_trace_attributes import ( validate_error_trace_attributes, ) -from testing_support.validators.validate_ml_event_count import validate_ml_event_count -from testing_support.validators.validate_ml_events import validate_ml_events +from testing_support.validators.validate_custom_events import validate_custom_events from testing_support.validators.validate_transaction_metrics import ( validate_transaction_metrics, ) @@ -40,7 +40,7 @@ from newrelic.api.background_task import background_task from newrelic.common.object_names import callable_name -disabled_ml_insights_settings = {"ml_insights_events.enabled": False} +disabled_custom_insights_settings = {"custom_insights_events.enabled": False} @pytest.fixture(scope="session", params=[False, True], ids=["Bytes", "Stream"]) @@ -92,8 +92,8 @@ def expected_client_error(model_id): @reset_core_stats_engine() def test_bedrock_embedding(set_trace_info, exercise_model, expected_events): - @validate_ml_events(expected_events) - @validate_ml_event_count(count=1) + @validate_custom_events(expected_events) + @validate_custom_event_count(count=1) @validate_transaction_metrics( name="test_bedrock_embedding", custom_metrics=[ @@ -110,7 +110,7 @@ def _test(): @reset_core_stats_engine() -@validate_ml_event_count(count=0) +@validate_custom_event_count(count=0) def test_bedrock_embedding_outside_txn(exercise_model): exercise_model(prompt="This is an embedding test.") @@ -119,9 +119,9 @@ def test_bedrock_embedding_outside_txn(exercise_model): _client_error_name = callable_name(_client_error) -@override_application_settings(disabled_ml_insights_settings) +@override_application_settings(disabled_custom_insights_settings) @reset_core_stats_engine() -@validate_ml_event_count(count=0) +@validate_custom_event_count(count=0) @validate_transaction_metrics( name="test_bedrock_embeddings:test_bedrock_embedding_disabled_settings", custom_metrics=[ diff --git a/tests/mlmodel_openai/test_chat_completion.py b/tests/mlmodel_openai/test_chat_completion.py index 6f3762a826..c864e4f030 100644 --- a/tests/mlmodel_openai/test_chat_completion.py +++ b/tests/mlmodel_openai/test_chat_completion.py @@ -16,9 +16,9 @@ from testing_support.fixtures import ( override_application_settings, reset_core_stats_engine, + validate_custom_event_count, ) -from testing_support.validators.validate_ml_event_count import validate_ml_event_count -from testing_support.validators.validate_ml_events import validate_ml_events +from testing_support.validators.validate_custom_events import validate_custom_events from testing_support.validators.validate_transaction_metrics import ( validate_transaction_metrics, ) @@ -26,7 +26,7 @@ from newrelic.api.background_task import background_task from newrelic.api.transaction import add_custom_attribute -disabled_ml_insights_settings = {"ml_insights_events.enabled": False} +disabled_custom_insights_settings = {"custom_insights_events.enabled": False} _test_openai_chat_completion_messages = ( {"role": "system", "content": "You are a scientist."}, @@ -129,9 +129,9 @@ @reset_core_stats_engine() -@validate_ml_events(chat_completion_recorded_events) +@validate_custom_events(chat_completion_recorded_events) # One summary event, one system message, one user message, and one response message from the assistant -@validate_ml_event_count(count=4) +@validate_custom_event_count(count=4) @validate_transaction_metrics( name="test_chat_completion:test_openai_chat_completion_sync_in_txn_with_convo_id", custom_metrics=[ @@ -244,9 +244,9 @@ def test_openai_chat_completion_sync_in_txn_with_convo_id(set_trace_info): @reset_core_stats_engine() -@validate_ml_events(chat_completion_recorded_events_no_convo_id) +@validate_custom_events(chat_completion_recorded_events_no_convo_id) # One summary event, one system message, one user message, and one response message from the assistant -@validate_ml_event_count(count=4) +@validate_custom_event_count(count=4) @background_task() def test_openai_chat_completion_sync_in_txn_no_convo_id(set_trace_info): set_trace_info() @@ -256,7 +256,7 @@ def test_openai_chat_completion_sync_in_txn_no_convo_id(set_trace_info): @reset_core_stats_engine() -@validate_ml_event_count(count=0) +@validate_custom_event_count(count=0) def test_openai_chat_completion_sync_outside_txn(): add_custom_attribute("conversation_id", "my-awesome-id") openai.ChatCompletion.create( @@ -264,18 +264,18 @@ def test_openai_chat_completion_sync_outside_txn(): ) -@override_application_settings(disabled_ml_insights_settings) +@override_application_settings(disabled_custom_insights_settings) @reset_core_stats_engine() -@validate_ml_event_count(count=0) +@validate_custom_event_count(count=0) @validate_transaction_metrics( - name="test_chat_completion:test_openai_chat_completion_sync_ml_insights_disabled", + name="test_chat_completion:test_openai_chat_completion_sync_custom_events_insights_disabled", custom_metrics=[ ("Python/ML/OpenAI/%s" % openai.__version__, 1), ], background_task=True, ) @background_task() -def test_openai_chat_completion_sync_ml_insights_disabled(set_trace_info): +def test_openai_chat_completion_sync_custom_events_insights_disabled(set_trace_info): set_trace_info() openai.ChatCompletion.create( model="gpt-3.5-turbo", messages=_test_openai_chat_completion_messages, temperature=0.7, max_tokens=100 @@ -283,8 +283,8 @@ def test_openai_chat_completion_sync_ml_insights_disabled(set_trace_info): @reset_core_stats_engine() -@validate_ml_events(chat_completion_recorded_events_no_convo_id) -@validate_ml_event_count(count=4) +@validate_custom_events(chat_completion_recorded_events_no_convo_id) +@validate_custom_event_count(count=4) @background_task() def test_openai_chat_completion_async_conversation_id_unset(loop, set_trace_info): set_trace_info() @@ -297,8 +297,8 @@ def test_openai_chat_completion_async_conversation_id_unset(loop, set_trace_info @reset_core_stats_engine() -@validate_ml_events(chat_completion_recorded_events) -@validate_ml_event_count(count=4) +@validate_custom_events(chat_completion_recorded_events) +@validate_custom_event_count(count=4) @validate_transaction_metrics( name="test_chat_completion:test_openai_chat_completion_async_conversation_id_set", custom_metrics=[ @@ -319,7 +319,7 @@ def test_openai_chat_completion_async_conversation_id_set(loop, set_trace_info): @reset_core_stats_engine() -@validate_ml_event_count(count=0) +@validate_custom_event_count(count=0) def test_openai_chat_completion_async_outside_transaction(loop): loop.run_until_complete( openai.ChatCompletion.acreate( @@ -328,18 +328,18 @@ def test_openai_chat_completion_async_outside_transaction(loop): ) -@override_application_settings(disabled_ml_insights_settings) +@override_application_settings(disabled_custom_insights_settings) @reset_core_stats_engine() -@validate_ml_event_count(count=0) +@validate_custom_event_count(count=0) @validate_transaction_metrics( - name="test_chat_completion:test_openai_chat_completion_async_disabled_ml_settings", + name="test_chat_completion:test_openai_chat_completion_async_disabled_custom_event_settings", custom_metrics=[ ("Python/ML/OpenAI/%s" % openai.__version__, 1), ], background_task=True, ) @background_task() -def test_openai_chat_completion_async_disabled_ml_settings(loop): +def test_openai_chat_completion_async_disabled_custom_event_settings(loop): loop.run_until_complete( openai.ChatCompletion.acreate( model="gpt-3.5-turbo", messages=_test_openai_chat_completion_messages, temperature=0.7, max_tokens=100 diff --git a/tests/mlmodel_openai/test_embeddings.py b/tests/mlmodel_openai/test_embeddings.py index 180052b0de..38b51d23f9 100644 --- a/tests/mlmodel_openai/test_embeddings.py +++ b/tests/mlmodel_openai/test_embeddings.py @@ -16,17 +16,16 @@ from testing_support.fixtures import ( # override_application_settings, override_application_settings, reset_core_stats_engine, + validate_custom_event_count, ) -from testing_support.validators.validate_ml_event_count import validate_ml_event_count -from testing_support.validators.validate_ml_events import validate_ml_events +from testing_support.validators.validate_custom_events import validate_custom_events from testing_support.validators.validate_transaction_metrics import ( validate_transaction_metrics, ) from newrelic.api.background_task import background_task -disabled_ml_insights_settings = {"ml_insights_events.enabled": False} - +disabled_custom_insights_settings = {"custom_insights_events.enabled": False} embedding_recorded_events = [ ( @@ -62,8 +61,8 @@ @reset_core_stats_engine() -@validate_ml_events(embedding_recorded_events) -@validate_ml_event_count(count=1) +@validate_custom_events(embedding_recorded_events) +@validate_custom_event_count(count=1) @validate_transaction_metrics( name="test_embeddings:test_openai_embedding_sync", custom_metrics=[ @@ -78,30 +77,30 @@ def test_openai_embedding_sync(set_trace_info): @reset_core_stats_engine() -@validate_ml_event_count(count=0) +@validate_custom_event_count(count=0) def test_openai_embedding_sync_outside_txn(): openai.Embedding.create(input="This is an embedding test.", model="text-embedding-ada-002") -@override_application_settings(disabled_ml_insights_settings) +@override_application_settings(disabled_custom_insights_settings) @reset_core_stats_engine() -@validate_ml_event_count(count=0) +@validate_custom_event_count(count=0) @validate_transaction_metrics( - name="test_embeddings:test_openai_chat_completion_sync_disabled_settings", + name="test_embeddings:test_openai_embedding_sync_disabled_settings", custom_metrics=[ ("Python/ML/OpenAI/%s" % openai.__version__, 1), ], background_task=True, ) @background_task() -def test_openai_chat_completion_sync_disabled_settings(set_trace_info): +def test_openai_embedding_sync_disabled_settings(set_trace_info): set_trace_info() openai.Embedding.create(input="This is an embedding test.", model="text-embedding-ada-002") @reset_core_stats_engine() -@validate_ml_events(embedding_recorded_events) -@validate_ml_event_count(count=1) +@validate_custom_events(embedding_recorded_events) +@validate_custom_event_count(count=1) @validate_transaction_metrics( name="test_embeddings:test_openai_embedding_async", custom_metrics=[ @@ -119,25 +118,25 @@ def test_openai_embedding_async(loop, set_trace_info): @reset_core_stats_engine() -@validate_ml_event_count(count=0) +@validate_custom_event_count(count=0) def test_openai_embedding_async_outside_transaction(loop): loop.run_until_complete( openai.Embedding.acreate(input="This is an embedding test.", model="text-embedding-ada-002") ) -@override_application_settings(disabled_ml_insights_settings) +@override_application_settings(disabled_custom_insights_settings) @reset_core_stats_engine() -@validate_ml_event_count(count=0) +@validate_custom_event_count(count=0) @validate_transaction_metrics( - name="test_embeddings:test_openai_embedding_async_disabled_ml_insights_events", + name="test_embeddings:test_openai_embedding_async_disabled_custom_insights_events", custom_metrics=[ ("Python/ML/OpenAI/%s" % openai.__version__, 1), ], background_task=True, ) @background_task() -def test_openai_embedding_async_disabled_ml_insights_events(loop): +def test_openai_embedding_async_disabled_custom_insights_events(loop): loop.run_until_complete( openai.Embedding.acreate(input="This is an embedding test.", model="text-embedding-ada-002") ) diff --git a/tests/mlmodel_openai/test_get_llm_message_ids.py b/tests/mlmodel_openai/test_get_llm_message_ids.py index e20245128e..af073f7300 100644 --- a/tests/mlmodel_openai/test_get_llm_message_ids.py +++ b/tests/mlmodel_openai/test_get_llm_message_ids.py @@ -13,12 +13,10 @@ # limitations under the License. import openai -from testing_support.fixtures import reset_core_stats_engine -from testing_support.validators.validate_ml_event_count import validate_ml_event_count - from newrelic.api.background_task import background_task from newrelic.api.ml_model import get_llm_message_ids, record_llm_feedback_event from newrelic.api.transaction import add_custom_attribute, current_transaction +from testing_support.fixtures import reset_core_stats_engine, validate_custom_event_count _test_openai_chat_completion_messages_1 = ( {"role": "system", "content": "You are a scientist."}, @@ -170,7 +168,7 @@ async def _run(): @reset_core_stats_engine() # Three chat completion messages and one chat completion summary for each create call (8 in total) # Three feedback events for the first create call -@validate_ml_event_count(11) +@validate_custom_event_count(11) @background_task() def test_get_llm_message_ids_mulitple_sync(set_trace_info): set_trace_info() @@ -203,7 +201,7 @@ def test_get_llm_message_ids_mulitple_sync(set_trace_info): @reset_core_stats_engine() -@validate_ml_event_count(11) +@validate_custom_event_count(11) @background_task() def test_get_llm_message_ids_mulitple_sync_no_conversation_id(set_trace_info): set_trace_info() diff --git a/tests/testing_support/validators/validate_custom_events.py b/tests/testing_support/validators/validate_custom_events.py new file mode 100644 index 0000000000..206ce08f1a --- /dev/null +++ b/tests/testing_support/validators/validate_custom_events.py @@ -0,0 +1,109 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import time + +from testing_support.fixtures import catch_background_exceptions + +from newrelic.common.object_wrapper import function_wrapper, transient_function_wrapper +from newrelic.packages import six + + +def validate_custom_events(events): + @function_wrapper + def _validate_wrapper(wrapped, instance, args, kwargs): + record_called = [] + recorded_events = [] + + @transient_function_wrapper("newrelic.core.stats_engine", "StatsEngine.record_transaction") + @catch_background_exceptions + def _validate_custom_events(wrapped, instance, args, kwargs): + record_called.append(True) + try: + result = wrapped(*args, **kwargs) + except: + raise + recorded_events[:] = [] + recorded_events.extend(list(instance._custom_events)) + + return result + + _new_wrapper = _validate_custom_events(wrapped) + val = _new_wrapper(*args, **kwargs) + assert record_called + found_events = copy.copy(recorded_events) + + record_called[:] = [] + recorded_events[:] = [] + + for expected in events: + matching_custom_events = 0 + mismatches = [] + for captured in found_events: + if _check_event_attributes(expected, captured, mismatches): + matching_custom_events += 1 + assert matching_custom_events == 1, _event_details(matching_custom_events, found_events, mismatches) + + return val + + return _validate_wrapper + + +def _check_event_attributes(expected, captured, mismatches): + assert len(captured) == 2 # [intrinsic, user attributes] + + intrinsics = captured[0] + + if intrinsics["type"] != expected[0]["type"]: + mismatches.append("key: type, value:<%s><%s>" % (expected[0]["type"], captured[0].get("type", None))) + return False + + now = time.time() + + if not (isinstance(intrinsics["timestamp"], int) and intrinsics["timestamp"] <= 1000.0 * now): + mismatches.append("key: timestamp, value:<%s>" % intrinsics["timestamp"]) + return False + + captured_keys = set(six.iterkeys(captured[1])) + expected_keys = set(six.iterkeys(expected[1])) + extra_keys = captured_keys - expected_keys + + if extra_keys: + mismatches.append("extra_keys: %s" % str(tuple(extra_keys))) + return False + + for key, value in six.iteritems(expected[1]): + if key in captured[1]: + captured_value = captured[1].get(key, None) + else: + mismatches.append("key: %s, value:<%s><%s>" % (key, value, captured[1].get(key, None))) + return False + + if value is not None: + if value != captured_value: + mismatches.append("key: %s, value:<%s><%s>" % (key, value, captured_value)) + return False + + return True + + +def _event_details(matching_custom_events, captured, mismatches): + details = [ + "matching_custom_events=%d" % matching_custom_events, + "mismatches=%s" % mismatches, + "captured_events=%s" % captured, + ] + + return "\n".join(details) diff --git a/tox.ini b/tox.ini index 25f602d455..0197fa5170 100644 --- a/tox.ini +++ b/tox.ini @@ -326,7 +326,7 @@ deps = framework_sanic-sanic{200904,210300,2109,2112,2203,2290}: websockets<11 ; For test_exception_in_middleware test, anyio is used: ; https://github.com/encode/starlette/pull/1157 - ; but anyiolatest creates breaking changes to our tests + ; but anyiolatest creates breaking changes to our tests ; (but not the instrumentation): ; https://github.com/agronholm/anyio/releases/tag/4.0.0 framework_starlette: anyio<4