From c7b3c97a27a2bdf9c13790283a109721a285fdcf Mon Sep 17 00:00:00 2001 From: Karthik Kalyanaraman Date: Wed, 31 Jul 2024 00:09:23 -0700 Subject: [PATCH 01/38] [WIP] Initial commit for OpenAI instrumentation --- .../LICENSE | 201 ++++++++ .../README.rst | 26 ++ .../pyproject.toml | 53 +++ .../instrumentation/openai/__init__.py | 74 +++ .../instrumentation/openai/package.py | 16 + .../instrumentation/openai/patch.py | 438 ++++++++++++++++++ .../instrumentation/openai/span_attributes.py | 212 +++++++++ .../instrumentation/openai/utils.py | 117 +++++ .../instrumentation/openai/version.py | 15 + .../test-requirements.txt | 8 + .../tests/__init__.py | 0 11 files changed, 1160 insertions(+) create mode 100644 instrumentation/opentelemetry-instrumentation-openai/LICENSE create mode 100644 instrumentation/opentelemetry-instrumentation-openai/README.rst create mode 100644 instrumentation/opentelemetry-instrumentation-openai/pyproject.toml create mode 100644 instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py create mode 100644 instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/package.py create mode 100644 instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py create mode 100644 instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/span_attributes.py create mode 100644 instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/utils.py create mode 100644 instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/version.py create mode 100644 instrumentation/opentelemetry-instrumentation-openai/test-requirements.txt create mode 100644 instrumentation/opentelemetry-instrumentation-openai/tests/__init__.py diff --git a/instrumentation/opentelemetry-instrumentation-openai/LICENSE b/instrumentation/opentelemetry-instrumentation-openai/LICENSE new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-openai/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/instrumentation/opentelemetry-instrumentation-openai/README.rst b/instrumentation/opentelemetry-instrumentation-openai/README.rst new file mode 100644 index 0000000000..7cabed5b16 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-openai/README.rst @@ -0,0 +1,26 @@ +OpenTelemetry OpenAI Instrumentation +=================================== + +|pypi| + +.. |pypi| image:: https://badge.fury.io/py/opentelemetry-instrumentation-openai.svg + :target: https://pypi.org/project/opentelemetry-instrumentation-openai/ + +Instrumentation with OpenAI that supports the openai library and is +specified to trace_integration using 'OpenAI'. + + +Installation +------------ + +:: + + pip install opentelemetry-instrumentation-openai + + +References +---------- +* `OpenTelemetry OpenAI Instrumentation `_ +* `OpenTelemetry Project `_ +* `OpenTelemetry Python Examples `_ + diff --git a/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml b/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml new file mode 100644 index 0000000000..bafb620577 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml @@ -0,0 +1,53 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "opentelemetry-instrumentation-openai" +dynamic = ["version"] +description = "OpenTelemetry OpenAI instrumentation" +readme = "README.rst" +license = "Apache-2.0" +requires-python = ">=3.8" +authors = [ + { name = "OpenTelemetry Authors", email = "cncf-opentelemetry-contributors@lists.cncf.io" }, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] +dependencies = [ + "opentelemetry-api ~= 1.12", + "opentelemetry-instrumentation == 0.48b0.dev", +] + +[project.optional-dependencies] +instruments = [ + "openai ~= 1.37.1", +] + +[project.entry-points.opentelemetry_instrumentor] +openai = "opentelemetry.instrumentation.openai:OpenAIInstrumentor" + +[project.urls] +Homepage = "https://github.com/open-telemetry/opentelemetry-python-contrib/tree/main/instrumentation/opentelemetry-instrumentation-openai" + +[tool.hatch.version] +path = "src/opentelemetry/instrumentation/openai/version.py" + +[tool.hatch.build.targets.sdist] +include = [ + "/src", + "/tests", +] + +[tool.hatch.build.targets.wheel] +packages = ["src/opentelemetry"] diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py new file mode 100644 index 0000000000..fe7d433ac1 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py @@ -0,0 +1,74 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +OpenAI client instrumentation supporting `openai`, it can be enabled by +using ``OpenAIInstrumentor``. + +.. _openai: https://pypi.org/project/openai/ + +Usage +----- + +.. code:: python + + from openai import OpenAI + from opentelemetry.instrumentation.openai import OpenAIInstrumentor + + OpenAIInstrumentor().instrument() + + client = OpenAI() + response = client.chat.completions.create( + model="gpt-4o-mini", + messages=[ + {"role": "user", "content": "Write a short poem on open telemetry."}, + ], + ) + +API +--- +""" + +import importlib.metadata +from typing import Collection + +from opentelemetry.instrumentation.instrumentor import BaseInstrumentor +from opentelemetry.instrumentation.openai.package import _instruments +from opentelemetry.trace import get_tracer +from wrapt import wrap_function_wrapper +from langtrace_python_sdk.instrumentation.openai.patch import ( + chat_completions_create +) + + +class OpenAIInstrumentor(BaseInstrumentor): + + def instrumentation_dependencies(self) -> Collection[str]: + return _instruments + + def _instrument(self, **kwargs): + """Enable OpenAI instrumentation. + """ + tracer_provider = kwargs.get("tracer_provider") + tracer = get_tracer(__name__, "", tracer_provider) + version = importlib.metadata.version("openai") + + wrap_function_wrapper( + "openai.resources.chat.completions", + "Completions.create", + chat_completions_create("openai.chat.completions.create", version, tracer), + ) + + def _uninstrument(self, **kwargs): + pass diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/package.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/package.py new file mode 100644 index 0000000000..9dd45c3b43 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/package.py @@ -0,0 +1,16 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +_instruments = ("openai ~= 1.37.1",) diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py new file mode 100644 index 0000000000..7e57aea25f --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py @@ -0,0 +1,438 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from opentelemetry import trace +from opentelemetry.trace import SpanKind, Span +from opentelemetry.trace.status import Status, StatusCode +from opentelemetry.trace.propagation import set_span_in_context +from openai._types import NOT_GIVEN +from span_attributes import SpanAttributes, LLMSpanAttributes, Event +from utils import estimate_tokens, silently_fail, extract_content, calculate_prompt_tokens + + +def chat_completions_create(original_method, version, tracer): + """Wrap the `create` method of the `ChatCompletion` class to trace it.""" + + def traced_method(wrapped, instance, args, kwargs): + llm_prompts = [] + for item in kwargs.get("messages", []): + tools = get_tool_calls(item) + if tools is not None: + tool_calls = [] + for tool_call in tools: + tool_call_dict = { + "id": tool_call.id if hasattr(tool_call, "id") else "", + "type": tool_call.type if hasattr(tool_call, "type") else "", + } + if hasattr(tool_call, "function"): + tool_call_dict["function"] = { + "name": ( + tool_call.function.name + if hasattr(tool_call.function, "name") + else "" + ), + "arguments": ( + tool_call.function.arguments + if hasattr(tool_call.function, "arguments") + else "" + ), + } + tool_calls.append(tool_call_dict) + llm_prompts.append(tool_calls) + else: + llm_prompts.append(item) + + span_attributes = { + **get_llm_request_attributes(kwargs, prompts=llm_prompts), + } + + attributes = LLMSpanAttributes(**span_attributes) + + span = tracer.start_span( + "openai.completion", + kind=SpanKind.CLIENT, + context=set_span_in_context(trace.get_current_span()), + ) + _set_input_attributes(span, kwargs, attributes) + + try: + result = wrapped(*args, **kwargs) + if is_streaming(kwargs): + prompt_tokens = 0 + for message in kwargs.get("messages", {}): + prompt_tokens += calculate_prompt_tokens( + json.dumps(str(message)), kwargs.get("model") + ) + + if ( + kwargs.get("functions") is not None + and kwargs.get("functions") != NOT_GIVEN + ): + for function in kwargs.get("functions"): + prompt_tokens += calculate_prompt_tokens( + json.dumps(function), kwargs.get("model") + ) + + return StreamWrapper( + result, + span, + prompt_tokens, + function_call=kwargs.get("functions") is not None, + tool_calls=kwargs.get("tools") is not None, + ) + else: + _set_response_attributes(span, kwargs, result) + span.set_status(StatusCode.OK) + span.end() + return result + + except Exception as error: + span.record_exception(error) + span.set_status(Status(StatusCode.ERROR, str(error))) + span.end() + raise + + return traced_method + + +def get_tool_calls(item): + if isinstance(item, dict): + if "tool_calls" in item and item["tool_calls"] is not None: + return item["tool_calls"] + return None + + else: + if hasattr(item, "tool_calls") and item.tool_calls is not None: + return item.tool_calls + return None + + +@silently_fail +def _set_input_attributes(span, kwargs, attributes): + tools = [] + for field, value in attributes.model_dump(by_alias=True).items(): + set_span_attribute(span, field, value) + + if kwargs.get("functions") is not None and kwargs.get("functions") != NOT_GIVEN: + for function in kwargs.get("functions"): + tools.append(json.dumps({"type": "function", "function": function})) + + if kwargs.get("tools") is not None and kwargs.get("tools") != NOT_GIVEN: + tools.append(json.dumps(kwargs.get("tools"))) + + if tools: + set_span_attribute(span, SpanAttributes.LLM_TOOLS, json.dumps(tools)) + + +@silently_fail +def _set_response_attributes(span, kwargs, result): + set_span_attribute(span, SpanAttributes.LLM_RESPONSE_MODEL, result.model) + if hasattr(result, "choices") and result.choices is not None: + responses = [ + { + "role": ( + choice.message.role + if choice.message and choice.message.role + else "assistant" + ), + "content": extract_content(choice), + **( + {"content_filter_results": choice["content_filter_results"]} + if "content_filter_results" in choice + else {} + ), + } + for choice in result.choices + ] + set_event_completion(span, responses) + + if ( + hasattr(result, "system_fingerprint") + and result.system_fingerprint is not None + and result.system_fingerprint != NOT_GIVEN + ): + set_span_attribute( + span, + SpanAttributes.LLM_SYSTEM_FINGERPRINT, + result.system_fingerprint, + ) + # Get the usage + if hasattr(result, "usage") and result.usage is not None: + usage = result.usage + if usage is not None: + set_span_attribute( + span, + SpanAttributes.LLM_USAGE_PROMPT_TOKENS, + result.usage.prompt_tokens, + ) + set_span_attribute( + span, + SpanAttributes.LLM_USAGE_COMPLETION_TOKENS, + result.usage.completion_tokens, + ) + set_span_attribute( + span, + SpanAttributes.LLM_USAGE_TOTAL_TOKENS, + result.usage.total_tokens, + ) + + +def set_event_prompt(span: Span, prompt): + span.add_event( + name=SpanAttributes.LLM_CONTENT_PROMPT, + attributes={ + SpanAttributes.LLM_PROMPTS: prompt, + }, + ) + + +def set_span_attributes(span: Span, attributes: dict): + for field, value in attributes.model_dump(by_alias=True).items(): + set_span_attribute(span, field, value) + + +def set_event_completion(span: Span, result_content): + span.add_event( + name=SpanAttributes.LLM_CONTENT_COMPLETION, + attributes={ + SpanAttributes.LLM_COMPLETIONS: json.dumps(result_content), + }, + ) + + +def set_event_completion_chunk(span: Span, chunk): + span.add_event( + name=SpanAttributes.LLM_CONTENT_COMPLETION_CHUNK, + attributes={ + SpanAttributes.LLM_CONTENT_COMPLETION_CHUNK: json.dumps(chunk), + }, + ) + + +def set_span_attribute(span: Span, name, value): + if value is not None: + if value != "" or value != NOT_GIVEN: + if name == SpanAttributes.LLM_PROMPTS: + set_event_prompt(span, value) + else: + span.set_attribute(name, value) + return + + +def is_streaming(kwargs): + return not ( + kwargs.get("stream") is False + or kwargs.get("stream") is None + or kwargs.get("stream") == NOT_GIVEN + ) + + +def get_llm_request_attributes(kwargs, prompts=None, model=None, operation_name="chat"): + + user = kwargs.get("user", None) + if prompts is None: + prompts = ( + [{"role": user or "user", "content": kwargs.get("prompt")}] + if "prompt" in kwargs + else None + ) + top_k = ( + kwargs.get("n", None) + or kwargs.get("k", None) + or kwargs.get("top_k", None) + or kwargs.get("top_n", None) + ) + + top_p = kwargs.get("p", None) or kwargs.get("top_p", None) + tools = kwargs.get("tools", None) + return { + SpanAttributes.LLM_OPERATION_NAME: operation_name, + SpanAttributes.LLM_REQUEST_MODEL: model or kwargs.get("model"), + SpanAttributes.LLM_IS_STREAMING: kwargs.get("stream"), + SpanAttributes.LLM_REQUEST_TEMPERATURE: kwargs.get("temperature"), + SpanAttributes.LLM_TOP_K: top_k, + SpanAttributes.LLM_PROMPTS: json.dumps(prompts) if prompts else None, + SpanAttributes.LLM_USER: user, + SpanAttributes.LLM_REQUEST_TOP_P: top_p, + SpanAttributes.LLM_REQUEST_MAX_TOKENS: kwargs.get("max_tokens"), + SpanAttributes.LLM_SYSTEM_FINGERPRINT: kwargs.get("system_fingerprint"), + SpanAttributes.LLM_PRESENCE_PENALTY: kwargs.get("presence_penalty"), + SpanAttributes.LLM_FREQUENCY_PENALTY: kwargs.get("frequency_penalty"), + SpanAttributes.LLM_REQUEST_SEED: kwargs.get("seed"), + SpanAttributes.LLM_TOOLS: json.dumps(tools) if tools else None, + SpanAttributes.LLM_TOOL_CHOICE: kwargs.get("tool_choice"), + SpanAttributes.LLM_REQUEST_LOGPROPS: kwargs.get("logprobs"), + SpanAttributes.LLM_REQUEST_LOGITBIAS: kwargs.get("logit_bias"), + SpanAttributes.LLM_REQUEST_TOP_LOGPROPS: kwargs.get("top_logprobs"), + } + + +class StreamWrapper: + span: Span + + def __init__( + self, stream, span, prompt_tokens, function_call=False, tool_calls=False + ): + self.stream = stream + self.span = span + self.prompt_tokens = prompt_tokens + self.function_call = function_call + self.tool_calls = tool_calls + self.result_content = [] + self.completion_tokens = 0 + self._span_started = False + self.setup() + + def setup(self): + if not self._span_started: + self.span.add_event(Event.STREAM_START.value) + self._span_started = True + + def cleanup(self): + if self._span_started: + self.span.add_event(Event.STREAM_END.value) + set_span_attribute( + self.span, + SpanAttributes.LLM_USAGE_PROMPT_TOKENS, + self.prompt_tokens, + ) + set_span_attribute( + self.span, + SpanAttributes.LLM_USAGE_COMPLETION_TOKENS, + self.completion_tokens, + ) + set_span_attribute( + self.span, + SpanAttributes.LLM_USAGE_TOTAL_TOKENS, + self.prompt_tokens + self.completion_tokens, + ) + set_event_completion( + self.span, + [ + { + "role": "assistant", + "content": "".join(self.result_content), + } + ], + ) + + self.span.set_status(StatusCode.OK) + self.span.end() + self._span_started = False + + def __enter__(self): + self.setup() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.cleanup() + + async def __aenter__(self): + self.setup() + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + self.cleanup() + + def __iter__(self): + return self + + def __next__(self): + try: + chunk = next(self.stream) + self.process_chunk(chunk) + return chunk + except StopIteration: + self.cleanup() + raise + + def __aiter__(self): + return self + + async def __anext__(self): + try: + chunk = await self.stream.__anext__() + self.process_chunk(chunk) + return chunk + except StopAsyncIteration: + self.cleanup() + raise StopAsyncIteration + + def process_chunk(self, chunk): + if hasattr(chunk, "model") and chunk.model is not None: + set_span_attribute( + self.span, + SpanAttributes.LLM_RESPONSE_MODEL, + chunk.model, + ) + + if hasattr(chunk, "choices") and chunk.choices is not None: + content = [] + if not self.function_call and not self.tool_calls: + for choice in chunk.choices: + if choice.delta and choice.delta.content is not None: + token_counts = estimate_tokens(choice.delta.content) + self.completion_tokens += token_counts + content = [choice.delta.content] + elif self.function_call: + for choice in chunk.choices: + if ( + choice.delta + and choice.delta.function_call is not None + and choice.delta.function_call.arguments is not None + ): + token_counts = estimate_tokens( + choice.delta.function_call.arguments + ) + self.completion_tokens += token_counts + content = [choice.delta.function_call.arguments] + elif self.tool_calls: + for choice in chunk.choices: + if choice.delta and choice.delta.tool_calls is not None: + toolcalls = choice.delta.tool_calls + content = [] + for tool_call in toolcalls: + if ( + tool_call + and tool_call.function is not None + and tool_call.function.arguments is not None + ): + token_counts = estimate_tokens( + tool_call.function.arguments + ) + self.completion_tokens += token_counts + content.append(tool_call.function.arguments) + set_event_completion_chunk( + self.span, + "".join(content) if len(content) > 0 and content[0] is not None else "", + ) + if content: + self.result_content.append(content[0]) + + if hasattr(chunk, "text"): + token_counts = estimate_tokens(chunk.text) + self.completion_tokens += token_counts + content = [chunk.text] + set_event_completion_chunk( + self.span, + "".join(content) if len(content) > 0 and content[0] is not None else "", + ) + + if content: + self.result_content.append(content[0]) + + if hasattr(chunk, "usage_metadata"): + self.completion_tokens = chunk.usage_metadata.candidates_token_count + self.prompt_tokens = chunk.usage_metadata.prompt_token_count diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/span_attributes.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/span_attributes.py new file mode 100644 index 0000000000..d4c7d70ec4 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/span_attributes.py @@ -0,0 +1,212 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations +from enum import Enum +from typing import List, Optional +from pydantic import BaseModel, ConfigDict, Field + + +class SpanAttributes: + LLM_SYSTEM = "gen_ai.system" + LLM_OPERATION_NAME = "gen_ai.operation.name" + LLM_REQUEST_MODEL = "gen_ai.request.model" + LLM_REQUEST_MAX_TOKENS = "gen_ai.request.max_tokens" + LLM_REQUEST_TEMPERATURE = "gen_ai.request.temperature" + LLM_REQUEST_TOP_P = "gen_ai.request.top_p" + LLM_SYSTEM_FINGERPRINT = "gen_ai.system_fingerprint" + LLM_REQUEST_DOCUMENTS = "gen_ai.request.documents" + LLM_REQUEST_SEARCH_REQUIRED = "gen_ai.request.is_search_required" + LLM_PROMPTS = "gen_ai.prompt" + LLM_CONTENT_PROMPT = "gen_ai.content.prompt" + LLM_COMPLETIONS = "gen_ai.completion" + LLM_CONTENT_COMPLETION = "gen_ai.content.completion" + LLM_RESPONSE_MODEL = "gen_ai.response.model" + LLM_USAGE_COMPLETION_TOKENS = "gen_ai.usage.output_tokens" + LLM_USAGE_PROMPT_TOKENS = "gen_ai.usage.input_tokens" + LLM_USAGE_TOTAL_TOKENS = "gen_ai.request.total_tokens" + LLM_USAGE_TOKEN_TYPE = "gen_ai.usage.token_type" + LLM_USAGE_SEARCH_UNITS = "gen_ai.usage.search_units" + LLM_GENERATION_ID = "gen_ai.generation_id" + LLM_TOKEN_TYPE = "gen_ai.token.type" + LLM_RESPONSE_ID = "gen_ai.response_id" + LLM_URL = "url.full" + LLM_PATH = "url.path" + LLM_RESPONSE_FORMAT = "gen_ai.response.format" + LLM_IMAGE_SIZE = "gen_ai.image.size" + LLM_REQUEST_ENCODING_FORMATS = "gen_ai.request.encoding_formats" + LLM_REQUEST_DIMENSIONS = "gen_ai.request.dimensions" + LLM_REQUEST_SEED = "gen_ai.request.seed" + LLM_REQUEST_TOP_LOGPROPS = "gen_ai.request.top_props" + LLM_REQUEST_LOGPROPS = "gen_ai.request.log_props" + LLM_REQUEST_LOGITBIAS = "gen_ai.request.logit_bias" + LLM_REQUEST_TYPE = "gen_ai.request.type" + LLM_HEADERS = "gen_ai.headers" + LLM_USER = "gen_ai.user" + LLM_TOOLS = "gen_ai.request.tools" + LLM_TOOL_CHOICE = "gen_ai.request.tool_choice" + LLM_TOOL_RESULTS = "gen_ai.request.tool_results" + LLM_TOP_K = "gen_ai.request.top_k" + LLM_IS_STREAMING = "gen_ai.request.stream" + LLM_FREQUENCY_PENALTY = "gen_ai.request.frequency_penalty" + LLM_PRESENCE_PENALTY = "gen_ai.request.presence_penalty" + LLM_CHAT_STOP_SEQUENCES = "gen_ai.chat.stop_sequences" + LLM_REQUEST_FUNCTIONS = "gen_ai.request.functions" + LLM_REQUEST_REPETITION_PENALTY = "gen_ai.request.repetition_penalty" + LLM_RESPONSE_FINISH_REASON = "gen_ai.response.finish_reasons" + LLM_RESPONSE_STOP_REASON = "gen_ai.response.stop_reason" + LLM_CONTENT_COMPLETION_CHUNK = "gen_ai.completion.chunk" + + +class Event(Enum): + STREAM_START = "stream.start" + STREAM_OUTPUT = "stream.output" + STREAM_END = "stream.end" + RESPONSE = "response" + + +class LLMSpanAttributes(BaseModel): + model_config = ConfigDict(extra="allow") + gen_ai_operation_name: str = Field( + ..., + alias='gen_ai.operation.name', + description='The name of the operation being performed.', + ) + gen_ai_request_model: str = Field( + ..., + alias='gen_ai.request.model', + description='Model name from the input request', + ) + gen_ai_response_model: Optional[str] = Field( + None, alias='gen_ai.response.model', description='Model name from the response' + ) + gen_ai_request_temperature: Optional[float] = Field( + None, + alias='gen_ai.request.temperature', + description='Temperature value from the input request', + ) + gen_ai_request_logit_bias: Optional[str] = Field( + None, + alias='gen_ai.request.logit_bias', + description='Likelihood bias of the specified tokens the input request.', + ) + gen_ai_request_logprobs: Optional[bool] = Field( + None, + alias='gen_ai.request.logprobs', + description='Logprobs flag returns log probabilities.', + ) + gen_ai_request_top_logprobs: Optional[float] = Field( + None, + alias='gen_ai.request.top_logprobs', + description='Integer between 0 and 5 specifying the number of most likely tokens to return.', + ) + gen_ai_request_top_p: Optional[float] = Field( + None, + alias='gen_ai.request.top_p', + description='Top P value from the input request', + ) + gen_ai_request_top_k: Optional[float] = Field( + None, + alias='gen_ai.request.top_k', + description='Top K results to return from the input request', + ) + gen_ai_user: Optional[str] = Field( + None, alias='gen_ai.user', description='User ID from the input request' + ) + gen_ai_prompt: Optional[str] = Field( + None, alias='gen_ai.prompt', description='Prompt text from the input request' + ) + gen_ai_completion: Optional[str] = Field( + None, + alias='gen_ai.completion', + description='Completion text from the response. This will be an array of json objects with the following format {"role": "", "content": ""}. Role can be one of the following values: [system, user, assistant, tool]', + ) + gen_ai_request_stream: Optional[bool] = Field( + None, + alias='gen_ai.request.stream', + description='Stream flag from the input request', + ) + gen_ai_request_encoding_formats: Optional[List[str]] = Field( + None, + alias='gen_ai.request.encoding_formats', + description="Encoding formats from the input request. Allowed values: ['float', 'int8','uint8', 'binary', 'ubinary', 'base64']", + ) + gen_ai_completion_chunk: Optional[str] = Field( + None, + alias='gen_ai.completion.chunk', + description='Chunk text from the response', + ) + gen_ai_response_finish_reasons: Optional[List[str]] = Field( + None, + alias='gen_ai.response.finish_reasons', + description='Array of reasons the model stopped generating tokens, corresponding to each generation received', + ) + gen_ai_system_fingerprint: Optional[str] = Field( + None, + alias='gen_ai.system_fingerprint', + description='System fingerprint of the system that generated the response', + ) + gen_ai_request_tool_choice: Optional[str] = Field( + None, + alias='gen_ai.request.tool_choice', + description='Tool choice from the input request', + ) + gen_ai_response_tool_calls: Optional[str] = Field( + None, + alias='gen_ai.response.tool_calls', + description='Array of tool calls from the response json stringified', + ) + gen_ai_request_max_tokens: Optional[float] = Field( + None, + alias='gen_ai.request.max_tokens', + description='The maximum number of tokens the LLM generates for a request.', + ) + gen_ai_usage_input_tokens: Optional[float] = Field( + None, + alias='gen_ai.usage.input_tokens', + description='The number of tokens used in the llm prompt.', + ) + gen_ai_usage_total_tokens: Optional[float] = Field( + None, + alias='gen_ai.usage.total_tokens', + description='The total number of tokens used in the llm request.', + ) + gen_ai_usage_output_tokens: Optional[float] = Field( + None, + alias='gen_ai.usage.output_tokens', + description='The number of tokens in the llm response.', + ) + gen_ai_request_seed: Optional[str] = Field( + None, alias='gen_ai.request.seed', description='Seed from the input request' + ) + gen_ai_request_frequency_penalty: Optional[float] = Field( + None, + alias='gen_ai.request.frequency_penalty', + description='Frequency penalty from the input request', + ) + gen_ai_request_presence_penalty: Optional[float] = Field( + None, + alias='gen_ai.request.presence_penalty', + description='Presence penalty from the input request', + ) + gen_ai_request_tools: Optional[str] = Field( + None, + alias='gen_ai.request.tools', + description='An array of tools from the input request json stringified', + ) + gen_ai_request_tool_results: Optional[str] = Field( + None, + alias='gen_ai.request.tool_results', + description='An array of tool results from the input request json stringified', + ) diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/utils.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/utils.py new file mode 100644 index 0000000000..1a824d463a --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/utils.py @@ -0,0 +1,117 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from tiktoken import get_encoding + + +def estimate_tokens_using_tiktoken(prompt, model): + """ + Estimate the number of tokens in a prompt using tiktoken.""" + encoding = get_encoding(model) + tokens = encoding.encode(prompt) + return len(tokens) + + +def estimate_tokens(prompt): + """ + Estimate the number of tokens in a prompt.""" + if prompt and len(prompt) > 0: + # Simplified token estimation: count the words. + return len([word for word in prompt.split() if word]) + return 0 + + +TIKTOKEN_MODEL_MAPPING = { + "gpt-4": "cl100k_base", + "gpt-4-32k": "cl100k_base", + "gpt-4-0125-preview": "cl100k_base", + "gpt-4-1106-preview": "cl100k_base", + "gpt-4-1106-vision-preview": "cl100k_base", + "gpt-4o": "0200k_base", + "gpt-4o-mini": "0200k_base", +} + + +def silently_fail(func): + """ + A decorator that catches exceptions thrown by the decorated function and logs them as warnings. + """ + + logger = logging.getLogger(func.__module__) + + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except Exception as exception: + logger.warning( + "Failed to execute %s, error: %s", func.__name__, str(exception) + ) + + return wrapper + + +def extract_content(choice): + # Check if choice.message exists and has a content attribute + if ( + hasattr(choice, "message") + and hasattr(choice.message, "content") + and choice.message.content is not None + ): + return choice.message.content + + # Check if choice.message has tool_calls and extract information accordingly + elif ( + hasattr(choice, "message") + and hasattr(choice.message, "tool_calls") + and choice.message.tool_calls is not None + ): + result = [ + { + "id": tool_call.id, + "type": tool_call.type, + "function": { + "name": tool_call.function.name, + "arguments": tool_call.function.arguments, + }, + } + for tool_call in choice.message.tool_calls + ] + return result + + # Check if choice.message has a function_call and extract information accordingly + elif ( + hasattr(choice, "message") + and hasattr(choice.message, "function_call") + and choice.message.function_call is not None + ): + return { + "name": choice.message.function_call.name, + "arguments": choice.message.function_call.arguments, + } + + # Return an empty string if none of the above conditions are met + else: + return "" + + +def calculate_prompt_tokens(prompt_content, model): + """ + Calculate the number of tokens in a prompt. If the model is supported by tiktoken, use it for the estimation. + """ + try: + tiktoken_model = TIKTOKEN_MODEL_MAPPING[model] + return estimate_tokens_using_tiktoken(prompt_content, tiktoken_model) + except Exception: + return estimate_tokens(prompt_content) # Fallback method diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/version.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/version.py new file mode 100644 index 0000000000..021133daf1 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/version.py @@ -0,0 +1,15 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = "0.0.1dev" diff --git a/instrumentation/opentelemetry-instrumentation-openai/test-requirements.txt b/instrumentation/opentelemetry-instrumentation-openai/test-requirements.txt new file mode 100644 index 0000000000..6b3d9ef39d --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-openai/test-requirements.txt @@ -0,0 +1,8 @@ +openai==1.37.1 +Deprecated==1.2.14 +importlib-metadata==6.11.0 +packaging==24.0 +pytest==7.4.4 +wrapt==1.16.0 +# -e opentelemetry-instrumentation +# -e instrumentation/opentelemetry-instrumentation-openai diff --git a/instrumentation/opentelemetry-instrumentation-openai/tests/__init__.py b/instrumentation/opentelemetry-instrumentation-openai/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 From e1bca1aa4db9c500fa42d00d1a2a159c7670f49d Mon Sep 17 00:00:00 2001 From: Ali Waleed <134522290+alizenhom@users.noreply.github.com> Date: Mon, 12 Aug 2024 10:24:12 +0300 Subject: [PATCH 02/38] Loosen openai version for instrumentation + linting --- .../pyproject.toml | 4 +- .../instrumentation/openai/__init__.py | 10 +- .../instrumentation/openai/package.py | 2 +- .../instrumentation/openai/patch.py | 59 ++++++++-- .../instrumentation/openai/span_attributes.py | 106 +++++++++--------- .../instrumentation/openai/utils.py | 4 +- .../instrumentation/openai/version.py | 2 +- 7 files changed, 116 insertions(+), 71 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml b/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml index bafb620577..dd508f9414 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml +++ b/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml @@ -26,12 +26,12 @@ classifiers = [ ] dependencies = [ "opentelemetry-api ~= 1.12", - "opentelemetry-instrumentation == 0.48b0.dev", + "opentelemetry-instrumentation == 0.47b0", ] [project.optional-dependencies] instruments = [ - "openai ~= 1.37.1", + "openai >= 0.27.0", ] [project.entry-points.opentelemetry_instrumentor] diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py index fe7d433ac1..d30064b51e 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py @@ -48,7 +48,7 @@ from opentelemetry.trace import get_tracer from wrapt import wrap_function_wrapper from langtrace_python_sdk.instrumentation.openai.patch import ( - chat_completions_create + chat_completions_create, ) @@ -58,16 +58,16 @@ def instrumentation_dependencies(self) -> Collection[str]: return _instruments def _instrument(self, **kwargs): - """Enable OpenAI instrumentation. - """ + """Enable OpenAI instrumentation.""" tracer_provider = kwargs.get("tracer_provider") tracer = get_tracer(__name__, "", tracer_provider) version = importlib.metadata.version("openai") - wrap_function_wrapper( "openai.resources.chat.completions", "Completions.create", - chat_completions_create("openai.chat.completions.create", version, tracer), + chat_completions_create( + "openai.chat.completions.create", version, tracer + ), ) def _uninstrument(self, **kwargs): diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/package.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/package.py index 9dd45c3b43..dcfded3025 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/package.py +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/package.py @@ -13,4 +13,4 @@ # limitations under the License. -_instruments = ("openai ~= 1.37.1",) +_instruments = ("openai >= 0.27.0",) diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py index 7e57aea25f..c63bdab256 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py @@ -19,7 +19,12 @@ from opentelemetry.trace.propagation import set_span_in_context from openai._types import NOT_GIVEN from span_attributes import SpanAttributes, LLMSpanAttributes, Event -from utils import estimate_tokens, silently_fail, extract_content, calculate_prompt_tokens +from utils import ( + estimate_tokens, + silently_fail, + extract_content, + calculate_prompt_tokens, +) def chat_completions_create(original_method, version, tracer): @@ -34,7 +39,11 @@ def traced_method(wrapped, instance, args, kwargs): for tool_call in tools: tool_call_dict = { "id": tool_call.id if hasattr(tool_call, "id") else "", - "type": tool_call.type if hasattr(tool_call, "type") else "", + "type": ( + tool_call.type + if hasattr(tool_call, "type") + else "" + ), } if hasattr(tool_call, "function"): tool_call_dict["function"] = { @@ -125,9 +134,14 @@ def _set_input_attributes(span, kwargs, attributes): for field, value in attributes.model_dump(by_alias=True).items(): set_span_attribute(span, field, value) - if kwargs.get("functions") is not None and kwargs.get("functions") != NOT_GIVEN: + if ( + kwargs.get("functions") is not None + and kwargs.get("functions") != NOT_GIVEN + ): for function in kwargs.get("functions"): - tools.append(json.dumps({"type": "function", "function": function})) + tools.append( + json.dumps({"type": "function", "function": function}) + ) if kwargs.get("tools") is not None and kwargs.get("tools") != NOT_GIVEN: tools.append(json.dumps(kwargs.get("tools"))) @@ -149,7 +163,11 @@ def _set_response_attributes(span, kwargs, result): ), "content": extract_content(choice), **( - {"content_filter_results": choice["content_filter_results"]} + { + "content_filter_results": choice[ + "content_filter_results" + ] + } if "content_filter_results" in choice else {} ), @@ -239,7 +257,9 @@ def is_streaming(kwargs): ) -def get_llm_request_attributes(kwargs, prompts=None, model=None, operation_name="chat"): +def get_llm_request_attributes( + kwargs, prompts=None, model=None, operation_name="chat" +): user = kwargs.get("user", None) if prompts is None: @@ -267,7 +287,9 @@ def get_llm_request_attributes(kwargs, prompts=None, model=None, operation_name= SpanAttributes.LLM_USER: user, SpanAttributes.LLM_REQUEST_TOP_P: top_p, SpanAttributes.LLM_REQUEST_MAX_TOKENS: kwargs.get("max_tokens"), - SpanAttributes.LLM_SYSTEM_FINGERPRINT: kwargs.get("system_fingerprint"), + SpanAttributes.LLM_SYSTEM_FINGERPRINT: kwargs.get( + "system_fingerprint" + ), SpanAttributes.LLM_PRESENCE_PENALTY: kwargs.get("presence_penalty"), SpanAttributes.LLM_FREQUENCY_PENALTY: kwargs.get("frequency_penalty"), SpanAttributes.LLM_REQUEST_SEED: kwargs.get("seed"), @@ -283,7 +305,12 @@ class StreamWrapper: span: Span def __init__( - self, stream, span, prompt_tokens, function_call=False, tool_calls=False + self, + stream, + span, + prompt_tokens, + function_call=False, + tool_calls=False, ): self.stream = stream self.span = span @@ -416,7 +443,11 @@ def process_chunk(self, chunk): content.append(tool_call.function.arguments) set_event_completion_chunk( self.span, - "".join(content) if len(content) > 0 and content[0] is not None else "", + ( + "".join(content) + if len(content) > 0 and content[0] is not None + else "" + ), ) if content: self.result_content.append(content[0]) @@ -427,12 +458,18 @@ def process_chunk(self, chunk): content = [chunk.text] set_event_completion_chunk( self.span, - "".join(content) if len(content) > 0 and content[0] is not None else "", + ( + "".join(content) + if len(content) > 0 and content[0] is not None + else "" + ), ) if content: self.result_content.append(content[0]) if hasattr(chunk, "usage_metadata"): - self.completion_tokens = chunk.usage_metadata.candidates_token_count + self.completion_tokens = ( + chunk.usage_metadata.candidates_token_count + ) self.prompt_tokens = chunk.usage_metadata.prompt_token_count diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/span_attributes.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/span_attributes.py index d4c7d70ec4..d85f149cc4 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/span_attributes.py +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/span_attributes.py @@ -80,133 +80,139 @@ class LLMSpanAttributes(BaseModel): model_config = ConfigDict(extra="allow") gen_ai_operation_name: str = Field( ..., - alias='gen_ai.operation.name', - description='The name of the operation being performed.', + alias="gen_ai.operation.name", + description="The name of the operation being performed.", ) gen_ai_request_model: str = Field( ..., - alias='gen_ai.request.model', - description='Model name from the input request', + alias="gen_ai.request.model", + description="Model name from the input request", ) gen_ai_response_model: Optional[str] = Field( - None, alias='gen_ai.response.model', description='Model name from the response' + None, + alias="gen_ai.response.model", + description="Model name from the response", ) gen_ai_request_temperature: Optional[float] = Field( None, - alias='gen_ai.request.temperature', - description='Temperature value from the input request', + alias="gen_ai.request.temperature", + description="Temperature value from the input request", ) gen_ai_request_logit_bias: Optional[str] = Field( None, - alias='gen_ai.request.logit_bias', - description='Likelihood bias of the specified tokens the input request.', + alias="gen_ai.request.logit_bias", + description="Likelihood bias of the specified tokens the input request.", ) gen_ai_request_logprobs: Optional[bool] = Field( None, - alias='gen_ai.request.logprobs', - description='Logprobs flag returns log probabilities.', + alias="gen_ai.request.logprobs", + description="Logprobs flag returns log probabilities.", ) gen_ai_request_top_logprobs: Optional[float] = Field( None, - alias='gen_ai.request.top_logprobs', - description='Integer between 0 and 5 specifying the number of most likely tokens to return.', + alias="gen_ai.request.top_logprobs", + description="Integer between 0 and 5 specifying the number of most likely tokens to return.", ) gen_ai_request_top_p: Optional[float] = Field( None, - alias='gen_ai.request.top_p', - description='Top P value from the input request', + alias="gen_ai.request.top_p", + description="Top P value from the input request", ) gen_ai_request_top_k: Optional[float] = Field( None, - alias='gen_ai.request.top_k', - description='Top K results to return from the input request', + alias="gen_ai.request.top_k", + description="Top K results to return from the input request", ) gen_ai_user: Optional[str] = Field( - None, alias='gen_ai.user', description='User ID from the input request' + None, alias="gen_ai.user", description="User ID from the input request" ) gen_ai_prompt: Optional[str] = Field( - None, alias='gen_ai.prompt', description='Prompt text from the input request' + None, + alias="gen_ai.prompt", + description="Prompt text from the input request", ) gen_ai_completion: Optional[str] = Field( None, - alias='gen_ai.completion', + alias="gen_ai.completion", description='Completion text from the response. This will be an array of json objects with the following format {"role": "", "content": ""}. Role can be one of the following values: [system, user, assistant, tool]', ) gen_ai_request_stream: Optional[bool] = Field( None, - alias='gen_ai.request.stream', - description='Stream flag from the input request', + alias="gen_ai.request.stream", + description="Stream flag from the input request", ) gen_ai_request_encoding_formats: Optional[List[str]] = Field( None, - alias='gen_ai.request.encoding_formats', + alias="gen_ai.request.encoding_formats", description="Encoding formats from the input request. Allowed values: ['float', 'int8','uint8', 'binary', 'ubinary', 'base64']", ) gen_ai_completion_chunk: Optional[str] = Field( None, - alias='gen_ai.completion.chunk', - description='Chunk text from the response', + alias="gen_ai.completion.chunk", + description="Chunk text from the response", ) gen_ai_response_finish_reasons: Optional[List[str]] = Field( None, - alias='gen_ai.response.finish_reasons', - description='Array of reasons the model stopped generating tokens, corresponding to each generation received', + alias="gen_ai.response.finish_reasons", + description="Array of reasons the model stopped generating tokens, corresponding to each generation received", ) gen_ai_system_fingerprint: Optional[str] = Field( None, - alias='gen_ai.system_fingerprint', - description='System fingerprint of the system that generated the response', + alias="gen_ai.system_fingerprint", + description="System fingerprint of the system that generated the response", ) gen_ai_request_tool_choice: Optional[str] = Field( None, - alias='gen_ai.request.tool_choice', - description='Tool choice from the input request', + alias="gen_ai.request.tool_choice", + description="Tool choice from the input request", ) gen_ai_response_tool_calls: Optional[str] = Field( None, - alias='gen_ai.response.tool_calls', - description='Array of tool calls from the response json stringified', + alias="gen_ai.response.tool_calls", + description="Array of tool calls from the response json stringified", ) gen_ai_request_max_tokens: Optional[float] = Field( None, - alias='gen_ai.request.max_tokens', - description='The maximum number of tokens the LLM generates for a request.', + alias="gen_ai.request.max_tokens", + description="The maximum number of tokens the LLM generates for a request.", ) gen_ai_usage_input_tokens: Optional[float] = Field( None, - alias='gen_ai.usage.input_tokens', - description='The number of tokens used in the llm prompt.', + alias="gen_ai.usage.input_tokens", + description="The number of tokens used in the llm prompt.", ) gen_ai_usage_total_tokens: Optional[float] = Field( None, - alias='gen_ai.usage.total_tokens', - description='The total number of tokens used in the llm request.', + alias="gen_ai.usage.total_tokens", + description="The total number of tokens used in the llm request.", ) gen_ai_usage_output_tokens: Optional[float] = Field( None, - alias='gen_ai.usage.output_tokens', - description='The number of tokens in the llm response.', + alias="gen_ai.usage.output_tokens", + description="The number of tokens in the llm response.", ) gen_ai_request_seed: Optional[str] = Field( - None, alias='gen_ai.request.seed', description='Seed from the input request' + None, + alias="gen_ai.request.seed", + description="Seed from the input request", ) gen_ai_request_frequency_penalty: Optional[float] = Field( None, - alias='gen_ai.request.frequency_penalty', - description='Frequency penalty from the input request', + alias="gen_ai.request.frequency_penalty", + description="Frequency penalty from the input request", ) gen_ai_request_presence_penalty: Optional[float] = Field( None, - alias='gen_ai.request.presence_penalty', - description='Presence penalty from the input request', + alias="gen_ai.request.presence_penalty", + description="Presence penalty from the input request", ) gen_ai_request_tools: Optional[str] = Field( None, - alias='gen_ai.request.tools', - description='An array of tools from the input request json stringified', + alias="gen_ai.request.tools", + description="An array of tools from the input request json stringified", ) gen_ai_request_tool_results: Optional[str] = Field( None, - alias='gen_ai.request.tool_results', - description='An array of tool results from the input request json stringified', + alias="gen_ai.request.tool_results", + description="An array of tool results from the input request json stringified", ) diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/utils.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/utils.py index 1a824d463a..b7ada750a4 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/utils.py +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/utils.py @@ -56,7 +56,9 @@ def wrapper(*args, **kwargs): return func(*args, **kwargs) except Exception as exception: logger.warning( - "Failed to execute %s, error: %s", func.__name__, str(exception) + "Failed to execute %s, error: %s", + func.__name__, + str(exception), ) return wrapper diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/version.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/version.py index 021133daf1..bd9c51f813 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/version.py +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "0.0.1dev" +__version__ = "0.47b0" From 94c10f455efa7f79ced3da49a502b86d82f98c25 Mon Sep 17 00:00:00 2001 From: Ali Waleed <134522290+alizenhom@users.noreply.github.com> Date: Mon, 12 Aug 2024 10:26:35 +0300 Subject: [PATCH 03/38] fix wrong patch.py import --- .../src/opentelemetry/instrumentation/openai/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py index d30064b51e..2f4a721c7f 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py @@ -47,9 +47,7 @@ from opentelemetry.instrumentation.openai.package import _instruments from opentelemetry.trace import get_tracer from wrapt import wrap_function_wrapper -from langtrace_python_sdk.instrumentation.openai.patch import ( - chat_completions_create, -) +from .patch import chat_completions_create class OpenAIInstrumentor(BaseInstrumentor): From bb97ec917f91c40021bcb36408de9f6e90f25c76 Mon Sep 17 00:00:00 2001 From: Ali Waleed <134522290+alizenhom@users.noreply.github.com> Date: Mon, 12 Aug 2024 10:29:27 +0300 Subject: [PATCH 04/38] add missing dependecies tiktoken & pydantic --- .../opentelemetry-instrumentation-openai/README.rst | 2 +- .../opentelemetry-instrumentation-openai/pyproject.toml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/instrumentation/opentelemetry-instrumentation-openai/README.rst b/instrumentation/opentelemetry-instrumentation-openai/README.rst index 7cabed5b16..02e5922127 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/README.rst +++ b/instrumentation/opentelemetry-instrumentation-openai/README.rst @@ -1,5 +1,5 @@ OpenTelemetry OpenAI Instrumentation -=================================== +==================================== |pypi| diff --git a/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml b/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml index dd508f9414..4aeff65c23 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml +++ b/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml @@ -27,6 +27,9 @@ classifiers = [ dependencies = [ "opentelemetry-api ~= 1.12", "opentelemetry-instrumentation == 0.47b0", + "tiktoken>=0.1.1", + "pydantic>=1.8" + ] [project.optional-dependencies] From e15d4438bb4d122b23bde7d2a1872cfcd55e95cd Mon Sep 17 00:00:00 2001 From: Ali Waleed <134522290+alizenhom@users.noreply.github.com> Date: Mon, 12 Aug 2024 10:33:37 +0300 Subject: [PATCH 05/38] remove async support from `StreamWrapper` until further notice --- .../instrumentation/openai/patch.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py index c63bdab256..8d28defc87 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py @@ -366,13 +366,6 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): self.cleanup() - async def __aenter__(self): - self.setup() - return self - - async def __aexit__(self, exc_type, exc_val, exc_tb): - self.cleanup() - def __iter__(self): return self @@ -385,18 +378,6 @@ def __next__(self): self.cleanup() raise - def __aiter__(self): - return self - - async def __anext__(self): - try: - chunk = await self.stream.__anext__() - self.process_chunk(chunk) - return chunk - except StopAsyncIteration: - self.cleanup() - raise StopAsyncIteration - def process_chunk(self, chunk): if hasattr(chunk, "model") and chunk.model is not None: set_span_attribute( From 1efdfcd74b828808ce112d9780d19c0b759465bd Mon Sep 17 00:00:00 2001 From: Ali Waleed <134522290+alizenhom@users.noreply.github.com> Date: Tue, 13 Aug 2024 11:38:04 +0300 Subject: [PATCH 06/38] addressing comments: 1. clean up prompt calculations to be deduced from last streaming chunk 2. save correct span name 3. remove recording exceptions and setting status to ok 4. remove saving stream chunks in events --- .../instrumentation/openai/patch.py | 128 +++++------------- 1 file changed, 33 insertions(+), 95 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py index 8d28defc87..66105b7ae9 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py @@ -13,21 +13,19 @@ # limitations under the License. import json +from typing import Optional, Union from opentelemetry import trace from opentelemetry.trace import SpanKind, Span from opentelemetry.trace.status import Status, StatusCode from opentelemetry.trace.propagation import set_span_in_context -from openai._types import NOT_GIVEN -from span_attributes import SpanAttributes, LLMSpanAttributes, Event -from utils import ( - estimate_tokens, - silently_fail, - extract_content, - calculate_prompt_tokens, -) +from openai import NOT_GIVEN +from .span_attributes import LLMSpanAttributes, SpanAttributes +from .utils import silently_fail, extract_content +from opentelemetry.trace import Tracer -def chat_completions_create(original_method, version, tracer): + +def chat_completions_create(original_method, version, tracer: Tracer): """Wrap the `create` method of the `ChatCompletion` class to trace it.""" def traced_method(wrapped, instance, args, kwargs): @@ -69,8 +67,10 @@ def traced_method(wrapped, instance, args, kwargs): attributes = LLMSpanAttributes(**span_attributes) + span_name = f"{attributes.gen_ai_operation_name} {attributes.gen_ai_request_model}" + span = tracer.start_span( - "openai.completion", + name=span_name, kind=SpanKind.CLIENT, context=set_span_in_context(trace.get_current_span()), ) @@ -79,36 +79,18 @@ def traced_method(wrapped, instance, args, kwargs): try: result = wrapped(*args, **kwargs) if is_streaming(kwargs): - prompt_tokens = 0 - for message in kwargs.get("messages", {}): - prompt_tokens += calculate_prompt_tokens( - json.dumps(str(message)), kwargs.get("model") - ) - - if ( - kwargs.get("functions") is not None - and kwargs.get("functions") != NOT_GIVEN - ): - for function in kwargs.get("functions"): - prompt_tokens += calculate_prompt_tokens( - json.dumps(function), kwargs.get("model") - ) - return StreamWrapper( result, span, - prompt_tokens, function_call=kwargs.get("functions") is not None, tool_calls=kwargs.get("tools") is not None, ) else: _set_response_attributes(span, kwargs, result) - span.set_status(StatusCode.OK) span.end() return result except Exception as error: - span.record_exception(error) span.set_status(Status(StatusCode.ERROR, str(error))) span.end() raise @@ -118,21 +100,14 @@ def traced_method(wrapped, instance, args, kwargs): def get_tool_calls(item): if isinstance(item, dict): - if "tool_calls" in item and item["tool_calls"] is not None: - return item["tool_calls"] - return None - + return item.get("tool_calls") else: - if hasattr(item, "tool_calls") and item.tool_calls is not None: - return item.tool_calls - return None + return getattr(item, "tool_calls", None) @silently_fail -def _set_input_attributes(span, kwargs, attributes): +def _set_input_attributes(span, kwargs, attributes: LLMSpanAttributes): tools = [] - for field, value in attributes.model_dump(by_alias=True).items(): - set_span_attribute(span, field, value) if ( kwargs.get("functions") is not None @@ -149,6 +124,9 @@ def _set_input_attributes(span, kwargs, attributes): if tools: set_span_attribute(span, SpanAttributes.LLM_TOOLS, json.dumps(tools)) + for field, value in attributes.model_dump(by_alias=True).items(): + set_span_attribute(span, field, value) + @silently_fail def _set_response_attributes(span, kwargs, result): @@ -230,15 +208,6 @@ def set_event_completion(span: Span, result_content): ) -def set_event_completion_chunk(span: Span, chunk): - span.add_event( - name=SpanAttributes.LLM_CONTENT_COMPLETION_CHUNK, - attributes={ - SpanAttributes.LLM_CONTENT_COMPLETION_CHUNK: json.dumps(chunk), - }, - ) - - def set_span_attribute(span: Span, name, value): if value is not None: if value != "" or value != NOT_GIVEN: @@ -250,18 +219,18 @@ def set_span_attribute(span: Span, name, value): def is_streaming(kwargs): - return not ( - kwargs.get("stream") is False - or kwargs.get("stream") is None - or kwargs.get("stream") == NOT_GIVEN - ) + return non_numerical_value_is_set(kwargs.get("stream")) + + +def non_numerical_value_is_set(value: Optional[Union[bool, str]]): + return bool(value) and value != NOT_GIVEN def get_llm_request_attributes( kwargs, prompts=None, model=None, operation_name="chat" ): - user = kwargs.get("user", None) + user = kwargs.get("user") if prompts is None: prompts = ( [{"role": user or "user", "content": kwargs.get("prompt")}] @@ -269,14 +238,14 @@ def get_llm_request_attributes( else None ) top_k = ( - kwargs.get("n", None) - or kwargs.get("k", None) - or kwargs.get("top_k", None) - or kwargs.get("top_n", None) + kwargs.get("n") + or kwargs.get("k") + or kwargs.get("top_k") + or kwargs.get("top_n") ) - top_p = kwargs.get("p", None) or kwargs.get("top_p", None) - tools = kwargs.get("tools", None) + top_p = kwargs.get("p") or kwargs.get("top_p") + tools = kwargs.get("tools") return { SpanAttributes.LLM_OPERATION_NAME: operation_name, SpanAttributes.LLM_REQUEST_MODEL: model or kwargs.get("model"), @@ -308,7 +277,7 @@ def __init__( self, stream, span, - prompt_tokens, + prompt_tokens=None, function_call=False, tool_calls=False, ): @@ -324,12 +293,10 @@ def __init__( def setup(self): if not self._span_started: - self.span.add_event(Event.STREAM_START.value) self._span_started = True def cleanup(self): if self._span_started: - self.span.add_event(Event.STREAM_END.value) set_span_attribute( self.span, SpanAttributes.LLM_USAGE_PROMPT_TOKENS, @@ -391,8 +358,6 @@ def process_chunk(self, chunk): if not self.function_call and not self.tool_calls: for choice in chunk.choices: if choice.delta and choice.delta.content is not None: - token_counts = estimate_tokens(choice.delta.content) - self.completion_tokens += token_counts content = [choice.delta.content] elif self.function_call: for choice in chunk.choices: @@ -401,10 +366,6 @@ def process_chunk(self, chunk): and choice.delta.function_call is not None and choice.delta.function_call.arguments is not None ): - token_counts = estimate_tokens( - choice.delta.function_call.arguments - ) - self.completion_tokens += token_counts content = [choice.delta.function_call.arguments] elif self.tool_calls: for choice in chunk.choices: @@ -417,40 +378,17 @@ def process_chunk(self, chunk): and tool_call.function is not None and tool_call.function.arguments is not None ): - token_counts = estimate_tokens( - tool_call.function.arguments - ) - self.completion_tokens += token_counts content.append(tool_call.function.arguments) - set_event_completion_chunk( - self.span, - ( - "".join(content) - if len(content) > 0 and content[0] is not None - else "" - ), - ) + if content: self.result_content.append(content[0]) if hasattr(chunk, "text"): - token_counts = estimate_tokens(chunk.text) - self.completion_tokens += token_counts content = [chunk.text] - set_event_completion_chunk( - self.span, - ( - "".join(content) - if len(content) > 0 and content[0] is not None - else "" - ), - ) if content: self.result_content.append(content[0]) - if hasattr(chunk, "usage_metadata"): - self.completion_tokens = ( - chunk.usage_metadata.candidates_token_count - ) - self.prompt_tokens = chunk.usage_metadata.prompt_token_count + if getattr(chunk, "usage"): + self.completion_tokens = chunk.usage.completion_tokens + self.prompt_tokens = chunk.usage.prompt_tokens From d04edad521ca4864270faa3f29848be8c43e0a3c Mon Sep 17 00:00:00 2001 From: Ali Waleed Date: Thu, 5 Sep 2024 11:14:30 +0300 Subject: [PATCH 07/38] Refactoring Openai instrumentation --- .../pyproject.toml | 4 +- .../instrumentation/openai/__init__.py | 13 +- .../instrumentation/openai/patch.py | 235 ++++-------------- .../instrumentation/openai/span_attributes.py | 11 +- .../instrumentation/openai/utils.py | 181 ++++++++++---- .../instrumentation/openai/version.py | 2 +- .../pyproject.toml | 1 + 7 files changed, 195 insertions(+), 252 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml b/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml index 4aeff65c23..0172e2a15f 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml +++ b/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml @@ -26,8 +26,8 @@ classifiers = [ ] dependencies = [ "opentelemetry-api ~= 1.12", - "opentelemetry-instrumentation == 0.47b0", - "tiktoken>=0.1.1", + "opentelemetry-instrumentation == 0.48b0.dev", + "opentelemetry-semantic-conventions == 0.48b0.dev", "pydantic>=1.8" ] diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py index 2f4a721c7f..9e83463a21 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py @@ -46,7 +46,7 @@ from opentelemetry.instrumentation.instrumentor import BaseInstrumentor from opentelemetry.instrumentation.openai.package import _instruments from opentelemetry.trace import get_tracer -from wrapt import wrap_function_wrapper +from wrapt import wrap_function_wrapper as _W from .patch import chat_completions_create @@ -59,13 +59,10 @@ def _instrument(self, **kwargs): """Enable OpenAI instrumentation.""" tracer_provider = kwargs.get("tracer_provider") tracer = get_tracer(__name__, "", tracer_provider) - version = importlib.metadata.version("openai") - wrap_function_wrapper( - "openai.resources.chat.completions", - "Completions.create", - chat_completions_create( - "openai.chat.completions.create", version, tracer - ), + _W( + module="openai.resources.chat.completions", + name="Completions.create", + wrapper=chat_completions_create(tracer), ) def _uninstrument(self, **kwargs): diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py index 66105b7ae9..f0c574f848 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py @@ -12,54 +12,38 @@ # See the License for the specific language governing permissions and # limitations under the License. -import json -from typing import Optional, Union + from opentelemetry import trace from opentelemetry.trace import SpanKind, Span from opentelemetry.trace.status import Status, StatusCode from opentelemetry.trace.propagation import set_span_in_context -from openai import NOT_GIVEN -from .span_attributes import LLMSpanAttributes, SpanAttributes -from .utils import silently_fail, extract_content +from .span_attributes import LLMSpanAttributes, SpanAttributes +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAIAttributes, +) +from .utils import ( + silently_fail, + extract_content, + get_llm_request_attributes, + is_streaming, + set_span_attribute, + set_event_completion, + extract_tools_prompt, +) from opentelemetry.trace import Tracer -def chat_completions_create(original_method, version, tracer: Tracer): +def chat_completions_create(tracer: Tracer): """Wrap the `create` method of the `ChatCompletion` class to trace it.""" def traced_method(wrapped, instance, args, kwargs): + llm_prompts = [] + for item in kwargs.get("messages", []): - tools = get_tool_calls(item) - if tools is not None: - tool_calls = [] - for tool_call in tools: - tool_call_dict = { - "id": tool_call.id if hasattr(tool_call, "id") else "", - "type": ( - tool_call.type - if hasattr(tool_call, "type") - else "" - ), - } - if hasattr(tool_call, "function"): - tool_call_dict["function"] = { - "name": ( - tool_call.function.name - if hasattr(tool_call.function, "name") - else "" - ), - "arguments": ( - tool_call.function.arguments - if hasattr(tool_call.function, "arguments") - else "" - ), - } - tool_calls.append(tool_call_dict) - llm_prompts.append(tool_calls) - else: - llm_prompts.append(item) + tools_prompt = extract_tools_prompt(item) + llm_prompts.append(tools_prompt if tools_prompt else item) span_attributes = { **get_llm_request_attributes(kwargs, prompts=llm_prompts), @@ -74,7 +58,7 @@ def traced_method(wrapped, instance, args, kwargs): kind=SpanKind.CLIENT, context=set_span_in_context(trace.get_current_span()), ) - _set_input_attributes(span, kwargs, attributes) + _set_input_attributes(span, attributes) try: result = wrapped(*args, **kwargs) @@ -86,52 +70,31 @@ def traced_method(wrapped, instance, args, kwargs): tool_calls=kwargs.get("tools") is not None, ) else: - _set_response_attributes(span, kwargs, result) + _set_response_attributes(span, result) span.end() return result except Exception as error: span.set_status(Status(StatusCode.ERROR, str(error))) + span.set_attribute("error.type", error.__class__.__name__) span.end() raise return traced_method -def get_tool_calls(item): - if isinstance(item, dict): - return item.get("tool_calls") - else: - return getattr(item, "tool_calls", None) - - @silently_fail -def _set_input_attributes(span, kwargs, attributes: LLMSpanAttributes): - tools = [] - - if ( - kwargs.get("functions") is not None - and kwargs.get("functions") != NOT_GIVEN - ): - for function in kwargs.get("functions"): - tools.append( - json.dumps({"type": "function", "function": function}) - ) - - if kwargs.get("tools") is not None and kwargs.get("tools") != NOT_GIVEN: - tools.append(json.dumps(kwargs.get("tools"))) - - if tools: - set_span_attribute(span, SpanAttributes.LLM_TOOLS, json.dumps(tools)) - +def _set_input_attributes(span, attributes: LLMSpanAttributes): for field, value in attributes.model_dump(by_alias=True).items(): set_span_attribute(span, field, value) @silently_fail -def _set_response_attributes(span, kwargs, result): - set_span_attribute(span, SpanAttributes.LLM_RESPONSE_MODEL, result.model) - if hasattr(result, "choices") and result.choices is not None: +def _set_response_attributes(span, result): + set_span_attribute( + span, GenAIAttributes.GEN_AI_RESPONSE_MODEL, result.model + ) + if getattr(result, "choices", None): responses = [ { "role": ( @@ -154,120 +117,30 @@ def _set_response_attributes(span, kwargs, result): ] set_event_completion(span, responses) - if ( - hasattr(result, "system_fingerprint") - and result.system_fingerprint is not None - and result.system_fingerprint != NOT_GIVEN - ): + if getattr(result, "system_fingerprint", None): set_span_attribute( span, SpanAttributes.LLM_SYSTEM_FINGERPRINT, result.system_fingerprint, ) - # Get the usage - if hasattr(result, "usage") and result.usage is not None: - usage = result.usage - if usage is not None: - set_span_attribute( - span, - SpanAttributes.LLM_USAGE_PROMPT_TOKENS, - result.usage.prompt_tokens, - ) - set_span_attribute( - span, - SpanAttributes.LLM_USAGE_COMPLETION_TOKENS, - result.usage.completion_tokens, - ) - set_span_attribute( - span, - SpanAttributes.LLM_USAGE_TOTAL_TOKENS, - result.usage.total_tokens, - ) - - -def set_event_prompt(span: Span, prompt): - span.add_event( - name=SpanAttributes.LLM_CONTENT_PROMPT, - attributes={ - SpanAttributes.LLM_PROMPTS: prompt, - }, - ) - -def set_span_attributes(span: Span, attributes: dict): - for field, value in attributes.model_dump(by_alias=True).items(): - set_span_attribute(span, field, value) - - -def set_event_completion(span: Span, result_content): - span.add_event( - name=SpanAttributes.LLM_CONTENT_COMPLETION, - attributes={ - SpanAttributes.LLM_COMPLETIONS: json.dumps(result_content), - }, - ) - - -def set_span_attribute(span: Span, name, value): - if value is not None: - if value != "" or value != NOT_GIVEN: - if name == SpanAttributes.LLM_PROMPTS: - set_event_prompt(span, value) - else: - span.set_attribute(name, value) - return - - -def is_streaming(kwargs): - return non_numerical_value_is_set(kwargs.get("stream")) - - -def non_numerical_value_is_set(value: Optional[Union[bool, str]]): - return bool(value) and value != NOT_GIVEN - - -def get_llm_request_attributes( - kwargs, prompts=None, model=None, operation_name="chat" -): - - user = kwargs.get("user") - if prompts is None: - prompts = ( - [{"role": user or "user", "content": kwargs.get("prompt")}] - if "prompt" in kwargs - else None + # Get the usage + if getattr(result, "usage", None): + set_span_attribute( + span, + GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS, + result.usage.prompt_tokens, + ) + set_span_attribute( + span, + GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS, + result.usage.completion_tokens, + ) + set_span_attribute( + span, + "gen_ai.usage.total_tokens", + result.usage.total_tokens, ) - top_k = ( - kwargs.get("n") - or kwargs.get("k") - or kwargs.get("top_k") - or kwargs.get("top_n") - ) - - top_p = kwargs.get("p") or kwargs.get("top_p") - tools = kwargs.get("tools") - return { - SpanAttributes.LLM_OPERATION_NAME: operation_name, - SpanAttributes.LLM_REQUEST_MODEL: model or kwargs.get("model"), - SpanAttributes.LLM_IS_STREAMING: kwargs.get("stream"), - SpanAttributes.LLM_REQUEST_TEMPERATURE: kwargs.get("temperature"), - SpanAttributes.LLM_TOP_K: top_k, - SpanAttributes.LLM_PROMPTS: json.dumps(prompts) if prompts else None, - SpanAttributes.LLM_USER: user, - SpanAttributes.LLM_REQUEST_TOP_P: top_p, - SpanAttributes.LLM_REQUEST_MAX_TOKENS: kwargs.get("max_tokens"), - SpanAttributes.LLM_SYSTEM_FINGERPRINT: kwargs.get( - "system_fingerprint" - ), - SpanAttributes.LLM_PRESENCE_PENALTY: kwargs.get("presence_penalty"), - SpanAttributes.LLM_FREQUENCY_PENALTY: kwargs.get("frequency_penalty"), - SpanAttributes.LLM_REQUEST_SEED: kwargs.get("seed"), - SpanAttributes.LLM_TOOLS: json.dumps(tools) if tools else None, - SpanAttributes.LLM_TOOL_CHOICE: kwargs.get("tool_choice"), - SpanAttributes.LLM_REQUEST_LOGPROPS: kwargs.get("logprobs"), - SpanAttributes.LLM_REQUEST_LOGITBIAS: kwargs.get("logit_bias"), - SpanAttributes.LLM_REQUEST_TOP_LOGPROPS: kwargs.get("top_logprobs"), - } class StreamWrapper: @@ -277,7 +150,7 @@ def __init__( self, stream, span, - prompt_tokens=None, + prompt_tokens=0, function_call=False, tool_calls=False, ): @@ -299,17 +172,17 @@ def cleanup(self): if self._span_started: set_span_attribute( self.span, - SpanAttributes.LLM_USAGE_PROMPT_TOKENS, + GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS, self.prompt_tokens, ) set_span_attribute( self.span, - SpanAttributes.LLM_USAGE_COMPLETION_TOKENS, + GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS, self.completion_tokens, ) set_span_attribute( self.span, - SpanAttributes.LLM_USAGE_TOTAL_TOKENS, + "gen_ai.usage.total_tokens", self.prompt_tokens + self.completion_tokens, ) set_event_completion( @@ -346,14 +219,14 @@ def __next__(self): raise def process_chunk(self, chunk): - if hasattr(chunk, "model") and chunk.model is not None: + if getattr(chunk, "model", None): set_span_attribute( self.span, - SpanAttributes.LLM_RESPONSE_MODEL, + GenAIAttributes.GEN_AI_RESPONSE_MODEL, chunk.model, ) - if hasattr(chunk, "choices") and chunk.choices is not None: + if getattr(chunk, "choices", None): content = [] if not self.function_call and not self.tool_calls: for choice in chunk.choices: @@ -383,12 +256,12 @@ def process_chunk(self, chunk): if content: self.result_content.append(content[0]) - if hasattr(chunk, "text"): + if getattr(chunk, "text", None): content = [chunk.text] if content: self.result_content.append(content[0]) - if getattr(chunk, "usage"): + if getattr(chunk, "usage", None): self.completion_tokens = chunk.usage.completion_tokens self.prompt_tokens = chunk.usage.prompt_tokens diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/span_attributes.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/span_attributes.py index d85f149cc4..0edf2dd400 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/span_attributes.py +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/span_attributes.py @@ -14,17 +14,12 @@ from __future__ import annotations from enum import Enum -from typing import List, Optional +from typing import Dict, List, Optional from pydantic import BaseModel, ConfigDict, Field class SpanAttributes: - LLM_SYSTEM = "gen_ai.system" - LLM_OPERATION_NAME = "gen_ai.operation.name" - LLM_REQUEST_MODEL = "gen_ai.request.model" - LLM_REQUEST_MAX_TOKENS = "gen_ai.request.max_tokens" - LLM_REQUEST_TEMPERATURE = "gen_ai.request.temperature" - LLM_REQUEST_TOP_P = "gen_ai.request.top_p" + LLM_SYSTEM_FINGERPRINT = "gen_ai.system_fingerprint" LLM_REQUEST_DOCUMENTS = "gen_ai.request.documents" LLM_REQUEST_SEARCH_REQUIRED = "gen_ai.request.is_search_required" @@ -98,7 +93,7 @@ class LLMSpanAttributes(BaseModel): alias="gen_ai.request.temperature", description="Temperature value from the input request", ) - gen_ai_request_logit_bias: Optional[str] = Field( + gen_ai_request_logit_bias: Optional[Dict[str, int]] = Field( None, alias="gen_ai.request.logit_bias", description="Likelihood bias of the specified tokens the input request.", diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/utils.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/utils.py index b7ada750a4..2b3e35b0a2 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/utils.py +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/utils.py @@ -12,36 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import logging -from tiktoken import get_encoding +from typing import Optional, Union +from openai import NOT_GIVEN +from .span_attributes import SpanAttributes +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAIAttributes, +) - -def estimate_tokens_using_tiktoken(prompt, model): - """ - Estimate the number of tokens in a prompt using tiktoken.""" - encoding = get_encoding(model) - tokens = encoding.encode(prompt) - return len(tokens) - - -def estimate_tokens(prompt): - """ - Estimate the number of tokens in a prompt.""" - if prompt and len(prompt) > 0: - # Simplified token estimation: count the words. - return len([word for word in prompt.split() if word]) - return 0 - - -TIKTOKEN_MODEL_MAPPING = { - "gpt-4": "cl100k_base", - "gpt-4-32k": "cl100k_base", - "gpt-4-0125-preview": "cl100k_base", - "gpt-4-1106-preview": "cl100k_base", - "gpt-4-1106-vision-preview": "cl100k_base", - "gpt-4o": "0200k_base", - "gpt-4o-mini": "0200k_base", -} +from opentelemetry.trace import Span def silently_fail(func): @@ -65,20 +45,16 @@ def wrapper(*args, **kwargs): def extract_content(choice): + if getattr(choice, "message", None) is None: + return "" + # Check if choice.message exists and has a content attribute - if ( - hasattr(choice, "message") - and hasattr(choice.message, "content") - and choice.message.content is not None - ): + message = choice.message + if getattr(message, "content", None): return choice.message.content # Check if choice.message has tool_calls and extract information accordingly - elif ( - hasattr(choice, "message") - and hasattr(choice.message, "tool_calls") - and choice.message.tool_calls is not None - ): + elif getattr(message, "tool_calls", None): result = [ { "id": tool_call.id, @@ -93,11 +69,7 @@ def extract_content(choice): return result # Check if choice.message has a function_call and extract information accordingly - elif ( - hasattr(choice, "message") - and hasattr(choice.message, "function_call") - and choice.message.function_call is not None - ): + elif getattr(message, "function_call", None): return { "name": choice.message.function_call.name, "arguments": choice.message.function_call.arguments, @@ -108,12 +80,117 @@ def extract_content(choice): return "" -def calculate_prompt_tokens(prompt_content, model): - """ - Calculate the number of tokens in a prompt. If the model is supported by tiktoken, use it for the estimation. - """ - try: - tiktoken_model = TIKTOKEN_MODEL_MAPPING[model] - return estimate_tokens_using_tiktoken(prompt_content, tiktoken_model) - except Exception: - return estimate_tokens(prompt_content) # Fallback method +def extract_tools_prompt(item): + tool_calls = getattr(item, "tool_calls", None) + if tool_calls is None: + return + + calls = [] + for tool_call in tool_calls: + tool_call_dict = { + "id": getattr(tool_call, "id", ""), + "type": getattr(tool_call, "type", ""), + } + + if hasattr(tool_call, "function"): + tool_call_dict["function"] = { + "name": getattr(tool_call.function, "name", ""), + "arguments": getattr(tool_call.function, "arguments", ""), + } + calls.append(tool_call_dict) + return calls + + +def set_event_prompt(span: Span, prompt): + span.add_event( + name="gen_ai.content.prompt", + attributes={ + GenAIAttributes.GEN_AI_PROMPT: prompt, + }, + ) + + +def set_span_attributes(span: Span, attributes: dict): + for field, value in attributes.model_dump(by_alias=True).items(): + set_span_attribute(span, field, value) + + +def set_event_completion(span: Span, result_content): + span.add_event( + name="gen_ai.content.completion", + attributes={ + GenAIAttributes.GEN_AI_COMPLETION: json.dumps(result_content), + }, + ) + + +def set_span_attribute(span: Span, name, value): + if non_numerical_value_is_set(value) is False: + return + + if name == GenAIAttributes.GEN_AI_PROMPT: + set_event_prompt(span, value) + else: + span.set_attribute(name, value) + + +def is_streaming(kwargs): + return non_numerical_value_is_set(kwargs.get("stream")) + + +def non_numerical_value_is_set(value: Optional[Union[bool, str]]): + return bool(value) and value != NOT_GIVEN + + +def get_llm_request_attributes( + kwargs, + prompts=None, + model=None, + operation_name=GenAIAttributes.GenAiOperationNameValues.CHAT.value, +): + + user = kwargs.get("user") + if prompts is None: + prompts = ( + [{"role": user or "user", "content": kwargs.get("prompt")}] + if "prompt" in kwargs + else None + ) + top_k = ( + kwargs.get("n") + or kwargs.get("k") + or kwargs.get("top_k") + or kwargs.get("top_n") + ) + + top_p = kwargs.get("p") or kwargs.get("top_p") + tools = kwargs.get("tools") + + return { + GenAIAttributes.GEN_AI_OPERATION_NAME: operation_name, + GenAIAttributes.GEN_AI_SYSTEM: GenAIAttributes.GenAiSystemValues.OPENAI.value, + GenAIAttributes.GEN_AI_REQUEST_MODEL: model or kwargs.get("model"), + GenAIAttributes.GEN_AI_REQUEST_TEMPERATURE: kwargs.get("temperature"), + GenAIAttributes.GEN_AI_REQUEST_TOP_K: top_k, + GenAIAttributes.GEN_AI_PROMPT: ( + json.dumps(prompts) if prompts else None + ), + GenAIAttributes.GEN_AI_REQUEST_TOP_P: top_p, + GenAIAttributes.GEN_AI_REQUEST_MAX_TOKENS: kwargs.get("max_tokens"), + GenAIAttributes.GEN_AI_REQUEST_PRESENCE_PENALTY: kwargs.get( + "presence_penalty" + ), + GenAIAttributes.GEN_AI_REQUEST_FREQUENCY_PENALTY: kwargs.get( + "frequency_penalty" + ), + SpanAttributes.LLM_SYSTEM_FINGERPRINT: kwargs.get( + "system_fingerprint" + ), + SpanAttributes.LLM_IS_STREAMING: kwargs.get("stream"), + SpanAttributes.LLM_USER: user, + SpanAttributes.LLM_TOOLS: json.dumps(tools) if tools else None, + SpanAttributes.LLM_TOOL_CHOICE: kwargs.get("tool_choice"), + SpanAttributes.LLM_REQUEST_LOGPROPS: kwargs.get("logprobs"), + SpanAttributes.LLM_REQUEST_LOGITBIAS: kwargs.get("logit_bias"), + SpanAttributes.LLM_REQUEST_TOP_LOGPROPS: kwargs.get("top_logprobs"), + } diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/version.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/version.py index bd9c51f813..4d21fcb843 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/version.py +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "0.47b0" +__version__ = "0.48b0.dev" diff --git a/opentelemetry-contrib-instrumentations/pyproject.toml b/opentelemetry-contrib-instrumentations/pyproject.toml index 6c7716bad2..4bce5fe8c3 100644 --- a/opentelemetry-contrib-instrumentations/pyproject.toml +++ b/opentelemetry-contrib-instrumentations/pyproject.toml @@ -56,6 +56,7 @@ dependencies = [ "opentelemetry-instrumentation-logging==0.48b0.dev", "opentelemetry-instrumentation-mysql==0.48b0.dev", "opentelemetry-instrumentation-mysqlclient==0.48b0.dev", + "opentelemetry-instrumentation-openai==0.48b0.dev", "opentelemetry-instrumentation-pika==0.48b0.dev", "opentelemetry-instrumentation-psycopg==0.48b0.dev", "opentelemetry-instrumentation-psycopg2==0.48b0.dev", From ec3c32030dce0e6ca591a228f28ddad21b0c77b1 Mon Sep 17 00:00:00 2001 From: Ali Waleed Date: Thu, 5 Sep 2024 12:29:34 +0300 Subject: [PATCH 08/38] remove `SpanAttributes` and refactor streamwrapper --- .../instrumentation/openai/__init__.py | 6 +- .../instrumentation/openai/patch.py | 143 +++++++++++------- .../instrumentation/openai/span_attributes.py | 54 ------- .../instrumentation/openai/utils.py | 12 -- 4 files changed, 95 insertions(+), 120 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py index 9e83463a21..b256893993 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py @@ -40,13 +40,13 @@ --- """ -import importlib.metadata + from typing import Collection from opentelemetry.instrumentation.instrumentor import BaseInstrumentor from opentelemetry.instrumentation.openai.package import _instruments from opentelemetry.trace import get_tracer -from wrapt import wrap_function_wrapper as _W +from wrapt import wrap_function_wrapper from .patch import chat_completions_create @@ -59,7 +59,7 @@ def _instrument(self, **kwargs): """Enable OpenAI instrumentation.""" tracer_provider = kwargs.get("tracer_provider") tracer = get_tracer(__name__, "", tracer_provider) - _W( + wrap_function_wrapper( module="openai.resources.chat.completions", name="Completions.create", wrapper=chat_completions_create(tracer), diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py index f0c574f848..e9495a5f76 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py @@ -17,8 +17,7 @@ from opentelemetry.trace import SpanKind, Span from opentelemetry.trace.status import Status, StatusCode from opentelemetry.trace.propagation import set_span_in_context - -from .span_attributes import LLMSpanAttributes, SpanAttributes +from .span_attributes import LLMSpanAttributes from opentelemetry.semconv._incubating.attributes import ( gen_ai_attributes as GenAIAttributes, ) @@ -94,7 +93,9 @@ def _set_response_attributes(span, result): set_span_attribute( span, GenAIAttributes.GEN_AI_RESPONSE_MODEL, result.model ) + print(result) if getattr(result, "choices", None): + choices = result.choices responses = [ { "role": ( @@ -113,16 +114,19 @@ def _set_response_attributes(span, result): else {} ), } - for choice in result.choices + for choice in choices ] + for choice in choices: + if choice.finish_reason: + set_span_attribute( + span, + GenAIAttributes.GEN_AI_RESPONSE_FINISH_REASONS, + choice.finish_reason, + ) set_event_completion(span, responses) - if getattr(result, "system_fingerprint", None): - set_span_attribute( - span, - SpanAttributes.LLM_SYSTEM_FINGERPRINT, - result.system_fingerprint, - ) + if getattr(result, "id", None): + set_span_attribute(span, GenAIAttributes.GEN_AI_RESPONSE_ID, result.id) # Get the usage if getattr(result, "usage", None): @@ -145,6 +149,8 @@ def _set_response_attributes(span, result): class StreamWrapper: span: Span + response_id: str = "" + response_model: str = "" def __init__( self, @@ -170,6 +176,20 @@ def setup(self): def cleanup(self): if self._span_started: + if self.response_model: + set_span_attribute( + self.span, + GenAIAttributes.GEN_AI_RESPONSE_MODEL, + self.response_model, + ) + + if self.response_id: + set_span_attribute( + self.span, + GenAIAttributes.GEN_AI_RESPONSE_ID, + self.response_id, + ) + set_span_attribute( self.span, GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS, @@ -218,50 +238,71 @@ def __next__(self): self.cleanup() raise - def process_chunk(self, chunk): - if getattr(chunk, "model", None): - set_span_attribute( - self.span, - GenAIAttributes.GEN_AI_RESPONSE_MODEL, - chunk.model, - ) + def set_response_model(self, chunk): + if self.response_model: + return - if getattr(chunk, "choices", None): - content = [] - if not self.function_call and not self.tool_calls: - for choice in chunk.choices: - if choice.delta and choice.delta.content is not None: - content = [choice.delta.content] - elif self.function_call: - for choice in chunk.choices: - if ( - choice.delta - and choice.delta.function_call is not None - and choice.delta.function_call.arguments is not None - ): - content = [choice.delta.function_call.arguments] - elif self.tool_calls: - for choice in chunk.choices: - if choice.delta and choice.delta.tool_calls is not None: - toolcalls = choice.delta.tool_calls - content = [] - for tool_call in toolcalls: - if ( - tool_call - and tool_call.function is not None - and tool_call.function.arguments is not None - ): - content.append(tool_call.function.arguments) - - if content: - self.result_content.append(content[0]) - - if getattr(chunk, "text", None): - content = [chunk.text] - - if content: - self.result_content.append(content[0]) + if getattr(chunk, "model", None): + self.response_model = chunk.model + + def set_response_id(self, chunk): + if self.response_id: + return + + if getattr(chunk, "id", None): + self.response_id = chunk.id + + def build_streaming_response(self, chunk): + if getattr(chunk, "choices", None) is None: + return + + choices = chunk.choices + content = [] + if not self.function_call and not self.tool_calls: + for choice in choices: + if choice.delta and choice.delta.content is not None: + content = [choice.delta.content] + + elif self.function_call: + for choice in choices: + if ( + choice.delta + and choice.delta.function_call is not None + and choice.delta.function_call.arguments is not None + ): + content = [choice.delta.function_call.arguments] + + elif self.tool_calls: + for choice in choices: + if choice.delta and choice.delta.tool_calls is not None: + toolcalls = choice.delta.tool_calls + content = [] + for tool_call in toolcalls: + if ( + tool_call + and tool_call.function is not None + and tool_call.function.arguments is not None + ): + content.append(tool_call.function.arguments) + + for choice in choices: + finish_reason = choice.finish_reason + if finish_reason: + set_span_attribute( + self.span, + GenAIAttributes.GEN_AI_RESPONSE_FINISH_REASONS, + finish_reason, + ) + if content: + self.result_content.append(content[0]) + def set_usage(self, chunk): if getattr(chunk, "usage", None): self.completion_tokens = chunk.usage.completion_tokens self.prompt_tokens = chunk.usage.prompt_tokens + + def process_chunk(self, chunk): + self.set_response_id(chunk) + self.set_response_model(chunk) + self.build_streaming_response(chunk) + self.set_usage(chunk) diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/span_attributes.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/span_attributes.py index 0edf2dd400..1e0bdeeb10 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/span_attributes.py +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/span_attributes.py @@ -13,64 +13,10 @@ # limitations under the License. from __future__ import annotations -from enum import Enum from typing import Dict, List, Optional from pydantic import BaseModel, ConfigDict, Field -class SpanAttributes: - - LLM_SYSTEM_FINGERPRINT = "gen_ai.system_fingerprint" - LLM_REQUEST_DOCUMENTS = "gen_ai.request.documents" - LLM_REQUEST_SEARCH_REQUIRED = "gen_ai.request.is_search_required" - LLM_PROMPTS = "gen_ai.prompt" - LLM_CONTENT_PROMPT = "gen_ai.content.prompt" - LLM_COMPLETIONS = "gen_ai.completion" - LLM_CONTENT_COMPLETION = "gen_ai.content.completion" - LLM_RESPONSE_MODEL = "gen_ai.response.model" - LLM_USAGE_COMPLETION_TOKENS = "gen_ai.usage.output_tokens" - LLM_USAGE_PROMPT_TOKENS = "gen_ai.usage.input_tokens" - LLM_USAGE_TOTAL_TOKENS = "gen_ai.request.total_tokens" - LLM_USAGE_TOKEN_TYPE = "gen_ai.usage.token_type" - LLM_USAGE_SEARCH_UNITS = "gen_ai.usage.search_units" - LLM_GENERATION_ID = "gen_ai.generation_id" - LLM_TOKEN_TYPE = "gen_ai.token.type" - LLM_RESPONSE_ID = "gen_ai.response_id" - LLM_URL = "url.full" - LLM_PATH = "url.path" - LLM_RESPONSE_FORMAT = "gen_ai.response.format" - LLM_IMAGE_SIZE = "gen_ai.image.size" - LLM_REQUEST_ENCODING_FORMATS = "gen_ai.request.encoding_formats" - LLM_REQUEST_DIMENSIONS = "gen_ai.request.dimensions" - LLM_REQUEST_SEED = "gen_ai.request.seed" - LLM_REQUEST_TOP_LOGPROPS = "gen_ai.request.top_props" - LLM_REQUEST_LOGPROPS = "gen_ai.request.log_props" - LLM_REQUEST_LOGITBIAS = "gen_ai.request.logit_bias" - LLM_REQUEST_TYPE = "gen_ai.request.type" - LLM_HEADERS = "gen_ai.headers" - LLM_USER = "gen_ai.user" - LLM_TOOLS = "gen_ai.request.tools" - LLM_TOOL_CHOICE = "gen_ai.request.tool_choice" - LLM_TOOL_RESULTS = "gen_ai.request.tool_results" - LLM_TOP_K = "gen_ai.request.top_k" - LLM_IS_STREAMING = "gen_ai.request.stream" - LLM_FREQUENCY_PENALTY = "gen_ai.request.frequency_penalty" - LLM_PRESENCE_PENALTY = "gen_ai.request.presence_penalty" - LLM_CHAT_STOP_SEQUENCES = "gen_ai.chat.stop_sequences" - LLM_REQUEST_FUNCTIONS = "gen_ai.request.functions" - LLM_REQUEST_REPETITION_PENALTY = "gen_ai.request.repetition_penalty" - LLM_RESPONSE_FINISH_REASON = "gen_ai.response.finish_reasons" - LLM_RESPONSE_STOP_REASON = "gen_ai.response.stop_reason" - LLM_CONTENT_COMPLETION_CHUNK = "gen_ai.completion.chunk" - - -class Event(Enum): - STREAM_START = "stream.start" - STREAM_OUTPUT = "stream.output" - STREAM_END = "stream.end" - RESPONSE = "response" - - class LLMSpanAttributes(BaseModel): model_config = ConfigDict(extra="allow") gen_ai_operation_name: str = Field( diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/utils.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/utils.py index 2b3e35b0a2..4b02922c32 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/utils.py +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/utils.py @@ -16,7 +16,6 @@ import logging from typing import Optional, Union from openai import NOT_GIVEN -from .span_attributes import SpanAttributes from opentelemetry.semconv._incubating.attributes import ( gen_ai_attributes as GenAIAttributes, ) @@ -164,7 +163,6 @@ def get_llm_request_attributes( ) top_p = kwargs.get("p") or kwargs.get("top_p") - tools = kwargs.get("tools") return { GenAIAttributes.GEN_AI_OPERATION_NAME: operation_name, @@ -183,14 +181,4 @@ def get_llm_request_attributes( GenAIAttributes.GEN_AI_REQUEST_FREQUENCY_PENALTY: kwargs.get( "frequency_penalty" ), - SpanAttributes.LLM_SYSTEM_FINGERPRINT: kwargs.get( - "system_fingerprint" - ), - SpanAttributes.LLM_IS_STREAMING: kwargs.get("stream"), - SpanAttributes.LLM_USER: user, - SpanAttributes.LLM_TOOLS: json.dumps(tools) if tools else None, - SpanAttributes.LLM_TOOL_CHOICE: kwargs.get("tool_choice"), - SpanAttributes.LLM_REQUEST_LOGPROPS: kwargs.get("logprobs"), - SpanAttributes.LLM_REQUEST_LOGITBIAS: kwargs.get("logit_bias"), - SpanAttributes.LLM_REQUEST_TOP_LOGPROPS: kwargs.get("top_logprobs"), } From 706c6f21014d6e9ba7026191df9fd4f4a51ef01d Mon Sep 17 00:00:00 2001 From: Ali Waleed Date: Fri, 6 Sep 2024 15:00:02 +0300 Subject: [PATCH 09/38] change instrumentation name & fix some nits --- .../instrumentation/openai/__init__.py | 7 +++++- .../instrumentation/openai/patch.py | 25 ++++--------------- .../instrumentation/openai/version.py | 2 +- 3 files changed, 12 insertions(+), 22 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py index b256893993..737e4063e7 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py @@ -58,7 +58,12 @@ def instrumentation_dependencies(self) -> Collection[str]: def _instrument(self, **kwargs): """Enable OpenAI instrumentation.""" tracer_provider = kwargs.get("tracer_provider") - tracer = get_tracer(__name__, "", tracer_provider) + tracer = get_tracer( + __name__, + "", + tracer_provider, + schema_url="https://opentelemetry.io/schemas/1.27.0", + ) wrap_function_wrapper( module="openai.resources.chat.completions", name="Completions.create", diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py index e9495a5f76..1bef48be12 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py @@ -13,13 +13,12 @@ # limitations under the License. -from opentelemetry import trace from opentelemetry.trace import SpanKind, Span from opentelemetry.trace.status import Status, StatusCode -from opentelemetry.trace.propagation import set_span_in_context from .span_attributes import LLMSpanAttributes from opentelemetry.semconv._incubating.attributes import ( gen_ai_attributes as GenAIAttributes, + error_attributes as ErrorAttributes, ) from .utils import ( silently_fail, @@ -52,11 +51,7 @@ def traced_method(wrapped, instance, args, kwargs): span_name = f"{attributes.gen_ai_operation_name} {attributes.gen_ai_request_model}" - span = tracer.start_span( - name=span_name, - kind=SpanKind.CLIENT, - context=set_span_in_context(trace.get_current_span()), - ) + span = tracer.start_span(name=span_name, kind=SpanKind.CLIENT) _set_input_attributes(span, attributes) try: @@ -75,7 +70,9 @@ def traced_method(wrapped, instance, args, kwargs): except Exception as error: span.set_status(Status(StatusCode.ERROR, str(error))) - span.set_attribute("error.type", error.__class__.__name__) + span.set_attribute( + ErrorAttributes.ERROR_TYPE, type(error).__qualname__ + ) span.end() raise @@ -93,7 +90,6 @@ def _set_response_attributes(span, result): set_span_attribute( span, GenAIAttributes.GEN_AI_RESPONSE_MODEL, result.model ) - print(result) if getattr(result, "choices", None): choices = result.choices responses = [ @@ -140,11 +136,6 @@ def _set_response_attributes(span, result): GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS, result.usage.completion_tokens, ) - set_span_attribute( - span, - "gen_ai.usage.total_tokens", - result.usage.total_tokens, - ) class StreamWrapper: @@ -200,11 +191,6 @@ def cleanup(self): GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS, self.completion_tokens, ) - set_span_attribute( - self.span, - "gen_ai.usage.total_tokens", - self.prompt_tokens + self.completion_tokens, - ) set_event_completion( self.span, [ @@ -215,7 +201,6 @@ def cleanup(self): ], ) - self.span.set_status(StatusCode.OK) self.span.end() self._span_started = False diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/version.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/version.py index 4d21fcb843..ee5a6342e7 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/version.py +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "0.48b0.dev" +__version__ = "0.49b0.dev" From 71aaeb6e1224949c920574d01015f0f21ca6f51c Mon Sep 17 00:00:00 2001 From: Ali Waleed Date: Fri, 6 Sep 2024 15:01:56 +0300 Subject: [PATCH 10/38] change openai package name --- .../opentelemetry-instrumentation-openai/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml b/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml index 0172e2a15f..d54858623a 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml +++ b/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml @@ -3,7 +3,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [project] -name = "opentelemetry-instrumentation-openai" +name = "opentelemetry-instrumentation-openai-v2" dynamic = ["version"] description = "OpenTelemetry OpenAI instrumentation" readme = "README.rst" From 8495a24a4f3fcfc07a58fa0dc8571e72a9496ad9 Mon Sep 17 00:00:00 2001 From: Ali Waleed Date: Fri, 6 Sep 2024 15:55:20 +0300 Subject: [PATCH 11/38] cleanup setting prompt events & finish reasons --- .../instrumentation/openai/patch.py | 40 +++++++++++-------- .../instrumentation/openai/utils.py | 29 ++------------ 2 files changed, 26 insertions(+), 43 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py index 1bef48be12..9215a1b000 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py @@ -16,9 +16,11 @@ from opentelemetry.trace import SpanKind, Span from opentelemetry.trace.status import Status, StatusCode from .span_attributes import LLMSpanAttributes +from opentelemetry.semconv.attributes import ( + error_attributes as ErrorAttributes, +) from opentelemetry.semconv._incubating.attributes import ( gen_ai_attributes as GenAIAttributes, - error_attributes as ErrorAttributes, ) from .utils import ( silently_fail, @@ -28,8 +30,10 @@ set_span_attribute, set_event_completion, extract_tools_prompt, + set_event_prompt, ) from opentelemetry.trace import Tracer +import json def chat_completions_create(tracer: Tracer): @@ -43,9 +47,7 @@ def traced_method(wrapped, instance, args, kwargs): tools_prompt = extract_tools_prompt(item) llm_prompts.append(tools_prompt if tools_prompt else item) - span_attributes = { - **get_llm_request_attributes(kwargs, prompts=llm_prompts), - } + span_attributes = {**get_llm_request_attributes(kwargs)} attributes = LLMSpanAttributes(**span_attributes) @@ -53,6 +55,7 @@ def traced_method(wrapped, instance, args, kwargs): span = tracer.start_span(name=span_name, kind=SpanKind.CLIENT) _set_input_attributes(span, attributes) + set_event_prompt(span, json.dumps(llm_prompts)) try: result = wrapped(*args, **kwargs) @@ -112,13 +115,15 @@ def _set_response_attributes(span, result): } for choice in choices ] + finish_reasons = [] for choice in choices: - if choice.finish_reason: - set_span_attribute( - span, - GenAIAttributes.GEN_AI_RESPONSE_FINISH_REASONS, - choice.finish_reason, - ) + finish_reasons.append(choice.finish_reason or "error") + + set_span_attribute( + span, + GenAIAttributes.GEN_AI_RESPONSE_FINISH_REASONS, + finish_reasons, + ) set_event_completion(span, responses) if getattr(result, "id", None): @@ -270,14 +275,15 @@ def build_streaming_response(self, chunk): ): content.append(tool_call.function.arguments) + finish_reasons = [] for choice in choices: - finish_reason = choice.finish_reason - if finish_reason: - set_span_attribute( - self.span, - GenAIAttributes.GEN_AI_RESPONSE_FINISH_REASONS, - finish_reason, - ) + finish_reasons.append(choice.finish_reason or "error") + + set_span_attribute( + self.span, + GenAIAttributes.GEN_AI_RESPONSE_FINISH_REASONS, + finish_reasons, + ) if content: self.result_content.append(content[0]) diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/utils.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/utils.py index 4b02922c32..490e2c61e1 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/utils.py +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/utils.py @@ -127,10 +127,7 @@ def set_span_attribute(span: Span, name, value): if non_numerical_value_is_set(value) is False: return - if name == GenAIAttributes.GEN_AI_PROMPT: - set_event_prompt(span, value) - else: - span.set_attribute(name, value) + span.set_attribute(name, value) def is_streaming(kwargs): @@ -143,37 +140,17 @@ def non_numerical_value_is_set(value: Optional[Union[bool, str]]): def get_llm_request_attributes( kwargs, - prompts=None, model=None, operation_name=GenAIAttributes.GenAiOperationNameValues.CHAT.value, ): - user = kwargs.get("user") - if prompts is None: - prompts = ( - [{"role": user or "user", "content": kwargs.get("prompt")}] - if "prompt" in kwargs - else None - ) - top_k = ( - kwargs.get("n") - or kwargs.get("k") - or kwargs.get("top_k") - or kwargs.get("top_n") - ) - - top_p = kwargs.get("p") or kwargs.get("top_p") - return { GenAIAttributes.GEN_AI_OPERATION_NAME: operation_name, GenAIAttributes.GEN_AI_SYSTEM: GenAIAttributes.GenAiSystemValues.OPENAI.value, GenAIAttributes.GEN_AI_REQUEST_MODEL: model or kwargs.get("model"), GenAIAttributes.GEN_AI_REQUEST_TEMPERATURE: kwargs.get("temperature"), - GenAIAttributes.GEN_AI_REQUEST_TOP_K: top_k, - GenAIAttributes.GEN_AI_PROMPT: ( - json.dumps(prompts) if prompts else None - ), - GenAIAttributes.GEN_AI_REQUEST_TOP_P: top_p, + GenAIAttributes.GEN_AI_REQUEST_TOP_P: kwargs.get("p") + or kwargs.get("top_p"), GenAIAttributes.GEN_AI_REQUEST_MAX_TOKENS: kwargs.get("max_tokens"), GenAIAttributes.GEN_AI_REQUEST_PRESENCE_PENALTY: kwargs.get( "presence_penalty" From 885b7fd39412b179be35f23f1cf9a14a278496ef Mon Sep 17 00:00:00 2001 From: Ali Waleed Date: Fri, 6 Sep 2024 16:08:49 +0300 Subject: [PATCH 12/38] catch connection drops and reraise error in streaming --- .../instrumentation/openai/patch.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py index 9215a1b000..29e97c1355 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py @@ -214,7 +214,15 @@ def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): - self.cleanup() + try: + if exc_type is not None: + self.span.set_status(Status(StatusCode.ERROR, str(exc_val))) + self.span.set_attribute( + ErrorAttributes.ERROR_TYPE, exc_type.__name__ + ) + finally: + self.cleanup() + return False # Propagate the exception def __iter__(self): return self @@ -227,6 +235,13 @@ def __next__(self): except StopIteration: self.cleanup() raise + except Exception as error: + self.span.set_status(Status(StatusCode.ERROR, str(error))) + self.span.set_attribute( + ErrorAttributes.ERROR_TYPE, type(error).__qualname__ + ) + self.cleanup() + raise def set_response_model(self, chunk): if self.response_model: From 6ac04cbea1ab18c36c50080f43139c86eeb39327 Mon Sep 17 00:00:00 2001 From: Ali Waleed Date: Fri, 6 Sep 2024 17:59:58 +0300 Subject: [PATCH 13/38] run `tox -e generate` --- opentelemetry-contrib-instrumentations/pyproject.toml | 1 + .../src/opentelemetry/instrumentation/bootstrap_gen.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/opentelemetry-contrib-instrumentations/pyproject.toml b/opentelemetry-contrib-instrumentations/pyproject.toml index 9c40f9c2a3..22057ca2dd 100644 --- a/opentelemetry-contrib-instrumentations/pyproject.toml +++ b/opentelemetry-contrib-instrumentations/pyproject.toml @@ -56,6 +56,7 @@ dependencies = [ "opentelemetry-instrumentation-logging==0.49b0.dev", "opentelemetry-instrumentation-mysql==0.49b0.dev", "opentelemetry-instrumentation-mysqlclient==0.49b0.dev", + "opentelemetry-instrumentation-openai==0.49b0.dev", "opentelemetry-instrumentation-pika==0.49b0.dev", "opentelemetry-instrumentation-psycopg==0.49b0.dev", "opentelemetry-instrumentation-psycopg2==0.49b0.dev", diff --git a/opentelemetry-instrumentation/src/opentelemetry/instrumentation/bootstrap_gen.py b/opentelemetry-instrumentation/src/opentelemetry/instrumentation/bootstrap_gen.py index 8c4293f261..55561c0640 100644 --- a/opentelemetry-instrumentation/src/opentelemetry/instrumentation/bootstrap_gen.py +++ b/opentelemetry-instrumentation/src/opentelemetry/instrumentation/bootstrap_gen.py @@ -116,6 +116,10 @@ "library": "mysqlclient < 3", "instrumentation": "opentelemetry-instrumentation-mysqlclient==0.49b0.dev", }, + { + "library": "openai >= 0.27.0", + "instrumentation": "opentelemetry-instrumentation-openai-v2==0.49b0.dev", + }, { "library": "pika >= 0.12.0", "instrumentation": "opentelemetry-instrumentation-pika==0.49b0.dev", From 42370a7b4215e75138a4c806189367146cf302e3 Mon Sep 17 00:00:00 2001 From: Ali Waleed Date: Fri, 6 Sep 2024 18:05:07 +0300 Subject: [PATCH 14/38] run linter --- .../instrumentation/openai/__init__.py | 4 +++- .../instrumentation/openai/patch.py | 23 ++++++++++--------- .../instrumentation/openai/span_attributes.py | 2 ++ .../instrumentation/openai/utils.py | 3 ++- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py index 737e4063e7..86e826146e 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py @@ -43,10 +43,12 @@ from typing import Collection +from wrapt import wrap_function_wrapper + from opentelemetry.instrumentation.instrumentor import BaseInstrumentor from opentelemetry.instrumentation.openai.package import _instruments from opentelemetry.trace import get_tracer -from wrapt import wrap_function_wrapper + from .patch import chat_completions_create diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py index 29e97c1355..05ae685e4e 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py @@ -13,27 +13,28 @@ # limitations under the License. -from opentelemetry.trace import SpanKind, Span -from opentelemetry.trace.status import Status, StatusCode -from .span_attributes import LLMSpanAttributes -from opentelemetry.semconv.attributes import ( - error_attributes as ErrorAttributes, -) +import json + from opentelemetry.semconv._incubating.attributes import ( gen_ai_attributes as GenAIAttributes, ) +from opentelemetry.semconv.attributes import ( + error_attributes as ErrorAttributes, +) +from opentelemetry.trace import Span, SpanKind, Tracer +from opentelemetry.trace.status import Status, StatusCode + +from .span_attributes import LLMSpanAttributes from .utils import ( - silently_fail, extract_content, + extract_tools_prompt, get_llm_request_attributes, is_streaming, - set_span_attribute, set_event_completion, - extract_tools_prompt, set_event_prompt, + set_span_attribute, + silently_fail, ) -from opentelemetry.trace import Tracer -import json def chat_completions_create(tracer: Tracer): diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/span_attributes.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/span_attributes.py index 1e0bdeeb10..68512a8472 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/span_attributes.py +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/span_attributes.py @@ -13,7 +13,9 @@ # limitations under the License. from __future__ import annotations + from typing import Dict, List, Optional + from pydantic import BaseModel, ConfigDict, Field diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/utils.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/utils.py index 490e2c61e1..fcfe4d9665 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/utils.py +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/utils.py @@ -15,11 +15,12 @@ import json import logging from typing import Optional, Union + from openai import NOT_GIVEN + from opentelemetry.semconv._incubating.attributes import ( gen_ai_attributes as GenAIAttributes, ) - from opentelemetry.trace import Span From c5ef8c36effc005940dd11c79219e3b6029251e7 Mon Sep 17 00:00:00 2001 From: Ali Waleed Date: Fri, 6 Sep 2024 18:06:48 +0300 Subject: [PATCH 15/38] run `tox -e generate` --- instrumentation/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/instrumentation/README.md b/instrumentation/README.md index 22c4241bab..477bfd28d4 100644 --- a/instrumentation/README.md +++ b/instrumentation/README.md @@ -28,6 +28,7 @@ | [opentelemetry-instrumentation-logging](./opentelemetry-instrumentation-logging) | logging | No | experimental | [opentelemetry-instrumentation-mysql](./opentelemetry-instrumentation-mysql) | mysql-connector-python >= 8.0, < 10.0 | No | experimental | [opentelemetry-instrumentation-mysqlclient](./opentelemetry-instrumentation-mysqlclient) | mysqlclient < 3 | No | experimental +| [opentelemetry-instrumentation-openai](./opentelemetry-instrumentation-openai) | openai >= 0.27.0 | No | experimental | [opentelemetry-instrumentation-pika](./opentelemetry-instrumentation-pika) | pika >= 0.12.0 | No | experimental | [opentelemetry-instrumentation-psycopg](./opentelemetry-instrumentation-psycopg) | psycopg >= 3.1.0 | No | experimental | [opentelemetry-instrumentation-psycopg2](./opentelemetry-instrumentation-psycopg2) | psycopg2 >= 2.7.3.1 | No | experimental From d52460e4810625334043c2a67d23eab4af2569c6 Mon Sep 17 00:00:00 2001 From: Ali Waleed Date: Fri, 6 Sep 2024 20:05:03 +0300 Subject: [PATCH 16/38] add changelog --- CHANGELOG.md | 68 +++++++++++++++++++++++++++++++++++----------------- 1 file changed, 46 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 003f8054f6..84a450351e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +## Version 1.27.0/0.49b0 () + +### Added + +- `opentelemetry-instrumentation-openai` Instrumentation for OpenAI >= 0.27.0 + ([#2759](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/2759)) + +## Unreleased + ## Version 1.27.0/0.48b0 () ### Added @@ -206,7 +215,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#2420](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/2420)) - `opentelemetry-instrumentation-elasticsearch` Disabling instrumentation with native OTel support enabled ([#2524](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/2524)) -- `opentelemetry-instrumentation-asyncio` Check for __name__ attribute in the coroutine +- `opentelemetry-instrumentation-asyncio` Check for **name** attribute in the coroutine ([#2521](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/2521)) - `opentelemetry-instrumentation-requests` Fix wrong time unit for duration histogram ([#2553](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/2553)) @@ -221,6 +230,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#2146](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/2146)) ### Fixed + - `opentelemetry-instrumentation-celery` Allow Celery instrumentation to be installed multiple times ([#2342](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/2342)) - Align gRPC span status codes to OTEL specification @@ -238,8 +248,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - AwsLambdaInstrumentor sets `cloud.account.id` span attribute ([#2367](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/2367)) - ### Added + - `opentelemetry-instrumentation-fastapi` Add support for configuring header extraction via runtime constructor parameters ([#2241](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/2241)) @@ -250,7 +260,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `opentelemetry-resource-detector-azure` Added 10s timeout to VM Resource Detector ([#2119](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/2119)) - `opentelemetry-instrumentation-asyncpg` Allow AsyncPGInstrumentor to be instantiated multiple times -([#1791](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1791)) + ([#1791](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1791)) - `opentelemetry-instrumentation-confluent-kafka` Add support for higher versions until 2.3.0 of confluent_kafka ([#2132](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/2132)) - `opentelemetry-resource-detector-azure` Changed timeout to 4 seconds due to [timeout bug](https://github.com/open-telemetry/opentelemetry-python/issues/3644) @@ -334,6 +344,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#152](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/2013)) ## Version 1.19.0/0.40b0 (2023-07-13) + - `opentelemetry-instrumentation-asgi` Add `http.server.request.size` metric ([#1867](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1867)) @@ -380,6 +391,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#1823](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1823)) ### Added + - `opentelemetry-instrumentation-kafka-python` Add instrumentation to `consume` method ([#1786](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1786)) @@ -430,6 +442,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#1692](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1692)) ### Changed + - Update HTTP server/client instrumentation span names to comply with spec ([#1759](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1759)) @@ -467,7 +480,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Support `aio_pika` 9.x (([#1670](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1670]) -- `opentelemetry-instrumentation-redis` Add `sanitize_query` config option to allow query sanitization. ([#1572](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1572)) +- `opentelemetry-instrumentation-redis` Add `sanitize_query` config option to allow query sanitization. ([#1572](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1572)) - `opentelemetry-instrumentation-elasticsearch` Add optional db.statement query sanitization. ([#1598](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1598)) - `opentelemetry-instrumentation-celery` Record exceptions as events on the span. @@ -491,7 +504,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#1575](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1575)) - Fix SQLAlchemy uninstrumentation ([#1581](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1581)) -- `opentelemetry-instrumentation-grpc` Fix code()/details() of _OpentelemetryServicerContext. +- `opentelemetry-instrumentation-grpc` Fix code()/details() of \_OpentelemetryServicerContext. ([#1578](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1578)) - Fix aiopg instrumentation to work with aiopg < 2.0.0 ([#1473](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1473)) @@ -543,7 +556,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#1430](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1430)) - `opentelemetry-instrumentation-aiohttp-client` Allow overriding of status in response hook. ([#1394](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1394)) -- `opentelemetry-instrumentation-pymysql` Fix dbapi connection instrument wrapper has no _sock member. +- `opentelemetry-instrumentation-pymysql` Fix dbapi connection instrument wrapper has no \_sock member. ([#1424](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1424)) - `opentelemetry-instrumentation-dbapi` Fix the check for the connection already being instrumented in instrument_connection(). ([#1424](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1424)) @@ -628,7 +641,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add metric instrumentation in starlette ([#1327](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1327)) - ### Fixed - `opentelemetry-instrumentation-kafka-python`: wait for metadata @@ -641,7 +653,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#1208](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1208)) - `opentelemetry-instrumentation-aiohttp-client` Fix producing additional spans with each newly created ClientSession - ([#1246](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1246)) -- Add _is_opentelemetry_instrumented check in _InstrumentedFastAPI class +- Add \_is_opentelemetry_instrumented check in \_InstrumentedFastAPI class ([#1313](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1313)) - Fix uninstrumentation of existing app instances in FastAPI ([#1258](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1258)) @@ -660,6 +672,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#1203](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1203)) ### Added + - `opentelemetry-instrumentation-redis` add support to instrument RedisCluster clients ([#1177](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1177)) - `opentelemetry-instrumentation-sqlalchemy` Added span for the connection phase ([#1133](https://github.com/open-telemetry/opentelemetry-python-contrib/issues/1133)) @@ -672,11 +685,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [1.12.0rc2-0.32b0](https://github.com/open-telemetry/opentelemetry-python/releases/tag/v1.12.0rc2-0.32b0) - 2022-07-01 - - Pyramid: Only categorize 500s server exceptions as errors ([#1037](https://github.com/open-telemetry/opentelemetry-python-contrib/issues/1037)) ### Fixed + - Fix bug in system metrics by checking their configuration ([#1129](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1129)) - Adding escape call to fix [auto-instrumentation not producing spans on Windows](https://github.com/open-telemetry/opentelemetry-python/issues/2703). @@ -689,8 +702,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - fixed typo in `system.network.io` metric configuration ([#1135](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1135)) - ### Added + - `opentelemetry-instrumentation-aiohttp-client` Add support for optional custom trace_configs argument. ([1079](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1079)) - `opentelemetry-instrumentation-sqlalchemy` add support to instrument multiple engines @@ -714,10 +727,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Integrated sqlcommenter plugin into opentelemetry-instrumentation-django ([#896](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/896)) - ## Version 1.12.0rc1/0.31b0 (2022-05-17) ### Fixed + - `opentelemetry-instrumentation-aiohttp-client` make span attributes available to sampler ([#1072](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1072)) - `opentelemetry-instrumentation-aws-lambda` Fixed an issue - in some rare cases (API GW proxy integration test) @@ -730,6 +743,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `opentelemetry-sdk-extension-aws` change timeout for AWS EC2 and EKS metadata requests from 1000 seconds and 2000 seconds to 1 second ### Added + - `opentelemetry-instrument` and `opentelemetry-bootstrap` now include a `--version` flag ([#1065](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1065)) - `opentelemetry-instrumentation-redis` now instruments asynchronous Redis clients, if the installed redis-py includes async support (>=4.2.0). @@ -737,22 +751,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `opentelemetry-instrumentation-boto3sqs` added AWS's SQS instrumentation. ([#1081](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1081)) - ## Version 1.11.1/0.30b1 (2022-04-21) ### Added + - `opentelemetry-instrumentation-starlette` Capture custom request/response headers in span attributes ([#1046](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1046)) ### Fixed + - Prune autoinstrumentation sitecustomize module directory from PYTHONPATH immediately ([#1066](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1066)) - ## Version 1.11.0/0.30b0 (2022-04-18) ### Fixed -- `opentelemetry-instrumentation-pyramid` Fixed which package is the correct caller in _traced_init. + +- `opentelemetry-instrumentation-pyramid` Fixed which package is the correct caller in \_traced_init. ([#830](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/830)) - `opentelemetry-instrumentation-tornado` Fix Tornado errors mapping to 500 ([#1048](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1048)) @@ -786,7 +801,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `opentelemetry-instrumentation-pyramid` Pyramid: Capture custom request/response headers in span attributes ([#1022](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1022)) - ## Version 1.10.0/0.29b0 (2022-03-10) - `opentelemetry-instrumentation-wsgi` Capture custom request/response headers in span attributes @@ -800,7 +814,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `opentelemetry-instrumentation-aws-lambda` `SpanKind.SERVER` by default, add more cases for `SpanKind.CONSUMER` services. ([#926](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/926)) - `opentelemetry-instrumentation-sqlalchemy` added experimental sql commenter capability - ([#924](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/924)) + ([#924](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/924)) - `opentelemetry-contrib-instrumentations` added new meta-package that installs all contrib instrumentations. ([#681](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/681)) - `opentelemetry-instrumentation-dbapi` add experimental sql commenter capability @@ -839,12 +853,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Version 1.9.0/0.28b0 (2022-01-26) - ### Added - `opentelemetry-instrumentation-pyramid` Pyramid: Conditionally create SERVER spans ([#869](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/869)) -- `opentelemetry-instrumentation-grpc` added `trailing_metadata` to _OpenTelemetryServicerContext. +- `opentelemetry-instrumentation-grpc` added `trailing_metadata` to \_OpenTelemetryServicerContext. ([#871](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/871)) - `opentelemetry-instrumentation-asgi` now returns a `traceresponse` response header. ([#817](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/817)) @@ -878,12 +891,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `opentelemetry-instrumentation-aiohttp-client` aiohttp: Remove `span_name` from docs ([#857](https://github.com/open-telemetry/opentelemetry-python-contrib/issues/857)) - ## Version 1.8.0/0.27b0 (2021-12-17) ### Added -- `opentelemetry-instrumentation-aws-lambda` Adds support for configurable flush timeout via `OTEL_INSTRUMENTATION_AWS_LAMBDA_FLUSH_TIMEOUT` property. ([#825](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/825)) +- `opentelemetry-instrumentation-aws-lambda` Adds support for configurable flush timeout via `OTEL_INSTRUMENTATION_AWS_LAMBDA_FLUSH_TIMEOUT` property. ([#825](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/825)) - `opentelemetry-instrumentation-pika` Adds support for versions between `0.12.0` to `1.0.0`. ([#837](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/837)) ### Fixed @@ -953,13 +965,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#755](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/755)) ### Added + - `opentelemetry-instrumentation-pika` Add `publish_hook` and `consume_hook` callbacks passed as arguments to the instrument method ([#763](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/763)) - ## Version 1.6.1/0.25b1 (2021-10-18) ### Changed + - `opentelemetry-util-http` no longer contains an instrumentation entrypoint and will not be loaded automatically by the auto instrumentor. ([#745](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/745)) @@ -973,7 +986,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#760](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/760)) ## Version 1.6.0/0.25b0 (2021-10-13) + ### Added + - `opentelemetry-sdk-extension-aws` Release AWS Python SDK Extension as 1.0.0 ([#667](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/667)) - `opentelemetry-instrumentation-urllib3`, `opentelemetry-instrumentation-requests` @@ -1000,6 +1015,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#391](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/391)) ### Changed + - `opentelemetry-instrumentation-flask` Fix `RuntimeError: Working outside of request context` ([#734](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/734)) - `opentelemetry-propagators-aws-xray` Rename `AwsXRayFormat` to `AwsXRayPropagator` @@ -1030,6 +1046,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Version 1.5.0/0.24b0 (2021-08-26) ### Added + - `opentelemetry-sdk-extension-aws` Add AWS resource detectors to extension package ([#586](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/586)) - `opentelemetry-instrumentation-asgi`, `opentelemetry-instrumentation-aiohttp-client`, `openetelemetry-instrumentation-fastapi`, @@ -1048,10 +1065,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Version 1.4.0/0.23b0 (2021-07-21) ### Removed + - Move `opentelemetry-instrumentation` to the core repo. ([#595](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/595)) ### Changed + - `opentelemetry-instrumentation-falcon` added support for Falcon 3. ([#607](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/607)) - `opentelemetry-instrumentation-tornado` properly instrument work done in tornado on_finish method. @@ -1099,12 +1118,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#568](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/568)) ### Added + - `opentelemetry-instrumentation-httpx` Add `httpx` instrumentation ([#461](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/461)) ## Version 1.3.0/0.22b0 (2021-06-01) ### Changed + - `opentelemetry-bootstrap` not longer forcibly removes and re-installs libraries and their instrumentations. This means running bootstrap will not auto-upgrade existing dependencies and as a result not cause dependency conflicts. @@ -1121,6 +1142,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#488](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/488)) ### Added + - `opentelemetry-instrumentation-botocore` now supports context propagation for lambda invoke via Payload embedded headers. ([#458](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/458)) @@ -1130,6 +1152,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Version 1.2.0/0.21b0 (2021-05-11) ### Changed + - Instrumentation packages don't specify the libraries they instrument as dependencies anymore. Instead, they verify the correct version of libraries are installed at runtime. ([#475](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/475)) @@ -1681,6 +1704,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `opentelemetry-ext-http-requests` Updates for core library changes - `Added support for PyPy3` Initial release + ## [#1033](https://github.com/open-telemetryopentelemetry-python-contrib/issues/1033) ## Version 0.1a0 (2019-09-30) @@ -1695,7 +1719,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `opentelemetry-resource-detector-azure` Added 10s timeout to VM Resource Detector ([#2119](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/2119)) - `opentelemetry-instrumentation-asyncpg` Allow AsyncPGInstrumentor to be instantiated multiple times -([#1791](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1791)) + ([#1791](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1791)) - `opentelemetry-instrumentation-confluent-kafka` Add support for higher versions until 2.3.0 of confluent_kafka ([#2132](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/2132)) - `opentelemetry-resource-detector-azure` Changed timeout to 4 seconds due to [timeout bug](https://github.com/open-telemetry/opentelemetry-python/issues/3644) From 452d41a0cc11e63eb3b042abc19d52dbce9ebec6 Mon Sep 17 00:00:00 2001 From: Ali Waleed Date: Mon, 9 Sep 2024 13:50:53 +0300 Subject: [PATCH 17/38] test requirments + tox ini --- .../test-requirements.txt | 6 ++++-- tox.ini | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-openai/test-requirements.txt b/instrumentation/opentelemetry-instrumentation-openai/test-requirements.txt index 6b3d9ef39d..bca6bfbd0e 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/test-requirements.txt +++ b/instrumentation/opentelemetry-instrumentation-openai/test-requirements.txt @@ -3,6 +3,8 @@ Deprecated==1.2.14 importlib-metadata==6.11.0 packaging==24.0 pytest==7.4.4 +pytest-vcr==1.0.2 wrapt==1.16.0 -# -e opentelemetry-instrumentation -# -e instrumentation/opentelemetry-instrumentation-openai + +-e opentelemetry-instrumentation +-e instrumentation/opentelemetry-instrumentation-openai diff --git a/tox.ini b/tox.ini index 600929a6ec..02066afc19 100644 --- a/tox.ini +++ b/tox.ini @@ -6,6 +6,11 @@ envlist = ; Environments are organized by individual package, allowing ; for specifying supported Python versions per package. + ; instrumentation-openai + py3{8,9,10,11,12}-test-instrumentation-openai + pypy3-test-instrumentation-openai + lint-instrumentation-openai + ; opentelemetry-resource-detector-container py3{8,9,10,11,12}-test-resource-detector-container pypy3-test-resource-detector-container @@ -399,6 +404,12 @@ commands_pre = opentelemetry-instrumentation: pip install opentelemetry-test-utils@{env:CORE_REPO}\#egg=opentelemetry-test-utils&subdirectory=tests/opentelemetry-test-utils opentelemetry-instrumentation: pip install -r {toxinidir}/opentelemetry-instrumentation/test-requirements.txt + openai: pip install opentelemetry-api@{env:CORE_REPO}\#egg=opentelemetry-api&subdirectory=opentelemetry-api + openai: pip install opentelemetry-semantic-conventions@{env:CORE_REPO}\#egg=opentelemetry-semantic-conventions&subdirectory=opentelemetry-semantic-conventions + openai: pip install opentelemetry-sdk@{env:CORE_REPO}\#egg=opentelemetry-sdk&subdirectory=opentelemetry-sdk + openai: pip install opentelemetry-test-utils@{env:CORE_REPO}\#egg=opentelemetry-test-utils&subdirectory=tests/opentelemetry-test-utils + openai: pip install -r {toxinidir}/instrumentation/opentelemetry-instrumentation-openai/test-requirements.txt + distro: pip install opentelemetry-api@{env:CORE_REPO}\#egg=opentelemetry-api&subdirectory=opentelemetry-api distro: pip install opentelemetry-semantic-conventions@{env:CORE_REPO}\#egg=opentelemetry-semantic-conventions&subdirectory=opentelemetry-semantic-conventions distro: pip install opentelemetry-sdk@{env:CORE_REPO}\#egg=opentelemetry-sdk&subdirectory=opentelemetry-sdk @@ -445,6 +456,11 @@ commands_pre = kafka-pythonng: pip install opentelemetry-sdk@{env:CORE_REPO}\#egg=opentelemetry-sdk&subdirectory=opentelemetry-sdk kafka-pythonng: pip install -r {toxinidir}/instrumentation/opentelemetry-instrumentation-kafka-python/test-requirements-ng.txt + openai: pip install opentelemetry-api@{env:CORE_REPO}\#egg=opentelemetry-api&subdirectory=opentelemetry-api + openai: pip install opentelemetry-sdk@{env:CORE_REPO}\#egg=opentelemetry-sdk&subdirectory=opentelemetry-sdk + openai: pip install opentelemetry-semantic-conventions@{env:CORE_REPO}\#egg=opentelemetry-semantic-conventions&subdirectory=opentelemetry-semantic-conventions + openai: pip install -r {toxinidir}/instrumentation/opentelemetry-instrumentation-openai/test-requirements.txt + confluent-kafka: pip install opentelemetry-api@{env:CORE_REPO}\#egg=opentelemetry-api&subdirectory=opentelemetry-api confluent-kafka: pip install opentelemetry-semantic-conventions@{env:CORE_REPO}\#egg=opentelemetry-semantic-conventions&subdirectory=opentelemetry-semantic-conventions confluent-kafka: pip install opentelemetry-sdk@{env:CORE_REPO}\#egg=opentelemetry-sdk&subdirectory=opentelemetry-sdk From e9a76c4708fc6d4de160b0bd6c5049a5608409e6 Mon Sep 17 00:00:00 2001 From: Ali Waleed Date: Wed, 25 Sep 2024 12:59:15 +0300 Subject: [PATCH 18/38] remove LLMSpanAttributes validation layer --- .../pyproject.toml | 3 +- .../instrumentation/openai/__init__.py | 3 +- .../instrumentation/openai/patch.py | 14 +- .../instrumentation/openai/span_attributes.py | 161 ------------------ 4 files changed, 8 insertions(+), 173 deletions(-) delete mode 100644 instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/span_attributes.py diff --git a/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml b/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml index d54858623a..ed3e1f53f7 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml +++ b/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml @@ -27,8 +27,7 @@ classifiers = [ dependencies = [ "opentelemetry-api ~= 1.12", "opentelemetry-instrumentation == 0.48b0.dev", - "opentelemetry-semantic-conventions == 0.48b0.dev", - "pydantic>=1.8" + "opentelemetry-semantic-conventions == 0.48b0.dev" ] diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py index 86e826146e..439f1e184c 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py @@ -47,6 +47,7 @@ from opentelemetry.instrumentation.instrumentor import BaseInstrumentor from opentelemetry.instrumentation.openai.package import _instruments +from opentelemetry.semconv.schemas import Schemas from opentelemetry.trace import get_tracer from .patch import chat_completions_create @@ -64,7 +65,7 @@ def _instrument(self, **kwargs): __name__, "", tracer_provider, - schema_url="https://opentelemetry.io/schemas/1.27.0", + schema_url=Schemas.V1_27_0, ) wrap_function_wrapper( module="openai.resources.chat.completions", diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py index 05ae685e4e..ff7e3bbe01 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py @@ -24,7 +24,6 @@ from opentelemetry.trace import Span, SpanKind, Tracer from opentelemetry.trace.status import Status, StatusCode -from .span_attributes import LLMSpanAttributes from .utils import ( extract_content, extract_tools_prompt, @@ -49,13 +48,10 @@ def traced_method(wrapped, instance, args, kwargs): llm_prompts.append(tools_prompt if tools_prompt else item) span_attributes = {**get_llm_request_attributes(kwargs)} - - attributes = LLMSpanAttributes(**span_attributes) - - span_name = f"{attributes.gen_ai_operation_name} {attributes.gen_ai_request_model}" + span_name = f"{span_attributes[GenAIAttributes.GEN_AI_OPERATION_NAME]} {span_attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL]}" span = tracer.start_span(name=span_name, kind=SpanKind.CLIENT) - _set_input_attributes(span, attributes) + _set_input_attributes(span, span_attributes) set_event_prompt(span, json.dumps(llm_prompts)) try: @@ -84,8 +80,8 @@ def traced_method(wrapped, instance, args, kwargs): @silently_fail -def _set_input_attributes(span, attributes: LLMSpanAttributes): - for field, value in attributes.model_dump(by_alias=True).items(): +def _set_input_attributes(span, attributes): + for field, value in attributes.items(): set_span_attribute(span, field, value) @@ -219,7 +215,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): if exc_type is not None: self.span.set_status(Status(StatusCode.ERROR, str(exc_val))) self.span.set_attribute( - ErrorAttributes.ERROR_TYPE, exc_type.__name__ + ErrorAttributes.ERROR_TYPE, exc_type.__qualname__ ) finally: self.cleanup() diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/span_attributes.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/span_attributes.py deleted file mode 100644 index 68512a8472..0000000000 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/span_attributes.py +++ /dev/null @@ -1,161 +0,0 @@ -# Copyright The OpenTelemetry Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import annotations - -from typing import Dict, List, Optional - -from pydantic import BaseModel, ConfigDict, Field - - -class LLMSpanAttributes(BaseModel): - model_config = ConfigDict(extra="allow") - gen_ai_operation_name: str = Field( - ..., - alias="gen_ai.operation.name", - description="The name of the operation being performed.", - ) - gen_ai_request_model: str = Field( - ..., - alias="gen_ai.request.model", - description="Model name from the input request", - ) - gen_ai_response_model: Optional[str] = Field( - None, - alias="gen_ai.response.model", - description="Model name from the response", - ) - gen_ai_request_temperature: Optional[float] = Field( - None, - alias="gen_ai.request.temperature", - description="Temperature value from the input request", - ) - gen_ai_request_logit_bias: Optional[Dict[str, int]] = Field( - None, - alias="gen_ai.request.logit_bias", - description="Likelihood bias of the specified tokens the input request.", - ) - gen_ai_request_logprobs: Optional[bool] = Field( - None, - alias="gen_ai.request.logprobs", - description="Logprobs flag returns log probabilities.", - ) - gen_ai_request_top_logprobs: Optional[float] = Field( - None, - alias="gen_ai.request.top_logprobs", - description="Integer between 0 and 5 specifying the number of most likely tokens to return.", - ) - gen_ai_request_top_p: Optional[float] = Field( - None, - alias="gen_ai.request.top_p", - description="Top P value from the input request", - ) - gen_ai_request_top_k: Optional[float] = Field( - None, - alias="gen_ai.request.top_k", - description="Top K results to return from the input request", - ) - gen_ai_user: Optional[str] = Field( - None, alias="gen_ai.user", description="User ID from the input request" - ) - gen_ai_prompt: Optional[str] = Field( - None, - alias="gen_ai.prompt", - description="Prompt text from the input request", - ) - gen_ai_completion: Optional[str] = Field( - None, - alias="gen_ai.completion", - description='Completion text from the response. This will be an array of json objects with the following format {"role": "", "content": ""}. Role can be one of the following values: [system, user, assistant, tool]', - ) - gen_ai_request_stream: Optional[bool] = Field( - None, - alias="gen_ai.request.stream", - description="Stream flag from the input request", - ) - gen_ai_request_encoding_formats: Optional[List[str]] = Field( - None, - alias="gen_ai.request.encoding_formats", - description="Encoding formats from the input request. Allowed values: ['float', 'int8','uint8', 'binary', 'ubinary', 'base64']", - ) - gen_ai_completion_chunk: Optional[str] = Field( - None, - alias="gen_ai.completion.chunk", - description="Chunk text from the response", - ) - gen_ai_response_finish_reasons: Optional[List[str]] = Field( - None, - alias="gen_ai.response.finish_reasons", - description="Array of reasons the model stopped generating tokens, corresponding to each generation received", - ) - gen_ai_system_fingerprint: Optional[str] = Field( - None, - alias="gen_ai.system_fingerprint", - description="System fingerprint of the system that generated the response", - ) - gen_ai_request_tool_choice: Optional[str] = Field( - None, - alias="gen_ai.request.tool_choice", - description="Tool choice from the input request", - ) - gen_ai_response_tool_calls: Optional[str] = Field( - None, - alias="gen_ai.response.tool_calls", - description="Array of tool calls from the response json stringified", - ) - gen_ai_request_max_tokens: Optional[float] = Field( - None, - alias="gen_ai.request.max_tokens", - description="The maximum number of tokens the LLM generates for a request.", - ) - gen_ai_usage_input_tokens: Optional[float] = Field( - None, - alias="gen_ai.usage.input_tokens", - description="The number of tokens used in the llm prompt.", - ) - gen_ai_usage_total_tokens: Optional[float] = Field( - None, - alias="gen_ai.usage.total_tokens", - description="The total number of tokens used in the llm request.", - ) - gen_ai_usage_output_tokens: Optional[float] = Field( - None, - alias="gen_ai.usage.output_tokens", - description="The number of tokens in the llm response.", - ) - gen_ai_request_seed: Optional[str] = Field( - None, - alias="gen_ai.request.seed", - description="Seed from the input request", - ) - gen_ai_request_frequency_penalty: Optional[float] = Field( - None, - alias="gen_ai.request.frequency_penalty", - description="Frequency penalty from the input request", - ) - gen_ai_request_presence_penalty: Optional[float] = Field( - None, - alias="gen_ai.request.presence_penalty", - description="Presence penalty from the input request", - ) - gen_ai_request_tools: Optional[str] = Field( - None, - alias="gen_ai.request.tools", - description="An array of tools from the input request json stringified", - ) - gen_ai_request_tool_results: Optional[str] = Field( - None, - alias="gen_ai.request.tool_results", - description="An array of tool results from the input request json stringified", - ) From 3d5a2b34e8fbf0ceafff30b56424f7d84431bc43 Mon Sep 17 00:00:00 2001 From: Ali Waleed Date: Wed, 25 Sep 2024 14:26:20 +0300 Subject: [PATCH 19/38] add tests --- .../README.rst | 2 +- .../pyproject.toml | 4 +- .../{openai => openai_v2}/__init__.py | 2 +- .../{openai => openai_v2}/package.py | 0 .../{openai => openai_v2}/patch.py | 0 .../{openai => openai_v2}/utils.py | 0 .../{openai => openai_v2}/version.py | 0 .../tests/cassettes/test_chat_completion.yaml | 97 +++++++++++ .../test_chat_completion_streaming.yaml | 157 +++++++++++++++++ .../tests/conftest.py | 52 ++++++ .../tests/test_chat_completions.py | 162 ++++++++++++++++++ 11 files changed, 472 insertions(+), 4 deletions(-) rename instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/{openai => openai_v2}/__init__.py (96%) rename instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/{openai => openai_v2}/package.py (100%) rename instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/{openai => openai_v2}/patch.py (100%) rename instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/{openai => openai_v2}/utils.py (100%) rename instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/{openai => openai_v2}/version.py (100%) create mode 100644 instrumentation/opentelemetry-instrumentation-openai/tests/cassettes/test_chat_completion.yaml create mode 100644 instrumentation/opentelemetry-instrumentation-openai/tests/cassettes/test_chat_completion_streaming.yaml create mode 100644 instrumentation/opentelemetry-instrumentation-openai/tests/conftest.py create mode 100644 instrumentation/opentelemetry-instrumentation-openai/tests/test_chat_completions.py diff --git a/instrumentation/opentelemetry-instrumentation-openai/README.rst b/instrumentation/opentelemetry-instrumentation-openai/README.rst index 02e5922127..d3f2873622 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/README.rst +++ b/instrumentation/opentelemetry-instrumentation-openai/README.rst @@ -15,7 +15,7 @@ Installation :: - pip install opentelemetry-instrumentation-openai + pip install opentelemetry-instrumentation-openai-v2 References diff --git a/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml b/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml index ed3e1f53f7..9f08f11027 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml +++ b/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml @@ -37,13 +37,13 @@ instruments = [ ] [project.entry-points.opentelemetry_instrumentor] -openai = "opentelemetry.instrumentation.openai:OpenAIInstrumentor" +openai = "opentelemetry.instrumentation.openai_v2:OpenAIInstrumentor" [project.urls] Homepage = "https://github.com/open-telemetry/opentelemetry-python-contrib/tree/main/instrumentation/opentelemetry-instrumentation-openai" [tool.hatch.version] -path = "src/opentelemetry/instrumentation/openai/version.py" +path = "src/opentelemetry/instrumentation/openai_v2/version.py" [tool.hatch.build.targets.sdist] include = [ diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/__init__.py similarity index 96% rename from instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py rename to instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/__init__.py index 439f1e184c..85ab2fd4fd 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/__init__.py @@ -46,7 +46,7 @@ from wrapt import wrap_function_wrapper from opentelemetry.instrumentation.instrumentor import BaseInstrumentor -from opentelemetry.instrumentation.openai.package import _instruments +from opentelemetry.instrumentation.openai_v2.package import _instruments from opentelemetry.semconv.schemas import Schemas from opentelemetry.trace import get_tracer diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/package.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/package.py similarity index 100% rename from instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/package.py rename to instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/package.py diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/patch.py similarity index 100% rename from instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/patch.py rename to instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/patch.py diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/utils.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/utils.py similarity index 100% rename from instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/utils.py rename to instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/utils.py diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/version.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/version.py similarity index 100% rename from instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai/version.py rename to instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/version.py diff --git a/instrumentation/opentelemetry-instrumentation-openai/tests/cassettes/test_chat_completion.yaml b/instrumentation/opentelemetry-instrumentation-openai/tests/cassettes/test_chat_completion.yaml new file mode 100644 index 0000000000..34c5f81bef --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-openai/tests/cassettes/test_chat_completion.yaml @@ -0,0 +1,97 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": "Say this is a test three times"}], + "model": "gpt-4", "stream": false}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '112' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.47.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.47.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.5 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//hJBBS8NAEIXv+RXDntuSpLXV3BQPIihSxItI2CbTZO1md81MoFr632W3 + MakH8RLI+/Y93rxDBCBUKTIQRS25aJyeXt/cN+r26bH72q9f4v0Dzuu79a78KPQllWLiHXbzjgX/ + uGaFbZxGVtaccNGiZPSpySpdpcskThcBNLZE7W2V4+liGi+Tee+orSqQRAavEQDAIXx9N1PiXmQQ + T36UBolkhSIbHgGI1mqvCEmkiKVhMRlhYQ2jCXWfa0WgCCQwEs/gn//zlBa3HUnf3nRa9/pxqKVt + 5Vq7oZ4P+lYZRXXeoiRrfAVi60SgxwjgLZzf/bpIuNY2jnO2OzQ+MFmc4sQ49Bm86CFblnrU06u/ + THmJLJWmswnFqZ8y1RgQDyXDlYI+ibHJt8pU2LpWhUXDFsfoGwAA//8DAOtohWdIAgAA + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8c8a6a3c3e5f11ac-MRS + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Wed, 25 Sep 2024 10:43:45 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=oXnQwjslhtdFXjG89Oureoj0ycU2U3.JD0YOmXVf7Oo-1727261025-1.0.1.1-8zkkMufvVyON_EWorQBeCtOhIav5dyIQ7s5UoEMu2gTW.uaDA3owAxnO_LwCkccXJhC56ryfDhKmS49nV855yA; + path=/; expires=Wed, 25-Sep-24 11:13:45 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=YSEnhEKHk_5duU.2X6td9ALnCbun1O0M.YvR4OF5DgU-1727261025973-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + openai-organization: + - scale3-1 + openai-processing-ms: + - '1074' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '1000000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '999975' + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 1ms + x-request-id: + - req_c4bc5f6decb9ecc1d59dd8e9435531bf + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-openai/tests/cassettes/test_chat_completion_streaming.yaml b/instrumentation/opentelemetry-instrumentation-openai/tests/cassettes/test_chat_completion_streaming.yaml new file mode 100644 index 0000000000..5859360bd7 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-openai/tests/cassettes/test_chat_completion_streaming.yaml @@ -0,0 +1,157 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": "Say this is a test three times"}], + "model": "gpt-4", "stream": true, "stream_options": {"include_usage": true}}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '154' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.47.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.47.0 + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.5 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: 'data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"This"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" + is"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" + a"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" + test"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" + This"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" + is"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" + a"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" + test"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" + This"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" + is"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" + a"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" + test"},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null} + + + data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null} + + + data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[],"usage":{"prompt_tokens":14,"completion_tokens":15,"total_tokens":29,"completion_tokens_details":{"reasoning_tokens":0}}} + + + data: [DONE] + + + ' + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8c8a8f963be411a4-MRS + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Wed, 25 Sep 2024 11:09:15 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=hvo156nahnyIUFXHe9iuYr0tn0dKzveWlQN7suEYz9Q-1727262555-1.0.1.1-L1wMbo_r0VTMdA..XHJ_8JDmEIjnuzOW_umwSN1y.LlARvkoK3fluYzgsPa5W1Wd_.Hx0yB__0kriyR1pszyDw; + path=/; expires=Wed, 25-Sep-24 11:39:15 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=imSGB7bQZURuXPnMTuFDyO6GqwZI2ELFF_y9AyZcOwk-1727262555002-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + openai-organization: + - scale3-1 + openai-processing-ms: + - '195' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '1000000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '999975' + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 1ms + x-request-id: + - req_49e07a9a2909db3c1d58e009add4f3ad + status: + code: 200 + message: OK +version: 1 diff --git a/instrumentation/opentelemetry-instrumentation-openai/tests/conftest.py b/instrumentation/opentelemetry-instrumentation-openai/tests/conftest.py new file mode 100644 index 0000000000..0ed9279c9d --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-openai/tests/conftest.py @@ -0,0 +1,52 @@ +"""Unit tests configuration module.""" + +import os + +import pytest +from openai import OpenAI + +from opentelemetry import trace +from opentelemetry.instrumentation.openai_v2 import OpenAIInstrumentor +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) + + +@pytest.fixture(scope="session") +def exporter(): + exporter = InMemorySpanExporter() + processor = SimpleSpanProcessor(exporter) + + provider = TracerProvider() + provider.add_span_processor(processor) + trace.set_tracer_provider(provider) + + return exporter + + +@pytest.fixture(autouse=True) +def clear_exporter(exporter): + exporter.clear() + + +@pytest.fixture(autouse=True) +def environment(): + if not os.getenv("OPENAI_API_KEY"): + os.environ["OPENAI_API_KEY"] = "test-api-key" + + +@pytest.fixture +def openai_client(): + return OpenAI() + + +@pytest.fixture(scope="module") +def vcr_config(): + return {"filter_headers": ["authorization", "api-key"]} + + +@pytest.fixture(scope="session", autouse=True) +def instrument(): + OpenAIInstrumentor().instrument() diff --git a/instrumentation/opentelemetry-instrumentation-openai/tests/test_chat_completions.py b/instrumentation/opentelemetry-instrumentation-openai/tests/test_chat_completions.py new file mode 100644 index 0000000000..f298a018a6 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-openai/tests/test_chat_completions.py @@ -0,0 +1,162 @@ +import json + +import pytest + +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAIAttributes, +) + + +@pytest.mark.vcr() +def test_chat_completion(exporter, openai_client): + llm_model_value = "gpt-4" + messages_value = [ + {"role": "user", "content": "Say this is a test three times"} + ] + + kwargs = { + "model": llm_model_value, + "messages": messages_value, + "stream": False, + } + + response = openai_client.chat.completions.create(**kwargs) + spans = exporter.get_finished_spans() + chat_completion_span = spans[-1] + # assert that the span name is correct + assert chat_completion_span.name == f"chat {llm_model_value}" + + attributes = chat_completion_span.attributes + operation_name = attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] + system = attributes[GenAIAttributes.GEN_AI_SYSTEM] + request_model = attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] + response_model = attributes[GenAIAttributes.GEN_AI_RESPONSE_MODEL] + response_id = attributes[GenAIAttributes.GEN_AI_RESPONSE_ID] + input_tokens = attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] + output_tokens = attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] + + # assert that the attributes are correct + assert ( + operation_name == GenAIAttributes.GenAiOperationNameValues.CHAT.value + ) + assert system == GenAIAttributes.GenAiSystemValues.OPENAI.value + assert request_model == llm_model_value + assert response_model == response.model + assert response_id == response.id + assert input_tokens == response.usage.prompt_tokens + assert output_tokens == response.usage.completion_tokens + + events = chat_completion_span.events + + # assert that the prompt and completion events are present + prompt_event = list( + filter( + lambda event: event.name == "gen_ai.content.prompt", + events, + ) + ) + completion_event = list( + filter( + lambda event: event.name == "gen_ai.content.completion", + events, + ) + ) + + assert prompt_event + assert completion_event + + # assert that the prompt and completion events have the correct attributes + assert prompt_event[0].attributes[ + GenAIAttributes.GEN_AI_PROMPT + ] == json.dumps(messages_value) + + assert ( + json.loads( + completion_event[0].attributes[GenAIAttributes.GEN_AI_COMPLETION] + )[0]["content"] + == response.choices[0].message.content + ) + + +@pytest.mark.vcr() +def test_chat_completion_streaming(exporter, openai_client): + llm_model_value = "gpt-4" + messages_value = [ + {"role": "user", "content": "Say this is a test three times"} + ] + + kwargs = { + "model": llm_model_value, + "messages": messages_value, + "stream": True, + "stream_options": {"include_usage": True}, + } + + response_stream_usage = None + response_stream_model = None + response_stream_id = None + response_stream_result = "" + response = openai_client.chat.completions.create(**kwargs) + for chunk in response: + if chunk.choices: + response_stream_result += chunk.choices[0].delta.content or "" + + # get the last chunk + if getattr(chunk, "usage", None): + response_stream_usage = chunk.usage + response_stream_model = chunk.model + response_stream_id = chunk.id + + spans = exporter.get_finished_spans() + streaming_span = spans[-1] + + assert streaming_span.name == f"chat {llm_model_value}" + attributes = streaming_span.attributes + + operation_name = attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] + system = attributes[GenAIAttributes.GEN_AI_SYSTEM] + request_model = attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] + response_model = attributes[GenAIAttributes.GEN_AI_RESPONSE_MODEL] + response_id = attributes[GenAIAttributes.GEN_AI_RESPONSE_ID] + input_tokens = attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] + output_tokens = attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] + assert ( + operation_name == GenAIAttributes.GenAiOperationNameValues.CHAT.value + ) + assert system == GenAIAttributes.GenAiSystemValues.OPENAI.value + assert request_model == llm_model_value + assert response_model == response_stream_model + assert response_id == response_stream_id + assert input_tokens == response_stream_usage.prompt_tokens + assert output_tokens == response_stream_usage.completion_tokens + + events = streaming_span.events + + # assert that the prompt and completion events are present + prompt_event = list( + filter( + lambda event: event.name == "gen_ai.content.prompt", + events, + ) + ) + completion_event = list( + filter( + lambda event: event.name == "gen_ai.content.completion", + events, + ) + ) + + assert prompt_event + assert completion_event + + # assert that the prompt and completion events have the correct attributes + assert prompt_event[0].attributes[ + GenAIAttributes.GEN_AI_PROMPT + ] == json.dumps(messages_value) + + assert ( + json.loads( + completion_event[0].attributes[GenAIAttributes.GEN_AI_COMPLETION] + )[0]["content"] + == response_stream_result + ) From b583aa097cd20ac797417acc38caa1a063dc747a Mon Sep 17 00:00:00 2001 From: Ali Waleed Date: Fri, 27 Sep 2024 17:36:11 +0300 Subject: [PATCH 20/38] enhance build settings --- CHANGELOG.md | 2 -- eachdist.ini | 1 + instrumentation/README.md | 2 +- .../opentelemetry-instrumentation-openai/README.rst | 4 ++-- .../opentelemetry-instrumentation-openai/pyproject.toml | 6 +++--- .../src/opentelemetry/instrumentation/openai_v2/__init__.py | 2 +- .../src/opentelemetry/instrumentation/openai_v2/version.py | 2 +- 7 files changed, 9 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 17c6bd4e9e..a65e549cbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,8 +14,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `opentelemetry-instrumentation-openai` Instrumentation for OpenAI >= 0.27.0 ([#2759](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/2759)) -## Unreleased - ### Added - `opentelemetry-instrumentation-fastapi` Add autoinstrumentation mechanism tests. diff --git a/eachdist.ini b/eachdist.ini index 79c865b334..0ebc2053c7 100644 --- a/eachdist.ini +++ b/eachdist.ini @@ -50,6 +50,7 @@ packages= opentelemetry-resource-detector-azure opentelemetry-sdk-extension-aws opentelemetry-propagator-aws-xray + opentelemetry-instrumentation-openai [lintroots] extraroots=examples/*,scripts/ diff --git a/instrumentation/README.md b/instrumentation/README.md index 2a564f9344..fbe9c6d69a 100644 --- a/instrumentation/README.md +++ b/instrumentation/README.md @@ -29,7 +29,7 @@ | [opentelemetry-instrumentation-logging](./opentelemetry-instrumentation-logging) | logging | No | experimental | [opentelemetry-instrumentation-mysql](./opentelemetry-instrumentation-mysql) | mysql-connector-python >= 8.0, < 10.0 | No | experimental | [opentelemetry-instrumentation-mysqlclient](./opentelemetry-instrumentation-mysqlclient) | mysqlclient < 3 | No | experimental -| [opentelemetry-instrumentation-openai](./opentelemetry-instrumentation-openai) | openai >= 0.27.0 | No | experimental +| [opentelemetry-instrumentation-openai](./opentelemetry-instrumentation-openai-v2) | openai >= 0.27.0 | No | experimental | [opentelemetry-instrumentation-pika](./opentelemetry-instrumentation-pika) | pika >= 0.12.0 | No | experimental | [opentelemetry-instrumentation-psycopg](./opentelemetry-instrumentation-psycopg) | psycopg >= 3.1.0 | No | experimental | [opentelemetry-instrumentation-psycopg2](./opentelemetry-instrumentation-psycopg2) | psycopg2 >= 2.7.3.1 | No | experimental diff --git a/instrumentation/opentelemetry-instrumentation-openai/README.rst b/instrumentation/opentelemetry-instrumentation-openai/README.rst index d3f2873622..cd7e9b3922 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/README.rst +++ b/instrumentation/opentelemetry-instrumentation-openai/README.rst @@ -3,8 +3,8 @@ OpenTelemetry OpenAI Instrumentation |pypi| -.. |pypi| image:: https://badge.fury.io/py/opentelemetry-instrumentation-openai.svg - :target: https://pypi.org/project/opentelemetry-instrumentation-openai/ +.. |pypi| image:: https://badge.fury.io/py/opentelemetry-instrumentation-openai-v2.svg + :target: https://pypi.org/project/opentelemetry-instrumentation-openai-v2/ Instrumentation with OpenAI that supports the openai library and is specified to trace_integration using 'OpenAI'. diff --git a/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml b/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml index 9f08f11027..ddd6532ea7 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml +++ b/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "hatchling.build" [project] name = "opentelemetry-instrumentation-openai-v2" dynamic = ["version"] -description = "OpenTelemetry OpenAI instrumentation" +description = "OpenTelemetry Official OpenAI instrumentation" readme = "README.rst" license = "Apache-2.0" requires-python = ">=3.8" @@ -26,8 +26,8 @@ classifiers = [ ] dependencies = [ "opentelemetry-api ~= 1.12", - "opentelemetry-instrumentation == 0.48b0.dev", - "opentelemetry-semantic-conventions == 0.48b0.dev" + "opentelemetry-instrumentation ~= 0.48b0", + "opentelemetry-semantic-conventions ~= 0.48b0" ] diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/__init__.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/__init__.py index 85ab2fd4fd..e5df94a348 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/__init__.py @@ -65,7 +65,7 @@ def _instrument(self, **kwargs): __name__, "", tracer_provider, - schema_url=Schemas.V1_27_0, + schema_url=Schemas.V1_27_0.value, ) wrap_function_wrapper( module="openai.resources.chat.completions", diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/version.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/version.py index ee5a6342e7..0e95b73df6 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/version.py +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "0.49b0.dev" +__version__ = "2.0.0.dev" From f2a5cfa344c9d2c302e0eee737c6e1f81754d832 Mon Sep 17 00:00:00 2001 From: Ali Waleed Date: Fri, 27 Sep 2024 18:03:04 +0300 Subject: [PATCH 21/38] address test comments --- .../tests/cassettes/test_chat_completion.yaml | 45 +++++------ .../test_chat_completion_streaming.yaml | 80 +++++-------------- .../tests/conftest.py | 15 +++- .../tests/test_chat_completions.py | 12 +-- 4 files changed, 58 insertions(+), 94 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-openai/tests/cassettes/test_chat_completion.yaml b/instrumentation/opentelemetry-instrumentation-openai/tests/cassettes/test_chat_completion.yaml index 34c5f81bef..3bfd94e415 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/tests/cassettes/test_chat_completion.yaml +++ b/instrumentation/opentelemetry-instrumentation-openai/tests/cassettes/test_chat_completion.yaml @@ -1,7 +1,7 @@ interactions: - request: - body: '{"messages": [{"role": "user", "content": "Say this is a test three times"}], - "model": "gpt-4", "stream": false}' + body: '{"messages": [{"role": "user", "content": "Say this is a test"}], "model": + "gpt-4", "stream": false}' headers: accept: - application/json @@ -10,7 +10,7 @@ interactions: connection: - keep-alive content-length: - - '112' + - '100' content-type: - application/json host: @@ -35,44 +35,39 @@ interactions: uri: https://api.openai.com/v1/chat/completions response: body: - string: !!binary | - H4sIAAAAAAAAAwAAAP//hJBBS8NAEIXv+RXDntuSpLXV3BQPIihSxItI2CbTZO1md81MoFr632W3 - MakH8RLI+/Y93rxDBCBUKTIQRS25aJyeXt/cN+r26bH72q9f4v0Dzuu79a78KPQllWLiHXbzjgX/ - uGaFbZxGVtaccNGiZPSpySpdpcskThcBNLZE7W2V4+liGi+Tee+orSqQRAavEQDAIXx9N1PiXmQQ - T36UBolkhSIbHgGI1mqvCEmkiKVhMRlhYQ2jCXWfa0WgCCQwEs/gn//zlBa3HUnf3nRa9/pxqKVt - 5Vq7oZ4P+lYZRXXeoiRrfAVi60SgxwjgLZzf/bpIuNY2jnO2OzQ+MFmc4sQ49Bm86CFblnrU06u/ - THmJLJWmswnFqZ8y1RgQDyXDlYI+ibHJt8pU2LpWhUXDFsfoGwAA//8DAOtohWdIAgAA + string: "{\n \"id\": \"chatcmpl-AC6ajKKHHpvf6x2Qm35t6m3QE8qli\",\n \"object\": + \"chat.completion\",\n \"created\": 1727448637,\n \"model\": \"gpt-4-0613\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"This is a test.\",\n \"refusal\": + null\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n + \ }\n ],\n \"usage\": {\n \"prompt_tokens\": 12,\n \"completion_tokens\": + 5,\n \"total_tokens\": 17,\n \"completion_tokens_details\": {\n \"reasoning_tokens\": + 0\n }\n },\n \"system_fingerprint\": null\n}\n" headers: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8c8a6a3c3e5f11ac-MRS + - 8c9c4e9b7fb674d8-PMO Connection: - keep-alive - Content-Encoding: - - gzip Content-Type: - application/json Date: - - Wed, 25 Sep 2024 10:43:45 GMT + - Fri, 27 Sep 2024 14:50:37 GMT Server: - cloudflare - Set-Cookie: - - __cf_bm=oXnQwjslhtdFXjG89Oureoj0ycU2U3.JD0YOmXVf7Oo-1727261025-1.0.1.1-8zkkMufvVyON_EWorQBeCtOhIav5dyIQ7s5UoEMu2gTW.uaDA3owAxnO_LwCkccXJhC56ryfDhKmS49nV855yA; - path=/; expires=Wed, 25-Sep-24 11:13:45 GMT; domain=.api.openai.com; HttpOnly; - Secure; SameSite=None - - _cfuvid=YSEnhEKHk_5duU.2X6td9ALnCbun1O0M.YvR4OF5DgU-1727261025973-0.0.1.1-604800000; - path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Set-Cookie: test_set_cookie Transfer-Encoding: - chunked X-Content-Type-Options: - nosniff access-control-expose-headers: - X-Request-ID - openai-organization: - - scale3-1 + content-length: + - '551' + openai-organization: test_organization openai-processing-ms: - - '1074' + - '434' openai-version: - '2020-10-01' strict-transport-security: @@ -84,13 +79,13 @@ interactions: x-ratelimit-remaining-requests: - '9999' x-ratelimit-remaining-tokens: - - '999975' + - '999977' x-ratelimit-reset-requests: - 6ms x-ratelimit-reset-tokens: - 1ms x-request-id: - - req_c4bc5f6decb9ecc1d59dd8e9435531bf + - req_5f2690abaf909a9f047488694d44495e status: code: 200 message: OK diff --git a/instrumentation/opentelemetry-instrumentation-openai/tests/cassettes/test_chat_completion_streaming.yaml b/instrumentation/opentelemetry-instrumentation-openai/tests/cassettes/test_chat_completion_streaming.yaml index 5859360bd7..c61133739b 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/tests/cassettes/test_chat_completion_streaming.yaml +++ b/instrumentation/opentelemetry-instrumentation-openai/tests/cassettes/test_chat_completion_streaming.yaml @@ -1,7 +1,7 @@ interactions: - request: - body: '{"messages": [{"role": "user", "content": "Say this is a test three times"}], - "model": "gpt-4", "stream": true, "stream_options": {"include_usage": true}}' + body: '{"messages": [{"role": "user", "content": "Say this is a test"}], "model": + "gpt-4", "stream": true, "stream_options": {"include_usage": true}}' headers: accept: - application/json @@ -10,7 +10,7 @@ interactions: connection: - keep-alive content-length: - - '154' + - '142' content-type: - application/json host: @@ -35,69 +35,31 @@ interactions: uri: https://api.openai.com/v1/chat/completions response: body: - string: 'data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null} + string: 'data: {"id":"chatcmpl-AC6akONKCxc8HS63qZ08HyjeTSq6p","object":"chat.completion.chunk","created":1727448638,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"This"},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-AC6akONKCxc8HS63qZ08HyjeTSq6p","object":"chat.completion.chunk","created":1727448638,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"This"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AC6akONKCxc8HS63qZ08HyjeTSq6p","object":"chat.completion.chunk","created":1727448638,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" is"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AC6akONKCxc8HS63qZ08HyjeTSq6p","object":"chat.completion.chunk","created":1727448638,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" a"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" + data: {"id":"chatcmpl-AC6akONKCxc8HS63qZ08HyjeTSq6p","object":"chat.completion.chunk","created":1727448638,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" test"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-AC6akONKCxc8HS63qZ08HyjeTSq6p","object":"chat.completion.chunk","created":1727448638,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" - This"},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-AC6akONKCxc8HS63qZ08HyjeTSq6p","object":"chat.completion.chunk","created":1727448638,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null} - data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" - is"},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" - a"},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" - test"},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" - This"},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" - is"},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" - a"},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":" - test"},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{"content":"."},"logprobs":null,"finish_reason":null}],"usage":null} - - - data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null} - - - data: {"id":"chatcmpl-ABKBOah1RoDknkRPBm3mgtT3raLaI","object":"chat.completion.chunk","created":1727262554,"model":"gpt-4-0613","system_fingerprint":null,"choices":[],"usage":{"prompt_tokens":14,"completion_tokens":15,"total_tokens":29,"completion_tokens_details":{"reasoning_tokens":0}}} + data: {"id":"chatcmpl-AC6akONKCxc8HS63qZ08HyjeTSq6p","object":"chat.completion.chunk","created":1727448638,"model":"gpt-4-0613","system_fingerprint":null,"choices":[],"usage":{"prompt_tokens":12,"completion_tokens":5,"total_tokens":17,"completion_tokens_details":{"reasoning_tokens":0}}} data: [DONE] @@ -108,31 +70,25 @@ interactions: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8c8a8f963be411a4-MRS + - 8c9c4ea489d57948-PMO Connection: - keep-alive Content-Type: - text/event-stream; charset=utf-8 Date: - - Wed, 25 Sep 2024 11:09:15 GMT + - Fri, 27 Sep 2024 14:50:38 GMT Server: - cloudflare - Set-Cookie: - - __cf_bm=hvo156nahnyIUFXHe9iuYr0tn0dKzveWlQN7suEYz9Q-1727262555-1.0.1.1-L1wMbo_r0VTMdA..XHJ_8JDmEIjnuzOW_umwSN1y.LlARvkoK3fluYzgsPa5W1Wd_.Hx0yB__0kriyR1pszyDw; - path=/; expires=Wed, 25-Sep-24 11:39:15 GMT; domain=.api.openai.com; HttpOnly; - Secure; SameSite=None - - _cfuvid=imSGB7bQZURuXPnMTuFDyO6GqwZI2ELFF_y9AyZcOwk-1727262555002-0.0.1.1-604800000; - path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Set-Cookie: test_set_cookie Transfer-Encoding: - chunked X-Content-Type-Options: - nosniff access-control-expose-headers: - X-Request-ID - openai-organization: - - scale3-1 + openai-organization: test_organization openai-processing-ms: - - '195' + - '161' openai-version: - '2020-10-01' strict-transport-security: @@ -144,13 +100,13 @@ interactions: x-ratelimit-remaining-requests: - '9999' x-ratelimit-remaining-tokens: - - '999975' + - '999977' x-ratelimit-reset-requests: - 6ms x-ratelimit-reset-tokens: - 1ms x-request-id: - - req_49e07a9a2909db3c1d58e009add4f3ad + - req_3fa9ac9f3693c712e4c377e26d203e58 status: code: 200 message: OK diff --git a/instrumentation/opentelemetry-instrumentation-openai/tests/conftest.py b/instrumentation/opentelemetry-instrumentation-openai/tests/conftest.py index 0ed9279c9d..c6e1ee86f4 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/tests/conftest.py +++ b/instrumentation/opentelemetry-instrumentation-openai/tests/conftest.py @@ -44,9 +44,22 @@ def openai_client(): @pytest.fixture(scope="module") def vcr_config(): - return {"filter_headers": ["authorization", "api-key"]} + return { + "filter_headers": ["authorization", "api-key"], + "decode_compressed_response": True, + "before_record_response": scrub_response_headers, + } @pytest.fixture(scope="session", autouse=True) def instrument(): OpenAIInstrumentor().instrument() + + +def scrub_response_headers(response): + """ + This scrubs sensitive response headers. Note they are case-sensitive! + """ + response["headers"]["openai-organization"] = "test_organization" + response["headers"]["Set-Cookie"] = "test_set_cookie" + return response diff --git a/instrumentation/opentelemetry-instrumentation-openai/tests/test_chat_completions.py b/instrumentation/opentelemetry-instrumentation-openai/tests/test_chat_completions.py index f298a018a6..2c37237fc3 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/tests/test_chat_completions.py +++ b/instrumentation/opentelemetry-instrumentation-openai/tests/test_chat_completions.py @@ -10,9 +10,7 @@ @pytest.mark.vcr() def test_chat_completion(exporter, openai_client): llm_model_value = "gpt-4" - messages_value = [ - {"role": "user", "content": "Say this is a test three times"} - ] + messages_value = [{"role": "user", "content": "Say this is a test"}] kwargs = { "model": llm_model_value, @@ -22,6 +20,8 @@ def test_chat_completion(exporter, openai_client): response = openai_client.chat.completions.create(**kwargs) spans = exporter.get_finished_spans() + # ignore by default added spans etc: http spans. + # we are only interested in the last span which is the chat completion span chat_completion_span = spans[-1] # assert that the span name is correct assert chat_completion_span.name == f"chat {llm_model_value}" @@ -81,9 +81,7 @@ def test_chat_completion(exporter, openai_client): @pytest.mark.vcr() def test_chat_completion_streaming(exporter, openai_client): llm_model_value = "gpt-4" - messages_value = [ - {"role": "user", "content": "Say this is a test three times"} - ] + messages_value = [{"role": "user", "content": "Say this is a test"}] kwargs = { "model": llm_model_value, @@ -108,6 +106,8 @@ def test_chat_completion_streaming(exporter, openai_client): response_stream_id = chunk.id spans = exporter.get_finished_spans() + # ignore by default added spans etc: http spans. + # we are only interested in the last span which is the chat completion span streaming_span = spans[-1] assert streaming_span.name == f"chat {llm_model_value}" From 3bdfd8f741cfa6ca152c559b337c7071d41d0d99 Mon Sep 17 00:00:00 2001 From: Ali Waleed Date: Tue, 8 Oct 2024 14:13:55 +0300 Subject: [PATCH 22/38] run `tox -e generate` & `tox -e generate-workflows` --- .github/workflows/lint_0.yml | 18 ++ .github/workflows/test_0.yml | 216 +++++++++--------- .github/workflows/test_1.yml | 108 +++++++++ instrumentation/README.md | 2 +- .../pyproject.toml | 2 +- .../instrumentation/bootstrap_gen.py | 2 +- 6 files changed, 237 insertions(+), 111 deletions(-) diff --git a/.github/workflows/lint_0.yml b/.github/workflows/lint_0.yml index 2236dc422c..8fcf16dae1 100644 --- a/.github/workflows/lint_0.yml +++ b/.github/workflows/lint_0.yml @@ -16,6 +16,24 @@ env: jobs: + lint-instrumentation-openai: + name: instrumentation-openai + runs-on: ubuntu-latest + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install tox + run: pip install tox + + - name: Run tests + run: tox -e lint-instrumentation-openai + lint-resource-detector-container: name: resource-detector-container runs-on: ubuntu-latest diff --git a/.github/workflows/test_0.yml b/.github/workflows/test_0.yml index d251737227..f1a2f1c846 100644 --- a/.github/workflows/test_0.yml +++ b/.github/workflows/test_0.yml @@ -16,6 +16,114 @@ env: jobs: + py38-test-instrumentation-openai_ubuntu-latest: + name: instrumentation-openai 3.8 Ubuntu + runs-on: ubuntu-latest + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.8 + uses: actions/setup-python@v5 + with: + python-version: "3.8" + + - name: Install tox + run: pip install tox + + - name: Run tests + run: tox -e py38-test-instrumentation-openai -- -ra + + py39-test-instrumentation-openai_ubuntu-latest: + name: instrumentation-openai 3.9 Ubuntu + runs-on: ubuntu-latest + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: "3.9" + + - name: Install tox + run: pip install tox + + - name: Run tests + run: tox -e py39-test-instrumentation-openai -- -ra + + py310-test-instrumentation-openai_ubuntu-latest: + name: instrumentation-openai 3.10 Ubuntu + runs-on: ubuntu-latest + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install tox + run: pip install tox + + - name: Run tests + run: tox -e py310-test-instrumentation-openai -- -ra + + py311-test-instrumentation-openai_ubuntu-latest: + name: instrumentation-openai 3.11 Ubuntu + runs-on: ubuntu-latest + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install tox + run: pip install tox + + - name: Run tests + run: tox -e py311-test-instrumentation-openai -- -ra + + py312-test-instrumentation-openai_ubuntu-latest: + name: instrumentation-openai 3.12 Ubuntu + runs-on: ubuntu-latest + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install tox + run: pip install tox + + - name: Run tests + run: tox -e py312-test-instrumentation-openai -- -ra + + pypy3-test-instrumentation-openai_ubuntu-latest: + name: instrumentation-openai pypy-3.8 Ubuntu + runs-on: ubuntu-latest + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python pypy-3.8 + uses: actions/setup-python@v5 + with: + python-version: "pypy-3.8" + + - name: Install tox + run: pip install tox + + - name: Run tests + run: tox -e pypy3-test-instrumentation-openai -- -ra + py38-test-resource-detector-container_ubuntu-latest: name: resource-detector-container 3.8 Ubuntu runs-on: ubuntu-latest @@ -4407,111 +4515,3 @@ jobs: - name: Run tests run: tox -e py311-test-instrumentation-pymemcache-2 -- -ra - - py311-test-instrumentation-pymemcache-3_ubuntu-latest: - name: instrumentation-pymemcache-3 3.11 Ubuntu - runs-on: ubuntu-latest - steps: - - name: Checkout repo @ SHA - ${{ github.sha }} - uses: actions/checkout@v4 - - - name: Set up Python 3.11 - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Install tox - run: pip install tox - - - name: Run tests - run: tox -e py311-test-instrumentation-pymemcache-3 -- -ra - - py311-test-instrumentation-pymemcache-4_ubuntu-latest: - name: instrumentation-pymemcache-4 3.11 Ubuntu - runs-on: ubuntu-latest - steps: - - name: Checkout repo @ SHA - ${{ github.sha }} - uses: actions/checkout@v4 - - - name: Set up Python 3.11 - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Install tox - run: pip install tox - - - name: Run tests - run: tox -e py311-test-instrumentation-pymemcache-4 -- -ra - - py312-test-instrumentation-pymemcache-0_ubuntu-latest: - name: instrumentation-pymemcache-0 3.12 Ubuntu - runs-on: ubuntu-latest - steps: - - name: Checkout repo @ SHA - ${{ github.sha }} - uses: actions/checkout@v4 - - - name: Set up Python 3.12 - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - - name: Install tox - run: pip install tox - - - name: Run tests - run: tox -e py312-test-instrumentation-pymemcache-0 -- -ra - - py312-test-instrumentation-pymemcache-1_ubuntu-latest: - name: instrumentation-pymemcache-1 3.12 Ubuntu - runs-on: ubuntu-latest - steps: - - name: Checkout repo @ SHA - ${{ github.sha }} - uses: actions/checkout@v4 - - - name: Set up Python 3.12 - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - - name: Install tox - run: pip install tox - - - name: Run tests - run: tox -e py312-test-instrumentation-pymemcache-1 -- -ra - - py312-test-instrumentation-pymemcache-2_ubuntu-latest: - name: instrumentation-pymemcache-2 3.12 Ubuntu - runs-on: ubuntu-latest - steps: - - name: Checkout repo @ SHA - ${{ github.sha }} - uses: actions/checkout@v4 - - - name: Set up Python 3.12 - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - - name: Install tox - run: pip install tox - - - name: Run tests - run: tox -e py312-test-instrumentation-pymemcache-2 -- -ra - - py312-test-instrumentation-pymemcache-3_ubuntu-latest: - name: instrumentation-pymemcache-3 3.12 Ubuntu - runs-on: ubuntu-latest - steps: - - name: Checkout repo @ SHA - ${{ github.sha }} - uses: actions/checkout@v4 - - - name: Set up Python 3.12 - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - - name: Install tox - run: pip install tox - - - name: Run tests - run: tox -e py312-test-instrumentation-pymemcache-3 -- -ra diff --git a/.github/workflows/test_1.yml b/.github/workflows/test_1.yml index 30ca4e67d2..30e02c5634 100644 --- a/.github/workflows/test_1.yml +++ b/.github/workflows/test_1.yml @@ -16,6 +16,114 @@ env: jobs: + py311-test-instrumentation-pymemcache-3_ubuntu-latest: + name: instrumentation-pymemcache-3 3.11 Ubuntu + runs-on: ubuntu-latest + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install tox + run: pip install tox + + - name: Run tests + run: tox -e py311-test-instrumentation-pymemcache-3 -- -ra + + py311-test-instrumentation-pymemcache-4_ubuntu-latest: + name: instrumentation-pymemcache-4 3.11 Ubuntu + runs-on: ubuntu-latest + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install tox + run: pip install tox + + - name: Run tests + run: tox -e py311-test-instrumentation-pymemcache-4 -- -ra + + py312-test-instrumentation-pymemcache-0_ubuntu-latest: + name: instrumentation-pymemcache-0 3.12 Ubuntu + runs-on: ubuntu-latest + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install tox + run: pip install tox + + - name: Run tests + run: tox -e py312-test-instrumentation-pymemcache-0 -- -ra + + py312-test-instrumentation-pymemcache-1_ubuntu-latest: + name: instrumentation-pymemcache-1 3.12 Ubuntu + runs-on: ubuntu-latest + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install tox + run: pip install tox + + - name: Run tests + run: tox -e py312-test-instrumentation-pymemcache-1 -- -ra + + py312-test-instrumentation-pymemcache-2_ubuntu-latest: + name: instrumentation-pymemcache-2 3.12 Ubuntu + runs-on: ubuntu-latest + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install tox + run: pip install tox + + - name: Run tests + run: tox -e py312-test-instrumentation-pymemcache-2 -- -ra + + py312-test-instrumentation-pymemcache-3_ubuntu-latest: + name: instrumentation-pymemcache-3 3.12 Ubuntu + runs-on: ubuntu-latest + steps: + - name: Checkout repo @ SHA - ${{ github.sha }} + uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install tox + run: pip install tox + + - name: Run tests + run: tox -e py312-test-instrumentation-pymemcache-3 -- -ra + py312-test-instrumentation-pymemcache-4_ubuntu-latest: name: instrumentation-pymemcache-4 3.12 Ubuntu runs-on: ubuntu-latest diff --git a/instrumentation/README.md b/instrumentation/README.md index fbe9c6d69a..2a564f9344 100644 --- a/instrumentation/README.md +++ b/instrumentation/README.md @@ -29,7 +29,7 @@ | [opentelemetry-instrumentation-logging](./opentelemetry-instrumentation-logging) | logging | No | experimental | [opentelemetry-instrumentation-mysql](./opentelemetry-instrumentation-mysql) | mysql-connector-python >= 8.0, < 10.0 | No | experimental | [opentelemetry-instrumentation-mysqlclient](./opentelemetry-instrumentation-mysqlclient) | mysqlclient < 3 | No | experimental -| [opentelemetry-instrumentation-openai](./opentelemetry-instrumentation-openai-v2) | openai >= 0.27.0 | No | experimental +| [opentelemetry-instrumentation-openai](./opentelemetry-instrumentation-openai) | openai >= 0.27.0 | No | experimental | [opentelemetry-instrumentation-pika](./opentelemetry-instrumentation-pika) | pika >= 0.12.0 | No | experimental | [opentelemetry-instrumentation-psycopg](./opentelemetry-instrumentation-psycopg) | psycopg >= 3.1.0 | No | experimental | [opentelemetry-instrumentation-psycopg2](./opentelemetry-instrumentation-psycopg2) | psycopg2 >= 2.7.3.1 | No | experimental diff --git a/opentelemetry-contrib-instrumentations/pyproject.toml b/opentelemetry-contrib-instrumentations/pyproject.toml index 302e323275..3a9c94c295 100644 --- a/opentelemetry-contrib-instrumentations/pyproject.toml +++ b/opentelemetry-contrib-instrumentations/pyproject.toml @@ -57,7 +57,7 @@ dependencies = [ "opentelemetry-instrumentation-logging==0.49b0.dev", "opentelemetry-instrumentation-mysql==0.49b0.dev", "opentelemetry-instrumentation-mysqlclient==0.49b0.dev", - "opentelemetry-instrumentation-openai==0.49b0.dev", + "opentelemetry-instrumentation-openai==2.0.0.dev", "opentelemetry-instrumentation-pika==0.49b0.dev", "opentelemetry-instrumentation-psycopg==0.49b0.dev", "opentelemetry-instrumentation-psycopg2==0.49b0.dev", diff --git a/opentelemetry-instrumentation/src/opentelemetry/instrumentation/bootstrap_gen.py b/opentelemetry-instrumentation/src/opentelemetry/instrumentation/bootstrap_gen.py index e4791775a3..3fd0bc39e3 100644 --- a/opentelemetry-instrumentation/src/opentelemetry/instrumentation/bootstrap_gen.py +++ b/opentelemetry-instrumentation/src/opentelemetry/instrumentation/bootstrap_gen.py @@ -122,7 +122,7 @@ }, { "library": "openai >= 0.27.0", - "instrumentation": "opentelemetry-instrumentation-openai-v2==0.49b0.dev", + "instrumentation": "opentelemetry-instrumentation-openai-v2==2.0.0.dev", }, { "library": "pika >= 0.12.0", From 41cbfd0b344b9d80db3e608bf7da85b41129faa5 Mon Sep 17 00:00:00 2001 From: Karthik Kalyanaraman <105607645+karthikscale3@users.noreply.github.com> Date: Wed, 9 Oct 2024 00:59:54 -0700 Subject: [PATCH 23/38] Update instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/__init__.py Co-authored-by: Riccardo Magliocchetti --- .../src/opentelemetry/instrumentation/openai_v2/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/__init__.py b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/__init__.py index e5df94a348..c196cae6ea 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/__init__.py @@ -74,4 +74,4 @@ def _instrument(self, **kwargs): ) def _uninstrument(self, **kwargs): - pass + unwrap(openai.resources.chat.completions.Completions, "create") From 578653de2ea8cfe490b3ab76ccd4f575a59c8df7 Mon Sep 17 00:00:00 2001 From: Ali Waleed Date: Thu, 17 Oct 2024 19:01:23 +0300 Subject: [PATCH 24/38] change folder name to v2 --- .../LICENSE | 0 .../README.rst | 0 .../pyproject.toml | 0 .../src/opentelemetry/instrumentation/openai_v2/__init__.py | 3 ++- .../src/opentelemetry/instrumentation/openai_v2/package.py | 2 +- .../src/opentelemetry/instrumentation/openai_v2/patch.py | 0 .../src/opentelemetry/instrumentation/openai_v2/utils.py | 0 .../src/opentelemetry/instrumentation/openai_v2/version.py | 0 .../test-requirements.txt | 2 +- .../tests/__init__.py | 0 .../tests/cassettes/test_chat_completion.yaml | 0 .../tests/cassettes/test_chat_completion_streaming.yaml | 0 .../tests/conftest.py | 0 .../tests/test_chat_completions.py | 0 14 files changed, 4 insertions(+), 3 deletions(-) rename instrumentation/{opentelemetry-instrumentation-openai => opentelemetry-instrumentation-openai-v2}/LICENSE (100%) rename instrumentation/{opentelemetry-instrumentation-openai => opentelemetry-instrumentation-openai-v2}/README.rst (100%) rename instrumentation/{opentelemetry-instrumentation-openai => opentelemetry-instrumentation-openai-v2}/pyproject.toml (100%) rename instrumentation/{opentelemetry-instrumentation-openai => opentelemetry-instrumentation-openai-v2}/src/opentelemetry/instrumentation/openai_v2/__init__.py (94%) rename instrumentation/{opentelemetry-instrumentation-openai => opentelemetry-instrumentation-openai-v2}/src/opentelemetry/instrumentation/openai_v2/package.py (94%) rename instrumentation/{opentelemetry-instrumentation-openai => opentelemetry-instrumentation-openai-v2}/src/opentelemetry/instrumentation/openai_v2/patch.py (100%) rename instrumentation/{opentelemetry-instrumentation-openai => opentelemetry-instrumentation-openai-v2}/src/opentelemetry/instrumentation/openai_v2/utils.py (100%) rename instrumentation/{opentelemetry-instrumentation-openai => opentelemetry-instrumentation-openai-v2}/src/opentelemetry/instrumentation/openai_v2/version.py (100%) rename instrumentation/{opentelemetry-instrumentation-openai => opentelemetry-instrumentation-openai-v2}/test-requirements.txt (92%) rename instrumentation/{opentelemetry-instrumentation-openai => opentelemetry-instrumentation-openai-v2}/tests/__init__.py (100%) rename instrumentation/{opentelemetry-instrumentation-openai => opentelemetry-instrumentation-openai-v2}/tests/cassettes/test_chat_completion.yaml (100%) rename instrumentation/{opentelemetry-instrumentation-openai => opentelemetry-instrumentation-openai-v2}/tests/cassettes/test_chat_completion_streaming.yaml (100%) rename instrumentation/{opentelemetry-instrumentation-openai => opentelemetry-instrumentation-openai-v2}/tests/conftest.py (100%) rename instrumentation/{opentelemetry-instrumentation-openai => opentelemetry-instrumentation-openai-v2}/tests/test_chat_completions.py (100%) diff --git a/instrumentation/opentelemetry-instrumentation-openai/LICENSE b/instrumentation/opentelemetry-instrumentation-openai-v2/LICENSE similarity index 100% rename from instrumentation/opentelemetry-instrumentation-openai/LICENSE rename to instrumentation/opentelemetry-instrumentation-openai-v2/LICENSE diff --git a/instrumentation/opentelemetry-instrumentation-openai/README.rst b/instrumentation/opentelemetry-instrumentation-openai-v2/README.rst similarity index 100% rename from instrumentation/opentelemetry-instrumentation-openai/README.rst rename to instrumentation/opentelemetry-instrumentation-openai-v2/README.rst diff --git a/instrumentation/opentelemetry-instrumentation-openai/pyproject.toml b/instrumentation/opentelemetry-instrumentation-openai-v2/pyproject.toml similarity index 100% rename from instrumentation/opentelemetry-instrumentation-openai/pyproject.toml rename to instrumentation/opentelemetry-instrumentation-openai-v2/pyproject.toml diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/__init__.py b/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/__init__.py similarity index 94% rename from instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/__init__.py rename to instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/__init__.py index c196cae6ea..05732a010f 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/__init__.py @@ -47,6 +47,7 @@ from opentelemetry.instrumentation.instrumentor import BaseInstrumentor from opentelemetry.instrumentation.openai_v2.package import _instruments +from opentelemetry.instrumentation.utils import unwrap from opentelemetry.semconv.schemas import Schemas from opentelemetry.trace import get_tracer @@ -74,4 +75,4 @@ def _instrument(self, **kwargs): ) def _uninstrument(self, **kwargs): - unwrap(openai.resources.chat.completions.Completions, "create") + unwrap("openai.resources.chat.completions.Completions", "create") diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/package.py b/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/package.py similarity index 94% rename from instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/package.py rename to instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/package.py index dcfded3025..b1df3b87b2 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/package.py +++ b/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/package.py @@ -13,4 +13,4 @@ # limitations under the License. -_instruments = ("openai >= 0.27.0",) +_instruments = ("openai >= 1.0.0",) diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/patch.py b/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/patch.py similarity index 100% rename from instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/patch.py rename to instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/patch.py diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/utils.py b/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/utils.py similarity index 100% rename from instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/utils.py rename to instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/utils.py diff --git a/instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/version.py b/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/version.py similarity index 100% rename from instrumentation/opentelemetry-instrumentation-openai/src/opentelemetry/instrumentation/openai_v2/version.py rename to instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/version.py diff --git a/instrumentation/opentelemetry-instrumentation-openai/test-requirements.txt b/instrumentation/opentelemetry-instrumentation-openai-v2/test-requirements.txt similarity index 92% rename from instrumentation/opentelemetry-instrumentation-openai/test-requirements.txt rename to instrumentation/opentelemetry-instrumentation-openai-v2/test-requirements.txt index bca6bfbd0e..0b15775d2c 100644 --- a/instrumentation/opentelemetry-instrumentation-openai/test-requirements.txt +++ b/instrumentation/opentelemetry-instrumentation-openai-v2/test-requirements.txt @@ -1,4 +1,4 @@ -openai==1.37.1 +openai==1.0.0 Deprecated==1.2.14 importlib-metadata==6.11.0 packaging==24.0 diff --git a/instrumentation/opentelemetry-instrumentation-openai/tests/__init__.py b/instrumentation/opentelemetry-instrumentation-openai-v2/tests/__init__.py similarity index 100% rename from instrumentation/opentelemetry-instrumentation-openai/tests/__init__.py rename to instrumentation/opentelemetry-instrumentation-openai-v2/tests/__init__.py diff --git a/instrumentation/opentelemetry-instrumentation-openai/tests/cassettes/test_chat_completion.yaml b/instrumentation/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_chat_completion.yaml similarity index 100% rename from instrumentation/opentelemetry-instrumentation-openai/tests/cassettes/test_chat_completion.yaml rename to instrumentation/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_chat_completion.yaml diff --git a/instrumentation/opentelemetry-instrumentation-openai/tests/cassettes/test_chat_completion_streaming.yaml b/instrumentation/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_chat_completion_streaming.yaml similarity index 100% rename from instrumentation/opentelemetry-instrumentation-openai/tests/cassettes/test_chat_completion_streaming.yaml rename to instrumentation/opentelemetry-instrumentation-openai-v2/tests/cassettes/test_chat_completion_streaming.yaml diff --git a/instrumentation/opentelemetry-instrumentation-openai/tests/conftest.py b/instrumentation/opentelemetry-instrumentation-openai-v2/tests/conftest.py similarity index 100% rename from instrumentation/opentelemetry-instrumentation-openai/tests/conftest.py rename to instrumentation/opentelemetry-instrumentation-openai-v2/tests/conftest.py diff --git a/instrumentation/opentelemetry-instrumentation-openai/tests/test_chat_completions.py b/instrumentation/opentelemetry-instrumentation-openai-v2/tests/test_chat_completions.py similarity index 100% rename from instrumentation/opentelemetry-instrumentation-openai/tests/test_chat_completions.py rename to instrumentation/opentelemetry-instrumentation-openai-v2/tests/test_chat_completions.py From 578a9425193223834662bba9df02d0aa3c47b736 Mon Sep 17 00:00:00 2001 From: Ali Waleed Date: Thu, 17 Oct 2024 19:08:02 +0300 Subject: [PATCH 25/38] adjust all naming to -v2 --- CHANGELOG.md | 2 +- eachdist.ini | 2 +- instrumentation/README.md | 2 +- .../opentelemetry-instrumentation-openai-v2/pyproject.toml | 5 ++--- .../test-requirements.txt | 2 +- opentelemetry-contrib-instrumentations/pyproject.toml | 2 +- .../src/opentelemetry/instrumentation/bootstrap_gen.py | 2 +- tox.ini | 4 ++-- 8 files changed, 10 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90f0b3c90b..7eef01e49d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- `opentelemetry-instrumentation-openai` Instrumentation for OpenAI >= 0.27.0 +- `opentelemetry-instrumentation-openai-v2` Instrumentation for OpenAI >= 0.27.0 ([#2759](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/2759)) ### Added diff --git a/eachdist.ini b/eachdist.ini index 5b045b6996..c04fb571ce 100644 --- a/eachdist.ini +++ b/eachdist.ini @@ -50,7 +50,7 @@ packages= opentelemetry-resource-detector-azure opentelemetry-sdk-extension-aws opentelemetry-propagator-aws-xray - opentelemetry-instrumentation-openai + opentelemetry-instrumentation-openai-v2 opentelemetry-instrumentation-test [lintroots] diff --git a/instrumentation/README.md b/instrumentation/README.md index f8d188cec3..097550a217 100644 --- a/instrumentation/README.md +++ b/instrumentation/README.md @@ -29,7 +29,7 @@ | [opentelemetry-instrumentation-logging](./opentelemetry-instrumentation-logging) | logging | No | experimental | [opentelemetry-instrumentation-mysql](./opentelemetry-instrumentation-mysql) | mysql-connector-python >= 8.0, < 10.0 | No | experimental | [opentelemetry-instrumentation-mysqlclient](./opentelemetry-instrumentation-mysqlclient) | mysqlclient < 3 | No | experimental -| [opentelemetry-instrumentation-openai](./opentelemetry-instrumentation-openai) | openai >= 0.27.0 | No | experimental +| [opentelemetry-instrumentation-openai](./opentelemetry-instrumentation-openai-v2) | openai >= 1.0.0 | No | experimental | [opentelemetry-instrumentation-pika](./opentelemetry-instrumentation-pika) | pika >= 0.12.0 | No | experimental | [opentelemetry-instrumentation-psycopg](./opentelemetry-instrumentation-psycopg) | psycopg >= 3.1.0 | No | experimental | [opentelemetry-instrumentation-psycopg2](./opentelemetry-instrumentation-psycopg2) | psycopg2 >= 2.7.3.1 | No | experimental diff --git a/instrumentation/opentelemetry-instrumentation-openai-v2/pyproject.toml b/instrumentation/opentelemetry-instrumentation-openai-v2/pyproject.toml index ddd6532ea7..b342cdade7 100644 --- a/instrumentation/opentelemetry-instrumentation-openai-v2/pyproject.toml +++ b/instrumentation/opentelemetry-instrumentation-openai-v2/pyproject.toml @@ -28,19 +28,18 @@ dependencies = [ "opentelemetry-api ~= 1.12", "opentelemetry-instrumentation ~= 0.48b0", "opentelemetry-semantic-conventions ~= 0.48b0" - ] [project.optional-dependencies] instruments = [ - "openai >= 0.27.0", + "openai >= 1.0.0", ] [project.entry-points.opentelemetry_instrumentor] openai = "opentelemetry.instrumentation.openai_v2:OpenAIInstrumentor" [project.urls] -Homepage = "https://github.com/open-telemetry/opentelemetry-python-contrib/tree/main/instrumentation/opentelemetry-instrumentation-openai" +Homepage = "https://github.com/open-telemetry/opentelemetry-python-contrib/tree/main/instrumentation/opentelemetry-instrumentation-openai-v2" [tool.hatch.version] path = "src/opentelemetry/instrumentation/openai_v2/version.py" diff --git a/instrumentation/opentelemetry-instrumentation-openai-v2/test-requirements.txt b/instrumentation/opentelemetry-instrumentation-openai-v2/test-requirements.txt index 0b15775d2c..9385ece64e 100644 --- a/instrumentation/opentelemetry-instrumentation-openai-v2/test-requirements.txt +++ b/instrumentation/opentelemetry-instrumentation-openai-v2/test-requirements.txt @@ -7,4 +7,4 @@ pytest-vcr==1.0.2 wrapt==1.16.0 -e opentelemetry-instrumentation --e instrumentation/opentelemetry-instrumentation-openai +-e instrumentation/opentelemetry-instrumentation-openai-v2 diff --git a/opentelemetry-contrib-instrumentations/pyproject.toml b/opentelemetry-contrib-instrumentations/pyproject.toml index 3912681d96..2b6b2cfedb 100644 --- a/opentelemetry-contrib-instrumentations/pyproject.toml +++ b/opentelemetry-contrib-instrumentations/pyproject.toml @@ -57,7 +57,7 @@ dependencies = [ "opentelemetry-instrumentation-logging==0.49b0.dev", "opentelemetry-instrumentation-mysql==0.49b0.dev", "opentelemetry-instrumentation-mysqlclient==0.49b0.dev", - "opentelemetry-instrumentation-openai==2.0.0.dev", + "opentelemetry-instrumentation-openai-v2==2.0.0.dev", "opentelemetry-instrumentation-pika==0.49b0.dev", "opentelemetry-instrumentation-psycopg==0.49b0.dev", "opentelemetry-instrumentation-psycopg2==0.49b0.dev", diff --git a/opentelemetry-instrumentation/src/opentelemetry/instrumentation/bootstrap_gen.py b/opentelemetry-instrumentation/src/opentelemetry/instrumentation/bootstrap_gen.py index 3a3aedfaa9..95d25026f9 100644 --- a/opentelemetry-instrumentation/src/opentelemetry/instrumentation/bootstrap_gen.py +++ b/opentelemetry-instrumentation/src/opentelemetry/instrumentation/bootstrap_gen.py @@ -121,7 +121,7 @@ "instrumentation": "opentelemetry-instrumentation-mysqlclient==0.49b0.dev", }, { - "library": "openai >= 0.27.0", + "library": "openai >= 1.0.0", "instrumentation": "opentelemetry-instrumentation-openai-v2==2.0.0.dev", }, { diff --git a/tox.ini b/tox.ini index 94fd1bc387..2c85207415 100644 --- a/tox.ini +++ b/tox.ini @@ -413,7 +413,7 @@ commands_pre = openai: pip install opentelemetry-semantic-conventions@{env:CORE_REPO}\#egg=opentelemetry-semantic-conventions&subdirectory=opentelemetry-semantic-conventions openai: pip install opentelemetry-sdk@{env:CORE_REPO}\#egg=opentelemetry-sdk&subdirectory=opentelemetry-sdk openai: pip install opentelemetry-test-utils@{env:CORE_REPO}\#egg=opentelemetry-test-utils&subdirectory=tests/opentelemetry-test-utils - openai: pip install -r {toxinidir}/instrumentation/opentelemetry-instrumentation-openai/test-requirements.txt + openai: pip install -r {toxinidir}/instrumentation/opentelemetry-instrumentation-openai-v2/test-requirements.txt distro: pip install opentelemetry-api@{env:CORE_REPO}\#egg=opentelemetry-api&subdirectory=opentelemetry-api distro: pip install opentelemetry-semantic-conventions@{env:CORE_REPO}\#egg=opentelemetry-semantic-conventions&subdirectory=opentelemetry-semantic-conventions @@ -470,7 +470,7 @@ commands_pre = openai: pip install opentelemetry-api@{env:CORE_REPO}\#egg=opentelemetry-api&subdirectory=opentelemetry-api openai: pip install opentelemetry-sdk@{env:CORE_REPO}\#egg=opentelemetry-sdk&subdirectory=opentelemetry-sdk openai: pip install opentelemetry-semantic-conventions@{env:CORE_REPO}\#egg=opentelemetry-semantic-conventions&subdirectory=opentelemetry-semantic-conventions - openai: pip install -r {toxinidir}/instrumentation/opentelemetry-instrumentation-openai/test-requirements.txt + openai: pip install -r {toxinidir}/instrumentation/opentelemetry-instrumentation-openai-v2/test-requirements.txt confluent-kafka: pip install opentelemetry-api@{env:CORE_REPO}\#egg=opentelemetry-api&subdirectory=opentelemetry-api confluent-kafka: pip install opentelemetry-semantic-conventions@{env:CORE_REPO}\#egg=opentelemetry-semantic-conventions&subdirectory=opentelemetry-semantic-conventions From 51f2438709784bb6385b5e4fedcb0b5323a32393 Mon Sep 17 00:00:00 2001 From: Ali Waleed Date: Thu, 17 Oct 2024 19:08:57 +0300 Subject: [PATCH 26/38] run `tox -e generate` --- instrumentation/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instrumentation/README.md b/instrumentation/README.md index 097550a217..9cdc0a52bb 100644 --- a/instrumentation/README.md +++ b/instrumentation/README.md @@ -29,7 +29,7 @@ | [opentelemetry-instrumentation-logging](./opentelemetry-instrumentation-logging) | logging | No | experimental | [opentelemetry-instrumentation-mysql](./opentelemetry-instrumentation-mysql) | mysql-connector-python >= 8.0, < 10.0 | No | experimental | [opentelemetry-instrumentation-mysqlclient](./opentelemetry-instrumentation-mysqlclient) | mysqlclient < 3 | No | experimental -| [opentelemetry-instrumentation-openai](./opentelemetry-instrumentation-openai-v2) | openai >= 1.0.0 | No | experimental +| [opentelemetry-instrumentation-openai-v2](./opentelemetry-instrumentation-openai-v2) | openai >= 1.0.0 | No | experimental | [opentelemetry-instrumentation-pika](./opentelemetry-instrumentation-pika) | pika >= 0.12.0 | No | experimental | [opentelemetry-instrumentation-psycopg](./opentelemetry-instrumentation-psycopg) | psycopg >= 3.1.0 | No | experimental | [opentelemetry-instrumentation-psycopg2](./opentelemetry-instrumentation-psycopg2) | psycopg2 >= 2.7.3.1 | No | experimental From 8b58f27ca60408d8fa53ce3c94358062de163796 Mon Sep 17 00:00:00 2001 From: Ali Waleed Date: Thu, 17 Oct 2024 19:17:12 +0300 Subject: [PATCH 27/38] adjust tests --- .github/workflows/test_0.yml | 36 ++++++++++++++++++------------------ tox.ini | 4 ++-- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/test_0.yml b/.github/workflows/test_0.yml index f1a2f1c846..df74e90021 100644 --- a/.github/workflows/test_0.yml +++ b/.github/workflows/test_0.yml @@ -16,8 +16,8 @@ env: jobs: - py38-test-instrumentation-openai_ubuntu-latest: - name: instrumentation-openai 3.8 Ubuntu + py38-test-instrumentation-openai-v2_ubuntu-latest: + name: instrumentation-openai-v2 3.8 Ubuntu runs-on: ubuntu-latest steps: - name: Checkout repo @ SHA - ${{ github.sha }} @@ -32,10 +32,10 @@ jobs: run: pip install tox - name: Run tests - run: tox -e py38-test-instrumentation-openai -- -ra + run: tox -e py38-test-instrumentation-openai-v2 -- -ra - py39-test-instrumentation-openai_ubuntu-latest: - name: instrumentation-openai 3.9 Ubuntu + py39-test-instrumentation-openai-v2_ubuntu-latest: + name: instrumentation-openai-v2 3.9 Ubuntu runs-on: ubuntu-latest steps: - name: Checkout repo @ SHA - ${{ github.sha }} @@ -50,10 +50,10 @@ jobs: run: pip install tox - name: Run tests - run: tox -e py39-test-instrumentation-openai -- -ra + run: tox -e py39-test-instrumentation-openai-v2 -- -ra - py310-test-instrumentation-openai_ubuntu-latest: - name: instrumentation-openai 3.10 Ubuntu + py310-test-instrumentation-openai-v2_ubuntu-latest: + name: instrumentation-openai-v2 3.10 Ubuntu runs-on: ubuntu-latest steps: - name: Checkout repo @ SHA - ${{ github.sha }} @@ -68,10 +68,10 @@ jobs: run: pip install tox - name: Run tests - run: tox -e py310-test-instrumentation-openai -- -ra + run: tox -e py310-test-instrumentation-openai-v2 -- -ra - py311-test-instrumentation-openai_ubuntu-latest: - name: instrumentation-openai 3.11 Ubuntu + py311-test-instrumentation-openai-v2_ubuntu-latest: + name: instrumentation-openai-v2 3.11 Ubuntu runs-on: ubuntu-latest steps: - name: Checkout repo @ SHA - ${{ github.sha }} @@ -86,10 +86,10 @@ jobs: run: pip install tox - name: Run tests - run: tox -e py311-test-instrumentation-openai -- -ra + run: tox -e py311-test-instrumentation-openai-v2 -- -ra - py312-test-instrumentation-openai_ubuntu-latest: - name: instrumentation-openai 3.12 Ubuntu + py312-test-instrumentation-openai-v2_ubuntu-latest: + name: instrumentation-openai-v2 3.12 Ubuntu runs-on: ubuntu-latest steps: - name: Checkout repo @ SHA - ${{ github.sha }} @@ -104,10 +104,10 @@ jobs: run: pip install tox - name: Run tests - run: tox -e py312-test-instrumentation-openai -- -ra + run: tox -e py312-test-instrumentation-openai-v2 -- -ra - pypy3-test-instrumentation-openai_ubuntu-latest: - name: instrumentation-openai pypy-3.8 Ubuntu + pypy3-test-instrumentation-openai-v2_ubuntu-latest: + name: instrumentation-openai-v2 pypy-3.8 Ubuntu runs-on: ubuntu-latest steps: - name: Checkout repo @ SHA - ${{ github.sha }} @@ -122,7 +122,7 @@ jobs: run: pip install tox - name: Run tests - run: tox -e pypy3-test-instrumentation-openai -- -ra + run: tox -e pypy3-test-instrumentation-openai-v2 -- -ra py38-test-resource-detector-container_ubuntu-latest: name: resource-detector-container 3.8 Ubuntu diff --git a/tox.ini b/tox.ini index 2c85207415..74e7146b1d 100644 --- a/tox.ini +++ b/tox.ini @@ -7,8 +7,8 @@ envlist = ; for specifying supported Python versions per package. ; instrumentation-openai - py3{8,9,10,11,12}-test-instrumentation-openai - pypy3-test-instrumentation-openai + py3{8,9,10,11,12}-test-instrumentation-openai-v2 + pypy3-test-instrumentation-openai-v2 lint-instrumentation-openai ; opentelemetry-resource-detector-container From d467eb1fae387d3f7fbed06026a1adb9cefe2dbb Mon Sep 17 00:00:00 2001 From: Ali Waleed Date: Thu, 17 Oct 2024 19:41:23 +0300 Subject: [PATCH 28/38] set attributes only when span is recording --- .../src/opentelemetry/instrumentation/openai_v2/patch.py | 5 +++-- .../src/opentelemetry/instrumentation/openai_v2/utils.py | 9 ++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/patch.py b/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/patch.py index ff7e3bbe01..2f3bc6b107 100644 --- a/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/patch.py +++ b/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/patch.py @@ -51,8 +51,9 @@ def traced_method(wrapped, instance, args, kwargs): span_name = f"{span_attributes[GenAIAttributes.GEN_AI_OPERATION_NAME]} {span_attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL]}" span = tracer.start_span(name=span_name, kind=SpanKind.CLIENT) - _set_input_attributes(span, span_attributes) - set_event_prompt(span, json.dumps(llm_prompts)) + if span.is_recording(): + _set_input_attributes(span, span_attributes) + set_event_prompt(span, json.dumps(llm_prompts)) try: result = wrapped(*args, **kwargs) diff --git a/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/utils.py b/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/utils.py index fcfe4d9665..bc7ff37e1e 100644 --- a/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/utils.py +++ b/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/utils.py @@ -21,7 +21,6 @@ from opentelemetry.semconv._incubating.attributes import ( gen_ai_attributes as GenAIAttributes, ) -from opentelemetry.trace import Span def silently_fail(func): @@ -101,7 +100,7 @@ def extract_tools_prompt(item): return calls -def set_event_prompt(span: Span, prompt): +def set_event_prompt(span, prompt): span.add_event( name="gen_ai.content.prompt", attributes={ @@ -110,12 +109,12 @@ def set_event_prompt(span: Span, prompt): ) -def set_span_attributes(span: Span, attributes: dict): +def set_span_attributes(span, attributes: dict): for field, value in attributes.model_dump(by_alias=True).items(): set_span_attribute(span, field, value) -def set_event_completion(span: Span, result_content): +def set_event_completion(span, result_content): span.add_event( name="gen_ai.content.completion", attributes={ @@ -124,7 +123,7 @@ def set_event_completion(span: Span, result_content): ) -def set_span_attribute(span: Span, name, value): +def set_span_attribute(span, name, value): if non_numerical_value_is_set(value) is False: return From ad7f1985d286421a05e03fa71a6e0ef92e8ef680 Mon Sep 17 00:00:00 2001 From: Ali Waleed Date: Thu, 17 Oct 2024 19:54:29 +0300 Subject: [PATCH 29/38] `model` fallback to `gpt-3.5-turbo` --- .../src/opentelemetry/instrumentation/openai_v2/utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/utils.py b/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/utils.py index bc7ff37e1e..7122465914 100644 --- a/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/utils.py +++ b/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/utils.py @@ -140,14 +140,15 @@ def non_numerical_value_is_set(value: Optional[Union[bool, str]]): def get_llm_request_attributes( kwargs, - model=None, operation_name=GenAIAttributes.GenAiOperationNameValues.CHAT.value, ): return { GenAIAttributes.GEN_AI_OPERATION_NAME: operation_name, GenAIAttributes.GEN_AI_SYSTEM: GenAIAttributes.GenAiSystemValues.OPENAI.value, - GenAIAttributes.GEN_AI_REQUEST_MODEL: model or kwargs.get("model"), + GenAIAttributes.GEN_AI_REQUEST_MODEL: kwargs.get( + "model", "gpt-3.5-turbo" + ), GenAIAttributes.GEN_AI_REQUEST_TEMPERATURE: kwargs.get("temperature"), GenAIAttributes.GEN_AI_REQUEST_TOP_P: kwargs.get("p") or kwargs.get("top_p"), From bbee1094288ef4c7db615624dfe92017093d8958 Mon Sep 17 00:00:00 2001 From: Ali Waleed Date: Sat, 19 Oct 2024 17:04:52 +0300 Subject: [PATCH 30/38] adjust `-v2` for linting --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index e02986e81e..c6bb6a17a4 100644 --- a/tox.ini +++ b/tox.ini @@ -9,7 +9,7 @@ envlist = ; instrumentation-openai py3{8,9,10,11,12}-test-instrumentation-openai-v2 pypy3-test-instrumentation-openai-v2 - lint-instrumentation-openai + lint-instrumentation-openai-v2 ; opentelemetry-resource-detector-container py3{8,9,10,11,12}-test-resource-detector-container From 9ac90f9f9b7bbc533fc46482e5efc2220a6a6631 Mon Sep 17 00:00:00 2001 From: Ali Waleed Date: Sat, 19 Oct 2024 17:08:45 +0300 Subject: [PATCH 31/38] make sure span is recording before setting attributes --- .../opentelemetry/instrumentation/openai_v2/patch.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/patch.py b/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/patch.py index 2f3bc6b107..40fad91cd9 100644 --- a/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/patch.py +++ b/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/patch.py @@ -65,15 +65,17 @@ def traced_method(wrapped, instance, args, kwargs): tool_calls=kwargs.get("tools") is not None, ) else: - _set_response_attributes(span, result) + if span.is_recording(): + _set_response_attributes(span, result) span.end() return result except Exception as error: span.set_status(Status(StatusCode.ERROR, str(error))) - span.set_attribute( - ErrorAttributes.ERROR_TYPE, type(error).__qualname__ - ) + if span.is_recording(): + span.set_attribute( + ErrorAttributes.ERROR_TYPE, type(error).__qualname__ + ) span.end() raise From 2549f257f677c3abc771e6239a33af4ff4a250a4 Mon Sep 17 00:00:00 2001 From: Ali Waleed Date: Sat, 19 Oct 2024 17:14:35 +0300 Subject: [PATCH 32/38] pass span_attributes when creating span inside `start_span` --- .../src/opentelemetry/instrumentation/openai_v2/patch.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/patch.py b/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/patch.py index 40fad91cd9..e139a799cb 100644 --- a/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/patch.py +++ b/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/patch.py @@ -50,7 +50,9 @@ def traced_method(wrapped, instance, args, kwargs): span_attributes = {**get_llm_request_attributes(kwargs)} span_name = f"{span_attributes[GenAIAttributes.GEN_AI_OPERATION_NAME]} {span_attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL]}" - span = tracer.start_span(name=span_name, kind=SpanKind.CLIENT) + span = tracer.start_span( + name=span_name, kind=SpanKind.CLIENT, attributes=span_attributes + ) if span.is_recording(): _set_input_attributes(span, span_attributes) set_event_prompt(span, json.dumps(llm_prompts)) From 1dacf8dbfaaa8dcda0449fdf217e524b2e187b7b Mon Sep 17 00:00:00 2001 From: Ali Waleed Date: Mon, 21 Oct 2024 11:16:48 +0300 Subject: [PATCH 33/38] adjust unwrap + add pydantic to test reqs --- .../src/opentelemetry/instrumentation/openai_v2/__init__.py | 4 +++- .../test-requirements.txt | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/__init__.py b/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/__init__.py index 05732a010f..4583eb4981 100644 --- a/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/__init__.py @@ -75,4 +75,6 @@ def _instrument(self, **kwargs): ) def _uninstrument(self, **kwargs): - unwrap("openai.resources.chat.completions.Completions", "create") + import openai + + unwrap(openai, "resources.chat.completions.Completions.create") diff --git a/instrumentation/opentelemetry-instrumentation-openai-v2/test-requirements.txt b/instrumentation/opentelemetry-instrumentation-openai-v2/test-requirements.txt index 9385ece64e..a8124a8f89 100644 --- a/instrumentation/opentelemetry-instrumentation-openai-v2/test-requirements.txt +++ b/instrumentation/opentelemetry-instrumentation-openai-v2/test-requirements.txt @@ -1,4 +1,5 @@ openai==1.0.0 +pydantic==2.8.2 Deprecated==1.2.14 importlib-metadata==6.11.0 packaging==24.0 From 404841044c29d9cd1617c1f2d36321b79186fcb9 Mon Sep 17 00:00:00 2001 From: Ali Waleed Date: Mon, 21 Oct 2024 12:28:41 +0300 Subject: [PATCH 34/38] bump openai support to `1.26.0` --- .../opentelemetry/instrumentation/openai_v2/package.py | 2 +- .../test-requirements.txt | 2 +- .../tests/test_chat_completions.py | 9 ++------- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/package.py b/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/package.py index b1df3b87b2..b53e25f7df 100644 --- a/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/package.py +++ b/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/package.py @@ -13,4 +13,4 @@ # limitations under the License. -_instruments = ("openai >= 1.0.0",) +_instruments = ("openai >= 1.26.0",) diff --git a/instrumentation/opentelemetry-instrumentation-openai-v2/test-requirements.txt b/instrumentation/opentelemetry-instrumentation-openai-v2/test-requirements.txt index a8124a8f89..198226dcfa 100644 --- a/instrumentation/opentelemetry-instrumentation-openai-v2/test-requirements.txt +++ b/instrumentation/opentelemetry-instrumentation-openai-v2/test-requirements.txt @@ -1,4 +1,4 @@ -openai==1.0.0 +openai==1.26.0 pydantic==2.8.2 Deprecated==1.2.14 importlib-metadata==6.11.0 diff --git a/instrumentation/opentelemetry-instrumentation-openai-v2/tests/test_chat_completions.py b/instrumentation/opentelemetry-instrumentation-openai-v2/tests/test_chat_completions.py index 2c37237fc3..2b0f6a7f8c 100644 --- a/instrumentation/opentelemetry-instrumentation-openai-v2/tests/test_chat_completions.py +++ b/instrumentation/opentelemetry-instrumentation-openai-v2/tests/test_chat_completions.py @@ -20,9 +20,7 @@ def test_chat_completion(exporter, openai_client): response = openai_client.chat.completions.create(**kwargs) spans = exporter.get_finished_spans() - # ignore by default added spans etc: http spans. - # we are only interested in the last span which is the chat completion span - chat_completion_span = spans[-1] + chat_completion_span = spans[0] # assert that the span name is correct assert chat_completion_span.name == f"chat {llm_model_value}" @@ -34,7 +32,6 @@ def test_chat_completion(exporter, openai_client): response_id = attributes[GenAIAttributes.GEN_AI_RESPONSE_ID] input_tokens = attributes[GenAIAttributes.GEN_AI_USAGE_INPUT_TOKENS] output_tokens = attributes[GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] - # assert that the attributes are correct assert ( operation_name == GenAIAttributes.GenAiOperationNameValues.CHAT.value @@ -106,9 +103,7 @@ def test_chat_completion_streaming(exporter, openai_client): response_stream_id = chunk.id spans = exporter.get_finished_spans() - # ignore by default added spans etc: http spans. - # we are only interested in the last span which is the chat completion span - streaming_span = spans[-1] + streaming_span = spans[0] assert streaming_span.name == f"chat {llm_model_value}" attributes = streaming_span.attributes From 8fc43367dd973cffaae4014e0c2898ce39b932be Mon Sep 17 00:00:00 2001 From: Ali Waleed Date: Mon, 21 Oct 2024 12:35:45 +0300 Subject: [PATCH 35/38] run `tox -e generate` & `tox -e generate-workflows` --- .github/workflows/lint_0.yml | 6 +++--- instrumentation/README.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/lint_0.yml b/.github/workflows/lint_0.yml index 8fcf16dae1..1fd3198785 100644 --- a/.github/workflows/lint_0.yml +++ b/.github/workflows/lint_0.yml @@ -16,8 +16,8 @@ env: jobs: - lint-instrumentation-openai: - name: instrumentation-openai + lint-instrumentation-openai-v2: + name: instrumentation-openai-v2 runs-on: ubuntu-latest steps: - name: Checkout repo @ SHA - ${{ github.sha }} @@ -32,7 +32,7 @@ jobs: run: pip install tox - name: Run tests - run: tox -e lint-instrumentation-openai + run: tox -e lint-instrumentation-openai-v2 lint-resource-detector-container: name: resource-detector-container diff --git a/instrumentation/README.md b/instrumentation/README.md index 9cdc0a52bb..db437fe518 100644 --- a/instrumentation/README.md +++ b/instrumentation/README.md @@ -29,7 +29,7 @@ | [opentelemetry-instrumentation-logging](./opentelemetry-instrumentation-logging) | logging | No | experimental | [opentelemetry-instrumentation-mysql](./opentelemetry-instrumentation-mysql) | mysql-connector-python >= 8.0, < 10.0 | No | experimental | [opentelemetry-instrumentation-mysqlclient](./opentelemetry-instrumentation-mysqlclient) | mysqlclient < 3 | No | experimental -| [opentelemetry-instrumentation-openai-v2](./opentelemetry-instrumentation-openai-v2) | openai >= 1.0.0 | No | experimental +| [opentelemetry-instrumentation-openai-v2](./opentelemetry-instrumentation-openai-v2) | openai >= 1.26.0 | No | experimental | [opentelemetry-instrumentation-pika](./opentelemetry-instrumentation-pika) | pika >= 0.12.0 | No | experimental | [opentelemetry-instrumentation-psycopg](./opentelemetry-instrumentation-psycopg) | psycopg >= 3.1.0 | No | experimental | [opentelemetry-instrumentation-psycopg2](./opentelemetry-instrumentation-psycopg2) | psycopg2 >= 2.7.3.1 | No | experimental From 9e273f6b864ec2c63f18535437c95f342e4f1c72 Mon Sep 17 00:00:00 2001 From: Ali Waleed Date: Tue, 22 Oct 2024 07:03:14 +0300 Subject: [PATCH 36/38] add uninstrument in tests + remove any none values from span attributes --- .../src/opentelemetry/instrumentation/openai_v2/patch.py | 7 ------- .../src/opentelemetry/instrumentation/openai_v2/utils.py | 5 ++++- .../tests/conftest.py | 5 +++++ 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/patch.py b/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/patch.py index e139a799cb..be3534e233 100644 --- a/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/patch.py +++ b/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/patch.py @@ -54,7 +54,6 @@ def traced_method(wrapped, instance, args, kwargs): name=span_name, kind=SpanKind.CLIENT, attributes=span_attributes ) if span.is_recording(): - _set_input_attributes(span, span_attributes) set_event_prompt(span, json.dumps(llm_prompts)) try: @@ -84,12 +83,6 @@ def traced_method(wrapped, instance, args, kwargs): return traced_method -@silently_fail -def _set_input_attributes(span, attributes): - for field, value in attributes.items(): - set_span_attribute(span, field, value) - - @silently_fail def _set_response_attributes(span, result): set_span_attribute( diff --git a/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/utils.py b/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/utils.py index 7122465914..7d0eddc58d 100644 --- a/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/utils.py +++ b/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/utils.py @@ -143,7 +143,7 @@ def get_llm_request_attributes( operation_name=GenAIAttributes.GenAiOperationNameValues.CHAT.value, ): - return { + attributes = { GenAIAttributes.GEN_AI_OPERATION_NAME: operation_name, GenAIAttributes.GEN_AI_SYSTEM: GenAIAttributes.GenAiSystemValues.OPENAI.value, GenAIAttributes.GEN_AI_REQUEST_MODEL: kwargs.get( @@ -160,3 +160,6 @@ def get_llm_request_attributes( "frequency_penalty" ), } + + # filter out None values + return {k: v for k, v in attributes.items() if v is not None} diff --git a/instrumentation/opentelemetry-instrumentation-openai-v2/tests/conftest.py b/instrumentation/opentelemetry-instrumentation-openai-v2/tests/conftest.py index c6e1ee86f4..976d58d7d8 100644 --- a/instrumentation/opentelemetry-instrumentation-openai-v2/tests/conftest.py +++ b/instrumentation/opentelemetry-instrumentation-openai-v2/tests/conftest.py @@ -56,6 +56,11 @@ def instrument(): OpenAIInstrumentor().instrument() +@pytest.fixture(scope="session", autouse=True) +def uninstrument(): + OpenAIInstrumentor().uninstrument() + + def scrub_response_headers(response): """ This scrubs sensitive response headers. Note they are case-sensitive! From 8e667dee8d867775db9297943923a4bac3a617a2 Mon Sep 17 00:00:00 2001 From: Ali Waleed Date: Tue, 22 Oct 2024 07:08:08 +0300 Subject: [PATCH 37/38] cleanup --- .../src/opentelemetry/instrumentation/openai_v2/patch.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/patch.py b/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/patch.py index be3534e233..e139a799cb 100644 --- a/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/patch.py +++ b/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/patch.py @@ -54,6 +54,7 @@ def traced_method(wrapped, instance, args, kwargs): name=span_name, kind=SpanKind.CLIENT, attributes=span_attributes ) if span.is_recording(): + _set_input_attributes(span, span_attributes) set_event_prompt(span, json.dumps(llm_prompts)) try: @@ -83,6 +84,12 @@ def traced_method(wrapped, instance, args, kwargs): return traced_method +@silently_fail +def _set_input_attributes(span, attributes): + for field, value in attributes.items(): + set_span_attribute(span, field, value) + + @silently_fail def _set_response_attributes(span, result): set_span_attribute( From 592c18e981980a7c9f421c901ee15b8db83591cd Mon Sep 17 00:00:00 2001 From: Ali Waleed Date: Tue, 22 Oct 2024 10:29:36 +0300 Subject: [PATCH 38/38] adjust `unwrap` --- .../src/opentelemetry/instrumentation/openai_v2/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/__init__.py b/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/__init__.py index 4583eb4981..64f74eaefe 100644 --- a/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/__init__.py @@ -77,4 +77,4 @@ def _instrument(self, **kwargs): def _uninstrument(self, **kwargs): import openai - unwrap(openai, "resources.chat.completions.Completions.create") + unwrap(openai.resources.chat.completions.Completions, "create")