From 49fb41ec8bbae44cfc4b6d40b18112c224f13572 Mon Sep 17 00:00:00 2001 From: Aaron Abbott Date: Wed, 12 May 2021 16:23:17 +0000 Subject: [PATCH] add CompositeCloudTraceW3CPropagator and update docs --- .../cloud_trace_propagator.rst | 11 + .../cloud_trace_propagator/README.rst | 42 ---- .../examples/cloud_trace_propagator/client.py | 41 ---- .../examples/cloud_trace_propagator/server.py | 55 ----- docs/examples/flask_e2e/README.rst | 2 + docs/examples/flask_e2e/client.py | 4 +- docs/examples/flask_e2e/server.py | 4 +- docs/index.rst | 5 +- opentelemetry-propagator-gcp/CHANGELOG.md | 2 + .../cloud_trace_propagator/__init__.py | 147 +++++++++++++- .../tests/test_cloud_trace_propagator.py | 189 +++++++++++++++++- 11 files changed, 351 insertions(+), 151 deletions(-) create mode 100644 docs/cloud_trace_propagator/cloud_trace_propagator.rst delete mode 100644 docs/examples/cloud_trace_propagator/README.rst delete mode 100644 docs/examples/cloud_trace_propagator/client.py delete mode 100644 docs/examples/cloud_trace_propagator/server.py diff --git a/docs/cloud_trace_propagator/cloud_trace_propagator.rst b/docs/cloud_trace_propagator/cloud_trace_propagator.rst new file mode 100644 index 00000000..6a2b2ab3 --- /dev/null +++ b/docs/cloud_trace_propagator/cloud_trace_propagator.rst @@ -0,0 +1,11 @@ +OpenTelemetry Google Cloud Trace Propagator +=========================================== + +.. image:: https://badge.fury.io/py/opentelemetry-propagator-gcp.svg + :target: https://badge.fury.io/py/opentelemetry-propagator-trace + +.. automodule:: opentelemetry.propagators.cloud_trace_propagator + :members: + :undoc-members: + :show-inheritance: + :noindex: diff --git a/docs/examples/cloud_trace_propagator/README.rst b/docs/examples/cloud_trace_propagator/README.rst deleted file mode 100644 index dfeb8aeb..00000000 --- a/docs/examples/cloud_trace_propagator/README.rst +++ /dev/null @@ -1,42 +0,0 @@ -Cloud Trace Propagator Example -============================== - -These examples show how to make OpenTelemetry use the -``X-Cloud-Trace-Context`` header for context propagation. - - -Basic Example -------------- - -To use this feature you first need to: - * Create a Google Cloud project. You can `create one here `_. - * Enable Cloud Trace API (listed in the Cloud Console as Stackdriver Trace API) in the project `here `_. If the page says "API Enabled" then you're done! No need to do anything. - * Enable Default Application Credentials by creating setting `GOOGLE_APPLICATION_CREDENTIALS `_ or by `installing gcloud sdk `_ and calling ``gcloud auth application-default login``. - -* Installation - -.. code-block:: sh - - pip install opentelemetry-api \ - opentelemetry-sdk \ - opentelemetry-exporter-gcp-trace \ - opentelemetry-exporter-gcp-monitoring \ - opentelemetry-propagator-gcp - -* Create a server that uses the Cloud Trace header - -.. literalinclude:: server.py - :language: python - :lines: 1- - -* Make a client that uses the Cloud Trace header - -.. literalinclude:: client.py - :language: python - :lines: 1- - - -Checking Output --------------------------- - -After running these examples, you can go to `Cloud Trace overview `_ to see the results. diff --git a/docs/examples/cloud_trace_propagator/client.py b/docs/examples/cloud_trace_propagator/client.py deleted file mode 100644 index 2efc83a9..00000000 --- a/docs/examples/cloud_trace_propagator/client.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2021 The OpenTelemetry Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import opentelemetry.ext.requests -import requests -from opentelemetry import trace -from opentelemetry.exporter.cloud_trace import CloudTraceSpanExporter -from opentelemetry.propagate import set_global_textmap -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import SimpleSpanProcessor -from opentelemetry.tools.cloud_trace_propagator import ( - CloudTraceFormatPropagator, -) - -# Instrumenting requests -opentelemetry.ext.requests.RequestsInstrumentor().instrument() - -# Tracer boilerplate -trace.set_tracer_provider(TracerProvider()) -trace.get_tracer_provider().add_span_processor( - SimpleSpanProcessor(CloudTraceSpanExporter()) -) - -# Using the X-Cloud-Trace-Context header -set_global_textmap(CloudTraceFormatPropagator()) - -tracer = trace.get_tracer(__name__) -with tracer.start_as_current_span("client_span"): - response = requests.get("http://localhost:5000/") diff --git a/docs/examples/cloud_trace_propagator/server.py b/docs/examples/cloud_trace_propagator/server.py deleted file mode 100644 index 8f92b86c..00000000 --- a/docs/examples/cloud_trace_propagator/server.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2021 The OpenTelemetry Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import opentelemetry.ext.requests -from opentelemetry import trace -from opentelemetry.exporter.cloud_trace import CloudTraceSpanExporter -from opentelemetry.ext.flask import FlaskInstrumentor -from opentelemetry.propagate import set_global_textmap -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import SimpleSpanProcessor -from opentelemetry.tools.cloud_trace_propagator import ( - CloudTraceFormatPropagator, -) - -from flask import Flask - -# Instrumenting requests -opentelemetry.ext.requests.RequestsInstrumentor().instrument() - -# Instrumenting flask -app = Flask(__name__) -FlaskInstrumentor().instrument_app(app) - -# Tracer boilerplate -trace.set_tracer_provider(TracerProvider()) -trace.get_tracer_provider().add_span_processor( - SimpleSpanProcessor(CloudTraceSpanExporter()) -) - -# Using the X-Cloud-Trace-Context header -set_global_textmap(CloudTraceFormatPropagator()) - - -@app.route("/") -def hello_world(): - tracer = trace.get_tracer(__name__) - with tracer.start_as_current_span("server_span"): - return "Hello World!" - - -if __name__ == "__main__": - port = 5000 - app.run(port=port) diff --git a/docs/examples/flask_e2e/README.rst b/docs/examples/flask_e2e/README.rst index a3a84337..4af5aab9 100644 --- a/docs/examples/flask_e2e/README.rst +++ b/docs/examples/flask_e2e/README.rst @@ -1,3 +1,5 @@ +.. _flask-e2e: + ============================= End-to-End Example with Flask ============================= diff --git a/docs/examples/flask_e2e/client.py b/docs/examples/flask_e2e/client.py index b9fd3ebe..7da01ac3 100644 --- a/docs/examples/flask_e2e/client.py +++ b/docs/examples/flask_e2e/client.py @@ -19,12 +19,12 @@ from opentelemetry.instrumentation.requests import RequestsInstrumentor from opentelemetry.propagate import set_global_textmap from opentelemetry.propagators.cloud_trace_propagator import ( - CloudTraceFormatPropagator, + CompositeCloudTraceW3CPropagator, ) from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor -set_global_textmap(CloudTraceFormatPropagator()) +set_global_textmap(CompositeCloudTraceW3CPropagator()) tracer_provider = TracerProvider() cloud_trace_exporter = CloudTraceSpanExporter() diff --git a/docs/examples/flask_e2e/server.py b/docs/examples/flask_e2e/server.py index 5a5ac938..d57d5047 100644 --- a/docs/examples/flask_e2e/server.py +++ b/docs/examples/flask_e2e/server.py @@ -23,7 +23,7 @@ from opentelemetry.instrumentation.flask import FlaskInstrumentor from opentelemetry.propagate import set_global_textmap from opentelemetry.propagators.cloud_trace_propagator import ( - CloudTraceFormatPropagator, + CompositeCloudTraceW3CPropagator, ) from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor @@ -32,7 +32,7 @@ # [START opentelemetry_flask_setup_propagator] -set_global_textmap(CloudTraceFormatPropagator()) +set_global_textmap(CompositeCloudTraceW3CPropagator()) # [END opentelemetry_flask_setup_propagator] diff --git a/docs/index.rst b/docs/index.rst index a8746196..d39c4c43 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -47,11 +47,12 @@ To install the GCP trace propagator: .. toctree:: :maxdepth: 1 - :caption: Exporters - :name: exporters + :caption: Packages + :name: packages cloud_monitoring/cloud_monitoring cloud_trace/cloud_trace + cloud_trace_propagator/cloud_trace_propagator .. toctree:: diff --git a/opentelemetry-propagator-gcp/CHANGELOG.md b/opentelemetry-propagator-gcp/CHANGELOG.md index 00917ecc..565c4ce7 100644 --- a/opentelemetry-propagator-gcp/CHANGELOG.md +++ b/opentelemetry-propagator-gcp/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +- Add CompositeCloudTraceW3CPropagator + ([#140](https://github.com/GoogleCloudPlatform/opentelemetry-operations-python/pull/140)) - Fix propagator modifying context if failed to extract ([#139](https://github.com/GoogleCloudPlatform/opentelemetry-operations-python/pull/139)) diff --git a/opentelemetry-propagator-gcp/src/opentelemetry/propagators/cloud_trace_propagator/__init__.py b/opentelemetry-propagator-gcp/src/opentelemetry/propagators/cloud_trace_propagator/__init__.py index ec300da0..c4b1bf08 100644 --- a/opentelemetry-propagator-gcp/src/opentelemetry/propagators/cloud_trace_propagator/__init__.py +++ b/opentelemetry-propagator-gcp/src/opentelemetry/propagators/cloud_trace_propagator/__init__.py @@ -11,21 +11,166 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# + +""" +This module contains OpenTelemetry propagators with support for the Cloud Trace +`X-Cloud-Trace-Context`_ format. + +It is recommended to use :class:`CompositeCloudTraceW3CPropagator`, which +combines the default OpenTelemetry supported propagation mechanisms (`W3C +TraceContext `_ and `Baggage +`_) with :class:`CloudTraceFormatPropagator`. +This way, your application will be able to propagate context to and from Google +and non-Google services. + +See :ref:`flask-e2e` for a full example using this propagator. + +Usage +----- + +.. code-block:: python + + from opentelemetry.propagate import set_global_textmap + from opentelemetry.propagators.cloud_trace_propagator import ( + CompositeCloudTraceW3CPropagator, + ) + + # set as the global OpenTelemetry propagator + set_global_textmap(CompositeCloudTraceW3CPropagator()) + +.. _X-Cloud-Trace-Context: https://cloud.google.com/trace/docs/setup#force-trace +""" import re import typing +import logging import opentelemetry.trace as trace +from opentelemetry.baggage.propagation import W3CBaggagePropagator from opentelemetry.context.context import Context from opentelemetry.propagators import textmap from opentelemetry.trace.span import SpanContext, TraceFlags, format_trace_id +from opentelemetry.propagators import composite, textmap +from opentelemetry.trace.propagation import ( + get_current_span, + set_span_in_context, +) +from opentelemetry.trace.propagation.tracecontext import ( + TraceContextTextMapPropagator, +) +from opentelemetry.trace.span import ( + INVALID_SPAN, + DEFAULT_TRACE_STATE, + INVALID_SPAN_CONTEXT, + SpanContext, + TraceFlags, + format_trace_id, +) _TRACE_CONTEXT_HEADER_NAME = "x-cloud-trace-context" _TRACE_CONTEXT_HEADER_FORMAT = r"(?P[0-9a-f]{32})\/(?P[\d]{1,20});o=(?P\d+)" _TRACE_CONTEXT_HEADER_RE = re.compile(_TRACE_CONTEXT_HEADER_FORMAT) _FIELDS = {_TRACE_CONTEXT_HEADER_NAME} +logger = logging.getLogger(__name__) + + +class CloudTraceW3CPropagator(textmap.TextMapPropagator): + """Propagator to support both OTel W3C defaults and `X-Cloud-Trace-Context`_ + format. + + We recommend using this propagator to support a wide range of propagation + scenarios. This propagator combines the output of: + + - W3C Trace Context propagator + - W3C Baggage propagator + - Cloud Trace format propagator + + If the trace and span IDs output by W3C Trace Context and + `X-Cloud-Trace-Context`_ match, the TraceFlags and TraceState are merged as + well. + + .. _X-Cloud-Trace-Context: https://cloud.google.com/trace/docs/setup#force-trace + """ + + def __init__(self) -> None: + self._trace_context_propagator = TraceContextTextMapPropagator() + self._baggage_propagator = W3CBaggagePropagator() + self._cloud_trace_propagator = CloudTraceFormatPropagator() + + def extract( + self, + carrier: textmap.CarrierT, + context: typing.Optional[Context] = None, + getter: textmap.Getter = textmap.default_getter, + ) -> Context: + w3c_context = self._trace_context_propagator.extract( + carrier, context, getter + ) + w3c_context = self._baggage_propagator.extract( + carrier, w3c_context, getter + ) + cloud_trace_context = self._cloud_trace_propagator.extract( + carrier, w3c_context, getter + ) + + traceparent_span_context = get_current_span( + w3c_context + ).get_span_context() + cloud_trace_span_context = get_current_span( + cloud_trace_context + ).get_span_context() + + combined_context = cloud_trace_context + + # If the cloud trace and w3c span contexts have the same trace and span + # IDs, merge in w3c trace flags and trace state + if ( + traceparent_span_context is not INVALID_SPAN_CONTEXT + and cloud_trace_span_context is not INVALID_SPAN_CONTEXT + ): + if ( + traceparent_span_context.trace_id + == cloud_trace_span_context.trace_id + and traceparent_span_context.span_id + == cloud_trace_span_context.span_id + ): + combined_context = trace.set_span_in_context( + trace.NonRecordingSpan( + SpanContext( + trace_id=cloud_trace_span_context.trace_id, + span_id=cloud_trace_span_context.span_id, + is_remote=True, + trace_flags=TraceFlags( + cloud_trace_span_context.trace_flags + | traceparent_span_context.trace_flags + ), + trace_state=traceparent_span_context.trace_state, + ) + ), + combined_context, + ) + else: + logger.warning( + "Trace and span IDs from traceparent and cloud trace propagators do not match. " + "Using the value from cloud trace propagator." + ) + + return context + + def inject( + self, + carrier: textmap.CarrierT, + context: typing.Optional[Context] = None, + setter: textmap.Setter = textmap.default_setter, + ) -> None: + for propagator in ( + self._trace_context_propagator, + self._baggage_propagator, + self._cloud_trace_propagator, + ): + propagator.inject(carrier=carrier, context=context, setter=setter) + class CloudTraceFormatPropagator(textmap.TextMapPropagator): """This class is for injecting into a carrier the SpanContext in Google diff --git a/opentelemetry-propagator-gcp/tests/test_cloud_trace_propagator.py b/opentelemetry-propagator-gcp/tests/test_cloud_trace_propagator.py index b5db753c..588df251 100644 --- a/opentelemetry-propagator-gcp/tests/test_cloud_trace_propagator.py +++ b/opentelemetry-propagator-gcp/tests/test_cloud_trace_propagator.py @@ -11,15 +11,22 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# + import unittest +<<<<<<< HEAD +======= +from typing import Type +from unittest import mock +>>>>>>> 1ecda74 (add CompositeCloudTraceW3CPropagator and update docs) import opentelemetry.trace as trace +from opentelemetry import baggage from opentelemetry.context import get_current from opentelemetry.context.context import Context from opentelemetry.propagators.cloud_trace_propagator import ( _TRACE_CONTEXT_HEADER_NAME, CloudTraceFormatPropagator, + CompositeCloudTraceW3CPropagator, ) from opentelemetry.propagators.textmap import default_getter from opentelemetry.trace.span import ( @@ -27,29 +34,40 @@ INVALID_TRACE_ID, SpanContext, TraceFlags, + TraceState, + format_span_id, format_trace_id, ) +_TRACEPARENT_HEADER_NAME = "traceparent" +_BAGGAGE_HEADER_NAME = "baggage" + class TestCloudTraceFormatPropagator(unittest.TestCase): + PropagatorCls: Type[ + CloudTraceFormatPropagator + ] = CloudTraceFormatPropagator + def setUp(self): - self.propagator = CloudTraceFormatPropagator() + self.propagator = self.PropagatorCls() self.valid_trace_id = 281017822499060589596062859815111849546 self.valid_span_id = 17725314949316355921 self.too_long_id = 111111111111111111111111111111111111111111111 - def _extract(self, header_value): + def _extract(self, header_value, header_key=_TRACE_CONTEXT_HEADER_NAME): """Test helper""" - header = {_TRACE_CONTEXT_HEADER_NAME: [header_value]} + header = {header_key: [header_value]} new_context = self.propagator.extract( carrier=header, getter=default_getter ) return new_context - def _extract_span_context(self, header_value): + def _extract_span_context( + self, header_value, header_key=_TRACE_CONTEXT_HEADER_NAME, + ): """Test helper""" return trace.get_current_span( - self._extract(header_value) + self._extract(header_value, header_key) ).get_span_context() def _inject(self, span=None): @@ -229,3 +247,162 @@ def test_inject_with_valid_context(self): format_trace_id(self.valid_trace_id), self.valid_span_id, 1, ), ) + +class TestCompositeCloudTraceW3CPropagator(TestCloudTraceFormatPropagator): + PropagatorCls = CompositeCloudTraceW3CPropagator + + baggage_key = "hello" + baggage_value = "world" + + def test_inject_tracecontext(self): + span_context = SpanContext( + trace_id=self.valid_trace_id, + span_id=self.valid_span_id, + is_remote=True, + trace_flags=TraceFlags(1), + ) + ctx = trace.set_span_in_context( + trace.NonRecordingSpan(span_context), get_current() + ) + output = {} + self.propagator.inject(output, context=ctx) + + self.assertIn(_TRACEPARENT_HEADER_NAME, output) + self.assertEqual( + output[_TRACEPARENT_HEADER_NAME], + "00-{}-{}-01".format( + format_trace_id(self.valid_trace_id), + format_span_id(self.valid_span_id), + ), + ) + + def test_extract_tracecontext(self): + header = "00-{}-{}-01".format( + format_trace_id(self.valid_trace_id), + format_span_id(self.valid_span_id), + ) + new_span_context = self._extract_span_context( + header, _TRACEPARENT_HEADER_NAME + ) + self.assertEqual(new_span_context.trace_id, self.valid_trace_id) + self.assertEqual(new_span_context.span_id, self.valid_span_id) + self.assertEqual(new_span_context.trace_flags, TraceFlags(1)) + self.assertTrue(new_span_context.is_remote) + + def test_inject_baggage(self): + ctx = baggage.set_baggage( + self.baggage_key, self.baggage_value, get_current() + ) + output = {} + self.propagator.inject(output, context=ctx) + self.assertIn(_BAGGAGE_HEADER_NAME, output) + self.assertEqual( + output[_BAGGAGE_HEADER_NAME], + "{}={}".format(self.baggage_key, self.baggage_value), + ) + + def test_extract_baggage(self): + header = "{}={}".format(self.baggage_key, self.baggage_value) + new_context = self._extract(header, _BAGGAGE_HEADER_NAME) + self.assertEqual( + baggage.get_baggage(self.baggage_key, new_context), + self.baggage_value, + ) + + def test_merge_contexts_if_already_present_and_match(self): + """This could happen in the case of a composite propagator of + traceparent + x-cloud-trace-context. They could represent the same + remote parent, but one of the two forces sampling (trace flags 0x01). We + should respect that. Also, tracestate header is not supported by + x-cloud-trace-context, so we should preserve it. + """ + + existing_trace_state = TraceState([("hello", "world")]) + + def extract_with_existing( + existing_traceflags: TraceFlags, + cloudtrace_traceflags: TraceFlags, + existing_trace_id: int, + cloudtrace_trace_id: int, + ) -> SpanContext: + existing_span = trace.NonRecordingSpan( + SpanContext( + trace_id=existing_trace_id, + span_id=self.valid_span_id, + is_remote=True, + trace_flags=existing_traceflags, + trace_state=existing_trace_state, + ) + ) + headers = { + _TRACE_CONTEXT_HEADER_NAME: [ + "{}/{};o={}".format( + format_trace_id(cloudtrace_trace_id), + self.valid_span_id, + cloudtrace_traceflags, + ) + ] + } + existing_context = set_span_in_context(existing_span, Context()) + return get_current_span( + self.propagator.extract( + carrier=headers, context=existing_context + ) + ).get_span_context() + + # trace flags and trace state should be preserved with matching span contexts + new_span_context = extract_with_existing( + TraceFlags(1), + TraceFlags(0), + self.valid_trace_id, + self.valid_trace_id, + ) + self.assertEqual(new_span_context.trace_flags, TraceFlags(1)) + self.assertEqual(new_span_context.trace_state, existing_trace_state) + + # trace flags and trace state should be preserved with matching span contexts + new_span_context = extract_with_existing( + TraceFlags(0), + TraceFlags(1), + self.valid_trace_id, + self.valid_trace_id, + ) + self.assertEqual(new_span_context.trace_flags, TraceFlags(1)) + self.assertEqual(new_span_context.trace_state, existing_trace_state) + + # everything should be overwritten with different span contexts + new_span_context = extract_with_existing( + TraceFlags(1), TraceFlags(0), self.valid_trace_id, 0xDEADBEEF, + ) + self.assertEqual(new_span_context.trace_flags, TraceFlags(0)) + self.assertNotEqual(new_span_context.trace_state, existing_trace_state) + + @mock.patch("opentelemetry.propagators.cloud_trace_propagator.logger") + def test_warn_if_overwriting_existing_span_context( + self, mock_logger: mock.Mock + ): + headers = { + _TRACE_CONTEXT_HEADER_NAME: [ + "{}/{};o=1".format( + format_trace_id(self.valid_trace_id), self.valid_span_id + ) + ] + } + + # should not warn if not overwriting + self.propagator.extract(carrier=headers, context=Context()) + mock_logger.warning.assert_not_called() + + # should warn if overwriting + existing_span = trace.NonRecordingSpan( + SpanContext( + trace_id=self.valid_trace_id, + span_id=self.valid_span_id, + is_remote=True, + trace_flags=TraceFlags(1), + ) + ) + context = set_span_in_context(existing_span, Context()) + self.propagator.extract(carrier=headers, context=context) + mock_logger.warning.assert_called_once() +