Add true auto-instrumentation support to opentelemetry-instrument

This commit extends the instrument command so it automatically
configures tracing with a provider, span processor and exporter. Most of
the component used can be customized with env vars or CLI arguments.

Details can be found on opentelemetry-instrumentation's README package.

Fixes open-telemetry#663

instrumentation/opentelemetry-instrumentation-django/src/opentelemetry/instrumentation/django/middleware.py

@@ -43,8 +43,7 @@
 
 
 class _DjangoMiddleware(MiddlewareMixin):
-    """Django Middleware for OpenTelemetry
-    """
+    """Django Middleware for OpenTelemetry"""
 
     _environ_activation_key = (
         "opentelemetry-instrumentor-django.activation_key"

opentelemetry-instrumentation/CHANGELOG.md

@@ -2,7 +2,8 @@
 
 ## Unreleased
 
-- Fixed boostrap command to correctly install opentelemetry-instrumentation-falcon instead of opentelemetry-instrumentation-flask
+- Fixed boostrap command to correctly install opentelemetry-instrumentation-falcon instead of opentelemetry-instrumentation-flask. ([#1138](https://github.com/open-telemetry/opentelemetry-python/pull/1138))
+- Added support for `OTEL_EXPORTER` to the `opentelemetry-instrument` command ([#1036](https://github.com/open-telemetry/opentelemetry-python/pull/1036))
 
 ## Version 0.13b0
 

opentelemetry-instrumentation/README.rst

@@ -16,30 +16,70 @@ Installation
 
 This package provides a couple of commands that help automatically instruments a program:
 
+
 opentelemetry-instrument
 ------------------------
 
 ::
 
     opentelemetry-instrument python program.py
 
+The instrument command will try to automatically detect packages used by your python program
+and when possible, apply automatic tracing instrumentation on them. This means your program
+will get automatic distrubuted tracing for free without having to make any code changes
+at all. This will also configure a global tracer and tracing exporter without you having to
+make any code changes. By default, the instrument command will use the OTLP exporter but
+this can be overrided when needed.
+
+The command supports the following configuration options as CLI arguments and environments vars:
+
+
+* ``--exporter`` or ``OTEL_EXPORTER``
+
+Used to specify which trace exporter to use. Can be set to one or more
+of the well-known exporter names (see below) or a fully
+qualified Python import path to a span  exporter implementation.
+
+    - Defaults to `otlp`.
+    - Can be set to `none` to disbale automatic tracer initialization. 
+
+You can pass multiple values to configure multiple exporters e.g, ``zipkin,prometheus`` 
+
+Well known trace exporter names:
+
+    - datadog
+    - jaeger
+    - opencensus
+    - otlp
+    - otlp_span
+    - otlp_metric
+    - zipkin
+
+``otlp`` is an alias for ``otlp_span,otlp_metric``.
+
+* ``--service-name`` or ``OTEL_SERVICE_NAME``
+
+When present the value is passed on to the relevant exporter initializer as ``service_name`` argument.
+
 The code in ``program.py`` needs to use one of the packages for which there is
 an OpenTelemetry integration. For a list of the available integrations please
 check `here <https://opentelemetry-python.readthedocs.io/en/stable/index.html#integrations>`_
 
+Examples
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+::
+
+    opentelemetry-instrument -e otlp flask run --port=3000
 
-opentelemetry-bootstrap
------------------------
+The above command will pass ``-e otlp`` to the instrument command and ``--port=3000`` to ``flask run``.
 
 ::
 
-    opentelemetry-bootstrap --action=install|requirements
+    opentelemetry-instrument -e zipkin,otlp celery -A tasks worker --loglevel=info
 
-This commands inspects the active Python site-packages and figures out which
-instrumentation packages the user might want to install. By default it prints out
-a list of the suggested instrumentation packages which can be added to a requirements.txt
-file. It also supports installing the suggested packages when run with :code:`--action=install`
-flag.
+The above command will configure global trace provider, attach zipkin and otlp exporters to it and then
+start celery with the rest of the arguments. 
 
 References
 ----------

opentelemetry-instrumentation/src/opentelemetry/instrumentation/auto_instrumentation/__init__.py

@@ -14,16 +14,71 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import argparse
 from logging import getLogger
 from os import environ, execl, getcwd
 from os.path import abspath, dirname, pathsep
 from shutil import which
-from sys import argv
+
+from opentelemetry.instrumentation import symbols
 
 logger = getLogger(__file__)
 
 
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description="""
+        opentelemetry-instrument automatically instruments a Python
+        program and it's dependencies and then runs the program.
+        """
+    )
+
+    parser.add_argument(
+        "-e",
+        "--exporter",
+        required=False,
+        help="""
+        Uses the specified exporter to export spans.
+
+        Must be one of the following:
+            - Name of a well-known trace exporter. Choices are:
+                {0}
+            - A fully qualified python import path to a trace exporter
+              implementation or a callable that returns a new instance
+              of a trace exporter.
+        """.format(
+            symbols.trace_exporters
+        ),
+    )
+
+    parser.add_argument(
+        "-s",
+        "--service-name",
+        required=False,
+        help="""
+        The service name that should be passed to a trace exporter.
+        """,
+    )
+
+    parser.add_argument("command", help="Your Python application.")
+    parser.add_argument(
+        "command_args",
+        help="Arguments for your application.",
+        nargs=argparse.REMAINDER,
+    )
+    return parser.parse_args()
+
+
+def load_config_from_cli_args(args):
+    if args.exporter:
+        environ["OTEL_EXPORTER"] = args.exporter
+    if args.service_name:
+        environ["OTEL_SERVICE_NAME"] = args.service_name
+
+
 def run() -> None:
+    args = parse_args()
+    load_config_from_cli_args(args)
 
     python_path = environ.get("PYTHONPATH")
 
@@ -49,6 +104,5 @@ def run() -> None:
 
     environ["PYTHONPATH"] = pathsep.join(python_path)
 
-    executable = which(argv[1])
-
-    execl(executable, executable, *argv[2:])
+    executable = which(args.command)
+    execl(executable, executable, *args.command_args)

opentelemetry-instrumentation/src/opentelemetry/instrumentation/auto_instrumentation/components.py

@@ -0,0 +1,157 @@
+# Copyright The OpenTelemetry Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from logging import getLogger
+from typing import Sequence, Tuple
+
+from opentelemetry import trace
+from opentelemetry.configuration import Configuration
+from opentelemetry.instrumentation import symbols
+from opentelemetry.sdk.metrics.export import MetricsExporter
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import (
+    BatchExportSpanProcessor,
+    SpanExporter,
+    SpanProcessor,
+)
+
+logger = getLogger(__file__)
+
+_DEFAULT_EXPORTER = symbols.exporter_otlp
+
+known_exporters = {
+    symbols.exporter_otlp: (
+        "opentelemetry.exporter.otlp.trace_exporter.OTLPSpanExporter",
+        "opentelemetry.exporter.otlp.metrics_exporter.OTLPMetricsExporter",
+    ),
+    symbols.exporter_dd: (
+        "opentelemetry.exporter.datadog.DatadogSpanExporter",
+    ),
+    symbols.exporter_oc: (
+        "opentelemetry.exporter.opencensus.trace_exporter.OpenCensusSpanExporter",
+    ),
+    symbols.exporter_otlp_span: (
+        "opentelemetry.exporter.otlp.trace_exporter.OTLPSpanExporter",
+    ),
+    symbols.exporter_otlp_metric: (
+        "opentelemetry.exporter.otlp.metrics_exporter.OTLPMetricsExporter"
+    ),
+    symbols.exporter_jaeger: (
+        "opentelemetry.exporter.jaeger.JaegerSpanExporter",
+    ),
+    symbols.exporter_zipkin: (
+        "opentelemetry.exporter.zipkin.ZipkinSpanExporter",
+    ),
+    symbols.exporter_prometheus: (
+        "opentelemetry.exporter.prometheus.PrometheusMetricsExporter",
+    ),
+}
+
+
+def _import(import_path: str) -> any:
+    split_path = import_path.rsplit(".", 1)
+    if len(split_path) < 2:
+        raise ImportError(
+            "could not import module or class: {0}".format(import_path)
+        )
+    module, class_name = split_path
+    mod = __import__(module, fromlist=[class_name])
+    return getattr(mod, class_name)
+
+
+def get_service_name() -> str:
+    return Configuration().SERVICE_NAME or ""
+
+
+def get_exporter_names() -> Sequence[str]:
+    exporter = Configuration().EXPORTER or _DEFAULT_EXPORTER
+    if exporter.lower().strip() == "none":
+        return []
+
+    return [e.strip() for e in exporter.split(",")]
+
+
+def get_tracer_provider_class() -> trace.TracerProvider:
+    return TracerProvider
+
+
+def get_processor_class_for_exporter(exporter_name: str) -> SpanProcessor:
+    if exporter_name == symbols.exporter_dd:
+        return _import(
+            "opentelemetry.exporter.datadog.DatadogExportSpanProcessor"
+        )
+    return BatchExportSpanProcessor
+
+
+def init_tracing(exporters: Sequence[SpanExporter]):
+    service_name = get_service_name()
+    provider = get_tracer_provider_class()(
+        resource=Resource.create({"service.name": service_name}),
+    )
+    trace.set_tracer_provider(provider)
+
+    for exporter_name, exporter_class in exporters.items():
+        processor_class = get_processor_class_for_exporter(exporter_name)
+
+        exporter_args = {}
+        if exporter_name == symbols.exporter_dd:
+            exporter_args["service"] = service_name
+        elif exporter_name not in [
+            symbols.exporter_otlp,
+            symbols.exporter_otlp_span,
+        ]:
+            exporter_args["service_name"] = service_name
+
+        provider.add_span_processor(
+            processor_class(exporter_class(**exporter_args))
+        )
+
+
+def init_metrics(exporters: Sequence[MetricsExporter]):
+    if exporters:
+        logger.warning("automatic metric initialization is not supported yet.")
+
+
+def import_exporters(
+    exporter_names: Sequence[str],
+) -> Tuple[Sequence[SpanExporter], Sequence[MetricsExporter]]:
+    trace_exporters, metric_exporters = {}, {}
+    for exporter_name in exporter_names:
+        for exporter_path in known_exporters.get(
+            exporter_name, [exporter_name]
+        ):
+            exporter_impl = _import(exporter_path)
+            if issubclass(exporter_impl, SpanExporter):
+                trace_exporters[exporter_name] = exporter_impl
+            elif issubclass(exporter_impl, MetricsExporter):
+                metric_exporters[exporter_name] = exporter_impl
+            else:
+                raise RuntimeError(
+                    "{0} ({1}) is neither a trace exporter nor a metric exporter".format(
+                        exporter_name, exporter_path
+                    )
+                )
+    return trace_exporters, metric_exporters
+
+
+def initialize_components():
+    exporter_names = get_exporter_names()
+    trace_exporters, metric_exporters = import_exporters(exporter_names)
+    init_tracing(trace_exporters)
+
+    # We don't support automatic initialization for metric yet but have added
+    # some boilerplate in order to make sure current implementation does not
+    # lock us out of supporting metrics later without major surgery.
+    init_metrics(metric_exporters)

opentelemetry-instrumentation/src/opentelemetry/instrumentation/auto_instrumentation/sitecustomize.py

@@ -12,17 +12,45 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
+import sys
 from logging import getLogger
 
 from pkg_resources import iter_entry_points
 
+from opentelemetry.instrumentation.auto_instrumentation.components import (
+    initialize_components,
+)
+
 logger = getLogger(__file__)
 
 
-for entry_point in iter_entry_points("opentelemetry_instrumentor"):
-    try:
-        entry_point.load()().instrument()  # type: ignore
-        logger.debug("Instrumented %s", entry_point.name)
+def auto_instrument():
+    for entry_point in iter_entry_points("opentelemetry_instrumentor"):
+        try:
+            entry_point.load()().instrument()  # type: ignore
+            logger.debug("Instrumented %s", entry_point.name)
+
+        except Exception:  # pylint: disable=broad-except
+            logger.exception("Instrumenting of %s failed", entry_point.name)
+
+
+def initialize():
+    initialize_components()
+    auto_instrument()
+
+
+if (
+    hasattr(sys, "argv")
+    and sys.argv[0].split(os.path.sep)[-1] == "celery"
+    and "worker" in sys.argv
+):
+    from celery.signals import worker_process_init  # pylint:disable=E0401
+
+    @worker_process_init.connect(weak=False)
+    def init_celery(*args, **kwargs):
+        initialize()
+
 
-    except Exception:  # pylint: disable=broad-except
-        logger.exception("Instrumenting of %s failed", entry_point.name)
+else:
+    initialize()