Add true auto-instrumentation support to opentelemetry-instrument

This commit extends the instrument command so it automatically
configures tracing with a provider, span processor and exporter. Most of
the component used can be customized with env vars or CLI arguments.

Details can be found on opentelemetry-instrumentation's README package.

Fixes #663

exporter/opentelemetry-exporter-jaeger/setup.cfg

@@ -48,3 +48,7 @@ where = src
 
 [options.extras_require]
 test =
+
+[options.entry_points]
+opentelemetry_exporter =
+    jaeger = opentelemetry.exporter.jaeger:JaegerSpanExporter

exporter/opentelemetry-exporter-opencensus/setup.cfg

@@ -50,3 +50,7 @@ where = src
 
 [options.extras_require]
 test =
+
+[options.entry_points]
+opentelemetry_exporter =
+    opencensus = opentelemetry.exporter.opencensus.trace_exporter:OpenCensusSpanExporter

exporter/opentelemetry-exporter-otlp/setup.cfg

@@ -52,3 +52,8 @@ test =
 
 [options.packages.find]
 where = src
+
+[options.entry_points]
+opentelemetry_exporter =
+    otlp_span = opentelemetry.exporter.otlp.trace_exporter:OTLPSpanExporter
+    otlp_metric = opentelemetry.exporter.otlp.metrics_exporter:OTLPMetricsExporter

exporter/opentelemetry-exporter-prometheus/setup.cfg

@@ -48,3 +48,7 @@ where = src
 
 [options.extras_require]
 test =
+
+[options.entry_points]
+opentelemetry_exporter =
+    prometheus = opentelemetry.exporter.prometheus:PrometheusMetricsExporter

exporter/opentelemetry-exporter-zipkin/setup.cfg

@@ -48,3 +48,7 @@ where = src
 
 [options.extras_require]
 test =
+
+[options.entry_points]
+opentelemetry_exporter =
+    zipkin = opentelemetry.exporter.zipkin:ZipkinSpanExporter

opentelemetry-instrumentation/CHANGELOG.md

@@ -2,11 +2,13 @@
 
 ## Unreleased
 
+- Added support for `OTEL_EXPORTER` to the `opentelemetry-instrument` command ([#1036](https://github.com/open-telemetry/opentelemetry-python/pull/1036))
+
 ## Version 0.14b0
 
 Released 2020-10-13
 
-- Fixed boostrap command to correctly install opentelemetry-instrumentation-falcon instead of opentelemetry-instrumentation-flask
+- Fixed boostrap command to correctly install opentelemetry-instrumentation-falcon instead of opentelemetry-instrumentation-flask. ([#1138](https://github.com/open-telemetry/opentelemetry-python/pull/1138))
 
 ## Version 0.13b0
 

opentelemetry-instrumentation/README.rst

@@ -16,30 +16,68 @@ Installation
 
 This package provides a couple of commands that help automatically instruments a program:
 
+
 opentelemetry-instrument
 ------------------------
 
 ::
 
     opentelemetry-instrument python program.py
 
+The instrument command will try to automatically detect packages used by your python program
+and when possible, apply automatic tracing instrumentation on them. This means your program
+will get automatic distributed tracing for free without having to make any code changes
+at all. This will also configure a global tracer and tracing exporter without you having to
+make any code changes. By default, the instrument command will use the OTLP exporter but
+this can be overriden when needed.
+
+The command supports the following configuration options as CLI arguments and environment vars:
+
+
+* ``--exporter`` or ``OTEL_EXPORTER``
+
+Used to specify which trace exporter to use. Can be set to one or more
+of the well-known exporter names (see below).
+
+    - Defaults to `otlp`.
+    - Can be set to `none` to disable automatic tracer initialization. 
+
+You can pass multiple values to configure multiple exporters e.g, ``zipkin,prometheus`` 
+
+Well known trace exporter names:
+
+    - jaeger
+    - opencensus
+    - otlp
+    - otlp_span
+    - otlp_metric
+    - zipkin
+
+``otlp`` is an alias for ``otlp_span,otlp_metric``.
+
+* ``--service-name`` or ``OTEL_SERVICE_NAME``
+
+When present the value is passed on to the relevant exporter initializer as ``service_name`` argument.
+
 The code in ``program.py`` needs to use one of the packages for which there is
 an OpenTelemetry integration. For a list of the available integrations please
 check `here <https://opentelemetry-python.readthedocs.io/en/stable/index.html#integrations>`_
 
+Examples
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+::
+
+    opentelemetry-instrument -e otlp flask run --port=3000
 
-opentelemetry-bootstrap
------------------------
+The above command will pass ``-e otlp`` to the instrument command and ``--port=3000`` to ``flask run``.
 
 ::
 
-    opentelemetry-bootstrap --action=install|requirements
+    opentelemetry-instrument -e zipkin,otlp celery -A tasks worker --loglevel=info
 
-This commands inspects the active Python site-packages and figures out which
-instrumentation packages the user might want to install. By default it prints out
-a list of the suggested instrumentation packages which can be added to a requirements.txt
-file. It also supports installing the suggested packages when run with :code:`--action=install`
-flag.
+The above command will configure global trace provider, attach zipkin and otlp exporters to it and then
+start celery with the rest of the arguments. 
 
 References
 ----------

opentelemetry-instrumentation/src/opentelemetry/instrumentation/auto_instrumentation/__init__.py

@@ -14,16 +14,67 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import argparse
 from logging import getLogger
 from os import environ, execl, getcwd
 from os.path import abspath, dirname, pathsep
 from shutil import which
-from sys import argv
 
 logger = getLogger(__file__)
 
 
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description="""
+        opentelemetry-instrument automatically instruments a Python
+        program and it's dependencies and then runs the program.
+        """
+    )
+
+    parser.add_argument(
+        "-e",
+        "--exporter",
+        required=False,
+        help="""
+        Uses the specified exporter to export spans or metrics.
+        Accepts multiple exporters as comma separated values.
+
+        Examples:
+
+            -e=otlp
+            -e=otlp_span,prometheus
+            -e=jaeger,otlp_metric
+        """,
+    )
+
+    parser.add_argument(
+        "-s",
+        "--service-name",
+        required=False,
+        help="""
+        The service name that should be passed to a trace exporter.
+        """,
+    )
+
+    parser.add_argument("command", help="Your Python application.")
+    parser.add_argument(
+        "command_args",
+        help="Arguments for your application.",
+        nargs=argparse.REMAINDER,
+    )
+    return parser.parse_args()
+
+
+def load_config_from_cli_args(args):
+    if args.exporter:
+        environ["OTEL_EXPORTER"] = args.exporter
+    if args.service_name:
+        environ["OTEL_SERVICE_NAME"] = args.service_name
+
+
 def run() -> None:
+    args = parse_args()
+    load_config_from_cli_args(args)
 
     python_path = environ.get("PYTHONPATH")
 
@@ -49,6 +100,5 @@ def run() -> None:
 
     environ["PYTHONPATH"] = pathsep.join(python_path)
 
-    executable = which(argv[1])
-
-    execl(executable, executable, *argv[2:])
+    executable = which(args.command)
+    execl(executable, executable, *args.command_args)

opentelemetry-instrumentation/src/opentelemetry/instrumentation/auto_instrumentation/components.py

@@ -0,0 +1,121 @@
+# Copyright The OpenTelemetry Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from logging import getLogger
+from typing import Sequence, Tuple
+
+from pkg_resources import iter_entry_points
+
+from opentelemetry import trace
+from opentelemetry.configuration import Configuration
+from opentelemetry.sdk.metrics.export import MetricsExporter
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import (
+    BatchExportSpanProcessor,
+    SpanExporter,
+)
+
+logger = getLogger(__file__)
+
+EXPORTER_OTLP = "otlp"
+EXPORTER_OTLP_SPAN = "otlp_span"
+EXPORTER_OTLP_METRIC = "otlp_metric"
+_DEFAULT_EXPORTER = EXPORTER_OTLP
+
+
+def get_service_name() -> str:
+    return Configuration().SERVICE_NAME or ""
+
+
+def get_exporter_names() -> Sequence[str]:
+    exporter = Configuration().EXPORTER or _DEFAULT_EXPORTER
+    if exporter.lower().strip() == "none":
+        return []
+
+    names = []
+    for exp in exporter.split(","):
+        name = exp.strip()
+        if name == EXPORTER_OTLP:
+            names.append(EXPORTER_OTLP_SPAN)
+            names.append(EXPORTER_OTLP_METRIC)
+        else:
+            names.append(name)
+    return names
+
+
+def init_tracing(exporters: Sequence[SpanExporter]):
+    service_name = get_service_name()
+    provider = TracerProvider(
+        resource=Resource.create({"service.name": service_name}),
+    )
+    trace.set_tracer_provider(provider)
+
+    for exporter_name, exporter_class in exporters.items():
+        exporter_args = {}
+        if exporter_name not in [
+            EXPORTER_OTLP,
+            EXPORTER_OTLP_SPAN,
+        ]:
+            exporter_args["service_name"] = service_name
+
+        provider.add_span_processor(
+            BatchExportSpanProcessor(exporter_class(**exporter_args))
+        )
+
+
+def init_metrics(exporters: Sequence[MetricsExporter]):
+    if exporters:
+        logger.warning("automatic metric initialization is not supported yet.")
+
+
+def import_exporters(
+    exporter_names: Sequence[str],
+) -> Tuple[Sequence[SpanExporter], Sequence[MetricsExporter]]:
+    trace_exporters, metric_exporters = {}, {}
+
+    exporters = {
+        ep.name: ep for ep in iter_entry_points("opentelemetry_exporter")
+    }
+
+    for exporter_name in exporter_names:
+        entry_point = exporters.get(exporter_name, None)
+        if not entry_point:
+            raise RuntimeError(
+                "Requested exporter not found: {0}".format(exporter_name)
+            )
+
+        exporter_impl = entry_point.load()
+        if issubclass(exporter_impl, SpanExporter):
+            trace_exporters[exporter_name] = exporter_impl
+        elif issubclass(exporter_impl, MetricsExporter):
+            metric_exporters[exporter_name] = exporter_impl
+        else:
+            raise RuntimeError(
+                "{0} is neither a trace exporter nor a metric exporter".format(
+                    exporter_name
+                )
+            )
+    return trace_exporters, metric_exporters
+
+
+def initialize_components():
+    exporter_names = get_exporter_names()
+    trace_exporters, metric_exporters = import_exporters(exporter_names)
+    init_tracing(trace_exporters)
+
+    # We don't support automatic initialization for metric yet but have added
+    # some boilerplate in order to make sure current implementation does not
+    # lock us out of supporting metrics later without major surgery.
+    init_metrics(metric_exporters)

opentelemetry-instrumentation/src/opentelemetry/instrumentation/auto_instrumentation/sitecustomize.py

@@ -12,17 +12,48 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
+import sys
 from logging import getLogger
 
 from pkg_resources import iter_entry_points
 
+from opentelemetry.instrumentation.auto_instrumentation.components import (
+    initialize_components,
+)
+
 logger = getLogger(__file__)
 
 
-for entry_point in iter_entry_points("opentelemetry_instrumentor"):
-    try:
-        entry_point.load()().instrument()  # type: ignore
-        logger.debug("Instrumented %s", entry_point.name)
+def auto_instrument():
+    for entry_point in iter_entry_points("opentelemetry_instrumentor"):
+        try:
+            entry_point.load()().instrument()  # type: ignore
+            logger.debug("Instrumented %s", entry_point.name)
+        except Exception as exc:  # pylint: disable=broad-except
+            logger.exception("Instrumenting of %s failed", entry_point.name)
+            raise exc
+
 
+def initialize():
+    try:
+        initialize_components()
+        auto_instrument()
     except Exception:  # pylint: disable=broad-except
-        logger.exception("Instrumenting of %s failed", entry_point.name)
+        logger.exception("Failed to auto initialize opentelemetry")
+
+
+if (
+    hasattr(sys, "argv")
+    and sys.argv[0].split(os.path.sep)[-1] == "celery"
+    and "worker" in sys.argv[1:]
+):
+    from celery.signals import worker_process_init  # pylint:disable=E0401
+
+    @worker_process_init.connect(weak=False)
+    def init_celery(*args, **kwargs):
+        initialize()
+
+
+else:
+    initialize()