Lambda exec wrapper calls upstream OTel Python auto instr script

python/src/otel/otel_sdk/otel-instrument

@@ -1,44 +1,127 @@
-#!/usr/bin/env python3
-# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
-# SPDX-License-Identifier: MIT-0
-
-from os import environ, system
-import sys
-
-# the path to the interpreter and all of the originally intended arguments
-args = sys.argv[1:]
-
-# enable OTel wrapper
-environ["ORIG_HANDLER"] = environ.get("_HANDLER")
-environ["_HANDLER"] = "otel_wrapper.lambda_handler"
-
-# config default traces exporter if missing
-environ.setdefault("OTEL_TRACES_EXPORTER", "otlp_proto_grpc_span")
-
-# set service name
-if environ.get("OTEL_RESOURCE_ATTRIBUTES") is None:
-    environ["OTEL_RESOURCE_ATTRIBUTES"] = "service.name=%s" % (
-        environ.get("AWS_LAMBDA_FUNCTION_NAME")
-    )
-elif "service.name=" not in environ.get("OTEL_RESOURCE_ATTRIBUTES"):
-    environ["OTEL_RESOURCE_ATTRIBUTES"] = "service.name=%s,%s" % (
-        environ.get("AWS_LAMBDA_FUNCTION_NAME"),
-        environ.get("OTEL_RESOURCE_ATTRIBUTES"),
-    )
-
-# TODO: Remove if sdk support resource detector env variable configuration.
-lambda_resource_attributes = (
-    "cloud.region=%s,cloud.provider=aws,faas.name=%s,faas.version=%s"
-    % (
-        environ.get("AWS_REGION"),
-        environ.get("AWS_LAMBDA_FUNCTION_NAME"),
-        environ.get("AWS_LAMBDA_FUNCTION_VERSION"),
-    )
-)
-environ["OTEL_RESOURCE_ATTRIBUTES"] = "%s,%s" % (
-    lambda_resource_attributes,
-    environ.get("OTEL_RESOURCE_ATTRIBUTES"),
-)
-
-# start the runtime with the extra options
-system(" ".join(args))
+#!/bin/bash
+
+# Copyright The OpenTelemetry Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+: <<'END_DOCUMENTATION'
+`otel-instrument`
+
+This script configures and sets up OpenTelemetry Python with the values we
+expect will be used by the common user. It does this by setting the environment
+variables OpenTelemetry uses, and then initializing OpenTelemetry using the
+`opentelemetry-instrument` auto instrumentation script from the
+`opentelemetry-instrumentation` package.
+
+Additionally, this configuration assumes the user is using packages conforming
+to the `opentelemetry-instrumentation` and `opentelemetry-sdk` specifications.
+
+DO NOT use this script for anything else besides SETTING ENVIRONMENT VARIABLES.
+
+See more:
+https://docs.aws.amazon.com/lambda/latest/dg/runtimes-modify.html#runtime-wrapper
+
+Usage
+-----
+We expect this file to be at the root of a Lambda Layer. Having it anywhere else
+seems to mean AWS Lambda cannot find it.
+
+In the configuration of an AWS Lambda function with this file at the
+root level of a Lambda Layer:
+
+.. code::
+
+    AWS_LAMBDA_EXEC_WRAPPER = /opt/otel-instrument
+
+END_DOCUMENTATION
+
+# Use constants to access the environment variables we want to use in this
+# script.
+
+# See more:
+# https://docs.aws.amazon.com/lambda/latest/dg/configuration-envvars.html#configuration-envvars-runtime
+
+# - Reserved environment variables
+
+# - - $AWS_LAMBDA_FUNCTION_NAME
+# - - $LAMBDA_RUNTIME_DIR
+
+# - Unreserved environment variables
+
+# - - $PYTHONPATH
+
+# Update the python paths for packages with `sys.path` and `PYTHONPATH`
+
+# - We know that the path to the Lambda Layer OpenTelemetry Python packages are
+#   well defined, so we can add them to the PYTHONPATH.
+#
+#   See more:
+#   https://docs.aws.amazon.com/lambda/latest/dg/configuration-layers.html#configuration-layers-path
+
+export LAMBDA_LAYER_PKGS_DIR="/opt/python";
+
+# - Set Lambda Layer python packages in PYTHONPATH so `opentelemetry-instrument`
+#   script can find them (it needs to find `opentelemetry` to find the auto
+#   instrumentation `run()` method later)
+
+export PYTHONPATH="$LAMBDA_LAYER_PKGS_DIR:$PYTHONPATH";
+
+# - Set Lambda runtime python packages in PYTHONPATH so
+#   `opentelemetry-instrument` script can find them during auto instrumentation
+#   and instrument them.
+
+export PYTHONPATH="$LAMBDA_RUNTIME_DIR:$PYTHONPATH";
+
+# Configure OpenTelemetry Python with environment variables
+
+# - Uses the default `OTEL_TRACES_EXPORTER` which is set to `otlp_proto_grpc`
+
+# - Set the service name
+
+if [ -z ${OTEL_SERVICE_NAME} ]; then
+    export OTEL_SERVICE_NAME=$AWS_LAMBDA_FUNCTION_NAME;
+fi
+
+# - Set the Resource Detectors (Resource Attributes)
+#
+#   TODO: waiting on OTel Python support for configuring Resource Detectors from
+#   an environment variable. Replace the bottom code with the following when
+#   this is possible.
+#
+#   export OTEL_RESOURCE_DETECTORS="aws_lambda";
+#
+export LAMBDA_RESOURCE_ATTRIBUTES="cloud.region=$AWS_REGION,cloud.provider=aws,faas.name=$AWS_LAMBDA_FUNCTION_NAME,faas.version=$AWS_LAMBDA_FUNCTION_VERSION";
+
+if [ -z ${OTEL_RESOURCE_ATTRIBUTES} ]; then
+    export OTEL_RESOURCE_ATTRIBUTES=$LAMBDA_RESOURCE_ATTRIBUTES;
+else
+    export OTEL_RESOURCE_ATTRIBUTES="$LAMBDA_RESOURCE_ATTRIBUTES,$OTEL_RESOURCE_ATTRIBUTES";
+fi
+
+# - Uses the default `OTEL_PROPAGATORS` which is set to `tracecontext,baggage`
+
+# - Use a wrapper because AWS Lambda's `python3 /var/runtime/bootstrap.py` will
+#   use `imp.load_module` to load the function from the `_HANDLER` environment
+#   variable. This RELOADS the module and REMOVES any instrumentation patching
+#   done earlier. So we delay instrumentation until `boostrap.py` imports
+#   `otel_wrapper.py` at which we know the patching will be picked up.
+#
+#   See more:
+#   https://docs.python.org/3/library/imp.html#imp.load_module
+
+export ORIG_HANDLER=$_HANDLER;
+export _HANDLER="otel_wrapper.lambda_handler";
+
+# - Call the upstream auto instrumentation script
+
+python3 $LAMBDA_LAYER_PKGS_DIR/bin/opentelemetry-instrument "$@"

python/src/otel/otel_sdk/otel_wrapper.py

@@ -1,110 +1,66 @@
-import logging
-import os
+# Copyright The OpenTelemetry Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+`otel_wrapper.py`
+
+This file serves as a wrapper over the user's Lambda function.
+
+Usage
+-----
+Patch the reserved `_HANDLER` Lambda environment variable to point to this
+file's `otel_wrapper.lambda_handler` property. Do this having saved the original
+`_HANDLER` in the `ORIG_HANDLER` environment variable. Doing this makes it so
+that **on import of this file, the handler is instrumented**.
+
+Instrumenting any earlier will cause the instrumentation to be lost because the
+AWS Service uses `imp.load_module` to import the handler which RELOADS the
+module. This is why AwsLambdaInstrumentor cannot be instrumented with the
+`opentelemetry-instrument` script.
+
+See more:
+https://docs.python.org/3/library/imp.html#imp.load_module
+
+"""
 
+import os
 from importlib import import_module
-from pkg_resources import iter_entry_points
 
-from opentelemetry.instrumentation.dependencies import get_dist_dependency_conflicts
 from opentelemetry.instrumentation.aws_lambda import AwsLambdaInstrumentor
-from opentelemetry.environment_variables import OTEL_PYTHON_DISABLED_INSTRUMENTATIONS
-from opentelemetry.instrumentation.distro import BaseDistro, DefaultDistro
-
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-# TODO: waiting OTel Python supports env variable config for resource detector
-# from opentelemetry.resource import AwsLambdaResourceDetector
-# from opentelemetry.sdk.resources import Resource
-# resource = Resource.create().merge(AwsLambdaResourceDetector().detect())
-# trace.get_tracer_provider.resource = resource
-
-def _load_distros() -> BaseDistro:
-    for entry_point in iter_entry_points("opentelemetry_distro"):
-        try:
-            distro = entry_point.load()()
-            if not isinstance(distro, BaseDistro):
-                logger.debug(
-                    "%s is not an OpenTelemetry Distro. Skipping",
-                    entry_point.name,
-                )
-                continue
-            logger.debug(
-                "Distribution %s will be configured", entry_point.name
-            )
-            return distro
-        except Exception as exc:  # pylint: disable=broad-except
-            logger.debug("Distribution %s configuration failed", entry_point.name)
-    return DefaultDistro()
-
-def _load_instrumentors(distro):
-    package_to_exclude = os.environ.get(OTEL_PYTHON_DISABLED_INSTRUMENTATIONS, [])
-    if isinstance(package_to_exclude, str):
-        package_to_exclude = package_to_exclude.split(",")
-        # to handle users entering "requests , flask" or "requests, flask" with spaces
-        package_to_exclude = [x.strip() for x in package_to_exclude]
-
-    for entry_point in iter_entry_points("opentelemetry_instrumentor"):
-        if entry_point.name in package_to_exclude:
-            logger.debug(
-                "Instrumentation skipped for library %s", entry_point.name
-            )
-            continue
-
-        try:
-            conflict = get_dist_dependency_conflicts(entry_point.dist)
-            if conflict:
-                logger.debug(
-                    "Skipping instrumentation %s: %s",
-                    entry_point.name,
-                    conflict,
-                )
-                continue
-
-            # tell instrumentation to not run dep checks again as we already did it above
-            distro.load_instrumentor(entry_point, skip_dep_check=True)
-            logger.info("Instrumented %s", entry_point.name)
-        except Exception as exc:  # pylint: disable=broad-except
-            logger.debug("Instrumenting of %s failed", entry_point.name)
-
-def _load_configurators():
-    configured = None
-    for entry_point in iter_entry_points("opentelemetry_configurator"):
-        if configured is not None:
-            logger.warning(
-                "Configuration of %s not loaded, %s already loaded",
-                entry_point.name,
-                configured,
-            )
-            continue
-        try:
-            entry_point.load()().configure()  # type: ignore
-            configured = entry_point.name
-        except Exception as exc:  # pylint: disable=broad-except
-            logger.debug("Configuration of %s failed", entry_point.name)
 
 
 def modify_module_name(module_name):
     """Returns a valid modified module to get imported"""
     return ".".join(module_name.split("/"))
 
+
 class HandlerError(Exception):
     pass
 
-distro = _load_distros()
-distro.configure()
-_load_configurators()
-_load_instrumentors(distro)
-# TODO: move to python-contrib
-AwsLambdaInstrumentor().instrument(skip_dep_check=True)
 
-path = os.environ.get("ORIG_HANDLER", None)
+AwsLambdaInstrumentor().instrument()
+
+path = os.environ.get("ORIG_HANDLER")
+
 if path is None:
     raise HandlerError("ORIG_HANDLER is not defined.")
-parts = path.rsplit(".", 1)
-if len(parts) != 2:
-    raise HandlerError("Value %s for ORIG_HANDLER has invalid format." % path)
 
-(mod_name, handler_name) = parts
+try:
+    (mod_name, handler_name) = path.rsplit(".", 1)
+except ValueError as e:
+    raise HandlerError("Bad path '{}' for ORIG_HANDLER: {}".format(path, str(e)))
+
 modified_mod_name = modify_module_name(mod_name)
 handler_module = import_module(modified_mod_name)
 lambda_handler = getattr(handler_module, handler_name)

python/src/otel/setup.cfg

@@ -50,5 +50,7 @@ test =
 where = otel_sdk
 
 [options.entry_points]
-opentelemetry_instrumentor =
-    aws_lambda = opentelemetry.instrumentation.aws_lambda:AwsLambdaInstrumentor
+# NOTE: (NathanielRN) DO NOT add AwsLambdaInstrumentor entry point because
+# current AWS Lambda implementation reloads a fresh import of the user's Lambda
+# handler. Auto Instrumentation runs _before_ and if it instruments the handler
+# that patching will be lost.