diff --git a/python/src/otel/otel_sdk/otel-instrument b/python/src/otel/otel_sdk/otel-instrument index 8da0177b72..080e629941 100755 --- a/python/src/otel/otel_sdk/otel-instrument +++ b/python/src/otel/otel_sdk/otel-instrument @@ -1,44 +1,122 @@ -#!/usr/bin/env python3 -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: MIT-0 - -from os import environ, system -import sys - -# the path to the interpreter and all of the originally intended arguments -args = sys.argv[1:] - -# enable OTel wrapper -environ["ORIG_HANDLER"] = environ.get("_HANDLER") -environ["_HANDLER"] = "otel_wrapper.lambda_handler" - -# config default traces exporter if missing -environ.setdefault("OTEL_TRACES_EXPORTER", "otlp_proto_grpc_span") - -# set service name -if environ.get("OTEL_RESOURCE_ATTRIBUTES") is None: - environ["OTEL_RESOURCE_ATTRIBUTES"] = "service.name=%s" % ( - environ.get("AWS_LAMBDA_FUNCTION_NAME") - ) -elif "service.name=" not in environ.get("OTEL_RESOURCE_ATTRIBUTES"): - environ["OTEL_RESOURCE_ATTRIBUTES"] = "service.name=%s,%s" % ( - environ.get("AWS_LAMBDA_FUNCTION_NAME"), - environ.get("OTEL_RESOURCE_ATTRIBUTES"), - ) - -# TODO: Remove if sdk support resource detector env variable configuration. -lambda_resource_attributes = ( - "cloud.region=%s,cloud.provider=aws,faas.name=%s,faas.version=%s" - % ( - environ.get("AWS_REGION"), - environ.get("AWS_LAMBDA_FUNCTION_NAME"), - environ.get("AWS_LAMBDA_FUNCTION_VERSION"), - ) -) -environ["OTEL_RESOURCE_ATTRIBUTES"] = "%s,%s" % ( - lambda_resource_attributes, - environ.get("OTEL_RESOURCE_ATTRIBUTES"), -) - -# start the runtime with the extra options -system(" ".join(args)) +#!/bin/bash + +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +: <<'END_DOCUMENTATION' +`otel-instrument` + +This script configures and sets up OpenTelemetry Python with the values we +expect will be used by the common user. It does this by setting the environment +variables OpenTelemetry uses, and then initializing OpenTelemetry using the +`opentelemetry-instrument` auto instrumentation script from the +`opentelemetry-instrumentation` package. + +Additionally, this configuration assumes the user is using packages conforming +to the `opentelemetry-instrumentation` and `opentelemetry-sdk` specifications. + +DO NOT use this script for anything else besides SETTING ENVIRONMENT VARIABLES. + +See more: +https://docs.aws.amazon.com/lambda/latest/dg/runtimes-modify.html#runtime-wrapper + +Usage +----- +We expect this file to be at the root of a Lambda Layer. Having it anywhere else +seems to mean AWS Lambda cannot find it. + +In the configuration of an AWS Lambda function with this file at the +root level of a Lambda Layer: + +.. code:: + + AWS_LAMBDA_EXEC_WRAPPER = /opt/otel-instrument + +END_DOCUMENTATION + +# Use constants to access the environment variables we want to use in this +# script. + +# See more: +# https://docs.aws.amazon.com/lambda/latest/dg/configuration-envvars.html#configuration-envvars-runtime + +# - Reserved environment variables + +# - - $AWS_LAMBDA_FUNCTION_NAME +# - - $LAMBDA_RUNTIME_DIR + +# - Unreserved environment variables + +# - - $PYTHONPATH + +# Update the python paths for packages with `sys.path` and `PYTHONPATH` + +# - We know that the path to the Lambda Layer OpenTelemetry Python packages are +# well defined, so we can add them to the PYTHONPATH. +# +# See more: +# https://docs.aws.amazon.com/lambda/latest/dg/configuration-layers.html#configuration-layers-path + +export LAMBDA_LAYER_PKGS_DIR="/opt/python"; + +# - Set Lambda Layer python packages in PYTHONPATH so `opentelemetry-instrument` +# script can find them (it needs to find `opentelemetry` to find the auto +# instrumentation `run()` method later) + +export PYTHONPATH="$LAMBDA_LAYER_PKGS_DIR:$PYTHONPATH"; + +# - Set Lambda runtime python packages in PYTHONPATH so +# `opentelemetry-instrument` script can find them during auto instrumentation +# and instrument them. + +export PYTHONPATH="$LAMBDA_RUNTIME_DIR:$PYTHONPATH"; + +# Configure OpenTelemetry Python with environment variables + +# - Uses the default `OTEL_TRACES_EXPORTER` which is set to `otlp_proto_grpc` + +# - Set the service name + +if [ -z ${OTEL_SERVICE_NAME} ]; then + export OTEL_SERVICE_NAME=$AWS_LAMBDA_FUNCTION_NAME; +fi + +# - Set the Resource Detectors (Resource Attributes) +# +# TODO: waiting on OTel Python support for configuring Resource Detectors from +# an environment variable. Replace the bottom code with the following when +# this is possible. +# +# export OTEL_RESOURCE_DETECTORS="aws_lambda"; +# + +export OTEL_RESOURCE_ATTRIBUTES="cloud.region=$AWS_REGION,cloud.provider=aws,faas.name=$AWS_LAMBDA_FUNCTION_NAME,faas.version=$AWS_LAMBDA_FUNCTION_VERSION,$OTEL_RESOURCE_ATTRIBUTES"; + +# - Uses the default `OTEL_PROPAGATORS` which is set to `tracecontext,baggage` + +# - Use a wrapper because AWS Lambda's `python3 /var/runtime/bootstrap.py` will +# use `imp.load_module` to load the function from the `_HANDLER` environment +# variable. This RELOADS the module and REMOVES any instrumentation patching +# done earlier. So we delay instrumentation until `boostrap.py` imports +# `otel_wrapper.py` at which we know the patching will be picked up. +# +# See more: +# https://docs.python.org/3/library/imp.html#imp.load_module + +export ORIG_HANDLER=$_HANDLER; +export _HANDLER="otel_wrapper.lambda_handler"; + +# - Call the upstream auto instrumentation script + +python3 $LAMBDA_LAYER_PKGS_DIR/bin/opentelemetry-instrument "$@" diff --git a/python/src/otel/otel_sdk/otel_wrapper.py b/python/src/otel/otel_sdk/otel_wrapper.py index 7f477a1dbd..295410406c 100644 --- a/python/src/otel/otel_sdk/otel_wrapper.py +++ b/python/src/otel/otel_sdk/otel_wrapper.py @@ -1,110 +1,66 @@ -import logging -import os +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +`otel_wrapper.py` + +This file serves as a wrapper over the user's Lambda function. + +Usage +----- +Patch the reserved `_HANDLER` Lambda environment variable to point to this +file's `otel_wrapper.lambda_handler` property. Do this having saved the original +`_HANDLER` in the `ORIG_HANDLER` environment variable. Doing this makes it so +that **on import of this file, the handler is instrumented**. + +Instrumenting any earlier will cause the instrumentation to be lost because the +AWS Service uses `imp.load_module` to import the handler which RELOADS the +module. This is why AwsLambdaInstrumentor cannot be instrumented with the +`opentelemetry-instrument` script. + +See more: +https://docs.python.org/3/library/imp.html#imp.load_module + +""" +import os from importlib import import_module -from pkg_resources import iter_entry_points -from opentelemetry.instrumentation.dependencies import get_dist_dependency_conflicts from opentelemetry.instrumentation.aws_lambda import AwsLambdaInstrumentor -from opentelemetry.environment_variables import OTEL_PYTHON_DISABLED_INSTRUMENTATIONS -from opentelemetry.instrumentation.distro import BaseDistro, DefaultDistro - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -# TODO: waiting OTel Python supports env variable config for resource detector -# from opentelemetry.resource import AwsLambdaResourceDetector -# from opentelemetry.sdk.resources import Resource -# resource = Resource.create().merge(AwsLambdaResourceDetector().detect()) -# trace.get_tracer_provider.resource = resource - -def _load_distros() -> BaseDistro: - for entry_point in iter_entry_points("opentelemetry_distro"): - try: - distro = entry_point.load()() - if not isinstance(distro, BaseDistro): - logger.debug( - "%s is not an OpenTelemetry Distro. Skipping", - entry_point.name, - ) - continue - logger.debug( - "Distribution %s will be configured", entry_point.name - ) - return distro - except Exception as exc: # pylint: disable=broad-except - logger.debug("Distribution %s configuration failed", entry_point.name) - return DefaultDistro() - -def _load_instrumentors(distro): - package_to_exclude = os.environ.get(OTEL_PYTHON_DISABLED_INSTRUMENTATIONS, []) - if isinstance(package_to_exclude, str): - package_to_exclude = package_to_exclude.split(",") - # to handle users entering "requests , flask" or "requests, flask" with spaces - package_to_exclude = [x.strip() for x in package_to_exclude] - - for entry_point in iter_entry_points("opentelemetry_instrumentor"): - if entry_point.name in package_to_exclude: - logger.debug( - "Instrumentation skipped for library %s", entry_point.name - ) - continue - - try: - conflict = get_dist_dependency_conflicts(entry_point.dist) - if conflict: - logger.debug( - "Skipping instrumentation %s: %s", - entry_point.name, - conflict, - ) - continue - - # tell instrumentation to not run dep checks again as we already did it above - distro.load_instrumentor(entry_point, skip_dep_check=True) - logger.info("Instrumented %s", entry_point.name) - except Exception as exc: # pylint: disable=broad-except - logger.debug("Instrumenting of %s failed", entry_point.name) - -def _load_configurators(): - configured = None - for entry_point in iter_entry_points("opentelemetry_configurator"): - if configured is not None: - logger.warning( - "Configuration of %s not loaded, %s already loaded", - entry_point.name, - configured, - ) - continue - try: - entry_point.load()().configure() # type: ignore - configured = entry_point.name - except Exception as exc: # pylint: disable=broad-except - logger.debug("Configuration of %s failed", entry_point.name) def modify_module_name(module_name): """Returns a valid modified module to get imported""" return ".".join(module_name.split("/")) + class HandlerError(Exception): pass -distro = _load_distros() -distro.configure() -_load_configurators() -_load_instrumentors(distro) -# TODO: move to python-contrib -AwsLambdaInstrumentor().instrument(skip_dep_check=True) -path = os.environ.get("ORIG_HANDLER", None) +AwsLambdaInstrumentor().instrument() + +path = os.environ.get("ORIG_HANDLER") + if path is None: raise HandlerError("ORIG_HANDLER is not defined.") -parts = path.rsplit(".", 1) -if len(parts) != 2: - raise HandlerError("Value %s for ORIG_HANDLER has invalid format." % path) -(mod_name, handler_name) = parts +try: + (mod_name, handler_name) = path.rsplit(".", 1) +except ValueError as e: + raise HandlerError("Bad path '{}' for ORIG_HANDLER: {}".format(path, str(e))) + modified_mod_name = modify_module_name(mod_name) handler_module = import_module(modified_mod_name) lambda_handler = getattr(handler_module, handler_name) diff --git a/python/src/otel/setup.cfg b/python/src/otel/setup.cfg index a8e712fa97..25f00d78c0 100644 --- a/python/src/otel/setup.cfg +++ b/python/src/otel/setup.cfg @@ -50,5 +50,7 @@ test = where = otel_sdk [options.entry_points] -opentelemetry_instrumentor = - aws_lambda = opentelemetry.instrumentation.aws_lambda:AwsLambdaInstrumentor +# NOTE: (NathanielRN) DO NOT add AwsLambdaInstrumentor entry point because +# current AWS Lambda implementation reloads a fresh import of the user's Lambda +# handler. Auto Instrumentation runs _before_ and if it instruments the handler +# that patching will be lost. diff --git a/python/src/otel/tests/test_otel.py b/python/src/otel/tests/test_otel.py index 89c9088233..4f5b945cf4 100644 --- a/python/src/otel/tests/test_otel.py +++ b/python/src/otel/tests/test_otel.py @@ -11,9 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import fileinput import os +import subprocess import sys from importlib import import_module +from shutil import which from unittest import mock from opentelemetry.instrumentation.aws_lambda import AwsLambdaInstrumentor @@ -33,6 +36,8 @@ INSTRUMENTATION_SRC_DIR = os.path.join( *(os.path.dirname(__file__), "..", "otel_sdk") ) +ORIG_HANDLER = "ORIG_HANDLER" +TOX_PYTHON_DIRECTORY = os.path.dirname(os.path.dirname(which("python3"))) class MockLambdaContext: @@ -67,6 +72,13 @@ def __init__(self, aws_request_id, invoked_function_arn): MOCK_W3C_TRACE_STATE_VALUE = "test_value" +def replace_in_file(filename, old_text, new_text): + with fileinput.FileInput(filename, inplace=True) as file_object: + for line in file_object: + # This directs the output to the file, not the console + print(line.replace(old_text, new_text), end="") + + def mock_aws_lambda_exec_wrapper(): """Mocks automatically instrumenting user Lambda function by pointing `AWS_LAMBDA_EXEC_WRAPPER` to the `otel-instrument` script. @@ -77,8 +89,40 @@ def mock_aws_lambda_exec_wrapper(): See more: https://aws-otel.github.io/docs/getting-started/lambda/lambda-python """ - # NOTE: AwsLambdaInstrumentor().instrument() is done at this point - exec(open(os.path.join(INSTRUMENTATION_SRC_DIR, "otel-instrument")).read()) + + # NOTE: Because we run as a subprocess, the python packages are NOT patched + # with instrumentation. In this test we just make sure we can complete auto + # instrumentation without error and the correct environment variabels are + # set. A future improvement might have us run `opentelemetry-instrument` in + # this process to imitate `otel-instrument`, but our lambda handler does not + # call other instrumented libraries so we have no use for it for now. + + print_environ_program = ( + "from os import environ;" + "print(f\"ORIG_HANDLER={environ['ORIG_HANDLER']}\");" + "print(f\"_HANDLER={environ['_HANDLER']}\");" + ) + + completed_subprocess = subprocess.run( + [ + os.path.join(INSTRUMENTATION_SRC_DIR, "otel-instrument"), + "python3", + "-c", + print_environ_program, + ], + check=True, + stdout=subprocess.PIPE, + text=True, + ) + + # NOTE: Because `otel-instrument` cannot affect this python environment, we + # parse the stdout produced by our test python program to update the + # environment in this parent python process. + + for env_var_line in completed_subprocess.stdout.split("\n"): + if env_var_line: + env_key, env_value = env_var_line.split("=") + os.environ[env_key] = env_value def mock_execute_lambda(event=None): @@ -104,6 +148,11 @@ class TestAwsLambdaInstrumentor(TestBase): def setUpClass(cls): super().setUpClass() sys.path.append(INSTRUMENTATION_SRC_DIR) + replace_in_file( + os.path.join(INSTRUMENTATION_SRC_DIR, "otel-instrument"), + 'export LAMBDA_LAYER_PKGS_DIR="/opt/python"', + f'export LAMBDA_LAYER_PKGS_DIR="{TOX_PYTHON_DIRECTORY}"', + ) def setUp(self): super().setUp() @@ -128,6 +177,11 @@ def tearDown(self): def tearDownClass(cls): super().tearDownClass() sys.path.remove(INSTRUMENTATION_SRC_DIR) + replace_in_file( + os.path.join(INSTRUMENTATION_SRC_DIR, "otel-instrument"), + f'export LAMBDA_LAYER_PKGS_DIR="{TOX_PYTHON_DIRECTORY}"', + 'export LAMBDA_LAYER_PKGS_DIR="/opt/python"', + ) def test_active_tracing(self): test_env_patch = mock.patch.dict( diff --git a/python/src/tox.ini b/python/src/tox.ini index 90b5317558..91aca6e7a9 100644 --- a/python/src/tox.ini +++ b/python/src/tox.ini @@ -14,7 +14,6 @@ passenv = TOXENV setenv = OTEL_PYTHON_TRACER_PROVIDER=sdk_tracer_provider - OTEL_TRACES_EXPORTER="console_span" ; override CORE_REPO_SHA via env variable when testing other branches/commits than main ; i.e: CORE_REPO_SHA=dde62cebffe519c35875af6d06fae053b3be65ec tox -e CORE_REPO_SHA={env:CORE_REPO_SHA:main}