diff --git a/local_testing/.gitignore b/local_testing/.gitignore new file mode 100644 index 0000000..5384ea6 --- /dev/null +++ b/local_testing/.gitignore @@ -0,0 +1,92 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +function.zip +lambda_output.txt +nr_tmp_env.sh +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# IPython Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# dotenv +.env + +# virtualenv +venv/ +ENV/ + +# Spyder project settings +.spyderproject + +# Rope project settings +.ropeproject \ No newline at end of file diff --git a/local_testing/README.md b/local_testing/README.md new file mode 100644 index 0000000..4458da7 --- /dev/null +++ b/local_testing/README.md @@ -0,0 +1,8 @@ +## Local Testing + +To test the extension on a AWS test account, follow the steps: +1. Configure the credentials for the AWS test account `aws configure` +2. Run `./publish.sh` to publish the layers to your test account `us-west-2` region +3. Publish script will create 4 lambda layers for runtimes - Python 3.12 [[Amazon Linux 2023](https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html)] & Python 3.11 [[Amazon Linux 2](https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html)] and architectures - x86 & ARM +3. Run `./test.sh` to create lambda with test layer published in step 2 +4. Go to your AWS test account and check the logs of the lambda with suffix - `NR_EXTENSION_TEST_LAMBDA_` for any error in extension diff --git a/local_testing/function.py b/local_testing/function.py new file mode 100644 index 0000000..9079eb7 --- /dev/null +++ b/local_testing/function.py @@ -0,0 +1,6 @@ +def handler(event, context): + # Your code here + return { + 'statusCode': 200, + 'body': 'Hello from Lambda!' + } diff --git a/local_testing/newrelic_lambda_wrapper.py b/local_testing/newrelic_lambda_wrapper.py new file mode 100644 index 0000000..4a8c89b --- /dev/null +++ b/local_testing/newrelic_lambda_wrapper.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- + +import importlib +import os +import sys +import warnings + +os.environ.setdefault("NEW_RELIC_APP_NAME", os.getenv("AWS_LAMBDA_FUNCTION_NAME", "")) +os.environ.setdefault("NEW_RELIC_NO_CONFIG_FILE", "true") +os.environ.setdefault("NEW_RELIC_DISTRIBUTED_TRACING_ENABLED", "true") +os.environ.setdefault("NEW_RELIC_SERVERLESS_MODE_ENABLED", "true") +os.environ.setdefault( + "NEW_RELIC_TRUSTED_ACCOUNT_KEY", os.getenv("NEW_RELIC_ACCOUNT_ID", "") +) + +# The agent will load some environment variables on module import so we need +# to perform the import after setting the necessary environment variables. +import newrelic.agent # noqa +from newrelic_lambda.lambda_handler import lambda_handler # noqa + +newrelic.agent.initialize() + + +class IOpipeNoOp(object): + def __call__(self, *args, **kwargs): + warnings.warn( + "Use of context.iopipe.* is no longer supported. " + "Please see New Relic Python agent documentation here: " + "https://docs.newrelic.com/docs/agents/python-agent" + ) + + def __getattr__(self, name): + return IOpipeNoOp() + + +def get_handler(): + if ( + "NEW_RELIC_LAMBDA_HANDLER" not in os.environ + or not os.environ["NEW_RELIC_LAMBDA_HANDLER"] + ): + raise ValueError( + "No value specified in NEW_RELIC_LAMBDA_HANDLER environment variable" + ) + + try: + module_path, handler_name = os.environ["NEW_RELIC_LAMBDA_HANDLER"].rsplit( + ".", 1 + ) + except ValueError: + raise ValueError( + "Improperly formated handler value: %s" + % os.environ["NEW_RELIC_LAMBDA_HANDLER"] + ) + + try: + # Use the same check as + # https://github.com/aws/aws-lambda-python-runtime-interface-client/blob/97dee252434edc56be4cafd54a9af1e7fa041eaf/awslambdaric/bootstrap.py#L33 + if module_path.split(".")[0] in sys.builtin_module_names: + raise ImportError( + "Cannot use built-in module %s as a handler module" % module_path + ) + + module = importlib.import_module(module_path.replace("/", ".")) + except Exception as e: + raise ImportError("Failed to import module '%s': %s" % (module_path, e)) + + try: + handler = getattr(module, handler_name) + except AttributeError: + raise AttributeError( + "No handler '%s' in module '%s'" % (handler_name, module_path) + ) + + return handler + + +# Greedily load the handler during cold start, so we don't pay for it on first invoke +wrapped_handler = get_handler() + + +@lambda_handler() +def handler(event, context): + context.iopipe = IOpipeNoOp() + return wrapped_handler(event, context) diff --git a/local_testing/publish.sh b/local_testing/publish.sh new file mode 100755 index 0000000..c116355 --- /dev/null +++ b/local_testing/publish.sh @@ -0,0 +1,197 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +# https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html +# Python 3.12 | Amazon Linux 2023 +# Python 3.11 | Amazon Linux 2 + +BUILD_DIR=python +DIST_DIR=dist + +BUCKET_PREFIX=nr-extension-test-layers + +PY311_DIST_ARM64=$DIST_DIR/python311.arm64.zip +PY311_DIST_X86_64=$DIST_DIR/python311.x86_64.zip + +PY312_DIST_ARM64=$DIST_DIR/python312.arm64.zip +PY312_DIST_X86_64=$DIST_DIR/python312.x86_64.zip + +REGIONS_X86=(us-west-2) +REGIONS_ARM=(us-west-2) + +EXTENSION_DIST_DIR=extensions +EXTENSION_DIST_ZIP=extension.zip +EXTENSION_DIST_PREVIEW_FILE=preview-extensions-ggqizro707 + +TMP_ENV_FILE_NAME=nr_tmp_env.sh + +function fetch_extension { + arch=$1 + url="https://github.com/newrelic/newrelic-lambda-extension/releases/download/v2.3.11/newrelic-lambda-extension.${arch}.zip" + rm -rf $EXTENSION_DIST_DIR $EXTENSION_DIST_ZIP + curl -L $url -o $EXTENSION_DIST_ZIP +} + +function download_extension { + fetch_extension $@ + + unzip $EXTENSION_DIST_ZIP -d . + rm -f $EXTENSION_DIST_ZIP +} + +function layer_name_str() { + rt_part="LambdaExtension" + arch_part="" + + case $1 in + "python3.11") + rt_part="Python311" + ;; + "python3.12") + rt_part="Python312" + ;; + esac + + case $2 in + "arm64") + arch_part="ARM64" + ;; + "x86_64") + arch_part="X86" + ;; + esac + + echo "NewRelic${rt_part}${arch_part}" +} + + +function hash_file() { + if command -v md5sum &> /dev/null ; then + md5sum $1 | awk '{ print $1 }' + else + md5 -q $1 + fi +} + + +function s3_prefix() { + name="nr-test-extension" + + case $1 in + "python3.11") + name="nr-python3.11" + ;; + "python3.12") + name="nr-python3.12" + ;; + esac + + echo $name +} + +function publish_layer { + layer_archive=$1 + region=$2 + runtime_name=$3 + arch=$4 + + layer_name=$( layer_name_str $runtime_name $arch ) + + hash=$( hash_file $layer_archive | awk '{ print $1 }' ) + + bucket_name="${BUCKET_PREFIX}-${region}" + s3_key="$( s3_prefix $runtime_name )/${hash}.${arch}.zip" + + echo "Uploading ${layer_archive} to s3://${bucket_name}/${s3_key}" + aws --region "$region" s3 cp $layer_archive "s3://${bucket_name}/${s3_key}" + + echo "Publishing ${runtime_name} layer to ${region}" + layer_output=$(aws lambda publish-layer-version \ + --layer-name ${layer_name} \ + --content "S3Bucket=${bucket_name},S3Key=${s3_key}" \ + --description "New Relic Test Layer for ${runtime_name} (${arch})" \ + --license-info "Apache-2.0" \ + --region "$region" \ + --output json) + + layer_version=$(echo $layer_output | jq -r '.Version') + layer_arn=$(echo $layer_output | jq -r '.LayerArn') + + echo "Published ${runtime_name} layer version ${layer_version} to ${region}" + echo "Layer ARN: ${layer_arn}" + full_layer_arn="${layer_arn}:${layer_version}" + + echo "Published ${runtime_name} layer version ${layer_version} to ${region}" + echo "Full Layer ARN with version: ${full_layer_arn}" + + arch_upper=$(echo "$arch" | tr '[:lower:]' '[:upper:]') + runtime_nodots=$(echo "${runtime_name//./}" | tr '[:lower:]' '[:upper:]') + + env_var_name="LAYER_ARN_${arch_upper}_${runtime_nodots}" + echo $env_var_name + declare "$env_var_name=$full_layer_arn" + + echo "export $env_var_name='$full_layer_arn'" >> $TMP_ENV_FILE_NAME +} + + +function build_python_version { + version=$1 + arch=$2 + dist_dir=$3 + + echo "Building New Relic layer for python$version ($arch)" + rm -rf $BUILD_DIR $dist_dir + mkdir -p $DIST_DIR + pip3 install --no-cache-dir -qU newrelic newrelic-lambda -t $BUILD_DIR/lib/python$version/site-packages + cp newrelic_lambda_wrapper.py $BUILD_DIR/lib/python$version/site-packages/newrelic_lambda_wrapper.py + find $BUILD_DIR -name '__pycache__' -exec rm -rf {} + + download_extension $arch + zip -rq $dist_dir $BUILD_DIR $EXTENSION_DIST_DIR $EXTENSION_DIST_PREVIEW_FILE + rm -rf $BUILD_DIR $EXTENSION_DIST_DIR $EXTENSION_DIST_PREVIEW_FILE + echo "Build complete: ${dist_dir}" +} + +function publish_python_version { + dist_dir=$1 + arch=$2 + version=$3 + regions=("${@:4}") + + if [ ! -f $dist_dir ]; then + echo "Package not found: ${dist_dir}" + exit 1 + fi + + for region in "${regions[@]}"; do + publish_layer $dist_dir $region python$version $arch + done +} + +if [ -f "$TMP_ENV_FILE_NAME" ]; then + rm -r "$TMP_ENV_FILE_NAME" +else + echo "File $TMP_ENV_FILE_NAME does not exist." +fi + + +# Build and publish for python3.11 arm64 +echo "Building and publishing for Python 3.11 ARM64..." +build_python_version "3.11" "arm64" $PY311_DIST_ARM64 +publish_python_version $PY311_DIST_ARM64 "arm64" "3.11" "${REGIONS_ARM[@]}" + +# Build and publish for python3.11 x86_64 +echo "Building and publishing for Python 3.11 x86_64..." +build_python_version "3.11" "x86_64" $PY311_DIST_X86_64 +publish_python_version $PY311_DIST_X86_64 "x86_64" "3.11" "${REGIONS_X86[@]}" + +# Build and publish for python3.12 arm64 +echo "Building and publishing for Python 3.12 ARM64..." +build_python_version "3.12" "arm64" $PY311_DIST_ARM64 +publish_python_version $PY311_DIST_ARM64 "arm64" "3.12" "${REGIONS_ARM[@]}" + +# Build and publish for python3.12 x86_64 +echo "Building and publishing for Python 3.11 x86_64..." +build_python_version "3.12" "x86_64" $PY311_DIST_X86_64 +publish_python_version $PY311_DIST_X86_64 "x86_64" "3.12" "${REGIONS_X86[@]}" \ No newline at end of file diff --git a/local_testing/test.sh b/local_testing/test.sh index f5173fa..f15cd51 100755 --- a/local_testing/test.sh +++ b/local_testing/test.sh @@ -1,79 +1,87 @@ #!/bin/bash # Variables -LAYER_NAME="MyLambdaLayer" -LAYER_ZIP="newrelic_lambda_layer.zip" -FUNCTION_NAME="Saket_sample_Integration_test" FUNCTION_FILE="function.zip" -ROLE_NAME="LambdaExecutionRole" -POLICY_NAME="AWSLambdaBasicExecutionRole" -REGION="us-east-1" # Change to your preferred AWS region -LOG_GROUP_NAME="/aws/lambda/${FUNCTION_NAME}" +ROLE_NAME="nr_extension_test_lambda_execution_role" +REGION="us-west-2" # Preferred AWS region TRUST_POLICY="trust-policy.json" +lambda_result_file="lambda_output.txt" -# Create trust policy file -cat > $TRUST_POLICY << EOL -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Service": "lambda.amazonaws.com" - }, - "Action": "sts:AssumeRole" - } - ] -} -EOL +NEW_RELIC_ACCOUNT_ID="${NEW_RELIC_ACCOUNT_ID}" +NEW_RELIC_LAMBDA_EXTENSION_ENABLED="True" +NEW_RELIC_LAMBDA_HANDLER="${NEW_RELIC_LAMBDA_HANDLER}" +NEW_RELIC_LICENSE_KEY_SECRET="${NEW_RELIC_LICENSE_KEY_SECRET}" +NEW_RELIC_LOG_ENDPOINT="${NEW_RELIC_LOG_ENDPOINT}" +NEW_RELIC_TELEMETRY_ENDPOINT="${NEW_RELIC_TELEMETRY_ENDPOINT}" -# Create IAM Role for Lambda Execution -EXECUTION_ROLE_ARN=$(aws iam create-role --role-name $ROLE_NAME --assume-role-policy-document file://$TRUST_POLICY --query 'Role.Arn' --output text) +source nr_tmp_env.sh -# Attach the basic execution policy to the role -aws iam attach-role-policy --role-name $ROLE_NAME --policy-arn arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole +zip function.zip function.py -# Wait a bit for the IAM role to propagate -sleep 10 - -# Create Lambda Layer -LAYER_ARN=$(aws lambda publish-layer-version --layer-name $LAYER_NAME --region $REGION --zip-file fileb://$LAYER_ZIP --query 'LayerVersionArn' --output text) -echo $LAYER_ARN -echo "Created lambda layer" -# Create Lambda Function -FUNCTION_ARN=$(aws lambda create-function --function-name $FUNCTION_NAME --zip-file fileb://$FUNCTION_FILE --handler function.handler --runtime python3.8 --role $EXECUTION_ROLE_ARN --layers $LAYER_ARN --query 'FunctionArn' --output text --region $REGION) -echo $FUNCTION_ARN -echo "Created lambda function" -# Invoke Lambda Function -aws lambda invoke --function-name $FUNCTION_NAME --region $REGION --payload '{}' output.txt -echo "Lambda Invoke function successful" +role_exists=$(aws iam get-role --role-name "$ROLE_NAME" --query 'Role.Arn' --output text --region "$REGION" 2>&1) +if [[ $role_exists == arn:aws:iam::* ]]; then + echo "IAM role already exists." + EXECUTION_ROLE_ARN=$role_exists +else + EXECUTION_ROLE_ARN=$(aws iam create-role --role-name "$ROLE_NAME" --assume-role-policy-document "file://$TRUST_POLICY" --query 'Role.Arn' --output text --region "$REGION") + if [ $? -ne 0 ]; then + echo "Error creating IAM role." + exit 1 + fi -Wait a bit for logs to generate -sleep 10 + aws iam attach-role-policy --role-name "$ROLE_NAME" --policy-arn "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" --region "$REGION" + echo "Attached policy to role." -# Get Logs -LOG_STREAM_NAME=$(aws logs describe-log-streams --log-group-name "$LOG_GROUP_NAME" --region "$REGION" --max-items 1 --order-by LastEventTime --descending --query 'logStreams[0].logStreamName' --output text) -echo $LOG_STREAM_NAME -# Assuming LOG_STREAM_NAME might have the value "None" at the end or be invalid -CLEANED_LOG_STREAM_NAME=$(echo "$LOG_STREAM_NAME" | sed 's/[[:space:]]*None$//' | tr -d '\r') + echo "Waiting for IAM role to propagate." + sleep 10 +fi -if [[ -z "$CLEANED_LOG_STREAM_NAME" ]]; then - echo "Log stream name is invalid or not found." -else - echo "cleaned log" - echo $CLEANED_LOG_STREAM_NAME - aws logs get-log-events --log-group-name "$LOG_GROUP_NAME" --region "$REGION" --log-stream-name "$CLEANED_LOG_STREAM_NAME" --output text +if [ ! -f "$FUNCTION_FILE" ]; then + echo "Lambda function file $FUNCTION_FILE does not exist." + exit 1 fi +runtimes=("python3.11" "python3.12") +architectures=("x86_64" "arm64") -# Cleanup steps (Lambda function, layer, IAM role) -aws lambda delete-function --function-name $FUNCTION_NAME --region $REGION -LAYER_VERSION=$(echo $LAYER_ARN | awk -F':' '{print $NF}') -aws lambda delete-layer-version --layer-name $LAYER_NAME --version-number $LAYER_VERSION --region $REGION -aws iam detach-role-policy --role-name $ROLE_NAME --policy-arn arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole -aws iam delete-role --role-name $ROLE_NAME +for arch in "${architectures[@]}"; do + for runtime in "${runtimes[@]}"; do + FUNCTION_NAME_SUFFIX="${arch}_${runtime//./}" + FUNCTION_NAME="NR_EXTENSION_TEST_LAMBDA_${FUNCTION_NAME_SUFFIX}" + arch_upper=$(echo "$arch" | tr '[:lower:]' '[:upper:]') + runtime_nodots=$(echo "${runtime//./}" | tr '[:lower:]' '[:upper:]') -# Optional: Delete Log Group -aws logs delete-log-group --log-group-name $LOG_GROUP_NAME + env_var_name="LAYER_ARN_${arch_upper}_${runtime_nodots}" + final_layer_arn="${!env_var_name}" + echo "$final_layer_arn" + if aws lambda get-function --function-name "$FUNCTION_NAME" --region "$REGION" 2>/dev/null; then + echo "Lambda function $FUNCTION_NAME already exists." + else + FUNCTION_ARN=$(aws lambda create-function \ + --function-name "$FUNCTION_NAME" \ + --zip-file "fileb://$FUNCTION_FILE" \ + --handler "function.handler" \ + --runtime "$runtime" \ + --architectures "$arch" \ + --role "$EXECUTION_ROLE_ARN" \ + --layers "$final_layer_arn" \ + --environment Variables="{NEW_RELIC_ACCOUNT_ID=$NEW_RELIC_ACCOUNT_ID,NEW_RELIC_LAMBDA_EXTENSION_ENABLED=$NEW_RELIC_LAMBDA_EXTENSION_ENABLED,NEW_RELIC_LAMBDA_HANDLER=$NEW_RELIC_LAMBDA_HANDLER,NEW_RELIC_LICENSE_KEY_SECRET=$NEW_RELIC_LICENSE_KEY_SECRET,NEW_RELIC_LOG_ENDPOINT=$NEW_RELIC_LOG_ENDPOINT,NEW_RELIC_TELEMETRY_ENDPOINT=$NEW_RELIC_TELEMETRY_ENDPOINT}" \ + --query 'FunctionArn' \ + --output text \ + --region "$REGION") + if [ $? -ne 0 ]; then + echo "Error creating Lambda function $FUNCTION_NAME." + exit 1 + fi + echo "Created lambda function: $FUNCTION_ARN" -echo "Lambda function, layer, and IAM role cleanup completed." + fi + sleep 10 + aws lambda invoke --function-name "$FUNCTION_NAME" --region "$REGION" --payload '{}' "$lambda_result_file" + if [ $? -eq 0 ]; then + echo "Lambda function invoked successfully. Output in $lambda_result_file" + else + echo "Error invoking Lambda function." + fi + done +done diff --git a/local_testing/trust-policy.json b/local_testing/trust-policy.json new file mode 100644 index 0000000..fb84ae9 --- /dev/null +++ b/local_testing/trust-policy.json @@ -0,0 +1,12 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } + ] +}