From 7932c01c7de0b777e35d1aa8c2d70b09e75f744d Mon Sep 17 00:00:00 2001 From: Aaron Abbott Date: Wed, 31 Jul 2024 15:08:25 -0400 Subject: [PATCH] Run e2e tests on Cloud Functions (#349) --- .dockerignore | 2 + cloudbuild-e2e-cloud-functions-gen2.yaml | 49 ++++++++++++++++++++ e2e-test-server/Dockerfile | 4 +- e2e-test-server/constraints.txt | 6 +++ e2e-test-server/e2e_test_server/constants.py | 7 ++- e2e-test-server/e2e_test_server/server.py | 17 ++++++- e2e-test-server/main.py | 6 ++- e2e-test-server/requirements-dockerfile.txt | 5 ++ e2e-test-server/requirements-shared.txt | 10 ++++ e2e-test-server/requirements.txt | 21 ++++----- 10 files changed, 108 insertions(+), 19 deletions(-) create mode 100644 cloudbuild-e2e-cloud-functions-gen2.yaml create mode 100644 e2e-test-server/requirements-dockerfile.txt create mode 100644 e2e-test-server/requirements-shared.txt diff --git a/.dockerignore b/.dockerignore index 82e9cab0..90e3fc3d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,3 +3,5 @@ docs/ .tox **/venv/ .mypy_cache/ +**/__pycache__/ +e2e-test-server/wheels/ diff --git a/cloudbuild-e2e-cloud-functions-gen2.yaml b/cloudbuild-e2e-cloud-functions-gen2.yaml new file mode 100644 index 00000000..aed7fdc0 --- /dev/null +++ b/cloudbuild-e2e-cloud-functions-gen2.yaml @@ -0,0 +1,49 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +steps: + # Build function source zip with vendored wheels of packages in this monorepo + - name: python:3.11-slim + id: build + script: | + set -xe + apt-get update + apt-get install -y zip + + cd e2e-test-server/ + # package monorepo libraries into wheels in local wheels/ directory + pip wheel \ + --no-deps \ + --wheel-dir wheels \ + ../opentelemetry-exporter-gcp-trace/ \ + ../opentelemetry-resourcedetector-gcp/ \ + ../opentelemetry-propagator-gcp + + zip -qr function-source.zip . + + # Run the test + - name: $_TEST_RUNNER_IMAGE + id: run-tests-cloud-run + dir: / + env: ["PROJECT_ID=$PROJECT_ID"] + args: + - cloud-functions-gen2 + - --functionsource=/workspace/e2e-test-server/function-source.zip + - --runtime=python311 + - --entrypoint=cloud_functions_handler + +logsBucket: gs://opentelemetry-ops-e2e-cloud-build-logs +substitutions: + _TEST_RUNNER_IMAGE: gcr.io/${PROJECT_ID}/opentelemetry-operations-e2e-testing:0.19.0 + _TEST_SERVER_IMAGE: gcr.io/${PROJECT_ID}/opentelemetry-operations-python-e2e-test-server:${SHORT_SHA} diff --git a/e2e-test-server/Dockerfile b/e2e-test-server/Dockerfile index a7d69724..800db1c0 100644 --- a/e2e-test-server/Dockerfile +++ b/e2e-test-server/Dockerfile @@ -31,8 +31,8 @@ COPY opentelemetry-propagator-gcp opentelemetry-propagator-gcp COPY opentelemetry-resourcedetector-gcp opentelemetry-resourcedetector-gcp WORKDIR $SRC/e2e-test-server # copy requirements/constraints -COPY e2e-test-server/requirements.txt e2e-test-server/constraints.txt ./ -RUN python -m venv venv && ./venv/bin/pip install -r requirements.txt +COPY e2e-test-server/*.txt ./ +RUN python -m venv venv && ./venv/bin/pip install -r requirements-dockerfile.txt FROM python-base WORKDIR $SRC/e2e-test-server diff --git a/e2e-test-server/constraints.txt b/e2e-test-server/constraints.txt index 12e33716..f71f5048 100644 --- a/e2e-test-server/constraints.txt +++ b/e2e-test-server/constraints.txt @@ -4,8 +4,11 @@ cachetools==5.3.1 certifi==2023.7.22 charset-normalizer==3.3.0 click==8.1.7 +cloudevents==1.11.0 Deprecated==1.2.14 +deprecation==2.1.0 Flask==3.0.0 +functions-framework==3.8.0 google-api-core==2.12.0 google-auth==2.23.3 google-cloud-pubsub==2.18.4 @@ -14,6 +17,7 @@ googleapis-common-protos==1.61.0 grpc-google-iam-v1==0.12.6 grpcio==1.59.0 grpcio-status==1.59.0 +gunicorn==22.0.0 idna==3.4 importlib-metadata==6.8.0 itsdangerous==2.1.2 @@ -22,6 +26,7 @@ MarkupSafe==2.1.3 opentelemetry-api==1.20.0 opentelemetry-sdk==1.20.0 opentelemetry-semantic-conventions==0.41b0 +packaging==24.1 proto-plus==1.22.3 protobuf==4.24.4 pyasn1==0.5.0 @@ -33,6 +38,7 @@ rsa==4.9 typing_extensions==4.8.0 urllib3==2.0.6 waitress==2.1.2 +watchdog==4.0.1 Werkzeug==3.0.0 wrapt==1.15.0 zipp==3.17.0 diff --git a/e2e-test-server/e2e_test_server/constants.py b/e2e-test-server/e2e_test_server/constants.py index 5b4edca5..63c28f06 100644 --- a/e2e-test-server/e2e_test_server/constants.py +++ b/e2e-test-server/e2e_test_server/constants.py @@ -15,12 +15,11 @@ import enum import os -from pydantic import BaseModel - class SubscriptionMode(enum.Enum): PULL = "pull" PUSH = "push" + UNDEFINED = None INSTRUMENTING_MODULE_NAME = "opentelemetry-ops-e2e-test-server" @@ -29,10 +28,10 @@ class SubscriptionMode(enum.Enum): TEST_ID = "test_id" TRACE_ID = "trace_id" SUBSCRIPTION_MODE: SubscriptionMode = SubscriptionMode( - os.environ["SUBSCRIPTION_MODE"] + os.environ.get("SUBSCRIPTION_MODE") ) PROJECT_ID = os.environ["PROJECT_ID"] -REQUEST_SUBSCRIPTION_NAME = os.environ["REQUEST_SUBSCRIPTION_NAME"] +REQUEST_SUBSCRIPTION_NAME = os.environ.get("REQUEST_SUBSCRIPTION_NAME") RESPONSE_TOPIC_NAME = os.environ["RESPONSE_TOPIC_NAME"] PUSH_PORT = ( os.environ["PUSH_PORT"] diff --git a/e2e-test-server/e2e_test_server/server.py b/e2e-test-server/e2e_test_server/server.py index 4e11b712..52105630 100644 --- a/e2e-test-server/e2e_test_server/server.py +++ b/e2e-test-server/e2e_test_server/server.py @@ -12,10 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import base64 import logging from typing import Literal +from cloudevents.http.event import CloudEvent +import functions_framework from google.cloud import pubsub_v1 from google.cloud.pubsub_v1.subscriber.message import Message from google.rpc import code_pb2 @@ -168,3 +169,17 @@ def index() -> Response: return Response(status=400) serve(app, port=PUSH_PORT) + + +@functions_framework.cloud_event +def cloud_functions_handler(cloud_event: CloudEvent) -> None: + """Handles pub/sub push message on Cloud Functions""" + # cloud_event.data is of type MessagePublishedData, i.e. it contains the pub/sub message + # https://github.com/googleapis/google-cloudevents/blob/v2.1.6/proto/google/events/cloud/pubsub/v1/data.proto#L26 + payload = types.PubsubPushPayload(**cloud_event.data) + + ack_or_nack = handle_message( + payload.message.to_pubsub_message(), _Responder() + ) + if ack_or_nack == "nack": + raise Exception("Nacking message") diff --git a/e2e-test-server/main.py b/e2e-test-server/main.py index ead0377d..7c0d65a5 100644 --- a/e2e-test-server/main.py +++ b/e2e-test-server/main.py @@ -15,6 +15,7 @@ import logging from e2e_test_server import server +from e2e_test_server.server import cloud_functions_handler from e2e_test_server.constants import SUBSCRIPTION_MODE, SubscriptionMode if __name__ == "__main__": @@ -22,5 +23,8 @@ if SUBSCRIPTION_MODE is SubscriptionMode.PULL: server.pubsub_pull() - else: + elif SUBSCRIPTION_MODE is SubscriptionMode.PUSH: server.pubsub_push() + +# Expose handler to functions-framework +__all__ = ["cloud_functions_handler"] diff --git a/e2e-test-server/requirements-dockerfile.txt b/e2e-test-server/requirements-dockerfile.txt new file mode 100644 index 00000000..11f15f9b --- /dev/null +++ b/e2e-test-server/requirements-dockerfile.txt @@ -0,0 +1,5 @@ +# Used for installing from the monorepo in the Dockerfile +-r requirements-shared.txt +../opentelemetry-exporter-gcp-trace +../opentelemetry-propagator-gcp +../opentelemetry-resourcedetector-gcp diff --git a/e2e-test-server/requirements-shared.txt b/e2e-test-server/requirements-shared.txt new file mode 100644 index 00000000..49179431 --- /dev/null +++ b/e2e-test-server/requirements-shared.txt @@ -0,0 +1,10 @@ +# Shared by all requirements files +-c constraints.txt +opentelemetry-sdk +opentelemetry-api +Flask +google-cloud-pubsub +googleapis-common-protos +pydantic +waitress +functions-framework diff --git a/e2e-test-server/requirements.txt b/e2e-test-server/requirements.txt index ed175141..dc8e9e62 100644 --- a/e2e-test-server/requirements.txt +++ b/e2e-test-server/requirements.txt @@ -1,11 +1,10 @@ --c constraints.txt -../opentelemetry-exporter-gcp-trace -../opentelemetry-propagator-gcp -../opentelemetry-resourcedetector-gcp -opentelemetry-sdk -opentelemetry-api -Flask -google-cloud-pubsub -googleapis-common-protos -pydantic -waitress +# Used for Cloud Functions and GAE, has to be named requirements.txt +-r requirements-shared.txt + + # allow installing dev versions so pip doesn't pull from pypi +--pre +# install monorepo packages from local vendored wheels directory +--find-links wheels/ +opentelemetry-exporter-gcp-trace +opentelemetry-propagator-gcp +opentelemetry-resourcedetector-gcp