Run e2e tests on Cloud Functions (#349)

GoogleCloudPlatform · Jul 31, 2024 · 7932c01 · 7932c01
1 parent 871ad2c
commit 7932c01
Show file tree

Hide file tree

Showing 10 changed files with 108 additions and 19 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -3,3 +3,5 @@ docs/
 .tox
 **/venv/
 .mypy_cache/
+**/__pycache__/
+e2e-test-server/wheels/
diff --git a/cloudbuild-e2e-cloud-functions-gen2.yaml b/cloudbuild-e2e-cloud-functions-gen2.yaml
@@ -0,0 +1,49 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+steps:
+  # Build function source zip with vendored wheels of packages in this monorepo
+  - name: python:3.11-slim
+    id: build
+    script: |
+      set -xe
+      apt-get update
+      apt-get install -y zip
+
+      cd e2e-test-server/
+      # package monorepo libraries into wheels in local wheels/ directory
+      pip wheel \
+        --no-deps \
+        --wheel-dir wheels \
+        ../opentelemetry-exporter-gcp-trace/ \
+        ../opentelemetry-resourcedetector-gcp/ \
+        ../opentelemetry-propagator-gcp
+
+      zip -qr function-source.zip .
+
+  # Run the test
+  - name: $_TEST_RUNNER_IMAGE
+    id: run-tests-cloud-run
+    dir: /
+    env: ["PROJECT_ID=$PROJECT_ID"]
+    args:
+      - cloud-functions-gen2
+      - --functionsource=/workspace/e2e-test-server/function-source.zip
+      - --runtime=python311
+      - --entrypoint=cloud_functions_handler
+
+logsBucket: gs://opentelemetry-ops-e2e-cloud-build-logs
+substitutions:
+  _TEST_RUNNER_IMAGE: gcr.io/${PROJECT_ID}/opentelemetry-operations-e2e-testing:0.19.0
+  _TEST_SERVER_IMAGE: gcr.io/${PROJECT_ID}/opentelemetry-operations-python-e2e-test-server:${SHORT_SHA}
diff --git a/e2e-test-server/Dockerfile b/e2e-test-server/Dockerfile
@@ -31,8 +31,8 @@ COPY opentelemetry-propagator-gcp opentelemetry-propagator-gcp
 COPY opentelemetry-resourcedetector-gcp opentelemetry-resourcedetector-gcp
 WORKDIR $SRC/e2e-test-server
 # copy requirements/constraints
-COPY e2e-test-server/requirements.txt e2e-test-server/constraints.txt ./
-RUN python -m venv venv && ./venv/bin/pip install -r requirements.txt
+COPY e2e-test-server/*.txt ./
+RUN python -m venv venv && ./venv/bin/pip install -r requirements-dockerfile.txt
 
 FROM python-base
 WORKDIR $SRC/e2e-test-server

diff --git a/e2e-test-server/constraints.txt b/e2e-test-server/constraints.txt
@@ -4,8 +4,11 @@ cachetools==5.3.1
 certifi==2023.7.22
 charset-normalizer==3.3.0
 click==8.1.7
+cloudevents==1.11.0
 Deprecated==1.2.14
+deprecation==2.1.0
 Flask==3.0.0
+functions-framework==3.8.0
 google-api-core==2.12.0
 google-auth==2.23.3
 google-cloud-pubsub==2.18.4
@@ -14,6 +17,7 @@ googleapis-common-protos==1.61.0
 grpc-google-iam-v1==0.12.6
 grpcio==1.59.0
 grpcio-status==1.59.0
+gunicorn==22.0.0
 idna==3.4
 importlib-metadata==6.8.0
 itsdangerous==2.1.2
@@ -22,6 +26,7 @@ MarkupSafe==2.1.3
 opentelemetry-api==1.20.0
 opentelemetry-sdk==1.20.0
 opentelemetry-semantic-conventions==0.41b0
+packaging==24.1
 proto-plus==1.22.3
 protobuf==4.24.4
 pyasn1==0.5.0
@@ -33,6 +38,7 @@ rsa==4.9
 typing_extensions==4.8.0
 urllib3==2.0.6
 waitress==2.1.2
+watchdog==4.0.1
 Werkzeug==3.0.0
 wrapt==1.15.0
 zipp==3.17.0
diff --git a/e2e-test-server/e2e_test_server/constants.py b/e2e-test-server/e2e_test_server/constants.py
@@ -15,12 +15,11 @@
 import enum
 import os
 
-from pydantic import BaseModel
-
 
 class SubscriptionMode(enum.Enum):
     PULL = "pull"
     PUSH = "push"
+    UNDEFINED = None
 
 
 INSTRUMENTING_MODULE_NAME = "opentelemetry-ops-e2e-test-server"
@@ -29,10 +28,10 @@ class SubscriptionMode(enum.Enum):
 TEST_ID = "test_id"
 TRACE_ID = "trace_id"
 SUBSCRIPTION_MODE: SubscriptionMode = SubscriptionMode(
-    os.environ["SUBSCRIPTION_MODE"]
+    os.environ.get("SUBSCRIPTION_MODE")
 )
 PROJECT_ID = os.environ["PROJECT_ID"]
-REQUEST_SUBSCRIPTION_NAME = os.environ["REQUEST_SUBSCRIPTION_NAME"]
+REQUEST_SUBSCRIPTION_NAME = os.environ.get("REQUEST_SUBSCRIPTION_NAME")
 RESPONSE_TOPIC_NAME = os.environ["RESPONSE_TOPIC_NAME"]
 PUSH_PORT = (
     os.environ["PUSH_PORT"]

diff --git a/e2e-test-server/e2e_test_server/server.py b/e2e-test-server/e2e_test_server/server.py
@@ -12,10 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import base64
 import logging
 from typing import Literal
 
+from cloudevents.http.event import CloudEvent
+import functions_framework
 from google.cloud import pubsub_v1
 from google.cloud.pubsub_v1.subscriber.message import Message
 from google.rpc import code_pb2
@@ -168,3 +169,17 @@ def index() -> Response:
             return Response(status=400)
 
     serve(app, port=PUSH_PORT)
+
+
+@functions_framework.cloud_event
+def cloud_functions_handler(cloud_event: CloudEvent) -> None:
+    """Handles pub/sub push message on Cloud Functions"""
+    # cloud_event.data is of type MessagePublishedData, i.e. it contains the pub/sub message
+    # https://github.com/googleapis/google-cloudevents/blob/v2.1.6/proto/google/events/cloud/pubsub/v1/data.proto#L26
+    payload = types.PubsubPushPayload(**cloud_event.data)
+
+    ack_or_nack = handle_message(
+        payload.message.to_pubsub_message(), _Responder()
+    )
+    if ack_or_nack == "nack":
+        raise Exception("Nacking message")
diff --git a/e2e-test-server/main.py b/e2e-test-server/main.py
@@ -15,12 +15,16 @@
 import logging
 
 from e2e_test_server import server
+from e2e_test_server.server import cloud_functions_handler
 from e2e_test_server.constants import SUBSCRIPTION_MODE, SubscriptionMode
 
 if __name__ == "__main__":
     logging.basicConfig(level=logging.INFO)
 
     if SUBSCRIPTION_MODE is SubscriptionMode.PULL:
         server.pubsub_pull()
-    else:
+    elif SUBSCRIPTION_MODE is SubscriptionMode.PUSH:
         server.pubsub_push()
+
+# Expose handler to functions-framework
+__all__ = ["cloud_functions_handler"]
diff --git a/e2e-test-server/requirements-dockerfile.txt b/e2e-test-server/requirements-dockerfile.txt
@@ -0,0 +1,5 @@
+# Used for installing from the monorepo in the Dockerfile
+-r requirements-shared.txt
+../opentelemetry-exporter-gcp-trace
+../opentelemetry-propagator-gcp
+../opentelemetry-resourcedetector-gcp
diff --git a/e2e-test-server/requirements-shared.txt b/e2e-test-server/requirements-shared.txt
@@ -0,0 +1,10 @@
+# Shared by all requirements files
+-c constraints.txt
+opentelemetry-sdk
+opentelemetry-api
+Flask
+google-cloud-pubsub
+googleapis-common-protos
+pydantic
+waitress
+functions-framework
diff --git a/e2e-test-server/requirements.txt b/e2e-test-server/requirements.txt
@@ -1,11 +1,10 @@
--c constraints.txt
-../opentelemetry-exporter-gcp-trace
-../opentelemetry-propagator-gcp
-../opentelemetry-resourcedetector-gcp
-opentelemetry-sdk
-opentelemetry-api
-Flask
-google-cloud-pubsub
-googleapis-common-protos
-pydantic
-waitress
+# Used for Cloud Functions and GAE, has to be named requirements.txt
+-r requirements-shared.txt
+
+ # allow installing dev versions so pip doesn't pull from pypi
+--pre
+# install monorepo packages from local vendored wheels directory
+--find-links wheels/
+opentelemetry-exporter-gcp-trace
+opentelemetry-propagator-gcp
+opentelemetry-resourcedetector-gcp