Update trace based tests run script (open-telemetry#1018)

* chore(tests): allow partial runs on trace-based test scripts * Adding option to run trace-based tests for selected services * Updating testing README * Fixing sanity checks * Fixing lint issues * Apply suggestions from code review Co-authored-by: Adnan Rahić <[email protected]> * Update README.md with suggestions * Updating Changelog * Fixing linter issue * Fixed changelog --------- Co-authored-by: Adnan Rahić <[email protected]> Co-authored-by: Carter Socha <[email protected]> Co-authored-by: Austin Parker <[email protected]> Co-authored-by: Juliano Costa <[email protected]>
jmichalak9 · Aug 7, 2023 · 740930b · 740930b
1 parent ba9b626
commit 740930b
Show file tree

Hide file tree

Showing 5 changed files with 71 additions and 17 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,8 @@ release.
 
 ## Unreleased
 
+* [tests] Update trace-based tests run script
+  ([#1018](https://github.com/open-telemetry/opentelemetry-demo/pull/1018))
 * Add cartServiceFailure feature flag triggering Cart Service errors
   ([#824](https://github.com/open-telemetry/opentelemetry-demo/pull/824))
 * [paymentservice] update JS SDKs to 1.12.0/0.38.0
@@ -45,6 +47,8 @@ release.
   ([#935](https://github.com/open-telemetry/opentelemetry-demo/pull/935))
 * [cartservice] update service to .NET 7
   ([#942](https://github.com/open-telemetry/opentelemetry-demo/pull/942))
+* [tests] Add trace-based testing examples
+  ([#877](https://github.com/open-telemetry/opentelemetry-demo/pull/877))
 * Introduce minimal mode to run demo
   ([#872](https://github.com/open-telemetry/opentelemetry-demo/pull/872))
 * [frontendproxy]Envoy expose a route for the collector to route frontend spans

diff --git a/Makefile b/Makefile
@@ -97,7 +97,7 @@ run-tests:
 	docker compose run traceBasedTests
 
 run-tracetesting:
-	docker compose run traceBasedTests
+	docker compose run traceBasedTests ${SERVICES_TO_TEST}
 
 .PHONY: generate-protobuf
 generate-protobuf:

diff --git a/test/README.md b/test/README.md
@@ -1,7 +1,49 @@
 # Service Testing
 
-Testing gRPC services as black boxes.
+There are two ways to test the service APIs in the OpenTelemetry Demo:
+
+1. Using black box-testing, calling gRPC services
+and validating their direct response
+2. Using Trace-based tests, calling both HTTP and
+gRPC services and validating their direct response as well as
+the distributed traces they generate
+
+## Testing gRPC services as black boxes
+
+To run the entire test suite as a black box, run the command:
+
+```sh
+docker compose run integrationTests
+```
+
+If you want to run tests for a specific service, run:
 
 1. Start the services you want to test with `docker compose up --build <service>`
-1. Run `npm install`
-1. Run `npm test` or `npx ava --match='<pattern>'` to match test names
+2. Run `npm install`
+3. Run `npm test` or `npx ava --match='<pattern>'` to match test names
+
+## Testing services with Trace-based tests
+
+To run the entire test suite of trace-based tests, run the command:
+
+```sh
+make run-tracetesting
+#or
+docker compose run traceBasedTests
+```
+
+To run tests for specific services, pass the name of the service as a
+parameter (using the folder names located [here](./tracetesting/)):
+
+```sh
+make run-tracetesting SERVICES_TO_TEST="service-1 service-2 ..."
+#or
+docker compose run traceBasedTests "service-1 service-2 ..."
+```
+
+For instance, if you need to run the tests for `ad-service` and
+`payment-service`, you can run them with:
+
+```sh
+make run-tracetesting SERVICES_TO_TEST="ad-service payment-service"
+```
diff --git a/test/tracetesting/Dockerfile b/test/tracetesting/Dockerfile
@@ -14,4 +14,4 @@ COPY ./pb ./pb
 
 WORKDIR /app/test/tracetesting
 
-CMD ["/bin/sh", "/app/test/tracetesting/run.bash"]
+ENTRYPOINT ["/bin/bash", "/app/test/tracetesting/run.bash"]
diff --git a/test/tracetesting/run.bash b/test/tracetesting/run.bash
@@ -47,12 +47,26 @@ EOF
 
 run_tracetest() {
   service_name=$1
-  test_file=./$service_name/all.yaml
+  transaction_file=./$service_name/all.yaml
 
-  tracetest --config ./cli-config.yml test run --definition $test_file --environment ./tracetesting-env.yaml --wait-for-result
+  tracetest --config ./cli-config.yml run transaction --file $transaction_file --environment ./tracetesting-env.yaml
   return $?
 }
 
+ALL_SERVICES=("ad-service" "cart-service" "currency-service" "checkout-service" "frontend-service" "email-service" "payment-service" "product-catalog-service" "recommendation-service" "shipping-service")
+CHOSEN_SERVICES=()
+
+while [[ $# -gt 0 ]]; do
+  CHOSEN_SERVICES+=("$1")
+  shift
+done
+
+if [ ${#CHOSEN_SERVICES[@]} -eq 0 ]; then
+  for service in "${ALL_SERVICES[@]}"; do
+    CHOSEN_SERVICES+=("$service")  
+  done
+fi
+
 check_if_tracetest_is_installed
 create_env_file
 
@@ -62,17 +76,11 @@ EXIT_STATUS=0
 
 echo ""
 echo "Running trace-based tests..."
+echo ""
 
-run_tracetest ad-service || EXIT_STATUS=$?
-run_tracetest cart-service || EXIT_STATUS=$?
-run_tracetest currency-service || EXIT_STATUS=$?
-run_tracetest checkout-service || EXIT_STATUS=$?
-run_tracetest frontend-service || EXIT_STATUS=$?
-run_tracetest email-service || EXIT_STATUS=$?
-run_tracetest payment-service || EXIT_STATUS=$?
-run_tracetest product-catalog-service || EXIT_STATUS=$?
-run_tracetest recommendation-service || EXIT_STATUS=$?
-run_tracetest shipping-service || EXIT_STATUS=$?
+for service in "${CHOSEN_SERVICES[@]}"; do
+  run_tracetest $service || EXIT_STATUS=$?
+done
 
 echo ""
 echo "Tests done! Exit code: $EXIT_STATUS"