Simplify e2e tests

Makefile

@@ -50,7 +50,7 @@ compile: ## Compile the pipeline to pipeline.yaml. Must specify pipeline=<traini
 	@cd pipelines/src && \
 	poetry run kfp dsl compile --py pipelines/${pipeline}/pipeline.py --output pipelines/${pipeline}/pipeline.yaml --function pipeline
 
-targets ?= "training serving"
+targets ?= training serving
 build: ## Build and push training and/or serving container(s) image using Docker. Specify targets=<training serving> e.g. targets=training or targets="training serving" (default)
 	@cd model && \
 	for target in $$targets ; do \
@@ -63,17 +63,18 @@ build: ## Build and push training and/or serving container(s) image using Docker
 	done 
 
 
-compile ?=true
-build ?= true 
-run: ## Compile or build pipeline and run pipeline in sandbox environment. Set compile=false to skip recompiling the pipeline and set build=false to skip rebuilding container images
-	@if [ "${compile}" ]; then \
+compile ?= true
+build ?= true
+wait ?= false
+run: ## Run pipeline in sandbox environment. Must specify pipeline=<training|prediction>. Optionally specify ENABLE_PIPELINE_CACHING=<true|false> (defaults to default Vertex caching behaviour) and wait=<true|false> (default = false). Set compile=false to skip recompiling the pipeline and set build=false to skip rebuilding container images
+	@if [ $(compile) = "true" ]; then \
 		$(MAKE) compile ; \
 	fi && \
-	if [ "${build}" ]; then \
+	if [ $(build) = "true" ]; then \
 		$(MAKE) build ; \
 	fi && \
-	cd pipelines/src  \
-	poetry run python -m pipelines.utils.trigger_pipeline --template_path=pipelines/${pipeline}/pipeline.yaml --display_name=${pipeline}
+	cd pipelines/src && \
+	poetry run python -m pipelines.utils.trigger_pipeline --template_path=pipelines/${pipeline}/pipeline.yaml --display_name=${pipeline} --wait=${wait}
 
 
 test: ## Run unit tests for a component group or for all component groups and the pipeline trigger code.
@@ -93,8 +94,3 @@ test: ## Run unit tests for a component group or for all component groups and th
 			cd ../.. ;\
 		done ; \
 	fi
-
-
-e2e-tests: ## Perform end-to-end (E2E) pipeline tests. Must specify pipeline=<training|prediction>. Optionally specify ENABLE_PIPELINE_CACHING=<true|false> (defaults to default Vertex caching behaviour).
-	@ cd pipelines && \
-	poetry run pytest --log-cli-level=INFO tests/$(pipeline)

README.md

@@ -177,13 +177,14 @@ make build target=serving
 You can run the training pipeline (for example) with:
 
 ```bash
-make run pipeline=training
+make run pipeline=training [ wait=<true|false> ]
 ```
 
 This will execute the pipeline using the chosen template on Vertex AI, namely it will:
 
 1. Compile the pipeline using the Kubeflow Pipelines SDK
 1. Trigger the pipeline with the help of `pipelines/trigger/main.py`
+1. (optional) Wait for pipeline to finish before returning if `wait` is set to `true` (default is false)
 
 #### Pipeline input parameters
 
@@ -193,8 +194,8 @@ When triggering ad hoc runs in your dev/sandbox environment, or when running the
 
 ## Testing
 
-Unit tests and end-to-end (E2E) pipeline tests are performed using [pytest](https://docs.pytest.org). 
-The unit tests for custom KFP components are run on each pull request, as well as the E2E tests. To run them on your local machine:
+Unit tests are performed using [pytest](https://docs.pytest.org).
+The unit tests for custom KFP components are run on each pull request. To run them on your local machine:
 
 ```
 make test
@@ -205,12 +206,6 @@ Alternatively, only test one of the component groups by running:
 make test GROUP=vertex-components
 ```
 
-To run end-to-end tests of a single pipeline, you can use:
-
-```
-make e2e-tests pipeline=<training|prediction>
-```
-
 There are also unit tests for the utility scripts in [pipelines/src/pipelines/utils](/pipelines/src/pipelines/utils/). To run them on your local machine:
 
 ```
@@ -241,14 +236,12 @@ vertex-pipelines-end-to-end-samples
 
 Make sure that you give the ML pipeline a unique name in the `@pipeline` decorator.
 
-To run your pipeline, use `make run` as before:
+To run your pipeline, use `make run` as before (optionally adding parameter to wait until pipeline is finished before returning - defaults to false):
 
 ```bash
-make run pipeline=your_new_pipeline
+make run pipeline=your_new_pipeline [ wait=<true|false> ]
 ```
 
-You will also need to add an E2E test - copy and paste the `training` or `prediction` example in [pipelines/tests/](/pipelines/tests/).
-
 Some of the scripts e.g. CI/CD pipelines assume only a training and prediction pipeline. You will need to adapt these to add in the compile, run and upload steps for your new pipeline in [cloudbuild/pr-checks.yaml](/cloudbuild/pr-checks.yaml), [cloudbuild/e2e-test.yaml](/cloudbuild/e2e-test.yaml) and [cloudbuild/release.yaml](/cloudbuild/release.yaml).
 
 ### Scheduling pipelines

cloudbuild/e2e-test.yaml

@@ -45,8 +45,8 @@ steps:
         curl -sSL https://install.python-poetry.org | python3 - && \
         export PATH="/builder/home/.local/bin:$$PATH" && \
         make install && \
-        make e2e-tests pipeline=training && \
-        make e2e-tests pipeline=prediction
+        make run pipeline=training build=false wait=true && \
+        make run pipeline=prediction build=false wait=true
     env:
       - ENABLE_PIPELINE_CACHING=${_TEST_ENABLE_PIPELINE_CACHING}
       - VERTEX_LOCATION=${_TEST_VERTEX_LOCATION}

docs/PRODUCTION.md

@@ -38,7 +38,7 @@ When you open the Pull Request, the CI pipeline (`pr-checks.yaml`) should be tri
 
 | :bulb: Remember    |
 |:-------------------|
-| Make sure to update any unit tests and end-to-end tests in line with your changes to the pipelines |
+| Make sure to update any unit tests in line with your changes to the pipelines |
 
 | :exclamation: IMPORTANT    |
 |:---------------------------|

pipelines/src/pipelines/utils/trigger_pipeline.py

@@ -20,12 +20,14 @@
 def trigger_pipeline(
     template_path: str,
     display_name: str,
+    wait: bool = False,
 ) -> aiplatform.PipelineJob:
     """Trigger a Vertex Pipeline run from a (local) compiled pipeline definition.
 
     Args:
         template_path (str): file path to the compiled YAML pipeline
         display_name (str): Display name to use for the PipelineJob
+        wait (bool): Wait for the pipeline to finish running
 
     Returns:
         aiplatform.PipelineJob: the Vertex PipelineJob object
@@ -67,6 +69,10 @@ def trigger_pipeline(
         network=network,
     )
 
+    if wait:
+        # Wait for pipeline to finish running before returning
+        pl.wait()
+
     return pl
 
 
@@ -84,10 +90,22 @@ def trigger_pipeline(
         type=str,
     )
 
+    parser.add_argument(
+        "--wait",
+        help="Wait for the pipeline to finish running",
+        type=str,
+    )
     # Get commandline args
     args = parser.parse_args()
 
+    if args.wait.lower() == "true":
+        wait = True
+    elif args.wait.lower() != "false":
+        raise ValueError("wait variable must be 'true' or 'false'")
+    wait = False
+
     trigger_pipeline(
         template_path=args.template_path,
         display_name=args.display_name,
+        wait=wait,
     )