Simplify component imports

.pre-commit-config.yaml

@@ -21,7 +21,6 @@ repos:
     rev: v0.0.9
     hooks:
       - id: terraform-fmt
-      # - id: shellcheck
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v1.2.3
@@ -43,7 +42,6 @@ repos:
     hooks:
       - id: yamllint
         args: [--format, parsable, --strict]
-        exclude: .*component.yaml
 
   - repo: https://github.com/psf/black
     rev: 22.3.0

.yamllint

@@ -14,34 +14,8 @@
 
 ---
 
-yaml-files:
-  - '*.yaml'
-  - '*.yml'
-  - .yamllint
+extends: default
 
 rules:
-  braces: enable
-  brackets: enable
-  colons: enable
-  commas: enable
-  comments:
-    level: warning
-  comments-indentation:
-    level: warning
-  document-end: disable
-  document-start:
-    level: warning
-  empty-lines: enable
-  empty-values: disable
-  hyphens: enable
-  indentation: enable
-  key-duplicates: enable
-  key-ordering: disable
-  line-length: disable
-  new-line-at-end-of-file: enable
-  new-lines: enable
-  octal-values: disable
-  quoted-strings: disable
-  trailing-spaces: enable
-  truthy:
-    level: warning
+  line-length:
+    max: 120

Makefile

@@ -29,36 +29,31 @@ setup: ## Set up local environment for Python development on pipelines
 
 test-trigger: ## Runs unit tests for the pipeline trigger code
 	@cd pipelines && \
-	pipenv install --dev && \
 	pipenv run python -m pytest tests/trigger
 
 compile-pipeline: ## Compile the pipeline to training.json or prediction.json. Must specify pipeline=<training|prediction>
-	@cd pipelines && \
+	@cd pipelines/src && \
 	pipenv run python -m pipelines.${PIPELINE_TEMPLATE}.${pipeline}.pipeline
 
-compile-components: ## Compile all the components in a component group
-	@cd pipeline_components/${GROUP} && \
-	pipenv install && \
-	for component in ${GROUP}/*/component.py ; do \
-		pipenv run python $$component ; \
-	done
+setup-components: ## Run unit tests for a component group
+	@cd "components/${GROUP}" && \
+	pipenv install --dev
 
-compile-all-components: ## Compile all pipeline components
+setup-all-components: ## Run unit tests for all pipeline components
 	@set -e && \
-	for component_group in pipeline_components/*/ ; do \
-		echo "Compiling components under $$component_group" && \
-		$(MAKE) compile-components GROUP=$$(basename $$component_group) ; \
+	for component_group in components/*/ ; do \
+		echo "Setup components under $$component_group" && \
+		$(MAKE) setup-components GROUP=$$(basename $$component_group) ; \
 	done
 
 test-components: ## Run unit tests for a component group
-	@cd pipeline_components/${GROUP} && \
-	pipenv install --dev && \
+	@cd "components/${GROUP}" && \
 	pipenv run pytest
 
 test-all-components: ## Run unit tests for all pipeline components
 	@set -e && \
-	for component_group in pipeline_components/*/ ; do \
-		echo "Running unit tests for components under $$component_group" && \
+	for component_group in components/*/ ; do \
+		echo "Test components under $$component_group" && \
 		$(MAKE) test-components GROUP=$$(basename $$component_group) ; \
 	done
 
@@ -70,14 +65,12 @@ sync-assets: ## Sync assets folder to GCS. Must specify pipeline=<training|predi
 run: ## Compile pipeline, copy assets to GCS, and run pipeline in sandbox environment. Must specify pipeline=<training|prediction>. Optionally specify enable_pipeline_caching=<true|false> (defaults to default Vertex caching behaviour)
 	@ $(MAKE) compile-pipeline && \
 	$(MAKE) sync-assets && \
-	cd pipelines && \
-	pipenv run python -m trigger.main --template_path=./$(pipeline).json --enable_caching=$(enable_pipeline_caching)
+	cd pipelines/src && \
+	pipenv run python -m pipelines.trigger --template_path=./$(pipeline).json --enable_caching=$(enable_pipeline_caching)
 
 e2e-tests: ## Compile pipeline, copy assets to GCS, and perform end-to-end (E2E) pipeline tests. Must specify pipeline=<training|prediction>. Optionally specify enable_pipeline_caching=<true|false> (defaults to default Vertex caching behaviour)
-	@ $(MAKE) compile-pipeline && \
-	$(MAKE) sync-assets && \
 	cd pipelines && \
-	pipenv run python -m pytest --log-cli-level=INFO tests/${PIPELINE_TEMPLATE}/$(pipeline) --enable_caching=$(enable_pipeline_caching)
+	pipenv run pytest --log-cli-level=INFO tests/${PIPELINE_TEMPLATE}/$(pipeline) --enable_caching=$(enable_pipeline_caching)
 
 env ?= dev
 deploy-infra: ## Deploy the Terraform infrastructure to your project. Requires VERTEX_PROJECT_ID and VERTEX_LOCATION env variables to be set in env.sh. Optionally specify env=<dev|test|prod> (default = dev)

cloudbuild/e2e-test.yaml

@@ -24,27 +24,20 @@ steps:
       - |
         mkdir -p ${COMMIT_SHA}/training/assets && \
         mkdir -p ${COMMIT_SHA}/prediction/assets && \
-        cp -r pipelines/pipelines/${_PIPELINE_TEMPLATE}/training/assets ${COMMIT_SHA}/training/ && \
-        cp -r pipelines/pipelines/${_PIPELINE_TEMPLATE}/prediction/assets ${COMMIT_SHA}/prediction/ && \
+        cp -r pipelines/src/pipelines/${_PIPELINE_TEMPLATE}/training/assets ${COMMIT_SHA}/training/ && \
+        cp -r pipelines/src/pipelines/${_PIPELINE_TEMPLATE}/prediction/assets ${COMMIT_SHA}/prediction/ && \
         gsutil cp -r ${COMMIT_SHA} ${_PIPELINE_PUBLISH_GCS_PATH}/${COMMIT_SHA}
 
   # Install Python deps
   # Run end-to-end (E2E) pipeline tests on both pipelines
   - name: python:3.7
     entrypoint: /bin/sh
-    dir: pipelines
     args:
       - -c
       - |
-        pip install pipenv && \
-        pipenv install --dev && \
-        cd ../ && \
-        make compile-all-components && \
-        make compile-pipeline pipeline=training && \
-        make compile-pipeline pipeline=prediction && \
-        cd pipelines && \
-        pipenv run python -m pytest --log-cli-level=INFO tests/${_PIPELINE_TEMPLATE}/training --enable_caching=${_TEST_ENABLE_PIPELINE_CACHING} && \
-        pipenv run python -m pytest --log-cli-level=INFO tests/${_PIPELINE_TEMPLATE}/prediction --enable_caching=${_TEST_ENABLE_PIPELINE_CACHING}
+        make setup && \
+        make e2e-tests pipeline=training enable_pipeline_caching=False && \
+        make e2e-tests pipeline=prediction enable_pipeline_caching=False
     env:
       - VERTEX_LOCATION=${_TEST_VERTEX_LOCATION}
       - VERTEX_PROJECT_ID=${_TEST_VERTEX_PROJECT_ID}

cloudbuild/pr-checks.yaml

@@ -26,10 +26,10 @@ steps:
         git init && \
         git add . && \
         make pre-commit && \
-        make compile-all-components && \
-        make test-all-components && \
         make compile-pipeline pipeline=training && \
-        make compile-pipeline pipeline=prediction
+        make compile-pipeline pipeline=prediction && \
+        make setup-all-components && \
+        make test-all-components
     env:
       - SKIP=terraform-fmt,git-dirty
       - PIPELINE_TEMPLATE=${_PIPELINE_TEMPLATE}

cloudbuild/trigger-tests.yaml

@@ -17,7 +17,7 @@ steps:
     args:
       - '-c'
       - |
-        pip install pipenv && \
+        make setup && \
         make test-trigger
     entrypoint: /bin/sh
 

pipeline_components/aiplatform/Pipfile → components/aiplatform-components/Pipfile

@@ -4,12 +4,12 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-kfp = "==1.8.9"
-google-cloud-pipeline-components = ">=1.0.0,<2.0.0"
-google-cloud-aiplatform = ">=1.24.1,<2.0.0"
+kfp = "==1.8.21"
 
 [dev-packages]
-pytest = ">=6.2.4,<7.0.0"
+google-cloud-aiplatform = ">= 1.24.1,<2.0.0"
+google-cloud-pipeline-components = ">= 1.0.0,<2.0.0"
+pytest = ">=7.3.1,<8.0.0"
 pre-commit = ">=2.14.1,<3.0.0"
 
 [requires]