feat(testing): use kustomize to patch deployments before deploy (#11294)

* Use kustomize to patch deployments before deploy Signed-off-by: carter.fendley <[email protected]> * Relocate scripts / manifests to github directory Signed-off-by: carter.fendley <[email protected]> --------- Signed-off-by: carter.fendley <[email protected]>
kubeflow · Oct 17, 2024 · be863a8 · be863a8
1 parent aec2856
commit be863a8
Show file tree

Hide file tree

Showing 29 changed files with 66 additions and 49 deletions.
diff --git a/.github/actions/kfp-cluster/action.yml b/.github/actions/kfp-cluster/action.yml
@@ -14,8 +14,8 @@ runs:
 
     - name: Build images
       shell: bash
-      run: ./scripts/deploy/github/build-images.sh
+      run: ./.github/resources/scripts/build-images.sh
 
     - name: Deploy KFP
       shell: bash
-      run: ./scripts/deploy/github/deploy-kfp.sh
+      run: ./.github/resources/scripts/deploy-kfp.sh
diff --git a/.github/actions/kfp-tekton-cluster/action.yml b/.github/actions/kfp-tekton-cluster/action.yml
@@ -14,8 +14,8 @@ runs:
 
     - name: Build images
       shell: bash
-      run: ./scripts/deploy/github/build-images.sh
+      run: ./.github/resources/scripts/build-images.sh
 
     - name: Deploy KFP
       shell: bash
-      run: ./scripts/deploy/github/deploy-kfp-tekton.sh
+      run: ./.github/resources/scripts/deploy-kfp-tekton.sh
diff --git a/.github/resources/manifests/argo/kustomization.yaml b/.github/resources/manifests/argo/kustomization.yaml
@@ -0,0 +1,19 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- ../../../../manifests/kustomize/env/platform-agnostic
+
+images:
+- name: gcr.io/ml-pipeline/api-server
+  newName: kind-registry:5000/apiserver
+  newTag: latest
+- name: gcr.io/ml-pipeline/persistenceagent
+  newName: kind-registry:5000/persistenceagent
+  newTag: latest
+- name: gcr.io/ml-pipeline/scheduledworkflow
+  newName: kind-registry:5000/scheduledworkflow
+  newTag: latest
+
+patchesStrategicMerge:
+- overlays/apiserver-env.yaml
diff --git a/.github/resources/manifests/argo/overlays/apiserver-env.yaml b/.github/resources/manifests/argo/overlays/apiserver-env.yaml
@@ -0,0 +1,14 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ml-pipeline
+spec:
+  template:
+    spec:
+      containers:
+        - name: ml-pipeline-api-server
+          env:
+            - name: V2_DRIVER_IMAGE
+              value: kind-registry:5000/driver
+            - name: V2_LAUNCHER_IMAGE
+              value: kind-registry:5000/launcher
diff --git a/...eploy/github/manifests/kustomization.yaml → ...urces/manifests/tekton/kustomization.yaml b/...eploy/github/manifests/kustomization.yaml → ...urces/manifests/tekton/kustomization.yaml
diff --git a/scripts/deploy/github/build-images.sh → .github/resources/scripts/build-images.sh b/scripts/deploy/github/build-images.sh → .github/resources/scripts/build-images.sh
diff --git a/scripts/deploy/github/deploy-kfp-tekton.sh → ...ub/resources/scripts/deploy-kfp-tekton.sh b/scripts/deploy/github/deploy-kfp-tekton.sh → ...ub/resources/scripts/deploy-kfp-tekton.sh
@@ -33,7 +33,8 @@ then
 fi
 
 # Deploy manifest
-kubectl apply -k "scripts/deploy/github/manifests" || EXIT_CODE=$?
+TEST_MANIFESTS=".github/resources/manifests/tekton"
+kubectl apply -k "${TEST_MANIFESTS}" || EXIT_CODE=$?
 if [[ $EXIT_CODE -ne 0 ]]
 then
   echo "Deploy unsuccessful. Failure applying $KUSTOMIZE_DIR."

diff --git a/scripts/deploy/github/deploy-kfp.sh → .github/resources/scripts/deploy-kfp.sh b/scripts/deploy/github/deploy-kfp.sh → .github/resources/scripts/deploy-kfp.sh
@@ -32,27 +32,15 @@ then
   exit $EXIT_CODE
 fi
 
-PLATFORM_AGNOSTIC_MANIFESTS="manifests/kustomize/env/platform-agnostic"
-
-kubectl apply -k "${PLATFORM_AGNOSTIC_MANIFESTS}" || EXIT_CODE=$?
+# Deploy manifest
+TEST_MANIFESTS=".github/resources/manifests/argo"
+kubectl apply -k "${TEST_MANIFESTS}" || EXIT_CODE=$?
 if [[ $EXIT_CODE -ne 0 ]]
 then
-  echo "Deploy unsuccessful. Failure applying ${PLATFORM_AGNOSTIC_MANIFESTS}."
+  echo "Deploy unsuccessful. Failure applying ${TEST_MANIFESTS}."
   exit 1
 fi
 
-echo "Patching deployments to use built docker images..."
-# Patch API server
-kubectl patch deployment ml-pipeline -p '{"spec": {"template": {"spec": {"containers": [{"name": "ml-pipeline-api-server", "image": "kind-registry:5000/apiserver"}]}}}}' -n kubeflow
-# Patch persistence agent
-kubectl patch deployment.apps/ml-pipeline-persistenceagent -p '{"spec": {"template": {"spec": {"containers": [{"name": "ml-pipeline-persistenceagent", "image": "kind-registry:5000/persistenceagent"}]}}}}' -n kubeflow
-# Patch scheduled workflow
-kubectl patch deployment.apps/ml-pipeline-scheduledworkflow -p '{"spec": {"template": {"spec": {"containers": [{"name": "ml-pipeline-scheduledworkflow", "image": "kind-registry:5000/scheduledworkflow"}]}}}}' -n kubeflow
-
-# Update environment variables to override driver / launcher
-kubectl set env deployments/ml-pipeline V2_DRIVER_IMAGE=kind-registry:5000/driver -n kubeflow
-kubectl set env deployments/ml-pipeline V2_LAUNCHER_IMAGE=kind-registry:5000/launcher -n kubeflow
-
 # Check if all pods are running - (10 minutes)
 wait_for_pods || EXIT_CODE=$?
 if [[ $EXIT_CODE -ne 0 ]]

diff --git a/scripts/deploy/github/e2e-test.sh → .github/resources/scripts/e2e-test.sh b/scripts/deploy/github/e2e-test.sh → .github/resources/scripts/e2e-test.sh
diff --git a/scripts/deploy/github/forward-port.sh → .github/resources/scripts/forward-port.sh b/scripts/deploy/github/forward-port.sh → .github/resources/scripts/forward-port.sh
diff --git a/scripts/deploy/github/helper-functions.sh → ...hub/resources/scripts/helper-functions.sh b/scripts/deploy/github/helper-functions.sh → ...hub/resources/scripts/helper-functions.sh
diff --git a/...loy/github/kfp-readiness/wait_for_pods.py → ...es/scripts/kfp-readiness/wait_for_pods.py b/...loy/github/kfp-readiness/wait_for_pods.py → ...es/scripts/kfp-readiness/wait_for_pods.py
diff --git a/scripts/deploy/github/re-tag.sh → .github/resources/scripts/re-tag.sh b/scripts/deploy/github/re-tag.sh → .github/resources/scripts/re-tag.sh
diff --git a/scripts/deploy/github/test-dynamic-loop.sh → ...ub/resources/scripts/test-dynamic-loop.sh b/scripts/deploy/github/test-dynamic-loop.sh → ...ub/resources/scripts/test-dynamic-loop.sh
diff --git a/scripts/deploy/github/test-env.sh → .github/resources/scripts/test-env.sh b/scripts/deploy/github/test-env.sh → .github/resources/scripts/test-env.sh
diff --git a/scripts/deploy/github/test-flip-coin.sh → .github/resources/scripts/test-flip-coin.sh b/scripts/deploy/github/test-flip-coin.sh → .github/resources/scripts/test-flip-coin.sh
diff --git a/scripts/deploy/github/test-pipeline.sh → .github/resources/scripts/test-pipeline.sh b/scripts/deploy/github/test-pipeline.sh → .github/resources/scripts/test-pipeline.sh
diff --git a/scripts/deploy/github/test-secret-as-env.sh → ...b/resources/scripts/test-secret-as-env.sh b/scripts/deploy/github/test-secret-as-env.sh → ...b/resources/scripts/test-secret-as-env.sh
diff --git a/...ts/deploy/github/test-secret-as-volume.sh → ...esources/scripts/test-secret-as-volume.sh b/...ts/deploy/github/test-secret-as-volume.sh → ...esources/scripts/test-secret-as-volume.sh
diff --git a/scripts/deploy/github/test-static-loop.sh → ...hub/resources/scripts/test-static-loop.sh b/scripts/deploy/github/test-static-loop.sh → ...hub/resources/scripts/test-static-loop.sh
diff --git a/scripts/deploy/github/test-volume.sh → .github/resources/scripts/test-volume.sh b/scripts/deploy/github/test-volume.sh → .github/resources/scripts/test-volume.sh
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
@@ -10,7 +10,7 @@ on:
       - '.github/workflows/backend.yml'
       - 'go.mod'
       - 'backend/**'
-      - 'scripts/deploy/github/**'
+      - '.github/resources/**'
       - 'manifests/kustomize/**'
 
 env:
@@ -48,23 +48,23 @@ jobs:
       - name: "flip coin test"
         run: |
           . .venv/bin/activate
-          TEST_SCRIPT="test-flip-coin.sh" ./scripts/deploy/github/e2e-test.sh
+          TEST_SCRIPT="test-flip-coin.sh" ./.github/resources/scripts/e2e-test.sh
       - name: "static loop test"
         run: |
           . .venv/bin/activate
-          TEST_SCRIPT="test-static-loop.sh" ./scripts/deploy/github/e2e-test.sh
+          TEST_SCRIPT="test-static-loop.sh" ./.github/resources/scripts/e2e-test.sh
       - name: "dynamic loop test"
         run: |
           . .venv/bin/activate
-          TEST_SCRIPT="test-dynamic-loop.sh" ./scripts/deploy/github/e2e-test.sh
+          TEST_SCRIPT="test-dynamic-loop.sh" ./.github/resources/scripts/e2e-test.sh
       - name: "use env"
         run: |
           . .venv/bin/activate
-          TEST_SCRIPT="test-env.sh" ./scripts/deploy/github/e2e-test.sh
+          TEST_SCRIPT="test-env.sh" ./.github/resources/scripts/e2e-test.sh
       - name: "use volume"
         run: |
           . .venv/bin/activate
-          TEST_SCRIPT="test-volume.sh" ./scripts/deploy/github/e2e-test.sh
+          TEST_SCRIPT="test-volume.sh" ./.github/resources/scripts/e2e-test.sh
       - name: Collect test results
         if: always()
         uses: actions/upload-artifact@v4

diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml
@@ -7,7 +7,7 @@ on:
   pull_request:
     paths:
       - '.github/workflows/e2e-test.yml'
-      - 'scripts/deploy/github/**'
+      - '.github/resources/**'
       - 'go.mod'
       - 'go.sum'
       - 'backend/**'
@@ -32,7 +32,7 @@ jobs:
         uses: ./.github/actions/kfp-cluster
 
       - name: Forward API port
-        run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
+        run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
 
       - name: Initialization tests v1
         working-directory: ./backend/test/initialization
@@ -60,7 +60,7 @@ jobs:
         uses: ./.github/actions/kfp-cluster
 
       - name: Forward API port
-        run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
+        run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
 
       - name: Initialization tests v2
         working-directory: ./backend/test/v2/initialization
@@ -88,7 +88,7 @@ jobs:
         uses: ./.github/actions/kfp-cluster
 
       - name: Forward API port
-        run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
+        run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
 
       - name: API integration tests v1
         working-directory: ./backend/test/integration
@@ -116,7 +116,7 @@ jobs:
         uses: ./.github/actions/kfp-cluster
 
       - name: Forward API port
-        run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
+        run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
 
       - name: API integration tests v2
         working-directory: ./backend/test/v2/integration
@@ -144,10 +144,10 @@ jobs:
         uses: ./.github/actions/kfp-cluster
 
       - name: Forward API port
-        run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
+        run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
 
       - name: Forward Frontend port
-        run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline-ui" 3000 3000
+        run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline-ui" 3000 3000
 
       - name: Build frontend integration tests image
         working-directory: ./test/frontend-integration-test
@@ -178,7 +178,7 @@ jobs:
         uses: ./.github/actions/kfp-cluster
 
       - name: Forward API port
-        run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
+        run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
 
       - name: Install prerequisites
         run: pip3 install -r ./test/sample-test/requirements.txt

diff --git a/.github/workflows/kfp-kubernetes-execution-tests.yml b/.github/workflows/kfp-kubernetes-execution-tests.yml
@@ -7,7 +7,7 @@ on:
   pull_request:
     paths:
       - '.github/workflows/kfp-kubernetes-execution-tests.yml'
-      - 'scripts/deploy/github/**'
+      - '.github/resources/**'
       - 'sdk/python/**'
       - 'api/v2alpha1/**'
       - 'kubernetes_platform/**'
@@ -28,7 +28,7 @@ jobs:
         uses: ./.github/actions/kfp-cluster
 
       - name: Forward API port
-        run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
+        run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
 
       - name: apt-get update
         run: sudo apt-get update

diff --git a/.github/workflows/kfp-samples.yml b/.github/workflows/kfp-samples.yml
@@ -6,7 +6,7 @@ on:
       - master
   pull_request:
     paths:
-      - 'scripts/deploy/github/**'
+      - '.github/resources/**'
       - 'samples/**'
       - 'backend/src/v2/**'
       - '.github/workflows/kfp-samples.yml'
@@ -28,7 +28,7 @@ jobs:
       uses: ./.github/actions/kfp-cluster
 
     - name: Forward API port
-      run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
+      run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
 
     - name: Run Samples Tests
       run: |

diff --git a/.github/workflows/kubeflow-pipelines-integration-v2.yml b/.github/workflows/kubeflow-pipelines-integration-v2.yml
@@ -7,7 +7,7 @@ on:
   pull_request:
     paths:
       - '.github/workflows/kubeflow-pipelines-integration-v2.yml'
-      - 'scripts/deploy/github/**'
+      - '.github/resources/**'
       - 'samples/core/parameterized_tfx_oss/**'
       - 'samples/core/dataflow/**'
       - 'backend/**'
@@ -32,7 +32,7 @@ jobs:
         uses: ./.github/actions/kfp-cluster
 
       - name: Forward API port
-        run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
+        run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
 
       - name: Run the Integration Tests
         run: |

diff --git a/.github/workflows/sdk-execution.yml b/.github/workflows/sdk-execution.yml
@@ -7,7 +7,7 @@ on:
   pull_request:
     paths:
       - '.github/workflows/sdk-execution.yml'
-      - 'scripts/deploy/github/**'
+      - '.github/resources/**'
       - 'sdk/python/**'
       - 'api/v2alpha1/**'
 
@@ -27,7 +27,7 @@ jobs:
         uses: ./.github/actions/kfp-cluster
 
       - name: Forward API port
-        run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
+        run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888
 
       - name: apt-get update
         run: sudo apt-get update

diff --git a/.github/workflows/upgrade-test.yml b/.github/workflows/upgrade-test.yml
@@ -7,7 +7,7 @@ on:
   pull_request:
     paths:
       - '.github/workflows/upgrade-test.yml'
-      - 'scripts/deploy/github/**'
+      - '.github/resources/**'
       - 'backend/**'
       - 'manifests/kustomize/**'
 

diff --git a/scripts/OWNERS b/scripts/OWNERS