Update e2e tests (#2161)

* Add sleep buffer to ensture the webhooks are ready before running the e2e tests Signed-off-by: Yi Chen <[email protected]> * Remove duplicate operator image build tasks Signed-off-by: Yi Chen <[email protected]> * Update e2e tests Signed-off-by: Yi Chen <[email protected]> * Update examples Signed-off-by: Yi Chen <[email protected]> --------- Signed-off-by: Yi Chen <[email protected]>
kubeflow · Sep 13, 2024 · e6a7805 · e6a7805
1 parent e58023b
commit e6a7805
Show file tree

Hide file tree

Showing 13 changed files with 66 additions and 86 deletions.
diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml
@@ -72,23 +72,8 @@ jobs:
       - name: Run unit tests
         run: make unit-test
 
-      - name: Build Spark-Operator Docker Image
-        run: make docker-build IMAGE_TAG=latest
-
-      - name: Check changes in resources used in docker file
-        run: |
-          DOCKERFILE_RESOURCES=$(cat Dockerfile | grep -P -o "COPY [a-zA-Z0-9].*? " | cut -c6-)
-          for resource in $DOCKERFILE_RESOURCES; do
-            # If the resource is different
-            if ! git diff  --quiet origin/master -- $resource; then
-              ## And the appVersion hasn't been updated
-              if ! git diff origin/master -- charts/spark-operator-chart/Chart.yaml | grep +appVersion; then
-                echo "resource used in docker.io/kubeflow/spark-operator has changed in $resource, need to update the appVersion in charts/spark-operator-chart/Chart.yaml"
-                git diff origin/master -- $resource;
-                echo "failing the build... " && false
-              fi
-            fi
-          done
+      - name: Build Spark operator
+        run: make build-operator
 
   build-helm-chart:
     runs-on: ubuntu-latest
@@ -131,7 +116,7 @@ jobs:
       - name: Run chart-testing (lint)
         if: steps.list-changed.outputs.changed == 'true'
         env:
-            BRANCH: ${{ steps.get_branch.outputs.BRANCH }}
+          BRANCH: ${{ steps.get_branch.outputs.BRANCH }}
         run: ct lint --check-version-increment=false --target-branch $BRANCH
 
       - name: Detect CRDs drift between chart and manifest

diff --git a/examples/spark-pi-configmap.yaml b/examples/spark-pi-configmap.yaml
@@ -33,18 +33,13 @@ spec:
     configMap:
       name: test-configmap
   driver:
-    labels:
-      version: 3.5.2
     cores: 1
-    coreLimit: 1200m
     memory: 512m
-    serviceAccount: spark-operator-spark
     volumeMounts:
     - name: config-vol
       mountPath: /opt/spark/config
+    serviceAccount: spark-operator-spark
   executor:
-    labels:
-      version: 3.5.2
     instances: 1
     cores: 1
     memory: 512m

diff --git a/examples/spark-pi-custom-resource.yaml b/examples/spark-pi-custom-resource.yaml
@@ -28,27 +28,13 @@ spec:
   sparkVersion: 3.5.2
   restartPolicy:
     type: Never
-  volumes:
-  - name: test-volume
-    hostPath:
-      path: /tmp
-      type: Directory
   driver:
-    labels:
-      version: 3.5.2
-    cores: 1
-    coreLimit: 1200m
+    coreRequest: "0.5"
+    coreLimit: 800m
     memory: 512m
     serviceAccount: spark-operator-spark
-    volumeMounts:
-    - name: test-volume
-      mountPath: /tmp
   executor:
-    labels:
-      version: 3.5.2
     instances: 1
-    cores: 1
+    coreRequest: "1200m"
+    coreLimit: 1500m
     memory: 512m
-    volumeMounts:
-    - name: test-volume
-      mountPath: /tmp
diff --git a/examples/spark-pi-dynamic-allocation.yaml b/examples/spark-pi-dynamic-allocation.yaml
@@ -26,21 +26,13 @@ spec:
   mainClass: org.apache.spark.examples.SparkPi
   mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.12-3.5.2.jar
   sparkVersion: 3.5.2
-  arguments:
-  - "50000"
   driver:
-    labels:
-      version: 3.5.2
     cores: 1
-    coreLimit: 1200m
     memory: 512m
     serviceAccount: spark-operator-spark
   executor:
-    labels:
-      version: 3.5.2
     instances: 1
     cores: 1
-    coreLimit: 1200m
     memory: 512m
   dynamicAllocation:
     enabled: true

diff --git a/examples/spark-pi-kube-scheduler.yaml b/examples/spark-pi-kube-scheduler.yaml
@@ -27,17 +27,11 @@ spec:
   mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.12-3.5.2.jar
   sparkVersion: 3.5.2
   driver:
-    labels:
-      version: 3.5.2
     cores: 1
-    coreLimit: 1200m
     memory: 512m
     serviceAccount: spark-operator-spark
   executor:
-    labels:
-      version: 3.5.2
     instances: 2
     cores: 1
-    coreLimit: 1200m
     memory: 512m
   batchScheduler: kube-scheduler
diff --git a/examples/spark-pi-prometheus.yaml b/examples/spark-pi-prometheus.yaml
@@ -33,7 +33,6 @@ spec:
     type: Never
   driver:
     cores: 1
-    coreLimit: 1200m
     memory: 512m
     labels:
       version: 3.1.1

diff --git a/examples/spark-pi-python.yaml b/examples/spark-pi-python.yaml
@@ -27,16 +27,10 @@ spec:
   mainApplicationFile: local:///opt/spark/examples/src/main/python/pi.py
   sparkVersion: 3.5.2
   driver:
-    labels:
-      version: 3.5.2
     cores: 1
-    coreLimit: 1200m
     memory: 512m
     serviceAccount: spark-operator-spark
   executor:
-    labels:
-      version: 3.5.2
     instances: 1
     cores: 1
-    coreLimit: 1200m
     memory: 512m
diff --git a/examples/spark-pi-scheduled.yaml b/examples/spark-pi-scheduled.yaml
@@ -33,16 +33,10 @@ spec:
     restartPolicy:
       type: Never
     driver:
-      labels:
-        version: 3.5.2
       cores: 1
-      coreLimit: 1200m
       memory: 512m
       serviceAccount: spark-operator-spark
     executor:
-      labels:
-        version: 3.5.2
       instances: 1
       cores: 1
-      coreLimit: 1200m
       memory: 512m
diff --git a/examples/spark-pi-volcano.yaml b/examples/spark-pi-volcano.yaml
@@ -27,17 +27,11 @@ spec:
   mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.12-3.5.2.jar
   sparkVersion: 3.5.2
   driver:
-    labels:
-      version: 3.5.2
     cores: 1
-    coreLimit: 1200m
     memory: 512m
     serviceAccount: spark-operator-spark
   executor:
-    labels:
-      version: 3.5.2
     instances: 2
     cores: 1
-    coreLimit: 1200m
     memory: 512m
   batchScheduler: volcano
diff --git a/examples/spark-pi-yunikorn.yaml b/examples/spark-pi-yunikorn.yaml
@@ -27,18 +27,12 @@ spec:
   mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.12-3.5.2.jar
   sparkVersion: 3.5.2
   driver:
-    labels:
-      version: 3.5.2
     cores: 1
-    coreLimit: 1200m
     memory: 512m
     serviceAccount: spark-operator-spark
   executor:
-    labels:
-      version: 3.5.2
     instances: 2
     cores: 1
-    coreLimit: 1200m
     memory: 512m
   batchScheduler: yunikorn
   batchSchedulerOptions:

diff --git a/examples/spark-pi.yaml b/examples/spark-pi.yaml
@@ -25,18 +25,18 @@ spec:
   imagePullPolicy: IfNotPresent
   mainClass: org.apache.spark.examples.SparkPi
   mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.12-3.5.2.jar
+  arguments:
+  - "5000"
   sparkVersion: 3.5.2
   driver:
     labels:
       version: 3.5.2
     cores: 1
-    coreLimit: 1200m
     memory: 512m
     serviceAccount: spark-operator-spark
   executor:
     labels:
       version: 3.5.2
     instances: 1
     cores: 1
-    coreLimit: 1200m
     memory: 512m
diff --git a/test/e2e/sparkapplication_test.go b/test/e2e/sparkapplication_test.go
@@ -26,11 +26,14 @@ import (
 	. "github.com/onsi/gomega"
 
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/equality"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/yaml"
 
 	"github.com/kubeflow/spark-operator/api/v1beta2"
+	"github.com/kubeflow/spark-operator/pkg/common"
 	"github.com/kubeflow/spark-operator/pkg/util"
 )
 
@@ -130,13 +133,43 @@ var _ = Describe("Example SparkApplication", func() {
 			}
 		})
 
-		It("Should complete successfully", func() {
+		It("Should complete successfully with configmap mounted", func() {
 			By("Waiting for SparkApplication to complete")
 			key := types.NamespacedName{Namespace: app.Namespace, Name: app.Name}
 			Expect(waitForSparkApplicationCompleted(ctx, key)).NotTo(HaveOccurred())
 
-			By("Checking out driver logs")
+			By("Checking out whether volumes are mounted to driver pod")
 			driverPodName := util.GetDriverPodName(app)
+			driverPodKey := types.NamespacedName{Namespace: app.Namespace, Name: driverPodName}
+			driverPod := &corev1.Pod{}
+			Expect(k8sClient.Get(ctx, driverPodKey, driverPod)).NotTo(HaveOccurred())
+			hasVolumes := false
+			hasVolumeMounts := false
+			for _, volume := range app.Spec.Volumes {
+				for _, podVolume := range driverPod.Spec.Volumes {
+					if volume.Name == podVolume.Name {
+						hasVolumes = true
+						break
+					}
+				}
+			}
+			for _, volumeMount := range app.Spec.Driver.VolumeMounts {
+				for _, container := range driverPod.Spec.Containers {
+					if container.Name != common.SparkDriverContainerName {
+						continue
+					}
+					for _, podVolumeMount := range container.VolumeMounts {
+						if equality.Semantic.DeepEqual(volumeMount, podVolumeMount) {
+							hasVolumeMounts = true
+							break
+						}
+					}
+				}
+			}
+			Expect(hasVolumes).To(BeTrue())
+			Expect(hasVolumeMounts).To(BeTrue())
+
+			By("Checking out driver logs")
 			bytes, err := clientset.CoreV1().Pods(app.Namespace).GetLogs(driverPodName, &corev1.PodLogOptions{}).Do(ctx).Raw()
 			Expect(err).NotTo(HaveOccurred())
 			Expect(bytes).NotTo(BeEmpty())
@@ -176,8 +209,26 @@ var _ = Describe("Example SparkApplication", func() {
 			key := types.NamespacedName{Namespace: app.Namespace, Name: app.Name}
 			Expect(waitForSparkApplicationCompleted(ctx, key)).NotTo(HaveOccurred())
 
-			By("Checking out driver logs")
+			By("Checking out whether resource requests and limits of driver pod are set")
 			driverPodName := util.GetDriverPodName(app)
+			driverPodKey := types.NamespacedName{Namespace: app.Namespace, Name: driverPodName}
+			driverPod := &corev1.Pod{}
+			Expect(k8sClient.Get(ctx, driverPodKey, driverPod)).NotTo(HaveOccurred())
+			for _, container := range driverPod.Spec.Containers {
+				if container.Name != common.SparkDriverContainerName {
+					continue
+				}
+				if app.Spec.Driver.CoreRequest != nil {
+					Expect(container.Resources.Requests.Cpu().Equal(resource.MustParse(*app.Spec.Driver.CoreRequest))).To(BeTrue())
+				}
+				if app.Spec.Driver.CoreLimit != nil {
+					Expect(container.Resources.Limits.Cpu().Equal(resource.MustParse(*app.Spec.Driver.CoreLimit))).To(BeTrue())
+				}
+				Expect(container.Resources.Requests.Memory).NotTo(BeNil())
+				Expect(container.Resources.Limits.Memory).NotTo(BeNil())
+			}
+
+			By("Checking out driver logs")
 			bytes, err := clientset.CoreV1().Pods(app.Namespace).GetLogs(driverPodName, &corev1.PodLogOptions{}).Do(ctx).Raw()
 			Expect(err).NotTo(HaveOccurred())
 			Expect(bytes).NotTo(BeEmpty())

diff --git a/test/e2e/suit_test.go b/test/e2e/suit_test.go
@@ -149,6 +149,8 @@ var _ = BeforeSuite(func() {
 	validatingWebhookKey := types.NamespacedName{Name: ValidatingWebhookName}
 	Expect(waitForMutatingWebhookReady(context.Background(), mutatingWebhookKey)).NotTo(HaveOccurred())
 	Expect(waitForValidatingWebhookReady(context.Background(), validatingWebhookKey)).NotTo(HaveOccurred())
+	// TODO: Remove this when there is a better way to ensure the webhooks are ready before running the e2e tests.
+	time.Sleep(10 * time.Second)
 })
 
 var _ = AfterSuite(func() {