diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index b37425701..663404025 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -72,23 +72,8 @@ jobs: - name: Run unit tests run: make unit-test - - name: Build Spark-Operator Docker Image - run: make docker-build IMAGE_TAG=latest - - - name: Check changes in resources used in docker file - run: | - DOCKERFILE_RESOURCES=$(cat Dockerfile | grep -P -o "COPY [a-zA-Z0-9].*? " | cut -c6-) - for resource in $DOCKERFILE_RESOURCES; do - # If the resource is different - if ! git diff --quiet origin/master -- $resource; then - ## And the appVersion hasn't been updated - if ! git diff origin/master -- charts/spark-operator-chart/Chart.yaml | grep +appVersion; then - echo "resource used in docker.io/kubeflow/spark-operator has changed in $resource, need to update the appVersion in charts/spark-operator-chart/Chart.yaml" - git diff origin/master -- $resource; - echo "failing the build... " && false - fi - fi - done + - name: Build Spark operator + run: make build-operator build-helm-chart: runs-on: ubuntu-latest @@ -131,7 +116,7 @@ jobs: - name: Run chart-testing (lint) if: steps.list-changed.outputs.changed == 'true' env: - BRANCH: ${{ steps.get_branch.outputs.BRANCH }} + BRANCH: ${{ steps.get_branch.outputs.BRANCH }} run: ct lint --check-version-increment=false --target-branch $BRANCH - name: Detect CRDs drift between chart and manifest diff --git a/examples/spark-pi-configmap.yaml b/examples/spark-pi-configmap.yaml index 06f844ddb..ddb812412 100644 --- a/examples/spark-pi-configmap.yaml +++ b/examples/spark-pi-configmap.yaml @@ -33,18 +33,13 @@ spec: configMap: name: test-configmap driver: - labels: - version: 3.5.2 cores: 1 - coreLimit: 1200m memory: 512m - serviceAccount: spark-operator-spark volumeMounts: - name: config-vol mountPath: /opt/spark/config + serviceAccount: spark-operator-spark executor: - labels: - version: 3.5.2 instances: 1 cores: 1 memory: 512m diff --git a/examples/spark-pi-custom-resource.yaml b/examples/spark-pi-custom-resource.yaml index 862148db1..1880cacfe 100644 --- a/examples/spark-pi-custom-resource.yaml +++ b/examples/spark-pi-custom-resource.yaml @@ -28,27 +28,13 @@ spec: sparkVersion: 3.5.2 restartPolicy: type: Never - volumes: - - name: test-volume - hostPath: - path: /tmp - type: Directory driver: - labels: - version: 3.5.2 - cores: 1 - coreLimit: 1200m + coreRequest: "0.5" + coreLimit: 800m memory: 512m serviceAccount: spark-operator-spark - volumeMounts: - - name: test-volume - mountPath: /tmp executor: - labels: - version: 3.5.2 instances: 1 - cores: 1 + coreRequest: "1200m" + coreLimit: 1500m memory: 512m - volumeMounts: - - name: test-volume - mountPath: /tmp diff --git a/examples/spark-pi-dynamic-allocation.yaml b/examples/spark-pi-dynamic-allocation.yaml index 8f98e7220..c2480cb5a 100644 --- a/examples/spark-pi-dynamic-allocation.yaml +++ b/examples/spark-pi-dynamic-allocation.yaml @@ -26,21 +26,13 @@ spec: mainClass: org.apache.spark.examples.SparkPi mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.12-3.5.2.jar sparkVersion: 3.5.2 - arguments: - - "50000" driver: - labels: - version: 3.5.2 cores: 1 - coreLimit: 1200m memory: 512m serviceAccount: spark-operator-spark executor: - labels: - version: 3.5.2 instances: 1 cores: 1 - coreLimit: 1200m memory: 512m dynamicAllocation: enabled: true diff --git a/examples/spark-pi-kube-scheduler.yaml b/examples/spark-pi-kube-scheduler.yaml index 4fe7bf573..c30b6734e 100644 --- a/examples/spark-pi-kube-scheduler.yaml +++ b/examples/spark-pi-kube-scheduler.yaml @@ -27,17 +27,11 @@ spec: mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.12-3.5.2.jar sparkVersion: 3.5.2 driver: - labels: - version: 3.5.2 cores: 1 - coreLimit: 1200m memory: 512m serviceAccount: spark-operator-spark executor: - labels: - version: 3.5.2 instances: 2 cores: 1 - coreLimit: 1200m memory: 512m batchScheduler: kube-scheduler diff --git a/examples/spark-pi-prometheus.yaml b/examples/spark-pi-prometheus.yaml index 29a447061..9e67943cd 100644 --- a/examples/spark-pi-prometheus.yaml +++ b/examples/spark-pi-prometheus.yaml @@ -33,7 +33,6 @@ spec: type: Never driver: cores: 1 - coreLimit: 1200m memory: 512m labels: version: 3.1.1 diff --git a/examples/spark-pi-python.yaml b/examples/spark-pi-python.yaml index bc8c47401..33b659eef 100644 --- a/examples/spark-pi-python.yaml +++ b/examples/spark-pi-python.yaml @@ -27,16 +27,10 @@ spec: mainApplicationFile: local:///opt/spark/examples/src/main/python/pi.py sparkVersion: 3.5.2 driver: - labels: - version: 3.5.2 cores: 1 - coreLimit: 1200m memory: 512m serviceAccount: spark-operator-spark executor: - labels: - version: 3.5.2 instances: 1 cores: 1 - coreLimit: 1200m memory: 512m diff --git a/examples/spark-pi-scheduled.yaml b/examples/spark-pi-scheduled.yaml index 442124049..3bcd053af 100644 --- a/examples/spark-pi-scheduled.yaml +++ b/examples/spark-pi-scheduled.yaml @@ -33,16 +33,10 @@ spec: restartPolicy: type: Never driver: - labels: - version: 3.5.2 cores: 1 - coreLimit: 1200m memory: 512m serviceAccount: spark-operator-spark executor: - labels: - version: 3.5.2 instances: 1 cores: 1 - coreLimit: 1200m memory: 512m diff --git a/examples/spark-pi-volcano.yaml b/examples/spark-pi-volcano.yaml index 871d82d93..889d32c36 100644 --- a/examples/spark-pi-volcano.yaml +++ b/examples/spark-pi-volcano.yaml @@ -27,17 +27,11 @@ spec: mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.12-3.5.2.jar sparkVersion: 3.5.2 driver: - labels: - version: 3.5.2 cores: 1 - coreLimit: 1200m memory: 512m serviceAccount: spark-operator-spark executor: - labels: - version: 3.5.2 instances: 2 cores: 1 - coreLimit: 1200m memory: 512m batchScheduler: volcano diff --git a/examples/spark-pi-yunikorn.yaml b/examples/spark-pi-yunikorn.yaml index 7f3f7d4dc..0cffd6aaa 100644 --- a/examples/spark-pi-yunikorn.yaml +++ b/examples/spark-pi-yunikorn.yaml @@ -27,18 +27,12 @@ spec: mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.12-3.5.2.jar sparkVersion: 3.5.2 driver: - labels: - version: 3.5.2 cores: 1 - coreLimit: 1200m memory: 512m serviceAccount: spark-operator-spark executor: - labels: - version: 3.5.2 instances: 2 cores: 1 - coreLimit: 1200m memory: 512m batchScheduler: yunikorn batchSchedulerOptions: diff --git a/examples/spark-pi.yaml b/examples/spark-pi.yaml index 4c0c15b62..034490e0c 100644 --- a/examples/spark-pi.yaml +++ b/examples/spark-pi.yaml @@ -25,12 +25,13 @@ spec: imagePullPolicy: IfNotPresent mainClass: org.apache.spark.examples.SparkPi mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.12-3.5.2.jar + arguments: + - "5000" sparkVersion: 3.5.2 driver: labels: version: 3.5.2 cores: 1 - coreLimit: 1200m memory: 512m serviceAccount: spark-operator-spark executor: @@ -38,5 +39,4 @@ spec: version: 3.5.2 instances: 1 cores: 1 - coreLimit: 1200m memory: 512m diff --git a/test/e2e/sparkapplication_test.go b/test/e2e/sparkapplication_test.go index 1f64049a8..113326cea 100644 --- a/test/e2e/sparkapplication_test.go +++ b/test/e2e/sparkapplication_test.go @@ -26,11 +26,14 @@ import ( . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/yaml" "github.com/kubeflow/spark-operator/api/v1beta2" + "github.com/kubeflow/spark-operator/pkg/common" "github.com/kubeflow/spark-operator/pkg/util" ) @@ -130,13 +133,43 @@ var _ = Describe("Example SparkApplication", func() { } }) - It("Should complete successfully", func() { + It("Should complete successfully with configmap mounted", func() { By("Waiting for SparkApplication to complete") key := types.NamespacedName{Namespace: app.Namespace, Name: app.Name} Expect(waitForSparkApplicationCompleted(ctx, key)).NotTo(HaveOccurred()) - By("Checking out driver logs") + By("Checking out whether volumes are mounted to driver pod") driverPodName := util.GetDriverPodName(app) + driverPodKey := types.NamespacedName{Namespace: app.Namespace, Name: driverPodName} + driverPod := &corev1.Pod{} + Expect(k8sClient.Get(ctx, driverPodKey, driverPod)).NotTo(HaveOccurred()) + hasVolumes := false + hasVolumeMounts := false + for _, volume := range app.Spec.Volumes { + for _, podVolume := range driverPod.Spec.Volumes { + if volume.Name == podVolume.Name { + hasVolumes = true + break + } + } + } + for _, volumeMount := range app.Spec.Driver.VolumeMounts { + for _, container := range driverPod.Spec.Containers { + if container.Name != common.SparkDriverContainerName { + continue + } + for _, podVolumeMount := range container.VolumeMounts { + if equality.Semantic.DeepEqual(volumeMount, podVolumeMount) { + hasVolumeMounts = true + break + } + } + } + } + Expect(hasVolumes).To(BeTrue()) + Expect(hasVolumeMounts).To(BeTrue()) + + By("Checking out driver logs") bytes, err := clientset.CoreV1().Pods(app.Namespace).GetLogs(driverPodName, &corev1.PodLogOptions{}).Do(ctx).Raw() Expect(err).NotTo(HaveOccurred()) Expect(bytes).NotTo(BeEmpty()) @@ -176,8 +209,26 @@ var _ = Describe("Example SparkApplication", func() { key := types.NamespacedName{Namespace: app.Namespace, Name: app.Name} Expect(waitForSparkApplicationCompleted(ctx, key)).NotTo(HaveOccurred()) - By("Checking out driver logs") + By("Checking out whether resource requests and limits of driver pod are set") driverPodName := util.GetDriverPodName(app) + driverPodKey := types.NamespacedName{Namespace: app.Namespace, Name: driverPodName} + driverPod := &corev1.Pod{} + Expect(k8sClient.Get(ctx, driverPodKey, driverPod)).NotTo(HaveOccurred()) + for _, container := range driverPod.Spec.Containers { + if container.Name != common.SparkDriverContainerName { + continue + } + if app.Spec.Driver.CoreRequest != nil { + Expect(container.Resources.Requests.Cpu().Equal(resource.MustParse(*app.Spec.Driver.CoreRequest))).To(BeTrue()) + } + if app.Spec.Driver.CoreLimit != nil { + Expect(container.Resources.Limits.Cpu().Equal(resource.MustParse(*app.Spec.Driver.CoreLimit))).To(BeTrue()) + } + Expect(container.Resources.Requests.Memory).NotTo(BeNil()) + Expect(container.Resources.Limits.Memory).NotTo(BeNil()) + } + + By("Checking out driver logs") bytes, err := clientset.CoreV1().Pods(app.Namespace).GetLogs(driverPodName, &corev1.PodLogOptions{}).Do(ctx).Raw() Expect(err).NotTo(HaveOccurred()) Expect(bytes).NotTo(BeEmpty()) diff --git a/test/e2e/suit_test.go b/test/e2e/suit_test.go index e409a1cbd..85dc3ed04 100644 --- a/test/e2e/suit_test.go +++ b/test/e2e/suit_test.go @@ -149,6 +149,8 @@ var _ = BeforeSuite(func() { validatingWebhookKey := types.NamespacedName{Name: ValidatingWebhookName} Expect(waitForMutatingWebhookReady(context.Background(), mutatingWebhookKey)).NotTo(HaveOccurred()) Expect(waitForValidatingWebhookReady(context.Background(), validatingWebhookKey)).NotTo(HaveOccurred()) + // TODO: Remove this when there is a better way to ensure the webhooks are ready before running the e2e tests. + time.Sleep(10 * time.Second) }) var _ = AfterSuite(func() {