Run e2e tests on Kind (#2148)

Signed-off-by: Jacob Salway <[email protected]>

.github/workflows/integration.yaml

@@ -91,7 +91,7 @@ jobs:
           done
 
   build-helm-chart:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - name: Determine branch name
         id: get_branch
@@ -163,37 +163,25 @@ jobs:
           minikube image load docker.io/kubeflow/spark-operator:local
           ct install
 
-  integration-test:
-    runs-on: ubuntu-22.04
+  e2e-test:
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout source code
         uses: actions/checkout@v4
         with:
-          fetch-depth: "0"
+          fetch-depth: 0
 
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version-file: "go.mod"
+          go-version-file: go.mod
 
-      - name: setup minikube
-        uses: manusa/[email protected]
-        with:
-          minikube version: v1.33.0
-          kubernetes version: v1.30.0
-          start args: --memory 6g --cpus=2 --addons ingress
-          github token: ${{ inputs.github-token }}
+      - name: Create a Kind cluster
+        run: make kind-create-cluster
 
-      - name: Build local spark-operator docker image for minikube testing
+      - name: Build and load image to Kind cluster
         run: |
-          docker build -t docker.io/kubeflow/spark-operator:local .
-          minikube image load docker.io/kubeflow/spark-operator:local
-
-      # The integration tests are currently broken see: https://github.com/kubeflow/spark-operator/issues/1416
-      # - name: Run chart-testing (integration test)
-      #   run: make integration-test
+          make kind-load-image IMAGE_TAG=local
 
-      - name: Setup tmate session
-        if: failure()
-        uses: mxschmitt/action-tmate@v3
-        timeout-minutes: 15
+      - name: Run e2e tests
+        run: make e2e-test

Makefile

@@ -246,7 +246,7 @@ endif
 .PHONY: kind-create-cluster
 kind-create-cluster: kind ## Create a kind cluster for integration tests.
 	if ! $(KIND) get clusters 2>/dev/null | grep -q "^$(KIND_CLUSTER_NAME)$$"; then \
-		kind create cluster --name $(KIND_CLUSTER_NAME) --config $(KIND_CONFIG_FILE) --kubeconfig $(KIND_KUBE_CONFIG); \
+		kind create cluster --name $(KIND_CLUSTER_NAME) --config $(KIND_CONFIG_FILE) --kubeconfig $(KIND_KUBE_CONFIG) --wait=1m; \
 	fi
 
 .PHONY: kind-load-image

charts/spark-operator-chart/ci/kind-config.yaml

@@ -0,0 +1,7 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+  - role: control-plane
+    image: kindest/node:v1.29.2
+  - role: worker
+    image: kindest/node:v1.29.2

test/e2e/suit_test.go

@@ -27,9 +27,9 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-
 	"helm.sh/helm/v3/pkg/action"
 	"helm.sh/helm/v3/pkg/chart/loader"
+	"helm.sh/helm/v3/pkg/chartutil"
 	"helm.sh/helm/v3/pkg/cli"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -128,7 +128,10 @@ var _ = BeforeSuite(func() {
 	chart, err := loader.Load(chartPath)
 	Expect(err).NotTo(HaveOccurred())
 	Expect(chart).NotTo(BeNil())
-	release, err := installAction.Run(chart, nil)
+	values, err := chartutil.ReadValuesFile(filepath.Join(chartPath, "ci", "ci-values.yaml"))
+	Expect(err).NotTo(HaveOccurred())
+	Expect(values).NotTo(BeNil())
+	release, err := installAction.Run(chart, values)
 	Expect(err).NotTo(HaveOccurred())
 	Expect(release).NotTo(BeNil())
 })