Add e2e test framework (#490)

.gitignore

@@ -25,6 +25,9 @@ cover.out
 # JUnit test output from ginkgo unit tests
 junit*.xml
 
+#e2e test files
+test/e2e/config/ibmcloud-e2e-envsubst.yaml
+
 # dep ensured 3rd code
 vendor/sigs.k8s.io/cluster-api/docs/book/*.json
 

Makefile

@@ -27,11 +27,15 @@ ARTIFACTS ?= $(REPO_ROOT)/_artifacts
 TOOLS_DIR := hack/tools
 TOOLS_BIN_DIR := $(TOOLS_DIR)/bin
 GO_INSTALL = ./scripts/go_install.sh
+E2E_CONF_FILE ?= $(REPO_ROOT)/test/e2e/config/ibmcloud-e2e.yaml
+E2E_CONF_FILE_ENVSUBST := $(REPO_ROOT)/test/e2e/config/ibmcloud-e2e-envsubst.yaml
 
 GOLANGCI_LINT := $(TOOLS_BIN_DIR)/golangci-lint
 KUSTOMIZE := $(TOOLS_BIN_DIR)/kustomize
 GOJQ := $(TOOLS_BIN_DIR)/gojq
 CONVERSION_GEN := $(TOOLS_BIN_DIR)/conversion-gen
+GINKGO := $(TOOLS_BIN_DIR)/ginkgo
+ENVSUBST := $(TOOLS_BIN_DIR)/envsubst
 
 STAGING_REGISTRY ?= gcr.io/k8s-staging-capi-ibmcloud
 STAGING_BUCKET ?= artifacts.k8s-staging-capi-ibmcloud.appspot.com
@@ -46,6 +50,7 @@ RELEASE_DIR := out
 TAG ?= dev
 ARCH ?= amd64
 ALL_ARCH ?= amd64 ppc64le
+PULL_POLICY ?= Always
 
 # main controller
 CORE_IMAGE_NAME ?= cluster-api-ibmcloud-controller
@@ -72,10 +77,6 @@ endif
 
 all: manager
 
-# Run tests
-test: generate fmt vet manifests
-	go test ./... -coverprofile cover.out
-
 # Build manager binary
 manager: generate fmt vet
 	go build -o bin/manager main.go
@@ -149,6 +150,33 @@ endif
 lint: $(GOLANGCI_LINT) ## Lint codebase
 	$(GOLANGCI_LINT) run -v --fast=false
 
+## --------------------------------------
+## Testing
+## --------------------------------------
+
+# Run unit tests
+test: generate fmt vet manifests
+	go test ./... -coverprofile cover.out
+
+# Allow overriding the e2e configurations
+GINKGO_FOCUS ?= Workload cluster creation
+GINKGO_NODES ?= 3
+GINKGO_NOCOLOR ?= false
+GINKGO_ARGS ?= -v -trace -progress -v -tags=e2e -focus=$(GINKGO_FOCUS) -nodes=$(GINKGO_NODES) --noColor=$(GINKGO_NOCOLOR)
+ARTIFACTS ?= $(REPO_ROOT)/_artifacts
+SKIP_CLEANUP ?= false
+SKIP_CREATE_MGMT_CLUSTER ?= false
+
+#Run the end-to-end tests
+.PHONY: test-e2e
+test-e2e: $(KUBECTL) $(GINKGO) $(ENVSUBST) e2e-image 
+	$(ENVSUBST) < $(E2E_CONF_FILE) > $(E2E_CONF_FILE_ENVSUBST) 
+	$(GINKGO) $(GINKGO_ARGS) ./test/e2e -- \
+        -e2e.artifacts-folder="$(ARTIFACTS)" \
+        -e2e.config="$(E2E_CONF_FILE_ENVSUBST)" \
+        -e2e.skip-resource-cleanup=$(SKIP_CLEANUP) \
+        -e2e.use-existing-cluster=$(SKIP_CREATE_MGMT_CLUSTER)
+
 ## --------------------------------------
 ## Docker
 ## --------------------------------------
@@ -166,6 +194,22 @@ docker-pull-prerequisites:
 	docker pull docker.io/docker/dockerfile:1.1-experimental
 	docker pull gcr.io/distroless/static:latest
 
+.PHONY: e2e-image
+e2e-image: docker-pull-prerequisites
+	docker build --tag=$(CORE_CONTROLLER_ORIGINAL_IMG):e2e .
+	$(MAKE) set-manifest-image MANIFEST_IMG=$(CORE_CONTROLLER_ORIGINAL_IMG):e2e TARGET_RESOURCE="./config/default/manager_image_patch.yaml"
+	$(MAKE) set-manifest-pull-policy PULL_POLICY=Never TARGET_RESOURCE="./config/default/manager_pull_policy.yaml"
+
+.PHONY: set-manifest-image
+set-manifest-image:
+	$(info Updating kustomize image patch file for default resource)
+	sed -i'' -e 's@image: .*@image: '"${MANIFEST_IMG}"'@' ./config/default/manager_image_patch.yaml
+
+.PHONY: set-manifest-pull-policy
+set-manifest-pull-policy:
+	$(info Updating kustomize pull policy file for default resource)
+	sed -i'' -e 's@imagePullPolicy: .*@imagePullPolicy: '"$(PULL_POLICY)"'@' ./config/default/manager_pull_policy.yaml	
+
 ## --------------------------------------
 ## Docker - All ARCH
 ## --------------------------------------

config/default/kustomization.yaml

@@ -33,6 +33,7 @@ patchesStrategicMerge:
 - manager_credentials_patch.yaml
 - manager_auth_proxy_patch.yaml
 - manager_image_patch.yaml
+- manager_pull_policy.yaml
 
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in 
 # crd/kustomization.yaml

config/default/manager_pull_policy.yaml

@@ -0,0 +1,11 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+      - name: manager
+        imagePullPolicy: Always

go.mod

@@ -21,5 +21,8 @@ require (
 	k8s.io/kube-openapi v0.0.0-20211110012726-3cc51fd1e909 // indirect
 	k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b
 	sigs.k8s.io/cluster-api v1.0.2
+	sigs.k8s.io/cluster-api/test v1.0.2
 	sigs.k8s.io/controller-runtime v0.10.3
 )
+
+replace sigs.k8s.io/cluster-api => sigs.k8s.io/cluster-api v1.0.2

hack/tools/Makefile

@@ -78,3 +78,11 @@ $(GOJQ): $(BIN_DIR) go.mod go.sum
 CONVERSION_GEN := $(BIN_DIR)/conversion-gen
 $(CONVERSION_GEN): $(BIN_DIR) go.mod go.sum
 	go build -tags=tools -o $@ k8s.io/code-generator/cmd/conversion-gen
+
+GINKGO := $(BIN_DIR)/ginkgo
+$(GINKGO): $(BIN_DIR) go.mod go.sum
+	go build -tags=tools -o $@ github.com/onsi/ginkgo/ginkgo
+
+ENVSUBST := $(BIN_DIR)/envsubst
+$(ENVSUBST): $(BIN_DIR) go.mod go.sum
+	go build -tags=tools -o $@ github.com/drone/envsubst/v2/cmd/envsubst

test/e2e/config/ibmcloud-e2e.yaml

@@ -0,0 +1,62 @@
+managementClusterName: capi-ibmcloud-e2e
+
+images:
+  # Use local built images for e2e tests
+  - name: gcr.io/k8s-staging-capi-ibmcloud/cluster-api-ibmcloud-controller:e2e
+    loadBehavior: mustLoad
+
+providers:
+  - name: cluster-api
+    type: CoreProvider
+    versions:
+    - name: v1.0.2
+      value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.0.2/core-components.yaml
+      type: url
+      files:
+      - sourcePath: "${PWD}/test/e2e/data/shared/metadata.yaml"
+  - name: kubeadm
+    type: BootstrapProvider
+    versions:
+    - name: v1.0.2
+      value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.0.2/bootstrap-components.yaml
+      type: url
+      files:
+      - sourcePath: "${PWD}/test/e2e/data/shared/metadata.yaml"
+  - name: kubeadm
+    type: ControlPlaneProvider
+    versions:
+    - name: v1.0.2
+      value: https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.0.2/control-plane-components.yaml
+      type: url
+      files:
+      - sourcePath: "${PWD}/test/e2e/data/shared/metadata.yaml"
+  - name: ibmcloud
+    type: InfrastructureProvider
+    versions:
+    - name: v0.2.0
+      value: "${PWD}/config/default"
+      files:
+      - sourcePath: "${PWD}/metadata.yaml"
+        targetName: "metadata.yaml"
+      - sourcePath: "${PWD}/templates/cluster-template-powervs.yaml"
+        targetName: "cluster-template-powervs.yaml"
+      - sourcePath: "${PWD}/templates/cluster-template.yaml"
+        targetName: "cluster-template-vpc.yaml"
+
+variables:
+  KUBERNETES_VERSION: "${KUBERNETES_VERSION:-v1.22.4}"
+  # Cluster Addons
+  CNI: "${PWD}/test/e2e/data/cni/calico/calico.yaml"
+  IP_FAMILY: "IPv4"
+
+intervals:
+  default/wait-controllers: ["3m", "10s"]
+  default/wait-cluster: ["20m", "10s"]
+  default/wait-control-plane: ["30m", "10s"]
+  default/wait-worker-nodes: ["30m", "10s"]
+  default/wait-delete-cluster: ["20m", "10s"]
+  default/wait-machine-upgrade: ["50m", "10s"]
+  default/wait-machine-remediation: ["30m", "10s"]
+  default/wait-deployment: ["5m", "10s"]
+  default/wait-job: ["5m", "10s"]
+  default/wait-service: ["3m", "10s"]