Skip to content

Commit

Permalink
🌱 test: implement cluster upgrade with Runtime SDK (#3069)
Browse files Browse the repository at this point in the history
* test: implement cluster upgrade with Runtime SDK and custom test extension

* review fixes
  • Loading branch information
chrischdi authored Jul 17, 2024
1 parent 6b27129 commit 4a7d0ff
Show file tree
Hide file tree
Showing 54 changed files with 1,506 additions and 95 deletions.
3 changes: 3 additions & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@ _artifacts
Makefile
**/Makefile

# add yaml files from internal/kubevip which are required for embedding.
!internal/kubevip/*.yaml

# ignores changes to test-only code to avoid extra rebuilds
test/e2e/**

Expand Down
4 changes: 2 additions & 2 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,12 @@ _artifacts/
test/e2e/data/infrastructure-vsphere-govmomi/main/**/clusterclass-quick-start.yaml
test/e2e/data/infrastructure-vsphere-govmomi/main/**/cluster-template*.yaml
test/e2e/data/infrastructure-vsphere-govmomi/*/cluster-template*.yaml
test/e2e/data/infrastructure-vsphere-govmomi/*/clusterclass-quick-start.yaml
test/e2e/data/infrastructure-vsphere-govmomi/*/clusterclass-quick-start*.yaml

test/e2e/data/infrastructure-vsphere-supervisor/main/**/clusterclass-quick-start-supervisor.yaml
test/e2e/data/infrastructure-vsphere-supervisor/main/**/cluster-template*.yaml
test/e2e/data/infrastructure-vsphere-supervisor/*/cluster-template*.yaml
test/e2e/data/infrastructure-vsphere-supervisor/*/clusterclass-quick-start-supervisor.yaml
test/e2e/data/infrastructure-vsphere-supervisor/*/clusterclass-quick-start*-supervisor.yaml

# env vars file used in getting-started.md and manifests generation
envvars.txt
Expand Down
25 changes: 25 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ BUILD_DIR := .build
TEST_DIR := test
VCSIM_DIR := test/infrastructure/vcsim
NETOP_DIR := test/infrastructure/net-operator
TEST_EXTENSION_DIR := test/extension
TOOLS_DIR := hack/tools
TOOLS_BIN_DIR := $(abspath $(TOOLS_DIR)/$(BIN_DIR))
FLAVOR_DIR := $(ROOT_DIR)/templates
Expand Down Expand Up @@ -233,6 +234,10 @@ VM_OPERATOR_ALL_ARCH = amd64 arm64
NET_OPERATOR_IMAGE_NAME ?= cluster-api-net-operator
NET_OPERATOR_IMG ?= $(STAGING_REGISTRY)/$(NET_OPERATOR_IMAGE_NAME)

# test-extension
TEST_EXTENSION_IMAGE_NAME ?= cluster-api-vsphere-test-extension
TEST_EXTENSION_IMG ?= $(STAGING_REGISTRY)/$(TEST_EXTENSION_IMAGE_NAME)

# boskosctl
BOSKOSCTL_IMG ?= gcr.io/k8s-staging-capi-vsphere/extra/boskosctl
BOSKOSCTL_IMG_TAG ?= $(shell git describe --always --dirty)
Expand Down Expand Up @@ -271,6 +276,7 @@ SUPERVISOR_WEBHOOK_ROOT ?= $(MANIFEST_ROOT)/supervisor/webhook
RBAC_ROOT ?= $(MANIFEST_ROOT)/rbac
VCSIM_RBAC_ROOT ?= $(VCSIM_DIR)/config/rbac
NETOP_RBAC_ROOT ?= $(NETOP_DIR)/config/rbac
TEST_EXTENSION_RBAC_ROOT ?= $(TEST_EXTENSION_DIR)/config/rbac

JANITOR_DIR ?= ./$(TOOLS_DIR)/janitor

Expand Down Expand Up @@ -318,6 +324,11 @@ generate-manifests: $(CONTROLLER_GEN) ## Generate manifests e.g. CRD, RBAC etc.
paths=./$(NETOP_DIR)/controllers/... \
output:rbac:dir=$(NETOP_RBAC_ROOT) \
rbac:roleName=manager-role
# test-extension is used for Runtime SDK tests
$(CONTROLLER_GEN) \
paths=./$(TEST_EXTENSION_DIR)/... \
output:rbac:dir=$(TEST_EXTENSION_RBAC_ROOT) \
rbac:roleName=manager-role
# vcsim crds are used for tests.
$(CONTROLLER_GEN) \
paths=./$(VCSIM_DIR)/api/v1alpha1 \
Expand Down Expand Up @@ -377,8 +388,10 @@ generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the mai
# generate clusterclass and cluster topology
cp "$(RELEASE_DIR)/main/clusterclass-template.yaml" "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/clusterclass/clusterclass-quick-start.yaml"
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/clusterclass" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/clusterclass-quick-start.yaml"
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/clusterclass-runtimesdk" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/clusterclass-quick-start-runtimesdk.yaml"
cp "$(RELEASE_DIR)/main/cluster-template-topology.yaml" "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/topology/cluster-template-topology.yaml"
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/topology" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-topology.yaml"
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/topology-runtimesdk" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-topology-runtimesdk.yaml"
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/install-on-bootstrap" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-install-on-bootstrap.yaml"
# for PCI passthrough template
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/pci" > "$(E2E_GOVMOMI_TEMPLATE_DIR)/main/cluster-template-pci.yaml"
Expand All @@ -392,8 +405,10 @@ generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the mai
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/base" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-supervisor.yaml"
cp "$(RELEASE_DIR)/main/clusterclass-template-supervisor.yaml" "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/clusterclass/clusterclass-quick-start-supervisor.yaml"
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/clusterclass" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/clusterclass-quick-start-supervisor.yaml"
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/clusterclass-runtimesdk" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/clusterclass-quick-start-supervisor-runtimesdk.yaml"
cp "$(RELEASE_DIR)/main/cluster-template-topology-supervisor.yaml" "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/topology/cluster-template-topology-supervisor.yaml"
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/topology" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-topology-supervisor.yaml"
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/topology-runtimesdk" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-topology-runtimesdk-supervisor.yaml"
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/conformance" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-conformance-supervisor.yaml"
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/install-on-bootstrap" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-install-on-bootstrap-supervisor.yaml"
"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/ownerrefs-finalizers" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-ownerrefs-finalizers-supervisor.yaml"
Expand Down Expand Up @@ -576,6 +591,15 @@ docker-build-net-operator: docker-pull-prerequisites ## Build the docker image f
$(MAKE) set-manifest-pull-policy TARGET_RESOURCE="./$(NETOP_DIR)/config/default/manager_pull_policy.yaml"; \
fi

.PHONY: docker-build-test-extension
docker-build-test-extension: docker-pull-prerequisites ## Build the docker image for test-extension controller manager
## reads Dockerfile from stdin to avoid an incorrectly cached Dockerfile (https://github.com/moby/buildkit/issues/1368)
cat $(TEST_EXTENSION_DIR)/Dockerfile | DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg ldflags="$(LDFLAGS)" . -t $(TEST_EXTENSION_IMG)-$(ARCH):$(TAG) --file -
@if [ "${DOCKER_BUILD_MODIFY_MANIFESTS}" = "true" ]; then \
$(MAKE) set-manifest-image MANIFEST_IMG=$(TEST_EXTENSION_IMG)-$(ARCH) MANIFEST_TAG=$(TAG) TARGET_RESOURCE="./$(TEST_EXTENSION_DIR)/config/default/manager_image_patch.yaml"; \
$(MAKE) set-manifest-pull-policy TARGET_RESOURCE="./$(TEST_EXTENSION_DIR)/config/default/manager_pull_policy.yaml"; \
fi

.PHONY: docker-build-boskosctl
docker-build-boskosctl:
cat hack/tools/boskosctl/Dockerfile | DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) . -t $(BOSKOSCTL_IMG):$(BOSKOSCTL_IMG_TAG) --file -
Expand Down Expand Up @@ -643,6 +667,7 @@ e2e-images: ## Build the e2e manager image
$(MAKE) REGISTRY=gcr.io/k8s-staging-capi-vsphere PULL_POLICY=IfNotPresent TAG=dev docker-build
$(MAKE) REGISTRY=gcr.io/k8s-staging-capi-vsphere PULL_POLICY=IfNotPresent TAG=dev docker-build-vcsim
$(MAKE) REGISTRY=gcr.io/k8s-staging-capi-vsphere PULL_POLICY=IfNotPresent TAG=dev docker-build-net-operator
$(MAKE) REGISTRY=gcr.io/k8s-staging-capi-vsphere PULL_POLICY=IfNotPresent TAG=dev docker-build-test-extension

.PHONY: e2e
e2e: e2e-images generate-e2e-templates
Expand Down
2 changes: 1 addition & 1 deletion config/base/manager_pull_policy.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,4 @@ spec:
spec:
containers:
- name: manager
imagePullPolicy: IfNotPresent
imagePullPolicy: Always
101 changes: 101 additions & 0 deletions internal/clusterclass/variables.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package clusterclass provides the shared functions for creating clusterclasses.
package clusterclass

import (
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
)

// GetClusterClassVariables provides the variables for the clusterclass.
// In govmomi mode it has additional variables.
func GetClusterClassVariables(govmomiMode bool) []clusterv1.ClusterClassVariable {
variables := []clusterv1.ClusterClassVariable{
{
Name: "sshKey",
Required: false,
Schema: clusterv1.VariableSchema{
OpenAPIV3Schema: clusterv1.JSONSchemaProps{
Description: "Public key to SSH onto the cluster nodes.",
Type: "string",
},
},
},
{
Name: "controlPlaneIpAddr",
Required: true,
Schema: clusterv1.VariableSchema{
OpenAPIV3Schema: clusterv1.JSONSchemaProps{
Type: "string",
Description: "Floating VIP for the control plane.",
},
},
},
{
Name: "controlPlanePort",
Required: true,
Schema: clusterv1.VariableSchema{
OpenAPIV3Schema: clusterv1.JSONSchemaProps{
Type: "integer",
Description: "Port for the control plane endpoint.",
},
},
},
{
Name: "kubeVipPodManifest",
Required: true,
Schema: clusterv1.VariableSchema{
OpenAPIV3Schema: clusterv1.JSONSchemaProps{
Type: "string",
Description: "kube-vip manifest for the control plane.",
},
},
},
}

if govmomiMode {
varForNoneSupervisorMode := []clusterv1.ClusterClassVariable{
{
Name: "infraServer",
Required: true,
Schema: clusterv1.VariableSchema{
OpenAPIV3Schema: clusterv1.JSONSchemaProps{
Type: "object",
Properties: map[string]clusterv1.JSONSchemaProps{
"url": {Type: "string"},
"thumbprint": {Type: "string"},
},
},
},
},
{
Name: "credsSecretName",
Required: true,
Schema: clusterv1.VariableSchema{
OpenAPIV3Schema: clusterv1.JSONSchemaProps{
Type: "string",
Description: "Secret containing the credentials for the infra cluster.",
},
},
},
}

variables = append(variables, varForNoneSupervisorMode...)
}

return variables
}
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

// Package kubevip provides the files required to run kube-vip in a cluster.
package kubevip

import (
Expand All @@ -38,12 +39,13 @@ var (
kubeVipPodRaw string
)

func newKubeVIPFiles() []bootstrapv1.File {
// Files returns the files required for a control plane node to run kube-vip.
func Files() []bootstrapv1.File {
return []bootstrapv1.File{
{
Owner: "root:root",
Path: "/etc/kubernetes/manifests/kube-vip.yaml",
Content: kubeVIPPodYAML(),
Content: PodYAML(),
Permissions: "0644",
},
// This file is part of the workaround for https://github.com/kube-vip/kube-vip/issues/692
Expand All @@ -63,7 +65,8 @@ func newKubeVIPFiles() []bootstrapv1.File {
}
}

func kubeVIPPodYAML() string {
// PodYAML returns the static pod manifest required to run kube-vip.
func PodYAML() string {
pod := &corev1.Pod{}

if err := yaml.Unmarshal([]byte(kubeVipPodRaw), pod); err != nil {
Expand Down
File renamed without changes.
File renamed without changes.
82 changes: 3 additions & 79 deletions packaging/flavorgen/flavors/clusterclass_generators.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (

infrav1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/v1beta1"
vmwarev1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/vmware/v1beta1"
"sigs.k8s.io/cluster-api-provider-vsphere/internal/clusterclass"
"sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/env"
"sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/kubevip"
"sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/util"
Expand All @@ -55,7 +56,7 @@ func newClusterClass() clusterv1.ClusterClass {
},
ControlPlane: getControlPlaneClass(),
Workers: getWorkersClass(),
Variables: getClusterClassVariables(false),
Variables: clusterclass.GetClusterClassVariables(true),
Patches: getClusterClassPatches(),
},
}
Expand All @@ -81,7 +82,7 @@ func newVMWareClusterClass() clusterv1.ClusterClass {
},
ControlPlane: getVMWareControlPlaneClass(),
Workers: getVMWareWorkersClass(),
Variables: getClusterClassVariables(true),
Variables: clusterclass.GetClusterClassVariables(false),
Patches: getVMWareClusterClassPatches(),
},
}
Expand Down Expand Up @@ -238,83 +239,6 @@ func getEnableSSHIntoNodesTemplate() *string {
return ptr.To(string(templateStr))
}

func getClusterClassVariables(supervisorMode bool) []clusterv1.ClusterClassVariable {
variables := []clusterv1.ClusterClassVariable{
{
Name: "sshKey",
Required: false,
Schema: clusterv1.VariableSchema{
OpenAPIV3Schema: clusterv1.JSONSchemaProps{
Description: "Public key to SSH onto the cluster nodes.",
Type: "string",
},
},
},
{
Name: "controlPlaneIpAddr",
Required: true,
Schema: clusterv1.VariableSchema{
OpenAPIV3Schema: clusterv1.JSONSchemaProps{
Type: "string",
Description: "Floating VIP for the control plane.",
},
},
},
{
Name: "controlPlanePort",
Required: true,
Schema: clusterv1.VariableSchema{
OpenAPIV3Schema: clusterv1.JSONSchemaProps{
Type: "integer",
Description: "Port for the control plane endpoint.",
},
},
},
{
Name: "kubeVipPodManifest",
Required: true,
Schema: clusterv1.VariableSchema{
OpenAPIV3Schema: clusterv1.JSONSchemaProps{
Type: "string",
Description: "kube-vip manifest for the control plane.",
},
},
},
}

if !supervisorMode {
varForNoneSupervisorMode := []clusterv1.ClusterClassVariable{
{
Name: "infraServer",
Required: true,
Schema: clusterv1.VariableSchema{
OpenAPIV3Schema: clusterv1.JSONSchemaProps{
Type: "object",
Properties: map[string]clusterv1.JSONSchemaProps{
"url": {Type: "string"},
"thumbprint": {Type: "string"},
},
},
},
},
{
Name: "credsSecretName",
Required: true,
Schema: clusterv1.VariableSchema{
OpenAPIV3Schema: clusterv1.JSONSchemaProps{
Type: "string",
Description: "Secret containing the credentials for the infra cluster.",
},
},
},
}

variables = append(variables, varForNoneSupervisorMode...)
}

return variables
}

func newVSphereClusterTemplate() infrav1.VSphereClusterTemplate {
return infrav1.VSphereClusterTemplate{
TypeMeta: metav1.TypeMeta{
Expand Down
4 changes: 3 additions & 1 deletion packaging/flavorgen/flavors/kubevip/kubevip.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,11 @@ package kubevip

import (
controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"

"sigs.k8s.io/cluster-api-provider-vsphere/internal/kubevip"
)

// PatchControlPlane adds kube-vip to a KubeadmControlPlane object.
func PatchControlPlane(cp *controlplanev1.KubeadmControlPlane) {
cp.Spec.KubeadmConfigSpec.Files = append(cp.Spec.KubeadmConfigSpec.Files, newKubeVIPFiles()...)
cp.Spec.KubeadmConfigSpec.Files = append(cp.Spec.KubeadmConfigSpec.Files, kubevip.Files()...)
}
5 changes: 3 additions & 2 deletions packaging/flavorgen/flavors/kubevip/topology.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,13 @@ import (
controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
"sigs.k8s.io/yaml"

"sigs.k8s.io/cluster-api-provider-vsphere/internal/kubevip"
"sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/util"
)

// TopologyVariable returns the ClusterClass variable for kube-vip.
func TopologyVariable() (*clusterv1.ClusterVariable, error) {
out, err := json.Marshal(kubeVIPPodYAML())
out, err := json.Marshal(kubevip.PodYAML())
if err != nil {
return nil, errors.Wrapf(err, "failed to json-encode variable kubeVipPod")
}
Expand All @@ -52,7 +53,7 @@ func TopologyVariable() (*clusterv1.ClusterVariable, error) {
func TopologyPatch() clusterv1.ClusterClassPatch {
patches := []clusterv1.JSONPatch{}

for _, f := range newKubeVIPFiles() {
for _, f := range kubevip.Files() {
p := clusterv1.JSONPatch{
Op: "add",
Path: "/spec/template/spec/kubeadmConfigSpec/files/-",
Expand Down
Loading

0 comments on commit 4a7d0ff

Please sign in to comment.