diff --git a/Makefile b/Makefile index ad6c1f0492..6c4751841f 100644 --- a/Makefile +++ b/Makefile @@ -158,6 +158,8 @@ e2e-templates: ## Generate e2e cluster templates "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/kustomization/pci > $(E2E_TEMPLATE_DIR)/cluster-template-pci.yaml # for DHCP overrides "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/kustomization/dhcp-overrides > $(E2E_TEMPLATE_DIR)/cluster-template-dhcp-overrides.yaml + # for IPAM template + "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build $(E2E_TEMPLATE_DIR)/kustomization/ipam > $(E2E_TEMPLATE_DIR)/cluster-template-ipam.yaml .PHONY: test-integration test-integration: e2e-image diff --git a/test/e2e/config/vsphere-dev.yaml b/test/e2e/config/vsphere-dev.yaml index 2f1674350e..2f4d428e9d 100644 --- a/test/e2e/config/vsphere-dev.yaml +++ b/test/e2e/config/vsphere-dev.yaml @@ -96,6 +96,19 @@ providers: - old: "imagePullPolicy: Always" new: "imagePullPolicy: IfNotPresent" + - name: incluster + type: IPAMProvider + versions: + - name: v0.1.0 + value: https://github.com/telekom/cluster-api-ipam-provider-in-cluster/releases/download/v0.1.0-alpha.1/ipam-components.yaml + type: "url" + contract: v1beta1 + files: + - sourcePath: "../data/cluster-api-ipam-provider-in-cluster/metadata.yaml" + replacements: + - old: "imagePullPolicy: Always" + new: "imagePullPolicy: IfNotPresent" + - name: vsphere type: InfrastructureProvider versions: @@ -126,6 +139,7 @@ providers: - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/cluster-template-pci.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/cluster-template-remote-management.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/cluster-template-storage-policy.yaml" + - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/cluster-template-ipam.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/cluster-template-topology.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/cluster-template-dhcp-overrides.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/clusterclass-quick-start.yaml" @@ -170,6 +184,12 @@ variables: # CAPV feature flags EXP_NODE_ANTI_AFFINITY: "true" EXP_NODE_LABELING: "true" + # Node IPAM settings + NODE_IPAM_POOL_SUBNET: 192.168.116.1/24 + NODE_IPAM_POOL_GATEWAY: 192.168.116.1 + NODE_IPAM_POOL_START: 192.168.116.151 + NODE_IPAM_POOL_END: 192.168.116.200 + NODE_IPAM_NAMESERVER: 10.142.7.1 intervals: default/wait-controllers: ["5m", "10s"] diff --git a/test/e2e/data/cluster-api-ipam-provider-in-cluster/metadata.yaml b/test/e2e/data/cluster-api-ipam-provider-in-cluster/metadata.yaml new file mode 100644 index 0000000000..11facb297c --- /dev/null +++ b/test/e2e/data/cluster-api-ipam-provider-in-cluster/metadata.yaml @@ -0,0 +1,6 @@ +apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3 +kind: Metadata +releaseSeries: + - major: 0 + minor: 1 + contract: v1beta1 diff --git a/test/e2e/data/infrastructure-vsphere/kustomization/ipam/ipam-template.yaml b/test/e2e/data/infrastructure-vsphere/kustomization/ipam/ipam-template.yaml new file mode 100644 index 0000000000..c5045a4e5c --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere/kustomization/ipam/ipam-template.yaml @@ -0,0 +1,34 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: VSphereMachineTemplate +metadata: + name: ${CLUSTER_NAME} + namespace: ${NAMESPACE} +spec: + template: + spec: + network: + devices: + - nameservers: ["${NODE_IPAM_NAMESERVER}"] + addressesFromPools: + - name: inclusterippool + apiGroup: ipam.cluster.x-k8s.io + kind: InClusterIPPool + networkName: '${VSPHERE_NETWORK}' +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: VSphereMachineTemplate +metadata: + name: ${CLUSTER_NAME}-worker + namespace: ${NAMESPACE} +spec: + template: + spec: + network: + devices: + - nameservers: ["${NODE_IPAM_NAMESERVER}"] + addressesFromPools: + - name: inclusterippool + apiGroup: ipam.cluster.x-k8s.io + kind: InClusterIPPool + networkName: '${VSPHERE_NETWORK}' diff --git a/test/e2e/data/infrastructure-vsphere/kustomization/ipam/kustomization.yaml b/test/e2e/data/infrastructure-vsphere/kustomization/ipam/kustomization.yaml new file mode 100644 index 0000000000..4354857ef6 --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere/kustomization/ipam/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../base + - pool.yaml +patchesStrategicMerge: + - ipam-template.yaml diff --git a/test/e2e/data/infrastructure-vsphere/kustomization/ipam/pool.yaml b/test/e2e/data/infrastructure-vsphere/kustomization/ipam/pool.yaml new file mode 100644 index 0000000000..7f3aa94b79 --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere/kustomization/ipam/pool.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: ipam.cluster.x-k8s.io/v1alpha1 +kind: InClusterIPPool +metadata: + name: inclusterippool + namespace: ${NAMESPACE} +spec: + subnet: ${NODE_IPAM_POOL_SUBNET} + gateway: ${NODE_IPAM_POOL_GATEWAY} + start: ${NODE_IPAM_POOL_START} + end: ${NODE_IPAM_POOL_END} diff --git a/test/e2e/ipam_test.go b/test/e2e/ipam_test.go new file mode 100644 index 0000000000..1d1d5b244b --- /dev/null +++ b/test/e2e/ipam_test.go @@ -0,0 +1,110 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "bytes" + "fmt" + "net" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + "k8s.io/utils/pointer" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" + "sigs.k8s.io/cluster-api/util" +) + +const ( + NodeIPAMPoolSubnet = "NODE_IPAM_POOL_SUBNET" + NodeIPAMPoolStart = "NODE_IPAM_POOL_START" + NodeIPAMPoolEnd = "NODE_IPAM_POOL_END" +) + +var _ = Describe("Clusters using FromPools get assigned addresses from IPAM", func() { + var namespace *v1.Namespace + var start, end string + + BeforeEach(func() { + start = e2eConfig.GetVariable(NodeIPAMPoolStart) + Expect(net.ParseIP(start)).NotTo(BeNil()) + end = e2eConfig.GetVariable(NodeIPAMPoolEnd) + Expect(net.ParseIP(end)).NotTo(BeNil()) + Expect(bootstrapClusterProxy).NotTo(BeNil(), "BootstrapClusterProxy can't be nil") + namespace = setupSpecNamespace("node-ipam") + }) + + AfterEach(func() { + cleanupSpecNamespace(namespace) + }) + + It("should create a cluster successfully", func() { + clusterName := fmt.Sprintf("cluster-%s", util.RandomString(6)) + + By("creating a workload cluster") + configCluster := ipamConfigCluster(clusterName, namespace.Name, 1, 1) + + clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: bootstrapClusterProxy, + ConfigCluster: configCluster, + WaitForClusterIntervals: e2eConfig.GetIntervals("", "wait-cluster"), + WaitForControlPlaneIntervals: e2eConfig.GetIntervals("", "wait-control-plane"), + WaitForMachineDeployments: e2eConfig.GetIntervals("", "wait-worker-nodes"), + }, &clusterctl.ApplyClusterTemplateAndWaitResult{}) + + By("Verifying that the cluster is using IPAM provided IP addresses") + list := getVSphereVMsForCluster(clusterName, namespace.Name) + Expect(list.Items).NotTo(BeEmpty()) + for _, vm := range list.Items { + path := fmt.Sprintf("/%s/vm/%s/%s", vm.Spec.Datacenter, vm.Spec.Folder, vm.Name) + vm, err := vsphereFinder.VirtualMachine(ctx, path) + Expect(err).ShouldNot(HaveOccurred()) + ip, err := vm.WaitForIP(ctx, true) + Expect(err).ShouldNot(HaveOccurred()) + Expect(IPIsInRange(ip, start, end)).To(BeTrue(), fmt.Sprintf("Expected IP %q to be between %q and %q", ip, start, end)) + } + }) +}) + +func IPIsInRange(ip, rangeStart, rangeEnd string) bool { + parsedIP := net.ParseIP(ip) + Expect(parsedIP).NotTo(BeNil()) + + parsedStart := net.ParseIP(rangeStart) + Expect(parsedStart).NotTo(BeNil()) + + parsedEnd := net.ParseIP(rangeEnd) + Expect(parsedEnd).NotTo(BeNil()) + + return bytes.Compare(parsedIP, parsedStart) >= 0 && bytes.Compare(parsedIP, parsedEnd) <= 0 +} + +func ipamConfigCluster(clusterName, namespace string, controlPlaneNodeCount, workerNodeCount int64) clusterctl.ConfigClusterInput { + return clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()), + ClusterctlConfigPath: clusterctlConfigPath, + KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: "ipam", + Namespace: namespace, + ClusterName: clusterName, + KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion), + ControlPlaneMachineCount: pointer.Int64Ptr(controlPlaneNodeCount), + WorkerMachineCount: pointer.Int64Ptr(workerNodeCount), + } +} diff --git a/test/helpers/framework.go b/test/helpers/framework.go index 6d182ee74f..69ffbc2cec 100644 --- a/test/helpers/framework.go +++ b/test/helpers/framework.go @@ -91,7 +91,10 @@ func InitBootstrapCluster(bootstrapClusterProxy framework.ClusterProxy, config * ClusterProxy: bootstrapClusterProxy, ClusterctlConfigPath: clusterctlConfig, InfrastructureProviders: config.InfrastructureProviders(), - LogFolder: filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()), + IPAMProviders: []string{ + "incluster:v0.1.0", + }, + LogFolder: filepath.Join(artifactFolder, "clusters", bootstrapClusterProxy.GetName()), }, config.GetIntervals(bootstrapClusterProxy.GetName(), "wait-controllers")...) }