Add IPv6 support

hack/ci/e2e.sh

@@ -102,12 +102,15 @@ create_cluster() {
 # necessary for conformance
 kind: Cluster
 apiVersion: kind.sigs.k8s.io/v1alpha3
+networking:
+  ipFamily: ${IP_FAMILY:-ipv4}
 nodes:
 # the control plane node
 - role: control-plane
 - role: worker
 - role: worker
 EOF
+
     # mark the cluster as up for cleanup
     # even if kind create fails, kind delete can clean up after it
     KIND_IS_UP=true
@@ -130,6 +133,34 @@ run_tests() {
     KUBECONFIG="$(kind get kubeconfig-path)"
     export KUBECONFIG
 
+    if [[ "${IP_FAMILY:-ipv4}" == "ipv6" ]]; then
+        # Create the CoreDNS config for offline and IPv6 clusters
+        # https://github.com/coredns/coredns/issues/2494#issuecomment-457215452
+        cat <<EOF | kubectl apply -f -
+---
+apiVersion: v1
+data:
+  Corefile: |
+    .:53 {
+        errors
+        health
+        rewrite name google.com my-google.default.svc.cluster.local
+        kubernetes cluster.local in-addr.arpa ip6.arpa {
+           pods insecure
+        }
+        prometheus :9153
+        cache 30
+        reload
+        loadbalance
+    }
+kind: ConfigMap
+metadata:
+  name: coredns
+  namespace: kube-system
+---
+EOF
+    fi
+
     # base kubetest args
     KUBETEST_ARGS="--provider=skeleton --test --check-version-skew=false"
 

pkg/cluster/config/fuzzer/fuzzer.go

@@ -42,6 +42,8 @@ func fuzzConfig(obj *config.Cluster, c fuzz.Continue) {
 	}}
 	obj.Networking.APIServerAddress = "127.0.0.1"
 	obj.Networking.PodSubnet = "10.244.0.0/16"
+	obj.Networking.ServiceSubnet = "10.96.0.0/12"
+	obj.Networking.IPFamily = "ipv4"
 }
 
 func fuzzNode(obj *config.Node, c fuzz.Continue) {

pkg/cluster/config/types.go

@@ -88,6 +88,8 @@ const (
 
 // Networking contains cluster wide network settings
 type Networking struct {
+	// IPFamily is the network cluster model, currently it can be ipv4 or ipv6
+	IPFamily ClusterIPFamily
 	// APIServerPort is the listen port on the host for the Kubernetes API Server
 	// Defaults to a random port on the host
 	APIServerPort int32
@@ -99,7 +101,20 @@ type Networking struct {
 	// PodSubnet is the CIDR used for pod IPs
 	// kind will select a default if unspecified
 	PodSubnet string
+	// ServiceSubnet is the CIDR used for services VIPs
+	// kind will select a default if unspecified
+	ServiceSubnet string
 	// If DisableDefaultCNI is true, kind will not install the default CNI setup.
 	// Instead the user should install their own CNI after creating the cluster.
 	DisableDefaultCNI bool
 }
+
+// ClusterIPFamily defines cluster network IP family
+type ClusterIPFamily string
+
+const (
+	// IPv4Family sets ClusterIPFamily to ipv4
+	IPv4Family ClusterIPFamily = "ipv4"
+	// IPv6Family sets ClusterIPFamily to ipv6
+	IPv6Family ClusterIPFamily = "ipv6"
+)

pkg/cluster/config/v1alpha3/types.go

@@ -88,6 +88,8 @@ const (
 
 // Networking contains cluster wide network settings
 type Networking struct {
+	// IPFamily is the network cluster model, currently it can be ipv4 or ipv6
+	IPFamily ClusterIPFamily `json:"ipFamily,omitempty"`
 	// APIServerPort is the listen port on the host for the Kubernetes API Server
 	// Defaults to a random port on the host
 	APIServerPort int32 `json:"apiServerPort,omitempty"`
@@ -99,7 +101,20 @@ type Networking struct {
 	// PodSubnet is the CIDR used for pod IPs
 	// kind will select a default if unspecified
 	PodSubnet string `json:"podSubnet,omitempty"`
+	// ServiceSubnet is the CIDR used for services VIPs
+	// kind will select a default if unspecified for IPv6
+	ServiceSubnet string `json:"serviceSubnet,omitempty"`
 	// If DisableDefaultCNI is true, kind will not install the default CNI setup.
 	// Instead the user should install their own CNI after creating the cluster.
 	DisableDefaultCNI bool `json:"disableDefaultCNI,omitempty"`
 }
+
+// ClusterIPFamily defines cluster network IP family
+type ClusterIPFamily string
+
+const (
+	// IPv4Family sets ClusterIPFamily to ipv4
+	IPv4Family ClusterIPFamily = "ipv4"
+	// IPv6Family sets ClusterIPFamily to ipv6
+	IPv6Family ClusterIPFamily = "ipv6"
+)

pkg/cluster/config/validate.go

@@ -54,6 +54,10 @@ func (c *Cluster) Validate() error {
 	if _, _, err := net.ParseCIDR(c.Networking.PodSubnet); err != nil {
 		errs = append(errs, errors.Wrapf(err, "invalid podSubnet"))
 	}
+	// serviceSubnet should be a valid CIDR
+	if _, _, err := net.ParseCIDR(c.Networking.ServiceSubnet); err != nil {
+		errs = append(errs, errors.Wrapf(err, "invalid serviceSubnet"))
+	}
 
 	if len(errs) > 0 {
 		return util.NewErrors(errs)

pkg/cluster/internal/create/actions/config/config.go

@@ -65,13 +65,19 @@ func (a *Action) Execute(ctx *actions.ActionContext) error {
 		return errors.Wrap(err, "failed to get kubernetes version from node")
 	}
 
-	// get the control plane endpoint
-	controlPlaneEndpoint, err := nodes.GetControlPlaneEndpoint(allNodes)
+	// get the control plane endpoint, in case the cluster has an external load balancer in
+	// front of the control-plane nodes
+	controlPlaneEndpoint, controlPlaneEndpointIPv6, err := nodes.GetControlPlaneEndpoint(allNodes)
 	if err != nil {
 		// TODO(bentheelder): logging here
 		return err
 	}
 
+	// configure the right protocol addresses
+	if ctx.Config.Networking.IPFamily == "ipv6" {
+		controlPlaneEndpoint = controlPlaneEndpointIPv6
+	}
+
 	// create kubeadm init config
 	fns := []func() error{}
 
@@ -83,7 +89,9 @@ func (a *Action) Execute(ctx *actions.ActionContext) error {
 		APIServerAddress:     ctx.Config.Networking.APIServerAddress,
 		Token:                kubeadm.Token,
 		PodSubnet:            ctx.Config.Networking.PodSubnet,
+		ServiceSubnet:        ctx.Config.Networking.ServiceSubnet,
 		ControlPlane:         true,
+		IPv6:                 ctx.Config.Networking.IPFamily == "ipv6",
 	}
 
 	fns = append(fns, func() error {
@@ -193,12 +201,16 @@ func setPatchNames(patches []string, jsonPatches []kustomize.PatchJSON6902) ([]s
 // writeKubeadmConfig writes the kubeadm configuration in the specified node
 func writeKubeadmConfig(cfg *config.Cluster, data kubeadm.ConfigData, node *nodes.Node) error {
 	// get the node ip address
-	nodeAddress, err := node.IP()
+	nodeAddress, nodeAddressIPv6, err := node.IP()
 	if err != nil {
 		return errors.Wrap(err, "failed to get IP for node")
 	}
 
 	data.NodeAddress = nodeAddress
+	// configure the right protocol addresses
+	if cfg.Networking.IPFamily == "ipv6" {
+		data.NodeAddress = nodeAddressIPv6
+	}
 
 	kubeadmConfig, err := getKubeadmConfig(cfg, data)
 

pkg/cluster/internal/create/actions/loadbalancer/loadbalancer.go

@@ -57,6 +57,12 @@ func (a *Action) Execute(ctx *actions.ActionContext) error {
 		return nil
 	}
 
+	// obtain IP family
+	ipv6 := false
+	if ctx.Config.Networking.IPFamily == "ipv6" {
+		ipv6 = true
+	}
+
 	// otherwise notify the user
 	ctx.Status.Start("Configuring the external load balancer ⚖️")
 	defer ctx.Status.End(false)
@@ -71,17 +77,23 @@ func (a *Action) Execute(ctx *actions.ActionContext) error {
 		return err
 	}
 	for _, n := range controlPlaneNodes {
-		controlPlaneIP, err := n.IP()
+		controlPlaneIPv4, controlPlaneIPv6, err := n.IP()
 		if err != nil {
 			return errors.Wrapf(err, "failed to get IP for node %s", n.Name())
 		}
-		backendServers[n.Name()] = fmt.Sprintf("%s:%d", controlPlaneIP, kubeadm.APIServerPort)
+		if controlPlaneIPv4 != "" && !ipv6 {
+			backendServers[n.Name()] = fmt.Sprintf("%s:%d", controlPlaneIPv4, kubeadm.APIServerPort)
+		}
+		if controlPlaneIPv6 != "" && ipv6 {
+			backendServers[n.Name()] = fmt.Sprintf("[%s]:%d", controlPlaneIPv6, kubeadm.APIServerPort)
+		}
 	}
 
 	// create loadbalancer config data
 	loadbalancerConfig, err := loadbalancer.Config(&loadbalancer.ConfigData{
 		ControlPlanePort: loadbalancer.ControlPlanePort,
 		BackendServers:   backendServers,
+		IPv6:             ipv6,
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to generate loadbalancer config data")

pkg/cluster/internal/create/nodes.go

@@ -103,9 +103,16 @@ func createNodeContainers(
 		desiredNode := desiredNode // capture loop variable
 		fns = append(fns, func() error {
 			// create the node into a container (~= docker run -d)
-			_, err := desiredNode.Create(clusterLabel)
+			node, err := desiredNode.Create(clusterLabel)
+			if err != nil {
+				return err
+			}
+			if desiredNode.IPv6 {
+				err = node.EnableIPv6()
+			}
 			return err
 		})
+
 	}
 	if err := concurrent.UntilError(fns); err != nil {
 		return err
@@ -125,6 +132,7 @@ type nodeSpec struct {
 	// TODO(bentheelder): replace with a cri.PortMapping when we have that
 	APIServerPort    int32
 	APIServerAddress string
+	IPv6             bool
 }
 
 func nodesToCreate(cfg *config.Cluster, clusterName string) []nodeSpec {
@@ -154,6 +162,11 @@ func nodesToCreate(cfg *config.Cluster, clusterName string) []nodeSpec {
 		}
 	}
 	isHA := controlPlanes > 1
+	// obtain IP family
+	ipv6 := false
+	if cfg.Networking.IPFamily == "ipv6" {
+		ipv6 = true
+	}
 
 	// add all of the config nodes as desired nodes
 	for _, configNode := range configNodes {
@@ -173,6 +186,7 @@ func nodesToCreate(cfg *config.Cluster, clusterName string) []nodeSpec {
 			ExtraMounts:      configNode.ExtraMounts,
 			APIServerAddress: apiServerAddress,
 			APIServerPort:    apiServerPort,
+			IPv6:             ipv6,
 		})
 	}
 
@@ -186,6 +200,7 @@ func nodesToCreate(cfg *config.Cluster, clusterName string) []nodeSpec {
 			ExtraMounts:      []cri.Mount{},
 			APIServerAddress: cfg.Networking.APIServerAddress,
 			APIServerPort:    cfg.Networking.APIServerPort,
+			IPv6:             ipv6,
 		})
 	}