From 765c313b6d231d7eb25f94e6434d22188b57643e Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Wed, 15 May 2019 14:01:47 +0200 Subject: [PATCH] Add IPv6 support This commits adds allows kind to create IPv6 Kubernetes clusters and makes the code ready to implement dual stack support. For simplicity, only one address of each ip family is considered. It adds 2 new options to the v1alpha3 API: ipFamily and serviceSubnet --- pkg/cluster/config/default.go | 22 ++++++- pkg/cluster/config/fuzzer/fuzzer.go | 2 + pkg/cluster/config/types.go | 15 +++++ pkg/cluster/config/v1alpha3/default.go | 22 ++++++- pkg/cluster/config/v1alpha3/types.go | 15 +++++ .../v1alpha3/zz_generated.conversion.go | 4 ++ pkg/cluster/config/validate.go | 4 ++ .../internal/create/actions/config/config.go | 18 +++++- .../actions/loadbalancer/loadbalancer.go | 16 ++++- pkg/cluster/internal/create/nodes.go | 17 +++++- pkg/cluster/internal/kubeadm/config.go | 59 +++++++++++++++++-- pkg/cluster/internal/loadbalancer/config.go | 4 ++ pkg/cluster/nodes/create.go | 6 +- pkg/cluster/nodes/node.go | 49 +++++++++++---- pkg/cluster/nodes/util.go | 21 ++++--- 15 files changed, 234 insertions(+), 40 deletions(-) diff --git a/pkg/cluster/config/default.go b/pkg/cluster/config/default.go index 68bfd57580..c8c557d742 100644 --- a/pkg/cluster/config/default.go +++ b/pkg/cluster/config/default.go @@ -41,14 +41,32 @@ func SetDefaults_Cluster(obj *Cluster) { }, } } - // default to listening on 127.0.0.1:randomPort - // TODO(bentheelder): this defaulting will need to be ipv6 aware as well + if obj.Networking.IPFamily == "" { + obj.Networking.IPFamily = "ipv4" + } + // default to listening on 127.0.0.1:randomPort on ipv4 + // and [::1]:randomPort on ipv6 if obj.Networking.APIServerAddress == "" { obj.Networking.APIServerAddress = "127.0.0.1" + if obj.Networking.IPFamily == "ipv6" { + obj.Networking.APIServerAddress = "::1" + } } // default the pod CIDR if obj.Networking.PodSubnet == "" { obj.Networking.PodSubnet = "10.244.0.0/16" + if obj.Networking.IPFamily == "ipv6" { + obj.Networking.PodSubnet = "fd00:10:244::/64" + } + } + // default the service CIDR using the kubeadm default + // https://github.com/kubernetes/kubernetes/blob/746404f82a28e55e0b76ffa7e40306fb88eb3317/cmd/kubeadm/app/apis/kubeadm/v1beta2/defaults.go#L32 + // Note: kubeadm is doing it already but this simplifies kind's logic + if obj.Networking.ServiceSubnet == "" { + obj.Networking.ServiceSubnet = "10.96.0.0/12" + if obj.Networking.IPFamily == "ipv6" { + obj.Networking.ServiceSubnet = "fd00:10:96::/112" + } } } diff --git a/pkg/cluster/config/fuzzer/fuzzer.go b/pkg/cluster/config/fuzzer/fuzzer.go index 7fed7c8d27..b6c5989f1b 100644 --- a/pkg/cluster/config/fuzzer/fuzzer.go +++ b/pkg/cluster/config/fuzzer/fuzzer.go @@ -42,6 +42,8 @@ func fuzzConfig(obj *config.Cluster, c fuzz.Continue) { }} obj.Networking.APIServerAddress = "127.0.0.1" obj.Networking.PodSubnet = "10.244.0.0/16" + obj.Networking.ServiceSubnet = "10.96.0.0/12" + obj.Networking.IPFamily = "ipv4" } func fuzzNode(obj *config.Node, c fuzz.Continue) { diff --git a/pkg/cluster/config/types.go b/pkg/cluster/config/types.go index f8d127981b..3f0dd4bcbc 100644 --- a/pkg/cluster/config/types.go +++ b/pkg/cluster/config/types.go @@ -88,6 +88,8 @@ const ( // Networking contains cluster wide network settings type Networking struct { + // IPFamily is the network cluster model, currently it can be ipv4 or ipv6 + IPFamily ClusterIPFamily // APIServerPort is the listen port on the host for the Kubernetes API Server // Defaults to a random port on the host APIServerPort int32 @@ -99,7 +101,20 @@ type Networking struct { // PodSubnet is the CIDR used for pod IPs // kind will select a default if unspecified PodSubnet string + // ServiceSubnet is the CIDR used for services VIPs + // kind will select a default if unspecified + ServiceSubnet string // If DisableDefaultCNI is true, kind will not install the default CNI setup. // Instead the user should install their own CNI after creating the cluster. DisableDefaultCNI bool } + +// ClusterIPFamily defines cluster network IP family +type ClusterIPFamily string + +const ( + // IPv4Family sets ClusterIPFamily to ipv4 + IPv4Family ClusterIPFamily = "ipv4" + // IPv6Family sets ClusterIPFamily to ipv6 + IPv6Family ClusterIPFamily = "ipv6" +) diff --git a/pkg/cluster/config/v1alpha3/default.go b/pkg/cluster/config/v1alpha3/default.go index 8e8f78d069..2c284e28b0 100644 --- a/pkg/cluster/config/v1alpha3/default.go +++ b/pkg/cluster/config/v1alpha3/default.go @@ -41,14 +41,32 @@ func SetDefaults_Cluster(obj *Cluster) { }, } } - // default to listening on 127.0.0.1:randomPort - // TODO(bentheelder): this defaulting will need to be ipv6 aware as well + if obj.Networking.IPFamily == "" { + obj.Networking.IPFamily = "ipv4" + } + // default to listening on 127.0.0.1:randomPort on ipv4 + // and [::1]:randomPort on ipv6 if obj.Networking.APIServerAddress == "" { obj.Networking.APIServerAddress = "127.0.0.1" + if obj.Networking.IPFamily == "ipv6" { + obj.Networking.APIServerAddress = "::1" + } } // default the pod CIDR if obj.Networking.PodSubnet == "" { obj.Networking.PodSubnet = "10.244.0.0/16" + if obj.Networking.IPFamily == "ipv6" { + obj.Networking.PodSubnet = "fd00:10:244::/64" + } + } + // default the service CIDR using the kubeadm default + // https://github.com/kubernetes/kubernetes/blob/746404f82a28e55e0b76ffa7e40306fb88eb3317/cmd/kubeadm/app/apis/kubeadm/v1beta2/defaults.go#L32 + // Note: kubeadm is doing it already but this simplifies kind's logic + if obj.Networking.ServiceSubnet == "" { + obj.Networking.ServiceSubnet = "10.96.0.0/12" + if obj.Networking.IPFamily == "ipv6" { + obj.Networking.ServiceSubnet = "fd00:10:96::/112" + } } } diff --git a/pkg/cluster/config/v1alpha3/types.go b/pkg/cluster/config/v1alpha3/types.go index 4a444ffe1f..7f18e9ea61 100644 --- a/pkg/cluster/config/v1alpha3/types.go +++ b/pkg/cluster/config/v1alpha3/types.go @@ -88,6 +88,8 @@ const ( // Networking contains cluster wide network settings type Networking struct { + // IPFamily is the network cluster model, currently it can be ipv4 or ipv6 + IPFamily ClusterIPFamily `json:"ipFamily,omitempty"` // APIServerPort is the listen port on the host for the Kubernetes API Server // Defaults to a random port on the host APIServerPort int32 `json:"apiServerPort,omitempty"` @@ -99,7 +101,20 @@ type Networking struct { // PodSubnet is the CIDR used for pod IPs // kind will select a default if unspecified PodSubnet string `json:"podSubnet,omitempty"` + // ServiceSubnet is the CIDR used for services VIPs + // kind will select a default if unspecified for IPv6 + ServiceSubnet string `json:"serviceSubnet,omitempty"` // If DisableDefaultCNI is true, kind will not install the default CNI setup. // Instead the user should install their own CNI after creating the cluster. DisableDefaultCNI bool `json:"disableDefaultCNI,omitempty"` } + +// ClusterIPFamily defines cluster network IP family +type ClusterIPFamily string + +const ( + // IPv4Family sets ClusterIPFamily to ipv4 + IPv4Family ClusterIPFamily = "ipv4" + // IPv6Family sets ClusterIPFamily to ipv6 + IPv6Family ClusterIPFamily = "ipv6" +) diff --git a/pkg/cluster/config/v1alpha3/zz_generated.conversion.go b/pkg/cluster/config/v1alpha3/zz_generated.conversion.go index 4625f566b4..05152a999a 100644 --- a/pkg/cluster/config/v1alpha3/zz_generated.conversion.go +++ b/pkg/cluster/config/v1alpha3/zz_generated.conversion.go @@ -101,9 +101,11 @@ func Convert_config_Cluster_To_v1alpha3_Cluster(in *config.Cluster, out *Cluster } func autoConvert_v1alpha3_Networking_To_config_Networking(in *Networking, out *config.Networking, s conversion.Scope) error { + out.IPFamily = config.ClusterIPFamily(in.IPFamily) out.APIServerPort = in.APIServerPort out.APIServerAddress = in.APIServerAddress out.PodSubnet = in.PodSubnet + out.ServiceSubnet = in.ServiceSubnet out.DisableDefaultCNI = in.DisableDefaultCNI return nil } @@ -114,9 +116,11 @@ func Convert_v1alpha3_Networking_To_config_Networking(in *Networking, out *confi } func autoConvert_config_Networking_To_v1alpha3_Networking(in *config.Networking, out *Networking, s conversion.Scope) error { + out.IPFamily = ClusterIPFamily(in.IPFamily) out.APIServerPort = in.APIServerPort out.APIServerAddress = in.APIServerAddress out.PodSubnet = in.PodSubnet + out.ServiceSubnet = in.ServiceSubnet out.DisableDefaultCNI = in.DisableDefaultCNI return nil } diff --git a/pkg/cluster/config/validate.go b/pkg/cluster/config/validate.go index 744fd87958..58be661e47 100644 --- a/pkg/cluster/config/validate.go +++ b/pkg/cluster/config/validate.go @@ -54,6 +54,10 @@ func (c *Cluster) Validate() error { if _, _, err := net.ParseCIDR(c.Networking.PodSubnet); err != nil { errs = append(errs, errors.Wrapf(err, "invalid podSubnet")) } + // serviceSubnet should be a valid CIDR + if _, _, err := net.ParseCIDR(c.Networking.ServiceSubnet); err != nil { + errs = append(errs, errors.Wrapf(err, "invalid serviceSubnet")) + } if len(errs) > 0 { return util.NewErrors(errs) diff --git a/pkg/cluster/internal/create/actions/config/config.go b/pkg/cluster/internal/create/actions/config/config.go index 410486d803..1d66629eec 100644 --- a/pkg/cluster/internal/create/actions/config/config.go +++ b/pkg/cluster/internal/create/actions/config/config.go @@ -65,13 +65,19 @@ func (a *Action) Execute(ctx *actions.ActionContext) error { return errors.Wrap(err, "failed to get kubernetes version from node") } - // get the control plane endpoint - controlPlaneEndpoint, err := nodes.GetControlPlaneEndpoint(allNodes) + // get the control plane endpoint, in case the cluster has an external load balancer in + // front of the control-plane nodes + controlPlaneEndpoint, controlPlaneEndpointIPv6, err := nodes.GetControlPlaneEndpoint(allNodes) if err != nil { // TODO(bentheelder): logging here return err } + // configure the right protocol addresses + if ctx.Config.Networking.IPFamily == "ipv6" { + controlPlaneEndpoint = controlPlaneEndpointIPv6 + } + // create kubeadm init config fns := []func() error{} @@ -83,7 +89,9 @@ func (a *Action) Execute(ctx *actions.ActionContext) error { APIServerAddress: ctx.Config.Networking.APIServerAddress, Token: kubeadm.Token, PodSubnet: ctx.Config.Networking.PodSubnet, + ServiceSubnet: ctx.Config.Networking.ServiceSubnet, ControlPlane: true, + IPv6: ctx.Config.Networking.IPFamily == "ipv6", } fns = append(fns, func() error { @@ -193,12 +201,16 @@ func setPatchNames(patches []string, jsonPatches []kustomize.PatchJSON6902) ([]s // writeKubeadmConfig writes the kubeadm configuration in the specified node func writeKubeadmConfig(cfg *config.Cluster, data kubeadm.ConfigData, node *nodes.Node) error { // get the node ip address - nodeAddress, err := node.IP() + nodeAddress, nodeAddressIPv6, err := node.IP() if err != nil { return errors.Wrap(err, "failed to get IP for node") } data.NodeAddress = nodeAddress + // configure the right protocol addresses + if cfg.Networking.IPFamily == "ipv6" { + data.NodeAddress = nodeAddressIPv6 + } kubeadmConfig, err := getKubeadmConfig(cfg, data) diff --git a/pkg/cluster/internal/create/actions/loadbalancer/loadbalancer.go b/pkg/cluster/internal/create/actions/loadbalancer/loadbalancer.go index 502d3acac2..3f79cadc28 100644 --- a/pkg/cluster/internal/create/actions/loadbalancer/loadbalancer.go +++ b/pkg/cluster/internal/create/actions/loadbalancer/loadbalancer.go @@ -57,6 +57,12 @@ func (a *Action) Execute(ctx *actions.ActionContext) error { return nil } + // obtain IP family + ipv6 := false + if ctx.Config.Networking.IPFamily == "ipv6" { + ipv6 = true + } + // otherwise notify the user ctx.Status.Start("Configuring the external load balancer ⚖️") defer ctx.Status.End(false) @@ -71,17 +77,23 @@ func (a *Action) Execute(ctx *actions.ActionContext) error { return err } for _, n := range controlPlaneNodes { - controlPlaneIP, err := n.IP() + controlPlaneIPv4, controlPlaneIPv6, err := n.IP() if err != nil { return errors.Wrapf(err, "failed to get IP for node %s", n.Name()) } - backendServers[n.Name()] = fmt.Sprintf("%s:%d", controlPlaneIP, kubeadm.APIServerPort) + if controlPlaneIPv4 != "" && !ipv6 { + backendServers[n.Name()] = fmt.Sprintf("%s:%d", controlPlaneIPv4, kubeadm.APIServerPort) + } + if controlPlaneIPv6 != "" && ipv6 { + backendServers[n.Name()] = fmt.Sprintf("[%s]:%d", controlPlaneIPv6, kubeadm.APIServerPort) + } } // create loadbalancer config data loadbalancerConfig, err := loadbalancer.Config(&loadbalancer.ConfigData{ ControlPlanePort: loadbalancer.ControlPlanePort, BackendServers: backendServers, + IPv6: ipv6, }) if err != nil { return errors.Wrap(err, "failed to generate loadbalancer config data") diff --git a/pkg/cluster/internal/create/nodes.go b/pkg/cluster/internal/create/nodes.go index 678e776ffa..08916d1db5 100644 --- a/pkg/cluster/internal/create/nodes.go +++ b/pkg/cluster/internal/create/nodes.go @@ -103,9 +103,16 @@ func createNodeContainers( desiredNode := desiredNode // capture loop variable fns = append(fns, func() error { // create the node into a container (~= docker run -d) - _, err := desiredNode.Create(clusterLabel) + node, err := desiredNode.Create(clusterLabel) + if err != nil { + return err + } + if desiredNode.IPv6 { + err = node.EnableIPv6() + } return err }) + } if err := concurrent.UntilError(fns); err != nil { return err @@ -125,6 +132,7 @@ type nodeSpec struct { // TODO(bentheelder): replace with a cri.PortMapping when we have that APIServerPort int32 APIServerAddress string + IPv6 bool } func nodesToCreate(cfg *config.Cluster, clusterName string) []nodeSpec { @@ -154,6 +162,11 @@ func nodesToCreate(cfg *config.Cluster, clusterName string) []nodeSpec { } } isHA := controlPlanes > 1 + // obtain IP family + ipv6 := false + if cfg.Networking.IPFamily == "ipv6" { + ipv6 = true + } // add all of the config nodes as desired nodes for _, configNode := range configNodes { @@ -173,6 +186,7 @@ func nodesToCreate(cfg *config.Cluster, clusterName string) []nodeSpec { ExtraMounts: configNode.ExtraMounts, APIServerAddress: apiServerAddress, APIServerPort: apiServerPort, + IPv6: ipv6, }) } @@ -186,6 +200,7 @@ func nodesToCreate(cfg *config.Cluster, clusterName string) []nodeSpec { ExtraMounts: []cri.Mount{}, APIServerAddress: cfg.Networking.APIServerAddress, APIServerPort: cfg.Networking.APIServerPort, + IPv6: ipv6, }) } diff --git a/pkg/cluster/internal/kubeadm/config.go b/pkg/cluster/internal/kubeadm/config.go index b36eba85ca..f12df31038 100644 --- a/pkg/cluster/internal/kubeadm/config.go +++ b/pkg/cluster/internal/kubeadm/config.go @@ -45,6 +45,10 @@ type ConfigData struct { Token string // The subnet used for pods PodSubnet string + // The subnet used for services + ServiceSubnet string + // IPv4 values take precedence over IPv6 by default, if true set IPv6 default values + IPv6 bool // DerivedConfigData is populated by Derive() // These auto-generated fields are available to Config templates, // but not meant to be set by hand @@ -104,9 +108,14 @@ apiServerExtraVolumes: # on docker for mac we have to expose the api server via port forward, # so we need to ensure the cert is valid for localhost so we can talk # to the cluster after rewriting the kubeconfig to point to localhost -apiServerCertSANs: [localhost, {{.APIServerAddress}}] +apiServerCertSANs: [localhost, "{{.APIServerAddress}}"] kubeletConfiguration: baseConfig: + # configure ipv6 addresses in IPv6 mode + {{ if .IPv6 -}} + address: "::" + healthzBindAddress: "::" + {{- end }} # disable disk resource management by default # kubelet will see the host disk that the inner container runtime # is ultimately backed by and attempt to recover disk space. @@ -151,6 +160,9 @@ metadata: kubernetesVersion: {{.KubernetesVersion}} clusterName: "{{.ClusterName}}" controlPlaneEndpoint: {{ .ControlPlaneEndpoint }} +networking: + podSubnet: "{{ .PodSubnet }}" + serviceSubnet: "{{ .ServiceSubnet }}" # we need nsswitch.conf so we use /etc/hosts # https://github.com/kubernetes/kubernetes/issues/69195 apiServerExtraVolumes: @@ -162,7 +174,7 @@ apiServerExtraVolumes: # on docker for mac we have to expose the api server via port forward, # so we need to ensure the cert is valid for localhost so we can talk # to the cluster after rewriting the kubeconfig to point to localhost -apiServerCertSANs: [localhost, {{.APIServerAddress}}] +apiServerCertSANs: [localhost, "{{.APIServerAddress}}"] controllerManagerExtraArgs: enable-hostpath-provisioner: "true" networking: @@ -210,6 +222,11 @@ apiVersion: kubelet.config.k8s.io/v1beta1 kind: KubeletConfiguration metadata: name: config +# configure ipv6 addresses in IPv6 mode +{{ if .IPv6 -}} +address: "::" +healthzBindAddress: "::" +{{- end }} # disable disk resource management by default # kubelet will see the host disk that the inner container runtime # is ultimately backed by and attempt to recover disk space. we don't want that. @@ -239,12 +256,24 @@ controlPlaneEndpoint: {{ .ControlPlaneEndpoint }} # so we need to ensure the cert is valid for localhost so we can talk # to the cluster after rewriting the kubeconfig to point to localhost apiServer: - certSANs: [localhost, {{.APIServerAddress}}] + certSANs: [localhost, "{{.APIServerAddress}}"] controllerManager: extraArgs: enable-hostpath-provisioner: "true" + # configure ipv6 default addresses for IPv6 clusters + {{ if .IPv6 -}} + bind-address: "::" + {{- end }} +scheduler: + extraArgs: + # configure ipv6 default addresses for IPv6 clusters + {{ if .IPv6 -}} + address: "::" + bind-address: "::1" + {{- end }} networking: podSubnet: "{{ .PodSubnet }}" + serviceSubnet: "{{ .ServiceSubnet }}" --- apiVersion: kubeadm.k8s.io/v1beta1 kind: InitConfiguration @@ -290,6 +319,11 @@ apiVersion: kubelet.config.k8s.io/v1beta1 kind: KubeletConfiguration metadata: name: config +# configure ipv6 addresses in IPv6 mode +{{ if .IPv6 -}} +address: "::" +healthzBindAddress: "::" +{{- end }} # disable disk resource management by default # kubelet will see the host disk that the inner container runtime # is ultimately backed by and attempt to recover disk space. we don't want that. @@ -319,12 +353,24 @@ controlPlaneEndpoint: {{ .ControlPlaneEndpoint }} # so we need to ensure the cert is valid for localhost so we can talk # to the cluster after rewriting the kubeconfig to point to localhost apiServer: - certSANs: [localhost, {{.APIServerAddress}}] + certSANs: [localhost, "{{.APIServerAddress}}"] controllerManager: extraArgs: enable-hostpath-provisioner: "true" + # configure ipv6 default addresses for IPv6 clusters + {{ if .IPv6 -}} + bind-address: "::" + {{- end }} +scheduler: + extraArgs: + # configure ipv6 default addresses for IPv6 clusters + {{ if .IPv6 -}} + address: "::" + bind-address: "::1" + {{- end }} networking: podSubnet: "{{ .PodSubnet }}" + serviceSubnet: "{{ .ServiceSubnet }}" --- apiVersion: kubeadm.k8s.io/v1beta2 kind: InitConfiguration @@ -370,6 +416,11 @@ apiVersion: kubelet.config.k8s.io/v1beta1 kind: KubeletConfiguration metadata: name: config +# configure ipv6 addresses in IPv6 mode +{{ if .IPv6 -}} +address: "::" +healthzBindAddress: "::" +{{- end }} # disable disk resource management by default # kubelet will see the host disk that the inner container runtime # is ultimately backed by and attempt to recover disk space. we don't want that. diff --git a/pkg/cluster/internal/loadbalancer/config.go b/pkg/cluster/internal/loadbalancer/config.go index cfcdefadcc..380ea7a43a 100644 --- a/pkg/cluster/internal/loadbalancer/config.go +++ b/pkg/cluster/internal/loadbalancer/config.go @@ -27,6 +27,7 @@ import ( type ConfigData struct { ControlPlanePort int BackendServers map[string]string + IPv6 bool } // DefaultConfigTemplate is the loadbalancer config template @@ -42,6 +43,9 @@ stream { server { listen {{ .ControlPlanePort }}; + {{ if .IPv6 -}} + listen [::]:{{ .ControlPlanePort }}; + {{- end }} proxy_pass tcp_backend; } } diff --git a/pkg/cluster/nodes/create.go b/pkg/cluster/nodes/create.go index 84a9811bab..8dc6db5250 100644 --- a/pkg/cluster/nodes/create.go +++ b/pkg/cluster/nodes/create.go @@ -59,11 +59,12 @@ func CreateControlPlaneNode(name, image, clusterLabel, listenAddress string, por port = p } + portMapping := net.JoinHostPort(listenAddress, fmt.Sprintf("%d", port)) + fmt.Sprintf(":%d", kubeadm.APIServerPort) node, err = createNode( name, image, clusterLabel, constants.ControlPlaneNodeRoleValue, mounts, // publish selected port for the API server "--expose", fmt.Sprintf("%d", port), - "-p", fmt.Sprintf("%s:%d:%d", listenAddress, port, kubeadm.APIServerPort), + "-p", portMapping, ) if err != nil { return node, err @@ -90,11 +91,12 @@ func CreateExternalLoadBalancerNode(name, image, clusterLabel, listenAddress str port = p } + portMapping := net.JoinHostPort(listenAddress, fmt.Sprintf("%d", port)) + fmt.Sprintf(":%d", loadbalancer.ControlPlanePort) node, err = createNode(name, image, clusterLabel, constants.ExternalLoadBalancerNodeRoleValue, nil, // publish selected port for the control plane "--expose", fmt.Sprintf("%d", port), - "-p", fmt.Sprintf("%s:%d:%d", listenAddress, port, loadbalancer.ControlPlanePort), + "-p", portMapping, ) if err != nil { return node, err diff --git a/pkg/cluster/nodes/node.go b/pkg/cluster/nodes/node.go index 0b44d0610f..14ac0473bb 100644 --- a/pkg/cluster/nodes/node.go +++ b/pkg/cluster/nodes/node.go @@ -62,7 +62,8 @@ func (n *Node) Command(command string, args ...string) exec.Cmd { type nodeCache struct { mu sync.RWMutex kubernetesVersion string - ip string + ipv4 string + ipv6 string ports map[int32]int32 role string } @@ -79,10 +80,10 @@ func (cache *nodeCache) KubeVersion() string { return cache.kubernetesVersion } -func (cache *nodeCache) IP() string { +func (cache *nodeCache) IP() (string, string) { cache.mu.RLock() defer cache.mu.RUnlock() - return cache.ip + return cache.ipv4, cache.ipv6 } func (cache *nodeCache) HostPort(p int32) (int32, bool) { @@ -147,25 +148,30 @@ func (n *Node) KubeVersion() (version string, err error) { } // IP returns the IP address of the node -func (n *Node) IP() (ip string, err error) { +func (n *Node) IP() (ipv4 string, ipv6 string, err error) { // use the cached version first - cachedIP := n.cache.IP() - if cachedIP != "" { - return cachedIP, nil + cachedIPv4, cachedIPv6 := n.cache.IP() + // TODO: this assumes there are always ipv4 and ipv6 cached addresses + if cachedIPv4 != "" && cachedIPv6 != "" { + return cachedIPv4, cachedIPv6, nil } // retrive the IP address of the node using docker inspect - lines, err := docker.Inspect(n.name, "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}") + lines, err := docker.Inspect(n.name, "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}") if err != nil { - return "", errors.Wrap(err, "failed to get file") + return "", "", errors.Wrap(err, "failed to get container details") } if len(lines) != 1 { - return "", errors.Errorf("file should only be one line, got %d lines", len(lines)) + return "", "", errors.Errorf("file should only be one line, got %d lines", len(lines)) + } + ips := strings.Split(lines[0], ",") + if len(ips) != 2 { + return "", "", errors.Errorf("container addresses should have 2 values, got %d values", len(ips)) } - ip = lines[0] n.cache.set(func(cache *nodeCache) { - cache.ip = ip + cache.ipv4 = ips[0] + cache.ipv6 = ips[1] }) - return ip, nil + return ips[0], ips[1], nil } // Ports returns a specific port mapping for the node @@ -275,3 +281,20 @@ func getProxyDetails() proxyDetails { } return details } + +// EnableIPv6 enables IPv6 inside the node container and in the inner docker daemon +func (n *Node) EnableIPv6() error { + // enable ipv6 + cmd := n.Command("sysctl", "net.ipv6.conf.all.disable_ipv6=0") + err := exec.RunLoggingOutputOnFail(cmd) + if err != nil { + return errors.Wrap(err, "failed to enable ipv6") + } + // enable ipv6 forwarding + cmd = n.Command("sysctl", "net.ipv6.conf.all.forwarding=1") + err = exec.RunLoggingOutputOnFail(cmd) + if err != nil { + return errors.Wrap(err, "failed to enable ipv6 forwarding") + } + return nil +} diff --git a/pkg/cluster/nodes/util.go b/pkg/cluster/nodes/util.go index dd079b0d76..a1680310d3 100644 --- a/pkg/cluster/nodes/util.go +++ b/pkg/cluster/nodes/util.go @@ -25,29 +25,28 @@ import ( "sigs.k8s.io/kind/pkg/cluster/internal/loadbalancer" ) -// GetControlPlaneEndpoint returns the control plane endpoint -// in case the cluster has an external load balancer it returns its ip, -// otherwise return the bootstrap node ip. -func GetControlPlaneEndpoint(allNodes []Node) (string, error) { +// GetControlPlaneEndpoint returns the control plane endpoints for IPv4 and IPv6 +// in case the cluster has an external load balancer in front of the control-plane nodes, +// otherwise return the bootstrap node IPs +func GetControlPlaneEndpoint(allNodes []Node) (string, string, error) { node, err := ExternalLoadBalancerNode(allNodes) if err != nil { - return "", err + return "", "", err } controlPlanePort := loadbalancer.ControlPlanePort // if there is no external load balancer use the bootstrap node if node == nil { node, err = BootstrapControlPlaneNode(allNodes) if err != nil { - return "", err + return "", "", err } controlPlanePort = kubeadm.APIServerPort } - // get the IP and port for the load balancer - controlPlaneIP, err := node.IP() + // gets the control plane IP addresses + controlPlaneIPv4, controlPlaneIPv6, err := node.IP() if err != nil { - return "", errors.Wrapf(err, "failed to get IP for node: %s", node.Name()) + return "", "", errors.Wrapf(err, "failed to get IPs for node: %s", node.Name()) } - - return fmt.Sprintf("%s:%d", controlPlaneIP, controlPlanePort), nil + return fmt.Sprintf("%s:%d", controlPlaneIPv4, controlPlanePort), fmt.Sprintf("[%s]:%d", controlPlaneIPv6, controlPlanePort), nil }