[Enhancement] Expose Host IP to services inside the cluster

cmd/cluster/clusterCreate.go

@@ -134,6 +134,7 @@ func NewCmdClusterCreate() *cobra.Command {
 	cmd.Flags().BoolVar(&updateCurrentContext, "switch-context", true, "Directly switch the default kubeconfig's current-context to the new cluster's context (requires --update-default-kubeconfig)")
 	cmd.Flags().BoolVar(&createClusterOpts.DisableLoadBalancer, "no-lb", false, "Disable the creation of a LoadBalancer in front of the server nodes")
 	cmd.Flags().BoolVar(&noRollback, "no-rollback", false, "Disable the automatic rollback actions, if anything goes wrong")
+	cmd.Flags().BoolVar(&createClusterOpts.PrepDisableHostIPInjection, "no-hostip", false, "Disable the automatic injection of the Host IP as 'host.k3d.internal' into the containers and CoreDNS")
 
 	/* Image Importing */
 	cmd.Flags().BoolVar(&createClusterOpts.DisableImageVolume, "no-image-volume", false, "Disable the creation of a volume for importing images")

cmd/util/filter.go

@@ -30,6 +30,8 @@ import (
 
 	k3d "github.com/rancher/k3d/v3/pkg/types"
 
+	"github.com/rancher/k3d/v3/pkg/util"
+
 	"regexp"
 )
 
@@ -99,7 +101,7 @@ func FilterNodes(nodes []*k3d.Node, filters []string) ([]*k3d.Node, error) {
 		}
 
 		// map capturing group names to submatches
-		submatches := mapSubexpNames(filterRegexp.SubexpNames(), match)
+		submatches := util.MapSubexpNames(filterRegexp.SubexpNames(), match)
 
 		// if one of the filters is 'all', we only return this and drop all others
 		if submatches["group"] == "all" {

cmd/util/util.go

@@ -20,13 +20,3 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
 package util
-
-// mapSubexpNames maps regex capturing group names to corresponding matches
-func mapSubexpNames(names, matches []string) map[string]string {
-	//names, matches = names[1:], matches[1:]
-	nameMatchMap := make(map[string]string, len(matches))
-	for index := range names {
-		nameMatchMap[names[index]] = matches[index]
-	}
-	return nameMatchMap
-}

pkg/cluster/cluster.go

@@ -44,10 +44,15 @@ import (
 // - some containerized k3s nodes
 // - a docker network
 func ClusterCreate(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster) error {
+	clusterCreateCtx := ctx
+	clusterPrepCtx := ctx
 	if cluster.CreateClusterOpts.Timeout > 0*time.Second {
-		var cancel context.CancelFunc
-		ctx, cancel = context.WithTimeout(ctx, cluster.CreateClusterOpts.Timeout)
-		defer cancel()
+		var cancelClusterCreateCtx context.CancelFunc
+		var cancelClusterPrepCtx context.CancelFunc
+		clusterCreateCtx, cancelClusterCreateCtx = context.WithTimeout(ctx, cluster.CreateClusterOpts.Timeout)
+		clusterPrepCtx, cancelClusterPrepCtx = context.WithTimeout(ctx, cluster.CreateClusterOpts.Timeout)
+		defer cancelClusterCreateCtx()
+		defer cancelClusterPrepCtx()
 	}
 
 	/*
@@ -74,7 +79,7 @@ func ClusterCreate(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clus
 	}
 
 	// create cluster network or use an existing one
-	networkID, networkExists, err := runtime.CreateNetworkIfNotPresent(ctx, cluster.Network.Name)
+	networkID, networkExists, err := runtime.CreateNetworkIfNotPresent(clusterCreateCtx, cluster.Network.Name)
 	if err != nil {
 		log.Errorln("Failed to create cluster network")
 		return err
@@ -102,7 +107,7 @@ func ClusterCreate(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clus
 	 */
 	if !cluster.CreateClusterOpts.DisableImageVolume {
 		imageVolumeName := fmt.Sprintf("%s-%s-images", k3d.DefaultObjectNamePrefix, cluster.Name)
-		if err := runtime.CreateVolume(ctx, imageVolumeName, map[string]string{k3d.LabelClusterName: cluster.Name}); err != nil {
+		if err := runtime.CreateVolume(clusterCreateCtx, imageVolumeName, map[string]string{k3d.LabelClusterName: cluster.Name}); err != nil {
 			log.Errorf("Failed to create image volume '%s' for cluster '%s'", imageVolumeName, cluster.Name)
 			return err
 		}
@@ -157,7 +162,7 @@ func ClusterCreate(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clus
 
 		// create node
 		log.Infof("Creating node '%s'", node.Name)
-		if err := NodeCreate(ctx, runtime, node, k3d.NodeCreateOpts{}); err != nil {
+		if err := NodeCreate(clusterCreateCtx, runtime, node, k3d.NodeCreateOpts{}); err != nil {
 			log.Errorln("Failed to create node")
 			return err
 		}
@@ -189,13 +194,13 @@ func ClusterCreate(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clus
 		// wait for the initnode to come up before doing anything else
 		for {
 			select {
-			case <-ctx.Done():
+			case <-clusterCreateCtx.Done():
 				log.Errorln("Failed to bring up initializing server node in time")
-				return fmt.Errorf(">>> %w", ctx.Err())
+				return fmt.Errorf(">>> %w", clusterCreateCtx.Err())
 			default:
 			}
 			log.Debugln("Waiting for initializing server node...")
-			logreader, err := runtime.GetNodeLogs(ctx, cluster.InitNode, time.Time{})
+			logreader, err := runtime.GetNodeLogs(clusterCreateCtx, cluster.InitNode, time.Time{})
 			if err != nil {
 				if logreader != nil {
 					logreader.Close()
@@ -219,7 +224,7 @@ func ClusterCreate(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clus
 	}
 
 	// vars to support waiting for server nodes to be ready
-	waitForServerWaitgroup, ctx := errgroup.WithContext(ctx)
+	waitForServerWaitgroup, clusterCreateCtx := errgroup.WithContext(clusterCreateCtx)
 
 	// create all other nodes, but skip the init node
 	for _, node := range cluster.Nodes {
@@ -257,7 +262,7 @@ func ClusterCreate(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clus
 				// TODO: avoid `level=fatal msg="starting kubernetes: preparing server: post join: a configuration change is already in progress (5)"`
 				// ... by scanning for this line in logs and restarting the container in case it appears
 				log.Debugf("Starting to wait for server node '%s'", serverNode.Name)
-				return NodeWaitForLogMessage(ctx, runtime, serverNode, k3d.ReadyLogMessageByRole[k3d.ServerRole], time.Time{})
+				return NodeWaitForLogMessage(clusterCreateCtx, runtime, serverNode, k3d.ReadyLogMessageByRole[k3d.ServerRole], time.Time{})
 			})
 		}
 	}
@@ -322,7 +327,7 @@ func ClusterCreate(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clus
 			}
 			cluster.Nodes = append(cluster.Nodes, lbNode) // append lbNode to list of cluster nodes, so it will be considered during rollback
 			log.Infof("Creating LoadBalancer '%s'", lbNode.Name)
-			if err := NodeCreate(ctx, runtime, lbNode, k3d.NodeCreateOpts{}); err != nil {
+			if err := NodeCreate(clusterCreateCtx, runtime, lbNode, k3d.NodeCreateOpts{}); err != nil {
 				log.Errorln("Failed to create loadbalancer")
 				return err
 			}
@@ -331,7 +336,7 @@ func ClusterCreate(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clus
 					// TODO: avoid `level=fatal msg="starting kubernetes: preparing server: post join: a configuration change is already in progress (5)"`
 					// ... by scanning for this line in logs and restarting the container in case it appears
 					log.Debugf("Starting to wait for loadbalancer node '%s'", lbNode.Name)
-					return NodeWaitForLogMessage(ctx, runtime, lbNode, k3d.ReadyLogMessageByRole[k3d.LoadBalancerRole], time.Time{})
+					return NodeWaitForLogMessage(clusterCreateCtx, runtime, lbNode, k3d.ReadyLogMessageByRole[k3d.LoadBalancerRole], time.Time{})
 				})
 			}
 		} else {
@@ -345,6 +350,36 @@ func ClusterCreate(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clus
 		return fmt.Errorf("Failed to bring up cluster")
 	}
 
+	/**********************************
+	 * Additional Cluster Preparation *
+	 **********************************/
+
+	/*
+	 * Networking Magic
+	 */
+
+	// add /etc/hosts and CoreDNS entry for host.k3d.internal, referring to the host system
+	if !cluster.CreateClusterOpts.PrepDisableHostIPInjection {
+		hostIP, err := GetHostIP(clusterPrepCtx, runtime, cluster)
+		if err != nil {
+			log.Errorln("Failed to get HostIP")
+			return err
+		}
+		hostsEntry := fmt.Sprintf("%s %s", hostIP, k3d.DefaultK3dInternalHostRecord)
+		log.Debugf("Adding extra host entry '%s'...", hostsEntry)
+		for _, node := range cluster.Nodes {
+			if err := runtime.ExecInNode(clusterPrepCtx, node, []string{"sh", "-c", fmt.Sprintf("echo '%s' >> /etc/hosts", hostsEntry)}); err != nil {
+				log.Warnf("Failed to add extra entry '%s' to /etc/hosts in node '%s'", hostsEntry, node.Name)
+			}
+		}
+
+		patchCmd := `test=$(kubectl get cm coredns -n kube-system --template='{{.data.NodeHosts}}' | sed -n -E -e '/[0-9\.]{4,12}\s+host\.k3d\.internal$/!p' -e '$a` + hostsEntry + `' | tr '\n' '^' | xargs -0 printf '{"data": {"NodeHosts":"%s"}}'| sed -E 's%\^%\\n%g') && kubectl patch cm coredns -n kube-system -p="$test"`
+		err = runtime.ExecInNode(clusterPrepCtx, cluster.Nodes[0], []string{"sh", "-c", patchCmd})
+		if err != nil {
+			log.Warnf("Failed to patch CoreDNS ConfigMap to include entry '%s': %+v", hostsEntry, err)
+		}
+	}
+
 	return nil
 }
 

pkg/cluster/host.go

@@ -0,0 +1,102 @@
+/*
+Copyright © 2020 The k3d Author(s)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+package cluster
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"net"
+	"regexp"
+	"runtime"
+
+	rt "github.com/rancher/k3d/v3/pkg/runtimes"
+	k3d "github.com/rancher/k3d/v3/pkg/types"
+	"github.com/rancher/k3d/v3/pkg/util"
+	log "github.com/sirupsen/logrus"
+)
+
+var nsLookupAddressRegexp = regexp.MustCompile(`^Address:\s+(?P<ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$`)
+
+// GetHostIP returns the routable IP address to be able to access services running on the host system from inside the cluster.
+// This depends on the Operating System and the chosen Runtime.
+func GetHostIP(ctx context.Context, rtime rt.Runtime, cluster *k3d.Cluster) (net.IP, error) {
+
+	// Docker Runtime
+	if rtime == rt.Docker {
+
+		log.Debugf("Runtime GOOS: %s", runtime.GOOS)
+
+		// "native" Docker on Linux
+		if runtime.GOOS == "linux" {
+			ip, err := rtime.GetHostIP(ctx, cluster.Network.Name)
+			if err != nil {
+				return nil, err
+			}
+			return ip, nil
+		}
+
+		// Docker (for Desktop) on MacOS or Windows
+		if runtime.GOOS == "windows" || runtime.GOOS == "darwin" {
+			ip, err := resolveHostnameFromInside(ctx, rtime, cluster.Nodes[0], "host.docker.internal")
+			if err != nil {
+				return nil, err
+			}
+			return ip, nil
+		}
+
+		// Catch all other GOOS cases
+		return nil, fmt.Errorf("GetHostIP only implemented for Linux, MacOS (Darwin) and Windows")
+
+	}
+
+	// Catch all other runtime selections
+	return nil, fmt.Errorf("GetHostIP only implemented for the docker runtime")
+
+}
+
+func resolveHostnameFromInside(ctx context.Context, rtime rt.Runtime, node *k3d.Node, hostname string) (net.IP, error) {
+
+	logreader, err := rtime.ExecInNodeGetLogs(ctx, node, []string{"sh", "-c", fmt.Sprintf("nslookup %s", hostname)})
+	if err != nil {
+		return nil, err
+	}
+
+	submatches := map[string]string{}
+	scanner := bufio.NewScanner(logreader)
+	for scanner.Scan() {
+		match := nsLookupAddressRegexp.FindStringSubmatch(scanner.Text())
+		if len(match) == 0 {
+			continue
+		}
+		submatches = util.MapSubexpNames(nsLookupAddressRegexp.SubexpNames(), match)
+		break
+	}
+	if _, ok := submatches["ip"]; !ok {
+		return nil, fmt.Errorf("Failed to read address for '%s' from nslookup response", hostname)
+	}
+
+	log.Debugf("Hostname '%s' -> Address '%s'", hostname, submatches["ip"])
+
+	return net.ParseIP(submatches["ip"]), nil
+
+}

pkg/runtimes/containerd/host.go

@@ -0,0 +1,32 @@
+/*
+Copyright © 2020 The k3d Author(s)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+package containerd
+
+import (
+	"context"
+	"net"
+)
+
+// GetHostIP returns the IP of the containerd host
+func (d Containerd) GetHostIP(ctx context.Context, network string) (net.IP, error) {
+	return nil, nil
+}

pkg/runtimes/containerd/node.go

@@ -23,6 +23,7 @@ THE SOFTWARE.
 package containerd
 
 import (
+	"bufio"
 	"context"
 	"io"
 	"time"
@@ -126,3 +127,8 @@ func (d Containerd) GetNodeLogs(ctx context.Context, node *k3d.Node, since time.
 func (d Containerd) ExecInNode(ctx context.Context, node *k3d.Node, cmd []string) error {
 	return nil
 }
+
+// ExecInNodeGetLogs execs a command inside a node and returns its logreader
+func (d Containerd) ExecInNodeGetLogs(ctx context.Context, node *k3d.Node, cmd []string) (*bufio.Reader, error) {
+	return nil, nil
+}

pkg/runtimes/docker/container.go

@@ -147,7 +147,7 @@ func getNodeContainer(ctx context.Context, node *k3d.Node) (*types.Container, er
 	for k, v := range node.Labels {
 		filters.Add("label", fmt.Sprintf("%s=%s", k, v))
 	}
-        // See https://github.com/moby/moby/issues/29997 for explanation around initial /
+	// See https://github.com/moby/moby/issues/29997 for explanation around initial /
 	filters.Add("name", fmt.Sprintf("^/?%s$", node.Name)) // regex filtering for exact name match
 
 	containers, err := docker.ContainerList(ctx, types.ContainerListOptions{

pkg/runtimes/docker/host.go

@@ -0,0 +1,43 @@
+/*
+Copyright © 2020 The k3d Author(s)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+package docker
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"runtime"
+)
+
+// GetHostIP returns the IP of the docker host (routable from inside the containers)
+func (d Docker) GetHostIP(ctx context.Context, network string) (net.IP, error) {
+	if runtime.GOOS == "linux" {
+		ip, err := GetGatewayIP(ctx, network)
+		if err != nil {
+			return nil, err
+		}
+		return ip, nil
+	}
+
+	return nil, fmt.Errorf("Docker Runtime: GetHostIP only implemented for Linux")
+
+}