Merge pull request #75 from andyz-dev/api-port

[Enhancement] Api port

cli/cluster.go

@@ -144,16 +144,20 @@ func createKubeConfigFile(cluster string) error {
 	// and trimming any NULL characters
 	trimBytes := bytes.Trim(readBytes[512:], "\x00")
 
-	// If running on a docker machine, replace localhost with
-	// docker machine's IP
-	dockerMachineIp, err := getDockerMachineIp()
-	if err != nil {
-		return err
-	}
-
-	if dockerMachineIp != "" {
+	// Fix up kubeconfig.yaml file.
+	//
+	// K3s generates the default kubeconfig.yaml with host name as 'localhost'.
+	// Change the host name to the name user specified via the --api-port argument.
+	//
+	// When user did not specify the host name and when we are running against a remote docker,
+	// set the host name to remote docker machine's IP address.
+	//
+	// Otherwise, the hostname remains as 'localhost'
+	apiHost := server[0].Labels["apihost"]
+
+	if apiHost != "" {
 		s := string(trimBytes)
-		s = strings.Replace(s, "localhost", dockerMachineIp, 1)
+		s = strings.Replace(s, "localhost", apiHost, 1)
 		trimBytes = []byte(s)
 	}
 	_, err = kubeconfigfile.Write(trimBytes)

cli/commands.go

@@ -93,26 +93,36 @@ func CreateCluster(c *cli.Context) error {
 	// environment variables
 	env := []string{"K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml"}
 	env = append(env, c.StringSlice("env")...)
-
-	k3sClusterSecret := ""
-	if c.Int("workers") > 0 {
-		k3sClusterSecret = fmt.Sprintf("K3S_CLUSTER_SECRET=%s", GenerateRandomString(20))
-		env = append(env, k3sClusterSecret)
-	}
+	env = append(env, fmt.Sprintf("K3S_CLUSTER_SECRET=%s", GenerateRandomString(20)))
 
 	// k3s server arguments
 	// TODO: --port will soon be --api-port since we want to re-use --port for arbitrary port mappings
 	if c.IsSet("port") {
 		log.Println("INFO: As of v2.0.0 --port will be used for arbitrary port mapping. Please use --api-port/-a instead for configuring the Api Port")
 	}
-	k3sServerArgs := []string{"--https-listen-port", c.String("api-port")}
-	if ip, err := getDockerMachineIp(); ip != "" || err != nil {
-		if err != nil {
+	apiPort, err := parseApiPort(c.String("api-port"))
+	if err != nil {
+		return err
+	}
+
+	k3sServerArgs := []string{"--https-listen-port", apiPort.Port}
+
+	// When the 'host' is not provided by --api-port, try to fill it using Docker Machine's IP address.
+	if apiPort.Host == "" {
+		if apiPort.Host, err = getDockerMachineIp(); err != nil {
 			return err
 		}
-		log.Printf("Add TLS SAN for %s", ip)
-		k3sServerArgs = append(k3sServerArgs, "--tls-san", ip)
+
+		// IP address is the same as the host
+		apiPort.HostIp = apiPort.Host
+	}
+
+	if apiPort.Host != "" {
+		// Add TLS SAN for non default host name
+		log.Printf("Add TLS SAN for %s", apiPort.Host)
+		k3sServerArgs = append(k3sServerArgs, "--tls-san", apiPort.Host)
 	}
+
 	if c.IsSet("server-arg") || c.IsSet("x") {
 		k3sServerArgs = append(k3sServerArgs, c.StringSlice("server-arg")...)
 	}
@@ -123,19 +133,23 @@ func CreateCluster(c *cli.Context) error {
 		log.Fatal(err)
 	}
 
+	clusterSpec := &ClusterSpec{
+		AgentArgs:         []string{},
+		ApiPort:           *apiPort,
+		AutoRestart:       c.Bool("auto-restart"),
+		ClusterName:       c.String("name"),
+		Env:               env,
+		Image:             image,
+		NodeToPortSpecMap: portmap,
+		PortAutoOffset:    c.Int("port-auto-offset"),
+		ServerArgs:        k3sServerArgs,
+		Verbose:           c.GlobalBool("verbose"),
+		Volumes:           c.StringSlice("volume"),
+	}
+
 	// create the server
 	log.Printf("Creating cluster [%s]", c.String("name"))
-	dockerID, err := createServer(
-		c.GlobalBool("verbose"),
-		image,
-		c.String("api-port"),
-		k3sServerArgs,
-		env,
-		c.String("name"),
-		c.StringSlice("volume"),
-		portmap,
-		c.Bool("auto-restart"),
-	)
+	dockerID, err := createServer(clusterSpec)
 	if err != nil {
 		deleteCluster()
 		return err
@@ -187,24 +201,9 @@ func CreateCluster(c *cli.Context) error {
 	// spin up the worker nodes
 	// TODO: do this concurrently in different goroutines
 	if c.Int("workers") > 0 {
-		k3sWorkerArgs := []string{}
-		env := []string{k3sClusterSecret}
-		env = append(env, c.StringSlice("env")...)
 		log.Printf("Booting %s workers for cluster %s", strconv.Itoa(c.Int("workers")), c.String("name"))
 		for i := 0; i < c.Int("workers"); i++ {
-			workerID, err := createWorker(
-				c.GlobalBool("verbose"),
-				image,
-				k3sWorkerArgs,
-				env,
-				c.String("name"),
-				c.StringSlice("volume"),
-				i,
-				c.String("api-port"),
-				portmap,
-				c.Int("port-auto-offset"),
-				c.Bool("auto-restart"),
-			)
+			workerID, err := createWorker(clusterSpec, i)
 			if err != nil {
 				deleteCluster()
 				return err

cli/container.go

@@ -20,6 +20,20 @@ import (
 	"github.com/docker/docker/client"
 )
 
+type ClusterSpec struct {
+	AgentArgs         []string
+	ApiPort           apiPort
+	AutoRestart       bool
+	ClusterName       string
+	Env               []string
+	Image             string
+	NodeToPortSpecMap map[string][]string
+	PortAutoOffset    int
+	ServerArgs        []string
+	Verbose           bool
+	Volumes           []string
+}
+
 func startContainer(verbose bool, config *container.Config, hostConfig *container.HostConfig, networkingConfig *network.NetworkingConfig, containerName string) (string, error) {
 	ctx := context.Background()
 
@@ -62,26 +76,32 @@ func startContainer(verbose bool, config *container.Config, hostConfig *containe
 	return resp.ID, nil
 }
 
-func createServer(verbose bool, image string, apiPort string, args []string, env []string,
-	name string, volumes []string, nodeToPortSpecMap map[string][]string, autoRestart bool) (string, error) {
-	log.Printf("Creating server using %s...\n", image)
+func createServer(spec *ClusterSpec) (string, error) {
+	log.Printf("Creating server using %s...\n", spec.Image)
 
 	containerLabels := make(map[string]string)
 	containerLabels["app"] = "k3d"
 	containerLabels["component"] = "server"
 	containerLabels["created"] = time.Now().Format("2006-01-02 15:04:05")
-	containerLabels["cluster"] = name
+	containerLabels["cluster"] = spec.ClusterName
 
-	containerName := GetContainerName("server", name, -1)
+	containerName := GetContainerName("server", spec.ClusterName, -1)
 
 	// ports to be assigned to the server belong to roles
 	// all, server or <server-container-name>
-	serverPorts, err := MergePortSpecs(nodeToPortSpecMap, "server", containerName)
+	serverPorts, err := MergePortSpecs(spec.NodeToPortSpecMap, "server", containerName)
 	if err != nil {
 		return "", err
 	}
 
-	apiPortSpec := fmt.Sprintf("0.0.0.0:%s:%s/tcp", apiPort, apiPort)
+	hostIp := "0.0.0.0"
+	containerLabels["apihost"] = "localhost"
+	if spec.ApiPort.Host != "" {
+		hostIp = spec.ApiPort.HostIp
+		containerLabels["apihost"] = spec.ApiPort.Host
+	}
+
+	apiPortSpec := fmt.Sprintf("%s:%s:%s/tcp", hostIp, spec.ApiPort.Port, spec.ApiPort.Port)
 
 	serverPorts = append(serverPorts, apiPortSpec)
 
@@ -95,31 +115,31 @@ func createServer(verbose bool, image string, apiPort string, args []string, env
 		Privileged:   true,
 	}
 
-	if autoRestart {
+	if spec.AutoRestart {
 		hostConfig.RestartPolicy.Name = "unless-stopped"
 	}
 
-	if len(volumes) > 0 && volumes[0] != "" {
-		hostConfig.Binds = volumes
+	if len(spec.Volumes) > 0 && spec.Volumes[0] != "" {
+		hostConfig.Binds = spec.Volumes
 	}
 
 	networkingConfig := &network.NetworkingConfig{
 		EndpointsConfig: map[string]*network.EndpointSettings{
-			k3dNetworkName(name): {
+			k3dNetworkName(spec.ClusterName): {
 				Aliases: []string{containerName},
 			},
 		},
 	}
 
 	config := &container.Config{
 		Hostname:     containerName,
-		Image:        image,
-		Cmd:          append([]string{"server"}, args...),
+		Image:        spec.Image,
+		Cmd:          append([]string{"server"}, spec.ServerArgs...),
 		ExposedPorts: serverPublishedPorts.ExposedPorts,
-		Env:          env,
+		Env:          spec.Env,
 		Labels:       containerLabels,
 	}
-	id, err := startContainer(verbose, config, hostConfig, networkingConfig, containerName)
+	id, err := startContainer(spec.Verbose, config, hostConfig, networkingConfig, containerName)
 	if err != nil {
 		return "", fmt.Errorf("ERROR: couldn't create container %s\n%+v", containerName, err)
 	}
@@ -128,32 +148,31 @@ func createServer(verbose bool, image string, apiPort string, args []string, env
 }
 
 // createWorker creates/starts a k3s agent node that connects to the server
-func createWorker(verbose bool, image string, args []string, env []string, name string, volumes []string,
-	postfix int, serverPort string, nodeToPortSpecMap map[string][]string, portAutoOffset int, autoRestart bool) (string, error) {
+func createWorker(spec *ClusterSpec, postfix int) (string, error) {
 	containerLabels := make(map[string]string)
 	containerLabels["app"] = "k3d"
 	containerLabels["component"] = "worker"
 	containerLabels["created"] = time.Now().Format("2006-01-02 15:04:05")
-	containerLabels["cluster"] = name
+	containerLabels["cluster"] = spec.ClusterName
 
-	containerName := GetContainerName("worker", name, postfix)
+	containerName := GetContainerName("worker", spec.ClusterName, postfix)
 
-	env = append(env, fmt.Sprintf("K3S_URL=https://k3d-%s-server:%s", name, serverPort))
+	env := append(spec.Env, fmt.Sprintf("K3S_URL=https://k3d-%s-server:%s", spec.ClusterName, spec.ApiPort.Port))
 
 	// ports to be assigned to the server belong to roles
 	// all, server or <server-container-name>
-	workerPorts, err := MergePortSpecs(nodeToPortSpecMap, "worker", containerName)
+	workerPorts, err := MergePortSpecs(spec.NodeToPortSpecMap, "worker", containerName)
 	if err != nil {
 		return "", err
 	}
 	workerPublishedPorts, err := CreatePublishedPorts(workerPorts)
 	if err != nil {
 		return "", err
 	}
-	if portAutoOffset > 0 {
+	if spec.PortAutoOffset > 0 {
 		// TODO: add some checks before to print a meaningful log message saying that we cannot map multiple container ports
 		// to the same host port without a offset
-		workerPublishedPorts = workerPublishedPorts.Offset(postfix + portAutoOffset)
+		workerPublishedPorts = workerPublishedPorts.Offset(postfix + spec.PortAutoOffset)
 	}
 
 	hostConfig := &container.HostConfig{
@@ -165,31 +184,31 @@ func createWorker(verbose bool, image string, args []string, env []string, name
 		Privileged:   true,
 	}
 
-	if autoRestart {
+	if spec.AutoRestart {
 		hostConfig.RestartPolicy.Name = "unless-stopped"
 	}
 
-	if len(volumes) > 0 && volumes[0] != "" {
-		hostConfig.Binds = volumes
+	if len(spec.Volumes) > 0 && spec.Volumes[0] != "" {
+		hostConfig.Binds = spec.Volumes
 	}
 
 	networkingConfig := &network.NetworkingConfig{
 		EndpointsConfig: map[string]*network.EndpointSettings{
-			k3dNetworkName(name): {
+			k3dNetworkName(spec.ClusterName): {
 				Aliases: []string{containerName},
 			},
 		},
 	}
 
 	config := &container.Config{
 		Hostname:     containerName,
-		Image:        image,
+		Image:        spec.Image,
 		Env:          env,
 		Labels:       containerLabels,
 		ExposedPorts: workerPublishedPorts.ExposedPorts,
 	}
 
-	id, err := startContainer(verbose, config, hostConfig, networkingConfig, containerName)
+	id, err := startContainer(spec.Verbose, config, hostConfig, networkingConfig, containerName)
 	if err != nil {
 		return "", fmt.Errorf("ERROR: couldn't start container %s\n%+v", containerName, err)
 	}

cli/util.go

@@ -3,10 +3,18 @@ package run
 import (
 	"fmt"
 	"math/rand"
+	"net"
+	"strconv"
 	"strings"
 	"time"
 )
 
+type apiPort struct {
+	Host   string
+	HostIp string
+	Port   string
+}
+
 const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
 const (
 	letterIdxBits = 6                    // 6 bits to represent a letter index
@@ -81,3 +89,34 @@ func ValidateHostname(name string) error {
 
 	return nil
 }
+
+func parseApiPort(portSpec string) (*apiPort, error) {
+	var port *apiPort
+	split := strings.Split(portSpec, ":")
+	if len(split) > 2 {
+		return nil, fmt.Errorf("api-port format error")
+	}
+
+	if len(split) == 1 {
+		port = &apiPort{Port: split[0]}
+	} else {
+		// Make sure 'host' can be resolved to an IP address
+		addrs, err := net.LookupHost(split[0])
+		if err != nil {
+			return nil, err
+		}
+		port = &apiPort{Host: split[0], HostIp: addrs[0], Port: split[1]}
+	}
+
+	// Verify 'port' is an integer and within port ranges
+	p, err := strconv.Atoi(port.Port)
+	if err != nil {
+		return nil, err
+	}
+
+	if p < 0 || p > 65535 {
+		return nil, fmt.Errorf("ERROR: --api-port port value out of range")
+	}
+
+	return port, nil
+}

main.go

@@ -96,11 +96,11 @@ func main() {
 					Name:  "version",
 					Usage: "Choose the k3s image version",
 				},
-				cli.IntFlag{
+				cli.StringFlag{
 					// TODO: only --api-port, -a soon since we want to use --port, -p for the --publish/--add-port functionality
 					Name:  "api-port, a, port, p",
-					Value: 6443,
-					Usage: "Map the Kubernetes ApiServer port to a local port (Note: --port/-p will be used for arbitrary port mapping as of v2.0.0, use --api-port/-a instead for setting the api port)",
+					Value: "6443",
+					Usage: "Specify the Kubernetes cluster API server port (Format: `[host:]port` (Note: --port/-p will be used for arbitrary port mapping as of v2.0.0, use --api-port/-a instead for setting the api port)",
 				},
 				cli.IntFlag{
 					Name:  "timeout, t",