From 1565e43365c67388a3937ea437f19de068bb988f Mon Sep 17 00:00:00 2001 From: Andy Zhou Date: Sat, 1 Jun 2019 15:19:41 -0700 Subject: [PATCH 1/9] Modify --api-port to take a stirng argument --api-port only takes the port as argument. This patch modifies it to take a string argument, in the form of host:port. --- main.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/main.go b/main.go index 64ebe3d0b..e706c2c77 100644 --- a/main.go +++ b/main.go @@ -96,11 +96,11 @@ func main() { Name: "version", Usage: "Choose the k3s image version", }, - cli.IntFlag{ + cli.StringFlag{ // TODO: only --api-port, -a soon since we want to use --port, -p for the --publish/--add-port functionality Name: "api-port, a, port, p", - Value: 6443, - Usage: "Map the Kubernetes ApiServer port to a local port (Note: --port/-p will be used for arbitrary port mapping as of v2.0.0, use --api-port/-a instead for setting the api port)", + Value: "6443", + Usage: "Specify the Kubernetes cluster API server port (Format: `[host:]port` (Note: --port/-p will be used for arbitrary port mapping as of v2.0.0, use --api-port/-a instead for setting the api port)", }, cli.IntFlag{ Name: "timeout, t", From b97ef230bd340ec166f40ed29cd404e21a82c1a5 Mon Sep 17 00:00:00 2001 From: Andy Zhou Date: Sat, 1 Jun 2019 15:36:29 -0700 Subject: [PATCH 2/9] Refactor The APIs of createServer() and createWorker() takes too many arguments. It is getting harder to maintain. Use a new struct ClusterSpec to make API simpler. This also reduces some code duplications. --- cli/commands.go | 50 ++++++++++++---------------------- cli/container.go | 70 ++++++++++++++++++++++++++++-------------------- 2 files changed, 58 insertions(+), 62 deletions(-) diff --git a/cli/commands.go b/cli/commands.go index d25011114..1430cc672 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -93,12 +93,7 @@ func CreateCluster(c *cli.Context) error { // environment variables env := []string{"K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml"} env = append(env, c.StringSlice("env")...) - - k3sClusterSecret := "" - if c.Int("workers") > 0 { - k3sClusterSecret = fmt.Sprintf("K3S_CLUSTER_SECRET=%s", GenerateRandomString(20)) - env = append(env, k3sClusterSecret) - } + env = append(env, fmt.Sprintf("K3S_CLUSTER_SECRET=%s", GenerateRandomString(20))) // k3s server arguments // TODO: --port will soon be --api-port since we want to re-use --port for arbitrary port mappings @@ -123,19 +118,23 @@ func CreateCluster(c *cli.Context) error { log.Fatal(err) } + clusterSpec := &ClusterSpec{ + AgentArgs: []string{}, + ApiPort: c.String("api-port"), + AutoRestart: c.Bool("auto-restart"), + ClusterName: c.String("name"), + Env: env, + Image: image, + NodeToPortSpecMap: portmap, + PortAutoOffset: c.Int("port-auto-offset"), + ServerArgs: k3sServerArgs, + Verbose: c.GlobalBool("verbose"), + Volumes: c.StringSlice("volume"), + } + // create the server log.Printf("Creating cluster [%s]", c.String("name")) - dockerID, err := createServer( - c.GlobalBool("verbose"), - image, - c.String("api-port"), - k3sServerArgs, - env, - c.String("name"), - c.StringSlice("volume"), - portmap, - c.Bool("auto-restart"), - ) + dockerID, err := createServer(clusterSpec) if err != nil { deleteCluster() return err @@ -187,24 +186,9 @@ func CreateCluster(c *cli.Context) error { // spin up the worker nodes // TODO: do this concurrently in different goroutines if c.Int("workers") > 0 { - k3sWorkerArgs := []string{} - env := []string{k3sClusterSecret} - env = append(env, c.StringSlice("env")...) log.Printf("Booting %s workers for cluster %s", strconv.Itoa(c.Int("workers")), c.String("name")) for i := 0; i < c.Int("workers"); i++ { - workerID, err := createWorker( - c.GlobalBool("verbose"), - image, - k3sWorkerArgs, - env, - c.String("name"), - c.StringSlice("volume"), - i, - c.String("api-port"), - portmap, - c.Int("port-auto-offset"), - c.Bool("auto-restart"), - ) + workerID, err := createWorker(clusterSpec, i) if err != nil { deleteCluster() return err diff --git a/cli/container.go b/cli/container.go index eb097b203..84bd7086c 100644 --- a/cli/container.go +++ b/cli/container.go @@ -20,6 +20,20 @@ import ( "github.com/docker/docker/client" ) +type ClusterSpec struct { + AgentArgs []string + ApiPort string + AutoRestart bool + ClusterName string + Env []string + Image string + NodeToPortSpecMap map[string][]string + PortAutoOffset int + ServerArgs []string + Verbose bool + Volumes []string +} + func startContainer(verbose bool, config *container.Config, hostConfig *container.HostConfig, networkingConfig *network.NetworkingConfig, containerName string) (string, error) { ctx := context.Background() @@ -62,26 +76,25 @@ func startContainer(verbose bool, config *container.Config, hostConfig *containe return resp.ID, nil } -func createServer(verbose bool, image string, apiPort string, args []string, env []string, - name string, volumes []string, nodeToPortSpecMap map[string][]string, autoRestart bool) (string, error) { - log.Printf("Creating server using %s...\n", image) +func createServer(spec *ClusterSpec) (string, error) { + log.Printf("Creating server using %s...\n", spec.Image) containerLabels := make(map[string]string) containerLabels["app"] = "k3d" containerLabels["component"] = "server" containerLabels["created"] = time.Now().Format("2006-01-02 15:04:05") - containerLabels["cluster"] = name + containerLabels["cluster"] = spec.ClusterName - containerName := GetContainerName("server", name, -1) + containerName := GetContainerName("server", spec.ClusterName, -1) // ports to be assigned to the server belong to roles // all, server or - serverPorts, err := MergePortSpecs(nodeToPortSpecMap, "server", containerName) + serverPorts, err := MergePortSpecs(spec.NodeToPortSpecMap, "server", containerName) if err != nil { return "", err } - apiPortSpec := fmt.Sprintf("0.0.0.0:%s:%s/tcp", apiPort, apiPort) + apiPortSpec := fmt.Sprintf("0.0.0.0:%s:%s/tcp", spec.ApiPort, spec.ApiPort) serverPorts = append(serverPorts, apiPortSpec) @@ -95,17 +108,17 @@ func createServer(verbose bool, image string, apiPort string, args []string, env Privileged: true, } - if autoRestart { + if spec.AutoRestart { hostConfig.RestartPolicy.Name = "unless-stopped" } - if len(volumes) > 0 && volumes[0] != "" { - hostConfig.Binds = volumes + if len(spec.Volumes) > 0 && spec.Volumes[0] != "" { + hostConfig.Binds = spec.Volumes } networkingConfig := &network.NetworkingConfig{ EndpointsConfig: map[string]*network.EndpointSettings{ - k3dNetworkName(name): { + k3dNetworkName(spec.ClusterName): { Aliases: []string{containerName}, }, }, @@ -113,13 +126,13 @@ func createServer(verbose bool, image string, apiPort string, args []string, env config := &container.Config{ Hostname: containerName, - Image: image, - Cmd: append([]string{"server"}, args...), + Image: spec.Image, + Cmd: append([]string{"server"}, spec.ServerArgs...), ExposedPorts: serverPublishedPorts.ExposedPorts, - Env: env, + Env: spec.Env, Labels: containerLabels, } - id, err := startContainer(verbose, config, hostConfig, networkingConfig, containerName) + id, err := startContainer(spec.Verbose, config, hostConfig, networkingConfig, containerName) if err != nil { return "", fmt.Errorf("ERROR: couldn't create container %s\n%+v", containerName, err) } @@ -128,21 +141,20 @@ func createServer(verbose bool, image string, apiPort string, args []string, env } // createWorker creates/starts a k3s agent node that connects to the server -func createWorker(verbose bool, image string, args []string, env []string, name string, volumes []string, - postfix int, serverPort string, nodeToPortSpecMap map[string][]string, portAutoOffset int, autoRestart bool) (string, error) { +func createWorker(spec *ClusterSpec, postfix int) (string, error) { containerLabels := make(map[string]string) containerLabels["app"] = "k3d" containerLabels["component"] = "worker" containerLabels["created"] = time.Now().Format("2006-01-02 15:04:05") - containerLabels["cluster"] = name + containerLabels["cluster"] = spec.ClusterName - containerName := GetContainerName("worker", name, postfix) + containerName := GetContainerName("worker", spec.ClusterName, postfix) - env = append(env, fmt.Sprintf("K3S_URL=https://k3d-%s-server:%s", name, serverPort)) + env := append(spec.Env, fmt.Sprintf("K3S_URL=https://k3d-%s-server:%s", spec.ClusterName, spec.ApiPort)) // ports to be assigned to the server belong to roles // all, server or - workerPorts, err := MergePortSpecs(nodeToPortSpecMap, "worker", containerName) + workerPorts, err := MergePortSpecs(spec.NodeToPortSpecMap, "worker", containerName) if err != nil { return "", err } @@ -150,10 +162,10 @@ func createWorker(verbose bool, image string, args []string, env []string, name if err != nil { return "", err } - if portAutoOffset > 0 { + if spec.PortAutoOffset > 0 { // TODO: add some checks before to print a meaningful log message saying that we cannot map multiple container ports // to the same host port without a offset - workerPublishedPorts = workerPublishedPorts.Offset(postfix + portAutoOffset) + workerPublishedPorts = workerPublishedPorts.Offset(postfix + spec.PortAutoOffset) } hostConfig := &container.HostConfig{ @@ -165,17 +177,17 @@ func createWorker(verbose bool, image string, args []string, env []string, name Privileged: true, } - if autoRestart { + if spec.AutoRestart { hostConfig.RestartPolicy.Name = "unless-stopped" } - if len(volumes) > 0 && volumes[0] != "" { - hostConfig.Binds = volumes + if len(spec.Volumes) > 0 && spec.Volumes[0] != "" { + hostConfig.Binds = spec.Volumes } networkingConfig := &network.NetworkingConfig{ EndpointsConfig: map[string]*network.EndpointSettings{ - k3dNetworkName(name): { + k3dNetworkName(spec.ClusterName): { Aliases: []string{containerName}, }, }, @@ -183,13 +195,13 @@ func createWorker(verbose bool, image string, args []string, env []string, name config := &container.Config{ Hostname: containerName, - Image: image, + Image: spec.Image, Env: env, Labels: containerLabels, ExposedPorts: workerPublishedPorts.ExposedPorts, } - id, err := startContainer(verbose, config, hostConfig, networkingConfig, containerName) + id, err := startContainer(spec.Verbose, config, hostConfig, networkingConfig, containerName) if err != nil { return "", fmt.Errorf("ERROR: couldn't start container %s\n%+v", containerName, err) } From f0c1b24e0b22860063e1334253ae445964874dd6 Mon Sep 17 00:00:00 2001 From: Andy Zhou Date: Sat, 1 Jun 2019 16:50:29 -0700 Subject: [PATCH 3/9] Add parseApiPort() Now that --api-port takes a string argument, add a parser function to handle error checking for this argument. --- cli/util.go | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/cli/util.go b/cli/util.go index b7a965163..9a354d2f4 100644 --- a/cli/util.go +++ b/cli/util.go @@ -3,7 +3,9 @@ package run import ( "fmt" "math/rand" + "net" "strings" + "strconv" "time" ) @@ -81,3 +83,39 @@ func ValidateHostname(name string) error { return nil } + +type apiPort struct { + Host string + Port string +} + +func parseApiPort(portSpec string) (*apiPort, error) { + var port *apiPort + split := strings.Split(portSpec, ":") + if len(split) > 2 { + return nil, fmt.Errorf("api-port format error") + } + + if len(split) == 1 { + port = &apiPort{Port: split[0]} + } else { + // Make sure 'host' can be resolved to an IP address + _, err := net.LookupHost(split[0]); + if err != nil { + return nil, err + } + port = &apiPort{Host: split[0], Port: split[1]} + } + + // Verify 'port' is an integer and within port ranges + p, err := strconv.Atoi(port.Port) + if err != nil { + return nil, err + } + + if p < 0 || p > 65535 { + return nil, fmt.Errorf("ERROR: --api-port port value out of range") + } + + return port, nil +} From 1e6ac3cce83297fc349137d5cf45a20f2159673f Mon Sep 17 00:00:00 2001 From: Andy Zhou Date: Sat, 1 Jun 2019 23:58:04 -0700 Subject: [PATCH 4/9] Enhance ClusterSpec Pass the apiPort sturct directly via ClusterSpec. --- cli/commands.go | 2 +- cli/container.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/commands.go b/cli/commands.go index 1430cc672..6952ef57b 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -120,7 +120,7 @@ func CreateCluster(c *cli.Context) error { clusterSpec := &ClusterSpec{ AgentArgs: []string{}, - ApiPort: c.String("api-port"), + ApiPort: apiPort{Host: "0.0.0.0", Port: c.String("api-port")}, AutoRestart: c.Bool("auto-restart"), ClusterName: c.String("name"), Env: env, diff --git a/cli/container.go b/cli/container.go index 84bd7086c..2e4e36089 100644 --- a/cli/container.go +++ b/cli/container.go @@ -22,7 +22,7 @@ import ( type ClusterSpec struct { AgentArgs []string - ApiPort string + ApiPort apiPort AutoRestart bool ClusterName string Env []string @@ -94,7 +94,7 @@ func createServer(spec *ClusterSpec) (string, error) { return "", err } - apiPortSpec := fmt.Sprintf("0.0.0.0:%s:%s/tcp", spec.ApiPort, spec.ApiPort) + apiPortSpec := fmt.Sprintf("0.0.0.0:%s:%s/tcp", spec.ApiPort.Port, spec.ApiPort.Port) serverPorts = append(serverPorts, apiPortSpec) From 58849e4a01b4f9b873a32fdc69187a70c31e7be1 Mon Sep 17 00:00:00 2001 From: Andy Zhou Date: Sun, 2 Jun 2019 11:03:31 -0700 Subject: [PATCH 5/9] Make use of the parseApiPort() function when creating cluster --- cli/commands.go | 28 +++++++++++++++++++++------- cli/container.go | 2 +- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/cli/commands.go b/cli/commands.go index 6952ef57b..62d07a560 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -100,14 +100,28 @@ func CreateCluster(c *cli.Context) error { if c.IsSet("port") { log.Println("INFO: As of v2.0.0 --port will be used for arbitrary port mapping. Please use --api-port/-a instead for configuring the Api Port") } - k3sServerArgs := []string{"--https-listen-port", c.String("api-port")} - if ip, err := getDockerMachineIp(); ip != "" || err != nil { - if err != nil { - return err + apiPort, err := parseApiPort(c.String("api-port")) + if err != nil { + return err + } + + k3sServerArgs := []string{"--https-listen-port", apiPort.Port} + + // When the 'host' is not provided by --api-port, try to fill it using Docker Machine's IP address. + if apiPort.Host == "" { + if apiPort.Host, err = getDockerMachineIp(); apiPort.Host != "" || err != nil { + if err != nil { + return err + } } - log.Printf("Add TLS SAN for %s", ip) - k3sServerArgs = append(k3sServerArgs, "--tls-san", ip) } + + if apiPort.Host != "" { + // Add TLS SAN for non default host name + log.Printf("Add TLS SAN for %s", apiPort.Host) + k3sServerArgs = append(k3sServerArgs, "--tls-san", apiPort.Host) + } + if c.IsSet("server-arg") || c.IsSet("x") { k3sServerArgs = append(k3sServerArgs, c.StringSlice("server-arg")...) } @@ -120,7 +134,7 @@ func CreateCluster(c *cli.Context) error { clusterSpec := &ClusterSpec{ AgentArgs: []string{}, - ApiPort: apiPort{Host: "0.0.0.0", Port: c.String("api-port")}, + ApiPort: *apiPort, AutoRestart: c.Bool("auto-restart"), ClusterName: c.String("name"), Env: env, diff --git a/cli/container.go b/cli/container.go index 2e4e36089..080081bc1 100644 --- a/cli/container.go +++ b/cli/container.go @@ -150,7 +150,7 @@ func createWorker(spec *ClusterSpec, postfix int) (string, error) { containerName := GetContainerName("worker", spec.ClusterName, postfix) - env := append(spec.Env, fmt.Sprintf("K3S_URL=https://k3d-%s-server:%s", spec.ClusterName, spec.ApiPort)) + env := append(spec.Env, fmt.Sprintf("K3S_URL=https://k3d-%s-server:%s", spec.ClusterName, spec.ApiPort.Port)) // ports to be assigned to the server belong to roles // all, server or From a72945af7107775d722b343fb4cdfc14b2cc55f4 Mon Sep 17 00:00:00 2001 From: Andy Zhou Date: Sun, 2 Jun 2019 12:05:21 -0700 Subject: [PATCH 6/9] Enhance docekr port mapping for --api-port --- cli/container.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cli/container.go b/cli/container.go index 080081bc1..a1182fe9f 100644 --- a/cli/container.go +++ b/cli/container.go @@ -94,7 +94,14 @@ func createServer(spec *ClusterSpec) (string, error) { return "", err } - apiPortSpec := fmt.Sprintf("0.0.0.0:%s:%s/tcp", spec.ApiPort.Port, spec.ApiPort.Port) + hostIp := "0.0.0.0" + containerLabels["apihost"] = "localhost" + if spec.ApiPort.Host != "" { + hostIp = spec.ApiPort.Host + containerLabels["apihost"] = spec.ApiPort.Host + } + + apiPortSpec := fmt.Sprintf("%s:%s:%s/tcp", hostIp, spec.ApiPort.Port, spec.ApiPort.Port) serverPorts = append(serverPorts, apiPortSpec) From 1080c0b1575b78eea24da1dea1f9e719030b3230 Mon Sep 17 00:00:00 2001 From: Andy Zhou Date: Sun, 2 Jun 2019 12:06:24 -0700 Subject: [PATCH 7/9] Fix up kubeconfig.yaml with --api-port host --- cli/cluster.go | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/cli/cluster.go b/cli/cluster.go index 96e544d05..aec873a14 100644 --- a/cli/cluster.go +++ b/cli/cluster.go @@ -144,16 +144,20 @@ func createKubeConfigFile(cluster string) error { // and trimming any NULL characters trimBytes := bytes.Trim(readBytes[512:], "\x00") - // If running on a docker machine, replace localhost with - // docker machine's IP - dockerMachineIp, err := getDockerMachineIp() - if err != nil { - return err - } - - if dockerMachineIp != "" { + // Fix up kubeconfig.yaml file. + // + // K3s generates the default kubeconfig.yaml with host name as 'localhost'. + // Change the host name to the name user specified via the --api-port argument. + // + // When user did not specify the host name and when we are running against a remote docker, + // set the host name to remote docker machine's IP address. + // + // Otherwise, the hostname remains as 'localhost' + apiHost := server[0].Labels["apihost"] + + if apiHost != "" { s := string(trimBytes) - s = strings.Replace(s, "localhost", dockerMachineIp, 1) + s = strings.Replace(s, "localhost", apiHost, 1) trimBytes = []byte(s) } _, err = kubeconfigfile.Write(trimBytes) From fc6a01d55a14a189a78c6272a2c08790028a0917 Mon Sep 17 00:00:00 2001 From: Andy Zhou Date: Mon, 3 Jun 2019 17:32:58 -0700 Subject: [PATCH 8/9] Allow --api-port to take host with either name or IP address To allow some flexibilities in how user specifies the --api-port argument. In case the 'host' is an string, We will try to convert it into an IP address for port mapping. IP address is also allowed. --- cli/container.go | 2 +- cli/util.go | 21 +++++++++++---------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/cli/container.go b/cli/container.go index a1182fe9f..db3a1c818 100644 --- a/cli/container.go +++ b/cli/container.go @@ -97,7 +97,7 @@ func createServer(spec *ClusterSpec) (string, error) { hostIp := "0.0.0.0" containerLabels["apihost"] = "localhost" if spec.ApiPort.Host != "" { - hostIp = spec.ApiPort.Host + hostIp = spec.ApiPort.HostIp containerLabels["apihost"] = spec.ApiPort.Host } diff --git a/cli/util.go b/cli/util.go index 9a354d2f4..51f1b18e7 100644 --- a/cli/util.go +++ b/cli/util.go @@ -4,11 +4,17 @@ import ( "fmt" "math/rand" "net" - "strings" "strconv" + "strings" "time" ) +type apiPort struct { + Host string + HostIp string + Port string +} + const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" const ( letterIdxBits = 6 // 6 bits to represent a letter index @@ -84,11 +90,6 @@ func ValidateHostname(name string) error { return nil } -type apiPort struct { - Host string - Port string -} - func parseApiPort(portSpec string) (*apiPort, error) { var port *apiPort split := strings.Split(portSpec, ":") @@ -97,14 +98,14 @@ func parseApiPort(portSpec string) (*apiPort, error) { } if len(split) == 1 { - port = &apiPort{Port: split[0]} + port = &apiPort{Port: split[0]} } else { // Make sure 'host' can be resolved to an IP address - _, err := net.LookupHost(split[0]); + addrs, err := net.LookupHost(split[0]) if err != nil { return nil, err } - port = &apiPort{Host: split[0], Port: split[1]} + port = &apiPort{Host: split[0], HostIp: addrs[0], Port: split[1]} } // Verify 'port' is an integer and within port ranges @@ -113,7 +114,7 @@ func parseApiPort(portSpec string) (*apiPort, error) { return nil, err } - if p < 0 || p > 65535 { + if p < 0 || p > 65535 { return nil, fmt.Errorf("ERROR: --api-port port value out of range") } From fa18c371559af00b886e39c8c6c1de48aa15549b Mon Sep 17 00:00:00 2001 From: Andy Zhou Date: Wed, 5 Jun 2019 09:18:13 -0700 Subject: [PATCH 9/9] Simplify Dockerm machine ip handling Based on review comments from @iwilltry42 --- cli/commands.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cli/commands.go b/cli/commands.go index 62d07a560..a7b505cde 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -109,11 +109,12 @@ func CreateCluster(c *cli.Context) error { // When the 'host' is not provided by --api-port, try to fill it using Docker Machine's IP address. if apiPort.Host == "" { - if apiPort.Host, err = getDockerMachineIp(); apiPort.Host != "" || err != nil { - if err != nil { - return err - } + if apiPort.Host, err = getDockerMachineIp(); err != nil { + return err } + + // IP address is the same as the host + apiPort.HostIp = apiPort.Host } if apiPort.Host != "" {