From 6fe75640bb4a9cde15fd954b760cfea1a9e33bda Mon Sep 17 00:00:00 2001 From: Andy Zhou Date: Tue, 7 May 2019 00:09:26 -0700 Subject: [PATCH 1/4] Add the PublishedPorts helping class This class holds the parsed results of the --publish options. Its methods helps the create clones of class, with mutations applied. Currently, there are two methods: Offset() change the host ports by a fixed amount. Addport() adds one additional port to the class. --- cli/container.go | 75 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/cli/container.go b/cli/container.go index 0541822f5..89947c1d7 100644 --- a/cli/container.go +++ b/cli/container.go @@ -21,6 +21,81 @@ import ( "github.com/docker/go-connections/nat" ) +type PublishedPorts struct { + ExposedPorts map[nat.Port]struct{} + PortBindings map[nat.Port][]nat.PortBinding +} + +// The factory function for PublishedPorts +func createPublishedPorts(specs []string) (*PublishedPorts, error) { + if len(specs) == 0 { + var newExposedPorts = make(map[nat.Port]struct{}, 1) + var newPortBindings = make(map[nat.Port][]nat.PortBinding, 1) + return &PublishedPorts{ExposedPorts: newExposedPorts, PortBindings: newPortBindings}, nil + } + + newExposedPorts, newPortBindings, err := nat.ParsePortSpecs(specs) + return &PublishedPorts{ExposedPorts: newExposedPorts, PortBindings: newPortBindings}, err +} + +// Create a new PublishedPort structure, with all host ports are changed by a fixed 'offset' +func (p PublishedPorts) Offset(offset int) (*PublishedPorts) { + var newExposedPorts = make(map[nat.Port]struct{}, len(p.ExposedPorts)) + var newPortBindings = make(map[nat.Port][]nat.PortBinding, len(p.PortBindings)) + + for k, v := range p.ExposedPorts { + newExposedPorts[k] = v + } + + for k, v := range p.PortBindings { + bindings := make([]nat.PortBinding, len(v)) + for i, b := range v { + port, _ := nat.ParsePort(b.HostPort) + bindings[i].HostIP = b.HostIP + bindings[i].HostPort = fmt.Sprintf("%d", port + offset) + } + newPortBindings[k] = bindings + } + + return &PublishedPorts{ExposedPorts: newExposedPorts, PortBindings: newPortBindings} +} + +// Create a new PublishedPort struct with one more port, based on 'portSpec' +func (p *PublishedPorts) AddPort(portSpec string) (*PublishedPorts, error) { + portMappings, err := nat.ParsePortSpec(portSpec) + if err != nil { + return nil, err + } + + var newExposedPorts = make(map[nat.Port]struct{}, len(p.ExposedPorts) + 1) + var newPortBindings = make(map[nat.Port][]nat.PortBinding, len(p.PortBindings) + 1) + + // Populate the new maps + for k, v := range p.ExposedPorts { + newExposedPorts[k] = v + } + + for k, v := range p.PortBindings { + newPortBindings[k] = v + } + + // Add new ports + for _, portMapping := range portMappings { + port := portMapping.Port + if _, exists := newExposedPorts[port]; !exists { + newExposedPorts[port] = struct{}{} + } + + bslice, exists := newPortBindings[port]; + if !exists { + bslice = []nat.PortBinding{} + } + newPortBindings[port] = append(bslice, portMapping.Binding) + } + + return &PublishedPorts{ExposedPorts: newExposedPorts, PortBindings: newPortBindings}, nil +} + func startContainer(verbose bool, config *container.Config, hostConfig *container.HostConfig, networkingConfig *network.NetworkingConfig, containerName string) (string, error) { ctx := context.Background() From f70a8b42f7b03bece845a5bf49193b0e16b4b7ee Mon Sep 17 00:00:00 2001 From: Andy Zhou Date: Tue, 7 May 2019 09:07:06 -0700 Subject: [PATCH 2/4] Add support for the --publish option for create subcommand Inspired by the docker CLI, --publish take same input as docker CLI and provides similar functions. For the k3s cluster server node, it behaves the same as docker cli; it exports the k3d server ports to the host ports. Handling for worker nodes will be added in the subsequent patches. This option can be used mutiple times for exposing more ports. --add-port is an alias to this option. --- cli/commands.go | 6 ++++++ cli/container.go | 22 +++++++++------------- main.go | 4 ++++ 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/cli/commands.go b/cli/commands.go index 8735fcb3e..16fdf79c8 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -71,6 +71,11 @@ func CreateCluster(c *cli.Context) error { k3sServerArgs = append(k3sServerArgs, c.StringSlice("server-arg")...) } + publishedPorts, err := createPublishedPorts(c.StringSlice("publish")) + if (err != nil) { + log.Fatalf("ERROR: failed to parse the publish parameter.\n%+v", err) + } + // create the server log.Printf("Creating cluster [%s]", c.String("name")) dockerID, err := createServer( @@ -81,6 +86,7 @@ func CreateCluster(c *cli.Context) error { env, c.String("name"), strings.Split(c.String("volume"), ","), + publishedPorts, ) if err != nil { log.Fatalf("ERROR: failed to create cluster\n%+v", err) diff --git a/cli/container.go b/cli/container.go index 89947c1d7..752ca5523 100644 --- a/cli/container.go +++ b/cli/container.go @@ -138,7 +138,8 @@ func startContainer(verbose bool, config *container.Config, hostConfig *containe return resp.ID, nil } -func createServer(verbose bool, image string, port string, args []string, env []string, name string, volumes []string) (string, error) { +func createServer(verbose bool, image string, port string, args []string, env []string, + name string, volumes []string, pPorts *PublishedPorts) (string, error) { log.Printf("Creating server using %s...\n", image) containerLabels := make(map[string]string) @@ -149,17 +150,14 @@ func createServer(verbose bool, image string, port string, args []string, env [] containerName := fmt.Sprintf("k3d-%s-server", name) - containerPort := nat.Port(fmt.Sprintf("%s/tcp", port)) + apiPortSpec := fmt.Sprintf("0.0.0.0:%s:%s/tcp", port, port) + serverPublishedPorts, err := pPorts.AddPort(apiPortSpec) + if (err != nil) { + log.Fatalf("Error: failed to parse API port spec %s \n%+v", apiPortSpec, err) + } hostConfig := &container.HostConfig{ - PortBindings: nat.PortMap{ - containerPort: []nat.PortBinding{ - { - HostIP: "0.0.0.0", - HostPort: port, - }, - }, - }, + PortBindings: serverPublishedPorts.PortBindings, Privileged: true, } @@ -179,9 +177,7 @@ func createServer(verbose bool, image string, port string, args []string, env [] Hostname: containerName, Image: image, Cmd: append([]string{"server"}, args...), - ExposedPorts: nat.PortSet{ - containerPort: struct{}{}, - }, + ExposedPorts: serverPublishedPorts.ExposedPorts, Env: env, Labels: containerLabels, } diff --git a/main.go b/main.go index ec8b28ded..aa3e40a9e 100644 --- a/main.go +++ b/main.go @@ -55,6 +55,10 @@ func main() { Name: "volume, v", Usage: "Mount one or more volumes into every node of the cluster (Docker notation: `source:destination[,source:destination]`)", }, + cli.StringSliceFlag{ + Name: "publish, add-port", + Usage: "publish k3s node ports to the host (Docker notation: `ip:public:private/proto`, use multiple options to expose more ports)", + }, cli.StringFlag{ Name: "version", Value: version.GetK3sVersion(), From 988fbdbdc5174f6923a9c4021ebfa421891feda1 Mon Sep 17 00:00:00 2001 From: Andy Zhou Date: Mon, 6 May 2019 16:22:00 -0700 Subject: [PATCH 3/4] Refactor createWorker() API Use postfix as int instead string. This make the following patch easier to read. --- cli/commands.go | 2 +- cli/container.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/commands.go b/cli/commands.go index 16fdf79c8..6fd46731d 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -148,7 +148,7 @@ func CreateCluster(c *cli.Context) error { env, c.String("name"), strings.Split(c.String("volume"), ","), - strconv.Itoa(i), + i, c.String("port"), ) if err != nil { diff --git a/cli/container.go b/cli/container.go index 752ca5523..12ef8ebd5 100644 --- a/cli/container.go +++ b/cli/container.go @@ -190,14 +190,14 @@ func createServer(verbose bool, image string, port string, args []string, env [] } // createWorker creates/starts a k3s agent node that connects to the server -func createWorker(verbose bool, image string, args []string, env []string, name string, volumes []string, postfix string, serverPort string) (string, error) { +func createWorker(verbose bool, image string, args []string, env []string, name string, volumes []string, postfix int, serverPort string) (string, error) { containerLabels := make(map[string]string) containerLabels["app"] = "k3d" containerLabels["component"] = "worker" containerLabels["created"] = time.Now().Format("2006-01-02 15:04:05") containerLabels["cluster"] = name - containerName := fmt.Sprintf("k3d-%s-worker-%s", name, postfix) + containerName := fmt.Sprintf("k3d-%s-worker-%d", name, postfix) env = append(env, fmt.Sprintf("K3S_URL=https://k3d-%s-server:%s", name, serverPort)) From 65f2820b3f2562c3df4f9934a1a010184546e496 Mon Sep 17 00:00:00 2001 From: Andy Zhou Date: Tue, 7 May 2019 11:07:56 -0700 Subject: [PATCH 4/4] Support publishing ports for the worker nodes All ports exposed by --publish will also be exported for all worker nodes. The host port will be auto indexed based worker id. For example: with the following command option: k3d create --publish 80:80 --publish 90:90/udp --workers 1 The exposed ports will be: host TCP port 80 -> k3s server TCP 80 host TCP port 90 -> k3s server TCP 90 host UDP port 81 -> k3s worker 0 UDP 80 host UDP port 91 -> k3s worker 0 UDP 90 --- cli/commands.go | 1 + cli/container.go | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/cli/commands.go b/cli/commands.go index 6fd46731d..812dfa775 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -150,6 +150,7 @@ func CreateCluster(c *cli.Context) error { strings.Split(c.String("volume"), ","), i, c.String("port"), + publishedPorts, ) if err != nil { return fmt.Errorf("ERROR: failed to create worker node for cluster %s\n%+v", c.String("name"), err) diff --git a/cli/container.go b/cli/container.go index 12ef8ebd5..843a6ce87 100644 --- a/cli/container.go +++ b/cli/container.go @@ -190,7 +190,8 @@ func createServer(verbose bool, image string, port string, args []string, env [] } // createWorker creates/starts a k3s agent node that connects to the server -func createWorker(verbose bool, image string, args []string, env []string, name string, volumes []string, postfix int, serverPort string) (string, error) { +func createWorker(verbose bool, image string, args []string, env []string, name string, volumes []string, + postfix int, serverPort string, pPorts *PublishedPorts) (string, error) { containerLabels := make(map[string]string) containerLabels["app"] = "k3d" containerLabels["component"] = "worker" @@ -201,11 +202,14 @@ func createWorker(verbose bool, image string, args []string, env []string, name env = append(env, fmt.Sprintf("K3S_URL=https://k3d-%s-server:%s", name, serverPort)) + workerPublishedPorts := pPorts.Offset(postfix + 1) + hostConfig := &container.HostConfig{ Tmpfs: map[string]string{ "/run": "", "/var/run": "", }, + PortBindings: workerPublishedPorts.PortBindings, Privileged: true, } @@ -226,6 +230,7 @@ func createWorker(verbose bool, image string, args []string, env []string, name Image: image, Env: env, Labels: containerLabels, + ExposedPorts: workerPublishedPorts.ExposedPorts, } id, err := startContainer(verbose, config, hostConfig, networkingConfig, containerName)