Add TLS support for libkv

This adds TLS support into the KV store for swarm.  The manage, join,
and list commands all have a new CLI argument, matching the docker engine
discovery backend.  This required adding the tlsconfig utility
package from docker engine.

Here's an example showing re-use of the cluster certs for the KV store:

    swarm manage --tlsverify \
        --tlscacert /etc/docker/ssl/ca.pem
        --tlscert /etc/docker/ssl/cert.pem
        --tlskey /etc/docker/ssl/key.pem
        --cluster-store-opt kv.cacertfile=/etc/docker/ssl/ca.pem
        --cluster-store-opt kv.certfile=/etc/docker/ssl/cert.pem
        --cluster-store-opt kv.keyfile=/etc/docker/ssl/key.pem
        --advertise 192.168.122.47:3376
        etcd://192.168.122.47:2379

Signed-off-by: Daniel Hiltgen <[email protected]>

cli/commands.go

@@ -14,7 +14,7 @@ var (
 			Name:      "list",
 			ShortName: "l",
 			Usage:     "List nodes in a cluster",
-			Flags:     []cli.Flag{flTimeout},
+			Flags:     []cli.Flag{flTimeout, flClusterStoreOpt},
 			Action:    list,
 		},
 		{
@@ -28,14 +28,14 @@ var (
 				flTLS, flTLSCaCert, flTLSCert, flTLSKey, flTLSVerify,
 				flHeartBeat,
 				flEnableCors,
-				flCluster, flClusterOpt},
+				flCluster, flClusterStoreOpt, flClusterOpt},
 			Action: manage,
 		},
 		{
 			Name:      "join",
 			ShortName: "j",
 			Usage:     "join a docker cluster",
-			Flags:     []cli.Flag{flJoinAdvertise, flHeartBeat, flTTL},
+			Flags:     []cli.Flag{flJoinAdvertise, flHeartBeat, flTTL, flClusterStoreOpt},
 			Action:    join,
 		},
 	}

cli/create.go

@@ -13,7 +13,7 @@ func create(c *cli.Context) {
 		log.Fatalf("the `create` command takes no arguments. See '%s create --help'.", c.App.Name)
 	}
 	discovery := &token.Discovery{}
-	discovery.Initialize("", 0, 0)
+	discovery.Initialize("", 0, 0, nil)
 	token, err := discovery.CreateCluster()
 	if err != nil {
 		log.Fatal(err)

cli/flags.go

@@ -112,6 +112,11 @@ var (
 		Usage: "cluster driver options",
 		Value: &cli.StringSlice{},
 	}
+	flClusterStoreOpt = cli.StringSliceFlag{
+		Name:  "cluster-store-opt",
+		Usage: "cluster store options",
+		Value: &cli.StringSlice{},
+	}
 
 	flLeaderElection = cli.BoolFlag{
 		Name:  "replication",

cli/join.go

@@ -42,7 +42,8 @@ func join(c *cli.Context) {
 	if ttl <= hb {
 		log.Fatal("--ttl must be strictly superior to the heartbeat value")
 	}
-	d, err := discovery.New(dflag, hb, ttl)
+
+	d, err := discovery.New(dflag, hb, ttl, getClusterStoreOpt(c))
 	if err != nil {
 		log.Fatal(err)
 	}

cli/list.go

@@ -19,7 +19,7 @@ func list(c *cli.Context) {
 		log.Fatalf("invalid --timeout: %v", err)
 	}
 
-	d, err := discovery.New(dflag, timeout, 0)
+	d, err := discovery.New(dflag, timeout, 0, getClusterStoreOpt(c))
 	if err != nil {
 		log.Fatal(err)
 	}

cli/manage.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"path"
+	"strings"
 	"time"
 
 	log "github.com/Sirupsen/logrus"
@@ -97,7 +98,7 @@ func loadTLSConfig(ca, cert, key string, verify bool) (*tls.Config, error) {
 }
 
 // Initialize the discovery service.
-func createDiscovery(uri string, c *cli.Context) discovery.Discovery {
+func createDiscovery(uri string, c *cli.Context, clusterStoreOpt []string) discovery.Discovery {
 	hb, err := time.ParseDuration(c.String("heartbeat"))
 	if err != nil {
 		log.Fatalf("invalid --heartbeat: %v", err)
@@ -107,14 +108,27 @@ func createDiscovery(uri string, c *cli.Context) discovery.Discovery {
 	}
 
 	// Set up discovery.
-	discovery, err := discovery.New(uri, hb, 0)
+	discovery, err := discovery.New(uri, hb, 0, getClusterStoreOpt(c))
 	if err != nil {
 		log.Fatal(err)
 	}
 
 	return discovery
 }
 
+func getClusterStoreOpt(c *cli.Context) map[string]string {
+	// Process the store options
+	options := map[string]string{}
+	for _, option := range c.StringSlice("cluster-store-opt") {
+		if !strings.Contains(option, "=") {
+			log.Fatal("--cluster-store-opt must contain key=value strings")
+		}
+		kvpair := strings.SplitN(option, "=", 2)
+		options[kvpair[0]] = kvpair[1]
+	}
+	return options
+}
+
 func setupReplication(c *cli.Context, cluster cluster.Cluster, server *api.Server, discovery discovery.Discovery, addr string, leaderTTL time.Duration, tlsConfig *tls.Config) {
 	kvDiscovery, ok := discovery.(*kvdiscovery.Discovery)
 	if !ok {
@@ -222,7 +236,7 @@ func manage(c *cli.Context) {
 	if uri == "" {
 		log.Fatalf("discovery required to manage a cluster. See '%s manage --help'.", c.App.Name)
 	}
-	discovery := createDiscovery(uri, c)
+	discovery := createDiscovery(uri, c, c.StringSlice("cluster-store-opt"))
 	s, err := strategy.New(c.String("strategy"))
 	if err != nil {
 		log.Fatal(err)

discovery/discovery.go

@@ -84,8 +84,8 @@ func (e Entries) Diff(cmp Entries) (Entries, Entries) {
 // The Discovery interface is implemented by Discovery backends which
 // manage swarm host entries.
 type Discovery interface {
-	// Initialize the discovery with URIs, a heartbeat and a ttl.
-	Initialize(string, time.Duration, time.Duration) error
+	// Initialize the discovery with URIs, a heartbeat, a ttl and optional settings.
+	Initialize(string, time.Duration, time.Duration, map[string]string) error
 
 	// Watch the discovery for entry changes.
 	// Returns a channel that will receive changes or an error.
@@ -133,12 +133,12 @@ func parse(rawurl string) (string, string) {
 
 // New returns a new Discovery given a URL, heartbeat and ttl settings.
 // Returns an error if the URL scheme is not supported.
-func New(rawurl string, heartbeat time.Duration, ttl time.Duration) (Discovery, error) {
+func New(rawurl string, heartbeat time.Duration, ttl time.Duration, clusterOpts map[string]string) (Discovery, error) {
 	scheme, uri := parse(rawurl)
 
 	if discovery, exists := discoveries[scheme]; exists {
 		log.WithFields(log.Fields{"name": scheme, "uri": uri}).Debug("Initializing discovery service")
-		err := discovery.Initialize(uri, heartbeat, ttl)
+		err := discovery.Initialize(uri, heartbeat, ttl, clusterOpts)
 		return discovery, err
 	}
 

discovery/file/file.go

@@ -25,7 +25,7 @@ func Init() {
 }
 
 // Initialize is exported
-func (s *Discovery) Initialize(path string, heartbeat time.Duration, ttl time.Duration) error {
+func (s *Discovery) Initialize(path string, heartbeat time.Duration, ttl time.Duration, _ map[string]string) error {
 	s.path = path
 	s.heartbeat = heartbeat
 	return nil

discovery/file/file_test.go

@@ -11,12 +11,12 @@ import (
 
 func TestInitialize(t *testing.T) {
 	d := &Discovery{}
-	d.Initialize("/path/to/file", 1000, 0)
+	d.Initialize("/path/to/file", 1000, 0, nil)
 	assert.Equal(t, d.path, "/path/to/file")
 }
 
 func TestNew(t *testing.T) {
-	d, err := discovery.New("file:///path/to/file", 0, 0)
+	d, err := discovery.New("file:///path/to/file", 0, 0, nil)
 	assert.NoError(t, err)
 	assert.Equal(t, d.(*Discovery).path, "/path/to/file")
 }
@@ -73,7 +73,7 @@ func TestWatch(t *testing.T) {
 
 	// Set up file discovery.
 	d := &Discovery{}
-	d.Initialize(tmp.Name(), 1000, 0)
+	d.Initialize(tmp.Name(), 1000, 0, nil)
 	stopCh := make(chan struct{})
 	ch, errCh := d.Watch(stopCh)
 

discovery/kv/kv.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	log "github.com/Sirupsen/logrus"
+	"github.com/docker/docker/pkg/tlsconfig"
 	"github.com/docker/libkv"
 	"github.com/docker/libkv/store"
 	"github.com/docker/libkv/store/consul"
@@ -47,7 +48,7 @@ func Init() {
 }
 
 // Initialize is exported
-func (s *Discovery) Initialize(uris string, heartbeat time.Duration, ttl time.Duration) error {
+func (s *Discovery) Initialize(uris string, heartbeat time.Duration, ttl time.Duration, clusterOpts map[string]string) error {
 	var (
 		parts = strings.SplitN(uris, "/", 2)
 		addrs = strings.Split(parts[0], ",")
@@ -63,9 +64,34 @@ func (s *Discovery) Initialize(uris string, heartbeat time.Duration, ttl time.Du
 	s.ttl = ttl
 	s.path = path.Join(s.prefix, discoveryPath)
 
+	var config *store.Config
+	if clusterOpts["kv.cacertfile"] != "" && clusterOpts["kv.certfile"] != "" && clusterOpts["kv.keyfile"] != "" {
+		log.Debug("Initializing discovery with TLS")
+		tlsConfig, err := tlsconfig.Client(tlsconfig.Options{
+			CAFile:   clusterOpts["kv.cacertfile"],
+			CertFile: clusterOpts["kv.certfile"],
+			KeyFile:  clusterOpts["kv.keyfile"],
+		})
+		if err != nil {
+			return err
+		}
+		config = &store.Config{
+			// Set ClientTLS to trigger https (bug in libkv/etcd)
+			ClientTLS: &store.ClientTLSConfig{
+				CACertFile: clusterOpts["kv.cacertfile"],
+				CertFile:   clusterOpts["kv.certfile"],
+				KeyFile:    clusterOpts["kv.keyfile"],
+			},
+			// The actual TLS config that will be used
+			TLS: tlsConfig,
+		}
+	} else {
+		log.Debug("Initializing discovery without TLS")
+	}
+
 	// Creates a new store, will ignore options given
 	// if not supported by the chosen store
-	s.store, err = libkv.NewStore(s.backend, addrs, &store.Config{})
+	s.store, err = libkv.NewStore(s.backend, addrs, config)
 	return err
 }