Plugable secret backend

This commit extends SwarmKit secret management with pluggable secret
backends support.

Following previous commits:
1. moby/swarmkit@eebac27
2. moby/moby@08f7cf0

Added driver parameter to `docker secret` command.
Specifically:
1. `docker secret create [secret_name] --driver [driver_name]`
2.  Displaying the driver in
```
    $ docker secret ls
    $ docker secret inspect [secret_name]
    $ docker secret inspect [secret_name] -pretty
```
3. Add driver parameter to secrets in `docker-compose`.

There is a bug in serialization of the secret data. Handled in
moby/moby#34157.

Signed-off-by: Liron Levin <[email protected]>

cli/command/formatter/secret.go

@@ -12,19 +12,20 @@ import (
 )
 
 const (
-	defaultSecretTableFormat           = "table {{.ID}}\t{{.Name}}\t{{.CreatedAt}}\t{{.UpdatedAt}}"
+	defaultSecretTableFormat           = "table {{.ID}}\t{{.Name}}\t{{.Driver}}\t{{.CreatedAt}}\t{{.UpdatedAt}}"
 	secretIDHeader                     = "ID"
 	secretCreatedHeader                = "CREATED"
 	secretUpdatedHeader                = "UPDATED"
-	secretInspectPrettyTemplate Format = `ID:			{{.ID}}
-Name:			{{.Name}}
+	secretInspectPrettyTemplate Format = `ID:              {{.ID}}
+Name:              {{.Name}}
 {{- if .Labels }}
 Labels:
 {{- range $k, $v := .Labels }}
  - {{ $k }}{{if $v }}={{ $v }}{{ end }}
 {{- end }}{{ end }}
-Created at:            	{{.CreatedAt}}
-Updated at:            	{{.UpdatedAt}}`
+Driver:            {{.Driver}}
+Created at:        {{.CreatedAt}}
+Updated at:        {{.UpdatedAt}}`
 )
 
 // NewSecretFormat returns a Format for rendering using a secret Context
@@ -61,6 +62,7 @@ func newSecretContext() *secretContext {
 	sCtx.header = map[string]string{
 		"ID":        secretIDHeader,
 		"Name":      nameHeader,
+		"Driver":    driverHeader,
 		"CreatedAt": secretCreatedHeader,
 		"UpdatedAt": secretUpdatedHeader,
 		"Labels":    labelsHeader,
@@ -89,6 +91,13 @@ func (c *secretContext) CreatedAt() string {
 	return units.HumanDuration(time.Now().UTC().Sub(c.s.Meta.CreatedAt)) + " ago"
 }
 
+func (c *secretContext) Driver() string {
+	if c.s.Spec.Driver == nil {
+		return ""
+	}
+	return c.s.Spec.Driver.Name
+}
+
 func (c *secretContext) UpdatedAt() string {
 	return units.HumanDuration(time.Now().UTC().Sub(c.s.Meta.UpdatedAt)) + " ago"
 }
@@ -153,6 +162,13 @@ func (ctx *secretInspectContext) Labels() map[string]string {
 	return ctx.Secret.Spec.Labels
 }
 
+func (ctx *secretInspectContext) Driver() string {
+	if ctx.Secret.Spec.Driver == nil {
+		return ""
+	}
+	return ctx.Secret.Spec.Driver.Name
+}
+
 func (ctx *secretInspectContext) CreatedAt() string {
 	return command.PrettyPrint(ctx.Secret.CreatedAt)
 }

cli/command/formatter/secret_test.go

@@ -28,9 +28,9 @@ func TestSecretContextFormatWrite(t *testing.T) {
 		},
 		// Table format
 		{Context{Format: NewSecretFormat("table", false)},
-			`ID                  NAME                CREATED                  UPDATED
-1                   passwords           Less than a second ago   Less than a second ago
-2                   id_rsa              Less than a second ago   Less than a second ago
+			`ID                  NAME                DRIVER              CREATED                  UPDATED
+1                   passwords                               Less than a second ago   Less than a second ago
+2                   id_rsa                                  Less than a second ago   Less than a second ago
 `},
 		{Context{Format: NewSecretFormat("table {{.Name}}", true)},
 			`NAME

cli/command/formatter/service.go

@@ -504,6 +504,9 @@ func (c *serviceContext) Replicas() string {
 }
 
 func (c *serviceContext) Image() string {
+	if c.service.Spec.TaskTemplate.ContainerSpec == nil {
+		return ""
+	}
 	image := c.service.Spec.TaskTemplate.ContainerSpec.Image
 	if ref, err := reference.ParseNormalizedNamed(image); err == nil {
 		// update image string for display, (strips any digest)

cli/command/secret/create.go

@@ -17,6 +17,7 @@ import (
 
 type createOptions struct {
 	name   string
+	driver string
 	file   string
 	labels opts.ListOpts
 }
@@ -27,17 +28,20 @@ func newSecretCreateCommand(dockerCli command.Cli) *cobra.Command {
 	}
 
 	cmd := &cobra.Command{
-		Use:   "create [OPTIONS] SECRET file|-",
+		Use:   "create [OPTIONS] SECRET [file|-]",
 		Short: "Create a secret from a file or STDIN as content",
-		Args:  cli.ExactArgs(2),
+		Args:  cli.RequiresRangeArgs(1, 2),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			options.name = args[0]
-			options.file = args[1]
+			if len(args) == 2 {
+				options.file = args[1]
+			}
 			return runSecretCreate(dockerCli, options)
 		},
 	}
 	flags := cmd.Flags()
 	flags.VarP(&options.labels, "label", "l", "Secret labels")
+	flags.StringVar(&options.driver, "driver", "", "Secret driver")
 
 	return cmd
 }
@@ -46,28 +50,26 @@ func runSecretCreate(dockerCli command.Cli, options createOptions) error {
 	client := dockerCli.Client()
 	ctx := context.Background()
 
-	var in io.Reader = dockerCli.In()
-	if options.file != "-" {
-		file, err := system.OpenSequential(options.file)
-		if err != nil {
-			return err
-		}
-		in = file
-		defer file.Close()
+	if options.driver != "" && options.file != "" {
+		return errors.Errorf("When using secret driver secret data must be empty")
 	}
 
-	secretData, err := ioutil.ReadAll(in)
+	secretData, err := readSecretData(dockerCli.In(), options.file)
 	if err != nil {
 		return errors.Errorf("Error reading content from %q: %v", options.file, err)
 	}
-
 	spec := swarm.SecretSpec{
 		Annotations: swarm.Annotations{
 			Name:   options.name,
 			Labels: opts.ConvertKVStringsToMap(options.labels.GetAll()),
 		},
 		Data: secretData,
 	}
+	if options.driver != "" {
+		spec.Driver = &swarm.Driver{
+			Name: options.driver,
+		}
+	}
 
 	r, err := client.SecretCreate(ctx, spec)
 	if err != nil {
@@ -77,3 +79,23 @@ func runSecretCreate(dockerCli command.Cli, options createOptions) error {
 	fmt.Fprintln(dockerCli.Out(), r.ID)
 	return nil
 }
+
+func readSecretData(in io.ReadCloser, file string) ([]byte, error) {
+	// Read secret value from external driver
+	if file == "" {
+		return nil, nil
+	}
+	if file != "-" {
+		var err error
+		in, err = system.OpenSequential(file)
+		if err != nil {
+			return nil, err
+		}
+		defer in.Close()
+	}
+	data, err := ioutil.ReadAll(in)
+	if err != nil {
+		return nil, err
+	}
+	return data, nil
+}

cli/command/secret/create_test.go

@@ -25,12 +25,8 @@ func TestSecretCreateErrors(t *testing.T) {
 		secretCreateFunc func(swarm.SecretSpec) (types.SecretCreateResponse, error)
 		expectedError    string
 	}{
-		{
-			args:          []string{"too_few"},
-			expectedError: "requires exactly 2 argument(s)",
-		},
 		{args: []string{"too", "many", "arguments"},
-			expectedError: "requires exactly 2 argument(s)",
+			expectedError: "requires at least 1 and at most 2",
 		},
 		{
 			args: []string{"name", filepath.Join("testdata", secretDataFile)},

cli/command/secret/inspect_test.go

@@ -164,6 +164,7 @@ func TestSecretInspectPretty(t *testing.T) {
 					}),
 					SecretID("secretID"),
 					SecretName("secretName"),
+					SecretDriver("driver"),
 					SecretCreatedAt(time.Time{}),
 					SecretUpdatedAt(time.Time{}),
 				), []byte{}, nil

cli/command/secret/ls_test.go

@@ -64,6 +64,7 @@ func TestSecretList(t *testing.T) {
 					SecretVersion(swarm.Version{Index: 11}),
 					SecretCreatedAt(time.Now().Add(-2*time.Hour)),
 					SecretUpdatedAt(time.Now().Add(-1*time.Hour)),
+					SecretDriver("driver"),
 				),
 			}, nil
 		},

cli/command/secret/testdata/secret-inspect-pretty.simple.golden

@@ -2,5 +2,6 @@ ID: secretID
 Name: secretName
 Labels:
 	- lbl1=value1
+Driver: driver
 Created at: 0001-01-01 00:00:00+0000 utc
 Updated at: 0001-01-01 00:00:00+0000 utc

cli/command/secret/testdata/secret-list-with-filter.golden

@@ -1,3 +1,3 @@
-ID                  NAME                CREATED             UPDATED
-ID-foo              foo                 2 hours ago         About an hour ago
-ID-bar              bar                 2 hours ago         About an hour ago
+ID                  NAME                 DRIVER                CREATED             UPDATED
+ID-foo              foo                                        2 hours ago         About an hour ago
+ID-bar              bar                                        2 hours ago         About an hour ago

cli/command/secret/testdata/secret-list.golden

@@ -1,3 +1,3 @@
-ID                  NAME                CREATED             UPDATED
-ID-foo              foo                 2 hours ago         About an hour ago
-ID-bar              bar                 2 hours ago         About an hour ago
+ID                  NAME                 DRIVER           CREATED             UPDATED
+ID-foo              foo                                  2 hours ago         About an hour ago
+ID-bar              bar                  driver          2 hours ago         About an hour ago

cli/command/service/inspect_test.go

@@ -41,7 +41,7 @@ func formatServiceInspect(t *testing.T, format formatter.Format, now time.Time)
 				Labels: map[string]string{"com.label": "foo"},
 			},
 			TaskTemplate: swarm.TaskSpec{
-				ContainerSpec: swarm.ContainerSpec{
+				ContainerSpec: &swarm.ContainerSpec{
 					Image: "foo/bar@sha256:this_is_a_test",
 				},
 				Networks: []swarm.NetworkAttachmentConfig{

cli/command/service/opts.go

@@ -592,7 +592,7 @@ func (options *serviceOptions) ToService(ctx context.Context, apiClient client.N
 			Labels: opts.ConvertKVStringsToMap(options.labels.GetAll()),
 		},
 		TaskTemplate: swarm.TaskSpec{
-			ContainerSpec: swarm.ContainerSpec{
+			ContainerSpec: &swarm.ContainerSpec{
 				Image:      options.image,
 				Args:       options.args,
 				Command:    options.entrypoint.Value(),

cli/command/service/update.go

@@ -270,7 +270,7 @@ func updateService(ctx context.Context, apiClient client.NetworkAPIClient, flags
 		}
 	}
 
-	cspec := &spec.TaskTemplate.ContainerSpec
+	cspec := spec.TaskTemplate.ContainerSpec
 	task := &spec.TaskTemplate
 
 	taskResources := func() *swarm.ResourceRequirements {

cli/command/service/update_test.go

@@ -19,8 +19,8 @@ func TestUpdateServiceArgs(t *testing.T) {
 	flags := newUpdateCommand(nil).Flags()
 	flags.Set("args", "the \"new args\"")
 
-	spec := &swarm.ServiceSpec{}
-	cspec := &spec.TaskTemplate.ContainerSpec
+	spec := &swarm.ServiceSpec{TaskTemplate: swarm.TaskSpec{ContainerSpec: &swarm.ContainerSpec{}}}
+	cspec := spec.TaskTemplate.ContainerSpec
 	cspec.Args = []string{"old", "args"}
 
 	updateService(nil, nil, flags, spec)
@@ -452,8 +452,8 @@ func TestUpdateSecretUpdateInPlace(t *testing.T) {
 }
 
 func TestUpdateReadOnly(t *testing.T) {
-	spec := &swarm.ServiceSpec{}
-	cspec := &spec.TaskTemplate.ContainerSpec
+	spec := &swarm.ServiceSpec{TaskTemplate: swarm.TaskSpec{ContainerSpec: &swarm.ContainerSpec{}}}
+	cspec := spec.TaskTemplate.ContainerSpec
 
 	// Update with --read-only=true, changed to true
 	flags := newUpdateCommand(nil).Flags()
@@ -474,8 +474,8 @@ func TestUpdateReadOnly(t *testing.T) {
 }
 
 func TestUpdateStopSignal(t *testing.T) {
-	spec := &swarm.ServiceSpec{}
-	cspec := &spec.TaskTemplate.ContainerSpec
+	spec := &swarm.ServiceSpec{TaskTemplate: swarm.TaskSpec{ContainerSpec: &swarm.ContainerSpec{}}}
+	cspec := spec.TaskTemplate.ContainerSpec
 
 	// Update with --stop-signal=SIGUSR1
 	flags := newUpdateCommand(nil).Flags()

cli/command/stack/deploy_bundlefile.go

@@ -64,7 +64,7 @@ func deployBundle(ctx context.Context, dockerCli command.Cli, opts deployOptions
 				Labels: convert.AddStackLabel(namespace, service.Labels),
 			},
 			TaskTemplate: swarm.TaskSpec{
-				ContainerSpec: swarm.ContainerSpec{
+				ContainerSpec: &swarm.ContainerSpec{
 					Image:   service.Image,
 					Command: service.Command,
 					Args:    service.Args,

cli/compose/convert/compose.go

@@ -101,17 +101,27 @@ func Secrets(namespace Namespace, secrets map[string]composetypes.SecretConfig)
 			continue
 		}
 
-		data, err := ioutil.ReadFile(secret.File)
-		if err != nil {
-			return nil, err
+		var data []byte
+		var err error
+		var driver *swarm.Driver
+		if secret.Driver == "" {
+			data, err = ioutil.ReadFile(secret.File)
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			driver = &swarm.Driver{
+				Name: secret.Driver,
+			}
 		}
 
 		result = append(result, swarm.SecretSpec{
 			Annotations: swarm.Annotations{
 				Name:   namespace.Scope(name),
 				Labels: AddStackLabel(namespace, secret.Labels),
 			},
-			Data: data,
+			Data:   data,
+			Driver: driver,
 		})
 	}
 	return result, nil

cli/compose/convert/compose_test.go

@@ -6,6 +6,7 @@ import (
 	composetypes "github.com/docker/cli/cli/compose/types"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/network"
+	"github.com/docker/docker/api/types/swarm"
 	"github.com/docker/docker/pkg/testutil/tempfile"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -105,6 +106,7 @@ func TestNetworks(t *testing.T) {
 
 func TestSecrets(t *testing.T) {
 	namespace := Namespace{name: "foo"}
+	const driver = "my-driver"
 
 	secretText := "this is the first secret"
 	secretFile := tempfile.NewTempFile(t, "convert-secrets", secretText)
@@ -120,12 +122,22 @@ func TestSecrets(t *testing.T) {
 				External: true,
 			},
 		},
+		"driver": {
+			Driver: driver,
+		},
 	}
 
 	specs, err := Secrets(namespace, source)
 	assert.NoError(t, err)
-	require.Len(t, specs, 1)
-	secret := specs[0]
+	require.Len(t, specs, 2)
+	secrets := make(map[string]swarm.SecretSpec)
+	for _, secret := range specs {
+		secrets[secret.Annotations.Name] = secret
+	}
+
+	assert.Equal(t, driver, secrets["foo_driver"].Driver.Name)
+
+	secret := secrets["foo_one"]
 	assert.Equal(t, "foo_one", secret.Name)
 	assert.Equal(t, map[string]string{
 		"monster":      "mash",

cli/compose/convert/service.go

@@ -128,7 +128,7 @@ func Service(
 			Labels: AddStackLabel(namespace, service.Deploy.Labels),
 		},
 		TaskTemplate: swarm.TaskSpec{
-			ContainerSpec: swarm.ContainerSpec{
+			ContainerSpec: &swarm.ContainerSpec{
 				Image:           service.Image,
 				Command:         service.Entrypoint,
 				Args:            service.Command,