Global Plugin Reload: OSS Changes Take II (#9347)

* Carefully move changes from the plugin-cluster-reload branch into this clean branch off master.

* Don't test this at this level, adequately covered in the api level tests

* Change PR link

* go.mod

* Vendoring

* Vendor api/sys_plugins.go

CHANGELOG.md

@@ -23,8 +23,8 @@ IMPROVEMENTS:
 * core: Added Password Policies for user-configurable password generation [[GH-8637](https://github.com/hashicorp/vault/pull/8637)]
 * core: New telemetry metrics covering token counts, token creation, KV secret counts, lease creation. [[GH-9239](https://github.com/hashicorp/vault/pull/9239)] [[GH-9250](https://github.com/hashicorp/vault/pull/9250)] [[GH-9244](https://github.com/hashicorp/vault/pull/9244)] [[GH-9052](https://github.com/hashicorp/vault/pull/9052)]
 * cli: Support reading TLS parameters from file for the `vault operator raft join` command. [[GH-9060](https://github.com/hashicorp/vault/pull/9060)]
-* plugin: Add SDK method, `Sys.ReloadPlugin`, and CLI command, `vault plugin reload`, 
-  for reloading plugins. [[GH-8777](https://github.com/hashicorp/vault/pull/8777)]
+* plugin: Add SDK method, `Sys.ReloadPlugin`, and CLI command, `vault plugin reload`, for reloading plugins. [[GH-8777](https://github.com/hashicorp/vault/pull/8777)]
+* plugin (enterprise): Add a scope field to plugin reload, which when global, reloads the plugin anywhere in a cluster. [[GH-9347](https://github.com/hashicorp/vault/pull/9347)] 
 * sdk/framework: Support accepting TypeFloat parameters over the API [[GH-8923](https://github.com/hashicorp/vault/pull/8923)]
 * secrets/aws: Add iam_groups parameter to role create/update [[GH-8811](https://github.com/hashicorp/vault/pull/8811)]
 * secrets/database: Add static role rotation for MongoDB Atlas database plugin [[GH-11](https://github.com/hashicorp/vault-plugin-database-mongodbatlas/pull/11)]

api/sys_plugins.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
+	"time"
 
 	"github.com/hashicorp/vault/sdk/helper/consts"
 	"github.com/mitchellh/mapstructure"
@@ -232,26 +233,98 @@ type ReloadPluginInput struct {
 
 	// Mounts is the array of string mount paths of the plugin backends to reload
 	Mounts []string `json:"mounts"`
+
+	// Scope is the scope of the plugin reload
+	Scope string `json:"scope"`
 }
 
-// ReloadPlugin reloads mounted plugin backends
-func (c *Sys) ReloadPlugin(i *ReloadPluginInput) error {
+// ReloadPlugin reloads mounted plugin backends, possibly returning
+// reloadId for a cluster scoped reload
+func (c *Sys) ReloadPlugin(i *ReloadPluginInput) (string, error) {
 	path := "/v1/sys/plugins/reload/backend"
 	req := c.c.NewRequest(http.MethodPut, path)
 
 	if err := req.SetJSONBody(i); err != nil {
-		return err
+		return "", err
 	}
 
 	ctx, cancelFunc := context.WithCancel(context.Background())
 	defer cancelFunc()
 
 	resp, err := c.c.RawRequestWithContext(ctx, req)
 	if err != nil {
-		return err
+		return "", err
 	}
 	defer resp.Body.Close()
-	return err
+
+	if i.Scope == "global" {
+		// Get the reload id
+		secret, parseErr := ParseSecret(resp.Body)
+		if parseErr != nil {
+			return "", err
+		}
+		if _, ok := secret.Data["reload_id"]; ok {
+			return secret.Data["reload_id"].(string), nil
+		}
+	}
+	return "", err
+}
+
+// ReloadStatus is the status of an individual node's plugin reload
+type ReloadStatus struct {
+	Timestamp time.Time `json:"timestamp" mapstructure:"timestamp"`
+	Success   bool      `json:"success" mapstructure:"success"`
+	Message   string    `json:"message" mapstructure:"message"`
+}
+
+// ReloadStatusResponse is the combined response of all known completed plugin reloads
+type ReloadStatusResponse struct {
+	ReloadID string                   `mapstructure:"reload_id"`
+	Results  map[string]*ReloadStatus `mapstructure:"results"`
+}
+
+// ReloadPluginStatusInput is used as input to the ReloadStatusPlugin function.
+type ReloadPluginStatusInput struct {
+	// ReloadID is the ID of the reload operation
+	ReloadID string `json:"reload_id"`
+}
+
+// ReloadPluginStatus retrieves the status of a reload operation
+func (c *Sys) ReloadPluginStatus(reloadStatusInput *ReloadPluginStatusInput) (*ReloadStatusResponse, error) {
+	path := "/v1/sys/plugins/reload/backend/status"
+	req := c.c.NewRequest(http.MethodGet, path)
+	req.Params.Add("reload_id", reloadStatusInput.ReloadID)
+
+	ctx, cancelFunc := context.WithCancel(context.Background())
+	defer cancelFunc()
+
+	resp, err := c.c.RawRequestWithContext(ctx, req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	if resp != nil {
+		secret, parseErr := ParseSecret(resp.Body)
+		if parseErr != nil {
+			return nil, err
+		}
+
+		var r ReloadStatusResponse
+		d, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
+			DecodeHook: mapstructure.StringToTimeHookFunc(time.RFC3339),
+			Result:     &r,
+		})
+		if err != nil {
+			return nil, err
+		}
+		err = d.Decode(secret.Data)
+		if err != nil {
+			return nil, err
+		}
+		return &r, nil
+	}
+	return nil, nil
+
 }
 
 // catalogPathByType is a helper to construct the proper API path by plugin type

command/commands.go

@@ -447,6 +447,11 @@ func initCommands(ui, serverCmdUi cli.Ui, runOpts *RunOptions) {
 				BaseCommand: getBaseCommand(),
 			}, nil
 		},
+		"plugin reload-status": func() (cli.Command, error) {
+			return &PluginReloadStatusCommand{
+				BaseCommand: getBaseCommand(),
+			}, nil
+		},
 		"policy": func() (cli.Command, error) {
 			return &PolicyCommand{
 				BaseCommand: getBaseCommand(),

command/plugin_reload.go

@@ -16,6 +16,7 @@ type PluginReloadCommand struct {
 	*BaseCommand
 	plugin string
 	mounts []string
+	scope  string
 }
 
 func (c *PluginReloadCommand) Synopsis() string {
@@ -58,6 +59,13 @@ func (c *PluginReloadCommand) Flags() *FlagSets {
 		Usage:      "Array or comma-separated string mount paths of the plugin backends to reload.",
 	})
 
+	f.StringVar(&StringVar{
+		Name:       "scope",
+		Target:     &c.scope,
+		Completion: complete.PredictAnything,
+		Usage:      "The scope of the reload, omitted for local, 'global', for replicated reloads",
+	})
+
 	return set
 }
 
@@ -84,6 +92,8 @@ func (c *PluginReloadCommand) Run(args []string) int {
 	case c.plugin != "" && len(c.mounts) > 0:
 		c.UI.Error(fmt.Sprintf("Too many arguments (expected 1, got %d)", len(args)))
 		return 1
+	case c.scope != "" && c.scope != "global":
+		c.UI.Error(fmt.Sprintf("Invalid reload scope: %s", c.scope))
 	}
 
 	client, err := c.Client()
@@ -92,18 +102,28 @@ func (c *PluginReloadCommand) Run(args []string) int {
 		return 2
 	}
 
-	if err := client.Sys().ReloadPlugin(&api.ReloadPluginInput{
+	rid, err := client.Sys().ReloadPlugin(&api.ReloadPluginInput{
 		Plugin: c.plugin,
 		Mounts: c.mounts,
-	}); err != nil {
+		Scope:  c.scope,
+	})
+	if err != nil {
 		c.UI.Error(fmt.Sprintf("Error reloading plugin/mounts: %s", err))
 		return 2
 	}
 
 	if len(c.mounts) > 0 {
-		c.UI.Output(fmt.Sprintf("Success! Reloaded mounts: %s", c.mounts))
+		if rid != "" {
+			c.UI.Output(fmt.Sprintf("Success! Reloading mounts: %s, reload_id: %s", c.mounts, rid))
+		} else {
+			c.UI.Output(fmt.Sprintf("Success! Reloaded mounts: %s", c.mounts))
+		}
 	} else {
-		c.UI.Output(fmt.Sprintf("Success! Reloaded plugin: %s", c.plugin))
+		if rid != "" {
+			c.UI.Output(fmt.Sprintf("Success! Reloading plugin: %s, reload_id: %s", c.plugin, rid))
+		} else {
+			c.UI.Output(fmt.Sprintf("Success! Reloaded plugin: %s", c.plugin))
+		}
 	}
 
 	return 0

command/plugin_reload_status.go

@@ -0,0 +1,91 @@
+package command
+
+import (
+	"fmt"
+	"github.com/hashicorp/vault/api"
+	"github.com/mitchellh/cli"
+	"github.com/posener/complete"
+	"strings"
+)
+
+var _ cli.Command = (*PluginReloadCommand)(nil)
+var _ cli.CommandAutocomplete = (*PluginReloadCommand)(nil)
+
+type PluginReloadStatusCommand struct {
+	*BaseCommand
+}
+
+func (c *PluginReloadStatusCommand) Synopsis() string {
+	return "Get the status of an active or recently completed global plugin reload"
+}
+
+func (c *PluginReloadStatusCommand) Help() string {
+	helpText := `
+Usage: vault plugin reload-status RELOAD_ID
+
+  Retrieves the status of a recent cluster plugin reload.  The reload id must be provided.
+
+	  $ vault plugin reload-status d60a3e83-a598-4f3a-879d-0ddd95f11d4e
+
+` + c.Flags().Help()
+
+	return strings.TrimSpace(helpText)
+}
+
+func (c *PluginReloadStatusCommand) Flags() *FlagSets {
+	return c.flagSet(FlagSetHTTP)
+}
+
+func (c *PluginReloadStatusCommand) AutocompleteArgs() complete.Predictor {
+	return complete.PredictNothing
+}
+
+func (c *PluginReloadStatusCommand) AutocompleteFlags() complete.Flags {
+	return c.Flags().Completions()
+}
+
+func (c *PluginReloadStatusCommand) Run(args []string) int {
+	f := c.Flags()
+
+	if err := f.Parse(args); err != nil {
+		c.UI.Error(err.Error())
+		return 1
+	}
+
+	args = f.Args()
+	switch {
+	case len(args) < 1:
+		c.UI.Error(fmt.Sprintf("Not enough arguments (expected 1, got %d)", len(args)))
+		return 1
+	case len(args) > 1:
+		c.UI.Error(fmt.Sprintf("Too many arguments (expected 1, got %d)", len(args)))
+		return 1
+	}
+
+	reloadId := strings.TrimSpace(args[0])
+
+	client, err := c.Client()
+	if err != nil {
+		c.UI.Error(err.Error())
+		return 2
+	}
+
+	r, err := client.Sys().ReloadPluginStatus(&api.ReloadPluginStatusInput{
+		ReloadID: reloadId,
+	})
+
+	if err != nil {
+		c.UI.Error(fmt.Sprintf("Error retrieving plugin reload status: %s", err))
+		return 2
+	}
+	out := []string{"Time | Participant | Success | Message "}
+	for i, s := range r.Results {
+		out = append(out, fmt.Sprintf("%s | %s | %t | %s ",
+			s.Timestamp.Format("15:04:05"),
+			i,
+			s.Success,
+			s.Message))
+	}
+	c.UI.Output(tableOutput(out, nil))
+	return 0
+}

command/plugin_reload_test.go

@@ -20,6 +20,17 @@ func testPluginReloadCommand(tb testing.TB) (*cli.MockUi, *PluginReloadCommand)
 	}
 }
 
+func testPluginReloadStatusCommand(tb testing.TB) (*cli.MockUi, *PluginReloadStatusCommand) {
+	tb.Helper()
+
+	ui := cli.NewMockUi()
+	return ui, &PluginReloadStatusCommand{
+		BaseCommand: &BaseCommand{
+			UI: ui,
+		},
+	}
+}
+
 func TestPluginReloadCommand_Run(t *testing.T) {
 	t.Parallel()
 
@@ -108,3 +119,46 @@ func TestPluginReloadCommand_Run(t *testing.T) {
 	})
 
 }
+
+func TestPluginReloadStatusCommand_Run(t *testing.T) {
+	t.Parallel()
+
+	cases := []struct {
+		name string
+		args []string
+		out  string
+		code int
+	}{
+		{
+			"not_enough_args",
+			nil,
+			"Not enough arguments",
+			1,
+		},
+	}
+
+	for _, tc := range cases {
+		tc := tc
+
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			client, closer := testVaultServer(t)
+			defer closer()
+
+			ui, cmd := testPluginReloadCommand(t)
+			cmd.client = client
+
+			args := append([]string{}, tc.args...)
+			code := cmd.Run(args)
+			if code != tc.code {
+				t.Errorf("expected %d to be %d", code, tc.code)
+			}
+
+			combined := ui.OutputWriter.String() + ui.ErrorWriter.String()
+			if !strings.Contains(combined, tc.out) {
+				t.Errorf("expected %q to contain %q", combined, tc.out)
+			}
+		})
+	}
+}

go.mod

@@ -84,7 +84,7 @@ require (
 	github.com/hashicorp/vault-plugin-secrets-kv v0.5.5
 	github.com/hashicorp/vault-plugin-secrets-mongodbatlas v0.1.2
 	github.com/hashicorp/vault-plugin-secrets-openldap v0.1.4-0.20200618161832-cae59ebde561
-	github.com/hashicorp/vault/api v1.0.5-0.20200519221902-385fac77e20f
+	github.com/hashicorp/vault/api v1.0.5-0.20200630135746-8163ff047c9a
 	github.com/hashicorp/vault/sdk v0.1.14-0.20200527182800-ad90e0b39d2f
 	github.com/influxdata/influxdb v0.0.0-20190411212539-d24b7ba8c4c4
 	github.com/jcmturner/gokrb5/v8 v8.0.0

go.sum

@@ -522,6 +522,7 @@ github.com/hashicorp/vault-plugin-auth-alicloud v0.5.5 h1:JYf3VYpKs7mOdtcwZWi73S
 github.com/hashicorp/vault-plugin-auth-alicloud v0.5.5/go.mod h1:sQ+VNwPQlemgXHXikYH6onfH9gPwDZ1GUVRLz0ZvHx8=
 github.com/hashicorp/vault-plugin-auth-azure v0.5.5 h1:kN79ai+aMVU9hUmwscHjmweW2fGa8V/t+ScIchPZGrk=
 github.com/hashicorp/vault-plugin-auth-azure v0.5.5/go.mod h1:RCVBsf8AJndh4c6iGZtvVZFui9SG0Bj9fnF0SodNIkw=
+github.com/hashicorp/vault-plugin-auth-azure v0.5.6-0.20200422235613-1b5c70f9ef68 h1:o4ekpvOmfRxCYE7+g4dV6FQc9H+Sl1jv4JoGDrLDKt0=
 github.com/hashicorp/vault-plugin-auth-azure v0.5.6-0.20200422235613-1b5c70f9ef68/go.mod h1:RCVBsf8AJndh4c6iGZtvVZFui9SG0Bj9fnF0SodNIkw=
 github.com/hashicorp/vault-plugin-auth-centrify v0.5.5 h1:YXxXt6o6I1rOkYW+hADK0vd+uVMj4C6Qs3jBrQlKQcY=
 github.com/hashicorp/vault-plugin-auth-centrify v0.5.5/go.mod h1:GfRoy7NHsuR/ogmZtbExdJXUwbfwcxPrS9xzkyy2J/c=
@@ -530,6 +531,7 @@ github.com/hashicorp/vault-plugin-auth-cf v0.5.4/go.mod h1:idkFYHc6ske2BE7fe00Sp
 github.com/hashicorp/vault-plugin-auth-gcp v0.5.1/go.mod h1:eLj92eX8MPI4vY1jaazVLF2sVbSAJ3LRHLRhF/pUmlI=
 github.com/hashicorp/vault-plugin-auth-gcp v0.6.1 h1:WXTuja3WC2BdZekYCnzuZGoVvZTAGH8kSDUHzOK2PQY=
 github.com/hashicorp/vault-plugin-auth-gcp v0.6.1/go.mod h1:8eBRzg+JIhAaDBfDndDAQKIhDrQ3WW8OPklxAYftNFs=
+github.com/hashicorp/vault-plugin-auth-gcp v0.6.2-0.20200428223335-82bd3a3ad5b3 h1:GTQYSsqv/2jjdTm0DawBSljMmZTc/js6zLer+9A5e2U=
 github.com/hashicorp/vault-plugin-auth-gcp v0.6.2-0.20200428223335-82bd3a3ad5b3/go.mod h1:U0fkAlxWTEyQ74lx8wlGdD493lP1DD/qpMjXgOEbwj0=
 github.com/hashicorp/vault-plugin-auth-jwt v0.6.2/go.mod h1:SFadxIfoLGzugEjwUUmUaCGbsYEz2/jJymZDDQjEqYg=
 github.com/hashicorp/vault-plugin-auth-jwt v0.7.0 h1:lHg02BB7IpUQbJStAPmGyS3KnZJC7PSEvc5LOZNPjHM=
@@ -548,6 +550,7 @@ github.com/hashicorp/vault-plugin-database-elasticsearch v0.5.4 h1:YE4qndazWmYGp
 github.com/hashicorp/vault-plugin-database-elasticsearch v0.5.4/go.mod h1:QjGrrxcRXv/4XkEZAlM0VMZEa3uxKAICFqDj27FP/48=
 github.com/hashicorp/vault-plugin-database-mongodbatlas v0.1.0-beta1.0.20200521152755-9cf156a44f9c h1:9pXwe7sEVhZ5C3U6egIrKaZBb5lD0FvLIjISEvpbQQA=
 github.com/hashicorp/vault-plugin-database-mongodbatlas v0.1.0-beta1.0.20200521152755-9cf156a44f9c/go.mod h1:HTXNzFr/SAVtJOs7jz0XxZ69jlKtaceEwp37l86UAQ0=
+github.com/hashicorp/vault-plugin-database-mongodbatlas v0.1.2-0.20200520204052-f840e9d4895c h1:P9rZXBJx+UHu/T8lK8NEtS2PGeSnyZ31zeOtkvGo4yo=
 github.com/hashicorp/vault-plugin-database-mongodbatlas v0.1.2-0.20200520204052-f840e9d4895c/go.mod h1:MP3kfr0N+7miOTZFwKv952b9VkXM4S2Q6YtQCiNKWq8=
 github.com/hashicorp/vault-plugin-secrets-ad v0.6.6-0.20200520202259-fc6b89630f9f/go.mod h1:kk98nB+cwDbt3I7UGQq3ota7+eHZrGSTQZfSRGpluvA=
 github.com/hashicorp/vault-plugin-secrets-ad v0.6.6 h1:GskxrCCL2flrBtnAeOsBV+whCaqnnM/+t/h1IyqukNo=