Run all builtins as plugins (#5536)

api/api_integration_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/hashicorp/vault/builtin/logical/database"
 	"github.com/hashicorp/vault/builtin/logical/pki"
 	"github.com/hashicorp/vault/builtin/logical/transit"
+	"github.com/hashicorp/vault/helper/builtinplugins"
 	"github.com/hashicorp/vault/logical"
 	"github.com/hashicorp/vault/vault"
 
@@ -56,6 +57,7 @@ func testVaultServerUnseal(t testing.TB) (*api.Client, []string, func()) {
 			"pki":            pki.Factory,
 			"transit":        transit.Factory,
 		},
+		BuiltinRegistry: builtinplugins.Registry,
 	})
 }
 

api/client.go

@@ -17,7 +17,7 @@ import (
 
 	"github.com/hashicorp/errwrap"
 	"github.com/hashicorp/go-cleanhttp"
-	retryablehttp "github.com/hashicorp/go-retryablehttp"
+	"github.com/hashicorp/go-retryablehttp"
 	"github.com/hashicorp/go-rootcerts"
 	"github.com/hashicorp/vault/helper/consts"
 	"github.com/hashicorp/vault/helper/parseutil"

api/sys_mounts.go

@@ -133,7 +133,6 @@ type MountInput struct {
 	Description string            `json:"description"`
 	Config      MountConfigInput  `json:"config"`
 	Local       bool              `json:"local"`
-	PluginName  string            `json:"plugin_name,omitempty"`
 	SealWrap    bool              `json:"seal_wrap" mapstructure:"seal_wrap"`
 	Options     map[string]string `json:"options"`
 }
@@ -144,7 +143,6 @@ type MountConfigInput struct {
 	Description               *string           `json:"description,omitempty" mapstructure:"description"`
 	MaxLeaseTTL               string            `json:"max_lease_ttl" mapstructure:"max_lease_ttl"`
 	ForceNoCache              bool              `json:"force_no_cache" mapstructure:"force_no_cache"`
-	PluginName                string            `json:"plugin_name,omitempty" mapstructure:"plugin_name"`
 	AuditNonHMACRequestKeys   []string          `json:"audit_non_hmac_request_keys,omitempty" mapstructure:"audit_non_hmac_request_keys"`
 	AuditNonHMACResponseKeys  []string          `json:"audit_non_hmac_response_keys,omitempty" mapstructure:"audit_non_hmac_response_keys"`
 	ListingVisibility         string            `json:"listing_visibility,omitempty" mapstructure:"listing_visibility"`
@@ -166,7 +164,6 @@ type MountConfigOutput struct {
 	DefaultLeaseTTL           int      `json:"default_lease_ttl" mapstructure:"default_lease_ttl"`
 	MaxLeaseTTL               int      `json:"max_lease_ttl" mapstructure:"max_lease_ttl"`
 	ForceNoCache              bool     `json:"force_no_cache" mapstructure:"force_no_cache"`
-	PluginName                string   `json:"plugin_name,omitempty" mapstructure:"plugin_name"`
 	AuditNonHMACRequestKeys   []string `json:"audit_non_hmac_request_keys,omitempty" mapstructure:"audit_non_hmac_request_keys"`
 	AuditNonHMACResponseKeys  []string `json:"audit_non_hmac_response_keys,omitempty" mapstructure:"audit_non_hmac_response_keys"`
 	ListingVisibility         string   `json:"listing_visibility,omitempty" mapstructure:"listing_visibility"`

api/sys_plugins.go

@@ -2,24 +2,43 @@ package api
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net/http"
+
+	"github.com/hashicorp/vault/helper/consts"
+	"github.com/mitchellh/mapstructure"
 )
 
 // ListPluginsInput is used as input to the ListPlugins function.
-type ListPluginsInput struct{}
+type ListPluginsInput struct {
+	// Type of the plugin. Required.
+	Type consts.PluginType `json:"type"`
+}
 
 // ListPluginsResponse is the response from the ListPlugins call.
 type ListPluginsResponse struct {
-	// Names is the list of names of the plugins.
-	Names []string `json:"names"`
+	// PluginsByType is the list of plugins by type.
+	PluginsByType map[consts.PluginType][]string `json:"types"`
+
+	// NamesDeprecated is the list of names of the plugins.
+	NamesDeprecated []string `json:"names"`
 }
 
 // ListPlugins lists all plugins in the catalog and returns their names as a
 // list of strings.
 func (c *Sys) ListPlugins(i *ListPluginsInput) (*ListPluginsResponse, error) {
-	path := "/v1/sys/plugins/catalog"
-	req := c.c.NewRequest("LIST", path)
+	path := ""
+	method := ""
+	if i.Type == consts.PluginTypeUnknown {
+		path = "/v1/sys/plugins/catalog"
+		method = "GET"
+	} else {
+		path = fmt.Sprintf("/v1/sys/plugins/catalog/%s", i.Type)
+		method = "LIST"
+	}
+
+	req := c.c.NewRequest(method, path)
 
 	ctx, cancelFunc := context.WithCancel(context.Background())
 	defer cancelFunc()
@@ -29,21 +48,76 @@ func (c *Sys) ListPlugins(i *ListPluginsInput) (*ListPluginsResponse, error) {
 	}
 	defer resp.Body.Close()
 
-	var result struct {
-		Data struct {
-			Keys []string `json:"keys"`
-		} `json:"data"`
-	}
-	if err := resp.DecodeJSON(&result); err != nil {
+	secret, err := ParseSecret(resp.Body)
+	if err != nil {
 		return nil, err
 	}
+	if secret == nil || secret.Data == nil {
+		return nil, errors.New("data from server response is empty")
+	}
 
-	return &ListPluginsResponse{Names: result.Data.Keys}, nil
+	if resp.StatusCode == 405 && req.Method == "GET" {
+		// We received an Unsupported Operation response from Vault, indicating
+		// Vault of an older version that doesn't support the READ method yet.
+		req.Method = "LIST"
+		resp, err := c.c.RawRequestWithContext(ctx, req)
+		if err != nil {
+			return nil, err
+		}
+		defer resp.Body.Close()
+		var result struct {
+			Data struct {
+				Keys []string `json:"keys"`
+			} `json:"data"`
+		}
+		if err := resp.DecodeJSON(&result); err != nil {
+			return nil, err
+		}
+		return &ListPluginsResponse{NamesDeprecated: result.Data.Keys}, nil
+	}
+
+	result := &ListPluginsResponse{
+		PluginsByType: make(map[consts.PluginType][]string),
+	}
+	if i.Type == consts.PluginTypeUnknown {
+		for pluginTypeStr, pluginsRaw := range secret.Data {
+			pluginType, err := consts.ParsePluginType(pluginTypeStr)
+			if err != nil {
+				return nil, err
+			}
+
+			pluginsIfc, ok := pluginsRaw.([]interface{})
+			if !ok {
+				return nil, fmt.Errorf("unable to parse plugins for %q type", pluginTypeStr)
+			}
+
+			plugins := make([]string, len(pluginsIfc))
+			for i, nameIfc := range pluginsIfc {
+				name, ok := nameIfc.(string)
+				if !ok {
+
+				}
+				plugins[i] = name
+			}
+			result.PluginsByType[pluginType] = plugins
+		}
+	} else {
+		var respKeys []string
+		if err := mapstructure.Decode(secret.Data["keys"], &respKeys); err != nil {
+			return nil, err
+		}
+		result.PluginsByType[i.Type] = respKeys
+	}
+
+	return result, nil
 }
 
 // GetPluginInput is used as input to the GetPlugin function.
 type GetPluginInput struct {
 	Name string `json:"-"`
+
+	// Type of the plugin. Required.
+	Type consts.PluginType `json:"type"`
 }
 
 // GetPluginResponse is the response from the GetPlugin call.
@@ -56,7 +130,7 @@ type GetPluginResponse struct {
 }
 
 func (c *Sys) GetPlugin(i *GetPluginInput) (*GetPluginResponse, error) {
-	path := fmt.Sprintf("/v1/sys/plugins/catalog/%s", i.Name)
+	path := fmt.Sprintf("/v1/sys/plugins/catalog/%s/%s", i.Type, i.Name)
 	req := c.c.NewRequest(http.MethodGet, path)
 
 	ctx, cancelFunc := context.WithCancel(context.Background())
@@ -82,6 +156,9 @@ type RegisterPluginInput struct {
 	// Name is the name of the plugin. Required.
 	Name string `json:"-"`
 
+	// Type of the plugin. Required.
+	Type consts.PluginType `json:"type"`
+
 	// Args is the list of args to spawn the process with.
 	Args []string `json:"args,omitempty"`
 
@@ -94,7 +171,7 @@ type RegisterPluginInput struct {
 
 // RegisterPlugin registers the plugin with the given information.
 func (c *Sys) RegisterPlugin(i *RegisterPluginInput) error {
-	path := fmt.Sprintf("/v1/sys/plugins/catalog/%s", i.Name)
+	path := fmt.Sprintf("/v1/sys/plugins/catalog/%s/%s", i.Type, i.Name)
 	req := c.c.NewRequest(http.MethodPut, path)
 	if err := req.SetJSONBody(i); err != nil {
 		return err
@@ -113,12 +190,15 @@ func (c *Sys) RegisterPlugin(i *RegisterPluginInput) error {
 type DeregisterPluginInput struct {
 	// Name is the name of the plugin. Required.
 	Name string `json:"-"`
+
+	// Type of the plugin. Required.
+	Type consts.PluginType `json:"type"`
 }
 
 // DeregisterPlugin removes the plugin with the given name from the plugin
 // catalog.
 func (c *Sys) DeregisterPlugin(i *DeregisterPluginInput) error {
-	path := fmt.Sprintf("/v1/sys/plugins/catalog/%s", i.Name)
+	path := fmt.Sprintf("/v1/sys/plugins/catalog/%s/%s", i.Type, i.Name)
 	req := c.c.NewRequest(http.MethodDelete, path)
 
 	ctx, cancelFunc := context.WithCancel(context.Background())

builtin/credential/app-id/backend_test.go

@@ -26,7 +26,7 @@ func TestBackend_basic(t *testing.T) {
 		return b, nil
 	}
 	logicaltest.Test(t, logicaltest.TestCase{
-		Factory: factory,
+		CredentialFactory: factory,
 		Steps: []logicaltest.TestStep{
 			testAccStepMapAppId(t),
 			testAccStepMapUserId(t),
@@ -65,7 +65,7 @@ func TestBackend_basic(t *testing.T) {
 
 func TestBackend_cidr(t *testing.T) {
 	logicaltest.Test(t, logicaltest.TestCase{
-		Factory: Factory,
+		CredentialFactory: Factory,
 		Steps: []logicaltest.TestStep{
 			testAccStepMapAppIdDisplayName(t),
 			testAccStepMapUserIdCidr(t, "192.168.1.0/16"),
@@ -78,7 +78,7 @@ func TestBackend_cidr(t *testing.T) {
 
 func TestBackend_displayName(t *testing.T) {
 	logicaltest.Test(t, logicaltest.TestCase{
-		Factory: Factory,
+		CredentialFactory: Factory,
 		Steps: []logicaltest.TestStep{
 			testAccStepMapAppIdDisplayName(t),
 			testAccStepMapUserId(t),

builtin/credential/aws/backend_test.go

@@ -508,8 +508,8 @@ func TestBackend_ConfigClient(t *testing.T) {
 	}
 
 	logicaltest.Test(t, logicaltest.TestCase{
-		AcceptanceTest: false,
-		Backend:        b,
+		AcceptanceTest:    false,
+		CredentialBackend: b,
 		Steps: []logicaltest.TestStep{
 			stepCreate,
 			stepInvalidAccessKey,

builtin/credential/cert/backend_test.go

@@ -220,7 +220,7 @@ func TestBackend_PermittedDNSDomainsIntermediateCA(t *testing.T) {
 	// Sign the intermediate CSR using /pki
 	secret, err = client.Logical().Write("pki/root/sign-intermediate", map[string]interface{}{
 		"permitted_dns_domains": ".myvault.com",
-		"csr":                   intermediateCSR,
+		"csr": intermediateCSR,
 	})
 	if err != nil {
 		t.Fatal(err)
@@ -840,7 +840,7 @@ func TestBackend_CertWrites(t *testing.T) {
 	}
 
 	tc := logicaltest.TestCase{
-		Backend: testFactory(t),
+		CredentialBackend: testFactory(t),
 		Steps: []logicaltest.TestStep{
 			testAccStepCert(t, "aaa", ca1, "foo", allowed{}, false),
 			testAccStepCert(t, "bbb", ca2, "foo", allowed{}, false),
@@ -863,7 +863,7 @@ func TestBackend_basic_CA(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 	logicaltest.Test(t, logicaltest.TestCase{
-		Backend: testFactory(t),
+		CredentialBackend: testFactory(t),
 		Steps: []logicaltest.TestStep{
 			testAccStepCert(t, "web", ca, "foo", allowed{}, false),
 			testAccStepLogin(t, connState),
@@ -898,7 +898,7 @@ func TestBackend_Basic_CRLs(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 	logicaltest.Test(t, logicaltest.TestCase{
-		Backend: testFactory(t),
+		CredentialBackend: testFactory(t),
 		Steps: []logicaltest.TestStep{
 			testAccStepCertNoLease(t, "web", ca, "foo"),
 			testAccStepLoginDefaultLease(t, connState),
@@ -923,7 +923,7 @@ func TestBackend_basic_singleCert(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 	logicaltest.Test(t, logicaltest.TestCase{
-		Backend: testFactory(t),
+		CredentialBackend: testFactory(t),
 		Steps: []logicaltest.TestStep{
 			testAccStepCert(t, "web", ca, "foo", allowed{}, false),
 			testAccStepLogin(t, connState),
@@ -948,7 +948,7 @@ func TestBackend_common_name_singleCert(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 	logicaltest.Test(t, logicaltest.TestCase{
-		Backend: testFactory(t),
+		CredentialBackend: testFactory(t),
 		Steps: []logicaltest.TestStep{
 			testAccStepCert(t, "web", ca, "foo", allowed{}, false),
 			testAccStepLogin(t, connState),
@@ -977,7 +977,7 @@ func TestBackend_ext_singleCert(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 	logicaltest.Test(t, logicaltest.TestCase{
-		Backend: testFactory(t),
+		CredentialBackend: testFactory(t),
 		Steps: []logicaltest.TestStep{
 			testAccStepCert(t, "web", ca, "foo", allowed{ext: "2.1.1.1:A UTF8String Extension"}, false),
 			testAccStepLogin(t, connState),
@@ -1032,7 +1032,7 @@ func TestBackend_dns_singleCert(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 	logicaltest.Test(t, logicaltest.TestCase{
-		Backend: testFactory(t),
+		CredentialBackend: testFactory(t),
 		Steps: []logicaltest.TestStep{
 			testAccStepCert(t, "web", ca, "foo", allowed{dns: "example.com"}, false),
 			testAccStepLogin(t, connState),
@@ -1063,7 +1063,7 @@ func TestBackend_email_singleCert(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 	logicaltest.Test(t, logicaltest.TestCase{
-		Backend: testFactory(t),
+		CredentialBackend: testFactory(t),
 		Steps: []logicaltest.TestStep{
 			testAccStepCert(t, "web", ca, "foo", allowed{emails: "[email protected]"}, false),
 			testAccStepLogin(t, connState),
@@ -1094,7 +1094,7 @@ func TestBackend_organizationalUnit_singleCert(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 	logicaltest.Test(t, logicaltest.TestCase{
-		Backend: testFactory(t),
+		CredentialBackend: testFactory(t),
 		Steps: []logicaltest.TestStep{
 			testAccStepCert(t, "web", ca, "foo", allowed{organizational_units: "engineering"}, false),
 			testAccStepLogin(t, connState),
@@ -1123,7 +1123,7 @@ func TestBackend_uri_singleCert(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 	logicaltest.Test(t, logicaltest.TestCase{
-		Backend: testFactory(t),
+		CredentialBackend: testFactory(t),
 		Steps: []logicaltest.TestStep{
 			testAccStepCert(t, "web", ca, "foo", allowed{uris: "spiffe://example.com/*"}, false),
 			testAccStepLogin(t, connState),
@@ -1151,7 +1151,7 @@ func TestBackend_mixed_constraints(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 	logicaltest.Test(t, logicaltest.TestCase{
-		Backend: testFactory(t),
+		CredentialBackend: testFactory(t),
 		Steps: []logicaltest.TestStep{
 			testAccStepCert(t, "1unconstrained", ca, "foo", allowed{}, false),
 			testAccStepCert(t, "2matching", ca, "foo", allowed{names: "*.example.com,whatever"}, false),
@@ -1172,7 +1172,7 @@ func TestBackend_untrusted(t *testing.T) {
 		t.Fatalf("error testing connection state: %v", err)
 	}
 	logicaltest.Test(t, logicaltest.TestCase{
-		Backend: testFactory(t),
+		CredentialBackend: testFactory(t),
 		Steps: []logicaltest.TestStep{
 			testAccStepLoginInvalid(t, connState),
 		},