diff --git a/api/api.go b/api/api.go index bb553b64576..84df7ec3793 100644 --- a/api/api.go +++ b/api/api.go @@ -65,6 +65,14 @@ type QueryOptions struct { // AuthToken is the secret ID of an ACL token AuthToken string + // PerPage is the number of entries to be returned in queries that support + // paginated lists. + PerPage int32 + + // NextToken is the token used indicate where to start paging for queries + // that support paginated lists. + NextToken string + // ctx is an optional context pass through to the underlying HTTP // request layer. Use Context() and WithContext() to manage this. ctx context.Context diff --git a/api/csi.go b/api/csi.go index 28da54bf0c8..5e5b3affbd8 100644 --- a/api/csi.go +++ b/api/csi.go @@ -28,6 +28,33 @@ func (v *CSIVolumes) List(q *QueryOptions) ([]*CSIVolumeListStub, *QueryMeta, er return resp, qm, nil } +// ListExternal returns all CSI volumes, as known to the storage provider +func (v *CSIVolumes) ListExternal(pluginID string, q *QueryOptions) (*CSIVolumeListExternalResponse, *QueryMeta, error) { + var resp *CSIVolumeListExternalResponse + + qp := url.Values{} + qp.Set("plugin_id", pluginID) + if q.NextToken != "" { + qp.Set("next_token", q.NextToken) + } + if q.PerPage != 0 { + qp.Set("per_page", fmt.Sprint(q.PerPage)) + } + + qm, err := v.client.query("/v1/volumes/external?"+qp.Encode(), &resp, q) + if err != nil { + return nil, nil, err + } + + sort.Sort(CSIVolumeExternalStubSort(resp.Volumes)) + return resp, qm, nil +} + +// PluginList returns all CSI volumes for the specified plugin id +func (v *CSIVolumes) PluginList(pluginID string) ([]*CSIVolumeListStub, *QueryMeta, error) { + return v.List(&QueryOptions{Prefix: pluginID}) +} + // Info is used to retrieve a single CSIVolume func (v *CSIVolumes) Info(id string, q *QueryOptions) (*CSIVolume, *QueryMeta, error) { var resp CSIVolume @@ -52,6 +79,21 @@ func (v *CSIVolumes) Deregister(id string, force bool, w *WriteOptions) error { return err } +func (v *CSIVolumes) Create(vol *CSIVolume, w *WriteOptions) ([]*CSIVolume, *WriteMeta, error) { + req := CSIVolumeCreateRequest{ + Volumes: []*CSIVolume{vol}, + } + + resp := &CSIVolumeCreateResponse{} + meta, err := v.client.write(fmt.Sprintf("/v1/volume/csi/%v/create", vol.ID), req, resp, w) + return resp.Volumes, meta, err +} + +func (v *CSIVolumes) Delete(externalVolID string, w *WriteOptions) error { + _, err := v.client.delete(fmt.Sprintf("/v1/volume/csi/%v/delete", url.PathEscape(externalVolID)), nil, w) + return err +} + func (v *CSIVolumes) Detach(volID, nodeID string, w *WriteOptions) error { _, err := v.client.delete(fmt.Sprintf("/v1/volume/csi/%v/detach?node=%v", url.PathEscape(volID), nodeID), nil, w) return err @@ -92,15 +134,23 @@ type CSISecrets map[string]string type CSIVolume struct { ID string Name string - ExternalID string `hcl:"external_id"` + ExternalID string `mapstructure:"external_id" hcl:"external_id"` Namespace string Topologies []*CSITopology AccessMode CSIVolumeAccessMode `hcl:"access_mode"` AttachmentMode CSIVolumeAttachmentMode `hcl:"attachment_mode"` MountOptions *CSIMountOptions `hcl:"mount_options"` - Secrets CSISecrets `hcl:"secrets"` - Parameters map[string]string `hcl:"parameters"` - Context map[string]string `hcl:"context"` + Secrets CSISecrets `mapstructure:"secrets" hcl:"secrets"` + Parameters map[string]string `mapstructure:"parameters" hcl:"parameters"` + Context map[string]string `mapstructure:"context" hcl:"context"` + Capacity int64 `hcl:"-"` + + // These fields are used as part of the volume creation request + RequestedCapacityMin int64 `hcl:"capacity_min"` + RequestedCapacityMax int64 `hcl:"capacity_max"` + RequestedCapabilities []*CSIVolumeCapability `hcl:"capability"` + CloneID string `mapstructure:"clone_id" hcl:"clone_id"` + SnapshotID string `mapstructure:"snapshot_id" hcl:"snapshot_id"` // ReadAllocs is a map of allocation IDs for tracking reader claim status. // The Allocation value will always be nil; clients can populate this data @@ -117,7 +167,7 @@ type CSIVolume struct { // Schedulable is true if all the denormalized plugin health fields are true Schedulable bool - PluginID string `hcl:"plugin_id"` + PluginID string `mapstructure:"plugin_id" hcl:"plugin_id"` Provider string ProviderVersion string ControllerRequired bool @@ -134,6 +184,13 @@ type CSIVolume struct { ExtraKeysHCL []string `hcl1:",unusedKeys" json:"-"` } +// CSIVolumeCapability is a requested attachment and access mode for a +// volume +type CSIVolumeCapability struct { + AccessMode CSIVolumeAccessMode `mapstructure:"access_mode" hcl:"access_mode"` + AttachmentMode CSIVolumeAttachmentMode `mapstructure:"attachment_mode" hcl:"attachment_mode"` +} + type CSIVolumeIndexSort []*CSIVolumeListStub func (v CSIVolumeIndexSort) Len() int { @@ -171,6 +228,50 @@ type CSIVolumeListStub struct { ModifyIndex uint64 } +type CSIVolumeListExternalResponse struct { + Volumes []*CSIVolumeExternalStub + NextToken string +} + +// CSIVolumeExternalStub is the storage provider's view of a volume, as +// returned from the controller plugin; all IDs are for external resources +type CSIVolumeExternalStub struct { + ExternalID string + CapacityBytes int64 + VolumeContext map[string]string + CloneID string + SnapshotID string + PublishedExternalNodeIDs []string + IsAbnormal bool + Status string +} + +// We can't sort external volumes by creation time because we don't get that +// data back from the storage provider. Sort by External ID within this page. +type CSIVolumeExternalStubSort []*CSIVolumeExternalStub + +func (v CSIVolumeExternalStubSort) Len() int { + return len(v) +} + +func (v CSIVolumeExternalStubSort) Less(i, j int) bool { + return v[i].ExternalID > v[j].ExternalID +} + +func (v CSIVolumeExternalStubSort) Swap(i, j int) { + v[i], v[j] = v[j], v[i] +} + +type CSIVolumeCreateRequest struct { + Volumes []*CSIVolume + WriteRequest +} + +type CSIVolumeCreateResponse struct { + Volumes []*CSIVolume + QueryMeta +} + type CSIVolumeRegisterRequest struct { Volumes []*CSIVolume WriteRequest diff --git a/client/client.go b/client/client.go index 6252e950045..5dad2ee974b 100644 --- a/client/client.go +++ b/client/client.go @@ -308,8 +308,12 @@ var ( noServersErr = errors.New("no servers") ) -// NewClient is used to create a new client from the given configuration -func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulProxies consulApi.SupportedProxiesAPI, consulService consulApi.ConsulServiceAPI) (*Client, error) { +// NewClient is used to create a new client from the given configuration. +// `rpcs` is a map of RPC names to RPC structs that, if non-nil, will be +// registered via https://golang.org/pkg/net/rpc/#Server.RegisterName in place +// of the client's normal RPC handlers. This allows server tests to override +// the behavior of the client. +func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulProxies consulApi.SupportedProxiesAPI, consulService consulApi.ConsulServiceAPI, rpcs map[string]interface{}) (*Client, error) { // Create the tls wrapper var tlsWrap tlsutil.RegionWrapper if cfg.TLSConfig.EnableRPC { @@ -384,7 +388,7 @@ func NewClient(cfg *config.Config, consulCatalog consul.CatalogAPI, consulProxie }) // Setup the clients RPC server - c.setupClientRpc() + c.setupClientRpc(rpcs) // Initialize the ACL state if err := c.clientACLResolver.init(); err != nil { diff --git a/client/client_test.go b/client/client_test.go index c00c18a4f6d..ee5215e4987 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -622,7 +622,7 @@ func TestClient_SaveRestoreState(t *testing.T) { c1.config.PluginLoader = catalog.TestPluginLoaderWithOptions(t, "", c1.config.Options, nil) c1.config.PluginSingletonLoader = singleton.NewSingletonLoader(logger, c1.config.PluginLoader) - c2, err := NewClient(c1.config, consulCatalog, nil, mockService) + c2, err := NewClient(c1.config, consulCatalog, nil, mockService, nil) if err != nil { t.Fatalf("err: %v", err) } diff --git a/client/csi_endpoint.go b/client/csi_endpoint.go index 8f86dfe7f8c..9a01762f1d1 100644 --- a/client/csi_endpoint.go +++ b/client/csi_endpoint.go @@ -39,24 +39,25 @@ func (c *CSI) ControllerValidateVolume(req *structs.ClientCSIControllerValidateV defer metrics.MeasureSince([]string{"client", "csi_controller", "validate_volume"}, time.Now()) if req.VolumeID == "" { - return errors.New("VolumeID is required") + return errors.New("CSI.ControllerValidateVolume: VolumeID is required") } if req.PluginID == "" { - return errors.New("PluginID is required") + return errors.New("CSI.ControllerValidateVolume: PluginID is required") } plugin, err := c.findControllerPlugin(req.PluginID) if err != nil { // the server's view of the plugin health is stale, so let it know it // should retry with another controller instance - return fmt.Errorf("%w: %v", nstructs.ErrCSIClientRPCRetryable, err) + return fmt.Errorf("CSI.ControllerValidateVolume: %w: %v", + nstructs.ErrCSIClientRPCRetryable, err) } defer plugin.Close() csiReq, err := req.ToCSIRequest() if err != nil { - return err + return fmt.Errorf("CSI.ControllerValidateVolume: %v", err) } ctx, cancelFn := c.requestContext() @@ -64,10 +65,14 @@ func (c *CSI) ControllerValidateVolume(req *structs.ClientCSIControllerValidateV // CSI ValidateVolumeCapabilities errors for timeout, codes.Unavailable and // codes.ResourceExhausted are retried; all other errors are fatal. - return plugin.ControllerValidateCapabilities(ctx, csiReq, + err = plugin.ControllerValidateCapabilities(ctx, csiReq, grpc_retry.WithPerRetryTimeout(CSIPluginRequestTimeout), grpc_retry.WithMax(3), grpc_retry.WithBackoff(grpc_retry.BackoffExponential(100*time.Millisecond))) + if err != nil { + return fmt.Errorf("CSI.ControllerValidateVolume: %v", err) + } + return nil } // ControllerAttachVolume is used to attach a volume from a CSI Cluster to @@ -84,7 +89,8 @@ func (c *CSI) ControllerAttachVolume(req *structs.ClientCSIControllerAttachVolum if err != nil { // the server's view of the plugin health is stale, so let it know it // should retry with another controller instance - return fmt.Errorf("%w: %v", nstructs.ErrCSIClientRPCRetryable, err) + return fmt.Errorf("CSI.ControllerAttachVolume: %w: %v", + nstructs.ErrCSIClientRPCRetryable, err) } defer plugin.Close() @@ -94,16 +100,16 @@ func (c *CSI) ControllerAttachVolume(req *structs.ClientCSIControllerAttachVolum // requests to plugins, and to aid with development. if req.VolumeID == "" { - return errors.New("VolumeID is required") + return errors.New("CSI.ControllerAttachVolume: VolumeID is required") } if req.ClientCSINodeID == "" { - return errors.New("ClientCSINodeID is required") + return errors.New("CSI.ControllerAttachVolume: ClientCSINodeID is required") } csiReq, err := req.ToCSIRequest() if err != nil { - return err + return fmt.Errorf("CSI.ControllerAttachVolume: %v", err) } // Submit the request for a volume to the CSI Plugin. @@ -116,7 +122,7 @@ func (c *CSI) ControllerAttachVolume(req *structs.ClientCSIControllerAttachVolum grpc_retry.WithMax(3), grpc_retry.WithBackoff(grpc_retry.BackoffExponential(100*time.Millisecond))) if err != nil { - return err + return fmt.Errorf("CSI.ControllerAttachVolume: %v", err) } resp.PublishContext = cresp.PublishContext @@ -131,7 +137,8 @@ func (c *CSI) ControllerDetachVolume(req *structs.ClientCSIControllerDetachVolum if err != nil { // the server's view of the plugin health is stale, so let it know it // should retry with another controller instance - return fmt.Errorf("%w: %v", nstructs.ErrCSIClientRPCRetryable, err) + return fmt.Errorf("CSI.ControllerDetachVolume: %w: %v", + nstructs.ErrCSIClientRPCRetryable, err) } defer plugin.Close() @@ -141,11 +148,11 @@ func (c *CSI) ControllerDetachVolume(req *structs.ClientCSIControllerDetachVolum // requests to plugins, and to aid with development. if req.VolumeID == "" { - return errors.New("VolumeID is required") + return errors.New("CSI.ControllerDetachVolume: VolumeID is required") } if req.ClientCSINodeID == "" { - return errors.New("ClientCSINodeID is required") + return errors.New("CSI.ControllerDetachVolume: ClientCSINodeID is required") } csiReq := req.ToCSIRequest() @@ -159,15 +166,148 @@ func (c *CSI) ControllerDetachVolume(req *structs.ClientCSIControllerDetachVolum grpc_retry.WithPerRetryTimeout(CSIPluginRequestTimeout), grpc_retry.WithMax(3), grpc_retry.WithBackoff(grpc_retry.BackoffExponential(100*time.Millisecond))) + if errors.Is(err, nstructs.ErrCSIClientRPCIgnorable) { + // if the controller detach previously happened but the server failed to + // checkpoint, we'll get an error from the plugin but can safely ignore it. + c.c.logger.Debug("could not unpublish volume: %v", err) + return nil + } if err != nil { - if errors.Is(err, nstructs.ErrCSIClientRPCIgnorable) { - // if the controller detach previously happened but the server failed to - // checkpoint, we'll get an error from the plugin but can safely ignore it. - c.c.logger.Debug("could not unpublish volume: %v", err) - return nil + return fmt.Errorf("CSI.ControllerDetachVolume: %v", err) + } + return err +} + +func (c *CSI) ControllerCreateVolume(req *structs.ClientCSIControllerCreateVolumeRequest, resp *structs.ClientCSIControllerCreateVolumeResponse) error { + defer metrics.MeasureSince([]string{"client", "csi_controller", "create_volume"}, time.Now()) + + plugin, err := c.findControllerPlugin(req.PluginID) + if err != nil { + // the server's view of the plugin health is stale, so let it know it + // should retry with another controller instance + return fmt.Errorf("CSI.ControllerCreateVolume: %w: %v", + nstructs.ErrCSIClientRPCRetryable, err) + } + defer plugin.Close() + + csiReq, err := req.ToCSIRequest() + if err != nil { + return fmt.Errorf("CSI.ControllerCreateVolume: %v", err) + } + + ctx, cancelFn := c.requestContext() + defer cancelFn() + + // CSI ControllerCreateVolume errors for timeout, codes.Unavailable and + // codes.ResourceExhausted are retried; all other errors are fatal. + cresp, err := plugin.ControllerCreateVolume(ctx, csiReq, + grpc_retry.WithPerRetryTimeout(CSIPluginRequestTimeout), + grpc_retry.WithMax(3), + grpc_retry.WithBackoff(grpc_retry.BackoffExponential(100*time.Millisecond))) + if err != nil { + return fmt.Errorf("CSI.ControllerCreateVolume: %v", err) + } + + if cresp == nil || cresp.Volume == nil { + c.c.logger.Warn("plugin did not return error or volume; this is a bug in the plugin and should be reported to the plugin author") + return fmt.Errorf("CSI.ControllerCreateVolume: plugin did not return error or volume") + } + resp.ExternalVolumeID = cresp.Volume.ExternalVolumeID + resp.CapacityBytes = cresp.Volume.CapacityBytes + resp.VolumeContext = cresp.Volume.VolumeContext + + return nil +} + +func (c *CSI) ControllerDeleteVolume(req *structs.ClientCSIControllerDeleteVolumeRequest, resp *structs.ClientCSIControllerDeleteVolumeResponse) error { + defer metrics.MeasureSince([]string{"client", "csi_controller", "delete_volume"}, time.Now()) + + plugin, err := c.findControllerPlugin(req.PluginID) + if err != nil { + // the server's view of the plugin health is stale, so let it know it + // should retry with another controller instance + return fmt.Errorf("CSI.ControllerDeleteVolume: %w: %v", + nstructs.ErrCSIClientRPCRetryable, err) + } + defer plugin.Close() + + csiReq := req.ToCSIRequest() + + ctx, cancelFn := c.requestContext() + defer cancelFn() + + // CSI ControllerDeleteVolume errors for timeout, codes.Unavailable and + // codes.ResourceExhausted are retried; all other errors are fatal. + err = plugin.ControllerDeleteVolume(ctx, csiReq, + grpc_retry.WithPerRetryTimeout(CSIPluginRequestTimeout), + grpc_retry.WithMax(3), + grpc_retry.WithBackoff(grpc_retry.BackoffExponential(100*time.Millisecond))) + if errors.Is(err, nstructs.ErrCSIClientRPCIgnorable) { + // if the volume was deleted out-of-band, we'll get an error from + // the plugin but can safely ignore it + c.c.logger.Debug("could not delete volume: %v", err) + return nil + } + if err != nil { + return fmt.Errorf("CSI.ControllerDeleteVolume: %v", err) + } + return err +} + +func (c *CSI) ControllerListVolumes(req *structs.ClientCSIControllerListVolumesRequest, resp *structs.ClientCSIControllerListVolumesResponse) error { + defer metrics.MeasureSince([]string{"client", "csi_controller", "list_volumes"}, time.Now()) + + plugin, err := c.findControllerPlugin(req.PluginID) + if err != nil { + // the server's view of the plugin health is stale, so let it know it + // should retry with another controller instance + return fmt.Errorf("CSI.ControllerListVolumes: %w: %v", + nstructs.ErrCSIClientRPCRetryable, err) + } + defer plugin.Close() + + csiReq := req.ToCSIRequest() + + ctx, cancelFn := c.requestContext() + defer cancelFn() + + // CSI ControllerListVolumes errors for timeout, codes.Unavailable and + // codes.ResourceExhausted are retried; all other errors are fatal. + cresp, err := plugin.ControllerListVolumes(ctx, csiReq, + grpc_retry.WithPerRetryTimeout(CSIPluginRequestTimeout), + grpc_retry.WithMax(3), + grpc_retry.WithBackoff(grpc_retry.BackoffExponential(100*time.Millisecond))) + if err != nil { + return fmt.Errorf("CSI.ControllerListVolumes: %v", err) + } + + resp.NextToken = cresp.NextToken + resp.Entries = []*nstructs.CSIVolumeExternalStub{} + + for _, entry := range cresp.Entries { + if entry.Volume == nil { + return fmt.Errorf("CSI.ControllerListVolumes: plugin returned an invalid entry") + } + vol := &nstructs.CSIVolumeExternalStub{ + ExternalID: entry.Volume.ExternalVolumeID, + CapacityBytes: entry.Volume.CapacityBytes, + VolumeContext: entry.Volume.VolumeContext, + CloneID: entry.Volume.ContentSource.CloneID, + SnapshotID: entry.Volume.ContentSource.SnapshotID, + } + if entry.Status != nil { + vol.PublishedExternalNodeIDs = entry.Status.PublishedNodeIds + vol.IsAbnormal = entry.Status.VolumeCondition.Abnormal + if entry.Status.VolumeCondition != nil { + vol.Status = entry.Status.VolumeCondition.Message + } + } + resp.Entries = append(resp.Entries, vol) + if req.MaxEntries != 0 && int32(len(resp.Entries)) == req.MaxEntries { + break } - return err } + return nil } @@ -180,13 +320,13 @@ func (c *CSI) NodeDetachVolume(req *structs.ClientCSINodeDetachVolumeRequest, re // real Nomad cluster. They serve as a defensive check before forwarding // requests to plugins, and to aid with development. if req.PluginID == "" { - return errors.New("PluginID is required") + return errors.New("CSI.NodeDetachVolume: PluginID is required") } if req.VolumeID == "" { - return errors.New("VolumeID is required") + return errors.New("CSI.NodeDetachVolume: VolumeID is required") } if req.AllocID == "" { - return errors.New("AllocID is required") + return errors.New("CSI.NodeDetachVolume: AllocID is required") } ctx, cancelFn := c.requestContext() @@ -194,7 +334,7 @@ func (c *CSI) NodeDetachVolume(req *structs.ClientCSINodeDetachVolumeRequest, re mounter, err := c.c.csimanager.MounterForPlugin(ctx, req.PluginID) if err != nil { - return err + return fmt.Errorf("CSI.NodeDetachVolume: %v", err) } usageOpts := &csimanager.UsageOptions{ @@ -208,7 +348,7 @@ func (c *CSI) NodeDetachVolume(req *structs.ClientCSINodeDetachVolumeRequest, re // if the unmounting previously happened but the server failed to // checkpoint, we'll get an error from Unmount but can safely // ignore it. - return err + return fmt.Errorf("CSI.NodeDetachVolume: %v", err) } return nil } diff --git a/client/csi_endpoint_test.go b/client/csi_endpoint_test.go index 766d6a800d4..f1ac7d88e7d 100644 --- a/client/csi_endpoint_test.go +++ b/client/csi_endpoint_test.go @@ -41,7 +41,7 @@ func TestCSIController_AttachVolume(t *testing.T) { PluginID: "some-garbage", }, }, - ExpectedErr: errors.New("CSI client error (retryable): plugin some-garbage for type csi-controller not found"), + ExpectedErr: errors.New("CSI.ControllerAttachVolume: CSI client error (retryable): plugin some-garbage for type csi-controller not found"), }, { Name: "validates volumeid is not empty", @@ -50,7 +50,7 @@ func TestCSIController_AttachVolume(t *testing.T) { PluginID: fakePlugin.Name, }, }, - ExpectedErr: errors.New("VolumeID is required"), + ExpectedErr: errors.New("CSI.ControllerAttachVolume: VolumeID is required"), }, { Name: "validates nodeid is not empty", @@ -60,7 +60,7 @@ func TestCSIController_AttachVolume(t *testing.T) { }, VolumeID: "1234-4321-1234-4321", }, - ExpectedErr: errors.New("ClientCSINodeID is required"), + ExpectedErr: errors.New("CSI.ControllerAttachVolume: ClientCSINodeID is required"), }, { Name: "validates AccessMode", @@ -73,7 +73,7 @@ func TestCSIController_AttachVolume(t *testing.T) { AttachmentMode: nstructs.CSIVolumeAttachmentModeFilesystem, AccessMode: nstructs.CSIVolumeAccessMode("foo"), }, - ExpectedErr: errors.New("Unknown volume access mode: foo"), + ExpectedErr: errors.New("CSI.ControllerAttachVolume: unknown volume access mode: foo"), }, { Name: "validates attachmentmode is not empty", @@ -86,7 +86,7 @@ func TestCSIController_AttachVolume(t *testing.T) { AccessMode: nstructs.CSIVolumeAccessModeMultiNodeReader, AttachmentMode: nstructs.CSIVolumeAttachmentMode("bar"), }, - ExpectedErr: errors.New("Unknown volume attachment mode: bar"), + ExpectedErr: errors.New("CSI.ControllerAttachVolume: unknown volume attachment mode: bar"), }, { Name: "returns transitive errors", @@ -102,7 +102,7 @@ func TestCSIController_AttachVolume(t *testing.T) { AccessMode: nstructs.CSIVolumeAccessModeSingleNodeWriter, AttachmentMode: nstructs.CSIVolumeAttachmentModeFilesystem, }, - ExpectedErr: errors.New("hello"), + ExpectedErr: errors.New("CSI.ControllerAttachVolume: hello"), }, { Name: "handles nil PublishContext", @@ -188,7 +188,7 @@ func TestCSIController_ValidateVolume(t *testing.T) { PluginID: fakePlugin.Name, }, }, - ExpectedErr: errors.New("VolumeID is required"), + ExpectedErr: errors.New("CSI.ControllerValidateVolume: VolumeID is required"), }, { Name: "returns plugin not found errors", @@ -198,7 +198,7 @@ func TestCSIController_ValidateVolume(t *testing.T) { }, VolumeID: "foo", }, - ExpectedErr: errors.New("CSI client error (retryable): plugin some-garbage for type csi-controller not found"), + ExpectedErr: errors.New("CSI.ControllerValidateVolume: CSI client error (retryable): plugin some-garbage for type csi-controller not found"), }, { Name: "validates attachmentmode", @@ -210,7 +210,7 @@ func TestCSIController_ValidateVolume(t *testing.T) { AttachmentMode: nstructs.CSIVolumeAttachmentMode("bar"), AccessMode: nstructs.CSIVolumeAccessModeMultiNodeReader, }, - ExpectedErr: errors.New("Unknown volume attachment mode: bar"), + ExpectedErr: errors.New("CSI.ControllerValidateVolume: unknown volume attachment mode: bar"), }, { Name: "validates AccessMode", @@ -222,7 +222,7 @@ func TestCSIController_ValidateVolume(t *testing.T) { AttachmentMode: nstructs.CSIVolumeAttachmentModeFilesystem, AccessMode: nstructs.CSIVolumeAccessMode("foo"), }, - ExpectedErr: errors.New("Unknown volume access mode: foo"), + ExpectedErr: errors.New("CSI.ControllerValidateVolume: unknown volume access mode: foo"), }, { Name: "returns transitive errors", @@ -237,7 +237,7 @@ func TestCSIController_ValidateVolume(t *testing.T) { AccessMode: nstructs.CSIVolumeAccessModeSingleNodeWriter, AttachmentMode: nstructs.CSIVolumeAttachmentModeFilesystem, }, - ExpectedErr: errors.New("hello"), + ExpectedErr: errors.New("CSI.ControllerValidateVolume: hello"), }, } @@ -287,7 +287,7 @@ func TestCSIController_DetachVolume(t *testing.T) { PluginID: "some-garbage", }, }, - ExpectedErr: errors.New("CSI client error (retryable): plugin some-garbage for type csi-controller not found"), + ExpectedErr: errors.New("CSI.ControllerDetachVolume: CSI client error (retryable): plugin some-garbage for type csi-controller not found"), }, { Name: "validates volumeid is not empty", @@ -296,7 +296,7 @@ func TestCSIController_DetachVolume(t *testing.T) { PluginID: fakePlugin.Name, }, }, - ExpectedErr: errors.New("VolumeID is required"), + ExpectedErr: errors.New("CSI.ControllerDetachVolume: VolumeID is required"), }, { Name: "validates nodeid is not empty", @@ -306,7 +306,7 @@ func TestCSIController_DetachVolume(t *testing.T) { }, VolumeID: "1234-4321-1234-4321", }, - ExpectedErr: errors.New("ClientCSINodeID is required"), + ExpectedErr: errors.New("CSI.ControllerDetachVolume: ClientCSINodeID is required"), }, { Name: "returns transitive errors", @@ -320,7 +320,7 @@ func TestCSIController_DetachVolume(t *testing.T) { VolumeID: "1234-4321-1234-4321", ClientCSINodeID: "abcde", }, - ExpectedErr: errors.New("hello"), + ExpectedErr: errors.New("CSI.ControllerDetachVolume: hello"), }, } @@ -353,6 +353,281 @@ func TestCSIController_DetachVolume(t *testing.T) { } } +func TestCSIController_CreateVolume(t *testing.T) { + t.Parallel() + + cases := []struct { + Name string + ClientSetupFunc func(*fake.Client) + Request *structs.ClientCSIControllerCreateVolumeRequest + ExpectedErr error + ExpectedResponse *structs.ClientCSIControllerCreateVolumeResponse + }{ + { + Name: "returns plugin not found errors", + Request: &structs.ClientCSIControllerCreateVolumeRequest{ + CSIControllerQuery: structs.CSIControllerQuery{ + PluginID: "some-garbage", + }, + }, + ExpectedErr: errors.New("CSI.ControllerCreateVolume: CSI client error (retryable): plugin some-garbage for type csi-controller not found"), + }, + { + Name: "validates AccessMode", + Request: &structs.ClientCSIControllerCreateVolumeRequest{ + CSIControllerQuery: structs.CSIControllerQuery{ + PluginID: fakePlugin.Name, + }, + Name: "1234-4321-1234-4321", + VolumeCapabilities: []*nstructs.CSIVolumeCapability{ + { + AttachmentMode: nstructs.CSIVolumeAttachmentModeFilesystem, + AccessMode: nstructs.CSIVolumeAccessMode("foo"), + }, + }, + }, + ExpectedErr: errors.New("CSI.ControllerCreateVolume: unknown volume access mode: foo"), + }, + { + Name: "validates attachmentmode is not empty", + Request: &structs.ClientCSIControllerCreateVolumeRequest{ + CSIControllerQuery: structs.CSIControllerQuery{ + PluginID: fakePlugin.Name, + }, + Name: "1234-4321-1234-4321", + VolumeCapabilities: []*nstructs.CSIVolumeCapability{ + { + AccessMode: nstructs.CSIVolumeAccessModeMultiNodeReader, + AttachmentMode: nstructs.CSIVolumeAttachmentMode("bar"), + }, + }, + }, + ExpectedErr: errors.New("CSI.ControllerCreateVolume: unknown volume attachment mode: bar"), + }, + { + Name: "returns transitive errors", + ClientSetupFunc: func(fc *fake.Client) { + fc.NextControllerCreateVolumeErr = errors.New("internal plugin error") + }, + Request: &structs.ClientCSIControllerCreateVolumeRequest{ + CSIControllerQuery: structs.CSIControllerQuery{ + PluginID: fakePlugin.Name, + }, + Name: "1234-4321-1234-4321", + VolumeCapabilities: []*nstructs.CSIVolumeCapability{ + { + AccessMode: nstructs.CSIVolumeAccessModeSingleNodeWriter, + AttachmentMode: nstructs.CSIVolumeAttachmentModeFilesystem, + }, + }, + }, + ExpectedErr: errors.New("CSI.ControllerCreateVolume: internal plugin error"), + }, + } + + for _, tc := range cases { + t.Run(tc.Name, func(t *testing.T) { + require := require.New(t) + client, cleanup := TestClient(t, nil) + defer cleanup() + + fakeClient := &fake.Client{} + if tc.ClientSetupFunc != nil { + tc.ClientSetupFunc(fakeClient) + } + + dispenserFunc := func(*dynamicplugins.PluginInfo) (interface{}, error) { + return fakeClient, nil + } + client.dynamicRegistry.StubDispenserForType( + dynamicplugins.PluginTypeCSIController, dispenserFunc) + + err := client.dynamicRegistry.RegisterPlugin(fakePlugin) + require.Nil(err) + + var resp structs.ClientCSIControllerCreateVolumeResponse + err = client.ClientRPC("CSI.ControllerCreateVolume", tc.Request, &resp) + require.Equal(tc.ExpectedErr, err) + if tc.ExpectedResponse != nil { + require.Equal(tc.ExpectedResponse, &resp) + } + }) + } +} + +func TestCSIController_DeleteVolume(t *testing.T) { + t.Parallel() + + cases := []struct { + Name string + ClientSetupFunc func(*fake.Client) + Request *structs.ClientCSIControllerDeleteVolumeRequest + ExpectedErr error + ExpectedResponse *structs.ClientCSIControllerDeleteVolumeResponse + }{ + { + Name: "returns plugin not found errors", + Request: &structs.ClientCSIControllerDeleteVolumeRequest{ + CSIControllerQuery: structs.CSIControllerQuery{ + PluginID: "some-garbage", + }, + }, + ExpectedErr: errors.New("CSI.ControllerDeleteVolume: CSI client error (retryable): plugin some-garbage for type csi-controller not found"), + }, + { + Name: "returns transitive errors", + ClientSetupFunc: func(fc *fake.Client) { + fc.NextControllerDeleteVolumeErr = errors.New("internal plugin error") + }, + Request: &structs.ClientCSIControllerDeleteVolumeRequest{ + CSIControllerQuery: structs.CSIControllerQuery{ + PluginID: fakePlugin.Name, + }, + ExternalVolumeID: "1234-4321-1234-4321", + }, + ExpectedErr: errors.New("CSI.ControllerDeleteVolume: internal plugin error"), + }, + } + + for _, tc := range cases { + t.Run(tc.Name, func(t *testing.T) { + require := require.New(t) + client, cleanup := TestClient(t, nil) + defer cleanup() + + fakeClient := &fake.Client{} + if tc.ClientSetupFunc != nil { + tc.ClientSetupFunc(fakeClient) + } + + dispenserFunc := func(*dynamicplugins.PluginInfo) (interface{}, error) { + return fakeClient, nil + } + client.dynamicRegistry.StubDispenserForType( + dynamicplugins.PluginTypeCSIController, dispenserFunc) + + err := client.dynamicRegistry.RegisterPlugin(fakePlugin) + require.Nil(err) + + var resp structs.ClientCSIControllerDeleteVolumeResponse + err = client.ClientRPC("CSI.ControllerDeleteVolume", tc.Request, &resp) + require.Equal(tc.ExpectedErr, err) + if tc.ExpectedResponse != nil { + require.Equal(tc.ExpectedResponse, &resp) + } + }) + } +} + +func TestCSIController_ListVolumes(t *testing.T) { + t.Parallel() + + cases := []struct { + Name string + ClientSetupFunc func(*fake.Client) + Request *structs.ClientCSIControllerListVolumesRequest + ExpectedErr error + ExpectedResponse *structs.ClientCSIControllerListVolumesResponse + }{ + { + Name: "returns plugin not found errors", + Request: &structs.ClientCSIControllerListVolumesRequest{ + CSIControllerQuery: structs.CSIControllerQuery{ + PluginID: "some-garbage", + }, + }, + ExpectedErr: errors.New("CSI.ControllerListVolumes: CSI client error (retryable): plugin some-garbage for type csi-controller not found"), + }, + { + Name: "returns transitive errors", + ClientSetupFunc: func(fc *fake.Client) { + fc.NextControllerListVolumesErr = errors.New("internal plugin error") + }, + Request: &structs.ClientCSIControllerListVolumesRequest{ + CSIControllerQuery: structs.CSIControllerQuery{ + PluginID: fakePlugin.Name, + }, + }, + ExpectedErr: errors.New("CSI.ControllerListVolumes: internal plugin error"), + }, + { + Name: "returns volumes", + ClientSetupFunc: func(fc *fake.Client) { + fc.NextControllerListVolumesResponse = &csi.ControllerListVolumesResponse{ + Entries: []*csi.ListVolumesResponse_Entry{ + { + Volume: &csi.Volume{ + CapacityBytes: 1000000, + ExternalVolumeID: "vol-1", + VolumeContext: map[string]string{"foo": "bar"}, + ContentSource: &csi.VolumeContentSource{ + SnapshotID: "snap-1", + }, + }, + Status: &csi.ListVolumesResponse_VolumeStatus{ + PublishedNodeIds: []string{"i-1234", "i-5678"}, + VolumeCondition: &csi.VolumeCondition{ + Message: "ok", + }, + }, + }, + }, + NextToken: "2", + } + }, + Request: &structs.ClientCSIControllerListVolumesRequest{ + CSIControllerQuery: structs.CSIControllerQuery{ + PluginID: fakePlugin.Name, + }, + StartingToken: "1", + MaxEntries: 100, + }, + ExpectedResponse: &structs.ClientCSIControllerListVolumesResponse{ + Entries: []*nstructs.CSIVolumeExternalStub{ + { + ExternalID: "vol-1", + CapacityBytes: 1000000, + VolumeContext: map[string]string{"foo": "bar"}, + SnapshotID: "snap-1", + PublishedExternalNodeIDs: []string{"i-1234", "i-5678"}, + Status: "ok", + }, + }, + NextToken: "2", + }, + }, + } + + for _, tc := range cases { + t.Run(tc.Name, func(t *testing.T) { + require := require.New(t) + client, cleanup := TestClient(t, nil) + defer cleanup() + + fakeClient := &fake.Client{} + if tc.ClientSetupFunc != nil { + tc.ClientSetupFunc(fakeClient) + } + + dispenserFunc := func(*dynamicplugins.PluginInfo) (interface{}, error) { + return fakeClient, nil + } + client.dynamicRegistry.StubDispenserForType( + dynamicplugins.PluginTypeCSIController, dispenserFunc) + + err := client.dynamicRegistry.RegisterPlugin(fakePlugin) + require.Nil(err) + + var resp structs.ClientCSIControllerListVolumesResponse + err = client.ClientRPC("CSI.ControllerListVolumes", tc.Request, &resp) + require.Equal(tc.ExpectedErr, err) + if tc.ExpectedResponse != nil { + require.Equal(tc.ExpectedResponse, &resp) + } + }) + } +} + func TestCSINode_DetachVolume(t *testing.T) { t.Parallel() @@ -374,14 +649,14 @@ func TestCSINode_DetachVolume(t *testing.T) { AccessMode: nstructs.CSIVolumeAccessModeMultiNodeReader, ReadOnly: true, }, - ExpectedErr: errors.New("plugin some-garbage for type csi-node not found"), + ExpectedErr: errors.New("CSI.NodeDetachVolume: plugin some-garbage for type csi-node not found"), }, { Name: "validates volumeid is not empty", Request: &structs.ClientCSINodeDetachVolumeRequest{ PluginID: fakeNodePlugin.Name, }, - ExpectedErr: errors.New("VolumeID is required"), + ExpectedErr: errors.New("CSI.NodeDetachVolume: VolumeID is required"), }, { Name: "validates nodeid is not empty", @@ -389,7 +664,7 @@ func TestCSINode_DetachVolume(t *testing.T) { PluginID: fakeNodePlugin.Name, VolumeID: "1234-4321-1234-4321", }, - ExpectedErr: errors.New("AllocID is required"), + ExpectedErr: errors.New("CSI.NodeDetachVolume: AllocID is required"), }, { Name: "returns transitive errors", @@ -402,7 +677,7 @@ func TestCSINode_DetachVolume(t *testing.T) { AllocID: "4321-1234-4321-1234", }, // we don't have a csimanager in this context - ExpectedErr: errors.New("plugin test-plugin for type csi-node not found"), + ExpectedErr: errors.New("CSI.NodeDetachVolume: plugin test-plugin for type csi-node not found"), }, } diff --git a/client/pluginmanager/csimanager/fingerprint.go b/client/pluginmanager/csimanager/fingerprint.go index b0da9c8fcf5..c35131a83c5 100644 --- a/client/pluginmanager/csimanager/fingerprint.go +++ b/client/pluginmanager/csimanager/fingerprint.go @@ -122,10 +122,18 @@ func (p *pluginFingerprinter) buildBasicFingerprint(ctx context.Context) (*struc } func applyCapabilitySetToControllerInfo(cs *csi.ControllerCapabilitySet, info *structs.CSIControllerInfo) { - info.SupportsReadOnlyAttach = cs.HasPublishReadonly + info.SupportsCreateDelete = cs.HasCreateDeleteVolume info.SupportsAttachDetach = cs.HasPublishUnpublishVolume info.SupportsListVolumes = cs.HasListVolumes + info.SupportsGetCapacity = cs.HasGetCapacity + info.SupportsCreateDeleteSnapshot = cs.HasCreateDeleteSnapshot + info.SupportsListSnapshots = cs.HasListSnapshots + info.SupportsClone = cs.HasCloneVolume + info.SupportsReadOnlyAttach = cs.HasPublishReadonly + info.SupportsExpand = cs.HasExpandVolume info.SupportsListVolumesAttachedNodes = cs.HasListVolumesPublishedNodes + info.SupportsCondition = cs.HasVolumeCondition + info.SupportsGet = cs.HasGetVolume } func (p *pluginFingerprinter) buildControllerFingerprint(ctx context.Context, base *structs.CSIInfo) (*structs.CSIInfo, error) { diff --git a/client/pluginmanager/csimanager/volume_test.go b/client/pluginmanager/csimanager/volume_test.go index bd5320daa0c..283ac8d5035 100644 --- a/client/pluginmanager/csimanager/volume_test.go +++ b/client/pluginmanager/csimanager/volume_test.go @@ -152,7 +152,7 @@ func TestVolumeManager_stageVolume(t *testing.T) { AttachmentMode: "nonsense", }, UsageOptions: &UsageOptions{}, - ExpectedErr: errors.New("Unknown volume attachment mode: nonsense"), + ExpectedErr: errors.New("unknown volume attachment mode: nonsense"), }, { Name: "Returns an error when an invalid AccessMode is provided", @@ -162,7 +162,7 @@ func TestVolumeManager_stageVolume(t *testing.T) { AccessMode: "nonsense", }, UsageOptions: &UsageOptions{}, - ExpectedErr: errors.New("Unknown volume access mode: nonsense"), + ExpectedErr: errors.New("unknown volume access mode: nonsense"), }, { Name: "Returns an error when the plugin returns an error", @@ -490,14 +490,14 @@ func TestVolumeManager_MountVolumeEvents(t *testing.T) { pubCtx := map[string]string{} _, err := manager.MountVolume(ctx, vol, alloc, usage, pubCtx) - require.Error(t, err, "Unknown volume attachment mode: ") + require.Error(t, err, "unknown volume attachment mode: ") require.Equal(t, 1, len(events)) e := events[0] require.Equal(t, "Mount volume", e.Message) require.Equal(t, "Storage", e.Subsystem) require.Equal(t, "vol", e.Details["volume_id"]) require.Equal(t, "false", e.Details["success"]) - require.Equal(t, "Unknown volume attachment mode: ", e.Details["error"]) + require.Equal(t, "unknown volume attachment mode: ", e.Details["error"]) events = events[1:] vol.AttachmentMode = structs.CSIVolumeAttachmentModeFilesystem diff --git a/client/rpc.go b/client/rpc.go index 11ea5bf919f..c106f5d4f4c 100644 --- a/client/rpc.go +++ b/client/rpc.go @@ -245,19 +245,24 @@ func (c *Client) streamingRpcConn(server *servers.Server, method string) (net.Co } // setupClientRpc is used to setup the Client's RPC endpoints -func (c *Client) setupClientRpc() { - // Initialize the RPC handlers - c.endpoints.ClientStats = &ClientStats{c} - c.endpoints.CSI = &CSI{c} - c.endpoints.FileSystem = NewFileSystemEndpoint(c) - c.endpoints.Allocations = NewAllocationsEndpoint(c) - c.endpoints.Agent = NewAgentEndpoint(c) - +func (c *Client) setupClientRpc(rpcs map[string]interface{}) { // Create the RPC Server c.rpcServer = rpc.NewServer() - // Register the endpoints with the RPC server - c.setupClientRpcServer(c.rpcServer) + // Initialize the RPC handlers + if rpcs != nil { + // override RPCs + for name, rpc := range rpcs { + c.rpcServer.RegisterName(name, rpc) + } + } else { + c.endpoints.ClientStats = &ClientStats{c} + c.endpoints.CSI = &CSI{c} + c.endpoints.FileSystem = NewFileSystemEndpoint(c) + c.endpoints.Allocations = NewAllocationsEndpoint(c) + c.endpoints.Agent = NewAgentEndpoint(c) + c.setupClientRpcServer(c.rpcServer) + } go c.rpcConnListener() } diff --git a/client/structs/csi.go b/client/structs/csi.go index ba6a46e0dc8..74848a23769 100644 --- a/client/structs/csi.go +++ b/client/structs/csi.go @@ -20,6 +20,12 @@ type CSIVolumeMountOptions struct { MountFlags []string } +// CSIControllerRequest interface lets us set embedded CSIControllerQuery +// fields in the server +type CSIControllerRequest interface { + SetControllerNodeID(string) +} + // CSIControllerQuery is used to specify various flags for queries against CSI // Controllers type CSIControllerQuery struct { @@ -30,6 +36,10 @@ type CSIControllerQuery struct { PluginID string } +func (c *CSIControllerQuery) SetControllerNodeID(nodeID string) { + c.ControllerNodeID = nodeID +} + type ClientCSIControllerValidateVolumeRequest struct { VolumeID string // note: this is the external ID @@ -175,6 +185,104 @@ func (c *ClientCSIControllerDetachVolumeRequest) ToCSIRequest() *csi.ControllerU type ClientCSIControllerDetachVolumeResponse struct{} +// ClientCSIControllerCreateVolumeRequest the RPC made from the server to a +// Nomad client to tell a CSI controller plugin on that client to perform +// CreateVolume +type ClientCSIControllerCreateVolumeRequest struct { + Name string + VolumeCapabilities []*structs.CSIVolumeCapability + Parameters map[string]string + Secrets structs.CSISecrets + CapacityMin int64 + CapacityMax int64 + SnapshotID string + CloneID string + // TODO: topology is not yet supported + // TopologyRequirement + + CSIControllerQuery +} + +func (req *ClientCSIControllerCreateVolumeRequest) ToCSIRequest() (*csi.ControllerCreateVolumeRequest, error) { + + creq := &csi.ControllerCreateVolumeRequest{ + Name: req.Name, + CapacityRange: &csi.CapacityRange{ + RequiredBytes: req.CapacityMin, + LimitBytes: req.CapacityMax, + }, + VolumeCapabilities: []*csi.VolumeCapability{}, + Parameters: req.Parameters, + Secrets: req.Secrets, + ContentSource: &csi.VolumeContentSource{ + CloneID: req.CloneID, + SnapshotID: req.SnapshotID, + }, + // TODO: topology is not yet supported + AccessibilityRequirements: &csi.TopologyRequirement{}, + } + for _, cap := range req.VolumeCapabilities { + ccap, err := csi.VolumeCapabilityFromStructs(cap.AttachmentMode, cap.AccessMode) + if err != nil { + return nil, err + } + creq.VolumeCapabilities = append(creq.VolumeCapabilities, ccap) + } + return creq, nil +} + +type ClientCSIControllerCreateVolumeResponse struct { + ExternalVolumeID string + CapacityBytes int64 + VolumeContext map[string]string + + // TODO: topology is not yet supported + // AccessibleTopology []*Topology +} + +// ClientCSIControllerDeleteVolumeRequest the RPC made from the server to a +// Nomad client to tell a CSI controller plugin on that client to perform +// DeleteVolume +type ClientCSIControllerDeleteVolumeRequest struct { + ExternalVolumeID string + Secrets structs.CSISecrets + + CSIControllerQuery +} + +func (req *ClientCSIControllerDeleteVolumeRequest) ToCSIRequest() *csi.ControllerDeleteVolumeRequest { + return &csi.ControllerDeleteVolumeRequest{ + ExternalVolumeID: req.ExternalVolumeID, + Secrets: req.Secrets, + } +} + +type ClientCSIControllerDeleteVolumeResponse struct{} + +// ClientCSIControllerListVolumesVolumeRequest the RPC made from the server to +// a Nomad client to tell a CSI controller plugin on that client to perform +// ListVolumes +type ClientCSIControllerListVolumesRequest struct { + // these pagination fields match the pagination fields of the plugins and + // not Nomad's own fields, for clarity when mapping between the two RPCs + MaxEntries int32 + StartingToken string + + CSIControllerQuery +} + +func (req *ClientCSIControllerListVolumesRequest) ToCSIRequest() *csi.ControllerListVolumesRequest { + return &csi.ControllerListVolumesRequest{ + MaxEntries: req.MaxEntries, + StartingToken: req.StartingToken, + } +} + +type ClientCSIControllerListVolumesResponse struct { + Entries []*structs.CSIVolumeExternalStub + NextToken string +} + // ClientCSINodeDetachVolumeRequest is the RPC made from the server to // a Nomad client to tell a CSI node plugin on that client to perform // NodeUnpublish and NodeUnstage. diff --git a/client/testing.go b/client/testing.go index 6ce3ddd29e0..94681f76e18 100644 --- a/client/testing.go +++ b/client/testing.go @@ -2,14 +2,18 @@ package client import ( "fmt" + "net" + "net/rpc" "time" "github.com/hashicorp/nomad/client/config" consulapi "github.com/hashicorp/nomad/client/consul" "github.com/hashicorp/nomad/client/fingerprint" + "github.com/hashicorp/nomad/client/servers" agentconsul "github.com/hashicorp/nomad/command/agent/consul" "github.com/hashicorp/nomad/helper/pluginutils/catalog" "github.com/hashicorp/nomad/helper/pluginutils/singleton" + "github.com/hashicorp/nomad/helper/pool" "github.com/hashicorp/nomad/helper/testlog" testing "github.com/mitchellh/go-testing-interface" ) @@ -21,6 +25,10 @@ import ( // and removed in the returned cleanup function. If they are overridden in the // callback then the caller still must run the returned cleanup func. func TestClient(t testing.T, cb func(c *config.Config)) (*Client, func() error) { + return TestClientWithRPCs(t, cb, nil) +} + +func TestClientWithRPCs(t testing.T, cb func(c *config.Config), rpcs map[string]interface{}) (*Client, func() error) { conf, cleanup := config.TestClientConfig(t) // Tighten the fingerprinter timeouts (must be done in client package @@ -46,7 +54,7 @@ func TestClient(t testing.T, cb func(c *config.Config)) (*Client, func() error) } mockCatalog := agentconsul.NewMockCatalog(logger) mockService := consulapi.NewMockConsulServiceClient(t, logger) - client, err := NewClient(conf, mockCatalog, nil, mockService) + client, err := NewClient(conf, mockCatalog, nil, mockService, rpcs) if err != nil { cleanup() t.Fatalf("err: %v", err) @@ -75,3 +83,51 @@ func TestClient(t testing.T, cb func(c *config.Config)) (*Client, func() error) } } } + +// TestRPCOnlyClient is a client that only pings to establish a connection +// with the server and then returns mock RPC responses for those interfaces +// passed in the `rpcs` parameter. Useful for testing client RPCs from the +// server. Returns the Client, a shutdown function, and any error. +func TestRPCOnlyClient(t testing.T, srvAddr net.Addr, rpcs map[string]interface{}) (*Client, func() error, error) { + var err error + conf, cleanup := config.TestClientConfig(t) + + client := &Client{config: conf, logger: testlog.HCLogger(t)} + client.servers = servers.New(client.logger, client.shutdownCh, client) + client.configCopy = client.config.Copy() + + client.rpcServer = rpc.NewServer() + for name, rpc := range rpcs { + client.rpcServer.RegisterName(name, rpc) + } + + client.connPool = pool.NewPool(testlog.HCLogger(t), 10*time.Second, 10, nil) + + cancelFunc := func() error { + ch := make(chan error) + + go func() { + defer close(ch) + client.connPool.Shutdown() + client.shutdownGroup.Wait() + cleanup() + }() + + select { + case <-ch: + return nil + case <-time.After(1 * time.Minute): + return fmt.Errorf("timed out while shutting down client") + } + } + + go client.rpcConnListener() + + _, err = client.SetServers([]string{srvAddr.String()}) + if err != nil { + return nil, cancelFunc, fmt.Errorf("could not set servers: %v", err) + } + client.shutdownGroup.Go(client.registerAndHeartbeat) + + return client, cancelFunc, nil +} diff --git a/command/agent/agent.go b/command/agent/agent.go index b98ce1ce7da..32c134bb134 100644 --- a/command/agent/agent.go +++ b/command/agent/agent.go @@ -861,7 +861,8 @@ func (a *Agent) setupClient() error { conf.StateDBFactory = state.GetStateDBFactory(conf.DevMode) } - nomadClient, err := client.NewClient(conf, a.consulCatalog, a.consulProxies, a.consulService) + nomadClient, err := client.NewClient( + conf, a.consulCatalog, a.consulProxies, a.consulService, nil) if err != nil { return fmt.Errorf("client setup failed: %v", err) } diff --git a/command/agent/csi_endpoint.go b/command/agent/csi_endpoint.go index 1a2b91e8237..c5bc9c7ee01 100644 --- a/command/agent/csi_endpoint.go +++ b/command/agent/csi_endpoint.go @@ -12,12 +12,29 @@ import ( func (s *HTTPServer) CSIVolumesRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) { switch req.Method { case http.MethodPut, http.MethodPost: - return s.csiVolumePut(resp, req) + return s.csiVolumeRegister(resp, req) case http.MethodGet: default: return nil, CodedError(405, ErrInvalidMethod) } + // Tokenize the suffix of the path to get the volume id + reqSuffix := strings.TrimPrefix(req.URL.Path, "/v1/volumes") + tokens := strings.Split(reqSuffix, "/") + if len(tokens) == 2 { + if tokens[1] == "external" { + return s.csiVolumesListExternal(resp, req) + } + return nil, CodedError(404, resourceNotFoundErr) + } else if len(tokens) > 2 { + return nil, CodedError(404, resourceNotFoundErr) + } + + return s.csiVolumesList(resp, req) +} + +func (s *HTTPServer) csiVolumesList(resp http.ResponseWriter, req *http.Request) (interface{}, error) { + // Type filters volume lists to a specific type. When support for non-CSI volumes is // introduced, we'll need to dispatch here query := req.URL.Query() @@ -30,19 +47,13 @@ func (s *HTTPServer) CSIVolumesRequest(resp http.ResponseWriter, req *http.Reque } args := structs.CSIVolumeListRequest{} - if s.parse(resp, req, &args.Region, &args.QueryOptions) { return nil, nil } - if prefix, ok := query["prefix"]; ok { - args.Prefix = prefix[0] - } - if plugin, ok := query["plugin_id"]; ok { - args.PluginID = plugin[0] - } - if node, ok := query["node_id"]; ok { - args.NodeID = node[0] - } + + args.Prefix = query.Get("prefix") + args.PluginID = query.Get("plugin_id") + args.NodeID = query.Get("node_id") var out structs.CSIVolumeListResponse if err := s.agent.RPC("CSIVolume.List", &args, &out); err != nil { @@ -53,6 +64,25 @@ func (s *HTTPServer) CSIVolumesRequest(resp http.ResponseWriter, req *http.Reque return out.Volumes, nil } +func (s *HTTPServer) csiVolumesListExternal(resp http.ResponseWriter, req *http.Request) (interface{}, error) { + + args := structs.CSIVolumeExternalListRequest{} + if s.parse(resp, req, &args.Region, &args.QueryOptions) { + return nil, nil + } + + query := req.URL.Query() + args.PluginID = query.Get("plugin_id") + + var out structs.CSIVolumeExternalListResponse + if err := s.agent.RPC("CSIVolume.ListExternal", &args, &out); err != nil { + return nil, err + } + + setMeta(resp, &out.QueryMeta) + return out.Volumes, nil +} + // CSIVolumeSpecificRequest dispatches GET and PUT func (s *HTTPServer) CSIVolumeSpecificRequest(resp http.ResponseWriter, req *http.Request) (interface{}, error) { // Tokenize the suffix of the path to get the volume id @@ -68,22 +98,30 @@ func (s *HTTPServer) CSIVolumeSpecificRequest(resp http.ResponseWriter, req *htt case http.MethodGet: return s.csiVolumeGet(id, resp, req) case http.MethodPut: - return s.csiVolumePut(resp, req) + return s.csiVolumeRegister(resp, req) case http.MethodDelete: - return s.csiVolumeDelete(id, resp, req) + return s.csiVolumeDeregister(id, resp, req) default: return nil, CodedError(405, ErrInvalidMethod) } } if len(tokens) == 2 { - if tokens[1] != "detach" { - return nil, CodedError(404, resourceNotFoundErr) - } - if req.Method != http.MethodDelete { + switch req.Method { + case http.MethodPut: + if tokens[1] == "create" { + return s.csiVolumeCreate(resp, req) + } + case http.MethodDelete: + if tokens[1] == "detach" { + return s.csiVolumeDetach(id, resp, req) + } + if tokens[1] == "delete" { + return s.csiVolumeDelete(id, resp, req) + } + default: return nil, CodedError(405, ErrInvalidMethod) } - return s.csiVolumeDetach(id, resp, req) } return nil, CodedError(404, resourceNotFoundErr) @@ -117,21 +155,17 @@ func (s *HTTPServer) csiVolumeGet(id string, resp http.ResponseWriter, req *http return vol, nil } -func (s *HTTPServer) csiVolumePut(resp http.ResponseWriter, req *http.Request) (interface{}, error) { +func (s *HTTPServer) csiVolumeRegister(resp http.ResponseWriter, req *http.Request) (interface{}, error) { switch req.Method { case http.MethodPost, http.MethodPut: default: return nil, CodedError(405, ErrInvalidMethod) } - args0 := structs.CSIVolumeRegisterRequest{} - if err := decodeBody(req, &args0); err != nil { + args := structs.CSIVolumeRegisterRequest{} + if err := decodeBody(req, &args); err != nil { return err, CodedError(400, err.Error()) } - - args := structs.CSIVolumeRegisterRequest{ - Volumes: args0.Volumes, - } s.parseWriteRequest(req, &args.WriteRequest) var out structs.CSIVolumeRegisterResponse @@ -144,7 +178,30 @@ func (s *HTTPServer) csiVolumePut(resp http.ResponseWriter, req *http.Request) ( return nil, nil } -func (s *HTTPServer) csiVolumeDelete(id string, resp http.ResponseWriter, req *http.Request) (interface{}, error) { +func (s *HTTPServer) csiVolumeCreate(resp http.ResponseWriter, req *http.Request) (interface{}, error) { + switch req.Method { + case http.MethodPost, http.MethodPut: + default: + return nil, CodedError(405, ErrInvalidMethod) + } + + args := structs.CSIVolumeCreateRequest{} + if err := decodeBody(req, &args); err != nil { + return err, CodedError(400, err.Error()) + } + s.parseWriteRequest(req, &args.WriteRequest) + + var out structs.CSIVolumeCreateResponse + if err := s.agent.RPC("CSIVolume.Create", &args, &out); err != nil { + return nil, err + } + + setMeta(resp, &out.QueryMeta) + + return out, nil +} + +func (s *HTTPServer) csiVolumeDeregister(id string, resp http.ResponseWriter, req *http.Request) (interface{}, error) { if req.Method != http.MethodDelete { return nil, CodedError(405, ErrInvalidMethod) } @@ -175,6 +232,26 @@ func (s *HTTPServer) csiVolumeDelete(id string, resp http.ResponseWriter, req *h return nil, nil } +func (s *HTTPServer) csiVolumeDelete(id string, resp http.ResponseWriter, req *http.Request) (interface{}, error) { + if req.Method != http.MethodDelete { + return nil, CodedError(405, ErrInvalidMethod) + } + + args := structs.CSIVolumeDeleteRequest{ + VolumeIDs: []string{id}, + } + s.parseWriteRequest(req, &args.WriteRequest) + + var out structs.CSIVolumeDeleteResponse + if err := s.agent.RPC("CSIVolume.Delete", &args, &out); err != nil { + return nil, err + } + + setMeta(resp, &out.QueryMeta) + + return nil, nil +} + func (s *HTTPServer) csiVolumeDetach(id string, resp http.ResponseWriter, req *http.Request) (interface{}, error) { if req.Method != http.MethodDelete { return nil, CodedError(405, ErrInvalidMethod) diff --git a/command/agent/csi_endpoint_test.go b/command/agent/csi_endpoint_test.go index 3415c9c52e3..bff28369780 100644 --- a/command/agent/csi_endpoint_test.go +++ b/command/agent/csi_endpoint_test.go @@ -58,7 +58,7 @@ func TestHTTP_CSIEndpointUtils(t *testing.T) { require.Equal(t, "bar", tops[0].Segments["foo"]) } -func TestHTTP_CSIEndpointVolume(t *testing.T) { +func TestHTTP_CSIEndpointRegisterVolume(t *testing.T) { t.Parallel() httpTest(t, nil, func(s *TestAgent) { server := s.Agent.Server() @@ -76,25 +76,55 @@ func TestHTTP_CSIEndpointVolume(t *testing.T) { body := encodeReq(args) req, err := http.NewRequest("PUT", "/v1/volumes", body) require.NoError(t, err) - resp := httptest.NewRecorder() _, err = s.Server.CSIVolumesRequest(resp, req) require.NoError(t, err, "put error") - require.Equal(t, 200, resp.Code) req, err = http.NewRequest("GET", "/v1/volume/csi/bar", nil) require.NoError(t, err) - resp = httptest.NewRecorder() raw, err := s.Server.CSIVolumeSpecificRequest(resp, req) require.NoError(t, err, "get error") - require.Equal(t, 200, resp.Code) - out, ok := raw.(*api.CSIVolume) require.True(t, ok) - require.Equal(t, 1, out.ControllersHealthy) require.Equal(t, 2, out.NodesHealthy) + + req, err = http.NewRequest("DELETE", "/v1/volume/csi/bar/detach", nil) + require.NoError(t, err) + resp = httptest.NewRecorder() + _, err = s.Server.CSIVolumeSpecificRequest(resp, req) + require.Equal(t, CodedError(400, "detach requires node ID"), err) + }) +} + +func TestHTTP_CSIEndpointCreateVolume(t *testing.T) { + t.Parallel() + httpTest(t, nil, func(s *TestAgent) { + server := s.Agent.Server() + cleanup := state.CreateTestCSIPlugin(server.State(), "foo") + defer cleanup() + + args := structs.CSIVolumeCreateRequest{ + Volumes: []*structs.CSIVolume{{ + ID: "baz", + PluginID: "foo", + AccessMode: structs.CSIVolumeAccessModeMultiNodeSingleWriter, + AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem, + }}, + } + body := encodeReq(args) + req, err := http.NewRequest("PUT", "/v1/volumes/create", body) + require.NoError(t, err) + resp := httptest.NewRecorder() + _, err = s.Server.CSIVolumesRequest(resp, req) + require.Error(t, err, "controller validate volume: No path to node") + + req, err = http.NewRequest("DELETE", "/v1/volume/csi/baz", nil) + require.NoError(t, err) + resp = httptest.NewRecorder() + _, err = s.Server.CSIVolumeSpecificRequest(resp, req) + require.Error(t, err, "volume not found: baz") }) } diff --git a/command/agent/http.go b/command/agent/http.go index b0a3b0895d8..c30c0de5253 100644 --- a/command/agent/http.go +++ b/command/agent/http.go @@ -692,9 +692,25 @@ func (s *HTTPServer) parse(resp http.ResponseWriter, req *http.Request, r *strin parseConsistency(req, b) parsePrefix(req, b) parseNamespace(req, &b.Namespace) + parsePagination(req, b) return parseWait(resp, req, b) } +// parsePagination parses the pagination fields for QueryOptions +func parsePagination(req *http.Request, b *structs.QueryOptions) { + query := req.URL.Query() + rawPerPage := query.Get("per_page") + if rawPerPage != "" { + perPage, err := strconv.Atoi(rawPerPage) + if err == nil { + b.PerPage = int32(perPage) + } + } + + nextToken := query.Get("next_token") + b.NextToken = nextToken +} + // parseWriteRequest is a convenience method for endpoints that need to parse a // write request. func (s *HTTPServer) parseWriteRequest(req *http.Request, w *structs.WriteRequest) { diff --git a/command/agent/http_test.go b/command/agent/http_test.go index c8ea3a732f0..bc71fcbc193 100644 --- a/command/agent/http_test.go +++ b/command/agent/http_test.go @@ -573,6 +573,49 @@ func TestParseBool(t *testing.T) { } } +func TestParsePagination(t *testing.T) { + t.Parallel() + s := makeHTTPServer(t, nil) + defer s.Shutdown() + + cases := []struct { + Input string + ExpectedNextToken string + ExpectedPerPage int32 + }{ + { + Input: "", + }, + { + Input: "next_token=a&per_page=3", + ExpectedNextToken: "a", + ExpectedPerPage: 3, + }, + { + Input: "next_token=a&next_token=b", + ExpectedNextToken: "a", + }, + { + Input: "per_page=a", + }, + } + + for i := range cases { + tc := cases[i] + t.Run("Input-"+tc.Input, func(t *testing.T) { + + req, err := http.NewRequest("GET", + "/v1/volumes/csi/external?"+tc.Input, nil) + + require.NoError(t, err) + opts := &structs.QueryOptions{} + parsePagination(req, opts) + require.Equal(t, tc.ExpectedNextToken, opts.NextToken) + require.Equal(t, tc.ExpectedPerPage, opts.PerPage) + }) + } +} + // TestHTTP_VerifyHTTPSClient asserts that a client certificate signed by the // appropriate CA is required when VerifyHTTPSClient=true. func TestHTTP_VerifyHTTPSClient(t *testing.T) { diff --git a/command/commands.go b/command/commands.go index 6f55efed7ac..3913ab54671 100644 --- a/command/commands.go +++ b/command/commands.go @@ -821,6 +821,16 @@ func Commands(metaPtr *Meta, agentUi cli.Ui) map[string]cli.CommandFactory { Meta: meta, }, nil }, + "volume create": func() (cli.Command, error) { + return &VolumeCreateCommand{ + Meta: meta, + }, nil + }, + "volume delete": func() (cli.Command, error) { + return &VolumeDeleteCommand{ + Meta: meta, + }, nil + }, } deprecated := map[string]cli.CommandFactory{ diff --git a/command/volume.go b/command/volume.go index 6260b0bfc6b..2d228ac36ce 100644 --- a/command/volume.go +++ b/command/volume.go @@ -32,6 +32,14 @@ Usage: nomad volume [options] $ nomad volume detach + Create an external volume and register it: + + $ nomad volume create + + Delete an external volume and deregister it: + + $ nomad volume delete + Please see the individual subcommand help for detailed usage information. ` return strings.TrimSpace(helpText) diff --git a/command/volume_create.go b/command/volume_create.go new file mode 100644 index 00000000000..9eaeff5cb86 --- /dev/null +++ b/command/volume_create.go @@ -0,0 +1,105 @@ +package command + +import ( + "fmt" + "io/ioutil" + "os" + "strings" + + "github.com/posener/complete" +) + +type VolumeCreateCommand struct { + Meta +} + +func (c *VolumeCreateCommand) Help() string { + helpText := ` +Usage: nomad volume create [options] + + Creates a volume in an external storage provider and registers it in Nomad. + + If the supplied path is "-" the volume file is read from stdin. Otherwise, it + is read from the file at the supplied path. + + When ACLs are enabled, this command requires a token with the + 'csi-write-volume' capability for the volume's namespace. + +General Options: + + ` + generalOptionsUsage(usageOptsDefault) + + return strings.TrimSpace(helpText) +} + +func (c *VolumeCreateCommand) AutocompleteFlags() complete.Flags { + return c.Meta.AutocompleteFlags(FlagSetClient) +} + +func (c *VolumeCreateCommand) AutocompleteArgs() complete.Predictor { + return complete.PredictFiles("*") +} + +func (c *VolumeCreateCommand) Synopsis() string { + return "Create an external volume" +} + +func (c *VolumeCreateCommand) Name() string { return "volume create" } + +func (c *VolumeCreateCommand) Run(args []string) int { + flags := c.Meta.FlagSet(c.Name(), FlagSetClient) + flags.Usage = func() { c.Ui.Output(c.Help()) } + + if err := flags.Parse(args); err != nil { + c.Ui.Error(fmt.Sprintf("Error parsing arguments %s", err)) + return 1 + } + + // Check that we get exactly one argument + args = flags.Args() + if l := len(args); l != 1 { + c.Ui.Error("This command takes one argument: ") + c.Ui.Error(commandErrorText(c)) + return 1 + } + + // Read the file contents + file := args[0] + var rawVolume []byte + var err error + if file == "-" { + rawVolume, err = ioutil.ReadAll(os.Stdin) + if err != nil { + c.Ui.Error(fmt.Sprintf("Failed to read stdin: %v", err)) + return 1 + } + } else { + rawVolume, err = ioutil.ReadFile(file) + if err != nil { + c.Ui.Error(fmt.Sprintf("Failed to read file: %v", err)) + return 1 + } + } + + ast, volType, err := parseVolumeType(string(rawVolume)) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error parsing the volume type: %s", err)) + return 1 + } + + // Get the HTTP client + client, err := c.Meta.Client() + if err != nil { + c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) + return 1 + } + + switch strings.ToLower(volType) { + case "csi": + code := c.csiCreate(client, ast) + return code + default: + c.Ui.Error(fmt.Sprintf("Error unknown volume type: %s", volType)) + return 1 + } +} diff --git a/command/volume_create_csi.go b/command/volume_create_csi.go new file mode 100644 index 00000000000..90a6c13d469 --- /dev/null +++ b/command/volume_create_csi.go @@ -0,0 +1,29 @@ +package command + +import ( + "fmt" + + "github.com/hashicorp/hcl/hcl/ast" + "github.com/hashicorp/nomad/api" +) + +func (c *VolumeCreateCommand) csiCreate(client *api.Client, ast *ast.File) int { + vol, err := csiDecodeVolume(ast) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error decoding the volume definition: %s", err)) + return 1 + } + + vols, _, err := client.CSIVolumes().Create(vol, nil) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error creating volume: %s", err)) + return 1 + } + for _, vol := range vols { + // note: the command only ever returns 1 volume from the API + c.Ui.Output(fmt.Sprintf( + "Created external volume %s with ID %s", vol.ExternalID, vol.ID)) + } + + return 0 +} diff --git a/command/volume_delete.go b/command/volume_delete.go new file mode 100644 index 00000000000..6ee508248e0 --- /dev/null +++ b/command/volume_delete.go @@ -0,0 +1,102 @@ +package command + +import ( + "fmt" + "strings" + + "github.com/hashicorp/nomad/api/contexts" + "github.com/posener/complete" +) + +type VolumeDeleteCommand struct { + Meta +} + +func (c *VolumeDeleteCommand) Help() string { + helpText := ` +Usage: nomad volume delete [options] + + Delete a volume from an external storage provider. The volume must still be + registered with Nomad in order to be deleted. Deleting will fail if the + volume is still in use by an allocation or in the process of being + unpublished. If the volume no longer exists, this command will silently + return without an error. + + When ACLs are enabled, this command requires a token with the + 'csi-write-volume' and 'csi-read-volume' capabilities for the volume's + namespace. + +General Options: + + ` + generalOptionsUsage(usageOptsDefault) + ` + +` + return strings.TrimSpace(helpText) +} + +func (c *VolumeDeleteCommand) AutocompleteFlags() complete.Flags { + return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient), + complete.Flags{}) +} + +func (c *VolumeDeleteCommand) AutocompleteArgs() complete.Predictor { + return complete.PredictFunc(func(a complete.Args) []string { + client, err := c.Meta.Client() + if err != nil { + return nil + } + + resp, _, err := client.Search().PrefixSearch(a.Last, contexts.Volumes, nil) + if err != nil { + return []string{} + } + matches := resp.Matches[contexts.Volumes] + + resp, _, err = client.Search().PrefixSearch(a.Last, contexts.Nodes, nil) + if err != nil { + return []string{} + } + matches = append(matches, resp.Matches[contexts.Nodes]...) + return matches + }) +} + +func (c *VolumeDeleteCommand) Synopsis() string { + return "Delete a volume" +} + +func (c *VolumeDeleteCommand) Name() string { return "volume delete" } + +func (c *VolumeDeleteCommand) Run(args []string) int { + flags := c.Meta.FlagSet(c.Name(), FlagSetClient) + flags.Usage = func() { c.Ui.Output(c.Help()) } + + if err := flags.Parse(args); err != nil { + c.Ui.Error(fmt.Sprintf("Error parsing arguments %s", err)) + return 1 + } + + // Check that we get exactly two arguments + args = flags.Args() + if l := len(args); l != 1 { + c.Ui.Error("This command takes one argument: ") + c.Ui.Error(commandErrorText(c)) + return 1 + } + volID := args[0] + + // Get the HTTP client + client, err := c.Meta.Client() + if err != nil { + c.Ui.Error(fmt.Sprintf("Error initializing client: %s", err)) + return 1 + } + + err = client.CSIVolumes().Delete(volID, nil) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error deleting volume: %s", err)) + return 1 + } + + return 0 +} diff --git a/command/volume_init.go b/command/volume_init.go index 19ee1fe142f..12402b9ef57 100644 --- a/command/volume_init.go +++ b/command/volume_init.go @@ -105,23 +105,59 @@ func (c *VolumeInitCommand) Run(args []string) int { } var defaultHclVolumeSpec = strings.TrimSpace(` -id = "ebs_prod_db1" -name = "database" -type = "csi" -external_id = "vol-23452345" -access_mode = "single-node-writer" -attachment_mode = "file-system" +id = "ebs_prod_db1" +name = "database" +type = "csi" +plugin_id = "plugin_id" + +# For 'nomad volume register', provide the external ID from the storage +# provider. This field should be omitted when creating a volume with +# 'nomad volume create' +external_id = "vol-23452345" + +# For 'nomad volume create', specify a snapshot ID or volume to clone. You can +# specify only one of these two fields. +snapshot_id = "snap-12345" +# clone_id = "vol-abcdef" + +# Optional: for 'nomad volume create', specify a maximum and minimum capacity. +# Registering an existing volume will record but ignore these fields. +capacity_min = "10GiB" +capacity_max = "20G" + +# Optional: for 'nomad volume create', specify one or more capabilities to +# validate. Registering an existing volume will record but ignore these fields. +capability { + access_mode = "single-node-writer" + attachment_mode = "file-system" +} + +capability { + access_mode = "single-node-reader" + attachment_mode = "block-device" +} +# Optional: for 'nomad volume create', specify mount options to +# validate. Registering an existing volume will record but ignore these +# fields. mount_options { fs_type = "ext4" mount_flags = ["ro"] } + +# Optional: provide any secrets specified by the plugin. secrets { example_secret = "xyzzy" } + +# Optional: provide a map of keys to string values expected by the plugin. parameters { skuname = "Premium_LRS" } + +# Optional: for 'nomad volume register', provide a map of keys to string +# values expected by the plugin. This field will populated automatically by +# 'nomad volume create'. context { endpoint = "http://192.168.1.101:9425" } @@ -132,23 +168,43 @@ var defaultJsonVolumeSpec = strings.TrimSpace(` "id": "ebs_prod_db1", "name": "database", "type": "csi", + "plugin_id": "plugin_id", "external_id": "vol-23452345", - "access_mode": "single-node-writer", - "attachment_mode": "file-system", - "mount_options": { - "fs_type": "ext4", - "mount_flags": [ - "ro" - ] - }, - "secrets": { - "example_secret": "xyzzy" - }, - "parameters": { - "skuname": "Premium_LRS" - }, - "context": { - "endpoint": "http://192.168.1.101:9425" - } + "snapshot_id": "snap-12345", + "capacity_min": "10GiB", + "capacity_max": "20G", + "capability": [ + { + "access_mode": "single-node-writer", + "attachment_mode": "file-system" + }, + { + "access_mode": "single-node-reader", + "attachment_mode": "block-device" + } + ], + "context": [ + { + "endpoint": "http://192.168.1.101:9425" + } + ], + "mount_options": [ + { + "fs_type": "ext4", + "mount_flags": [ + "ro" + ] + } + ], + "parameters": [ + { + "skuname": "Premium_LRS" + } + ], + "secrets": [ + { + "example_secret": "xyzzy" + } + ] } `) diff --git a/command/volume_register_csi.go b/command/volume_register_csi.go index a7f74f2f39a..b88950eb429 100644 --- a/command/volume_register_csi.go +++ b/command/volume_register_csi.go @@ -2,11 +2,14 @@ package command import ( "fmt" + "strings" + humanize "github.com/dustin/go-humanize" "github.com/hashicorp/hcl" "github.com/hashicorp/hcl/hcl/ast" "github.com/hashicorp/nomad/api" "github.com/hashicorp/nomad/helper" + "github.com/mitchellh/mapstructure" ) func (c *VolumeRegisterCommand) csiRegister(client *api.Client, ast *ast.File) int { @@ -24,21 +27,111 @@ func (c *VolumeRegisterCommand) csiRegister(client *api.Client, ast *ast.File) i return 0 } -// parseVolume is used to parse the quota specification from HCL func csiDecodeVolume(input *ast.File) (*api.CSIVolume, error) { - output := &api.CSIVolume{} - err := hcl.DecodeObject(output, input) + var err error + vol := &api.CSIVolume{} + + list, ok := input.Node.(*ast.ObjectList) + if !ok { + return nil, fmt.Errorf("error parsing: root should be an object") + } + + // Decode the full thing into a map[string]interface for ease + var m map[string]interface{} + err = hcl.DecodeObject(&m, list) if err != nil { return nil, err } - // api.CSIVolume doesn't have the type field, it's used only for dispatch in - // parseVolumeType - helper.RemoveEqualFold(&output.ExtraKeysHCL, "type") - err = helper.UnusedKeys(output) + // Need to manually parse these fields + delete(m, "capability") + delete(m, "mount_options") + delete(m, "capacity_max") + delete(m, "capacity_min") + delete(m, "type") + + // Decode the rest + err = mapstructure.WeakDecode(m, vol) if err != nil { return nil, err } - return output, nil + capacityMin, err := parseCapacityBytes(list.Filter("capacity_min")) + if err != nil { + return nil, fmt.Errorf("invalid capacity_min: %v", err) + } + vol.RequestedCapacityMin = capacityMin + capacityMax, err := parseCapacityBytes(list.Filter("capacity_max")) + if err != nil { + return nil, fmt.Errorf("invalid capacity_max: %v", err) + } + vol.RequestedCapacityMax = capacityMax + + capObj := list.Filter("capability") + if len(capObj.Items) > 0 { + + for _, o := range capObj.Elem().Items { + valid := []string{"access_mode", "attachment_mode"} + if err := helper.CheckHCLKeys(o.Val, valid); err != nil { + return nil, err + } + + ot, ok := o.Val.(*ast.ObjectType) + if !ok { + break + } + + var m map[string]interface{} + if err := hcl.DecodeObject(&m, ot.List); err != nil { + return nil, err + } + var cap *api.CSIVolumeCapability + if err := mapstructure.WeakDecode(&m, &cap); err != nil { + return nil, err + } + + vol.RequestedCapabilities = append(vol.RequestedCapabilities, cap) + } + } + + mObj := list.Filter("mount_options") + if len(mObj.Items) > 0 { + + for _, o := range mObj.Elem().Items { + valid := []string{"fs_type", "mount_flags"} + if err := helper.CheckHCLKeys(o.Val, valid); err != nil { + return nil, err + } + + ot, ok := o.Val.(*ast.ObjectType) + if !ok { + break + } + var opts *api.CSIMountOptions + if err := hcl.DecodeObject(&opts, ot.List); err != nil { + return nil, err + } + vol.MountOptions = opts + break + } + } + + return vol, nil +} + +func parseCapacityBytes(cap *ast.ObjectList) (int64, error) { + if len(cap.Items) > 0 { + for _, o := range cap.Elem().Items { + lit, ok := o.Val.(*ast.LiteralType) + if !ok { + break + } + b, err := humanize.ParseBytes(strings.Trim(lit.Token.Text, "\"")) + if err != nil { + return 0, fmt.Errorf("could not parse value as bytes: %v", err) + } + return int64(b), err + } + } + return 0, fmt.Errorf("could not parse value as bytes") } diff --git a/command/volume_register_test.go b/command/volume_register_test.go index 14152be1ceb..1499132be46 100644 --- a/command/volume_register_test.go +++ b/command/volume_register_test.go @@ -42,60 +42,90 @@ rando = "bar" } } -func TestCSIVolumeParse(t *testing.T) { +func TestCSIVolumeDecode(t *testing.T) { t.Parallel() cases := []struct { - hcl string - q *api.CSIVolume - err string + name string + hcl string + expected *api.CSIVolume + err string }{{ + name: "typical volume", hcl: ` -id = "foo" -type = "csi" -namespace = "n" -access_mode = "single-node-writer" -attachment_mode = "file-system" -plugin_id = "p" +id = "testvolume" +name = "test" +type = "csi" +plugin_id = "myplugin" + +capacity_min = "10 MiB" +capacity_max = "1G" +snapshot_id = "snap-12345" + +mount_options { + fs_type = "ext4" + mount_flags = ["ro"] +} + secrets { - mysecret = "secretvalue" + password = "xyzzy" +} + +parameters { + skuname = "Premium_LRS" +} + +capability { + access_mode = "single-node-writer" + attachment_mode = "file-system" +} + +capability { + access_mode = "single-node-reader-only" + attachment_mode = "block-device" } `, - q: &api.CSIVolume{ - ID: "foo", - Namespace: "n", - AccessMode: "single-node-writer", - AttachmentMode: "file-system", - PluginID: "p", - Secrets: api.CSISecrets{"mysecret": "secretvalue"}, - }, - err: "", - }, { - hcl: ` -{"id": "foo", "namespace": "n", "type": "csi", "access_mode": "single-node-writer", "attachment_mode": "file-system", -"plugin_id": "p"} -`, - q: &api.CSIVolume{ - ID: "foo", - Namespace: "n", - AccessMode: "single-node-writer", - AttachmentMode: "file-system", - PluginID: "p", + expected: &api.CSIVolume{ + ID: "testvolume", + Name: "test", + PluginID: "myplugin", + SnapshotID: "snap-12345", + RequestedCapacityMin: 10485760, + RequestedCapacityMax: 1000000000, + RequestedCapabilities: []*api.CSIVolumeCapability{ + { + AccessMode: api.CSIVolumeAccessModeSingleNodeWriter, + AttachmentMode: api.CSIVolumeAttachmentModeFilesystem, + }, + { + AccessMode: api.CSIVolumeAccessModeSingleNodeReader, + AttachmentMode: api.CSIVolumeAttachmentModeBlockDevice, + }, + }, + MountOptions: &api.CSIMountOptions{ + FSType: "ext4", + MountFlags: []string{"ro"}, + }, + Parameters: map[string]string{"skuname": "Premium_LRS"}, + Secrets: map[string]string{"password": "xyzzy"}, }, err: "", - }} + }, + } for _, c := range cases { - t.Run(c.hcl, func(t *testing.T) { + t.Run(c.name, func(t *testing.T) { ast, err := hcl.ParseString(c.hcl) require.NoError(t, err) vol, err := csiDecodeVolume(ast) - require.Equal(t, c.q, vol) if c.err == "" { require.NoError(t, err) } else { require.Contains(t, err.Error(), c.err) } + require.Equal(t, c.expected, vol) + }) + } } diff --git a/command/volume_status_csi.go b/command/volume_status_csi.go index d81191c072f..78502c6c103 100644 --- a/command/volume_status_csi.go +++ b/command/volume_status_csi.go @@ -18,25 +18,7 @@ func (c *VolumeStatusCommand) csiBanner() { func (c *VolumeStatusCommand) csiStatus(client *api.Client, id string) int { // Invoke list mode if no volume id if id == "" { - c.csiBanner() - vols, _, err := client.CSIVolumes().List(nil) - if err != nil { - c.Ui.Error(fmt.Sprintf("Error querying volumes: %s", err)) - return 1 - } - - if len(vols) == 0 { - // No output if we have no volumes - c.Ui.Error("No CSI volumes") - } else { - str, err := c.csiFormatVolumes(vols) - if err != nil { - c.Ui.Error(fmt.Sprintf("Error formatting: %s", err)) - return 1 - } - c.Ui.Output(str) - } - return 0 + return c.listVolumes(client) } // Prefix search for the volume @@ -77,6 +59,89 @@ func (c *VolumeStatusCommand) csiStatus(client *api.Client, id string) int { return 0 } +func (c *VolumeStatusCommand) listVolumes(client *api.Client) int { + + c.csiBanner() + vols, _, err := client.CSIVolumes().List(nil) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error querying volumes: %s", err)) + return 1 + } + + if len(vols) == 0 { + // No output if we have no volumes + c.Ui.Error("No CSI volumes") + } else { + str, err := c.csiFormatVolumes(vols) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error formatting: %s", err)) + return 1 + } + c.Ui.Output(str) + } + if !c.verbose { + return 0 + } + + plugins, _, err := client.CSIPlugins().List(nil) + if err != nil { + c.Ui.Error(fmt.Sprintf("Error querying CSI plugins: %s", err)) + return 1 + } + + if len(plugins) == 0 { + return 0 // No more output if we have no plugins + } + + var code int + q := &api.QueryOptions{PerPage: 30} // TODO: tune page size + + for _, plugin := range plugins { + if !plugin.ControllerRequired || plugin.ControllersHealthy < 1 { + continue // only controller plugins can support this query + } + for { + externalList, _, err := client.CSIVolumes().ListExternal(plugin.ID, q) + if err != nil { + c.Ui.Error(fmt.Sprintf( + "Error querying CSI external volumes for plugin %q: %s", plugin.ID, err)) + // we'll stop querying this plugin, but there may be more to + // query, so report and set the error code but move on to the + // next plugin + code = 1 + continue + } + rows := []string{} + if len(externalList.Volumes) > 0 { + rows[0] = "External ID|Condition|Nodes" + for i, v := range externalList.Volumes { + condition := "OK" + if v.IsAbnormal { + condition = fmt.Sprintf("Abnormal (%v)", v.Status) + } + + rows[i+1] = fmt.Sprintf("%s|%s|%s", + limit(v.ExternalID, c.length), + limit(condition, 20), + strings.Join(v.PublishedExternalNodeIDs, ","), + ) + } + c.Ui.Output(formatList(rows)) + } + + q.NextToken = externalList.NextToken + if q.NextToken == "" { + break + } + // we can't know the shape of arbitrarily-sized lists of volumes, + // so break after each page + c.Ui.Output("...") + } + } + + return code +} + func (c *VolumeStatusCommand) csiFormatVolumes(vols []*api.CSIVolumeListStub) (string, error) { // Sort the output by volume id sort.Slice(vols, func(i, j int) bool { return vols[i].ID < vols[j].ID }) diff --git a/nomad/client_csi_endpoint.go b/nomad/client_csi_endpoint.go index 932b35b5a49..2f07af6d687 100644 --- a/nomad/client_csi_endpoint.go +++ b/nomad/client_csi_endpoint.go @@ -22,80 +22,98 @@ type ClientCSI struct { func (a *ClientCSI) ControllerAttachVolume(args *cstructs.ClientCSIControllerAttachVolumeRequest, reply *cstructs.ClientCSIControllerAttachVolumeResponse) error { defer metrics.MeasureSince([]string{"nomad", "client_csi_controller", "attach_volume"}, time.Now()) - clientIDs, err := a.clientIDsForController(args.PluginID) + err := a.sendCSIControllerRPC(args.PluginID, + "CSI.ControllerAttachVolume", + "ClientCSI.ControllerAttachVolume", + args, reply) if err != nil { return fmt.Errorf("controller attach volume: %v", err) } + return nil +} - for _, clientID := range clientIDs { - args.ControllerNodeID = clientID - state, ok := a.srv.getNodeConn(clientID) - if !ok { - return findNodeConnAndForward(a.srv, - clientID, "ClientCSI.ControllerAttachVolume", args, reply) - } +func (a *ClientCSI) ControllerValidateVolume(args *cstructs.ClientCSIControllerValidateVolumeRequest, reply *cstructs.ClientCSIControllerValidateVolumeResponse) error { + defer metrics.MeasureSince([]string{"nomad", "client_csi_controller", "validate_volume"}, time.Now()) - err = NodeRpc(state.Session, "CSI.ControllerAttachVolume", args, reply) - if err == nil { - return nil - } - if a.isRetryable(err) { - a.logger.Debug("failed to reach controller on client", - "nodeID", clientID, "err", err) - continue - } - return fmt.Errorf("controller attach volume: %v", err) + err := a.sendCSIControllerRPC(args.PluginID, + "CSI.ControllerValidateVolume", + "ClientCSI.ControllerValidateVolume", + args, reply) + if err != nil { + return fmt.Errorf("controller validate volume: %v", err) } - return fmt.Errorf("controller attach volume: %v", err) + return nil } -func (a *ClientCSI) ControllerValidateVolume(args *cstructs.ClientCSIControllerValidateVolumeRequest, reply *cstructs.ClientCSIControllerValidateVolumeResponse) error { - defer metrics.MeasureSince([]string{"nomad", "client_csi_controller", "validate_volume"}, time.Now()) +func (a *ClientCSI) ControllerDetachVolume(args *cstructs.ClientCSIControllerDetachVolumeRequest, reply *cstructs.ClientCSIControllerDetachVolumeResponse) error { + defer metrics.MeasureSince([]string{"nomad", "client_csi_controller", "detach_volume"}, time.Now()) - clientIDs, err := a.clientIDsForController(args.PluginID) + err := a.sendCSIControllerRPC(args.PluginID, + "CSI.ControllerDetachVolume", + "ClientCSI.ControllerDetachVolume", + args, reply) if err != nil { - return fmt.Errorf("validate volume: %v", err) + return fmt.Errorf("controller detach volume: %v", err) } + return nil +} - for _, clientID := range clientIDs { - args.ControllerNodeID = clientID - state, ok := a.srv.getNodeConn(clientID) - if !ok { - return findNodeConnAndForward(a.srv, - clientID, "ClientCSI.ControllerValidateVolume", args, reply) - } +func (a *ClientCSI) ControllerCreateVolume(args *cstructs.ClientCSIControllerCreateVolumeRequest, reply *cstructs.ClientCSIControllerCreateVolumeResponse) error { + defer metrics.MeasureSince([]string{"nomad", "client_csi_controller", "create_volume"}, time.Now()) - err = NodeRpc(state.Session, "CSI.ControllerValidateVolume", args, reply) - if err == nil { - return nil - } - if a.isRetryable(err) { - a.logger.Debug("failed to reach controller on client", - "nodeID", clientID, "err", err) - continue - } - return fmt.Errorf("validate volume: %v", err) + err := a.sendCSIControllerRPC(args.PluginID, + "CSI.ControllerCreateVolume", + "ClientCSI.ControllerCreateVolume", + args, reply) + if err != nil { + return fmt.Errorf("controller create volume: %v", err) } - return fmt.Errorf("validate volume: %v", err) + return nil } -func (a *ClientCSI) ControllerDetachVolume(args *cstructs.ClientCSIControllerDetachVolumeRequest, reply *cstructs.ClientCSIControllerDetachVolumeResponse) error { - defer metrics.MeasureSince([]string{"nomad", "client_csi_controller", "detach_volume"}, time.Now()) +func (a *ClientCSI) ControllerDeleteVolume(args *cstructs.ClientCSIControllerDeleteVolumeRequest, reply *cstructs.ClientCSIControllerDeleteVolumeResponse) error { + defer metrics.MeasureSince([]string{"nomad", "client_csi_controller", "delete_volume"}, time.Now()) - clientIDs, err := a.clientIDsForController(args.PluginID) + err := a.sendCSIControllerRPC(args.PluginID, + "CSI.ControllerDeleteVolume", + "ClientCSI.ControllerDeleteVolume", + args, reply) if err != nil { - return fmt.Errorf("controller detach volume: %v", err) + return fmt.Errorf("controller delete volume: %v", err) + } + return nil +} + +func (a *ClientCSI) ControllerListVolumes(args *cstructs.ClientCSIControllerListVolumesRequest, reply *cstructs.ClientCSIControllerListVolumesResponse) error { + defer metrics.MeasureSince([]string{"nomad", "client_csi_controller", "list_volumes"}, time.Now()) + + err := a.sendCSIControllerRPC(args.PluginID, + "CSI.ControllerListVolumes", + "ClientCSI.ControllerListVolumes", + args, reply) + if err != nil { + return fmt.Errorf("controller list volumes: %v", err) + } + return nil +} + +func (a *ClientCSI) sendCSIControllerRPC(pluginID, method, fwdMethod string, args cstructs.CSIControllerRequest, reply interface{}) error { + + clientIDs, err := a.clientIDsForController(pluginID) + if err != nil { + return err } for _, clientID := range clientIDs { - args.ControllerNodeID = clientID + args.SetControllerNodeID(clientID) + state, ok := a.srv.getNodeConn(clientID) if !ok { return findNodeConnAndForward(a.srv, - clientID, "ClientCSI.ControllerDetachVolume", args, reply) + clientID, fwdMethod, args, reply) } - err = NodeRpc(state.Session, "CSI.ControllerDetachVolume", args, reply) + err = NodeRpc(state.Session, method, args, reply) if err == nil { return nil } @@ -104,9 +122,9 @@ func (a *ClientCSI) ControllerDetachVolume(args *cstructs.ClientCSIControllerDet "nodeID", clientID, "err", err) continue } - return fmt.Errorf("controller detach volume: %v", err) + return err } - return fmt.Errorf("controller detach volume: %v", err) + return err } // we can retry the same RPC on a different controller in the cases where the diff --git a/nomad/client_csi_endpoint_test.go b/nomad/client_csi_endpoint_test.go index 0852bc2f1cf..a67544c6c34 100644 --- a/nomad/client_csi_endpoint_test.go +++ b/nomad/client_csi_endpoint_test.go @@ -18,6 +18,63 @@ import ( "github.com/stretchr/testify/require" ) +// MockClientCSI is a mock for the nomad.ClientCSI RPC server (see +// nomad/client_csi_endpoint.go). This can be used with a TestRPCOnlyClient to +// return specific plugin responses back to server RPCs for testing. Note that +// responses that have no bodies have no "Next*Response" field and will always +// return an empty response body. +type MockClientCSI struct { + NextValidateError error + NextAttachError error + NextAttachResponse *cstructs.ClientCSIControllerAttachVolumeResponse + NextDetachError error + NextCreateError error + NextCreateResponse *cstructs.ClientCSIControllerCreateVolumeResponse + NextDeleteError error + NextListExternalError error + NextListExternalResponse *cstructs.ClientCSIControllerListVolumesResponse + NextNodeDetachError error +} + +func newMockClientCSI() *MockClientCSI { + return &MockClientCSI{ + NextAttachResponse: &cstructs.ClientCSIControllerAttachVolumeResponse{}, + NextCreateResponse: &cstructs.ClientCSIControllerCreateVolumeResponse{}, + NextListExternalResponse: &cstructs.ClientCSIControllerListVolumesResponse{}, + } +} + +func (c *MockClientCSI) ControllerValidateVolume(req *cstructs.ClientCSIControllerValidateVolumeRequest, resp *cstructs.ClientCSIControllerValidateVolumeResponse) error { + return c.NextValidateError +} + +func (c *MockClientCSI) ControllerAttachVolume(req *cstructs.ClientCSIControllerAttachVolumeRequest, resp *cstructs.ClientCSIControllerAttachVolumeResponse) error { + *resp = *c.NextAttachResponse + return c.NextAttachError +} + +func (c *MockClientCSI) ControllerDetachVolume(req *cstructs.ClientCSIControllerDetachVolumeRequest, resp *cstructs.ClientCSIControllerDetachVolumeResponse) error { + return c.NextDetachError +} + +func (c *MockClientCSI) ControllerCreateVolume(req *cstructs.ClientCSIControllerCreateVolumeRequest, resp *cstructs.ClientCSIControllerCreateVolumeResponse) error { + *resp = *c.NextCreateResponse + return c.NextCreateError +} + +func (c *MockClientCSI) ControllerDeleteVolume(req *cstructs.ClientCSIControllerDeleteVolumeRequest, resp *cstructs.ClientCSIControllerDeleteVolumeResponse) error { + return c.NextDeleteError +} + +func (c *MockClientCSI) ControllerListVolumes(req *cstructs.ClientCSIControllerListVolumesRequest, resp *cstructs.ClientCSIControllerListVolumesResponse) error { + *resp = *c.NextListExternalResponse + return c.NextListExternalError +} + +func (c *MockClientCSI) NodeDetachVolume(req *cstructs.ClientCSINodeDetachVolumeRequest, resp *cstructs.ClientCSINodeDetachVolumeResponse) error { + return c.NextNodeDetachError +} + func TestClientCSIController_AttachVolume_Local(t *testing.T) { t.Parallel() require := require.New(t) @@ -30,7 +87,7 @@ func TestClientCSIController_AttachVolume_Local(t *testing.T) { var resp structs.GenericResponse err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerAttachVolume", req, &resp) - require.NotNil(err) + require.Error(err) require.Contains(err.Error(), "no plugins registered for type") } @@ -46,7 +103,7 @@ func TestClientCSIController_AttachVolume_Forwarded(t *testing.T) { var resp structs.GenericResponse err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerAttachVolume", req, &resp) - require.NotNil(err) + require.Error(err) require.Contains(err.Error(), "no plugins registered for type") } @@ -62,7 +119,7 @@ func TestClientCSIController_DetachVolume_Local(t *testing.T) { var resp structs.GenericResponse err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerDetachVolume", req, &resp) - require.NotNil(err) + require.Error(err) require.Contains(err.Error(), "no plugins registered for type") } @@ -78,7 +135,7 @@ func TestClientCSIController_DetachVolume_Forwarded(t *testing.T) { var resp structs.GenericResponse err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerDetachVolume", req, &resp) - require.NotNil(err) + require.Error(err) require.Contains(err.Error(), "no plugins registered for type") } @@ -95,7 +152,7 @@ func TestClientCSIController_ValidateVolume_Local(t *testing.T) { var resp structs.GenericResponse err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerValidateVolume", req, &resp) - require.NotNil(err) + require.Error(err) require.Contains(err.Error(), "no plugins registered for type") } @@ -112,7 +169,105 @@ func TestClientCSIController_ValidateVolume_Forwarded(t *testing.T) { var resp structs.GenericResponse err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerValidateVolume", req, &resp) - require.NotNil(err) + require.Error(err) + require.Contains(err.Error(), "no plugins registered for type") +} + +func TestClientCSIController_CreateVolume_Local(t *testing.T) { + t.Parallel() + require := require.New(t) + codec, cleanup := setupLocal(t) + defer cleanup() + + req := &cstructs.ClientCSIControllerCreateVolumeRequest{ + CSIControllerQuery: cstructs.CSIControllerQuery{PluginID: "minnie"}, + } + + var resp structs.GenericResponse + err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerCreateVolume", req, &resp) + require.Error(err) + require.Contains(err.Error(), "no plugins registered for type") +} + +func TestClientCSIController_CreateVolume_Forwarded(t *testing.T) { + t.Parallel() + require := require.New(t) + codec, cleanup := setupForward(t) + defer cleanup() + + req := &cstructs.ClientCSIControllerCreateVolumeRequest{ + CSIControllerQuery: cstructs.CSIControllerQuery{PluginID: "minnie"}, + } + + var resp structs.GenericResponse + err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerCreateVolume", req, &resp) + require.Error(err) + require.Contains(err.Error(), "no plugins registered for type") +} + +func TestClientCSIController_DeleteVolume_Local(t *testing.T) { + t.Parallel() + require := require.New(t) + codec, cleanup := setupLocal(t) + defer cleanup() + + req := &cstructs.ClientCSIControllerDeleteVolumeRequest{ + ExternalVolumeID: "test", + CSIControllerQuery: cstructs.CSIControllerQuery{PluginID: "minnie"}, + } + + var resp structs.GenericResponse + err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerDeleteVolume", req, &resp) + require.Error(err) + require.Contains(err.Error(), "no plugins registered for type") +} + +func TestClientCSIController_DeleteVolume_Forwarded(t *testing.T) { + t.Parallel() + require := require.New(t) + codec, cleanup := setupForward(t) + defer cleanup() + + req := &cstructs.ClientCSIControllerDeleteVolumeRequest{ + ExternalVolumeID: "test", + CSIControllerQuery: cstructs.CSIControllerQuery{PluginID: "minnie"}, + } + + var resp structs.GenericResponse + err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerDeleteVolume", req, &resp) + require.Error(err) + require.Contains(err.Error(), "no plugins registered for type") +} + +func TestClientCSIController_ListVolumes_Local(t *testing.T) { + t.Parallel() + require := require.New(t) + codec, cleanup := setupLocal(t) + defer cleanup() + + req := &cstructs.ClientCSIControllerListVolumesRequest{ + CSIControllerQuery: cstructs.CSIControllerQuery{PluginID: "minnie"}, + } + + var resp structs.GenericResponse + err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerListVolumes", req, &resp) + require.Error(err) + require.Contains(err.Error(), "no plugins registered for type") +} + +func TestClientCSIController_ListVolumes_Forwarded(t *testing.T) { + t.Parallel() + require := require.New(t) + codec, cleanup := setupForward(t) + defer cleanup() + + req := &cstructs.ClientCSIControllerListVolumesRequest{ + CSIControllerQuery: cstructs.CSIControllerQuery{PluginID: "minnie"}, + } + + var resp structs.GenericResponse + err := msgpackrpc.CallWithCodec(codec, "ClientCSI.ControllerListVolumes", req, &resp) + require.Error(err) require.Contains(err.Error(), "no plugins registered for type") } @@ -163,9 +318,12 @@ func TestClientCSI_NodeForControllerPlugin(t *testing.T) { // returns a RPC client to the leader and a cleanup function. func setupForward(t *testing.T) (rpc.ClientCodec, func()) { - s1, cleanupS1 := TestServer(t, func(c *Config) { c.BootstrapExpect = 1 }) + s1, cleanupS1 := TestServer(t, func(c *Config) { c.BootstrapExpect = 2 }) + s2, cleanupS2 := TestServer(t, func(c *Config) { c.BootstrapExpect = 2 }) + TestJoin(t, s1, s2) testutil.WaitForLeader(t, s1.RPC) + testutil.WaitForLeader(t, s2.RPC) codec := rpcClient(t, s1) c1, cleanupC1 := client.TestClient(t, func(c *config.Config) { @@ -176,24 +334,22 @@ func setupForward(t *testing.T) (rpc.ClientCodec, func()) { select { case <-c1.Ready(): case <-time.After(10 * time.Second): - cleanupS1() cleanupC1() + cleanupS1() + cleanupS2() t.Fatal("client timedout on initialize") } - waitForNodes(t, s1, 1, 1) - - s2, cleanupS2 := TestServer(t, func(c *Config) { c.BootstrapExpect = 2 }) - TestJoin(t, s1, s2) - c2, cleanupC2 := client.TestClient(t, func(c *config.Config) { c.Servers = []string{s2.config.RPCAddr.String()} }) select { case <-c2.Ready(): case <-time.After(10 * time.Second): - cleanupS1() cleanupC1() + cleanupC2() + cleanupS1() + cleanupS2() t.Fatal("client timedout on initialize") } @@ -224,10 +380,10 @@ func setupForward(t *testing.T) (rpc.ClientCodec, func()) { s1.fsm.state.UpsertNode(structs.MsgTypeTestSetup, 1000, node1) cleanup := func() { - cleanupS1() cleanupC1() - cleanupS2() cleanupC2() + cleanupS2() + cleanupS1() } return codec, cleanup @@ -235,23 +391,46 @@ func setupForward(t *testing.T) (rpc.ClientCodec, func()) { // sets up a single server with a client, and registers a plugin to the client. func setupLocal(t *testing.T) (rpc.ClientCodec, func()) { - + var err error s1, cleanupS1 := TestServer(t, func(c *Config) { c.BootstrapExpect = 1 }) testutil.WaitForLeader(t, s1.RPC) codec := rpcClient(t, s1) - c1, cleanupC1 := client.TestClient(t, func(c *config.Config) { - c.Servers = []string{s1.config.RPCAddr.String()} - }) + mockCSI := newMockClientCSI() + mockCSI.NextValidateError = fmt.Errorf("no plugins registered for type") + mockCSI.NextAttachError = fmt.Errorf("no plugins registered for type") + mockCSI.NextDetachError = fmt.Errorf("no plugins registered for type") + mockCSI.NextCreateError = fmt.Errorf("no plugins registered for type") + mockCSI.NextDeleteError = fmt.Errorf("no plugins registered for type") + mockCSI.NextListExternalError = fmt.Errorf("no plugins registered for type") + + c1, cleanupC1 := client.TestClientWithRPCs(t, + func(c *config.Config) { + c.Servers = []string{s1.config.RPCAddr.String()} + }, + map[string]interface{}{"CSI": mockCSI}, + ) - // Wait for client initialization - select { - case <-c1.Ready(): - case <-time.After(10 * time.Second): + if err != nil { + cleanupC1() cleanupS1() + require.NoError(t, err, "could not setup test client") + } + + node1 := c1.Node() + node1.Attributes["nomad.version"] = "0.11.0" // client RPCs not supported on early versions + + req := &structs.NodeRegisterRequest{ + Node: node1, + WriteRequest: structs.WriteRequest{Region: "global"}, + } + var resp structs.NodeUpdateResponse + err = c1.RPC("Node.Register", req, &resp) + if err != nil { cleanupC1() - t.Fatal("client timedout on initialize") + cleanupS1() + require.NoError(t, err, "could not register client node") } waitForNodes(t, s1, 1, 1) @@ -266,15 +445,12 @@ func setupLocal(t *testing.T) (rpc.ClientCodec, func()) { } // update w/ plugin - node1 := c1.Node() - node1.Attributes["nomad.version"] = "0.11.0" // client RPCs not supported on early versions node1.CSIControllerPlugins = plugins - s1.fsm.state.UpsertNode(structs.MsgTypeTestSetup, 1000, node1) cleanup := func() { - cleanupS1() cleanupC1() + cleanupS1() } return codec, cleanup diff --git a/nomad/csi_endpoint.go b/nomad/csi_endpoint.go index a68eabb4fa6..22529165c35 100644 --- a/nomad/csi_endpoint.go +++ b/nomad/csi_endpoint.go @@ -108,8 +108,7 @@ func (v *CSIVolume) List(args *structs.CSIVolumeListRequest, reply *structs.CSIV return structs.ErrPermissionDenied } - metricsStart := time.Now() - defer metrics.MeasureSince([]string{"nomad", "volume", "list"}, metricsStart) + defer metrics.MeasureSince([]string{"nomad", "volume", "list"}, time.Now()) ns := args.RequestNamespace() opts := blockingOptions{ @@ -192,8 +191,7 @@ func (v *CSIVolume) Get(args *structs.CSIVolumeGetRequest, reply *structs.CSIVol return structs.ErrPermissionDenied } - metricsStart := time.Now() - defer metrics.MeasureSince([]string{"nomad", "volume", "get"}, metricsStart) + defer metrics.MeasureSince([]string{"nomad", "volume", "get"}, time.Now()) if args.ID == "" { return fmt.Errorf("missing volume ID") @@ -276,14 +274,13 @@ func (v *CSIVolume) Register(args *structs.CSIVolumeRegisterRequest, reply *stru return err } - metricsStart := time.Now() - defer metrics.MeasureSince([]string{"nomad", "volume", "register"}, metricsStart) + defer metrics.MeasureSince([]string{"nomad", "volume", "register"}, time.Now()) if !allowVolume(aclObj, args.RequestNamespace()) || !aclObj.AllowPluginRead() { return structs.ErrPermissionDenied } - if args.Volumes == nil || len(args.Volumes) == 0 { + if len(args.Volumes) == 0 { return fmt.Errorf("missing volume definition") } @@ -331,8 +328,7 @@ func (v *CSIVolume) Deregister(args *structs.CSIVolumeDeregisterRequest, reply * return err } - metricsStart := time.Now() - defer metrics.MeasureSince([]string{"nomad", "volume", "deregister"}, metricsStart) + defer metrics.MeasureSince([]string{"nomad", "volume", "deregister"}, time.Now()) ns := args.RequestNamespace() if !allowVolume(aclObj, ns) { @@ -369,8 +365,7 @@ func (v *CSIVolume) Claim(args *structs.CSIVolumeClaimRequest, reply *structs.CS return err } - metricsStart := time.Now() - defer metrics.MeasureSince([]string{"nomad", "volume", "claim"}, metricsStart) + defer metrics.MeasureSince([]string{"nomad", "volume", "claim"}, time.Now()) if !allowVolume(aclObj, args.RequestNamespace()) || !aclObj.AllowPluginRead() { return structs.ErrPermissionDenied @@ -446,9 +441,10 @@ func (v *CSIVolume) controllerPublishVolume(req *structs.CSIVolumeClaimRequest, return fmt.Errorf("%s: %s", structs.ErrUnknownAllocationPrefix, req.AllocationID) } - // if no plugin was returned then controller validation is not required. - // Here we can return nil. - if plug == nil { + // Some plugins support controllers for create/snapshot but not attach. So + // if there's no plugin or the plugin doesn't attach volumes, then we can + // skip the controller publish workflow and return nil. + if plug == nil || !plug.HasControllerCapability(structs.CSIControllerSupportsAttachDetach) { return nil } @@ -531,8 +527,7 @@ func (v *CSIVolume) Unpublish(args *structs.CSIVolumeUnpublishRequest, reply *st return err } - metricsStart := time.Now() - defer metrics.MeasureSince([]string{"nomad", "volume", "unpublish"}, metricsStart) + defer metrics.MeasureSince([]string{"nomad", "volume", "unpublish"}, time.Now()) allowVolume := acl.NamespaceValidator(acl.NamespaceCapabilityCSIMountVolume) aclObj, err := v.srv.WriteACLObj(&args.WriteRequest, true) @@ -685,11 +680,23 @@ func (v *CSIVolume) controllerUnpublishVolume(vol *structs.CSIVolume, claim *str return nil } + state := v.srv.fsm.State() + ws := memdb.NewWatchSet() + + plugin, err := state.CSIPluginByID(ws, vol.PluginID) + if err != nil { + return fmt.Errorf("could not query plugin: %v", err) + } else if plugin == nil { + return fmt.Errorf("no such plugin: %q", vol.PluginID) + } + if !plugin.HasControllerCapability(structs.CSIControllerSupportsAttachDetach) { + return nil + } + // we only send a controller detach if a Nomad client no longer has // any claim to the volume, so we need to check the status of claimed // allocations - state := v.srv.fsm.State() - vol, err := state.CSIVolumeDenormalize(memdb.NewWatchSet(), vol) + vol, err = state.CSIVolumeDenormalize(ws, vol) if err != nil { return err } @@ -795,6 +802,256 @@ func (v *CSIVolume) checkpointClaim(vol *structs.CSIVolume, claim *structs.CSIVo return nil } +func (v *CSIVolume) Create(args *structs.CSIVolumeCreateRequest, reply *structs.CSIVolumeCreateResponse) error { + + if done, err := v.srv.forward("CSIVolume.Create", args, args, reply); done { + return err + } + + defer metrics.MeasureSince([]string{"nomad", "volume", "create"}, time.Now()) + + allowVolume := acl.NamespaceValidator(acl.NamespaceCapabilityCSIWriteVolume) + aclObj, err := v.srv.WriteACLObj(&args.WriteRequest, false) + if err != nil { + return err + } + + if !allowVolume(aclObj, args.RequestNamespace()) || !aclObj.AllowPluginRead() { + return structs.ErrPermissionDenied + } + + if len(args.Volumes) == 0 { + return fmt.Errorf("missing volume definition") + } + + regArgs := &structs.CSIVolumeRegisterRequest{WriteRequest: args.WriteRequest} + + type validated struct { + vol *structs.CSIVolume + plugin *structs.CSIPlugin + } + validatedVols := []validated{} + + // This is the only namespace we ACL checked, force all the volumes to use it. + // We also validate that the plugin exists for each plugin, and validate the + // capabilities when the plugin has a controller. + for _, vol := range args.Volumes { + vol.Namespace = args.RequestNamespace() + if err = vol.Validate(); err != nil { + return err + } + plugin, err := v.pluginValidateVolume(regArgs, vol) + if err != nil { + return err + } + if !plugin.ControllerRequired { + return fmt.Errorf("plugin has no controller") + } + if !plugin.HasControllerCapability(structs.CSIControllerSupportsCreateDelete) { + return fmt.Errorf("plugin does not support creating volumes") + } + + validatedVols = append(validatedVols, validated{vol, plugin}) + } + + // Attempt to create all the validated volumes and write only successfully + // created volumes to raft. And we'll report errors for any failed volumes + // + // NOTE: creating the volume in the external storage provider can't be + // made atomic with the registration, and creating the volume provides + // values we want to write on the CSIVolume in raft anyways. For now + // we'll block the RPC on the external storage provider so that we can + // easily return meaningful errors to the user, but in the future we + // should consider creating registering first and creating a "volume + // eval" that can do the plugin RPCs async. + + var mErr multierror.Error + + for _, valid := range validatedVols { + err = v.createVolume(valid.vol, valid.plugin) + if err != nil { + multierror.Append(&mErr, err) + } else { + regArgs.Volumes = append(regArgs.Volumes, valid.vol) + } + } + + resp, index, err := v.srv.raftApply(structs.CSIVolumeRegisterRequestType, regArgs) + if err != nil { + v.logger.Error("csi raft apply failed", "error", err, "method", "register") + return err + } + if respErr, ok := resp.(error); ok { + multierror.Append(&mErr, respErr) + } + + err = mErr.ErrorOrNil() + if err != nil { + return err + } + + reply.Volumes = regArgs.Volumes + reply.Index = index + v.srv.setQueryMeta(&reply.QueryMeta) + return nil +} + +func (v *CSIVolume) createVolume(vol *structs.CSIVolume, plugin *structs.CSIPlugin) error { + + method := "ClientCSI.ControllerCreateVolume" + cReq := &cstructs.ClientCSIControllerCreateVolumeRequest{ + Name: vol.Name, + VolumeCapabilities: vol.RequestedCapabilities, + Parameters: vol.Parameters, + Secrets: vol.Secrets, + CapacityMin: vol.RequestedCapacityMin, + CapacityMax: vol.RequestedCapacityMax, + SnapshotID: vol.SnapshotID, + CloneID: vol.CloneID, + } + cReq.PluginID = plugin.ID + cResp := &cstructs.ClientCSIControllerCreateVolumeResponse{} + err := v.srv.RPC(method, cReq, cResp) + if err != nil { + return err + } + + vol.ExternalID = cResp.ExternalVolumeID + vol.Capacity = cResp.CapacityBytes + vol.Context = cResp.VolumeContext + return nil +} + +func (v *CSIVolume) Delete(args *structs.CSIVolumeDeleteRequest, reply *structs.CSIVolumeDeleteResponse) error { + if done, err := v.srv.forward("CSIVolume.Delete", args, args, reply); done { + return err + } + + defer metrics.MeasureSince([]string{"nomad", "volume", "delete"}, time.Now()) + + allowVolume := acl.NamespaceValidator(acl.NamespaceCapabilityCSIWriteVolume) + aclObj, err := v.srv.WriteACLObj(&args.WriteRequest, false) + if err != nil { + return err + } + + ns := args.RequestNamespace() + if !allowVolume(aclObj, ns) { + return structs.ErrPermissionDenied + } + + if len(args.VolumeIDs) == 0 { + return fmt.Errorf("missing volume IDs") + } + + for _, volID := range args.VolumeIDs { + + plugin, vol, err := v.volAndPluginLookup(args.Namespace, volID) + if err != nil { + if err == fmt.Errorf("volume not found: %s", volID) { + v.logger.Warn("volume %q to be deleted was already deregistered") + continue + } else { + return err + } + } + + // NOTE: deleting the volume in the external storage provider can't be + // made atomic with deregistration. We can't delete a volume that's + // not registered because we need to be able to lookup its plugin. + err = v.deleteVolume(vol, plugin) + if err != nil { + return err + } + } + + deregArgs := &structs.CSIVolumeDeregisterRequest{ + VolumeIDs: args.VolumeIDs, + WriteRequest: args.WriteRequest, + } + resp, index, err := v.srv.raftApply(structs.CSIVolumeDeregisterRequestType, deregArgs) + if err != nil { + v.logger.Error("csi raft apply failed", "error", err, "method", "deregister") + return err + } + if respErr, ok := resp.(error); ok { + return respErr + } + + reply.Index = index + v.srv.setQueryMeta(&reply.QueryMeta) + return nil +} + +func (v *CSIVolume) deleteVolume(vol *structs.CSIVolume, plugin *structs.CSIPlugin) error { + + method := "ClientCSI.ControllerDeleteVolume" + cReq := &cstructs.ClientCSIControllerDeleteVolumeRequest{ + ExternalVolumeID: vol.ExternalID, + Secrets: vol.Secrets, + } + cReq.PluginID = plugin.ID + cResp := &cstructs.ClientCSIControllerDeleteVolumeResponse{} + + return v.srv.RPC(method, cReq, cResp) +} + +func (v *CSIVolume) ListExternal(args *structs.CSIVolumeExternalListRequest, reply *structs.CSIVolumeExternalListResponse) error { + + if done, err := v.srv.forward("CSIVolume.ListExternal", args, args, reply); done { + return err + } + defer metrics.MeasureSince([]string{"nomad", "volume", "list_external"}, time.Now()) + + allowVolume := acl.NamespaceValidator(acl.NamespaceCapabilityCSIListVolume, + acl.NamespaceCapabilityCSIReadVolume, + acl.NamespaceCapabilityCSIMountVolume, + acl.NamespaceCapabilityListJobs) + aclObj, err := v.srv.QueryACLObj(&args.QueryOptions, false) + if err != nil { + return err + } + + // NOTE: this is the plugin's namespace, not the volume(s) because they + // might not even be registered + if !allowVolume(aclObj, args.RequestNamespace()) { + return structs.ErrPermissionDenied + } + snap, err := v.srv.fsm.State().Snapshot() + if err != nil { + return err + } + + plugin, err := snap.CSIPluginByID(nil, args.PluginID) + if err != nil { + return err + } + if plugin == nil { + return fmt.Errorf("no such plugin") + } + + method := "ClientCSI.ControllerListVolumes" + cReq := &cstructs.ClientCSIControllerListVolumesRequest{ + MaxEntries: args.PerPage, + StartingToken: args.NextToken, + } + cReq.PluginID = plugin.ID + cResp := &cstructs.ClientCSIControllerListVolumesResponse{} + + err = v.srv.RPC(method, cReq, cResp) + if err != nil { + return err + } + if args.PerPage > 0 { + reply.Volumes = cResp.Entries[:args.PerPage] + } else { + reply.Volumes = cResp.Entries + } + reply.NextToken = cResp.NextToken + + return nil +} + // CSIPlugin wraps the structs.CSIPlugin with request data and server context type CSIPlugin struct { srv *Server @@ -816,8 +1073,7 @@ func (v *CSIPlugin) List(args *structs.CSIPluginListRequest, reply *structs.CSIP return structs.ErrPermissionDenied } - metricsStart := time.Now() - defer metrics.MeasureSince([]string{"nomad", "plugin", "list"}, metricsStart) + defer metrics.MeasureSince([]string{"nomad", "plugin", "list"}, time.Now()) opts := blockingOptions{ queryOpts: &args.QueryOptions, @@ -865,8 +1121,7 @@ func (v *CSIPlugin) Get(args *structs.CSIPluginGetRequest, reply *structs.CSIPlu withAllocs := aclObj == nil || aclObj.AllowNsOp(args.RequestNamespace(), acl.NamespaceCapabilityReadJob) - metricsStart := time.Now() - defer metrics.MeasureSince([]string{"nomad", "plugin", "get"}, metricsStart) + defer metrics.MeasureSince([]string{"nomad", "plugin", "get"}, time.Now()) if args.ID == "" { return fmt.Errorf("missing plugin ID") @@ -926,8 +1181,7 @@ func (v *CSIPlugin) Delete(args *structs.CSIPluginDeleteRequest, reply *structs. return structs.ErrPermissionDenied } - metricsStart := time.Now() - defer metrics.MeasureSince([]string{"nomad", "plugin", "delete"}, metricsStart) + defer metrics.MeasureSince([]string{"nomad", "plugin", "delete"}, time.Now()) if args.ID == "" { return fmt.Errorf("missing plugin ID") diff --git a/nomad/csi_endpoint_test.go b/nomad/csi_endpoint_test.go index bb9ff5e8145..5443bbc6d52 100644 --- a/nomad/csi_endpoint_test.go +++ b/nomad/csi_endpoint_test.go @@ -7,6 +7,9 @@ import ( msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc" "github.com/hashicorp/nomad/acl" + "github.com/hashicorp/nomad/client" + cconfig "github.com/hashicorp/nomad/client/config" + cstructs "github.com/hashicorp/nomad/client/structs" "github.com/hashicorp/nomad/helper/uuid" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/state" @@ -159,11 +162,6 @@ func TestCSIVolumeEndpoint_Register(t *testing.T) { } resp1 := &structs.CSIVolumeRegisterResponse{} err := msgpackrpc.CallWithCodec(codec, "CSIVolume.Register", req1, resp1) - require.Error(t, err, "expected validation error") - - // Fix the registration so that it passes validation - vols[0].AttachmentMode = structs.CSIVolumeAttachmentModeFilesystem - err = msgpackrpc.CallWithCodec(codec, "CSIVolume.Register", req1, resp1) require.NoError(t, err) require.NotEqual(t, uint64(0), resp1.Index) @@ -435,14 +433,14 @@ func TestCSIVolumeEndpoint_ClaimWithController(t *testing.T) { claimResp := &structs.CSIVolumeClaimResponse{} err = msgpackrpc.CallWithCodec(codec, "CSIVolume.Claim", claimReq, claimResp) // Because the node is not registered - require.EqualError(t, err, "controller publish: attach volume: No path to node") + require.EqualError(t, err, "controller publish: attach volume: controller attach volume: No path to node") // The node SecretID is authorized for all policies claimReq.AuthToken = node.SecretID claimReq.Namespace = "" claimResp = &structs.CSIVolumeClaimResponse{} err = msgpackrpc.CallWithCodec(codec, "CSIVolume.Claim", claimReq, claimResp) - require.EqualError(t, err, "controller publish: attach volume: No path to node") + require.EqualError(t, err, "controller publish: attach volume: controller attach volume: No path to node") } func TestCSIVolumeEndpoint_Unpublish(t *testing.T) { @@ -496,7 +494,7 @@ func TestCSIVolumeEndpoint_Unpublish(t *testing.T) { { name: "unpublish previously detached node", startingState: structs.CSIVolumeClaimStateNodeDetached, - expectedErrMsg: "could not detach from controller: No path to node", + expectedErrMsg: "could not detach from controller: controller detach volume: No path to node", }, { name: "first unpublish", @@ -653,6 +651,353 @@ func TestCSIVolumeEndpoint_List(t *testing.T) { require.Equal(t, vols[1].ID, resp.Volumes[0].ID) } +func TestCSIVolumeEndpoint_Create(t *testing.T) { + t.Parallel() + var err error + srv, shutdown := TestServer(t, func(c *Config) { + c.NumSchedulers = 0 // Prevent automatic dequeue + }) + defer shutdown() + + testutil.WaitForLeader(t, srv.RPC) + + fake := newMockClientCSI() + fake.NextValidateError = nil + fake.NextCreateError = nil + fake.NextCreateResponse = &cstructs.ClientCSIControllerCreateVolumeResponse{ + ExternalVolumeID: "vol-12345", + CapacityBytes: 42, + VolumeContext: map[string]string{"plugincontext": "bar"}, + } + + client, cleanup := client.TestClientWithRPCs(t, + func(c *cconfig.Config) { + c.Servers = []string{srv.config.RPCAddr.String()} + }, + map[string]interface{}{"CSI": fake}, + ) + defer cleanup() + + node := client.Node() + node.Attributes["nomad.version"] = "0.11.0" // client RPCs not supported on early versions + + req0 := &structs.NodeRegisterRequest{ + Node: node, + WriteRequest: structs.WriteRequest{Region: "global"}, + } + var resp0 structs.NodeUpdateResponse + err = client.RPC("Node.Register", req0, &resp0) + require.NoError(t, err) + + testutil.WaitForResult(func() (bool, error) { + nodes := srv.connectedNodes() + return len(nodes) == 1, nil + }, func(err error) { + t.Fatalf("should have a client") + }) + + ns := structs.DefaultNamespace + + state := srv.fsm.State() + codec := rpcClient(t, srv) + index := uint64(1000) + + node.CSIControllerPlugins = map[string]*structs.CSIInfo{ + "minnie": { + PluginID: "minnie", + Healthy: true, + ControllerInfo: &structs.CSIControllerInfo{ + SupportsAttachDetach: true, + SupportsCreateDelete: true, + }, + RequiresControllerPlugin: true, + }, + } + node.CSINodePlugins = map[string]*structs.CSIInfo{ + "minnie": { + PluginID: "minnie", + Healthy: true, + NodeInfo: &structs.CSINodeInfo{}, + }, + } + index++ + require.NoError(t, state.UpsertNode(structs.MsgTypeTestSetup, index, node)) + + // Create the volume + volID := uuid.Generate() + vols := []*structs.CSIVolume{{ + ID: volID, + Name: "vol", + Namespace: "notTheNamespace", // overriden by WriteRequest + PluginID: "minnie", + AccessMode: structs.CSIVolumeAccessModeMultiNodeReader, // ignored in create + AttachmentMode: structs.CSIVolumeAttachmentModeFilesystem, // ignored in create + MountOptions: &structs.CSIMountOptions{ + FSType: "ext4", MountFlags: []string{"sensitive"}}, // ignored in create + Secrets: structs.CSISecrets{"mysecret": "secretvalue"}, + Parameters: map[string]string{"myparam": "paramvalue"}, + Context: map[string]string{"mycontext": "contextvalue"}, // dropped by create + }} + + // Create the create request + req1 := &structs.CSIVolumeCreateRequest{ + Volumes: vols, + WriteRequest: structs.WriteRequest{ + Region: "global", + Namespace: ns, + }, + } + resp1 := &structs.CSIVolumeCreateResponse{} + err = msgpackrpc.CallWithCodec(codec, "CSIVolume.Create", req1, resp1) + require.NoError(t, err) + + // Get the volume back out + req2 := &structs.CSIVolumeGetRequest{ + ID: volID, + QueryOptions: structs.QueryOptions{ + Region: "global", + }, + } + resp2 := &structs.CSIVolumeGetResponse{} + err = msgpackrpc.CallWithCodec(codec, "CSIVolume.Get", req2, resp2) + require.NoError(t, err) + require.Equal(t, resp1.Index, resp2.Index) + + vol := resp2.Volume + require.NotNil(t, vol) + require.Equal(t, volID, vol.ID) + + // these fields are set from the args + require.Equal(t, "csi.CSISecrets(map[mysecret:[REDACTED]])", + vol.Secrets.String()) + require.Equal(t, "csi.CSIOptions(FSType: ext4, MountFlags: [REDACTED])", + vol.MountOptions.String()) + require.Equal(t, ns, vol.Namespace) + + // these fields are set from the plugin and should have been written to raft + require.Equal(t, "vol-12345", vol.ExternalID) + require.Equal(t, int64(42), vol.Capacity) + require.Equal(t, "bar", vol.Context["plugincontext"]) + require.Equal(t, "", vol.Context["mycontext"]) +} + +func TestCSIVolumeEndpoint_Delete(t *testing.T) { + t.Parallel() + var err error + srv, shutdown := TestServer(t, func(c *Config) { + c.NumSchedulers = 0 // Prevent automatic dequeue + }) + defer shutdown() + + testutil.WaitForLeader(t, srv.RPC) + + fake := newMockClientCSI() + fake.NextDeleteError = fmt.Errorf("should not see this") + + client, cleanup := client.TestClientWithRPCs(t, + func(c *cconfig.Config) { + c.Servers = []string{srv.config.RPCAddr.String()} + }, + map[string]interface{}{"CSI": fake}, + ) + defer cleanup() + + node := client.Node() + node.Attributes["nomad.version"] = "0.11.0" // client RPCs not supported on early versions + + req0 := &structs.NodeRegisterRequest{ + Node: node, + WriteRequest: structs.WriteRequest{Region: "global"}, + } + var resp0 structs.NodeUpdateResponse + err = client.RPC("Node.Register", req0, &resp0) + require.NoError(t, err) + + testutil.WaitForResult(func() (bool, error) { + nodes := srv.connectedNodes() + return len(nodes) == 1, nil + }, func(err error) { + t.Fatalf("should have a client") + }) + + ns := structs.DefaultNamespace + + state := srv.fsm.State() + codec := rpcClient(t, srv) + index := uint64(1000) + + node.CSIControllerPlugins = map[string]*structs.CSIInfo{ + "minnie": { + PluginID: "minnie", + Healthy: true, + ControllerInfo: &structs.CSIControllerInfo{ + SupportsAttachDetach: true, + }, + RequiresControllerPlugin: true, + }, + } + node.CSINodePlugins = map[string]*structs.CSIInfo{ + "minnie": { + PluginID: "minnie", + Healthy: true, + NodeInfo: &structs.CSINodeInfo{}, + }, + } + index++ + require.NoError(t, state.UpsertNode(structs.MsgTypeTestSetup, index, node)) + + volID := uuid.Generate() + vols := []*structs.CSIVolume{{ + ID: volID, + Namespace: structs.DefaultNamespace, + PluginID: "minnie", + Secrets: structs.CSISecrets{"mysecret": "secretvalue"}, + }} + index++ + err = state.CSIVolumeRegister(index, vols) + require.NoError(t, err) + + // Delete volumes + + // Create an invalid delete request, ensure it doesn't hit the plugin + req1 := &structs.CSIVolumeDeleteRequest{ + VolumeIDs: []string{"bad", volID}, + WriteRequest: structs.WriteRequest{ + Region: "global", + Namespace: ns, + }, + } + resp1 := &structs.CSIVolumeCreateResponse{} + err = msgpackrpc.CallWithCodec(codec, "CSIVolume.Delete", req1, resp1) + require.EqualError(t, err, "volume not found: bad") + + // Make sure the valid volume wasn't deleted + req2 := &structs.CSIVolumeGetRequest{ + ID: volID, + QueryOptions: structs.QueryOptions{ + Region: "global", + }, + } + resp2 := &structs.CSIVolumeGetResponse{} + err = msgpackrpc.CallWithCodec(codec, "CSIVolume.Get", req2, resp2) + require.NoError(t, err) + require.NotNil(t, resp2.Volume) + + // Fix the delete request + fake.NextDeleteError = nil + req1.VolumeIDs = []string{volID} + err = msgpackrpc.CallWithCodec(codec, "CSIVolume.Delete", req1, resp1) + require.NoError(t, err) + + // Make sure it was deregistered + err = msgpackrpc.CallWithCodec(codec, "CSIVolume.Get", req2, resp2) + require.NoError(t, err) + require.Nil(t, resp2.Volume) +} + +func TestCSIVolumeEndpoint_ListExternal(t *testing.T) { + t.Parallel() + var err error + srv, shutdown := TestServer(t, func(c *Config) { + c.NumSchedulers = 0 // Prevent automatic dequeue + }) + defer shutdown() + + testutil.WaitForLeader(t, srv.RPC) + + fake := newMockClientCSI() + fake.NextDeleteError = fmt.Errorf("should not see this") + fake.NextListExternalResponse = &cstructs.ClientCSIControllerListVolumesResponse{ + Entries: []*structs.CSIVolumeExternalStub{ + { + ExternalID: "vol-12345", + CapacityBytes: 70000, + PublishedExternalNodeIDs: []string{"i-12345"}, + }, + { + ExternalID: "vol-abcde", + CapacityBytes: 50000, + IsAbnormal: true, + Status: "something went wrong", + }, + { + ExternalID: "vol-00000", + Status: "you should not see me", + }, + }, + NextToken: "page2", + } + + client, cleanup := client.TestClientWithRPCs(t, + func(c *cconfig.Config) { + c.Servers = []string{srv.config.RPCAddr.String()} + }, + map[string]interface{}{"CSI": fake}, + ) + defer cleanup() + + node := client.Node() + node.Attributes["nomad.version"] = "0.11.0" // client RPCs not supported on early versions + + req0 := &structs.NodeRegisterRequest{ + Node: node, + WriteRequest: structs.WriteRequest{Region: "global"}, + } + var resp0 structs.NodeUpdateResponse + err = client.RPC("Node.Register", req0, &resp0) + require.NoError(t, err) + + testutil.WaitForResult(func() (bool, error) { + nodes := srv.connectedNodes() + return len(nodes) == 1, nil + }, func(err error) { + t.Fatalf("should have a client") + }) + + state := srv.fsm.State() + codec := rpcClient(t, srv) + index := uint64(1000) + + node.CSIControllerPlugins = map[string]*structs.CSIInfo{ + "minnie": { + PluginID: "minnie", + Healthy: true, + ControllerInfo: &structs.CSIControllerInfo{ + SupportsAttachDetach: true, + }, + RequiresControllerPlugin: true, + }, + } + node.CSINodePlugins = map[string]*structs.CSIInfo{ + "minnie": { + PluginID: "minnie", + Healthy: true, + NodeInfo: &structs.CSINodeInfo{}, + }, + } + index++ + require.NoError(t, state.UpsertNode(structs.MsgTypeTestSetup, index, node)) + + // List external volumes; note that none of these exist in the state store + + req := &structs.CSIVolumeExternalListRequest{ + QueryOptions: structs.QueryOptions{ + Region: "global", + Namespace: structs.DefaultNamespace, + PerPage: 2, + NextToken: "page1", + }, + } + resp := &structs.CSIVolumeExternalListResponse{} + err = msgpackrpc.CallWithCodec(codec, "CSIVolume.ListExternal", req, resp) + require.NoError(t, err) + require.Len(t, resp.Volumes, 2) + require.Equal(t, "vol-12345", resp.Volumes[0].ExternalID) + require.Equal(t, "vol-abcde", resp.Volumes[1].ExternalID) + require.True(t, resp.Volumes[1].IsAbnormal) + require.Equal(t, "page2", resp.NextToken) +} + func TestCSIPluginEndpoint_RegisterViaFingerprint(t *testing.T) { t.Parallel() srv, shutdown := TestServer(t, func(c *Config) { diff --git a/nomad/state/testing.go b/nomad/state/testing.go index 39072010b02..554d21fb567 100644 --- a/nomad/state/testing.go +++ b/nomad/state/testing.go @@ -81,6 +81,8 @@ func createTestCSIPlugin(s *StateStore, id string, requiresController bool) func SupportsAttachDetach: true, SupportsListVolumes: true, SupportsListVolumesAttachedNodes: false, + SupportsCreateDeleteSnapshot: true, + SupportsListSnapshots: true, }, }, } diff --git a/nomad/structs/csi.go b/nomad/structs/csi.go index 71a61ef399e..1c99242c1cd 100644 --- a/nomad/structs/csi.go +++ b/nomad/structs/csi.go @@ -79,6 +79,13 @@ func (t *TaskCSIPluginConfig) Copy() *TaskCSIPluginConfig { return nt } +// CSIVolumeCapability is the requested attachment and access mode for a +// volume +type CSIVolumeCapability struct { + AttachmentMode CSIVolumeAttachmentMode + AccessMode CSIVolumeAccessMode +} + // CSIVolumeAttachmentMode chooses the type of storage api that will be used to // interact with the device. type CSIVolumeAttachmentMode string @@ -246,6 +253,15 @@ type CSIVolume struct { Secrets CSISecrets Parameters map[string]string Context map[string]string + Capacity int64 // bytes + + // These values are used only on volume creation but we record them + // so that we can diff the volume later + RequestedCapacityMin int64 // bytes + RequestedCapacityMax int64 // bytes + RequestedCapabilities []*CSIVolumeCapability + CloneID string + SnapshotID string // Allocations, tracking claim status ReadAllocs map[string]*Allocation // AllocID -> Allocation @@ -574,21 +590,8 @@ func (v *CSIVolume) Validate() error { if v.Namespace == "" { errs = append(errs, "missing namespace") } - if v.AccessMode == "" { - errs = append(errs, "missing access mode") - } - if v.AttachmentMode == "" { - errs = append(errs, "missing attachment mode") - } - if v.AttachmentMode == CSIVolumeAttachmentModeBlockDevice { - if v.MountOptions != nil { - if v.MountOptions.FSType != "" { - errs = append(errs, "mount options not allowed for block-device") - } - if v.MountOptions.MountFlags != nil && len(v.MountOptions.MountFlags) != 0 { - errs = append(errs, "mount options not allowed for block-device") - } - } + if v.SnapshotID != "" && v.CloneID != "" { + errs = append(errs, "only one of snapshot_id and clone_id is allowed") } // TODO: Volume Topologies are optional - We should check to see if the plugin @@ -630,6 +633,25 @@ type CSIVolumeDeregisterResponse struct { QueryMeta } +type CSIVolumeCreateRequest struct { + Volumes []*CSIVolume + WriteRequest +} + +type CSIVolumeCreateResponse struct { + Volumes []*CSIVolume + QueryMeta +} + +type CSIVolumeDeleteRequest struct { + VolumeIDs []string + WriteRequest +} + +type CSIVolumeDeleteResponse struct { + QueryMeta +} + type CSIVolumeClaimMode int const ( @@ -700,6 +722,38 @@ type CSIVolumeListResponse struct { QueryMeta } +// CSIVolumeExternalListRequest is a request to a controller plugin to list +// all the volumes known to the the storage provider. This request is +// paginated by the plugin and accepts the QueryOptions.PerPage and +// QueryOptions.NextToken fields +type CSIVolumeExternalListRequest struct { + PluginID string + QueryOptions +} + +type CSIVolumeExternalListResponse struct { + Volumes []*CSIVolumeExternalStub + NextToken string + QueryMeta +} + +// CSIVolumeExternalStub is the storage provider's view of a volume, as +// returned from the controller plugin; all IDs are for external resources +type CSIVolumeExternalStub struct { + ExternalID string + CapacityBytes int64 + VolumeContext map[string]string + CloneID string + SnapshotID string + + // TODO: topology support + // AccessibleTopology []*Topology + + PublishedExternalNodeIDs []string + IsAbnormal bool + Status string +} + type CSIVolumeGetRequest struct { ID string QueryOptions @@ -792,14 +846,102 @@ func (p *CSIPlugin) Copy() *CSIPlugin { return out } +type CSIControllerCapability byte + +const ( + // CSIControllerSupportsCreateDelete indicates plugin support for + // CREATE_DELETE_VOLUME + CSIControllerSupportsCreateDelete CSIControllerCapability = 0 + + // CSIControllerSupportsAttachDetach is true when the controller + // implements the methods required to attach and detach volumes. If this + // is false Nomad should skip the controller attachment flow. + CSIControllerSupportsAttachDetach CSIControllerCapability = 1 + + // CSIControllerSupportsListVolumes is true when the controller implements + // the ListVolumes RPC. NOTE: This does not guarantee that attached nodes + // will be returned unless SupportsListVolumesAttachedNodes is also true. + CSIControllerSupportsListVolumes CSIControllerCapability = 2 + + // CSIControllerSupportsGetCapacity indicates plugin support for + // GET_CAPACITY + CSIControllerSupportsGetCapacity CSIControllerCapability = 3 + + // CSIControllerSupportsCreateDeleteSnapshot indicates plugin support for + // CREATE_DELETE_SNAPSHOT + CSIControllerSupportsCreateDeleteSnapshot CSIControllerCapability = 4 + + // CSIControllerSupportsListSnapshots indicates plugin support for + // LIST_SNAPSHOTS + CSIControllerSupportsListSnapshots CSIControllerCapability = 5 + + // CSIControllerSupportsClone indicates plugin support for CLONE_VOLUME + CSIControllerSupportsClone CSIControllerCapability = 6 + + // CSIControllerSupportsReadOnlyAttach is set to true when the controller + // returns the ATTACH_READONLY capability. + CSIControllerSupportsReadOnlyAttach CSIControllerCapability = 7 + + // CSIControllerSupportsExpand indicates plugin support for EXPAND_VOLUME + CSIControllerSupportsExpand CSIControllerCapability = 8 + + // CSIControllerSupportsListVolumesAttachedNodes indicates whether the + // plugin will return attached nodes data when making ListVolume RPCs + // (plugin support for LIST_VOLUMES_PUBLISHED_NODES) + CSIControllerSupportsListVolumesAttachedNodes CSIControllerCapability = 9 + + // CSIControllerSupportsCondition indicates plugin support for + // VOLUME_CONDITION + CSIControllerSupportsCondition CSIControllerCapability = 10 + + // CSIControllerSupportsGet indicates plugin support for GET_VOLUME + CSIControllerSupportsGet CSIControllerCapability = 11 +) + +func (p *CSIPlugin) HasControllerCapability(cap CSIControllerCapability) bool { + if len(p.Controllers) < 1 { + return false + } + // we're picking the first controller because they should be uniform + // across the same version of the plugin + for _, c := range p.Controllers { + switch cap { + case CSIControllerSupportsCreateDelete: + return c.ControllerInfo.SupportsCreateDelete + case CSIControllerSupportsAttachDetach: + return c.ControllerInfo.SupportsAttachDetach + case CSIControllerSupportsListVolumes: + return c.ControllerInfo.SupportsListVolumes + case CSIControllerSupportsGetCapacity: + return c.ControllerInfo.SupportsGetCapacity + case CSIControllerSupportsCreateDeleteSnapshot: + return c.ControllerInfo.SupportsCreateDeleteSnapshot + case CSIControllerSupportsListSnapshots: + return c.ControllerInfo.SupportsListSnapshots + case CSIControllerSupportsClone: + return c.ControllerInfo.SupportsClone + case CSIControllerSupportsReadOnlyAttach: + return c.ControllerInfo.SupportsReadOnlyAttach + case CSIControllerSupportsExpand: + return c.ControllerInfo.SupportsExpand + case CSIControllerSupportsListVolumesAttachedNodes: + return c.ControllerInfo.SupportsListVolumesAttachedNodes + case CSIControllerSupportsCondition: + return c.ControllerInfo.SupportsCondition + case CSIControllerSupportsGet: + return c.ControllerInfo.SupportsGet + default: + return false + } + } + return false +} + // AddPlugin adds a single plugin running on the node. Called from state.NodeUpdate in a // transaction func (p *CSIPlugin) AddPlugin(nodeID string, info *CSIInfo) error { if info.ControllerInfo != nil { - p.ControllerRequired = info.RequiresControllerPlugin && - (info.ControllerInfo.SupportsAttachDetach || - info.ControllerInfo.SupportsReadOnlyAttach) - + p.ControllerRequired = info.RequiresControllerPlugin prev, ok := p.Controllers[nodeID] if ok { if prev == nil { diff --git a/nomad/structs/node.go b/nomad/structs/node.go index a49cda0310b..c8631ebff40 100644 --- a/nomad/structs/node.go +++ b/nomad/structs/node.go @@ -112,23 +112,50 @@ func (n *CSINodeInfo) Copy() *CSINodeInfo { // CSIControllerInfo is the fingerprinted data from a CSI Plugin that is specific to // the Controller API. type CSIControllerInfo struct { - // SupportsReadOnlyAttach is set to true when the controller returns the - // ATTACH_READONLY capability. - SupportsReadOnlyAttach bool - // SupportsPublishVolume is true when the controller implements the methods - // required to attach and detach volumes. If this is false Nomad should skip - // the controller attachment flow. + // SupportsCreateDelete indicates plugin support for CREATE_DELETE_VOLUME + SupportsCreateDelete bool + + // SupportsPublishVolume is true when the controller implements the + // methods required to attach and detach volumes. If this is false Nomad + // should skip the controller attachment flow. SupportsAttachDetach bool - // SupportsListVolumes is true when the controller implements the ListVolumes - // RPC. NOTE: This does not guaruntee that attached nodes will be returned - // unless SupportsListVolumesAttachedNodes is also true. + // SupportsListVolumes is true when the controller implements the + // ListVolumes RPC. NOTE: This does not guarantee that attached nodes will + // be returned unless SupportsListVolumesAttachedNodes is also true. SupportsListVolumes bool - // SupportsListVolumesAttachedNodes indicates whether the plugin will return - // attached nodes data when making ListVolume RPCs + // SupportsGetCapacity indicates plugin support for GET_CAPACITY + SupportsGetCapacity bool + + // SupportsCreateDeleteSnapshot indicates plugin support for + // CREATE_DELETE_SNAPSHOT + SupportsCreateDeleteSnapshot bool + + // SupportsListSnapshots indicates plugin support for LIST_SNAPSHOTS + SupportsListSnapshots bool + + // SupportsClone indicates plugin support for CLONE_VOLUME + SupportsClone bool + + // SupportsReadOnlyAttach is set to true when the controller returns the + // ATTACH_READONLY capability. + SupportsReadOnlyAttach bool + + // SupportsExpand indicates plugin support for EXPAND_VOLUME + SupportsExpand bool + + // SupportsListVolumesAttachedNodes indicates whether the plugin will + // return attached nodes data when making ListVolume RPCs (plugin support + // for LIST_VOLUMES_PUBLISHED_NODES) SupportsListVolumesAttachedNodes bool + + // SupportsCondition indicates plugin support for VOLUME_CONDITION + SupportsCondition bool + + // SupportsGet indicates plugin support for GET_VOLUME + SupportsGet bool } func (c *CSIControllerInfo) Copy() *CSIControllerInfo { diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 09ea6de94cc..ed885e7ebfd 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -293,6 +293,14 @@ type QueryOptions struct { // AuthToken is secret portion of the ACL token used for the request AuthToken string + // PerPage is the number of entries to be returned in queries that support + // paginated lists. + PerPage int32 + + // NextToken is the token used indicate where to start paging for queries + // that support paginated lists. + NextToken string + InternalRpcInfo } diff --git a/plugins/csi/client.go b/plugins/csi/client.go index d0bcb4ebd3e..f8c8ee9951e 100644 --- a/plugins/csi/client.go +++ b/plugins/csi/client.go @@ -68,6 +68,9 @@ type CSIControllerClient interface { ControllerPublishVolume(ctx context.Context, in *csipbv1.ControllerPublishVolumeRequest, opts ...grpc.CallOption) (*csipbv1.ControllerPublishVolumeResponse, error) ControllerUnpublishVolume(ctx context.Context, in *csipbv1.ControllerUnpublishVolumeRequest, opts ...grpc.CallOption) (*csipbv1.ControllerUnpublishVolumeResponse, error) ValidateVolumeCapabilities(ctx context.Context, in *csipbv1.ValidateVolumeCapabilitiesRequest, opts ...grpc.CallOption) (*csipbv1.ValidateVolumeCapabilitiesResponse, error) + CreateVolume(ctx context.Context, in *csipbv1.CreateVolumeRequest, opts ...grpc.CallOption) (*csipbv1.CreateVolumeResponse, error) + ListVolumes(ctx context.Context, in *csipbv1.ListVolumesRequest, opts ...grpc.CallOption) (*csipbv1.ListVolumesResponse, error) + DeleteVolume(ctx context.Context, in *csipbv1.DeleteVolumeRequest, opts ...grpc.CallOption) (*csipbv1.DeleteVolumeResponse, error) } // CSINodeClient defines the minimal CSI Node Plugin interface used @@ -383,6 +386,91 @@ func (c *client) ControllerValidateCapabilities(ctx context.Context, req *Contro return nil } +func (c *client) ControllerCreateVolume(ctx context.Context, req *ControllerCreateVolumeRequest, opts ...grpc.CallOption) (*ControllerCreateVolumeResponse, error) { + err := req.Validate() + if err != nil { + return nil, err + } + creq := req.ToCSIRepresentation() + resp, err := c.controllerClient.CreateVolume(ctx, creq, opts...) + + // these standard gRPC error codes are overloaded with CSI-specific + // meanings, so translate them into user-understandable terms + // https://github.com/container-storage-interface/spec/blob/master/spec.md#createvolume-errors + if err != nil { + code := status.Code(err) + switch code { + case codes.InvalidArgument: + return nil, fmt.Errorf( + "volume %q snapshot source %q is not compatible with these parameters: %v", + req.Name, req.ContentSource, err) + case codes.NotFound: + return nil, fmt.Errorf( + "volume %q content source %q does not exist: %v", + req.Name, req.ContentSource, err) + case codes.AlreadyExists: + return nil, fmt.Errorf( + "volume %q already exists but is incompatible with these parameters: %v", + req.Name, err) + case codes.ResourceExhausted: + return nil, fmt.Errorf( + "unable to provision %q in accessible_topology: %v", + req.Name, err) + case codes.OutOfRange: + return nil, fmt.Errorf( + "unsupported capacity_range for volume %q: %v", req.Name, err) + case codes.Internal: + return nil, fmt.Errorf( + "controller plugin returned an internal error, check the plugin allocation logs for more information: %v", err) + } + return nil, err + } + + return NewCreateVolumeResponse(resp), nil +} + +func (c *client) ControllerListVolumes(ctx context.Context, req *ControllerListVolumesRequest, opts ...grpc.CallOption) (*ControllerListVolumesResponse, error) { + err := req.Validate() + if err != nil { + return nil, err + } + creq := req.ToCSIRepresentation() + resp, err := c.controllerClient.ListVolumes(ctx, creq, opts...) + if err != nil { + code := status.Code(err) + switch code { + case codes.Aborted: + return nil, fmt.Errorf( + "invalid starting token %q: %v", req.StartingToken, err) + case codes.Internal: + return nil, fmt.Errorf( + "controller plugin returned an internal error, check the plugin allocation logs for more information: %v", err) + } + return nil, err + } + return NewListVolumesResponse(resp), nil +} + +func (c *client) ControllerDeleteVolume(ctx context.Context, req *ControllerDeleteVolumeRequest, opts ...grpc.CallOption) error { + err := req.Validate() + if err != nil { + return err + } + creq := req.ToCSIRepresentation() + _, err = c.controllerClient.DeleteVolume(ctx, creq, opts...) + if err != nil { + code := status.Code(err) + switch code { + case codes.FailedPrecondition: + return fmt.Errorf("volume %q is in use: %v", req.ExternalVolumeID, err) + case codes.Internal: + return fmt.Errorf( + "controller plugin returned an internal error, check the plugin allocation logs for more information: %v", err) + } + } + return err +} + // compareCapabilities returns an error if the 'got' capabilities aren't found // within the 'expected' capability. // diff --git a/plugins/csi/client_test.go b/plugins/csi/client_test.go index 53a1042fcb9..c2423e3ae16 100644 --- a/plugins/csi/client_test.go +++ b/plugins/csi/client_test.go @@ -222,7 +222,7 @@ func TestClient_RPC_ControllerGetCapabilities(t *testing.T) { { Type: &csipbv1.ControllerServiceCapability_Rpc{ Rpc: &csipbv1.ControllerServiceCapability_RPC{ - Type: csipbv1.ControllerServiceCapability_RPC_GET_CAPACITY, + Type: csipbv1.ControllerServiceCapability_RPC_UNKNOWN, }, }, }, @@ -698,7 +698,249 @@ func TestClient_RPC_ControllerValidateVolume(t *testing.T) { } }) } +} + +func TestClient_RPC_ControllerCreateVolume(t *testing.T) { + + cases := []struct { + Name string + CapacityRange *CapacityRange + ContentSource *VolumeContentSource + ResponseErr error + Response *csipbv1.CreateVolumeResponse + ExpectedErr error + }{ + { + Name: "handles underlying grpc errors", + ResponseErr: status.Errorf(codes.Internal, "some grpc error"), + ExpectedErr: fmt.Errorf("controller plugin returned an internal error, check the plugin allocation logs for more information: rpc error: code = Internal desc = some grpc error"), + }, + + { + Name: "handles error invalid capacity range", + CapacityRange: &CapacityRange{ + RequiredBytes: 1000, + LimitBytes: 500, + }, + ExpectedErr: errors.New("LimitBytes cannot be less than RequiredBytes"), + }, + + { + Name: "handles error invalid content source", + ContentSource: &VolumeContentSource{ + SnapshotID: "snap-12345", + CloneID: "vol-12345", + }, + ExpectedErr: errors.New( + "one of SnapshotID or CloneID must be set if ContentSource is set"), + }, + + { + Name: "handles success missing source and range", + Response: &csipbv1.CreateVolumeResponse{}, + }, + + { + Name: "handles success with capacity range and source", + CapacityRange: &CapacityRange{ + RequiredBytes: 500, + LimitBytes: 1000, + }, + ContentSource: &VolumeContentSource{ + SnapshotID: "snap-12345", + }, + Response: &csipbv1.CreateVolumeResponse{ + Volume: &csipbv1.Volume{ + CapacityBytes: 1000, + ContentSource: &csipbv1.VolumeContentSource{ + Type: &csipbv1.VolumeContentSource_Snapshot{ + Snapshot: &csipbv1.VolumeContentSource_SnapshotSource{ + SnapshotId: "snap-12345", + }, + }, + }, + }, + }, + }, + } + for _, tc := range cases { + t.Run(tc.Name, func(t *testing.T) { + _, cc, _, client := newTestClient() + defer client.Close() + + req := &ControllerCreateVolumeRequest{ + Name: "vol-123456", + CapacityRange: tc.CapacityRange, + VolumeCapabilities: []*VolumeCapability{ + { + AccessType: VolumeAccessTypeMount, + AccessMode: VolumeAccessModeMultiNodeMultiWriter, + }, + }, + Parameters: map[string]string{}, + Secrets: structs.CSISecrets{}, + ContentSource: tc.ContentSource, + AccessibilityRequirements: &TopologyRequirement{}, + } + + cc.NextCreateVolumeResponse = tc.Response + cc.NextErr = tc.ResponseErr + + resp, err := client.ControllerCreateVolume(context.TODO(), req) + if tc.ExpectedErr != nil { + require.EqualError(t, err, tc.ExpectedErr.Error()) + return + } + require.NoError(t, err, tc.Name) + if tc.Response == nil { + require.Nil(t, resp) + return + } + if tc.CapacityRange != nil { + require.Greater(t, resp.Volume.CapacityBytes, int64(0)) + } + if tc.ContentSource != nil { + require.Equal(t, tc.ContentSource.CloneID, resp.Volume.ContentSource.CloneID) + require.Equal(t, tc.ContentSource.SnapshotID, resp.Volume.ContentSource.SnapshotID) + } + }) + } +} + +func TestClient_RPC_ControllerDeleteVolume(t *testing.T) { + + cases := []struct { + Name string + Request *ControllerDeleteVolumeRequest + ResponseErr error + ExpectedErr error + }{ + { + Name: "handles underlying grpc errors", + Request: &ControllerDeleteVolumeRequest{ExternalVolumeID: "vol-12345"}, + ResponseErr: status.Errorf(codes.Internal, "some grpc error"), + ExpectedErr: fmt.Errorf("controller plugin returned an internal error, check the plugin allocation logs for more information: rpc error: code = Internal desc = some grpc error"), + }, + + { + Name: "handles error missing volume ID", + Request: &ControllerDeleteVolumeRequest{}, + ExpectedErr: errors.New("missing ExternalVolumeID"), + }, + + { + Name: "handles success", + Request: &ControllerDeleteVolumeRequest{ExternalVolumeID: "vol-12345"}, + }, + } + for _, tc := range cases { + t.Run(tc.Name, func(t *testing.T) { + _, cc, _, client := newTestClient() + defer client.Close() + + cc.NextErr = tc.ResponseErr + err := client.ControllerDeleteVolume(context.TODO(), tc.Request) + if tc.ExpectedErr != nil { + require.EqualError(t, err, tc.ExpectedErr.Error()) + return + } + require.NoError(t, err, tc.Name) + }) + } +} + +func TestClient_RPC_ControllerListVolume(t *testing.T) { + + cases := []struct { + Name string + Request *ControllerListVolumesRequest + ResponseErr error + ExpectedErr error + }{ + { + Name: "handles underlying grpc errors", + Request: &ControllerListVolumesRequest{}, + ResponseErr: status.Errorf(codes.Internal, "some grpc error"), + ExpectedErr: fmt.Errorf("controller plugin returned an internal error, check the plugin allocation logs for more information: rpc error: code = Internal desc = some grpc error"), + }, + + { + Name: "handles error invalid max entries", + Request: &ControllerListVolumesRequest{MaxEntries: -1}, + ExpectedErr: errors.New("MaxEntries cannot be negative"), + }, + { + Name: "handles success", + Request: &ControllerListVolumesRequest{}, + }, + } + + for _, tc := range cases { + t.Run(tc.Name, func(t *testing.T) { + _, cc, _, client := newTestClient() + defer client.Close() + + cc.NextErr = tc.ResponseErr + if tc.ResponseErr != nil { + // note: there's nothing interesting to assert here other than + // that we don't throw a NPE during transformation from + // protobuf to our struct + cc.NextListVolumesResponse = &csipbv1.ListVolumesResponse{ + Entries: []*csipbv1.ListVolumesResponse_Entry{ + { + Volume: &csipbv1.Volume{ + CapacityBytes: 1000000, + VolumeId: "vol-0", + VolumeContext: map[string]string{"foo": "bar"}, + + ContentSource: &csipbv1.VolumeContentSource{}, + AccessibleTopology: []*csipbv1.Topology{ + { + Segments: map[string]string{"rack": "A"}, + }, + }, + }, + }, + + { + Volume: &csipbv1.Volume{ + VolumeId: "vol-1", + AccessibleTopology: []*csipbv1.Topology{ + { + Segments: map[string]string{"rack": "A"}, + }, + }, + }, + }, + + { + Volume: &csipbv1.Volume{ + VolumeId: "vol-3", + ContentSource: &csipbv1.VolumeContentSource{ + Type: &csipbv1.VolumeContentSource_Snapshot{ + Snapshot: &csipbv1.VolumeContentSource_SnapshotSource{ + SnapshotId: "snap-12345", + }, + }, + }, + }, + }, + }, + NextToken: "abcdef", + } + } + + resp, err := client.ControllerListVolumes(context.TODO(), tc.Request) + if tc.ExpectedErr != nil { + require.EqualError(t, err, tc.ExpectedErr.Error()) + return + } + require.NoError(t, err, tc.Name) + require.NotNil(t, resp) + + }) + } } func TestClient_RPC_NodeStageVolume(t *testing.T) { diff --git a/plugins/csi/fake/client.go b/plugins/csi/fake/client.go index e593dba6f4e..5b67fa41022 100644 --- a/plugins/csi/fake/client.go +++ b/plugins/csi/fake/client.go @@ -50,6 +50,17 @@ type Client struct { NextControllerUnpublishVolumeErr error ControllerUnpublishVolumeCallCount int64 + NextControllerCreateVolumeResponse *csi.ControllerCreateVolumeResponse + NextControllerCreateVolumeErr error + ControllerCreateVolumeCallCount int64 + + NextControllerDeleteVolumeErr error + ControllerDeleteVolumeCallCount int64 + + NextControllerListVolumesResponse *csi.ControllerListVolumesResponse + NextControllerListVolumesErr error + ControllerListVolumesCallCount int64 + NextControllerValidateVolumeErr error ControllerValidateVolumeCallCount int64 @@ -168,6 +179,27 @@ func (c *Client) ControllerValidateCapabilities(ctx context.Context, req *csi.Co return c.NextControllerValidateVolumeErr } +func (c *Client) ControllerCreateVolume(ctx context.Context, in *csi.ControllerCreateVolumeRequest, opts ...grpc.CallOption) (*csi.ControllerCreateVolumeResponse, error) { + c.Mu.Lock() + defer c.Mu.Unlock() + c.ControllerCreateVolumeCallCount++ + return c.NextControllerCreateVolumeResponse, c.NextControllerCreateVolumeErr +} + +func (c *Client) ControllerDeleteVolume(ctx context.Context, req *csi.ControllerDeleteVolumeRequest, opts ...grpc.CallOption) error { + c.Mu.Lock() + defer c.Mu.Unlock() + c.ControllerDeleteVolumeCallCount++ + return c.NextControllerDeleteVolumeErr +} + +func (c *Client) ControllerListVolumes(ctx context.Context, req *csi.ControllerListVolumesRequest, opts ...grpc.CallOption) (*csi.ControllerListVolumesResponse, error) { + c.Mu.Lock() + defer c.Mu.Unlock() + c.ControllerListVolumesCallCount++ + return c.NextControllerListVolumesResponse, c.NextControllerListVolumesErr +} + func (c *Client) NodeGetCapabilities(ctx context.Context) (*csi.NodeCapabilitySet, error) { c.Mu.Lock() defer c.Mu.Unlock() diff --git a/plugins/csi/plugin.go b/plugins/csi/plugin.go index df68bdce22e..ae1532a1b7c 100644 --- a/plugins/csi/plugin.go +++ b/plugins/csi/plugin.go @@ -45,6 +45,18 @@ type CSIPlugin interface { // supports the requested capability. ControllerValidateCapabilities(ctx context.Context, req *ControllerValidateVolumeRequest, opts ...grpc.CallOption) error + // ControllerCreateVolume is used to create a remote volume in the + // external storage provider + ControllerCreateVolume(ctx context.Context, req *ControllerCreateVolumeRequest, opts ...grpc.CallOption) (*ControllerCreateVolumeResponse, error) + + // ControllerDeleteVolume is used to delete a remote volume in the + // external storage provider + ControllerDeleteVolume(ctx context.Context, req *ControllerDeleteVolumeRequest, opts ...grpc.CallOption) error + + // ControllerListVolumes is used to list all volumes available in the + // external storage provider + ControllerListVolumes(ctx context.Context, req *ControllerListVolumesRequest, opts ...grpc.CallOption) (*ControllerListVolumesResponse, error) + // NodeGetCapabilities is used to return the available capabilities from the // Node Service. NodeGetCapabilities(ctx context.Context) (*NodeCapabilitySet, error) @@ -268,10 +280,18 @@ func NewPluginCapabilitySet(capabilities *csipbv1.GetPluginCapabilitiesResponse) } type ControllerCapabilitySet struct { + HasCreateDeleteVolume bool HasPublishUnpublishVolume bool - HasPublishReadonly bool HasListVolumes bool + HasGetCapacity bool + HasCreateDeleteSnapshot bool + HasListSnapshots bool + HasCloneVolume bool + HasPublishReadonly bool + HasExpandVolume bool HasListVolumesPublishedNodes bool + HasVolumeCondition bool + HasGetVolume bool } func NewControllerCapabilitySet(resp *csipbv1.ControllerGetCapabilitiesResponse) *ControllerCapabilitySet { @@ -281,14 +301,30 @@ func NewControllerCapabilitySet(resp *csipbv1.ControllerGetCapabilitiesResponse) for _, pcap := range pluginCapabilities { if c := pcap.GetRpc(); c != nil { switch c.Type { + case csipbv1.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME: + cs.HasCreateDeleteVolume = true case csipbv1.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME: cs.HasPublishUnpublishVolume = true - case csipbv1.ControllerServiceCapability_RPC_PUBLISH_READONLY: - cs.HasPublishReadonly = true case csipbv1.ControllerServiceCapability_RPC_LIST_VOLUMES: cs.HasListVolumes = true + case csipbv1.ControllerServiceCapability_RPC_GET_CAPACITY: + cs.HasGetCapacity = true + case csipbv1.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT: + cs.HasCreateDeleteSnapshot = true + case csipbv1.ControllerServiceCapability_RPC_LIST_SNAPSHOTS: + cs.HasListSnapshots = true + case csipbv1.ControllerServiceCapability_RPC_CLONE_VOLUME: + cs.HasCloneVolume = true + case csipbv1.ControllerServiceCapability_RPC_PUBLISH_READONLY: + cs.HasPublishReadonly = true + case csipbv1.ControllerServiceCapability_RPC_EXPAND_VOLUME: + cs.HasExpandVolume = true case csipbv1.ControllerServiceCapability_RPC_LIST_VOLUMES_PUBLISHED_NODES: cs.HasListVolumesPublishedNodes = true + case csipbv1.ControllerServiceCapability_RPC_VOLUME_CONDITION: + cs.HasVolumeCondition = true + case csipbv1.ControllerServiceCapability_RPC_GET_VOLUME: + cs.HasGetVolume = true default: continue } @@ -392,6 +428,259 @@ func (r *ControllerUnpublishVolumeRequest) Validate() error { type ControllerUnpublishVolumeResponse struct{} +type ControllerCreateVolumeRequest struct { + // note that Name is intentionally differentiated from both CSIVolume.ID + // and ExternalVolumeID. This name is only a recommendation for the + // storage provider, and many will discard this suggestion + Name string + CapacityRange *CapacityRange + VolumeCapabilities []*VolumeCapability + Parameters map[string]string + Secrets structs.CSISecrets + ContentSource *VolumeContentSource + AccessibilityRequirements *TopologyRequirement +} + +func (r *ControllerCreateVolumeRequest) ToCSIRepresentation() *csipbv1.CreateVolumeRequest { + if r == nil { + return nil + } + caps := make([]*csipbv1.VolumeCapability, 0, len(r.VolumeCapabilities)) + for _, cap := range r.VolumeCapabilities { + caps = append(caps, cap.ToCSIRepresentation()) + } + req := &csipbv1.CreateVolumeRequest{ + Name: r.Name, + CapacityRange: r.CapacityRange.ToCSIRepresentation(), + VolumeCapabilities: caps, + Parameters: r.Parameters, + Secrets: r.Secrets, + VolumeContentSource: r.ContentSource.ToCSIRepresentation(), + AccessibilityRequirements: r.AccessibilityRequirements.ToCSIRepresentation(), + } + + return req +} + +func (r *ControllerCreateVolumeRequest) Validate() error { + if r.Name == "" { + return errors.New("missing Name") + } + if r.VolumeCapabilities == nil { + return errors.New("missing VolumeCapabilities") + } + if r.CapacityRange != nil { + if r.CapacityRange.LimitBytes == 0 && r.CapacityRange.RequiredBytes == 0 { + return errors.New( + "one of LimitBytes or RequiredBytes must be set if CapacityRange is set") + } + if r.CapacityRange.LimitBytes < r.CapacityRange.RequiredBytes { + return errors.New("LimitBytes cannot be less than RequiredBytes") + } + } + if r.ContentSource != nil { + if r.ContentSource.CloneID != "" && r.ContentSource.SnapshotID != "" { + return errors.New( + "one of SnapshotID or CloneID must be set if ContentSource is set") + } + } + return nil +} + +// VolumeContentSource is snapshot or volume that the plugin will use to +// create the new volume. At most one of these fields can be set, but nil (and +// not an empty struct) is expected by CSI plugins if neither field is set. +type VolumeContentSource struct { + SnapshotID string + CloneID string +} + +func (vcr *VolumeContentSource) ToCSIRepresentation() *csipbv1.VolumeContentSource { + if vcr == nil { + return nil + } + if vcr.CloneID != "" { + return &csipbv1.VolumeContentSource{ + Type: &csipbv1.VolumeContentSource_Volume{ + Volume: &csipbv1.VolumeContentSource_VolumeSource{ + VolumeId: vcr.CloneID, + }, + }, + } + } else if vcr.SnapshotID != "" { + return &csipbv1.VolumeContentSource{ + Type: &csipbv1.VolumeContentSource_Snapshot{ + Snapshot: &csipbv1.VolumeContentSource_SnapshotSource{ + SnapshotId: vcr.SnapshotID, + }, + }, + } + } + // Nomad's RPCs will hand us an empty struct, not nil + return nil +} + +func newVolumeContentSource(src *csipbv1.VolumeContentSource) *VolumeContentSource { + return &VolumeContentSource{ + SnapshotID: src.GetSnapshot().GetSnapshotId(), + CloneID: src.GetVolume().GetVolumeId(), + } +} + +type TopologyRequirement struct { + Requisite []*Topology + Preferred []*Topology +} + +func (tr *TopologyRequirement) ToCSIRepresentation() *csipbv1.TopologyRequirement { + if tr == nil { + return nil + } + result := &csipbv1.TopologyRequirement{ + Requisite: []*csipbv1.Topology{}, + Preferred: []*csipbv1.Topology{}, + } + for _, topo := range tr.Requisite { + result.Requisite = append(result.Requisite, + &csipbv1.Topology{Segments: topo.Segments}) + } + for _, topo := range tr.Preferred { + result.Preferred = append(result.Preferred, + &csipbv1.Topology{Segments: topo.Segments}) + } + return result +} + +func newTopologies(src []*csipbv1.Topology) []*Topology { + t := []*Topology{} + for _, topo := range src { + t = append(t, &Topology{Segments: topo.Segments}) + } + return t +} + +type ControllerCreateVolumeResponse struct { + Volume *Volume +} + +func NewCreateVolumeResponse(resp *csipbv1.CreateVolumeResponse) *ControllerCreateVolumeResponse { + vol := resp.GetVolume() + return &ControllerCreateVolumeResponse{Volume: &Volume{ + CapacityBytes: vol.GetCapacityBytes(), + ExternalVolumeID: vol.GetVolumeId(), + VolumeContext: vol.GetVolumeContext(), + ContentSource: newVolumeContentSource(vol.GetContentSource()), + }} +} + +type Volume struct { + CapacityBytes int64 + + // this is differentiated from VolumeID so as not to create confusion + // between the Nomad CSIVolume.ID and the storage provider's ID. + ExternalVolumeID string + VolumeContext map[string]string + ContentSource *VolumeContentSource + AccessibleTopology []*Topology +} + +type ControllerDeleteVolumeRequest struct { + ExternalVolumeID string + Secrets structs.CSISecrets +} + +func (r *ControllerDeleteVolumeRequest) ToCSIRepresentation() *csipbv1.DeleteVolumeRequest { + if r == nil { + return nil + } + return &csipbv1.DeleteVolumeRequest{ + VolumeId: r.ExternalVolumeID, + Secrets: r.Secrets, + } +} + +func (r *ControllerDeleteVolumeRequest) Validate() error { + if r.ExternalVolumeID == "" { + return errors.New("missing ExternalVolumeID") + } + return nil +} + +type ControllerListVolumesRequest struct { + MaxEntries int32 + StartingToken string +} + +func (r *ControllerListVolumesRequest) ToCSIRepresentation() *csipbv1.ListVolumesRequest { + if r == nil { + return nil + } + return &csipbv1.ListVolumesRequest{ + MaxEntries: r.MaxEntries, + StartingToken: r.StartingToken, + } +} + +func (r *ControllerListVolumesRequest) Validate() error { + if r.MaxEntries < 0 { + return errors.New("MaxEntries cannot be negative") + } + return nil +} + +type ControllerListVolumesResponse struct { + Entries []*ListVolumesResponse_Entry + NextToken string +} + +func NewListVolumesResponse(resp *csipbv1.ListVolumesResponse) *ControllerListVolumesResponse { + if resp == nil { + return &ControllerListVolumesResponse{} + } + entries := []*ListVolumesResponse_Entry{} + if resp.Entries != nil { + for _, entry := range resp.Entries { + vol := entry.GetVolume() + status := entry.GetStatus() + entries = append(entries, &ListVolumesResponse_Entry{ + Volume: &Volume{ + CapacityBytes: vol.CapacityBytes, + ExternalVolumeID: vol.VolumeId, + VolumeContext: vol.VolumeContext, + ContentSource: newVolumeContentSource(vol.ContentSource), + AccessibleTopology: newTopologies(vol.AccessibleTopology), + }, + Status: &ListVolumesResponse_VolumeStatus{ + PublishedNodeIds: status.GetPublishedNodeIds(), + VolumeCondition: &VolumeCondition{ + Abnormal: status.GetVolumeCondition().GetAbnormal(), + Message: status.GetVolumeCondition().GetMessage(), + }, + }, + }) + } + } + return &ControllerListVolumesResponse{ + Entries: entries, + NextToken: resp.NextToken, + } +} + +type ListVolumesResponse_Entry struct { + Volume *Volume + Status *ListVolumesResponse_VolumeStatus +} + +type ListVolumesResponse_VolumeStatus struct { + PublishedNodeIds []string + VolumeCondition *VolumeCondition +} + +type VolumeCondition struct { + Abnormal bool + Message string +} + type NodeCapabilitySet struct { HasStageUnstageVolume bool } @@ -478,7 +767,7 @@ func VolumeCapabilityFromStructs(sAccessType structs.CSIVolumeAttachmentMode, sA // final check during transformation into the requisite CSI Data type to // defend against development bugs and corrupted state - and incompatible // nomad versions in the future. - return nil, fmt.Errorf("Unknown volume attachment mode: %s", sAccessType) + return nil, fmt.Errorf("unknown volume attachment mode: %s", sAccessType) } var accessMode VolumeAccessMode @@ -498,7 +787,7 @@ func VolumeCapabilityFromStructs(sAccessType structs.CSIVolumeAttachmentMode, sA // final check during transformation into the requisite CSI Data type to // defend against development bugs and corrupted state - and incompatible // nomad versions in the future. - return nil, fmt.Errorf("Unknown volume access mode: %v", sAccessMode) + return nil, fmt.Errorf("unknown volume access mode: %v", sAccessMode) } return &VolumeCapability{ @@ -531,3 +820,18 @@ func (c *VolumeCapability) ToCSIRepresentation() *csipbv1.VolumeCapability { return vc } + +type CapacityRange struct { + RequiredBytes int64 + LimitBytes int64 +} + +func (c *CapacityRange) ToCSIRepresentation() *csipbv1.CapacityRange { + if c == nil { + return nil + } + return &csipbv1.CapacityRange{ + RequiredBytes: c.RequiredBytes, + LimitBytes: c.LimitBytes, + } +} diff --git a/plugins/csi/testing/client.go b/plugins/csi/testing/client.go index 9f3a13d25a4..be9139ecb93 100644 --- a/plugins/csi/testing/client.go +++ b/plugins/csi/testing/client.go @@ -2,6 +2,7 @@ package testing import ( "context" + "fmt" csipbv1 "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc" @@ -49,6 +50,9 @@ type ControllerClient struct { NextPublishVolumeResponse *csipbv1.ControllerPublishVolumeResponse NextUnpublishVolumeResponse *csipbv1.ControllerUnpublishVolumeResponse NextValidateVolumeCapabilitiesResponse *csipbv1.ValidateVolumeCapabilitiesResponse + NextCreateVolumeResponse *csipbv1.CreateVolumeResponse + NextDeleteVolumeResponse *csipbv1.DeleteVolumeResponse + NextListVolumesResponse *csipbv1.ListVolumesResponse } // NewControllerClient returns a new ControllerClient @@ -62,6 +66,9 @@ func (f *ControllerClient) Reset() { f.NextPublishVolumeResponse = nil f.NextUnpublishVolumeResponse = nil f.NextValidateVolumeCapabilitiesResponse = nil + f.NextCreateVolumeResponse = nil + f.NextDeleteVolumeResponse = nil + f.NextListVolumesResponse = nil } func (c *ControllerClient) ControllerGetCapabilities(ctx context.Context, in *csipbv1.ControllerGetCapabilitiesRequest, opts ...grpc.CallOption) (*csipbv1.ControllerGetCapabilitiesResponse, error) { @@ -80,6 +87,29 @@ func (c *ControllerClient) ValidateVolumeCapabilities(ctx context.Context, in *c return c.NextValidateVolumeCapabilitiesResponse, c.NextErr } +func (c *ControllerClient) CreateVolume(ctx context.Context, in *csipbv1.CreateVolumeRequest, opts ...grpc.CallOption) (*csipbv1.CreateVolumeResponse, error) { + if in.VolumeContentSource != nil { + if in.VolumeContentSource.Type == nil || (in.VolumeContentSource.Type == + &csipbv1.VolumeContentSource_Volume{ + Volume: &csipbv1.VolumeContentSource_VolumeSource{VolumeId: ""}, + }) || (in.VolumeContentSource.Type == + &csipbv1.VolumeContentSource_Snapshot{ + Snapshot: &csipbv1.VolumeContentSource_SnapshotSource{SnapshotId: ""}, + }) { + return nil, fmt.Errorf("empty content source should be nil") + } + } + return c.NextCreateVolumeResponse, c.NextErr +} + +func (c *ControllerClient) DeleteVolume(ctx context.Context, in *csipbv1.DeleteVolumeRequest, opts ...grpc.CallOption) (*csipbv1.DeleteVolumeResponse, error) { + return c.NextDeleteVolumeResponse, c.NextErr +} + +func (c *ControllerClient) ListVolumes(ctx context.Context, in *csipbv1.ListVolumesRequest, opts ...grpc.CallOption) (*csipbv1.ListVolumesResponse, error) { + return c.NextListVolumesResponse, c.NextErr +} + // NodeClient is a CSI Node client used for testing type NodeClient struct { NextErr error diff --git a/vendor/github.com/hashicorp/nomad/api/api.go b/vendor/github.com/hashicorp/nomad/api/api.go index bb553b64576..84df7ec3793 100644 --- a/vendor/github.com/hashicorp/nomad/api/api.go +++ b/vendor/github.com/hashicorp/nomad/api/api.go @@ -65,6 +65,14 @@ type QueryOptions struct { // AuthToken is the secret ID of an ACL token AuthToken string + // PerPage is the number of entries to be returned in queries that support + // paginated lists. + PerPage int32 + + // NextToken is the token used indicate where to start paging for queries + // that support paginated lists. + NextToken string + // ctx is an optional context pass through to the underlying HTTP // request layer. Use Context() and WithContext() to manage this. ctx context.Context diff --git a/vendor/github.com/hashicorp/nomad/api/csi.go b/vendor/github.com/hashicorp/nomad/api/csi.go index 28da54bf0c8..5e5b3affbd8 100644 --- a/vendor/github.com/hashicorp/nomad/api/csi.go +++ b/vendor/github.com/hashicorp/nomad/api/csi.go @@ -28,6 +28,33 @@ func (v *CSIVolumes) List(q *QueryOptions) ([]*CSIVolumeListStub, *QueryMeta, er return resp, qm, nil } +// ListExternal returns all CSI volumes, as known to the storage provider +func (v *CSIVolumes) ListExternal(pluginID string, q *QueryOptions) (*CSIVolumeListExternalResponse, *QueryMeta, error) { + var resp *CSIVolumeListExternalResponse + + qp := url.Values{} + qp.Set("plugin_id", pluginID) + if q.NextToken != "" { + qp.Set("next_token", q.NextToken) + } + if q.PerPage != 0 { + qp.Set("per_page", fmt.Sprint(q.PerPage)) + } + + qm, err := v.client.query("/v1/volumes/external?"+qp.Encode(), &resp, q) + if err != nil { + return nil, nil, err + } + + sort.Sort(CSIVolumeExternalStubSort(resp.Volumes)) + return resp, qm, nil +} + +// PluginList returns all CSI volumes for the specified plugin id +func (v *CSIVolumes) PluginList(pluginID string) ([]*CSIVolumeListStub, *QueryMeta, error) { + return v.List(&QueryOptions{Prefix: pluginID}) +} + // Info is used to retrieve a single CSIVolume func (v *CSIVolumes) Info(id string, q *QueryOptions) (*CSIVolume, *QueryMeta, error) { var resp CSIVolume @@ -52,6 +79,21 @@ func (v *CSIVolumes) Deregister(id string, force bool, w *WriteOptions) error { return err } +func (v *CSIVolumes) Create(vol *CSIVolume, w *WriteOptions) ([]*CSIVolume, *WriteMeta, error) { + req := CSIVolumeCreateRequest{ + Volumes: []*CSIVolume{vol}, + } + + resp := &CSIVolumeCreateResponse{} + meta, err := v.client.write(fmt.Sprintf("/v1/volume/csi/%v/create", vol.ID), req, resp, w) + return resp.Volumes, meta, err +} + +func (v *CSIVolumes) Delete(externalVolID string, w *WriteOptions) error { + _, err := v.client.delete(fmt.Sprintf("/v1/volume/csi/%v/delete", url.PathEscape(externalVolID)), nil, w) + return err +} + func (v *CSIVolumes) Detach(volID, nodeID string, w *WriteOptions) error { _, err := v.client.delete(fmt.Sprintf("/v1/volume/csi/%v/detach?node=%v", url.PathEscape(volID), nodeID), nil, w) return err @@ -92,15 +134,23 @@ type CSISecrets map[string]string type CSIVolume struct { ID string Name string - ExternalID string `hcl:"external_id"` + ExternalID string `mapstructure:"external_id" hcl:"external_id"` Namespace string Topologies []*CSITopology AccessMode CSIVolumeAccessMode `hcl:"access_mode"` AttachmentMode CSIVolumeAttachmentMode `hcl:"attachment_mode"` MountOptions *CSIMountOptions `hcl:"mount_options"` - Secrets CSISecrets `hcl:"secrets"` - Parameters map[string]string `hcl:"parameters"` - Context map[string]string `hcl:"context"` + Secrets CSISecrets `mapstructure:"secrets" hcl:"secrets"` + Parameters map[string]string `mapstructure:"parameters" hcl:"parameters"` + Context map[string]string `mapstructure:"context" hcl:"context"` + Capacity int64 `hcl:"-"` + + // These fields are used as part of the volume creation request + RequestedCapacityMin int64 `hcl:"capacity_min"` + RequestedCapacityMax int64 `hcl:"capacity_max"` + RequestedCapabilities []*CSIVolumeCapability `hcl:"capability"` + CloneID string `mapstructure:"clone_id" hcl:"clone_id"` + SnapshotID string `mapstructure:"snapshot_id" hcl:"snapshot_id"` // ReadAllocs is a map of allocation IDs for tracking reader claim status. // The Allocation value will always be nil; clients can populate this data @@ -117,7 +167,7 @@ type CSIVolume struct { // Schedulable is true if all the denormalized plugin health fields are true Schedulable bool - PluginID string `hcl:"plugin_id"` + PluginID string `mapstructure:"plugin_id" hcl:"plugin_id"` Provider string ProviderVersion string ControllerRequired bool @@ -134,6 +184,13 @@ type CSIVolume struct { ExtraKeysHCL []string `hcl1:",unusedKeys" json:"-"` } +// CSIVolumeCapability is a requested attachment and access mode for a +// volume +type CSIVolumeCapability struct { + AccessMode CSIVolumeAccessMode `mapstructure:"access_mode" hcl:"access_mode"` + AttachmentMode CSIVolumeAttachmentMode `mapstructure:"attachment_mode" hcl:"attachment_mode"` +} + type CSIVolumeIndexSort []*CSIVolumeListStub func (v CSIVolumeIndexSort) Len() int { @@ -171,6 +228,50 @@ type CSIVolumeListStub struct { ModifyIndex uint64 } +type CSIVolumeListExternalResponse struct { + Volumes []*CSIVolumeExternalStub + NextToken string +} + +// CSIVolumeExternalStub is the storage provider's view of a volume, as +// returned from the controller plugin; all IDs are for external resources +type CSIVolumeExternalStub struct { + ExternalID string + CapacityBytes int64 + VolumeContext map[string]string + CloneID string + SnapshotID string + PublishedExternalNodeIDs []string + IsAbnormal bool + Status string +} + +// We can't sort external volumes by creation time because we don't get that +// data back from the storage provider. Sort by External ID within this page. +type CSIVolumeExternalStubSort []*CSIVolumeExternalStub + +func (v CSIVolumeExternalStubSort) Len() int { + return len(v) +} + +func (v CSIVolumeExternalStubSort) Less(i, j int) bool { + return v[i].ExternalID > v[j].ExternalID +} + +func (v CSIVolumeExternalStubSort) Swap(i, j int) { + v[i], v[j] = v[j], v[i] +} + +type CSIVolumeCreateRequest struct { + Volumes []*CSIVolume + WriteRequest +} + +type CSIVolumeCreateResponse struct { + Volumes []*CSIVolume + QueryMeta +} + type CSIVolumeRegisterRequest struct { Volumes []*CSIVolume WriteRequest diff --git a/website/content/docs/commands/volume/create.mdx b/website/content/docs/commands/volume/create.mdx new file mode 100644 index 00000000000..c240156773f --- /dev/null +++ b/website/content/docs/commands/volume/create.mdx @@ -0,0 +1,158 @@ +--- +layout: docs +page_title: 'Commands: volume create' +description: | + Create volumes with CSI plugins. +--- + +# Command: volume create + +The `volume create` command creates external storage volumes with Nomad's +[Container Storage Interface (CSI)][csi] support. Only CSI plugins that +implement the [Controller][csi_plugins_internals] interface support this +command. The volume will also be [registered] when it is successfully created. + +## Usage + +```plaintext +nomad volume create [options] [file] +``` + +The `volume create` command requires a single argument, specifying the path to +a file containing a valid [volume specification][volume_specification]. This +file will be read and the volume will be submitted to Nomad for scheduling. If +the supplied path is "-", the volume file is read from STDIN. Otherwise it is +read from the file at the supplied path. + +When ACLs are enabled, this command requires a token with the +`csi-write-volume` capability for the volume's namespace. + +## General Options + +@include 'general_options.mdx' + +## Volume Specification + +The file may be provided as either HCL or JSON. An example HCL configuration: + +```hcl +id = "ebs_prod_db1" +name = "database" +type = "csi" +external_id = "vol-23452345" +plugin_id = "ebs-prod" +snapshot_id = "snap-12345" # or clone_id, see below + +capacity { + required = "100G" + limit = "200G" +} + +access_mode = "single-node-writer" +attachment_mode = "file-system" + +mount_options { + fs_type = "ext4" + mount_flags = ["ro"] +} + +secrets { + example_secret = "xyzzy" +} + +parameters { + skuname = "Premium_LRS" +} + +context { + endpoint = "http://192.168.1.101:9425" +} +``` + +## Volume Specification Parameters + +- `id` `(string: )` - The unique ID of the volume. This is how the + [`volume.source`][csi_volume_source] field in a job specification will refer + to the volume. + +- `name` `(string: )` - The display name of the volume. + +- `type` `(string: )` - The type of volume. Currently only `"csi"` + is supported. + +- `external_id` `(string: )` - The ID of the physical volume from + the storage provider. For example, the volume ID of an AWS EBS volume or + Digital Ocean volume. + +- `plugin_id` `(string: )` - The ID of the [CSI plugin][csi_plugin] + that manages this volume. + +- `snapshot_id` `(string: )` - If the storage provider supports + snapshots, the external ID of the snapshot to restore when creating this + volume. If omitted, the volume will be created from scratch. The + `snapshot_id` cannot be set if the `clone_id` field is set. + +- `clone_id` `(string: )` - If the storage provider supports + cloning, the external ID of the volume to clone when creating this + volume. If omitted, the volume will be created from scratch. The + `clone_id` cannot be set if the `snapshot_id` field is set. + +- `capacity_min` `(string: )` - Option for setting the capacity. The + volume must be at least this large, in bytes. The storage provider may + return a volume that is larger than this value. Accepts human-friendly + suffixes such as `"100GiB"`. This field may not be supported by all + storage providers. + +- `capacity_max` `(string: )` - Option for setting the capacity. The + volume must be no more than this large, in bytes. The storage provider may + return a volume that is smaller than this value. Accepts human-friendly + suffixes such as `"100GiB"`. This field may not be supported by all + storage providers. + +- `capability` `(optional)` - Option for validating the capbility of a + volume. You may provide multiple `capability` blocks, each of which must + have the following fields: + + - `access_mode` `(string: )` - Defines whether a volume should be + available concurrently. Can be one of `"single-node-reader-only"`, + `"single-node-writer"`, `"multi-node-reader-only"`, + `"multi-node-single-writer"`, or `"multi-node-multi-writer"`. Most CSI + plugins support only single-node modes. Consult the documentation of the + storage provider and CSI plugin. + + - `attachment_mode` `(string: )` - The storage API that will be used + by the volume. Most storage providers will support `"file-system"`, to mount + volumes using the CSI filesystem API. Some storage providers will support + `"block-device"`, which will mount the volume with the CSI block device API + within the container. + +- `mount_options` - Options for mounting `file-system` volumes that don't + already have a pre-formatted file system. Consult the documentation for your + storage provider and CSI plugin as to whether these options are required or + necessary. + + - `fs_type` `(string )` - File system type (ex. `"ext4"`) + - `mount_flags` `([]string: )` - The flags passed to `mount` + (ex. `["ro", "noatime"]`) + +- `secrets` (map:nil) - An optional + key-value map of strings used as credentials for publishing and + unpublishing volumes. + +- `parameters` (map:nil) - An optional + key-value map of strings passed directly to the CSI plugin to + configure the volume. The details of these parameters are specific + to each storage provider, so please see the specific plugin + documentation for more information. + +- `context` (map:nil) - This field is only used + during volume registration, and will be set automatically by the plugin when + `volume create` is successful. See [`volume register`]. + +[csi]: https://github.com/container-storage-interface/spec +[csi_plugins_internals]: /docs/internals/plugins/csi#csi-plugins +[volume_specification]: #volume-specification +[csi_plugin]: /docs/job-specification/csi_plugin +[csi_volume_source]: /docs/job-specification/volume#source +[registered]: /docs/commands/volume/register +[`volume register`]: /docs/commands/volume/register diff --git a/website/content/docs/commands/volume/delete.mdx b/website/content/docs/commands/volume/delete.mdx new file mode 100644 index 00000000000..35213d68ea5 --- /dev/null +++ b/website/content/docs/commands/volume/delete.mdx @@ -0,0 +1,38 @@ +--- +layout: docs +page_title: 'Commands: volume delete' +description: | + Delete volumes with CSI plugins. +--- + +# Command: volume delete + +The `volume delete` command deletes external storage volumes with Nomad's +[Container Storage Interface (CSI)][csi] support. Only CSI plugins that +implement the [Controller][csi_plugins_internals] interface support this +command. The volume will also be [deregistered] when it is successfully +deleted. + +## Usage + +```plaintext +nomad volume delete [options] [volume] +``` + +The `volume delete` command requires a single argument, specifying the ID of +volume to be deleted. The volume must still be [registered] with Nomad in +order to be deleted. Deleting will fail if the volume is still in use by an +allocation or in the process of being unpublished. If the volume no longer +exists, this command will silently return without an error. + +When ACLs are enabled, this command requires a token with the +`csi-write-volume` capability for the volume's namespace. + +## General Options + +@include 'general_options.mdx' + +[csi]: https://github.com/container-storage-interface/spec +[csi_plugins_internals]: /docs/internals/plugins/csi#csi-plugins +[deregistered]: /docs/commands/volume/deregister +[registered]: /docs/commands/volume/register diff --git a/website/content/docs/commands/volume/deregister.mdx b/website/content/docs/commands/volume/deregister.mdx index 60e2773a92c..e98b8208aa0 100644 --- a/website/content/docs/commands/volume/deregister.mdx +++ b/website/content/docs/commands/volume/deregister.mdx @@ -8,9 +8,10 @@ description: | # Command: volume deregister The `volume deregister` command deregisters external storage volumes with -Nomad's [Container Storage Interface (CSI)][csi] support. The volume -must exist on the remote storage provider before it can be deregistered -and used by a task. +Nomad's [Container Storage Interface (CSI)][csi] support. The volume will be +removed from Nomad's state store but not deleted from the external storage +provider. Note that deregistering a volume prevents Nomad from deleting it via +[`volume delete`] at a later time. ## Usage @@ -37,3 +38,4 @@ When ACLs are enabled, this command requires a token with the allocations. This does not detach the volume from client nodes. [csi]: https://github.com/container-storage-interface/spec +[`volume delete`]: /docs/commands/volume/delete diff --git a/website/content/docs/commands/volume/register.mdx b/website/content/docs/commands/volume/register.mdx index 68b78880e3d..1fe894bb1fd 100644 --- a/website/content/docs/commands/volume/register.mdx +++ b/website/content/docs/commands/volume/register.mdx @@ -7,10 +7,13 @@ description: | # Command: volume register -The `volume register` command registers external storage volumes with -Nomad's [Container Storage Interface (CSI)][csi] support. The volume -must exist on the remote storage provider before it can be registered -and used by a task. +The `volume register` command registers external storage volumes with Nomad's +[Container Storage Interface (CSI)][csi] support. The volume must exist on the +remote storage provider before it can be registered and used by a task. + +CSI plugins that implement the [Controller][csi_plugins_internals] interface +can be created via the [`volume create`] command, which will automatically +register the volume as well. ## Usage @@ -18,12 +21,12 @@ and used by a task. nomad volume register [options] [file] ``` -The `volume register` command requires a single argument, specifying -the path to a file containing a valid [volume -specification][volume_specification]. This file will be read and the -job will be submitted to Nomad for scheduling. If the supplied path is -"-", the job file is read from STDIN. Otherwise it is read from the -file at the supplied path. +The `volume register` command requires a single argument, specifying the path +to a file containing a valid [volume +specification][volume_specification]. This file will be read and the job will +be submitted to Nomad for scheduling. If the supplied path is "-", the job +file is read from STDIN. Otherwise it is read from the file at the supplied +path. When ACLs are enabled, this command requires a token with the `csi-write-volume` capability for the volume's namespace. @@ -37,66 +40,69 @@ When ACLs are enabled, this command requires a token with the The file may be provided as either HCL or JSON. An example HCL configuration: ```hcl -id = "ebs_prod_db1" -name = "database" -type = "csi" -external_id = "vol-23452345" -plugin_id = "ebs-prod" -access_mode = "single-node-writer" +id = "ebs_prod_db1" +name = "database" +type = "csi" +external_id = "vol-23452345" +plugin_id = "ebs-prod" +snapshot_id = "snap-12345" # or clone_id, see below +capacity = "100G" +access_mode = "single-node-writer" attachment_mode = "file-system" + mount_options { - fs_type = "ext4" - mount_flags = ["ro"] + fs_type = "ext4" + mount_flags = ["ro"] } + secrets { - example_secret = "xyzzy" + example_secret = "xyzzy" } + parameters { - skuname = "Premium_LRS" + skuname = "Premium_LRS" } + context { - endpoint = "http://192.168.1.101:9425" + endpoint = "http://192.168.1.101:9425" } ``` ## Volume Specification Parameters -- `id` `(string: )` - The unique ID of the volume. This will - be how [`volume`][csi_volume] stanzas in a jobspec refer to the volume. +- `id` `(string: )` - The unique ID of the volume. This is how the + [`volume.source`][csi_volume_source] field in a job specification will refer + to the volume. - `name` `(string: )` - The display name of the volume. -- `type` `(string: )` - The type of volume. Currently only - `"csi"` is supported. +- `type` `(string: )` - The type of volume. Currently only `"csi"` + is supported. -- `external_id` `(string: )` - The ID of the physical volume - from the storage provider. For example, the volume ID of an AWS EBS - volume or Digital Ocean volume. +- `external_id` `(string: )` - The ID of the physical volume from + the storage provider. For example, the volume ID of an AWS EBS volume or + Digital Ocean volume. -- `plugin_id` `(string: )` - The ID of the [CSI - plugin][csi_plugin] that manages this volume. +- `plugin_id` `(string: )` - The ID of the [CSI plugin][csi_plugin] + that manages this volume. -- `access_mode` `(string: )` - Defines whether a volume - should be available concurrently. Can be one of - `"single-node-reader-only"`, `"single-node-writer"`, - `"multi-node-reader-only"`, `"multi-node-single-writer"`, or - `"multi-node-multi-writer"`. Most CSI plugins support only - single-node modes. Consult the documentation of the storage provider - and CSI plugin. +- `snapshot_id` - This field is only used during volume creation, and is + ignored during volume registration. See [`volume create`]. -- `attachment_mode` `(string: )` - The storage API that will be used - by the volume. Most storage providers will support `"file-system"`, to mount - volumes using the CSI filesystem API. Some storage providers will support - `"block-device"`, which will mount the volume with the CSI block device API - within the container. +- `clone_id` - This field is only used during volume creation, and is ignored + during volume registration. See [`volume create`]. -- `mount_options` - Options for mounting `file-system` volumes that don't - already have a pre-formatted file system. Consult the documentation for your - storage provider and CSI plugin as to whether these options are required or - necessary. +- `capacity_min` - This field is only used during volume creation, and is + ignored during volume registration. See [`volume create`]. - - `fs_type`: file system type (ex. `"ext4"`) - - `mount_flags`: the flags passed to `mount` (ex. `"ro,noatime"`) +- `capacity_max` - This field is only used during volume creation, and is + ignored during volume registration. See [`volume create`]. + +- `capability` - This field is only used during volume creation, and is + ignored during volume registration. See [`volume create`]. + +- `mount_options` - This field is only used during volume creation, and is + ignored during volume registration. See [`volume create`]. - `secrets` (map:nil) - An optional key-value map of strings used as credentials for publishing and @@ -114,7 +120,9 @@ context { each storage provider, so please see the specific plugin documentation for more information. -[volume_specification]: #volume-specification [csi]: https://github.com/container-storage-interface/spec +[csi_plugins_internals]: /docs/internals/plugins/csi#csi-plugins +[volume_specification]: #volume-specification [csi_plugin]: /docs/job-specification/csi_plugin -[csi_volumes]: /docs/job-specification/volume +[csi_volume_source]: /docs/job-specification/volume#source +[`volume create`]: /docs/commands/volume/create diff --git a/website/content/docs/commands/volume/status.mdx b/website/content/docs/commands/volume/status.mdx index f87651f3ad2..6e4a092748d 100644 --- a/website/content/docs/commands/volume/status.mdx +++ b/website/content/docs/commands/volume/status.mdx @@ -45,8 +45,12 @@ namespace. being queried. Drops verbose volume allocation data from the output. -- `-verbose`: Show full information. Allocation create and modify - times are shown in `yyyy/mm/dd hh:mm:ss` format. +- `-verbose`: Show full information. Allocation create and modify times are + shown in `yyyy/mm/dd hh:mm:ss` format. When listing volumes, this flag will + cause Nomad to query the storage provider for volumes that are known to the + storage provider but not yet registered with Nomad. This may include volumes + that have been created by the [`volume create`] command that are not yet + schedulable. ## Examples @@ -58,6 +62,18 @@ ID Name Plugin ID Schedulable Access Mode ebs_prod_db1 database ebs-prod true single-node-writer ``` +List of all volumes, with external provider info: + +```shell-session +$ nomad volume [-type csi] status -verbose +ID Name Plugin ID Schedulable Access Mode +ebs_prod_db1 database ebs-prod true single-node-writer + +External ID Condition Nodes +vol-abcedf OK i-abc123f2,i-14a12df13 +vol-cd46df Abnormal (provider message here) i-14a12df13 +``` + Short view of a specific volume: ```shell-session @@ -108,3 +124,4 @@ b00fa322 28be17d5 write csi 0 run [csi]: https://github.com/container-storage-interface/spec [csi_plugin]: /docs/job-specification/csi_plugin +[`volume create`]: /docs/commands/volume/create diff --git a/website/data/docs-nav-data.json b/website/data/docs-nav-data.json index a353369c3cc..170cc45062b 100644 --- a/website/data/docs-nav-data.json +++ b/website/data/docs-nav-data.json @@ -723,6 +723,14 @@ "title": "Overview", "path": "commands/volume" }, + { + "title": "create", + "path": "commands/volume/create" + }, + { + "title": "delete", + "path": "commands/volume/delete" + }, { "title": "deregister", "path": "commands/volume/deregister"