diff --git a/CHANGELOG.md b/CHANGELOG.md index 9766930ab0..fb937101f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,13 @@ All of these changes are from [PR ### New and Improved +* list filtering: Listing now supports filtering results before being returned + to the user. The filtering takes place server side and uses boolean + expressions against the JSON representation of returned items. See [the + documentation](https://www.boundaryproject.io/docs/concepts/filtering/resource-listing) + for more details. ([PR](https://github.com/hashicorp/boundary/pull/952)) + ([PR](https://github.com/hashicorp/boundary/pull/957)) + ([PR](https://github.com/hashicorp/boundary/pull/967)) * server: Officially support reloading TLS parameters on `SIGHUP`. (This likely worked before but wasn't fully tested.) ([PR](https://github.com/hashicorp/boundary/pull/959)) diff --git a/api/output_string.go b/api/output_string.go index 95a105083b..9639dbbeba 100644 --- a/api/output_string.go +++ b/api/output_string.go @@ -57,7 +57,7 @@ func (d *OutputStringError) parseRequest() { h = fmt.Sprintf("Bearer $(boundary config get-token -keyring-type %s -token-name %s)", keyringType, tokenName) } } - d.parsedCurlString = fmt.Sprintf("%s-H \"%s: %s\" ", d.parsedCurlString, k, h) + d.parsedCurlString = fmt.Sprintf("%s-H \"%s: %s\"", d.parsedCurlString, k, h) } } @@ -65,10 +65,11 @@ func (d *OutputStringError) parseRequest() { // We need to escape single quotes since that's what we're using to // quote the body escapedBody := strings.Replace(string(body), "'", "'\"'\"'", -1) - d.parsedCurlString = fmt.Sprintf("%s-d '%s' ", d.parsedCurlString, escapedBody) + d.parsedCurlString = fmt.Sprintf(" %s-d '%s'", d.parsedCurlString, escapedBody) } - d.parsedCurlString = fmt.Sprintf("%s%s", d.parsedCurlString, d.Request.URL.String()) + // Filters can have shell characters so we use single quotes to surround the URL + d.parsedCurlString = fmt.Sprintf("%s '%s'", d.parsedCurlString, d.Request.URL.String()) } func (d *OutputStringError) CurlString() string { diff --git a/internal/cmd/base/base.go b/internal/cmd/base/base.go index f297cba70c..9d9b46c910 100644 --- a/internal/cmd/base/base.go +++ b/internal/cmd/base/base.go @@ -77,6 +77,7 @@ type Command struct { FlagHostCatalogId string FlagVersion int FlagRecursive bool + FlagFilter string client *api.Client } diff --git a/internal/cmd/commands/accountscmd/accounts.gen.go b/internal/cmd/commands/accountscmd/accounts.gen.go index 8879c1b936..a51863ac8c 100644 --- a/internal/cmd/commands/accountscmd/accounts.gen.go +++ b/internal/cmd/commands/accountscmd/accounts.gen.go @@ -94,7 +94,7 @@ var flagsMap = map[string][]string{ "delete": {"id"}, - "list": {"auth-method-id"}, + "list": {"auth-method-id", "filter"}, } func (c *Command) Flags() *base.FlagSets { @@ -176,6 +176,10 @@ func (c *Command) Run(args []string) int { opts = append(opts, accounts.WithDescription(c.FlagDescription)) } + if c.FlagFilter != "" { + opts = append(opts, accounts.WithFilter(c.FlagFilter)) + } + var version uint32 switch c.Func { diff --git a/internal/cmd/commands/accountscmd/password_accounts.gen.go b/internal/cmd/commands/accountscmd/password_accounts.gen.go index 7056644c3a..e17f796c0c 100644 --- a/internal/cmd/commands/accountscmd/password_accounts.gen.go +++ b/internal/cmd/commands/accountscmd/password_accounts.gen.go @@ -159,6 +159,10 @@ func (c *PasswordCommand) Run(args []string) int { opts = append(opts, accounts.WithDescription(c.FlagDescription)) } + if c.FlagFilter != "" { + opts = append(opts, accounts.WithFilter(c.FlagFilter)) + } + var version uint32 switch c.Func { diff --git a/internal/cmd/commands/authmethodscmd/authmethods.gen.go b/internal/cmd/commands/authmethodscmd/authmethods.gen.go index c6e3751670..f2ae52299e 100644 --- a/internal/cmd/commands/authmethodscmd/authmethods.gen.go +++ b/internal/cmd/commands/authmethodscmd/authmethods.gen.go @@ -92,7 +92,7 @@ var flagsMap = map[string][]string{ "delete": {"id"}, - "list": {"scope-id", "recursive"}, + "list": {"scope-id", "filter", "recursive"}, } func (c *Command) Flags() *base.FlagSets { @@ -163,6 +163,10 @@ func (c *Command) Run(args []string) int { opts = append(opts, authmethods.WithRecursive(true)) } + if c.FlagFilter != "" { + opts = append(opts, authmethods.WithFilter(c.FlagFilter)) + } + var version uint32 if ret := extraFlagsHandlingFunc(c, &opts); ret != 0 { diff --git a/internal/cmd/commands/authmethodscmd/password_authmethods.gen.go b/internal/cmd/commands/authmethodscmd/password_authmethods.gen.go index 5b6c59e9fb..5bc6f29490 100644 --- a/internal/cmd/commands/authmethodscmd/password_authmethods.gen.go +++ b/internal/cmd/commands/authmethodscmd/password_authmethods.gen.go @@ -164,6 +164,10 @@ func (c *PasswordCommand) Run(args []string) int { opts = append(opts, authmethods.WithRecursive(true)) } + if c.FlagFilter != "" { + opts = append(opts, authmethods.WithFilter(c.FlagFilter)) + } + var version uint32 switch c.Func { diff --git a/internal/cmd/commands/authtokenscmd/authtokens.gen.go b/internal/cmd/commands/authtokenscmd/authtokens.gen.go index ec4e9b43bb..c8dafcf2a0 100644 --- a/internal/cmd/commands/authtokenscmd/authtokens.gen.go +++ b/internal/cmd/commands/authtokenscmd/authtokens.gen.go @@ -92,7 +92,7 @@ var flagsMap = map[string][]string{ "delete": {"id"}, - "list": {"scope-id", "recursive"}, + "list": {"scope-id", "filter", "recursive"}, } func (c *Command) Flags() *base.FlagSets { @@ -159,6 +159,10 @@ func (c *Command) Run(args []string) int { opts = append(opts, authtokens.WithRecursive(true)) } + if c.FlagFilter != "" { + opts = append(opts, authtokens.WithFilter(c.FlagFilter)) + } + var version uint32 if ret := extraFlagsHandlingFunc(c, &opts); ret != 0 { diff --git a/internal/cmd/commands/groupscmd/groups.gen.go b/internal/cmd/commands/groupscmd/groups.gen.go index 674c356e7d..5af11740e8 100644 --- a/internal/cmd/commands/groupscmd/groups.gen.go +++ b/internal/cmd/commands/groupscmd/groups.gen.go @@ -104,7 +104,7 @@ var flagsMap = map[string][]string{ "delete": {"id"}, - "list": {"scope-id", "recursive"}, + "list": {"scope-id", "filter", "recursive"}, } func (c *Command) Flags() *base.FlagSets { @@ -195,6 +195,10 @@ func (c *Command) Run(args []string) int { opts = append(opts, groups.WithRecursive(true)) } + if c.FlagFilter != "" { + opts = append(opts, groups.WithFilter(c.FlagFilter)) + } + var version uint32 switch c.Func { diff --git a/internal/cmd/commands/hostcatalogscmd/hostcatalogs.gen.go b/internal/cmd/commands/hostcatalogscmd/hostcatalogs.gen.go index adce939165..12d258a35f 100644 --- a/internal/cmd/commands/hostcatalogscmd/hostcatalogs.gen.go +++ b/internal/cmd/commands/hostcatalogscmd/hostcatalogs.gen.go @@ -92,7 +92,7 @@ var flagsMap = map[string][]string{ "delete": {"id"}, - "list": {"scope-id", "recursive"}, + "list": {"scope-id", "filter", "recursive"}, } func (c *Command) Flags() *base.FlagSets { @@ -163,6 +163,10 @@ func (c *Command) Run(args []string) int { opts = append(opts, hostcatalogs.WithRecursive(true)) } + if c.FlagFilter != "" { + opts = append(opts, hostcatalogs.WithFilter(c.FlagFilter)) + } + var version uint32 if ret := extraFlagsHandlingFunc(c, &opts); ret != 0 { diff --git a/internal/cmd/commands/hostcatalogscmd/static_hostcatalogs.gen.go b/internal/cmd/commands/hostcatalogscmd/static_hostcatalogs.gen.go index 8e9b33e1cc..a11324e07b 100644 --- a/internal/cmd/commands/hostcatalogscmd/static_hostcatalogs.gen.go +++ b/internal/cmd/commands/hostcatalogscmd/static_hostcatalogs.gen.go @@ -162,6 +162,10 @@ func (c *StaticCommand) Run(args []string) int { opts = append(opts, hostcatalogs.WithRecursive(true)) } + if c.FlagFilter != "" { + opts = append(opts, hostcatalogs.WithFilter(c.FlagFilter)) + } + var version uint32 switch c.Func { diff --git a/internal/cmd/commands/hostscmd/hosts.gen.go b/internal/cmd/commands/hostscmd/hosts.gen.go index 111147ab18..bd39d028cc 100644 --- a/internal/cmd/commands/hostscmd/hosts.gen.go +++ b/internal/cmd/commands/hostscmd/hosts.gen.go @@ -92,7 +92,7 @@ var flagsMap = map[string][]string{ "delete": {"id"}, - "list": {"host-catalog-id"}, + "list": {"host-catalog-id", "filter"}, } func (c *Command) Flags() *base.FlagSets { @@ -174,6 +174,10 @@ func (c *Command) Run(args []string) int { opts = append(opts, hosts.WithDescription(c.FlagDescription)) } + if c.FlagFilter != "" { + opts = append(opts, hosts.WithFilter(c.FlagFilter)) + } + var version uint32 if ret := extraFlagsHandlingFunc(c, &opts); ret != 0 { diff --git a/internal/cmd/commands/hostscmd/static_hosts.gen.go b/internal/cmd/commands/hostscmd/static_hosts.gen.go index 68b4e2a265..48b8088254 100644 --- a/internal/cmd/commands/hostscmd/static_hosts.gen.go +++ b/internal/cmd/commands/hostscmd/static_hosts.gen.go @@ -159,6 +159,10 @@ func (c *StaticCommand) Run(args []string) int { opts = append(opts, hosts.WithDescription(c.FlagDescription)) } + if c.FlagFilter != "" { + opts = append(opts, hosts.WithFilter(c.FlagFilter)) + } + var version uint32 switch c.Func { diff --git a/internal/cmd/commands/hostsetscmd/hostsets.gen.go b/internal/cmd/commands/hostsetscmd/hostsets.gen.go index ba9aa7328d..031a5dcd36 100644 --- a/internal/cmd/commands/hostsetscmd/hostsets.gen.go +++ b/internal/cmd/commands/hostsetscmd/hostsets.gen.go @@ -94,7 +94,7 @@ var flagsMap = map[string][]string{ "delete": {"id"}, - "list": {"host-catalog-id"}, + "list": {"host-catalog-id", "filter"}, } func (c *Command) Flags() *base.FlagSets { @@ -176,6 +176,10 @@ func (c *Command) Run(args []string) int { opts = append(opts, hostsets.WithDescription(c.FlagDescription)) } + if c.FlagFilter != "" { + opts = append(opts, hostsets.WithFilter(c.FlagFilter)) + } + var version uint32 switch c.Func { diff --git a/internal/cmd/commands/hostsetscmd/static_hostsets.gen.go b/internal/cmd/commands/hostsetscmd/static_hostsets.gen.go index a57a36cef5..88e594dd60 100644 --- a/internal/cmd/commands/hostsetscmd/static_hostsets.gen.go +++ b/internal/cmd/commands/hostsetscmd/static_hostsets.gen.go @@ -157,6 +157,10 @@ func (c *StaticCommand) Run(args []string) int { opts = append(opts, hostsets.WithDescription(c.FlagDescription)) } + if c.FlagFilter != "" { + opts = append(opts, hostsets.WithFilter(c.FlagFilter)) + } + var version uint32 switch c.Func { diff --git a/internal/cmd/commands/rolescmd/roles.gen.go b/internal/cmd/commands/rolescmd/roles.gen.go index df3a9d725e..f18b364ce1 100644 --- a/internal/cmd/commands/rolescmd/roles.gen.go +++ b/internal/cmd/commands/rolescmd/roles.gen.go @@ -104,7 +104,7 @@ var flagsMap = map[string][]string{ "delete": {"id"}, - "list": {"scope-id", "recursive"}, + "list": {"scope-id", "filter", "recursive"}, } func (c *Command) Flags() *base.FlagSets { @@ -195,6 +195,10 @@ func (c *Command) Run(args []string) int { opts = append(opts, roles.WithRecursive(true)) } + if c.FlagFilter != "" { + opts = append(opts, roles.WithFilter(c.FlagFilter)) + } + var version uint32 switch c.Func { diff --git a/internal/cmd/commands/scopescmd/scopes.gen.go b/internal/cmd/commands/scopescmd/scopes.gen.go index a955dc8514..1bf4179ca2 100644 --- a/internal/cmd/commands/scopescmd/scopes.gen.go +++ b/internal/cmd/commands/scopescmd/scopes.gen.go @@ -104,7 +104,7 @@ var flagsMap = map[string][]string{ "delete": {"id"}, - "list": {"scope-id", "recursive"}, + "list": {"scope-id", "filter", "recursive"}, } func (c *Command) Flags() *base.FlagSets { @@ -195,6 +195,10 @@ func (c *Command) Run(args []string) int { opts = append(opts, scopes.WithRecursive(true)) } + if c.FlagFilter != "" { + opts = append(opts, scopes.WithFilter(c.FlagFilter)) + } + var version uint32 switch c.Func { diff --git a/internal/cmd/commands/server/listener_reload_test.go b/internal/cmd/commands/server/listener_reload_test.go index 322c7b5591..4b7af26585 100644 --- a/internal/cmd/commands/server/listener_reload_test.go +++ b/internal/cmd/commands/server/listener_reload_test.go @@ -104,10 +104,10 @@ func TestServer_ReloadListener(t *testing.T) { // Setup initial certs inBytes, err := ioutil.ReadFile(wd + "bundle1.pem") require.NoError(err) - require.NoError(ioutil.WriteFile(td+"/bundle.pem", inBytes, 0777)) + require.NoError(ioutil.WriteFile(td+"/bundle.pem", inBytes, 0o777)) relHcl := fmt.Sprintf(reloadConfig, cmd.DatabaseUrl, controllerKey, workerAuthKey, recoveryKey, td, td) - require.NoError(ioutil.WriteFile(td+"/reload.hcl", []byte(relHcl), 0777)) + require.NoError(ioutil.WriteFile(td+"/reload.hcl", []byte(relHcl), 0o777)) // Populate CA pool inBytes, _ = ioutil.ReadFile(td + "/bundle.pem") @@ -146,7 +146,7 @@ func TestServer_ReloadListener(t *testing.T) { inBytes, err = ioutil.ReadFile(wd + "bundle2.pem") require.NoError(err) - require.NoError(ioutil.WriteFile(td+"/bundle.pem", inBytes, 0777)) + require.NoError(ioutil.WriteFile(td+"/bundle.pem", inBytes, 0o777)) cmd.SighupCh <- struct{}{} select { diff --git a/internal/cmd/commands/sessionscmd/sessions.gen.go b/internal/cmd/commands/sessionscmd/sessions.gen.go index 5cad72ae22..7936d29e61 100644 --- a/internal/cmd/commands/sessionscmd/sessions.gen.go +++ b/internal/cmd/commands/sessionscmd/sessions.gen.go @@ -86,7 +86,7 @@ var flagsMap = map[string][]string{ "read": {"id"}, - "list": {"scope-id", "recursive"}, + "list": {"scope-id", "filter", "recursive"}, } func (c *Command) Flags() *base.FlagSets { @@ -153,6 +153,10 @@ func (c *Command) Run(args []string) int { opts = append(opts, sessions.WithRecursive(true)) } + if c.FlagFilter != "" { + opts = append(opts, sessions.WithFilter(c.FlagFilter)) + } + var version uint32 switch c.Func { diff --git a/internal/cmd/commands/targetscmd/targets.gen.go b/internal/cmd/commands/targetscmd/targets.gen.go index 302b1129e0..083c4e423f 100644 --- a/internal/cmd/commands/targetscmd/targets.gen.go +++ b/internal/cmd/commands/targetscmd/targets.gen.go @@ -95,7 +95,7 @@ var flagsMap = map[string][]string{ "delete": {"id"}, - "list": {"scope-id", "recursive"}, + "list": {"scope-id", "filter", "recursive"}, } func (c *Command) Flags() *base.FlagSets { @@ -182,6 +182,10 @@ func (c *Command) Run(args []string) int { opts = append(opts, targets.WithRecursive(true)) } + if c.FlagFilter != "" { + opts = append(opts, targets.WithFilter(c.FlagFilter)) + } + var version uint32 switch c.Func { diff --git a/internal/cmd/commands/targetscmd/tcp_targets.gen.go b/internal/cmd/commands/targetscmd/tcp_targets.gen.go index 45e544862a..12c351c180 100644 --- a/internal/cmd/commands/targetscmd/tcp_targets.gen.go +++ b/internal/cmd/commands/targetscmd/tcp_targets.gen.go @@ -164,6 +164,10 @@ func (c *TcpCommand) Run(args []string) int { opts = append(opts, targets.WithRecursive(true)) } + if c.FlagFilter != "" { + opts = append(opts, targets.WithFilter(c.FlagFilter)) + } + var version uint32 switch c.Func { diff --git a/internal/cmd/commands/userscmd/users.gen.go b/internal/cmd/commands/userscmd/users.gen.go index 485e5936f0..5e2890d894 100644 --- a/internal/cmd/commands/userscmd/users.gen.go +++ b/internal/cmd/commands/userscmd/users.gen.go @@ -104,7 +104,7 @@ var flagsMap = map[string][]string{ "delete": {"id"}, - "list": {"scope-id", "recursive"}, + "list": {"scope-id", "filter", "recursive"}, } func (c *Command) Flags() *base.FlagSets { @@ -195,6 +195,10 @@ func (c *Command) Run(args []string) int { opts = append(opts, users.WithRecursive(true)) } + if c.FlagFilter != "" { + opts = append(opts, users.WithFilter(c.FlagFilter)) + } + var version uint32 switch c.Func { diff --git a/internal/cmd/common/flags.go b/internal/cmd/common/flags.go index bf9151821c..693a2bd6a2 100644 --- a/internal/cmd/common/flags.go +++ b/internal/cmd/common/flags.go @@ -71,6 +71,12 @@ func PopulateCommonFlags(c *base.Command, f *base.FlagSet, resourceType string, Target: &c.FlagRecursive, Usage: "If set, the list operation will be applied recursively into child scopes, if supported by the type.", }) + case "filter": + f.StringVar(&base.StringVar{ + Name: "filter", + Target: &c.FlagFilter, + Usage: "If set, the list operation will be filtered before being returned. The filter operates against each item in the list. Using single quotes is recommended as filters contain double quotes. See https://www.boundaryproject.io/docs/concepts/filtering/resource-listing for details.", + }) } } } diff --git a/internal/cmd/gencli/templates.go b/internal/cmd/gencli/templates.go index 09ab60526d..10eae97e4c 100644 --- a/internal/cmd/gencli/templates.go +++ b/internal/cmd/gencli/templates.go @@ -183,7 +183,7 @@ var flags{{ camelCase .SubActionPrefix }}Map = map[string][]string{ "delete": {"id"}, {{ end }} {{ if eq $action "list" }} - "list": { "{{ kebabCase $input.Container }}-id" {{ if (eq $input.Container "Scope") }}, "recursive"{{ end }} }, + "list": { "{{ kebabCase $input.Container }}-id", "filter" {{ if (eq $input.Container "Scope") }}, "recursive"{{ end }} }, {{ end }} {{ end }} } @@ -296,6 +296,10 @@ func (c *{{ camelCase .SubActionPrefix }}Command) Run(args []string) int { } {{ end }} + if c.FlagFilter != "" { + opts = append(opts, {{ .Pkg }}.WithFilter(c.FlagFilter)) + } + {{ if .HasScopeName }} switch c.FlagScopeName { case "": diff --git a/website/content/docs/concepts/filtering/resource-listing.mdx b/website/content/docs/concepts/filtering/resource-listing.mdx new file mode 100644 index 0000000000..e6b358e278 --- /dev/null +++ b/website/content/docs/concepts/filtering/resource-listing.mdx @@ -0,0 +1,88 @@ +--- +layout: docs +page_title: Filtering - Resource Listing +sidebar_title: Resource Listing +description: |- + How to use filter list responses coming back from Boundary. +--- + +This page describes how to use filters when listing resources. This can be used +to reduce the returned set of resources when performing a list operation. + +~> This feature is intended to provide a userful service to clients; it does not +affect the database queries generated for the operation and as such is not +designed to provide greater efficiency. + +# List Filtering + +Starting in Boundary 0.1.8, when running a list action, a filter can be +specified. It uses the standard [filter syntax](/docs/concepts/filtering) used +elsewhere in Boundary. Unless otherwise specified for a given list endpoint, the +list of items being returned is looped through and the filter is run on the JSON +representation of that item. A good way to see what that data looks like is by +looking at representative JSON output on the command line; for example, the +following is the output of `boundary targets list -scope-id p_1234567890 -format json` on a dev instance (piped through `jq` for readability): + +```json +[ + { + "id": "ttcp_1234567890", + "scope_id": "p_1234567890", + "scope": { + "id": "p_1234567890", + "type": "project", + "name": "Generated project scope", + "description": "Provides an initial project scope in Boundary", + "parent_scope_id": "o_1234567890" + }, + "name": "Generated target", + "description": "Provides an initial target in Boundary", + "created_time": "2021-02-24T22:19:50.640476Z", + "updated_time": "2021-02-24T22:19:50.640476Z", + "version": 1, + "type": "tcp", + "session_max_seconds": 28800, + "session_connection_limit": -1, + "attributes": { + "default_port": 22 + }, + "authorized_actions": [ + "read", + "update", + "delete", + "add-host-sets", + "set-host-sets", + "remove-host-sets", + "authorize-session" + ] + } +] +``` + +As the filter tests each entry being returned, it places the data under test +within the filter at `/item`. + +On the CLI a filter can be given via `-filter`. + +~> Double quotes are part of the filter syntax; when using the CLI, it is likely +easier to surround the filter with single quotes than to deal with escaping +double quotes. + +When using the HTTP API, it is a `filter` query parameter. + +~> Ensure that the query parameter is properly escaped! Most HTTP libraries will +do this for you. If you're having trouble, try using the `-output-curl-string` +flag with the Boundary CLI: + +``` +$ boundary targets list -scope-id p_1234567890 -format json -filter '"authorize-session" in "/item/authorized_actions"' -output-curl-string +curl -H "Authorization: Bearer $(boundary config get-token -keyring-type pass -token-name default)"-H "Content-Type: application/json" 'http://127.0.0.1:9200/v1/targets?filter=%22authorize-session%22+in+%22%2Fitem%2Fauthorized_actions%22&scope_id=p_1234567890' +``` + +Following are some examples. + +- Resources in which the user is allowed to run an "update" action: + `"update" in "/item/authorized_actions"` + +- Resources matching a name pattern, but only those within an organization + scope: `"/item/name" matches "groupa-*" and "/item/scope/type" == "org"` diff --git a/website/data/docs-navigation.js b/website/data/docs-navigation.js index 3e50381bf8..544f302f12 100644 --- a/website/data/docs-navigation.js +++ b/website/data/docs-navigation.js @@ -44,7 +44,7 @@ export default [ }, { category: 'filtering', - content: ['worker-tags'], + content: ['resource-listing', 'worker-tags'], }, ], },