Add missing sync command flags (#409)

peak · Mar 25, 2022 · b43df69 · b43df69
1 parent a06e19a
commit b43df69
Show file tree

Hide file tree

Showing 16 changed files with 831 additions and 226 deletions.
diff --git a/command/app.go b/command/app.go
@@ -82,7 +82,7 @@ var app = &cli.App{
 
 		if retryCount < 0 {
 			err := fmt.Errorf("retry count cannot be a negative value")
-			printError(givenCommand(c), c.Command.Name, err)
+			printError(commandFromContext(c), c.Command.Name, err)
 			return err
 		}
 
@@ -154,8 +154,8 @@ func Commands() []*cli.Command {
 		NewSizeCommand(),
 		NewCatCommand(),
 		NewRunCommand(),
-		NewVersionCommand(),
 		NewSyncCommand(),
+		NewVersionCommand(),
 	}
 }
 

diff --git a/command/cat.go b/command/cat.go
@@ -36,7 +36,7 @@ func NewCatCommand() *cli.Command {
 		Before: func(c *cli.Context) error {
 			err := validateCatCommand(c)
 			if err != nil {
-				printError(givenCommand(c), c.Command.Name, err)
+				printError(commandFromContext(c), c.Command.Name, err)
 			}
 			return err
 		},
@@ -45,7 +45,7 @@ func NewCatCommand() *cli.Command {
 
 			src, err := url.New(c.Args().Get(0))
 			op := c.Command.Name
-			fullCommand := givenCommand(c)
+			fullCommand := commandFromContext(c)
 			if err != nil {
 				printError(fullCommand, op, err)
 				return err

diff --git a/command/context.go b/command/context.go
@@ -0,0 +1,106 @@
+package command
+
+import (
+	"flag"
+	"fmt"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/peak/s5cmd/storage/url"
+	"github.com/urfave/cli/v2"
+)
+
+func commandFromContext(c *cli.Context) string {
+	cmd := c.Command.FullName()
+
+	for _, f := range c.Command.Flags {
+		flagname := f.Names()[0]
+		for _, flagvalue := range contextValue(c, flagname) {
+			cmd = fmt.Sprintf("%s --%s=%v", cmd, flagname, flagvalue)
+		}
+	}
+
+	if c.Args().Len() > 0 {
+		cmd = fmt.Sprintf("%v %v", cmd, strings.Join(c.Args().Slice(), " "))
+	}
+
+	return cmd
+}
+
+// contextValue traverses context and its ancestor contexts to find
+// the flag value and returns string slice.
+func contextValue(c *cli.Context, flagname string) []string {
+	for _, c := range c.Lineage() {
+		if !c.IsSet(flagname) {
+			continue
+		}
+
+		val := c.Value(flagname)
+		switch val.(type) {
+		case cli.StringSlice:
+			return c.StringSlice(flagname)
+		case cli.Int64Slice, cli.IntSlice:
+			values := c.Int64Slice(flagname)
+			var result []string
+			for _, v := range values {
+				result = append(result, strconv.FormatInt(v, 10))
+			}
+			return result
+		case string:
+			return []string{c.String(flagname)}
+		case bool:
+			return []string{strconv.FormatBool(c.Bool(flagname))}
+		case int, int64:
+			return []string{strconv.FormatInt(c.Int64(flagname), 10)}
+		default:
+			return []string{fmt.Sprintf("%v", val)}
+		}
+	}
+
+	return nil
+}
+
+// generateCommand generates command string from given context, app command, default flags and urls.
+func generateCommand(c *cli.Context, cmd string, defaultFlags map[string]interface{}, urls ...*url.URL) (string, error) {
+	command := AppCommand(cmd)
+	flagset := flag.NewFlagSet(command.Name, flag.ContinueOnError)
+
+	var args []string
+	for _, url := range urls {
+		args = append(args, fmt.Sprintf("%q", url.String()))
+	}
+
+	flags := []string{}
+	for flagname, flagvalue := range defaultFlags {
+		flags = append(flags, fmt.Sprintf("--%s=%v", flagname, flagvalue))
+	}
+
+	isDefaultFlag := func(flagname string) bool {
+		_, ok := defaultFlags[flagname]
+		return ok
+	}
+
+	for _, f := range command.Flags {
+		flagname := f.Names()[0]
+		if isDefaultFlag(flagname) || !c.IsSet(flagname) {
+			continue
+		}
+
+		for _, flagvalue := range contextValue(c, flagname) {
+			flags = append(flags, fmt.Sprintf("--%s=%s", flagname, flagvalue))
+		}
+	}
+
+	sort.Strings(flags)
+	flags = append(flags, args...)
+	flags = append([]string{command.Name}, flags...)
+
+	err := flagset.Parse(flags)
+	if err != nil {
+		return "", err
+	}
+
+	cmdCtx := cli.NewContext(c.App, flagset, c)
+	return strings.TrimSpace(commandFromContext(cmdCtx)), nil
+}
diff --git a/command/context_test.go b/command/context_test.go
@@ -0,0 +1,197 @@
+package command
+
+import (
+	"flag"
+	"strings"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/peak/s5cmd/storage/url"
+	"github.com/urfave/cli/v2"
+)
+
+func TestGenerateCommand(t *testing.T) {
+	t.Parallel()
+
+	app := cli.NewApp()
+
+	testcases := []struct {
+		name            string
+		cmd             string
+		flags           []cli.Flag
+		defaultFlags    map[string]interface{}
+		ctx             *cli.Context
+		urls            []*url.URL
+		expectedCommand string
+	}{
+		{
+			name:  "empty-cli-flags",
+			cmd:   "cp",
+			flags: []cli.Flag{},
+			urls: []*url.URL{
+				mustNewURL(t, "s3://bucket/key1"),
+				mustNewURL(t, "s3://bucket/key2"),
+			},
+			expectedCommand: `cp "s3://bucket/key1" "s3://bucket/key2"`,
+		},
+		{
+			name:  "empty-cli-flags-with-default-flags",
+			cmd:   "cp",
+			flags: []cli.Flag{},
+			defaultFlags: map[string]interface{}{
+				"raw": true,
+				"acl": "public-read",
+			},
+			urls: []*url.URL{
+				mustNewURL(t, "s3://bucket/key1"),
+				mustNewURL(t, "s3://bucket/key2"),
+			},
+			expectedCommand: `cp --acl=public-read --raw=true "s3://bucket/key1" "s3://bucket/key2"`,
+		},
+		{
+			name: "same-flag-should-be-ignored-if-given-from-both-default-and-cli-flags",
+			cmd:  "cp",
+			flags: []cli.Flag{
+				&cli.BoolFlag{
+					Name:  "raw",
+					Value: true,
+				},
+			},
+			defaultFlags: map[string]interface{}{
+				"raw": true,
+			},
+			urls: []*url.URL{
+				mustNewURL(t, "s3://bucket/key1"),
+				mustNewURL(t, "s3://bucket/key2"),
+			},
+			expectedCommand: `cp --raw=true "s3://bucket/key1" "s3://bucket/key2"`,
+		},
+		{
+			name: "ignore-non-shared-flag",
+			cmd:  "cp",
+			flags: []cli.Flag{
+				&cli.BoolFlag{
+					Name:  "force-glacier-transfer",
+					Value: true,
+				},
+				&cli.BoolFlag{
+					Name:  "raw",
+					Value: true,
+				},
+				&cli.BoolFlag{
+					Name:  "flatten",
+					Value: true,
+				},
+				&cli.IntFlag{
+					Name:  "concurrency",
+					Value: 6,
+				},
+				// delete is not shared flag, will be ignored
+				&cli.BoolFlag{
+					Name:  "delete",
+					Value: true,
+				},
+				// size-only is not shared flag, will be ignored
+				&cli.BoolFlag{
+					Name:  "size-only",
+					Value: true,
+				},
+			},
+			urls: []*url.URL{
+				mustNewURL(t, "s3://bucket/key1"),
+				mustNewURL(t, "s3://bucket/key2"),
+			},
+			expectedCommand: `cp --concurrency=6 --flatten=true --force-glacier-transfer=true --raw=true "s3://bucket/key1" "s3://bucket/key2"`,
+		},
+		{
+			name: "string-slice-flag",
+			cmd:  "cp",
+			flags: []cli.Flag{
+				&cli.StringSliceFlag{
+					Name:  "exclude",
+					Value: cli.NewStringSlice("*.txt", "*.log"),
+				},
+			},
+			urls: []*url.URL{
+				mustNewURL(t, "/source/dir"),
+				mustNewURL(t, "s3://bucket/prefix/"),
+			},
+			expectedCommand: `cp --exclude=*.log --exclude=*.txt "/source/dir" "s3://bucket/prefix/"`,
+		},
+		{
+			name:  "command-with-multiple-args",
+			cmd:   "rm",
+			flags: []cli.Flag{},
+			urls: []*url.URL{
+				mustNewURL(t, "s3://bucket/key1"),
+				mustNewURL(t, "s3://bucket/key2"),
+				mustNewURL(t, "s3://bucket/prefix/key3"),
+				mustNewURL(t, "s3://bucket/prefix/key4"),
+			},
+			expectedCommand: `rm "s3://bucket/key1" "s3://bucket/key2" "s3://bucket/prefix/key3" "s3://bucket/prefix/key4"`,
+		},
+		{
+			name:  "command-args-with-spaces",
+			cmd:   "rm",
+			flags: []cli.Flag{},
+			urls: []*url.URL{
+				mustNewURL(t, "file with space"),
+				mustNewURL(t, "wow wow"),
+			},
+			expectedCommand: `rm "file with space" "wow wow"`,
+		},
+	}
+	for _, tc := range testcases {
+		tc := tc
+
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			command := AppCommand(tc.cmd)
+			set := flagSet(t, command.Name, tc.flags)
+			ctx := cli.NewContext(app, set, nil)
+
+			// urfave.Cli pass flags values to context before calling command.Action()
+			// and methods to update context are package-private, so write simple
+			// flag parser to update context value.
+			set.VisitAll(func(f *flag.Flag) {
+				value := strings.Trim(f.Value.String(), "[")
+				value = strings.Trim(value, "]")
+				for _, v := range strings.Fields(value) {
+					ctx.Set(f.Name, v)
+				}
+			})
+
+			got, err := generateCommand(ctx, command.Name, tc.defaultFlags, tc.urls...)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if diff := cmp.Diff(tc.expectedCommand, got); diff != "" {
+				t.Errorf("(-want +got):\n%v", diff)
+			}
+		})
+	}
+}
+
+func mustNewURL(t *testing.T, path string) *url.URL {
+	t.Helper()
+
+	u, err := url.New(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return u
+}
+
+func flagSet(t *testing.T, name string, flags []cli.Flag) *flag.FlagSet {
+	t.Helper()
+
+	set := flag.NewFlagSet(name, flag.ContinueOnError)
+	for _, f := range flags {
+		if err := f.Apply(set); err != nil {
+			t.Fatal(err)
+		}
+	}
+	return set
+}