feat: Implement copy dir support

cmd/byctl/cp.go

@@ -2,18 +2,20 @@ package main
 
 import (
 	"fmt"
+	"strings"
 
-	"github.com/beyondstorage/go-storage/v4/services"
-	"github.com/beyondstorage/go-storage/v4/types"
 	"github.com/docker/go-units"
 	"github.com/urfave/cli/v2"
 	"go.uber.org/zap"
 
 	"github.com/beyondstorage/beyond-ctl/operations"
+	"github.com/beyondstorage/go-storage/v4/services"
+	"github.com/beyondstorage/go-storage/v4/types"
 )
 
 const (
 	cpFlagMultipartThresholdName = "multipart-threshold"
+	cpFlagDirName                = "dir"
 )
 
 var cpFlags = []cli.Flag{
@@ -25,6 +27,14 @@ var cpFlags = []cli.Flag{
 		},
 		Value: "1GiB", // Use 1 GiB as the default value.
 	},
+	&cli.BoolFlag{
+		Name: cpFlagDirName,
+		Aliases: []string{
+			"d",
+			"D",
+		},
+		Usage: "Used to copy a directory",
+	},
 }
 
 var cpCmd = &cli.Command{
@@ -58,6 +68,10 @@ var cpCmd = &cli.Command{
 			return err
 		}
 
+		if c.Bool(cpFlagDirName) && !strings.HasSuffix(srcKey, "/") {
+			srcKey += "/"
+		}
+
 		src, err := services.NewStoragerFromString(srcConn)
 		if err != nil {
 			logger.Error("init src storager", zap.Error(err), zap.String("conn string", srcConn))
@@ -78,11 +92,6 @@ var cpCmd = &cli.Command{
 			return err
 		}
 
-		if srcObject.Mode.IsDir() {
-			logger.Error("copy dir is not supported for now")
-			return fmt.Errorf("copy dir not supported")
-		}
-
 		size, ok := srcObject.GetContentLength()
 		if !ok {
 			logger.Error("can't get object content length", zap.String("path", srcKey))
@@ -134,7 +143,9 @@ var cpCmd = &cli.Command{
 		}
 
 		var ch chan *operations.EmptyResult
-		if size < multipartThreshold {
+		if c.Bool(cpFlagDirName) {
+			ch, err = do.CopyDir(srcKey, dstKey, multipartThreshold)
+		} else if size < multipartThreshold {
 			ch, err = do.CopyFileViaWrite(srcKey, dstKey, size)
 		} else {
 			// TODO: we will support other copy method later.

cmd/byctl/flags.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+
 	"github.com/urfave/cli/v2"
 )
 

cmd/byctl/ls.go

@@ -5,12 +5,12 @@ import (
 	"path/filepath"
 	"time"
 
-	"github.com/beyondstorage/go-storage/v4/services"
-	"github.com/beyondstorage/go-storage/v4/types"
 	"github.com/urfave/cli/v2"
 	"go.uber.org/zap"
 
 	"github.com/beyondstorage/beyond-ctl/operations"
+	"github.com/beyondstorage/go-storage/v4/services"
+	"github.com/beyondstorage/go-storage/v4/types"
 )
 
 const (

cmd/byctl/profile.go

@@ -5,10 +5,9 @@ import (
 	"fmt"
 	"os"
 
-	"go.uber.org/zap"
-
 	"github.com/BurntSushi/toml"
 	"github.com/urfave/cli/v2"
+	"go.uber.org/zap"
 
 	"github.com/beyondstorage/beyond-ctl/config"
 )

cmd/byctl/rm.go

@@ -4,11 +4,11 @@ import (
 	"errors"
 	"fmt"
 
-	"github.com/beyondstorage/go-storage/v4/services"
 	"github.com/urfave/cli/v2"
 	"go.uber.org/zap"
 
 	"github.com/beyondstorage/beyond-ctl/operations"
+	"github.com/beyondstorage/go-storage/v4/services"
 )
 
 const (

cmd/byctl/utils.go

@@ -2,16 +2,16 @@ package main
 
 import (
 	"fmt"
-	"github.com/Xuanwo/go-bufferpool"
 	"time"
 
-	"github.com/beyondstorage/go-storage/v4/pairs"
-	"github.com/beyondstorage/go-storage/v4/types"
+	"github.com/Xuanwo/go-bufferpool"
 	"github.com/docker/go-units"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/time/rate"
 
 	"github.com/beyondstorage/beyond-ctl/config"
+	"github.com/beyondstorage/go-storage/v4/pairs"
+	"github.com/beyondstorage/go-storage/v4/types"
 )
 
 var pool = bufferpool.New(128)

operations/cp.go

@@ -4,11 +4,13 @@ import (
 	"fmt"
 	"io"
 	"sort"
+	"strings"
 	"sync"
 
+	"go.uber.org/zap"
+
 	"github.com/beyondstorage/go-storage/v4/pairs"
 	"github.com/beyondstorage/go-storage/v4/types"
-	"go.uber.org/zap"
 )
 
 // CopyFileViaWrite will copy a file via Write operation.
@@ -189,3 +191,74 @@ func (do *DualOperator) copyMultipart(
 	}
 	ch <- &PartResult{Part: p}
 }
+
+// CopyDir will copy a directory.
+func (do *DualOperator) CopyDir(src, dst string, multipartThreshold int64) (errch chan *EmptyResult, err error) {
+	errch = make(chan *EmptyResult, 4)
+
+	so := NewSingleOperator(do.src)
+	och, err := so.ListRecursively(src)
+	if err != nil {
+		errch <- &EmptyResult{Error: err}
+	}
+
+	if !strings.HasSuffix(dst, "/") {
+		dst += "/"
+	}
+
+	go func() {
+		defer close(errch)
+
+		wg := &sync.WaitGroup{}
+
+		for or := range och {
+			if or.Error != nil {
+				errch <- &EmptyResult{Error: err}
+				break
+			}
+			object := or.Object
+
+			if object.Mode.IsDir() {
+				continue
+			}
+
+			path := dst + strings.TrimPrefix(object.Path, src)
+			size := object.MustGetContentLength()
+
+			wg.Add(1)
+			err = so.pool.Submit(func() {
+				defer wg.Done()
+
+				if size < multipartThreshold {
+					ch, err := do.CopyFileViaWrite(object.Path, path, size)
+					if err != nil {
+						errch <- &EmptyResult{Error: err}
+					}
+					for er := range ch {
+						if er.Error != nil {
+							errch <- &EmptyResult{Error: err}
+						}
+					}
+				} else {
+					ch, err := do.CopyFileViaMultipart(object.Path, path, size)
+					if err != nil {
+						errch <- &EmptyResult{Error: err}
+					}
+					for er := range ch {
+						if er.Error != nil {
+							errch <- &EmptyResult{Error: err}
+						}
+					}
+				}
+			})
+			if err != nil {
+				errch <- &EmptyResult{Error: err}
+				break
+			}
+		}
+
+		wg.Wait()
+	}()
+
+	return errch, nil
+}

operations/operator.go

@@ -3,9 +3,10 @@ package operations
 import (
 	"fmt"
 
-	"github.com/beyondstorage/go-storage/v4/types"
 	"github.com/panjf2000/ants/v2"
 	"go.uber.org/zap"
+
+	"github.com/beyondstorage/go-storage/v4/types"
 )
 
 type SingleOperator struct {