WIP: tracing + refactor + gateway service.

cmd/reva/download.go

@@ -23,28 +23,27 @@ import (
 	"io"
 	"net/http"
 	"os"
-	"time"
 
 	"github.com/cheggaaa/pb"
 	rpcpb "github.com/cs3org/go-cs3apis/cs3/rpc"
 	storageproviderv0alphapb "github.com/cs3org/go-cs3apis/cs3/storageprovider/v0alpha"
+	"github.com/cs3org/reva/cmd/revad/svcs/httpsvcs/utils"
 )
 
 func downloadCommand() *command {
 	cmd := newCommand("download")
 	cmd.Description = func() string { return "download a remote file into the local filesystem" }
 	cmd.Usage = func() string { return "Usage: download [-flags] <remote_file> <local_file>" }
 	cmd.Action = func() error {
-		if cmd.NArg() < 3 {
+		if cmd.NArg() < 2 {
 			fmt.Println(cmd.Usage())
 			os.Exit(1)
 		}
 
-		provider := cmd.Args()[0]
-		remote := cmd.Args()[1]
-		local := cmd.Args()[2]
+		remote := cmd.Args()[0]
+		local := cmd.Args()[1]
 
-		client, err := getStorageProviderClient(provider)
+		client, err := getStorageProviderClient()
 		if err != nil {
 			return err
 		}
@@ -85,16 +84,12 @@ func downloadCommand() *command {
 
 		dataServerURL := res.DownloadEndpoint
 		// TODO(labkode): do a protocol switch
-		httpReq, err := http.NewRequest("GET", dataServerURL, nil)
+		httpReq, err := utils.NewRequest(ctx, "GET", dataServerURL, nil)
 		if err != nil {
 			return err
 		}
 
-		// TODO(labkode): harden http client
-		// https://medium.com/@nate510/don-t-use-go-s-default-http-client-4804cb19f779
-		httpClient := &http.Client{
-			Timeout: time.Second * 10,
-		}
+		httpClient := utils.GetHTTPClient(ctx)
 
 		httpRes, err := httpClient.Do(httpReq)
 		if err != nil {

cmd/reva/grpc.go

@@ -23,10 +23,6 @@ import (
 	"fmt"
 	"log"
 
-	"google.golang.org/grpc/metadata"
-
-	"github.com/pkg/errors"
-
 	appproviderv0alphapb "github.com/cs3org/go-cs3apis/cs3/appprovider/v0alpha"
 	appregistryv0alphapb "github.com/cs3org/go-cs3apis/cs3/appregistry/v0alpha"
 	authv0alphapb "github.com/cs3org/go-cs3apis/cs3/auth/v0alpha"
@@ -35,10 +31,15 @@ import (
 	storageproviderv0alphapb "github.com/cs3org/go-cs3apis/cs3/storageprovider/v0alpha"
 	storageregistryv0alphapb "github.com/cs3org/go-cs3apis/cs3/storageregistry/v0alpha"
 	usershareproviderv0alphapb "github.com/cs3org/go-cs3apis/cs3/usershareprovider/v0alpha"
-
+	"github.com/cs3org/reva/cmd/revad/svcs/grpcsvcs/pool"
+	"github.com/cs3org/reva/pkg/token"
+	"github.com/pkg/errors"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/metadata"
 )
 
+const defaultHeader = "x-access-token"
+
 func getAuthContext() context.Context {
 	ctx := context.Background()
 	// read token from file
@@ -47,7 +48,8 @@ func getAuthContext() context.Context {
 		log.Println(err)
 		return ctx
 	}
-	ctx = metadata.AppendToOutgoingContext(ctx, "x-access-token", t)
+	ctx = token.ContextSetToken(ctx, t)
+	ctx = metadata.AppendToOutgoingContext(ctx, defaultHeader, t)
 	return ctx
 }
 
@@ -74,20 +76,16 @@ func getAppRegistryClient() (appregistryv0alphapb.AppRegistryServiceClient, erro
 	return appregistryv0alphapb.NewAppRegistryServiceClient(conn), nil
 }
 
-func getUserShareProviderClient(host string) (usershareproviderv0alphapb.UserShareProviderServiceClient, error) {
-	conn, err := getConnToHost(host)
+func getUserShareProviderClient() (usershareproviderv0alphapb.UserShareProviderServiceClient, error) {
+	conn, err := getConn()
 	if err != nil {
 		return nil, err
 	}
 	return usershareproviderv0alphapb.NewUserShareProviderServiceClient(conn), nil
 }
 
-func getStorageProviderClient(host string) (storageproviderv0alphapb.StorageProviderServiceClient, error) {
-	conn, err := getConnToHost(host)
-	if err != nil {
-		return nil, err
-	}
-	return storageproviderv0alphapb.NewStorageProviderServiceClient(conn), nil
+func getStorageProviderClient() (storageproviderv0alphapb.StorageProviderServiceClient, error) {
+	return pool.GetStorageProviderServiceClient(conf.Host)
 }
 
 func getAuthClient() (authv0alphapb.AuthServiceClient, error) {

cmd/reva/import.go

@@ -34,20 +34,19 @@ import (
 func importCommand() *command {
 	cmd := newCommand("import")
 	cmd.Description = func() string { return "import metadata" }
-	cmd.Usage = func() string { return "Usage: import [-flags] <provider> <user export folder>" }
+	cmd.Usage = func() string { return "Usage: import [-flags] <user export folder>" }
 	cmd.Action = func() error {
-		if cmd.NArg() < 2 {
+		if cmd.NArg() < 1 {
 			fmt.Println(cmd.Usage())
 			os.Exit(1)
 		}
-		provider := cmd.Args()[0]
-		f := cmd.Args()[1]
+		f := cmd.Args()[0]
 
-		sClient, err := getStorageProviderClient(provider)
+		sClient, err := getStorageProviderClient()
 		if err != nil {
 			return err
 		}
-		uClient, err := getUserShareProviderClient(provider)
+		uClient, err := getUserShareProviderClient()
 		if err != nil {
 			return err
 		}

cmd/reva/ls.go

@@ -21,24 +21,26 @@ package main
 import (
 	"fmt"
 	"os"
+	"path"
 
 	storageproviderv0alphapb "github.com/cs3org/go-cs3apis/cs3/storageprovider/v0alpha"
 )
 
 func lsCommand() *command {
 	cmd := newCommand("ls")
-	cmd.Description = func() string { return "list a folder contents" }
-	cmd.Usage = func() string { return "Usage: ls [-flags] <provider> <folder_name>" }
+	cmd.Description = func() string { return "list a container contents" }
+	cmd.Usage = func() string { return "Usage: ls [-flags] <container_name>" }
 	longFlag := cmd.Bool("l", false, "long listing")
+	fullFlag := cmd.Bool("f", false, "shows full path")
+
 	cmd.Action = func() error {
-		if cmd.NArg() < 2 {
+		if cmd.NArg() < 1 {
 			fmt.Println(cmd.Usage())
 			os.Exit(1)
 		}
 
-		provider := cmd.Args()[0]
-		fn := cmd.Args()[1]
-		client, err := getStorageProviderClient(provider)
+		fn := cmd.Args()[0]
+		client, err := getStorageProviderClient()
 		if err != nil {
 			return err
 		}
@@ -56,10 +58,15 @@ func lsCommand() *command {
 
 		infos := res.Infos
 		for _, info := range infos {
+			p := info.Path
+			if !*fullFlag {
+				p = path.Base(info.Path)
+			}
+
 			if *longFlag {
-				fmt.Printf("%+v %d %d %v %s\n", info.PermissionSet, info.Mtime, info.Size, info.Id, info.Path)
+				fmt.Printf("%+v %d %d %v %s\n", info.PermissionSet, info.Mtime, info.Size, info.Id, p)
 			} else {
-				fmt.Println(info.Path)
+				fmt.Println(p)
 			}
 		}
 		return nil

cmd/reva/mkdir.go

@@ -28,19 +28,18 @@ import (
 
 func mkdirCommand() *command {
 	cmd := newCommand("mkdir")
-	cmd.Description = func() string { return "creates a folder" }
-	cmd.Usage = func() string { return "Usage: mkdir <folder_name> [-flags]" }
+	cmd.Description = func() string { return "creates a container" }
+	cmd.Usage = func() string { return "Usage: mkdir [-flags] <container_name>" }
 	cmd.Action = func() error {
-		if cmd.NArg() < 2 {
+		if cmd.NArg() < 1 {
 			fmt.Println(cmd.Usage())
 			os.Exit(1)
 		}
 
 		fn := cmd.Args()[0]
-		provider := cmd.Args()[1]
 
 		ctx := getAuthContext()
-		client, err := getStorageProviderClient(provider)
+		client, err := getStorageProviderClient()
 		if err != nil {
 			return err
 		}

cmd/reva/mv.go

@@ -31,17 +31,16 @@ func moveCommand() *command {
 	cmd.Description = func() string { return "moves/rename a file/folder" }
 	cmd.Usage = func() string { return "Usage: mv [-flags] <source> <destination>" }
 	cmd.Action = func() error {
-		if cmd.NArg() < 3 {
+		if cmd.NArg() < 2 {
 			fmt.Println(cmd.Usage())
 			os.Exit(1)
 		}
 
-		provider := cmd.Args()[0]
-		src := cmd.Args()[1]
-		dst := cmd.Args()[2]
+		src := cmd.Args()[0]
+		dst := cmd.Args()[1]
 
 		ctx := getAuthContext()
-		client, err := getStorageProviderClient(provider)
+		client, err := getStorageProviderClient()
 		if err != nil {
 			return err
 		}

cmd/reva/rm.go

@@ -31,15 +31,14 @@ func rmCommand() *command {
 	cmd.Description = func() string { return "removes a file or folder" }
 	cmd.Usage = func() string { return "Usage: rm [-flags] <file_name>" }
 	cmd.Action = func() error {
-		if cmd.NArg() < 2 {
+		if cmd.NArg() < 1 {
 			fmt.Println(cmd.Usage())
 			os.Exit(1)
 		}
 
-		provider := cmd.Args()[0]
-		fn := cmd.Args()[1]
+		fn := cmd.Args()[0]
 		ctx := getAuthContext()
-		client, err := getStorageProviderClient(provider)
+		client, err := getStorageProviderClient()
 		if err != nil {
 			return err
 		}

cmd/reva/stat.go

@@ -31,16 +31,15 @@ func statCommand() *command {
 	cmd.Description = func() string { return "get the metadata for a file or folder" }
 	cmd.Usage = func() string { return "Usage: stat [-flags] <file_name>" }
 	cmd.Action = func() error {
-		if cmd.NArg() < 2 {
+		if cmd.NArg() < 1 {
 			fmt.Println(cmd.Usage())
 			os.Exit(1)
 		}
 
-		provider := cmd.Args()[0]
-		fn := cmd.Args()[1]
+		fn := cmd.Args()[0]
 
 		ctx := getAuthContext()
-		client, err := getStorageProviderClient(provider)
+		client, err := getStorageProviderClient()
 		if err != nil {
 			return err
 		}

cmd/reva/upload.go

@@ -24,15 +24,13 @@ import (
 	"math"
 	"net/http"
 	"os"
-	"time"
-
-	"github.com/cs3org/reva/cmd/revad/svcs/grpcsvcs/storageprovidersvc"
-
-	"github.com/cs3org/reva/pkg/crypto"
 
 	"github.com/cheggaaa/pb"
 	rpcpb "github.com/cs3org/go-cs3apis/cs3/rpc"
 	storageproviderv0alphapb "github.com/cs3org/go-cs3apis/cs3/storageprovider/v0alpha"
+	"github.com/cs3org/reva/cmd/revad/svcs/grpcsvcs/storageprovidersvc"
+	"github.com/cs3org/reva/cmd/revad/svcs/httpsvcs/utils"
+	"github.com/cs3org/reva/pkg/crypto"
 )
 
 func uploadCommand() *command {
@@ -41,14 +39,15 @@ func uploadCommand() *command {
 	cmd.Usage = func() string { return "Usage: upload [-flags] <file_name> <remote_target>" }
 	xsFlag := cmd.String("xs", "negotiate", "compute checksum")
 	cmd.Action = func() error {
-		if cmd.NArg() < 3 {
+		ctx := getAuthContext()
+
+		if cmd.NArg() < 2 {
 			fmt.Println(cmd.Usage())
 			os.Exit(1)
 		}
 
-		provider := cmd.Args()[0]
-		fn := cmd.Args()[1]
-		target := cmd.Args()[2]
+		fn := cmd.Args()[0]
+		target := cmd.Args()[1]
 
 		fd, err := os.Open(fn)
 		if err != nil {
@@ -64,8 +63,7 @@ func uploadCommand() *command {
 
 		fmt.Printf("Local file size: %d bytes\n", md.Size())
 
-		ctx := getAuthContext()
-		client, err := getStorageProviderClient(provider)
+		client, err := getStorageProviderClient()
 		if err != nil {
 			return err
 		}
@@ -113,8 +111,7 @@ func uploadCommand() *command {
 		bar.Start()
 		reader := bar.NewProxyReader(fd)
 
-		// TODO(labkode): do a protocol switch
-		httpReq, err := http.NewRequest("PUT", dataServerURL, reader)
+		httpReq, err := utils.NewRequest(ctx, "PUT", dataServerURL, reader)
 		if err != nil {
 			return err
 		}
@@ -124,11 +121,7 @@ func uploadCommand() *command {
 		q.Add("xs_type", storageprovidersvc.GRPC2PKGXS(xsType).String())
 		httpReq.URL.RawQuery = q.Encode()
 
-		// TODO(labkode): harden http client
-		// https://medium.com/@nate510/don-t-use-go-s-default-http-client-4804cb19f779
-		httpClient := &http.Client{
-			Timeout: time.Second * 10,
-		}
+		httpClient := utils.GetHTTPClient(ctx)
 
 		httpRes, err := httpClient.Do(httpReq)
 		if err != nil {
@@ -160,6 +153,7 @@ func uploadCommand() *command {
 		info := res2.Info
 
 		fmt.Printf("File uploaded: %s:%s %d %s\n", info.Id.StorageId, info.Id.OpaqueId, info.Size, info.Path)
+
 		return nil
 	}
 	return cmd

cmd/revad/gateway.toml

@@ -0,0 +1,19 @@
+[core]
+log_file = "stderr"
+log_mode = "dev"
+max_cpus = "100%"
+tracing_enabled = true
+disable_http = true
+
+[log]
+level = "debug"
+mode = "console"
+
+[grpc]
+network = "tcp"
+address = "0.0.0.0:10000"
+enabled_services = ["gatewaysvc"]
+
+[grpc.services.gatewaysvc]
+storageregistrysvc = "localhost:9999"
+authsvc = "localhost:9999"