Skip to content

Commit

Permalink
Merge branch 'fix-56'
Browse files Browse the repository at this point in the history 
* fix-56: (32 commits)
  Updated README.md
  Priting login error early (#59)
  Added 'scw logout' command
  Improved help messages for login (#59)
  Update README.md
  golint fixes
  Removed spinlocks
  Refactored cp (#56)
  Updated debug
  Do not mix src/dst spawned process when having two remote endpoints in scw cp.
  Fix typo in README
  Update README.md
  Added .exe extension to windows binary
  Using i386 version for x86_64 release until we fix the static build
  Updated changelog
  Rename now uses generic PatchServer
  Added _patch method (#57)
  Added generic api.PatchServer method
  typo
  Added resolveIdentifier and getIdentifier api helpers
  ...
  • Loading branch information
@moul
moul committed Jun 12, 2015
2 parents 73011de + c8535d3 commit f0730dd
Show file tree
Hide file tree
Showing 5 changed files with 221 additions and 54 deletions.
18 changes: 11 additions & 7 deletions Makefile
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
# Go parameters
GOCMD ?= go
GOBUILD ?= $(GOCMD) build
GOBUILD ?= godep $(GOCMD) build
GOCLEAN ?= $(GOCMD) clean
GOINSTALL ?= $(GOCMD) install
GOTEST ?= $(GOCMD) test
GOTEST ?= godep $(GOCMD) test
GODEP ?= $(GOTEST) -i
GOFMT ?= gofmt -w

Expand All @@ -20,10 +20,10 @@ CLEAN_LIST = $(foreach int, $(SRC), $(int)_clean)
INSTALL_LIST = $(foreach int, $(SRC), $(int)_install)
IREF_LIST = $(foreach int, $(SRC), $(int)_iref)
TEST_LIST = $(foreach int, $(SRC), $(int)_test)
FMT_TEST = $(foreach int, $(SRC), $(int)_fmt)
FMT_LIST = $(foreach int, $(SRC), $(int)_fmt)


.PHONY: $(CLEAN_LIST) $(TEST_LIST) $(FMT_LIST) $(INSTALL_LIST) $(BUILD_LIST) $(IREF_LIST) scwversion/version.go
.PHONY: $(CLEAN_LIST) $(TEST_LIST) $(FMT_LIST) $(INSTALL_LIST) $(BUILD_LIST) $(IREF_LIST)


all: build
Expand All @@ -32,10 +32,14 @@ clean: $(CLEAN_LIST)
install: $(INSTALL_LIST)
test: $(TEST_LIST)
iref: $(IREF_LIST)
fmt: $(FMT_TEST)
fmt: $(FMT_LIST)


scwversion/version.go:
.git:
touch $@


scwversion/version.go: .git
@sed 's/\(.*GITCOMMIT.* = \).*/\1"$(REV)"/;s/\(.*VERSION.* = \).*/\1"$(TAG)"/' scwversion/version.tpl > $@.tmp
@if [ "$$(diff $@.tmp $@ 2>&1)" != "" ]; then mv $@.tmp $@; fi
@rm -f $@.tmp
Expand All @@ -58,7 +62,7 @@ $(IREF_LIST): %_iref: Godeps
$(GODEP) ./$*
$(TEST_LIST): %_test:
$(GOTEST) ./$*
$(FMT_TEST): %_fmt:
$(FMT_LIST): %_fmt:
$(GOFMT) ./$*


Expand Down
16 changes: 16 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,21 @@ running the command. Use '-' to write the data as a tar file to STDOUT.
Options:

-h, --help=false Print usage

Examples:

$ scw cp path/to/my/local/file myserver:path
$ scw cp myserver:path/to/file path/to/my/local/dir
$ scw cp myserver:path/to/file myserver2:path/to/dir
$ scw cp myserver:path/to/file - > myserver-pathtofile-backup.tar
$ scw cp myserver:path/to/file - | tar -tvf -
$ scw cp path/to/my/local/dir myserver:path
$ scw cp myserver:path/to/dir path/to/my/local/dir
$ scw cp myserver:path/to/dir myserver2:path/to/dir
$ scw cp myserver:path/to/dir - > myserver-pathtodir-backup.tar
$ scw cp myserver:path/to/dir - | tar -tvf -
$ cat archive.tar | scw cp - myserver:/path
$ tar -cvf - . | scw cp - myserver:path
```


Expand Down Expand Up @@ -925,6 +940,7 @@ Development in progress

#### Features

* Support of `scw cp` from {server-path,local-path,stdin} to {server-path,local-path,stdout} ([#56](https://github.com/scaleway/scaleway-cli/issues/56))
* Support of `_patch` experimental command ([#57](https://github.com/scaleway/scaleway-cli/issues/57))
* Support of `_completion` command (shell completion helper) ([#45](https://github.com/scaleway/scaleway-cli/issues/45))
* Returning more resource fields on `scw inspect` ([#50](https://github.com/scaleway/scaleway-cli/issues/50))
Expand Down
211 changes: 164 additions & 47 deletions cp.go
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
package main

import (
"fmt"
"io"
"os"
"os/exec"
"path"
"path/filepath"
"strings"

log "github.com/Sirupsen/logrus"
Expand All @@ -13,9 +14,23 @@ import (

var cmdCp = &Command{
Exec: runCp,
UsageLine: "cp [OPTIONS] SERVER:PATH HOSTDIR|-",
UsageLine: "cp [OPTIONS] SERVER:PATH|HOSTPATH|- SERVER:PATH|HOSTPATH|-",
Description: "Copy files/folders from a PATH on the server to a HOSTDIR on the host",
Help: "Copy files/folders from a PATH on the server to a HOSTDIR on the host\nrunning the command. Use '-' to write the data as a tar file to STDOUT.",
Examples: `
$ scw cp path/to/my/local/file myserver:path
$ scw cp myserver:path/to/file path/to/my/local/dir
$ scw cp myserver:path/to/file myserver2:path/to/dir
$ scw cp myserver:path/to/file - > myserver-pathtofile-backup.tar
$ scw cp myserver:path/to/file - | tar -tvf -
$ scw cp path/to/my/local/dir myserver:path
$ scw cp myserver:path/to/dir path/to/my/local/dir
$ scw cp myserver:path/to/dir myserver2:path/to/dir
$ scw cp myserver:path/to/dir - > myserver-pathtodir-backup.tar
$ scw cp myserver:path/to/dir - | tar -tvf -
$ cat archive.tar | scw cp - myserver:/path
$ tar -cvf - . | scw cp - myserver:path
`,
}

func init() {
Expand All @@ -25,72 +40,174 @@ func init() {
// Flags
var cpHelp bool // -h, --help flag

func runCp(cmd *Command, args []string) {
if cpHelp {
cmd.PrintUsage()
}
if len(args) != 2 {
cmd.PrintShortUsage()
}
func TarFromSource(api *ScalewayAPI, source string) (*io.ReadCloser, error) {
var tarOutputStream io.ReadCloser

hostPath := args[1]
// source is a server address + path (scp-like uri)
if strings.Index(source, ":") > -1 {
log.Debugf("Creating a tarball remotely and streaming it using SSH")
serverParts := strings.Split(source, ":")
if len(serverParts) != 2 {
return nil, fmt.Errorf("invalid source uri, see 'scw cp -h' for usage")
}

serverParts := strings.Split(args[0], ":")
if len(serverParts) != 2 {
log.Fatalf("usage: scw %s", cmd.UsageLine)
}
serverID := api.GetServerID(serverParts[0])

serverID := cmd.API.GetServerID(serverParts[0])
server, err := api.GetServer(serverID)
if err != nil {
return nil, err
}

server, err := cmd.API.GetServer(serverID)
if err != nil {
log.Fatalf("Failed to get server information for %s: %v", serverID, err)
}
dir, base := PathToTARPathparts(serverParts[1])
log.Debugf("Equivalent to 'scp root@%s:%s/%s ...'", server.PublicAddress.IP, dir, base)

// remoteCommand is executed on the remote server
// it streams a tarball raw content
remoteCommand := []string{"tar"}
remoteCommand = append(remoteCommand, "-C", dir)
if os.Getenv("DEBUG") == "1" {
remoteCommand = append(remoteCommand, "-v")
}
remoteCommand = append(remoteCommand, "-cf", "-")
remoteCommand = append(remoteCommand, base)

// execCmd contains the ssh connection + the remoteCommand
execCmd := append(NewSSHExecCmd(server.PublicAddress.IP, false, remoteCommand))
log.Debugf("Executing: ssh %s", strings.Join(execCmd, " "))
spawnSrc := exec.Command("ssh", execCmd...)

tarOutputStream, err = spawnSrc.StdoutPipe()
if err != nil {
return nil, err
}

tarErrorStream, err := spawnSrc.StderrPipe()
if err != nil {
return nil, err
}
defer tarErrorStream.Close()
io.Copy(os.Stderr, tarErrorStream)

// remoteCommand is executed on the remote server
// it streams a tarball raw content
remoteCommand := []string{"tar"}
remoteCommand = append(remoteCommand, "-C", path.Dir(serverParts[1]))
if os.Getenv("DEBUG") == "1" {
remoteCommand = append(remoteCommand, "-v")
err = spawnSrc.Start()
if err != nil {
return nil, err
}
defer spawnSrc.Wait()

return &tarOutputStream, nil
}
remoteCommand = append(remoteCommand, "-cf", "-")
remoteCommand = append(remoteCommand, path.Base(serverParts[1]))

// execCmd contains the ssh connection + the remoteCommand
execCmd := append(NewSSHExecCmd(server.PublicAddress.IP, false, remoteCommand))
log.Debugf("Executing: ssh %s", strings.Join(execCmd, " "))
spawn := exec.Command("ssh", execCmd...)
// source is stdin
if source == "-" {
log.Debugf("Streaming tarball from stdin")
tarOutputStream = os.Stdin
return &tarOutputStream, nil
}

tarOutputStream, err := spawn.StdoutPipe()
// source is a path on localhost
log.Debugf("Taring local path %s", source)
path, err := filepath.Abs(source)
if err != nil {
log.Fatal(err)
return nil, err
}
tarErrorStream, err := spawn.StderrPipe()
path, err = filepath.EvalSymlinks(path)
if err != nil {
log.Fatal(err)
return nil, err
}
log.Debugf("Real local path is %s", path)

err = spawn.Start()
dir, base := PathToTARPathparts(path)

tarOutputStream, err = archive.TarWithOptions(dir, &archive.TarOptions{
Compression: archive.Uncompressed,
IncludeFiles: []string{base},
})
if err != nil {
log.Fatalf("Failed to start ssh command: %v", err)
return nil, err
}
return &tarOutputStream, nil
}

func UntarToDest(api *ScalewayAPI, sourceStream *io.ReadCloser, destination string) error {
// destination is a server address + path (scp-like uri)
if strings.Index(destination, ":") > -1 {
log.Debugf("Streaming using ssh and untaring remotely")
serverParts := strings.Split(destination, ":")
if len(serverParts) != 2 {
return fmt.Errorf("invalid destination uri, see 'scw cp -h' for usage")
}

serverID := api.GetServerID(serverParts[0])

server, err := api.GetServer(serverID)
if err != nil {
return err
}

defer spawn.Wait()
// remoteCommand is executed on the remote server
// it streams a tarball raw content
remoteCommand := []string{"tar"}
remoteCommand = append(remoteCommand, "-C", serverParts[1])
if os.Getenv("DEBUG") == "1" {
remoteCommand = append(remoteCommand, "-v")
}
remoteCommand = append(remoteCommand, "-xf", "-")

io.Copy(os.Stderr, tarErrorStream)
// execCmd contains the ssh connection + the remoteCommand
execCmd := append(NewSSHExecCmd(server.PublicAddress.IP, false, remoteCommand))
log.Debugf("Executing: ssh %s", strings.Join(execCmd, " "))
spawnDst := exec.Command("ssh", execCmd...)

if hostPath == "-" {
log.Debugf("Writing tarOutputStream(%v) to os.Stdout(%v)", tarOutputStream, os.Stdout)
written, err := io.Copy(os.Stdout, tarOutputStream)
log.Debugf("%d bytes written", written)
untarInputStream, err := spawnDst.StdinPipe()
if err != nil {
log.Fatal(err)
return err
}
} else {
err = archive.Untar(tarOutputStream, hostPath, &archive.TarOptions{NoLchown: true})
defer untarInputStream.Close()

// spawnDst.Stderr = os.Stderr
// spawnDst.Stdout = os.Stdout

err = spawnDst.Start()
if err != nil {
log.Fatalf("Failed to untar the remote archive: %v", err)
return err
}

_, err = io.Copy(untarInputStream, *sourceStream)
return err
}

// destination is stdout
if destination == "-" { // stdout
log.Debugf("Writing sourceStream(%v) to os.Stdout(%v)", sourceStream, os.Stdout)
_, err := io.Copy(os.Stdout, *sourceStream)
return err
}

// destination is a path on localhost
log.Debugf("Untaring to local path: %s", destination)
err := archive.Untar(*sourceStream, destination, &archive.TarOptions{NoLchown: true})
return err
}

func runCp(cmd *Command, args []string) {
if cpHelp {
cmd.PrintUsage()
}
if len(args) != 2 {
cmd.PrintShortUsage()
}

if strings.Count(args[0], ":") > 1 || strings.Count(args[1], ":") > 1 {
log.Fatalf("usage: scw %s", cmd.UsageLine)
}

sourceStream, err := TarFromSource(cmd.API, args[0])
if err != nil {
log.Fatalf("Cannot tar from source '%s': %v", args[0], err)
}

err = UntarToDest(cmd.API, sourceStream, args[1])
if err != nil {
log.Fatalf("Cannot untar to destination '%s': %v", args[1], err)
}
}
7 changes: 7 additions & 0 deletions utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"net"
"os"
"os/exec"
"path"
"regexp"
"strings"
"time"
Expand Down Expand Up @@ -100,3 +101,9 @@ func wordify(str string) string {
str = strings.Trim(str, "_")
return str
}

// PathToTARPathparts returns the two parts of a unix path
func PathToTARPathparts(fullPath string) (string, string) {
fullPath = strings.TrimRight(fullPath, "/")
return path.Dir(fullPath), path.Base(fullPath)
}
23 changes: 23 additions & 0 deletions utils_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,3 +35,26 @@ func TestTruncIf(t *testing.T) {
t.Errorf("returned value is invalid [actual: %s][expected: %s]", actual, expected)
}
}

func TestPathToTARPathparts(t *testing.T) {
dir, base := PathToTARPathparts("/etc/passwd")
expected := []string{"/etc", "passwd"}
actual := []string{dir, base}
if actual[0] != expected[0] || actual[1] != expected[1] {
t.Errorf("returned value is invalid [actual: %s][expected: %s]", actual, expected)
}

dir, base = PathToTARPathparts("/etc")
expected = []string{"/", "etc"}
actual = []string{dir, base}
if actual[0] != expected[0] || actual[1] != expected[1] {
t.Errorf("returned value is invalid [actual: %s][expected: %s]", actual, expected)
}

dir, base = PathToTARPathparts("/etc/")
expected = []string{"/", "etc"}
actual = []string{dir, base}
if actual[0] != expected[0] || actual[1] != expected[1] {
t.Errorf("returned value is invalid [actual: %s][expected: %s]", actual, expected)
}
}

0 comments on commit f0730dd

Please sign in to comment.