Skip to content

Commit

Permalink
Merge pull request #26 from rlmcpherson/ini_file
Browse files Browse the repository at this point in the history 
Ini file
  • Loading branch information
@rlmcpherson
rlmcpherson committed Aug 12, 2014
2 parents b9e2f8b + ab6a72f commit afb7c63
Show file tree
Hide file tree
Showing 6 changed files with 103 additions and 54 deletions.
18 changes: 8 additions & 10 deletions gof3r/cp.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,18 +11,18 @@ import (
"github.com/rlmcpherson/s3gof3r"
)

type Cp struct {
type cpOpts struct {
CommonOpts
Header http.Header `long:"header" short:"m" description:"HTTP headers. May be used to set custom metadata, server-side encryption etc."`
Header http.Header `long:"header" short:"m" description:"HTTP headers. May be used to set custom metadata, server-side encryption etc." ini-name:"header"`
}

var cp Cp
var cp cpOpts

func (cp *Cp) Usage() string {
func (cp *cpOpts) Usage() string {
return "<source> <dest> [cp-OPTIONS]"
}

func (cp *Cp) Execute(args []string) (err error) {
func (cp *cpOpts) Execute(args []string) (err error) {

k, err := getAWSKeys()
if err != nil {
Expand Down Expand Up @@ -59,10 +59,9 @@ func (cp *Cp) Execute(args []string) (err error) {
src, err := func(src *url.URL) (io.ReadCloser, error) {
if src.Host == "" {
return os.Open(src.Path)
} else {
r, _, err := s3.Bucket(src.Host).GetReader(src.Path, conf)
return r, err
}
r, _, err := s3.Bucket(src.Host).GetReader(src.Path, conf)
return r, err
}(urls[0])
if err != nil {
return
Expand All @@ -71,9 +70,8 @@ func (cp *Cp) Execute(args []string) (err error) {
dst, err := func(dst *url.URL) (io.WriteCloser, error) {
if dst.Host == "" {
return os.Create(dst.Path)
} else {
return s3.Bucket(dst.Host).PutWriter(dst.Path, cp.Header, conf)
}
return s3.Bucket(dst.Host).PutWriter(dst.Path, cp.Header, conf)
}(urls[1])
if err != nil {
return
Expand Down
28 changes: 14 additions & 14 deletions gof3r/cp_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,59 +15,59 @@ const (
)

var tb = os.Getenv("TEST_BUCKET")
var defaultCp = &Cp{
var defaultcpOpts = &cpOpts{
CommonOpts: CommonOpts{EndPoint: "s3.amazonaws.com",
PartSize: mb}}

type cpTest struct {
*Cp
*cpOpts
args []string
err error
}

var cpTests = []cpTest{
{defaultCp,
{defaultcpOpts,
[]string{"cp_test.go", "s3://" + tb + "/t1"},
nil},
{defaultCp,
{defaultcpOpts,
[]string{"s3://" + tb + "/t1", "s3://" + tb + "/t2"},
nil},
{defaultCp,
{defaultcpOpts,
[]string{"s3://" + tb + "/t1", "s3://" + tb + "//t2"},
nil},
{defaultCp,
{defaultcpOpts,
[]string{"s3://" + tb + "/t1", "/dev/null"},
nil},
{defaultCp,
{defaultcpOpts,
[]string{"s3://" + tb + "/noexist", "/dev/null"},
errors.New("404")},
{&Cp{
{&cpOpts{
CommonOpts: CommonOpts{EndPoint: "s3-external-1.amazonaws.com",
PartSize: mb}},
[]string{"s3://" + tb + "/&exist", "/dev/null"},
errors.New("404")},
{&Cp{
{&cpOpts{
CommonOpts: CommonOpts{NoSSL: true,
PartSize: mb}},
[]string{"s3://" + tb + "/t1", "s3://" + tb + "/tdir/.tst"},
nil},
{&Cp{
{&cpOpts{
CommonOpts: CommonOpts{EndPoint: "s3.amazonaws.com",
PartSize: mb}},
[]string{"s3://" + tb + "/t1"},
errors.New("source and destination arguments required")},
{defaultCp,
{defaultcpOpts,
[]string{"s://" + tb + "/t1", "s3://" + tb + "/tdir/.tst"},
errors.New("parse error: s://")},
{defaultCp,
{defaultcpOpts,
[]string{"http://%%s", ""},
errors.New("parse error: parse http")},
{defaultCp,
{defaultcpOpts,
[]string{"s3://" + tb + "/t1", "s3://no-bucket/.tst"},
errors.New("bucket does not exist")},
}

func TestCpExecute(t *testing.T) {
func TestcpOptsExecute(t *testing.T) {

if tb == "" {
t.Fatal("TEST_BUCKET must be set in environment")
Expand Down
14 changes: 7 additions & 7 deletions gof3r/get.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,17 +9,17 @@ import (
"github.com/rlmcpherson/s3gof3r"
)

type Get struct {
Key string `long:"key" short:"k" description:"S3 object key" required:"true"`
Bucket string `long:"bucket" short:"b" description:"S3 bucket" required:"true"`
Path string `short:"p" long:"path" description:"Path to file. Defaults to standard output for streaming."`
type getOpts struct {
Key string `long:"key" short:"k" description:"S3 object key" required:"true" no-ini:"true"`
Bucket string `long:"bucket" short:"b" description:"S3 bucket" required:"true" no-ini:"true"`
Path string `short:"p" long:"path" description:"Path to file. Defaults to standard output for streaming." no-ini:"true"`
CommonOpts
VersionID string `short:"v" long:"versionId" description:"Version ID of the object. Incompatible with md5 check (use --no-md5)."`
VersionID string `short:"v" long:"versionId" description:"Version ID of the object. Incompatible with md5 check (use --no-md5)." no-ini:"true"`
}

var get Get
var get getOpts

func (get *Get) Execute(args []string) (err error) {
func (get *getOpts) Execute(args []string) (err error) {
conf := new(s3gof3r.Config)
*conf = *s3gof3r.DefaultConfig
k, err := getAWSKeys()
Expand Down
13 changes: 11 additions & 2 deletions gof3r/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,13 @@ import (
"runtime"
"time"

"github.com/jessevdk/go-flags"
"github.com/rlmcpherson/go-flags"
"github.com/rlmcpherson/s3gof3r"
)

const (
name = "gof3r"
version = "0.4.1"
version = "0.4.2"
)

func main() {
Expand All @@ -51,9 +51,18 @@ func main() {

start := time.Now()

// parse ini file
if err := parseIni(); err != nil {
fmt.Fprintln(os.Stderr, err)
}

// parser calls the Execute function for the command after parsing the command line options.
if _, err := parser.Parse(); err != nil {

if appOpts.WriteIni {
writeIni() // exits
}

// handling for flag parse errors
if ferr, ok := err.(*flags.Error); ok {
if ferr.Type == flags.ErrHelp {
Expand Down
70 changes: 56 additions & 14 deletions gof3r/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,38 +4,43 @@ import (
"fmt"
"log"
"os"
"os/user"

"github.com/jessevdk/go-flags"
"github.com/rlmcpherson/go-flags"
)

const (
iniFile = ".gof3r.ini"
)

// CommonOpts are Options common to all commands
type CommonOpts struct {
NoSSL bool `long:"no-ssl" description:"Do not use SSL for endpoint connection."`
NoMd5 bool `long:"no-md5" description:"Do not use md5 hash checking to ensure data integrity. By default, the md5 hash of is calculated concurrently during puts, stored at <bucket>.md5/<key>.md5, and verified on gets."`
Concurrency int `long:"concurrency" short:"c" default:"10" description:"Concurrency of transfers"`
PartSize int64 `long:"partsize" short:"s" description:"Initial size of concurrent parts, in bytes" default:"20971520"`
EndPoint string `long:"endpoint" description:"Amazon S3 endpoint" default:"s3.amazonaws.com"`
Debug bool `long:"debug" description:"Enable debug logging."`
NoSSL bool `long:"no-ssl" description:"Do not use SSL for endpoint connection." ini-name:"no-ssl"`
NoMd5 bool `long:"no-md5" description:"Do not use md5 hash checking to ensure data integrity. By default, the md5 hash of is calculated concurrently during puts, stored at <bucket>.md5/<key>.md5, and verified on gets." ini-name:"no-md5"`
Concurrency int `long:"concurrency" short:"c" default:"10" description:"Concurrency of transfers" ini-name:"concurrency"`
PartSize int64 `long:"partsize" short:"s" description:"Initial size of concurrent parts, in bytes" default:"20971520" ini-name:"partsize"`
EndPoint string `long:"endpoint" description:"Amazon S3 endpoint" default:"s3.amazonaws.com" ini-name:"endpoint"`
Debug bool `long:"debug" description:"Enable debug logging." ini-name:"debug"`
}

var AppOpts struct {
Version func() `long:"version" short:"v" description:"Print version"`
Man func() `long:"manpage" short:"m" description:"Create gof3r.man man page in current directory"`
var appOpts struct {
Version func() `long:"version" short:"v" description:"Print version"`
Man func() `long:"manpage" short:"m" description:"Create gof3r.man man page in current directory"`
WriteIni bool `long:"writeini" short:"i" description:"Write .gof3r.ini in current user's home directory" no-ini:"true"`
}

var parser = flags.NewParser(&AppOpts, (flags.HelpFlag | flags.PassDoubleDash))
var parser = flags.NewParser(&appOpts, (flags.HelpFlag | flags.PassDoubleDash))

func init() {

// set parser fields
parser.ShortDescription = "streaming, concurrent s3 client"

AppOpts.Version = func() {
appOpts.Version = func() {
fmt.Fprintf(os.Stderr, "%s version %s\n", name, version)
os.Exit(0)
}

AppOpts.Man = func() {
appOpts.Man = func() {
f, err := os.Create(name + ".man")
if err != nil {
log.Fatal(err)
Expand All @@ -45,3 +50,40 @@ func init() {
os.Exit(0)
}
}

func iniPath() (path string, exist bool, err error) {
usr, err := user.Current()
if err != nil {
return
}
path = fmt.Sprintf("%s/%s", usr.HomeDir, iniFile)
if _, staterr := os.Stat(path); !os.IsNotExist(staterr) {
exist = true
}
return
}

func parseIni() (err error) {
p, exist, err := iniPath()
if err != nil || !exist {
return
}
return flags.NewIniParser(parser).ParseFile(p)
}

func writeIni() {
p, exist, err := iniPath()
if err != nil {
log.Fatal(err)
}
if exist {
fmt.Fprintf(os.Stderr, "%s exists, refusing to overwrite.\n", p)
} else {
if err := flags.NewIniParser(parser).WriteFile(p,
(flags.IniIncludeComments | flags.IniIncludeDefaults | flags.IniCommentDefaults)); err != nil {
log.Fatal(err)
}
fmt.Fprintf(os.Stderr, "ini file written to %s\n", p)
}
os.Exit(0)
}
14 changes: 7 additions & 7 deletions gof3r/put.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,17 +9,17 @@ import (
"github.com/rlmcpherson/s3gof3r"
)

type Put struct {
Key string `long:"key" short:"k" description:"S3 object key" required:"true"`
Bucket string `long:"bucket" short:"b" description:"S3 bucket" required:"true"`
Path string `short:"p" long:"path" description:"Path to file. Defaults to standard input for streaming."`
type putOpts struct {
Key string `long:"key" short:"k" description:"S3 object key" required:"true" no-ini:"true"`
Bucket string `long:"bucket" short:"b" description:"S3 bucket" required:"true" no-ini:"true"`
Path string `short:"p" long:"path" description:"Path to file. Defaults to standard output for streaming." no-ini:"true"`
CommonOpts
Header http.Header `long:"header" short:"m" description:"HTTP headers. May be used to set custom metadata, server-side encryption etc."`
Header http.Header `long:"header" short:"m" description:"HTTP headers. May be used to set custom metadata, server-side encryption etc." ini-name:"header"`
}

var put Put
var put putOpts

func (put *Put) Execute(args []string) (err error) {
func (put *putOpts) Execute(args []string) (err error) {
conf := new(s3gof3r.Config)
*conf = *s3gof3r.DefaultConfig
k, err := getAWSKeys()
Expand Down

0 comments on commit afb7c63

Please sign in to comment.