diff --git a/CHANGELOG.md b/CHANGELOG.md index eca054be043..db48fcbe09b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ IMPROVEMENTS: [GH-2610] * client: Fingerprint all routable addresses on an interface including IPv6 addresses [GH-2536] + * client/artifact: Allow specifying a go-getter mode [GH-2781] + * client/artifact: Support non-Amazon S3-compatible sources [GH-2781] * client/template: Support reading env vars from templates [GH-2654] * config: Support Unix socket addresses for Consul [GH-2622] * discovery: Advertise driver-specified IP address and port [GH-2709] diff --git a/api/tasks.go b/api/tasks.go index aac90ee4070..bedbaa14972 100644 --- a/api/tasks.go +++ b/api/tasks.go @@ -2,6 +2,9 @@ package api import ( "fmt" + "path" + + "path/filepath" "strings" "time" @@ -323,12 +326,30 @@ func (t *Task) Canonicalize(tg *TaskGroup, job *Job) { type TaskArtifact struct { GetterSource *string `mapstructure:"source"` GetterOptions map[string]string `mapstructure:"options"` + GetterMode *string `mapstructure:"mode"` RelativeDest *string `mapstructure:"destination"` } func (a *TaskArtifact) Canonicalize() { + if a.GetterMode == nil { + a.GetterMode = helper.StringToPtr("any") + } + if a.GetterSource == nil { + // Shouldn't be possible, but we don't want to panic + a.GetterSource = helper.StringToPtr("") + } if a.RelativeDest == nil { - a.RelativeDest = helper.StringToPtr("local/") + switch *a.GetterMode { + case "file": + // File mode should default to local/filename + dest := *a.GetterSource + dest = path.Base(dest) + dest = filepath.Join("local", dest) + a.RelativeDest = &dest + default: + // Default to a directory + a.RelativeDest = helper.StringToPtr("local/") + } } } diff --git a/api/tasks_test.go b/api/tasks_test.go index 7756bfe528b..334a1cdf388 100644 --- a/api/tasks_test.go +++ b/api/tasks_test.go @@ -219,3 +219,17 @@ func TestTask_Constrain(t *testing.T) { t.Fatalf("expect: %#v, got: %#v", expect, task.Constraints) } } + +func TestTask_Artifact(t *testing.T) { + a := TaskArtifact{ + GetterSource: helper.StringToPtr("http://localhost/foo.txt"), + GetterMode: helper.StringToPtr("file"), + } + a.Canonicalize() + if *a.GetterMode != "file" { + t.Errorf("expected file but found %q", *a.GetterMode) + } + if *a.RelativeDest != "local/foo.txt" { + t.Errorf("expected local/foo.txt but found %q", *a.RelativeDest) + } +} diff --git a/client/getter/getter.go b/client/getter/getter.go index c55dafd2776..eeb9bb62c36 100644 --- a/client/getter/getter.go +++ b/client/getter/getter.go @@ -33,7 +33,7 @@ type EnvReplacer interface { } // getClient returns a client that is suitable for Nomad downloading artifacts. -func getClient(src, dst string) *gg.Client { +func getClient(src string, mode gg.ClientMode, dst string) *gg.Client { lock.Lock() defer lock.Unlock() @@ -50,7 +50,7 @@ func getClient(src, dst string) *gg.Client { return &gg.Client{ Src: src, Dst: dst, - Mode: gg.ClientModeAny, + Mode: mode, Getters: getters, } } @@ -97,7 +97,17 @@ func GetArtifact(taskEnv EnvReplacer, artifact *structs.TaskArtifact, taskDir st // Download the artifact dest := filepath.Join(taskDir, artifact.RelativeDest) - if err := getClient(url, dest).Get(); err != nil { + + // Convert from string getter mode to go-getter const + mode := gg.ClientModeAny + switch artifact.GetterMode { + case structs.GetterModeFile: + mode = gg.ClientModeFile + case structs.GetterModeDir: + mode = gg.ClientModeDir + } + + if err := getClient(url, mode, dest).Get(); err != nil { return newGetError(url, err, true) } diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index 7c975d617bd..b5bed914237 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -654,6 +654,7 @@ func ApiTaskToStructsTask(apiTask *api.Task, structsTask *structs.Task) { structsTask.Artifacts[k] = &structs.TaskArtifact{ GetterSource: *ta.GetterSource, GetterOptions: ta.GetterOptions, + GetterMode: *ta.GetterMode, RelativeDest: *ta.RelativeDest, } } diff --git a/command/agent/job_endpoint_test.go b/command/agent/job_endpoint_test.go index 75b8103d79a..3d189773a9c 100644 --- a/command/agent/job_endpoint_test.go +++ b/command/agent/job_endpoint_test.go @@ -4,6 +4,7 @@ import ( "net/http" "net/http/httptest" "reflect" + "strings" "testing" "time" @@ -12,6 +13,7 @@ import ( "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" + "github.com/kr/pretty" ) func TestHTTP_JobsList(t *testing.T) { @@ -993,6 +995,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { GetterOptions: map[string]string{ "a": "b", }, + GetterMode: helper.StringToPtr("dir"), RelativeDest: helper.StringToPtr("dest"), }, }, @@ -1178,6 +1181,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { GetterOptions: map[string]string{ "a": "b", }, + GetterMode: "dir", RelativeDest: "dest", }, }, @@ -1213,7 +1217,7 @@ func TestJobs_ApiJobToStructsJob(t *testing.T) { structsJob := ApiJobToStructJob(apiJob) - if !reflect.DeepEqual(expected, structsJob) { - t.Fatalf("bad %#v", structsJob) + if diff := pretty.Diff(expected, structsJob); len(diff) > 0 { + t.Fatalf("bad:\n%s", strings.Join(diff, "\n")) } } diff --git a/jobspec/parse.go b/jobspec/parse.go index 02f1dcbf1f9..34b6f32f4e9 100644 --- a/jobspec/parse.go +++ b/jobspec/parse.go @@ -788,6 +788,7 @@ func parseArtifacts(result *[]*api.TaskArtifact, list *ast.ObjectList) error { valid := []string{ "source", "options", + "mode", "destination", } if err := checkHCLKeys(o.Val, valid); err != nil { diff --git a/jobspec/parse_test.go b/jobspec/parse_test.go index 9fc57ba92c0..012108be521 100644 --- a/jobspec/parse_test.go +++ b/jobspec/parse_test.go @@ -166,6 +166,7 @@ func TestParse(t *testing.T) { GetterOptions: map[string]string{ "checksum": "md5:ff1cc0d3432dad54d607c1505fb7245c", }, + GetterMode: helper.StringToPtr("file"), }, }, Vault: &api.Vault{ diff --git a/jobspec/test-fixtures/basic.hcl b/jobspec/test-fixtures/basic.hcl index f7a38ee84f0..cfcfc5a1d81 100644 --- a/jobspec/test-fixtures/basic.hcl +++ b/jobspec/test-fixtures/basic.hcl @@ -140,6 +140,7 @@ job "binstore-storagelocker" { artifact { source = "http://bar.com/artifact" destination = "test/foo/" + mode = "file" options { checksum = "md5:ff1cc0d3432dad54d607c1505fb7245c" diff --git a/nomad/structs/diff_test.go b/nomad/structs/diff_test.go index 4ba261de3de..7e90dd9b00d 100644 --- a/nomad/structs/diff_test.go +++ b/nomad/structs/diff_test.go @@ -2469,6 +2469,7 @@ func TestTaskDiff(t *testing.T) { GetterOptions: map[string]string{ "bar": "baz", }, + GetterMode: "dir", RelativeDest: "bar", }, }, @@ -2487,6 +2488,7 @@ func TestTaskDiff(t *testing.T) { GetterOptions: map[string]string{ "bam": "baz", }, + GetterMode: "file", RelativeDest: "bam", }, }, @@ -2498,6 +2500,12 @@ func TestTaskDiff(t *testing.T) { Type: DiffTypeAdded, Name: "Artifact", Fields: []*FieldDiff{ + { + Type: DiffTypeAdded, + Name: "GetterMode", + Old: "", + New: "file", + }, { Type: DiffTypeAdded, Name: "GetterOptions[bam]", @@ -2522,6 +2530,12 @@ func TestTaskDiff(t *testing.T) { Type: DiffTypeDeleted, Name: "Artifact", Fields: []*FieldDiff{ + { + Type: DiffTypeDeleted, + Name: "GetterMode", + Old: "dir", + New: "", + }, { Type: DiffTypeDeleted, Name: "GetterOptions[bar]", diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index daa461fb4be..13a2c5015d9 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -78,6 +78,10 @@ const ( ProtocolVersion = "protocol" APIMajorVersion = "api.major" APIMinorVersion = "api.minor" + + GetterModeAny = "any" + GetterModeFile = "file" + GetterModeDir = "dir" ) // RPCInfo is used to describe common information about query @@ -3405,6 +3409,10 @@ type TaskArtifact struct { // go-getter. GetterOptions map[string]string + // GetterMode is the go-getter.ClientMode for fetching resources. + // Defaults to "any" but can be set to "file" or "dir". + GetterMode string + // RelativeDest is the download destination given relative to the task's // directory. RelativeDest string @@ -3453,6 +3461,17 @@ func (ta *TaskArtifact) Validate() error { mErr.Errors = append(mErr.Errors, fmt.Errorf("source must be specified")) } + switch ta.GetterMode { + case "": + // Default to any + ta.GetterMode = GetterModeAny + case GetterModeAny, GetterModeFile, GetterModeDir: + // Ok + default: + mErr.Errors = append(mErr.Errors, fmt.Errorf("invalid artifact mode %q; must be one of: %s, %s, %s", + ta.GetterMode, GetterModeAny, GetterModeFile, GetterModeDir)) + } + escaped, err := PathEscapesAllocDir("task", ta.RelativeDest) if err != nil { mErr.Errors = append(mErr.Errors, fmt.Errorf("invalid destination path: %v", err)) diff --git a/vendor/github.com/hashicorp/go-getter/README.md b/vendor/github.com/hashicorp/go-getter/README.md index 4a0b6a625d1..78239da54c9 100644 --- a/vendor/github.com/hashicorp/go-getter/README.md +++ b/vendor/github.com/hashicorp/go-getter/README.md @@ -222,13 +222,17 @@ None ### HTTP (`http`) -None +#### Basic Authentication + +To use HTTP basic authentication with go-getter, simply prepend `username:password@` to the +hostname in the URL such as `https://Aladdin:OpenSesame@www.example.com/index.html`. All special +characters, including the username and password, must be URL encoded. ### S3 (`s3`) S3 takes various access configurations in the URL. Note that it will also -read these from standard AWS environment variables if they're set. If -the query parameters are present, these take priority. +read these from standard AWS environment variables if they're set. S3 compliant servers like Minio +are also supported. If the query parameters are present, these take priority. * `aws_access_key_id` - AWS access key. * `aws_access_key_secret` - AWS access key secret. @@ -240,6 +244,14 @@ If you use go-getter and want to use an EC2 IAM Instance Profile to avoid using credentials, then just omit these and the profile, if available will be used automatically. +### Using S3 with Minio + If you use go-gitter for Minio support, you must consider the following: + + * `aws_access_key_id` (required) - Minio access key. + * `aws_access_key_secret` (required) - Minio access key secret. + * `region` (optional - defaults to us-east-1) - Region identifier to use. + * `version` (optional - fefaults to Minio default) - Configuration file format. + #### S3 Bucket Examples S3 has several addressing schemes used to reference your bucket. These are @@ -250,4 +262,5 @@ Some examples for these addressing schemes: - s3::https://s3-eu-west-1.amazonaws.com/bucket/foo - bucket.s3.amazonaws.com/foo - bucket.s3-eu-west-1.amazonaws.com/foo/bar +- "s3::http://127.0.0.1:9000/test-bucket/hello.txt?aws_access_key_id=KEYID&aws_access_key_secret=SECRETKEY®ion=us-east-2" diff --git a/vendor/github.com/hashicorp/go-getter/appveyor.yml b/vendor/github.com/hashicorp/go-getter/appveyor.yml index 159dad4dc25..ec48d45ec3d 100644 --- a/vendor/github.com/hashicorp/go-getter/appveyor.yml +++ b/vendor/github.com/hashicorp/go-getter/appveyor.yml @@ -1,5 +1,5 @@ version: "build-{branch}-{build}" -image: Visual Studio 2015 +image: Visual Studio 2017 clone_folder: c:\gopath\github.com\hashicorp\go-getter environment: GOPATH: c:\gopath diff --git a/vendor/github.com/hashicorp/go-getter/decompress_testing.go b/vendor/github.com/hashicorp/go-getter/decompress_testing.go index a2390e5d077..82b8ab4f6e8 100644 --- a/vendor/github.com/hashicorp/go-getter/decompress_testing.go +++ b/vendor/github.com/hashicorp/go-getter/decompress_testing.go @@ -11,7 +11,8 @@ import ( "runtime" "sort" "strings" - "testing" + + "github.com/mitchellh/go-testing-interface" ) // TestDecompressCase is a single test case for testing decompressors @@ -24,7 +25,7 @@ type TestDecompressCase struct { } // TestDecompressor is a helper function for testing generic decompressors. -func TestDecompressor(t *testing.T, d Decompressor, cases []TestDecompressCase) { +func TestDecompressor(t testing.T, d Decompressor, cases []TestDecompressCase) { for _, tc := range cases { t.Logf("Testing: %s", tc.Input) @@ -87,7 +88,7 @@ func TestDecompressor(t *testing.T, d Decompressor, cases []TestDecompressCase) } } -func testListDir(t *testing.T, path string) []string { +func testListDir(t testing.T, path string) []string { var result []string err := filepath.Walk(path, func(sub string, info os.FileInfo, err error) error { if err != nil { @@ -116,7 +117,7 @@ func testListDir(t *testing.T, path string) []string { return result } -func testMD5(t *testing.T, path string) string { +func testMD5(t testing.T, path string) string { f, err := os.Open(path) if err != nil { t.Fatalf("err: %s", err) diff --git a/vendor/github.com/hashicorp/go-getter/get_s3.go b/vendor/github.com/hashicorp/go-getter/get_s3.go index d3bffeb1737..ebb3217417d 100644 --- a/vendor/github.com/hashicorp/go-getter/get_s3.go +++ b/vendor/github.com/hashicorp/go-getter/get_s3.go @@ -28,7 +28,7 @@ func (g *S3Getter) ClientMode(u *url.URL) (ClientMode, error) { } // Create client config - config := g.getAWSConfig(region, creds) + config := g.getAWSConfig(region, u, creds) sess := session.New(config) client := s3.New(sess) @@ -84,7 +84,7 @@ func (g *S3Getter) Get(dst string, u *url.URL) error { return err } - config := g.getAWSConfig(region, creds) + config := g.getAWSConfig(region, u, creds) sess := session.New(config) client := s3.New(sess) @@ -139,7 +139,7 @@ func (g *S3Getter) GetFile(dst string, u *url.URL) error { return err } - config := g.getAWSConfig(region, creds) + config := g.getAWSConfig(region, u, creds) sess := session.New(config) client := s3.New(sess) return g.getObject(client, dst, bucket, path, version) @@ -174,7 +174,7 @@ func (g *S3Getter) getObject(client *s3.S3, dst, bucket, key, version string) er return err } -func (g *S3Getter) getAWSConfig(region string, creds *credentials.Credentials) *aws.Config { +func (g *S3Getter) getAWSConfig(region string, url *url.URL, creds *credentials.Credentials) *aws.Config { conf := &aws.Config{} if creds == nil { // Grab the metadata URL @@ -195,6 +195,14 @@ func (g *S3Getter) getAWSConfig(region string, creds *credentials.Credentials) * }) } + if creds != nil { + conf.Endpoint = &url.Host + conf.S3ForcePathStyle = aws.Bool(true) + if url.Scheme == "http" { + conf.DisableSSL = aws.Bool(true) + } + } + conf.Credentials = creds if region != "" { conf.Region = aws.String(region) @@ -204,29 +212,48 @@ func (g *S3Getter) getAWSConfig(region string, creds *credentials.Credentials) * } func (g *S3Getter) parseUrl(u *url.URL) (region, bucket, path, version string, creds *credentials.Credentials, err error) { - // Expected host style: s3.amazonaws.com. They always have 3 parts, - // although the first may differ if we're accessing a specific region. - hostParts := strings.Split(u.Host, ".") - if len(hostParts) != 3 { - err = fmt.Errorf("URL is not a valid S3 URL") - return - } + // This just check whether we are dealing with S3 or + // any other S3 compliant service. S3 has a predictable + // url as others do not + if strings.Contains(u.Host, "amazonaws.com") { + // Expected host style: s3.amazonaws.com. They always have 3 parts, + // although the first may differ if we're accessing a specific region. + hostParts := strings.Split(u.Host, ".") + if len(hostParts) != 3 { + err = fmt.Errorf("URL is not a valid S3 URL") + return + } - // Parse the region out of the first part of the host - region = strings.TrimPrefix(strings.TrimPrefix(hostParts[0], "s3-"), "s3") - if region == "" { - region = "us-east-1" - } + // Parse the region out of the first part of the host + region = strings.TrimPrefix(strings.TrimPrefix(hostParts[0], "s3-"), "s3") + if region == "" { + region = "us-east-1" + } - pathParts := strings.SplitN(u.Path, "/", 3) - if len(pathParts) != 3 { - err = fmt.Errorf("URL is not a valid S3 URL") - return - } + pathParts := strings.SplitN(u.Path, "/", 3) + if len(pathParts) != 3 { + err = fmt.Errorf("URL is not a valid S3 URL") + return + } + + bucket = pathParts[1] + path = pathParts[2] + version = u.Query().Get("version") - bucket = pathParts[1] - path = pathParts[2] - version = u.Query().Get("version") + } else { + pathParts := strings.SplitN(u.Path, "/", 3) + if len(pathParts) != 3 { + err = fmt.Errorf("URL is not a valid S3 complaint URL") + return + } + bucket = pathParts[1] + path = pathParts[2] + version = u.Query().Get("version") + region = u.Query().Get("region") + if region == "" { + region = "us-east-1" + } + } _, hasAwsId := u.Query()["aws_access_key_id"] _, hasAwsSecret := u.Query()["aws_access_key_secret"] diff --git a/vendor/github.com/hashicorp/go-sockaddr/GNUmakefile b/vendor/github.com/hashicorp/go-sockaddr/GNUmakefile new file mode 100644 index 00000000000..f3dfd24cfd3 --- /dev/null +++ b/vendor/github.com/hashicorp/go-sockaddr/GNUmakefile @@ -0,0 +1,65 @@ +TOOLS= golang.org/x/tools/cover +GOCOVER_TMPFILE?= $(GOCOVER_FILE).tmp +GOCOVER_FILE?= .cover.out +GOCOVERHTML?= coverage.html +FIND=`/usr/bin/which 2> /dev/null gfind find | /usr/bin/grep -v ^no | /usr/bin/head -n 1` +XARGS=`/usr/bin/which 2> /dev/null gxargs xargs | /usr/bin/grep -v ^no | /usr/bin/head -n 1` + +test:: $(GOCOVER_FILE) + @$(MAKE) -C cmd/sockaddr test + +cover:: coverage_report + +$(GOCOVER_FILE):: + @${FIND} . -type d ! -path '*cmd*' ! -path '*.git*' -print0 | ${XARGS} -0 -I % sh -ec "cd % && rm -f $(GOCOVER_TMPFILE) && go test -coverprofile=$(GOCOVER_TMPFILE)" + + @echo 'mode: set' > $(GOCOVER_FILE) + @${FIND} . -type f ! -path '*cmd*' ! -path '*.git*' -name "$(GOCOVER_TMPFILE)" -print0 | ${XARGS} -0 -n1 cat $(GOCOVER_TMPFILE) | grep -v '^mode: ' >> ${PWD}/$(GOCOVER_FILE) + +$(GOCOVERHTML): $(GOCOVER_FILE) + go tool cover -html=$(GOCOVER_FILE) -o $(GOCOVERHTML) + +coverage_report:: $(GOCOVER_FILE) + go tool cover -html=$(GOCOVER_FILE) + +audit_tools:: + @go get -u github.com/golang/lint/golint && echo "Installed golint:" + @go get -u github.com/fzipp/gocyclo && echo "Installed gocyclo:" + @go get -u github.com/remyoudompheng/go-misc/deadcode && echo "Installed deadcode:" + @go get -u github.com/client9/misspell/cmd/misspell && echo "Installed misspell:" + @go get -u github.com/gordonklaus/ineffassign && echo "Installed ineffassign:" + +audit:: + deadcode + go tool vet -all *.go + go tool vet -shadow=true *.go + golint *.go + ineffassign . + gocyclo -over 65 *.go + misspell *.go + +clean:: + rm -f $(GOCOVER_FILE) $(GOCOVERHTML) + +dev:: + @go build + @$(MAKE) -B -C cmd/sockaddr sockaddr + +install:: + @go install + @$(MAKE) -C cmd/sockaddr install + +doc:: + @echo Visit: http://127.0.0.1:6161/pkg/github.com/hashicorp/go-sockaddr/ + godoc -http=:6161 -goroot $GOROOT + +world:: + @set -e; \ + for os in solaris darwin freebsd linux windows; do \ + for arch in amd64; do \ + printf "Building on %s-%s\n" "$${os}" "$${arch}" ; \ + env GOOS="$${os}" GOARCH="$${arch}" go build -o /dev/null; \ + done; \ + done + + $(MAKE) -C cmd/sockaddr world diff --git a/vendor/github.com/mitchellh/go-testing-interface/LICENSE b/vendor/github.com/mitchellh/go-testing-interface/LICENSE new file mode 100644 index 00000000000..a3866a291fd --- /dev/null +++ b/vendor/github.com/mitchellh/go-testing-interface/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 Mitchell Hashimoto + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/mitchellh/go-testing-interface/README.md b/vendor/github.com/mitchellh/go-testing-interface/README.md new file mode 100644 index 00000000000..26781bbae88 --- /dev/null +++ b/vendor/github.com/mitchellh/go-testing-interface/README.md @@ -0,0 +1,52 @@ +# go-testing-interface + +go-testing-interface is a Go library that exports an interface that +`*testing.T` implements as well as a runtime version you can use in its +place. + +The purpose of this library is so that you can export test helpers as a +public API without depending on the "testing" package, since you can't +create a `*testing.T` struct manually. This lets you, for example, use the +public testing APIs to generate mock data at runtime, rather than just at +test time. + +## Usage & Example + +For usage and examples see the [Godoc](http://godoc.org/github.com/mitchellh/go-testing-interface). + +Given a test helper written using `go-testing-interface` like this: + + import "github.com/mitchellh/go-testing-interface" + + func TestHelper(t testing.T) { + t.Fatal("I failed") + } + +You can call the test helper in a real test easily: + + import "testing" + + func TestThing(t *testing.T) { + TestHelper(t) + } + +You can also call the test helper at runtime if needed: + + import "github.com/mitchellh/go-testing-interface" + + func main() { + TestHelper(&testing.RuntimeT{}) + } + +## Why?! + +**Why would I call a test helper that takes a *testing.T at runtime?** + +You probably shouldn't. The only use case I've seen (and I've had) for this +is to implement a "dev mode" for a service where the test helpers are used +to populate mock data, create a mock DB, perhaps run service dependencies +in-memory, etc. + +Outside of a "dev mode", I've never seen a use case for this and I think +there shouldn't be one since the point of the `testing.T` interface is that +you can fail immediately. diff --git a/vendor/github.com/mitchellh/go-testing-interface/testing.go b/vendor/github.com/mitchellh/go-testing-interface/testing.go new file mode 100644 index 00000000000..af6426a4dfd --- /dev/null +++ b/vendor/github.com/mitchellh/go-testing-interface/testing.go @@ -0,0 +1,70 @@ +package testing + +import ( + "fmt" + "log" +) + +// T is the interface that mimics the standard library *testing.T. +// +// In unit tests you can just pass a *testing.T struct. At runtime, outside +// of tests, you can pass in a RuntimeT struct from this package. +type T interface { + Error(args ...interface{}) + Errorf(format string, args ...interface{}) + Fatal(args ...interface{}) + Fatalf(format string, args ...interface{}) + Fail() + FailNow() + Failed() bool + Log(args ...interface{}) + Logf(format string, args ...interface{}) +} + +// RuntimeT implements T and can be instantiated and run at runtime to +// mimic *testing.T behavior. Unlike *testing.T, this will simply panic +// for calls to Fatal. For calls to Error, you'll have to check the errors +// list to determine whether to exit yourself. +type RuntimeT struct { + failed bool +} + +func (t *RuntimeT) Error(args ...interface{}) { + log.Println(fmt.Sprintln(args...)) + t.Fail() +} + +func (t *RuntimeT) Errorf(format string, args ...interface{}) { + log.Println(fmt.Sprintf(format, args...)) + t.Fail() +} + +func (t *RuntimeT) Fatal(args ...interface{}) { + log.Println(fmt.Sprintln(args...)) + t.FailNow() +} + +func (t *RuntimeT) Fatalf(format string, args ...interface{}) { + log.Println(fmt.Sprintf(format, args...)) + t.FailNow() +} + +func (t *RuntimeT) Fail() { + t.failed = true +} + +func (t *RuntimeT) FailNow() { + panic("testing.T failed, see logs for output (if any)") +} + +func (t *RuntimeT) Failed() bool { + return t.failed +} + +func (t *RuntimeT) Log(args ...interface{}) { + log.Println(fmt.Sprintln(args...)) +} + +func (t *RuntimeT) Logf(format string, args ...interface{}) { + log.Println(fmt.Sprintf(format, args...)) +} diff --git a/vendor/vendor.json b/vendor/vendor.json index b8da1cd876d..974466edf35 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -689,16 +689,16 @@ "revisionTime": "2017-06-02T22:43:19Z" }, { - "checksumSHA1": "hHoNT6CfqcDF1+yXRKnz/duuwIk=", + "checksumSHA1": "Sozy3aNAPBleUfiECj0jnaYjw2k=", "path": "github.com/hashicorp/go-getter", - "revision": "e48f67b534e614bf7fbd978fd0020f61a17b7527", - "revisionTime": "2017-04-05T22:15:29Z" + "revision": "2814e6fb2ca5b3bd950c97eff22553ecb3c7f77b", + "revisionTime": "2017-07-06T02:51:20Z" }, { "checksumSHA1": "9J+kDr29yDrwsdu2ULzewmqGjpA=", "path": "github.com/hashicorp/go-getter/helper/url", - "revision": "e48f67b534e614bf7fbd978fd0020f61a17b7527", - "revisionTime": "2017-04-05T22:15:29Z" + "revision": "2814e6fb2ca5b3bd950c97eff22553ecb3c7f77b", + "revisionTime": "2017-07-06T02:51:20Z" }, { "checksumSHA1": "zvmksNyW6g+Fd/bywd4vcn8rp+M=", @@ -981,6 +981,12 @@ "revision": "4fdf99ab29366514c69ccccddab5dc58b8d84062", "revisionTime": "2017-03-09T13:30:38Z" }, + { + "checksumSHA1": "j83WZHiKiPF86Hj5QMQdWboizls=", + "path": "github.com/mitchellh/go-testing-interface", + "revision": "477c2d05a845d8b55912a5a7993b9b24abcc5ef8", + "revisionTime": "2017-04-30T14:07:22Z" + }, { "path": "github.com/mitchellh/hashstructure", "revision": "1ef5c71b025aef149d12346356ac5973992860bc" diff --git a/website/source/docs/job-specification/artifact.html.md b/website/source/docs/job-specification/artifact.html.md index 6ecbfb3861b..cc083e31a11 100644 --- a/website/source/docs/job-specification/artifact.html.md +++ b/website/source/docs/job-specification/artifact.html.md @@ -46,10 +46,15 @@ unarchived before the starting the task. ## `artifact` Parameters -- `destination` `(string: "local/$1")` - Specifies the directory path to download the - artifact, relative to the root of the task's directory. If omitted, the - default value is to place the binary in `local/`. The destination is treated - as a directory and source files will be downloaded into that directory path. +- `destination` `(string: "local/")` - Specifies the directory path to download + the artifact, relative to the root of the task's directory. If omitted, the + default value is to place the artifact in `local/`. The destination is treated + as a directory unless `mode` is set to `file`. Source files will be downloaded + into that directory path. + +- `mode` `(string: "any")` - One of `any`, `file`, or `dir`. If set to `file` + the `destination` must be a file, not a directory. By default the + `destination` will be `local/`. - `options` `(map: nil)` - Specifies configuration parameters to fetch the artifact. The key-value pairs map directly to parameters appended to @@ -150,11 +155,13 @@ artifact { } ``` -### Download from an S3 Bucket +### Download from an S3-compatible Bucket These examples download artifacts from Amazon S3. There are several different types of [S3 bucket addressing][s3-bucket-addr] and [S3 region-specific -endpoints][s3-region-endpoints]. +endpoints][s3-region-endpoints]. As of Nomad 0.6 non-Amazon S3-compatible +endpoints like [Minio] are supported, but you must explicitly set the "s3::" +prefix. This example uses path-based notation on a publicly-accessible bucket: @@ -194,5 +201,6 @@ artifact { ``` [go-getter]: https://github.com/hashicorp/go-getter "HashiCorp go-getter Library" +[Minio]: https://www.minio.io/ [s3-bucket-addr]: http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html#access-bucket-intro "Amazon S3 Bucket Addressing" [s3-region-endpoints]: http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region "Amazon S3 Region Endpoints"