build, vendor: use go modules for vendoring (#1532)

.golangci.yml

@@ -0,0 +1,13 @@
+linters-settings:
+  goconst:
+    # minimal length of string constant, 3 by default
+    min-len: 3
+    # minimal occurrences count to trigger, 3 by default
+    min-occurrences: 5
+
+issues:
+  # Maximum issues count per one linter. Set to 0 to disable. Default is 50.
+  max-issues-per-linter: 0
+
+  # Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
+  max-same-issues: 0

.travis.yml

@@ -1,6 +1,10 @@
 language: go
 go_import_path: github.com/ethersphere/swarm
 sudo: false
+env:
+  global:
+    - GO111MODULE=on
+    - GOFLAGS=-mod=vendor
 branches:
   only:
     - master

Makefile

@@ -11,3 +11,16 @@ swarm:
 
 alltools:
 	build/env.sh go run build/ci.go install ./cmd/...
+
+# Wrap go modules vendor command to copy forked cgo libraries
+# from go module cache and correct their file permissons.
+.PHONY: vendor
+vendor: export GO111MODULE=on
+vendor:
+	@go mod vendor
+	@cp -rf "$(shell GO111MODULE=on go list -f {{.Dir}} github.com/karalabe/hid)/hidapi" vendor/github.com/karalabe/hid/hidapi
+	@chmod -R u+w vendor/github.com/karalabe/hid/hidapi
+	@cp -rf "$(shell GO111MODULE=on go list -f {{.Dir}} github.com/karalabe/hid)/libusb" vendor/github.com/karalabe/hid/libusb
+	@chmod -R u+w vendor/github.com/karalabe/hid/libusb
+	@cp -rf "$(shell GO111MODULE=on go list -f {{.Dir}} github.com/ethereum/go-ethereum/crypto/secp256k1)/libsecp256k1" vendor/github.com/ethereum/go-ethereum/crypto/secp256k1/libsecp256k1
+	@chmod -R u+w vendor/github.com/ethereum/go-ethereum/crypto/secp256k1/libsecp256k1

README.md

@@ -189,11 +189,11 @@ $ git clone [email protected]:nirname/swarm.git $GOPATH/src/github.com/ethersphere/
 
 ### Vendored Dependencies
 
-All dependencies are tracked in the `vendor` directory. We use `govendor` to manage them.
+All dependencies are tracked in the `vendor` directory. We use `go mod` to manage depenedencies, except for `go mod vendor` command. Vendoring is done by Makefile rule `make vendor` which uses `go mod vendor` and additionally copies cgo dependencies into `vendor` directory from go modules cache.
 
-If you want to add a new dependency, run `govendor fetch <import-path>`, then commit the result.
+If you want to add a new dependency, run `GO111MODULE=on go get <import-path>`, vendor it `make vednor`, then commit the result.
 
-If you want to update all dependencies to their latest upstream version, run `govendor fetch +v`.
+If you want to update all dependencies to their latest upstream version, run `GO111MODULE=on go get -u all` and vendor them with `make vendor`.
 
 
 ### Testing

api/api_test.go

@@ -96,7 +96,7 @@ func checkResponse(t *testing.T, resp *testResponse, exp *Response) {
 	}
 	if resp.Content != exp.Content {
 		// if !bytes.Equal(resp.Content, exp.Content)
-		t.Errorf("incorrect content. expected '%s...', got '%s...'", string(exp.Content), string(resp.Content))
+		t.Errorf("incorrect content. expected '%s...', got '%s...'", exp.Content, resp.Content)
 	}
 }
 

api/http/middleware.go

@@ -113,7 +113,7 @@ func InitUploadTag(h http.Handler, tags *chunk.Tags) http.Handler {
 			uri := GetURI(r.Context())
 			if uri != nil {
 				log.Debug("got uri from context")
-				if uri.Addr == "encrypt" {
+				if uri.Addr == encryptAddr {
 					estimatedTotal = calculateNumberOfChunks(r.ContentLength, true)
 				} else {
 					estimatedTotal = calculateNumberOfChunks(r.ContentLength, false)
@@ -138,7 +138,7 @@ func InitUploadTag(h http.Handler, tags *chunk.Tags) http.Handler {
 func InstrumentOpenTracing(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		uri := GetURI(r.Context())
-		if uri == nil || r.Method == "" || (uri != nil && uri.Scheme == "") {
+		if uri == nil || r.Method == "" || uri.Scheme == "" {
 			h.ServeHTTP(w, r) // soft fail
 			return
 		}

api/http/server.go

@@ -67,6 +67,9 @@ var (
 const (
 	SwarmTagHeaderName = "x-swarm-tag" // Presence of this in header indicates the tag
 	PinHeaderName      = "x-swarm-pin" // Presence of this in header indicates pinning required
+
+	encryptAddr    = "encrypt"
+	tarContentType = "application/x-tar"
 )
 
 type methodHandler map[string]http.Handler
@@ -193,7 +196,7 @@ type Server struct {
 
 func (s *Server) HandleBzzGet(w http.ResponseWriter, r *http.Request) {
 	log.Debug("handleBzzGet", "ruid", GetRUID(r.Context()), "uri", r.RequestURI)
-	if r.Header.Get("Accept") == "application/x-tar" {
+	if r.Header.Get("Accept") == tarContentType {
 		uri := GetURI(r.Context())
 		_, credentials, _ := r.BasicAuth()
 		reader, err := s.api.GetDirectoryTar(r.Context(), s.api.Decryptor(r.Context(), credentials), uri)
@@ -208,7 +211,7 @@ func (s *Server) HandleBzzGet(w http.ResponseWriter, r *http.Request) {
 		}
 		defer reader.Close()
 
-		w.Header().Set("Content-Type", "application/x-tar")
+		w.Header().Set("Content-Type", tarContentType)
 
 		fileName := uri.Addr
 		if found := path.Base(uri.Path); found != "" && found != "." && found != "/" {
@@ -256,7 +259,7 @@ func (s *Server) HandlePostRaw(w http.ResponseWriter, r *http.Request) {
 
 	toEncrypt := false
 	uri := GetURI(r.Context())
-	if uri.Addr == "encrypt" {
+	if uri.Addr == encryptAddr {
 		toEncrypt = true
 	}
 
@@ -269,7 +272,7 @@ func (s *Server) HandlePostRaw(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if uri.Addr != "" && uri.Addr != "encrypt" {
+	if uri.Addr != "" && uri.Addr != encryptAddr {
 		postRawFail.Inc(1)
 		respondError(w, r, "raw POST request addr can only be empty or \"encrypt\"", http.StatusBadRequest)
 		return
@@ -327,15 +330,15 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) {
 
 	toEncrypt := false
 	uri := GetURI(r.Context())
-	if uri.Addr == "encrypt" {
+	if uri.Addr == encryptAddr {
 		toEncrypt = true
 	}
 
 	// Set the pinCounter if there is a pin header present in the request
 	headerPin := r.Header.Get(PinHeaderName)
 
 	var addr storage.Address
-	if uri.Addr != "" && uri.Addr != "encrypt" {
+	if uri.Addr != "" && uri.Addr != encryptAddr {
 		addr, err = s.api.Resolve(r.Context(), uri.Addr)
 		if err != nil {
 			postFilesFail.Inc(1)
@@ -354,7 +357,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) {
 	}
 	newAddr, err := s.api.UpdateManifest(r.Context(), addr, func(mw *api.ManifestWriter) error {
 		switch contentType {
-		case "application/x-tar":
+		case tarContentType:
 			_, err := s.handleTarUpload(r, mw)
 			if err != nil {
 				respondError(w, r, fmt.Sprintf("error uploading tarball: %v", err), http.StatusInternalServerError)

api/http/server_test.go

@@ -101,7 +101,7 @@ func TestBzzWithFeed(t *testing.T) {
 	`)
 
 	// POST data to bzz and get back a content-addressed **manifest hash** pointing to it.
-	resp, err := http.Post(fmt.Sprintf("%s/bzz:/", srv.URL), "text/plain", bytes.NewReader([]byte(dataBytes)))
+	resp, err := http.Post(fmt.Sprintf("%s/bzz:/", srv.URL), "text/plain", bytes.NewReader(dataBytes))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -188,7 +188,7 @@ func TestBzzWithFeed(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !bytes.Equal(retrievedData, []byte(dataBytes)) {
+	if !bytes.Equal(retrievedData, dataBytes) {
 		t.Fatalf("retrieved data mismatch, expected %x, got %x", dataBytes, retrievedData)
 	}
 }
@@ -746,7 +746,7 @@ func testBzzTar(encrypted bool, t *testing.T) {
 	//post tar stream
 	url := srv.URL + "/bzz:/"
 	if encrypted {
-		url = url + "encrypt"
+		url = url + encryptAddr
 	}
 	req, err := http.NewRequest("POST", url, buf)
 	if err != nil {
@@ -863,7 +863,7 @@ func TestBzzCorrectTagEstimate(t *testing.T) {
 		defer cancel()
 		addr := ""
 		if v.toEncrypt {
-			addr = "encrypt"
+			addr = encryptAddr
 		}
 		req, err := http.NewRequest("POST", srv.URL+"/bzz:/"+addr, pr)
 		if err != nil {

build/ci.go

@@ -109,6 +109,14 @@ func executablePath(name string) string {
 func main() {
 	log.SetFlags(log.Lshortfile)
 
+	// Use modules in subcommands.
+	os.Setenv("GO111MODULE", "on")
+	goflgs := "-mod=vendor"
+	if v := os.Getenv("GOFLAGS"); v != "" && v != "-mod=vendor" {
+		goflgs = v + " " + goflgs
+	}
+	os.Setenv("GOFLAGS", goflgs)
+
 	if _, err := os.Stat(filepath.Join("build", "ci.go")); os.IsNotExist(err) {
 		log.Fatal("this script must be run from the root of the repository")
 	}
@@ -284,12 +292,13 @@ func doLint(cmdline []string) {
 		packages = flag.CommandLine.Args()
 	}
 	// Get metalinter and install all supported linters
-	build.MustRun(goTool("get", "gopkg.in/alecthomas/gometalinter.v2"))
-	build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), "--install")
+	lintcmd := goTool("get", "github.com/golangci/golangci-lint/cmd/golangci-lint")
+	lintcmd.Env = append(lintcmd.Env, "GO111MODULE=off") // do not interfere with project modules
+	build.MustRun(lintcmd)
 
 	// Run fast linters batched together
 	configs := []string{
-		"--vendor",
+		"run",
 		"--tests",
 		"--deadline=2m",
 		"--disable-all",
@@ -299,14 +308,16 @@ func doLint(cmdline []string) {
 		"--enable=gofmt",
 		"--enable=misspell",
 		"--enable=goconst",
-		"--min-occurrences=6", // for goconst
 	}
-	build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), append(configs, packages...)...)
+	build.MustRunCommand(filepath.Join(GOBIN, "golangci-lint"), append(configs, packages...)...)
 
 	// Run slow linters one by one
-	for _, linter := range []string{"unconvert", "gosimple"} {
-		configs = []string{"--vendor", "--tests", "--deadline=10m", "--disable-all", "--enable=" + linter}
-		build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), append(configs, packages...)...)
+	for _, linter := range []string{
+		"unconvert",
+		"gosimple",
+	} {
+		configs = []string{"run", "--tests", "--deadline=10m", "--disable-all", "--enable=" + linter}
+		build.MustRunCommand(filepath.Join(GOBIN, "golangci-lint"), append(configs, packages...)...)
 	}
 }
 
@@ -644,6 +655,7 @@ func doXgo(cmdline []string) {
 
 	// Make sure xgo is available for cross compilation
 	gogetxgo := goTool("get", "github.com/karalabe/xgo")
+	gogetxgo.Env = append(gogetxgo.Env, "GO111MODULE=off") // do not interfere with project modules
 	build.MustRun(gogetxgo)
 
 	// If all tools building is requested, build everything the builder wants

build/env.sh

@@ -21,6 +21,8 @@ fi
 # Set up the environment to use the workspace.
 GOPATH="$workspace"
 export GOPATH
+export GO111MODULE=on
+export GOFLAGS=-mod=vendor
 
 # Run the command inside the workspace.
 cd "$ethdir/swarm"

cmd/swarm-smoke/util.go

@@ -212,7 +212,7 @@ func uploadWithTag(data []byte, endpoint string, tag string) (string, error) {
 		Tag: tag,
 	}
 
-	return swarm.TarUpload("", &client.FileUploader{f}, "", false, false)
+	return swarm.TarUpload("", &client.FileUploader{File: f}, "", false, false)
 }
 
 func digest(r io.Reader) ([]byte, error) {

cmd/swarm-snapshot/create_test.go

@@ -127,8 +127,8 @@ func TestSnapshotCreate(t *testing.T) {
 			// as strings to every node sorted services
 			sort.Strings(wantServices)
 
-			for i, n := range snap.Nodes {
-				gotServices := n.Node.Config.Services
+			for i := 0; i < len(snap.Nodes); i++ {
+				gotServices := snap.Nodes[i].Node.Config.Services
 				sort.Strings(gotServices)
 				if fmt.Sprint(gotServices) != fmt.Sprint(wantServices) {
 					t.Errorf("got services %v for node %v, want %v", gotServices, i, wantServices)

cmd/swarm/access_test.go

@@ -43,12 +43,14 @@ import (
 const (
 	hashRegexp = `[a-f\d]{128}`
 	data       = "notsorandomdata"
+
+	goosWindows = "windows"
 )
 
 var DefaultCurve = crypto.S256()
 
 func TestACT(t *testing.T) {
-	if runtime.GOOS == "windows" {
+	if runtime.GOOS == goosWindows {
 		t.Skip()
 	}
 

cmd/swarm/export_test.go

@@ -53,7 +53,7 @@ const (
 // 5. imports the exported datastore
 // 6. fetches the uploaded random file from the second node
 func TestCLISwarmExportImport(t *testing.T) {
-	if runtime.GOOS == "windows" {
+	if runtime.GOOS == goosWindows {
 		t.Skip()
 	}
 	cluster := newTestCluster(t, 1)
@@ -123,7 +123,7 @@ func TestCLISwarmExportImport(t *testing.T) {
 // 5. import the dump
 // 6. file should be accessible
 func TestExportLegacyToNew(t *testing.T) {
-	if runtime.GOOS == "windows" {
+	if runtime.GOOS == goosWindows {
 		t.Skip() // this should be reenabled once the appveyor tests underlying issue is fixed
 	}
 	/*

cmd/swarm/manifest_test.go

@@ -32,7 +32,7 @@ import (
 // TestManifestChange tests manifest add, update and remove
 // cli commands without encryption.
 func TestManifestChange(t *testing.T) {
-	if runtime.GOOS == "windows" {
+	if runtime.GOOS == goosWindows {
 		t.Skip()
 	}
 
@@ -42,7 +42,7 @@ func TestManifestChange(t *testing.T) {
 // TestManifestChange tests manifest add, update and remove
 // cli commands with encryption enabled.
 func TestManifestChangeEncrypted(t *testing.T) {
-	if runtime.GOOS == "windows" {
+	if runtime.GOOS == goosWindows {
 		t.Skip()
 	}
 
@@ -410,7 +410,7 @@ func testManifestChange(t *testing.T, encrypt bool) {
 // TestNestedDefaultEntryUpdate tests if the default entry is updated
 // if the file in nested manifest used for it is also updated.
 func TestNestedDefaultEntryUpdate(t *testing.T) {
-	if runtime.GOOS == "windows" {
+	if runtime.GOOS == goosWindows {
 		t.Skip()
 	}
 
@@ -421,7 +421,7 @@ func TestNestedDefaultEntryUpdate(t *testing.T) {
 // of encrypted upload is updated if the file in nested manifest
 // used for it is also updated.
 func TestNestedDefaultEntryUpdateEncrypted(t *testing.T) {
-	if runtime.GOOS == "windows" {
+	if runtime.GOOS == goosWindows {
 		t.Skip()
 	}
 

cmd/swarm/run_test.go

@@ -246,7 +246,7 @@ func getTestAccount(t *testing.T, dir string) (conf *node.Config, account accoun
 	}
 
 	// use a unique IPCPath when running tests on Windows
-	if runtime.GOOS == "windows" {
+	if runtime.GOOS == goosWindows {
 		conf.IPCPath = fmt.Sprintf("bzzd-%s.ipc", account.Address.String())
 	}
 
@@ -258,7 +258,7 @@ func existingTestNode(t *testing.T, dir string, bzzaccount string) *testNode {
 	node := &testNode{Dir: dir}
 
 	// use a unique IPCPath when running tests on Windows
-	if runtime.GOOS == "windows" {
+	if runtime.GOOS == goosWindows {
 		conf.IPCPath = fmt.Sprintf("bzzd-%s.ipc", bzzaccount)
 	}
 

cmd/swarm/upload_test.go

@@ -42,7 +42,7 @@ func init() {
 }
 
 func TestSwarmUp(t *testing.T) {
-	if runtime.GOOS == "windows" {
+	if runtime.GOOS == goosWindows {
 		t.Skip()
 	}
 
@@ -173,7 +173,7 @@ func testDefault(t *testing.T, cluster *testCluster, toEncrypt bool) {
 		}
 	}
 
-	timeout := time.Duration(2 * time.Second)
+	timeout := 2 * time.Second
 	httpClient := http.Client{
 		Timeout: timeout,
 	}