Merge pull request #1234 from sudo-suhas/env-dep-cachedir

allow cachedir override using env var

CHANGELOG.md

@@ -4,6 +4,8 @@ NEW FEATURES:
 * Add support for importing from [glock](https://github.com/robfig/glock) based projects (#1422).
 * Add support for importing from [govendor](https://github.com/kardianos/govendor)
   based projects (#815).
+* Allow override of cache directory location using environment variable
+`DEPCACHEDIR`. ([#1234](https://github.com/golang/dep/pull/1234))
 
 BUG FIXES:
 

cmd/dep/integration_test.go

@@ -7,9 +7,11 @@ package main
 import (
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"testing"
 
@@ -52,6 +54,94 @@ func TestIntegration(t *testing.T) {
 	})
 }
 
+func TestDepCachedir(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		// This test is unreliable on Windows and fails at random which makes it very
+		// difficult to debug. It might have something to do with parallel execution.
+		// Since the test doesn't test any specific behavior of Windows, it should be okay
+		// to skip.
+		t.Skip("skipping on windows")
+	}
+	t.Parallel()
+
+	test.NeedsExternalNetwork(t)
+	test.NeedsGit(t)
+
+	wd, err := os.Getwd()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	initPath := filepath.Join("testdata", "cachedir")
+
+	t.Run("env-cachedir", func(t *testing.T) {
+		t.Parallel()
+		testProj := integration.NewTestProject(t, initPath, wd, runMain)
+		defer testProj.Cleanup()
+
+		testProj.TempDir("cachedir")
+		cachedir := testProj.Path("cachedir")
+		testProj.Setenv("DEPCACHEDIR", cachedir)
+
+		// Running `dep ensure` will pull in the dependency into cachedir.
+		err = testProj.DoRun([]string{"ensure"})
+		if err != nil {
+			// Log the error output from running `dep ensure`, could be useful.
+			t.Logf("`dep ensure` error output: \n%s", testProj.GetStderr())
+			t.Errorf("got an unexpected error: %s", err)
+		}
+
+		// Check that the cache was created in the cachedir. Our fixture has the dependency
+		// `github.com/sdboyer/deptest`
+		_, err = os.Stat(testProj.Path("cachedir", "sources", "https---github.com-sdboyer-deptest"))
+		if err != nil {
+			if os.IsNotExist(err) {
+				t.Error("expected cachedir to have been populated but none was found")
+			} else {
+				t.Errorf("got an unexpected error: %s", err)
+			}
+		}
+	})
+	t.Run("env-invalid-cachedir", func(t *testing.T) {
+		t.Parallel()
+		testProj := integration.NewTestProject(t, initPath, wd, runMain)
+		defer testProj.Cleanup()
+
+		var d []byte
+		tmpFp := testProj.Path("tmp-file")
+		ioutil.WriteFile(tmpFp, d, 0644)
+		cases := []string{
+			// invalid path
+			"\000",
+			// parent directory does not exist
+			testProj.Path("non-existent-fldr", "cachedir"),
+			// path is a regular file
+			tmpFp,
+			// invalid path, tmp-file is a regular file
+			testProj.Path("tmp-file", "cachedir"),
+		}
+
+		wantErr := "dep: $DEPCACHEDIR set to an invalid or inaccessible path"
+		for _, c := range cases {
+			testProj.Setenv("DEPCACHEDIR", c)
+
+			err = testProj.DoRun([]string{"ensure"})
+
+			if err == nil {
+				// Log the output from running `dep ensure`, could be useful.
+				t.Logf("test run output: \n%s\n%s", testProj.GetStdout(), testProj.GetStderr())
+				t.Error("unexpected result: \n\t(GOT) nil\n\t(WNT) exit status 1")
+			} else if stderr := testProj.GetStderr(); !strings.Contains(stderr, wantErr) {
+				t.Errorf(
+					"unexpected error output: \n\t(GOT) %s\n\t(WNT) %s",
+					strings.TrimSpace(stderr), wantErr,
+				)
+			}
+		}
+	})
+
+}
+
 // execCmd is a test.RunFunc which runs the program in another process.
 func execCmd(prog string, args []string, stdout, stderr io.Writer, dir string, env []string) error {
 	cmd := exec.Command(prog, args...)

cmd/dep/main.go

@@ -18,6 +18,7 @@ import (
 	"text/tabwriter"
 
 	"github.com/golang/dep"
+	"github.com/golang/dep/internal/fs"
 )
 
 var (
@@ -158,41 +159,55 @@ func (c *Config) Run() int {
 	for _, cmd := range commands {
 		if cmd.Name() == cmdName {
 			// Build flag set with global flags in there.
-			fs := flag.NewFlagSet(cmdName, flag.ContinueOnError)
-			fs.SetOutput(c.Stderr)
-			verbose := fs.Bool("v", false, "enable verbose logging")
+			flags := flag.NewFlagSet(cmdName, flag.ContinueOnError)
+			flags.SetOutput(c.Stderr)
+			verbose := flags.Bool("v", false, "enable verbose logging")
 
 			// Register the subcommand flags in there, too.
-			cmd.Register(fs)
+			cmd.Register(flags)
 
 			// Override the usage text to something nicer.
-			resetUsage(errLogger, fs, cmdName, cmd.Args(), cmd.LongHelp())
+			resetUsage(errLogger, flags, cmdName, cmd.Args(), cmd.LongHelp())
 
 			if printCommandHelp {
-				fs.Usage()
+				flags.Usage()
 				return errorExitCode
 			}
 
 			// Parse the flags the user gave us.
 			// flag package automatically prints usage and error message in err != nil
 			// or if '-h' flag provided
-			if err := fs.Parse(c.Args[2:]); err != nil {
+			if err := flags.Parse(c.Args[2:]); err != nil {
 				return errorExitCode
 			}
 
+			// Cachedir is loaded from env if present. `$GOPATH/pkg/dep` is used as the
+			// default cache location.
+			cachedir := getEnv(c.Env, "DEPCACHEDIR")
+			if cachedir != "" {
+				if err := fs.EnsureDir(cachedir, 0777); err != nil {
+					errLogger.Printf(
+						"dep: $DEPCACHEDIR set to an invalid or inaccessible path: %q\n", cachedir,
+					)
+					errLogger.Printf("dep: failed to ensure cache directory: %v\n", err)
+					return errorExitCode
+				}
+			}
+
 			// Set up dep context.
 			ctx := &dep.Ctx{
 				Out:            outLogger,
 				Err:            errLogger,
 				Verbose:        *verbose,
 				DisableLocking: getEnv(c.Env, "DEPNOLOCK") != "",
+				Cachedir:       cachedir,
 			}
 
 			GOPATHS := filepath.SplitList(getEnv(c.Env, "GOPATH"))
 			ctx.SetPaths(c.WorkingDir, GOPATHS...)
 
 			// Run the command with the post-flag-processing args.
-			if err := cmd.Run(ctx, fs.Args()); err != nil {
+			if err := cmd.Run(ctx, flags.Args()); err != nil {
 				errLogger.Printf("%v\n", err)
 				return errorExitCode
 			}

cmd/dep/testdata/cachedir/Gopkg.toml

@@ -0,0 +1,4 @@
+
+[[constraint]]
+  name = "github.com/sdboyer/deptest"
+  version = "1.0.0"

cmd/dep/testdata/cachedir/main.go

@@ -0,0 +1,12 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	_ "github.com/sdboyer/deptest"
+)
+
+func main() {
+}

context.go

@@ -40,6 +40,7 @@ type Ctx struct {
 	Out, Err       *log.Logger // Required loggers.
 	Verbose        bool        // Enables more verbose logging.
 	DisableLocking bool        // When set, no lock file will be created to protect against simultaneous dep processes.
+	Cachedir       string      // Cache directory loaded from environment.
 }
 
 // SetPaths sets the WorkingDir and GOPATHs fields. If GOPATHs is empty, then
@@ -87,8 +88,18 @@ func defaultGOPATH() string {
 // SourceManager produces an instance of gps's built-in SourceManager
 // initialized to log to the receiver's logger.
 func (c *Ctx) SourceManager() (*gps.SourceMgr, error) {
+	cachedir := c.Cachedir
+	if cachedir == "" {
+		// When `DEPCACHEDIR` isn't set in the env, use the default - `$GOPATH/pkg/dep`.
+		cachedir = filepath.Join(c.GOPATH, "pkg", "dep")
+		// Create the default cachedir if it does not exist.
+		if err := os.MkdirAll(cachedir, 0777); err != nil {
+			return nil, errors.Wrap(err, "failed to create default cache directory")
+		}
+	}
+
 	return gps.NewSourceManager(gps.SourceManagerConfig{
-		Cachedir:       filepath.Join(c.GOPATH, "pkg", "dep"),
+		Cachedir:       cachedir,
 		Logger:         c.Out,
 		DisableLocking: c.DisableLocking,
 	})

context_test.go

@@ -488,3 +488,42 @@ func TestDetectGOPATH(t *testing.T) {
 		}
 	}
 }
+
+func TestDepCachedir(t *testing.T) {
+	h := test.NewHelper(t)
+	defer h.Cleanup()
+
+	h.TempDir("cache")
+	// Create the directory for default cachedir location.
+	h.TempDir(filepath.Join("go", "pkg", "dep"))
+
+	testCachedir := h.Path("cache")
+	gopath := h.Path("go")
+	discardLgr := discardLogger()
+
+	cases := []struct {
+		cachedir     string
+		wantCachedir string
+	}{
+		// If `Cachedir` is not set in the context, it should use `$GOPATH/pkg/dep`.
+		{cachedir: "", wantCachedir: h.Path(filepath.Join("go", "pkg", "dep"))},
+		// If `Cachedir` is set in the context, it should use that.
+		{cachedir: testCachedir, wantCachedir: testCachedir},
+	}
+
+	for _, c := range cases {
+		ctx := &Ctx{
+			GOPATH:   gopath,
+			Cachedir: c.cachedir,
+			Out:      discardLgr,
+			Err:      discardLgr,
+		}
+		sm, err := ctx.SourceManager()
+		h.Must(err)
+		defer sm.Release()
+
+		if sm.Cachedir() != c.wantCachedir {
+			t.Errorf("expected cachedir to be %s, got %s", c.wantCachedir, sm.Cachedir())
+		}
+	}
+}

gps/manager_test.go

@@ -121,17 +121,23 @@ func TestSourceManagerInit(t *testing.T) {
 		t.Fatalf("Global cache lock file not cleared correctly on Release()")
 	}
 
+	err = os.MkdirAll(cpath, 0777)
+	if err != nil {
+		t.Errorf("Failed to re-create temp dir: %s", err)
+	}
+	defer func() {
+		err = os.RemoveAll(cpath)
+		if err != nil {
+			t.Errorf("removeAll failed: %s", err)
+		}
+	}()
 	// Set another one up at the same spot now, just to be sure
 	sm, err = NewSourceManager(cfg)
 	if err != nil {
-		t.Errorf("Creating a second SourceManager should have succeeded when the first was released, but failed with err %s", err)
+		t.Fatalf("Creating a second SourceManager should have succeeded when the first was released, but failed with err %s", err)
 	}
 
 	sm.Release()
-	err = os.RemoveAll(cpath)
-	if err != nil {
-		t.Errorf("removeAll failed: %s", err)
-	}
 }
 
 func TestSourceInit(t *testing.T) {

gps/source_manager.go

@@ -20,6 +20,7 @@ import (
 	"time"
 
 	"github.com/golang/dep/gps/pkgtree"
+	"github.com/golang/dep/internal/fs"
 	"github.com/nightlyone/lockfile"
 	"github.com/pkg/errors"
 	"github.com/sdboyer/constext"
@@ -197,7 +198,7 @@ func NewSourceManager(c SourceManagerConfig) (*SourceMgr, error) {
 		c.Logger = log.New(ioutil.Discard, "", 0)
 	}
 
-	err := os.MkdirAll(filepath.Join(c.Cachedir, "sources"), 0777)
+	err := fs.EnsureDir(filepath.Join(c.Cachedir, "sources"), 0777)
 	if err != nil {
 		return nil, err
 	}
@@ -291,6 +292,11 @@ func NewSourceManager(c SourceManagerConfig) (*SourceMgr, error) {
 	return sm, nil
 }
 
+// Cachedir returns the location of the cache directory.
+func (sm *SourceMgr) Cachedir() string {
+	return sm.cachedir
+}
+
 // UseDefaultSignalHandling sets up typical os.Interrupt signal handling for a
 // SourceMgr.
 func (sm *SourceMgr) UseDefaultSignalHandling() {

internal/fs/fs.go

@@ -480,6 +480,23 @@ func cloneSymlink(sl, dst string) error {
 	return os.Symlink(resolved, dst)
 }
 
+// EnsureDir tries to ensure that a directory is present at the given path. It first
+// checks if the directory already exists at the given path. If there isn't one, it tries
+// to create it with the given permissions. However, it does not try to create the
+// directory recursively.
+func EnsureDir(path string, perm os.FileMode) error {
+	_, err := IsDir(path)
+
+	if os.IsNotExist(err) {
+		err = os.Mkdir(path, perm)
+		if err != nil {
+			return errors.Wrapf(err, "failed to ensure directory at %q", path)
+		}
+	}
+
+	return err
+}
+
 // IsDir determines is the path given is a directory or not.
 func IsDir(name string) (bool, error) {
 	fi, err := os.Stat(name)