Updating mage setup command to download tools inside repository folde…

…r (#6)

.gitignore

@@ -1,4 +1,5 @@
 out/
+.setup/
 venv/
 .idea/
 __pycache__/

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/Masterminds/sprig v2.22.0+incompatible // indirect
 	github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496 // indirect
 	github.com/aws/aws-lambda-go v1.13.3
-	github.com/aws/aws-sdk-go v1.28.3
+	github.com/aws/aws-sdk-go v1.28.5
 	github.com/cenkalti/backoff v2.2.1+incompatible
 	github.com/go-openapi/analysis v0.19.7 // indirect
 	github.com/go-openapi/errors v0.19.3
@@ -34,18 +34,18 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.1 // indirect
 	github.com/olekukonko/tablewriter v0.0.4 // indirect
-	github.com/pkg/errors v0.8.1
+	github.com/pkg/errors v0.9.1
 	github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
 	github.com/stretchr/testify v1.4.0
 	github.com/tidwall/gjson v1.3.5
 	go.mongodb.org/mongo-driver v1.2.1 // indirect
 	go.uber.org/atomic v1.5.1 // indirect
 	go.uber.org/multierr v1.4.0 // indirect
 	go.uber.org/zap v1.13.0
-	golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d // indirect
+	golang.org/x/crypto v0.0.0-20200117160349-530e935923ad // indirect
 	golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f // indirect
 	golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa // indirect
-	golang.org/x/tools v0.0.0-20200116225955-84cebe10344f // indirect
+	golang.org/x/tools v0.0.0-20200119215504-eb0d8dd85bcc // indirect
 	gopkg.in/go-playground/assert.v1 v1.2.1 // indirect
 	gopkg.in/go-playground/validator.v9 v9.31.0
 	gopkg.in/russross/blackfriday.v2 v2.0.0 // indirect

go.sum

@@ -22,6 +22,8 @@ github.com/aws/aws-lambda-go v1.13.3 h1:SuCy7H3NLyp+1Mrfp+m80jcbi9KYWAs9/BXwppwR
 github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
 github.com/aws/aws-sdk-go v1.28.3 h1:FnkDp+fz4JHWUW3Ust2Wh89RpdGif077Wjis/sMrGKM=
 github.com/aws/aws-sdk-go v1.28.3/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/aws/aws-sdk-go v1.28.5 h1:yYeWPM8w5FoIj3Lo0BDZxRyDpTveKTq/qvnIEPBnev8=
+github.com/aws/aws-sdk-go v1.28.5/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
@@ -164,6 +166,8 @@ github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FW
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@@ -216,6 +220,8 @@ golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d h1:2+ZP7EfsZV7Vvmx3TIqSlSzATMkTAKqM14YGFPoSKjI=
 golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200117160349-530e935923ad h1:Jh8cai0fqIK+f6nG0UgPW5wFk8wmiMhM3AyciDBdtQg=
+golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE=
@@ -252,6 +258,10 @@ golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200116225955-84cebe10344f h1:ILR8hSRYwfkRGnA5a6c/vTjIFGkLibNIFxGiosQo+LM=
 golang.org/x/tools v0.0.0-20200116225955-84cebe10344f/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200117220505-0cba7a3a9ee9 h1:KOkk4e2xd5OeCDJGwacvr75ICCbCsShrHiqPEdsA9hg=
+golang.org/x/tools v0.0.0-20200117220505-0cba7a3a9ee9/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200119215504-eb0d8dd85bcc h1:ZA7KFRdqWZkBr0/YbHm1h08vDJ5gQdjVG/8L153z5c4=
+golang.org/x/tools v0.0.0-20200119215504-eb0d8dd85bcc/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

tools/mage/build_namespace.go

@@ -72,7 +72,7 @@ func (b Build) API() error {
 		}
 
 		args := []string{"generate", "client", "-q", "-t", path.Dir(spec), "-f", spec}
-		if err := sh.Run("swagger", args...); err != nil {
+		if err := sh.Run(path.Join(setupDirectory, "swagger"), args...); err != nil {
 			return err
 		}
 

tools/mage/fmt.go

@@ -65,7 +65,7 @@ func Fmt() error {
 	if mg.Verbose() {
 		args = append(args, "--verbose")
 	}
-	return sh.Run("venv/bin/yapf", append(args, pyTargets...)...)
+	return sh.Run(pythonLibPath("yapf"), append(args, pyTargets...)...)
 }
 
 func removeAllImportNewlines() error {

tools/mage/setup.go

@@ -30,31 +30,17 @@ import (
 )
 
 const (
-	golangciVersion = "1.21.0"
+	golangciVersion = "1.22.2"
 	swaggerVersion  = "0.21.0"
-
-	binDir = "/usr/local/bin"
 )
 
 // Setup Install development dependencies
 func Setup() error {
-	fmt.Println("setup: installing python3 venv")
-	if err := os.RemoveAll("venv"); err != nil {
-		return err
-	}
-	if err := sh.RunV("python3", "-m", "venv", "venv"); err != nil {
-		return err
-	}
-	args := []string{"install", "-r", "requirements.txt"}
-	if !mg.Verbose() {
-		args = append(args, "--quiet")
-	}
-	if err := sh.RunV("venv/bin/pip3", args...); err != nil {
+	if err := setupPythonVirtualEnv(); err != nil {
 		return err
 	}
-
 	// Some libraries are only needed for development, not for CI
-	if os.Getenv("CI") == "" {
+	if !isRunningInCI() {
 		fmt.Println("setup: installing goimports for local development")
 		if err := sh.RunV("go", "get", "golang.org/x/tools/cmd/goimports"); err != nil {
 			return err
@@ -72,11 +58,32 @@ func Setup() error {
 	return installGolangCiLint(env)
 }
 
+func setupPythonVirtualEnv() error {
+	fmt.Println("setup: installing python3 venv")
+	if err := os.RemoveAll(pythonVirtualEnvPath); err != nil {
+		return err
+	}
+	if err := sh.RunV("python3", "-m", "venv", pythonVirtualEnvPath); err != nil {
+		return err
+	}
+	args := []string{"install", "-r", "requirements.txt"}
+	if !mg.Verbose() {
+		args = append(args, "--quiet")
+	}
+	if err := sh.RunV(pythonLibPath("pip3"), args...); err != nil {
+		return err
+	}
+	return nil
+}
+
 func installSwagger(uname string) error {
 	fmt.Println("setup: installing go-swagger")
+	if err := os.MkdirAll(setupDirectory, os.ModePerm); err != nil {
+		return err
+	}
 	url := fmt.Sprintf("https://github.com/go-swagger/go-swagger/releases/download/v%s/swagger_%s_amd64",
 		swaggerVersion, strings.ToLower(uname))
-	binary := path.Join(binDir, "swagger")
+	binary := path.Join(setupDirectory, "swagger")
 	if err := sh.RunV("curl", "-o", binary, "-fL", url); err != nil {
 		return err
 	}
@@ -85,8 +92,8 @@ func installSwagger(uname string) error {
 
 func installGolangCiLint(uname string) error {
 	fmt.Println("setup: installing golangci-lint")
-	downloadDir := filepath.Join(os.TempDir(), "golangci")
-	if err := os.MkdirAll(downloadDir, 0755); err != nil {
+	downloadDir := filepath.Join(setupDirectory, "golangci")
+	if err := os.MkdirAll(downloadDir, os.ModePerm); err != nil {
 		return err
 	}
 
@@ -100,5 +107,10 @@ func installGolangCiLint(uname string) error {
 	if err := sh.RunV("tar", "-xzvf", path.Join(downloadDir, "ci.tar.gz"), "-C", downloadDir); err != nil {
 		return err
 	}
-	return sh.RunV("mv", path.Join(downloadDir, pkg, "golangci-lint"), binDir)
+	// moving golangci-lint from download folder to setupDirectory
+	if err := os.Rename(path.Join(downloadDir, pkg, "golangci-lint"), path.Join(setupDirectory, "golangci-lint")); err != nil {
+		return err
+	}
+	// deleting download folder
+	return os.RemoveAll(downloadDir)
 }

tools/mage/test_namespace.go

@@ -22,6 +22,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
 
@@ -72,7 +73,7 @@ func (t Test) Lint() error {
 	if mg.Verbose() {
 		args = append(args, "-v")
 	}
-	if err := sh.RunV("golangci-lint", args...); err != nil {
+	if err := sh.RunV(path.Join(setupDirectory, "golangci-lint"), args...); err != nil {
 		errs = append(errs, err)
 	}
 
@@ -84,7 +85,7 @@ func (t Test) Lint() error {
 	} else {
 		args = append(args, "--quiet")
 	}
-	if err := sh.Run("venv/bin/bandit", append(args, pyTargets...)...); err != nil {
+	if err := sh.Run(pythonLibPath("bandit"), append(args, pyTargets...)...); err != nil {
 		errs = append(errs, err)
 	}
 
@@ -94,7 +95,7 @@ func (t Test) Lint() error {
 	if mg.Verbose() {
 		args = append(args, "--verbose")
 	}
-	if output, err := sh.Output("venv/bin/yapf", append(args, pyTargets...)...); err != nil {
+	if output, err := sh.Output(pythonLibPath("yapf"), append(args, pyTargets...)...); err != nil {
 		errs = append(errs, fmt.Errorf("yapf diff: %d bytes (err: %v)", len(output), err))
 	}
 
@@ -105,12 +106,12 @@ func (t Test) Lint() error {
 	}
 	// pylint src
 	srcArgs := append(args, "--ignore", "tests", "--disable", strings.Join(pylintSrcDisabled, ","))
-	if err := sh.RunV("venv/bin/pylint", append(srcArgs, pyTargets...)...); err != nil {
+	if err := sh.RunV(pythonLibPath("pylint"), append(srcArgs, pyTargets...)...); err != nil {
 		errs = append(errs, err)
 	}
 	// pylint tests
 	testArgs := append(args, "--ignore", "src", "--disable", strings.Join(pylintTestsDisabled, ","))
-	if err := sh.RunV("venv/bin/pylint", append(testArgs, pyTargets...)...); err != nil {
+	if err := sh.RunV(pythonLibPath("pylint"), append(testArgs, pyTargets...)...); err != nil {
 		errs = append(errs, err)
 	}
 
@@ -120,7 +121,7 @@ func (t Test) Lint() error {
 	if mg.Verbose() {
 		args = append(args, "--verbose")
 	}
-	if err := sh.RunV("venv/bin/mypy", append(args, pyTargets...)...); err != nil {
+	if err := sh.RunV(pythonLibPath("mypy"), append(args, pyTargets...)...); err != nil {
 		errs = append(errs, err)
 	}
 
@@ -136,7 +137,7 @@ func (t Test) Lint() error {
 	if err != nil {
 		errs = append(errs, fmt.Errorf("filepath.Walk(deployments) failed: %v", err))
 	}
-	if err := sh.RunV("venv/bin/cfn-lint", templates...); err != nil {
+	if err := sh.RunV(pythonLibPath("cfn-lint"), templates...); err != nil {
 		errs = append(errs, err)
 	}
 
@@ -165,7 +166,7 @@ func (Test) Unit() error {
 
 	for _, target := range []string{"internal/core", "internal/compliance", "internal/log_analysis"} {
 		fmt.Println("test:unit python unittest", target)
-		if err := sh.RunV("venv/bin/python3", append(args, target)...); err != nil {
+		if err := sh.RunV(pythonLibPath("python3"), append(args, target)...); err != nil {
 			return err
 		}
 	}
@@ -228,7 +229,7 @@ func (t Test) Integration() error {
 	// Run Python integration tests unless a Go pkg is specified
 	if os.Getenv("PKG") == "" {
 		fmt.Println("test:integration: python engine")
-		return sh.RunV("venv/bin/python3", "internal/compliance/policy_engine/tests/integration.py")
+		return sh.RunV(pythonLibPath("python3"), "internal/compliance/policy_engine/tests/integration.py")
 	}
 	return nil
 }

tools/mage/util.go

@@ -24,6 +24,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
+	"path"
 	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -33,6 +34,11 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
+var (
+	setupDirectory       = "./.setup"
+	pythonVirtualEnvPath = path.Join(setupDirectory, "venv")
+)
+
 // Open and parse a yaml file.
 func loadYamlFile(path string, out interface{}) error {
 	contents, err := ioutil.ReadFile(path)
@@ -132,3 +138,14 @@ func download(url string) ([]byte, error) {
 
 	return ioutil.ReadAll(response.Body)
 }
+
+// isRunningInCI returns true if the mage command is running inside
+// CI environment
+func isRunningInCI() bool {
+	return os.Getenv("CI") != ""
+}
+
+// pythonLibPath the Python venv path of the given library
+func pythonLibPath(lib string) string {
+	return path.Join(pythonVirtualEnvPath, "bin", lib)
+}