Stabilize Go modules, format generated API files (#696)

go.mod

@@ -5,9 +5,8 @@ go 1.14
 require (
 	github.com/alecthomas/jsonschema v0.0.0-20200217214135-7152f22193c9
 	github.com/aws/aws-lambda-go v1.16.0
-	github.com/aws/aws-sdk-go v1.30.6
+	github.com/aws/aws-sdk-go v1.30.7
 	github.com/cenkalti/backoff v2.2.1+incompatible
-	github.com/fatih/color v1.9.0
 	github.com/go-openapi/errors v0.19.4
 	github.com/go-openapi/loads v0.19.5 // indirect
 	github.com/go-openapi/runtime v0.19.15
@@ -24,7 +23,6 @@ require (
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/leodido/go-urn v1.2.0 // indirect
 	github.com/magefile/mage v1.9.0
-	github.com/mattn/go-colorable v0.1.6 // indirect
 	github.com/mitchellh/mapstructure v1.2.2 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.1 // indirect
@@ -35,8 +33,7 @@ require (
 	go.mongodb.org/mongo-driver v1.3.2 // indirect
 	go.uber.org/zap v1.14.1
 	golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e // indirect
-	golang.org/x/sys v0.0.0-20200406155108-e3b113bbe6a4 // indirect
-	golang.org/x/tools v0.0.0-20200409210453-700752c24408 // indirect
+	golang.org/x/tools v0.0.0-20200407144507-5fc56a9a2104 // indirect
 	gopkg.in/go-playground/assert.v1 v1.2.1 // indirect
 	gopkg.in/go-playground/validator.v9 v9.31.0
 	gopkg.in/yaml.v2 v2.2.8

go.sum

@@ -16,8 +16,8 @@ github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496 h1:zV3ejI06
 github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
 github.com/aws/aws-lambda-go v1.16.0 h1:9+Pp1/6cjEXYhwadp8faFXKSOWt7/tHRCnQxQmKvVwM=
 github.com/aws/aws-lambda-go v1.16.0/go.mod h1:FEwgPLE6+8wcGBTe5cJN3JWurd1Ztm9zN4jsXsjzKKw=
-github.com/aws/aws-sdk-go v1.30.6 h1:GuWgFWWR9CF8mO9SM6N9oZt0vM0yzgPCMDDZOEQb8l4=
-github.com/aws/aws-sdk-go v1.30.6/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
+github.com/aws/aws-sdk-go v1.30.7 h1:IaXfqtioP6p9SFAnNfsqdNczbR5UNbYqvcZUSsCAdTY=
+github.com/aws/aws-sdk-go v1.30.7/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
@@ -26,8 +26,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
-github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
-github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
 github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
 github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI=
@@ -180,15 +178,6 @@ github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8
 github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
 github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
 github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
-github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
-github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
-github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE=
-github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
-github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
-github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM=
-github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
-github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
-github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.2.2 h1:dxe5oCinTXiTIcfgmZecdCzPmAJKd46KsCWc35r0TV4=
@@ -291,21 +280,14 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=
-golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200406155108-e3b113bbe6a4 h1:c1Sgqkh8v6ZxafNGG64r8C8UisIW2TKMJN8P86tKjr0=
-golang.org/x/sys v0.0.0-20200406155108-e3b113bbe6a4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -324,10 +306,8 @@ golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs=
 golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20200408132156-9ee5ef7a2c0d h1:2DXIdtvIYvvWOcAOsX81FwOUBoQoMZhosWn7KjXEl94=
-golang.org/x/tools v0.0.0-20200408132156-9ee5ef7a2c0d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200409210453-700752c24408 h1:ce4a1oU3GkqroNNqWtsLFadWIwt4+dcI+uCZZKwA1A4=
-golang.org/x/tools v0.0.0-20200409210453-700752c24408/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200407144507-5fc56a9a2104 h1:BgjF1Nn5zNEp8cxfwjYGMLT28bm1GD1Uir2/OnI1Wn4=
+golang.org/x/tools v0.0.0-20200407144507-5fc56a9a2104/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=

tools/mage/build_namespace.go

@@ -75,12 +75,24 @@ func (b Build) API() {
 		}
 		walk(client, handler)
 		walk(models, handler)
+
+		// Format generated files with our license header and import ordering.
+		// "swagger generate client" can embed the header, but it's simpler to keep the whole repo
+		// formatted the exact same way.
+		fmtLicense(client, models)
+		if err := gofmt(client, models); err != nil {
+			logger.Warnf("gofmt %s %s failed: %v", client, models, err)
+		}
 	}
 
 	logger.Info("build:api: generating web typescript from graphql")
 	if err := sh.Run("npm", "run", "graphql-codegen"); err != nil {
 		logger.Fatalf("graphql generation failed: %v", err)
 	}
+	fmtLicense("web/__generated__")
+	if err := prettier("web/__generated__/*"); err != nil {
+		logger.Warnf("prettier web/__generated__/ failed: %v", err)
+	}
 }
 
 // Lambda Compile Go Lambda function source
@@ -100,8 +112,23 @@ func (b Build) lambda() error {
 
 	logger.Infof("build:lambda: compiling %d Go Lambda functions (internal/.../main) using %s",
 		len(packages), runtime.Version())
+
+	// "go build" in parallel for each Lambda function.
+	//
+	// If you don't already have all go modules downloaded, this may fail because each goroutine will
+	// automatically modify the go.mod/go.sum files which will cause conflicts with itself.
+	//
+	// Run "go mod download" or "mage setup" before building to download the go modules.
+	// If you're adding a new module, run "go get ./..." before building to fetch the new module.
+	errs := make(chan error)
 	for _, pkg := range packages {
-		if err := buildPackage(pkg); err != nil {
+		go func(pkg string, errs chan error) {
+			errs <- buildPackage(pkg)
+		}(pkg, errs)
+	}
+
+	for range packages {
+		if err := <-errs; err != nil {
 			return err
 		}
 	}

tools/mage/deploy.go

@@ -35,7 +35,6 @@ import (
 	"github.com/aws/aws-sdk-go/service/cognitoidentityprovider"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/sts"
-	"github.com/fatih/color"
 	"github.com/magefile/mage/sh"
 
 	"github.com/panther-labs/panther/api/gateway/analysis/client"
@@ -147,7 +146,7 @@ func Deploy() {
 	}
 
 	logger.Infof("deploy: finished successfully in %s", time.Since(start))
-	color.Yellow("\nPanther URL = https://%s\n", outputs["LoadBalancerUrl"])
+	logger.Infof("***** Panther URL = https://%s", outputs["LoadBalancerUrl"])
 }
 
 // Fail the deploy early if there is a known issue with the user's environment.

tools/mage/fmt.go

@@ -41,7 +41,7 @@ var (
 // Fmt Format source files
 func Fmt() {
 	// Add license headers first (don't run in parallel with other formatters)
-	fmtLicense()
+	fmtLicenseAll()
 
 	results := make(chan goroutineResult)
 	count := 0
@@ -51,14 +51,19 @@ func Fmt() {
 		c <- goroutineResult{"fmt: gofmt", gofmt(goTargets...)}
 	}(results)
 
+	count++
+	go func(c chan goroutineResult) {
+		c <- goroutineResult{"fmt: go mod tidy", goModTidy()}
+	}(results)
+
 	count++
 	go func(c chan goroutineResult) {
 		c <- goroutineResult{"fmt: yapf", yapf(pyTargets...)}
 	}(results)
 
 	count++
 	go func(c chan goroutineResult) {
-		c <- goroutineResult{"fmt: prettier", prettier()}
+		c <- goroutineResult{"fmt: prettier", prettier("")}
 	}(results)
 
 	count++
@@ -107,6 +112,7 @@ func gofmt(paths ...string) error {
 	if err := sh.Run("goimports", args...); err != nil {
 		return fmt.Errorf("goimports failed: %v", err)
 	}
+
 	return nil
 }
 
@@ -132,6 +138,11 @@ func removeImportNewlines(path string) error {
 	return ioutil.WriteFile(path, bytes.Join(newLines, []byte("\n")), 0644)
 }
 
+// Tidy go.mod/go.sum
+func goModTidy() error {
+	return sh.Run("go", "mod", "tidy")
+}
+
 // Apply Python formatting to the given paths
 func yapf(paths ...string) error {
 	logger.Debug("fmt: python yapf " + strings.Join(paths, " "))
@@ -143,10 +154,12 @@ func yapf(paths ...string) error {
 }
 
 // Apply prettier formatting to web, markdown, and yml files
-func prettier() error {
-	files := "**/*.{ts,js,tsx,md,json,yaml,yml}"
-	logger.Debug("fmt: prettier " + files)
-	args := []string{"--write", files}
+func prettier(pathPattern string) error {
+	if pathPattern == "" {
+		pathPattern = "**/*.{ts,js,tsx,md,json,yaml,yml}"
+	}
+	logger.Debug("fmt: prettier " + pathPattern)
+	args := []string{"--write", pathPattern}
 	if !mg.Verbose() {
 		args = append(args, "--loglevel", "error")
 	}

tools/mage/fmt_license.go

@@ -44,14 +44,18 @@ func commentEachLine(prefix, text string) string {
 }
 
 // Add the license header to all applicable source files
-func fmtLicense() {
-	logger.Debugf("fmt: license header %s for %s", agplSource, strings.Join(licensePaths, " "))
+func fmtLicenseAll() {
+	fmtLicense(licensePaths...)
+}
+
+func fmtLicense(paths ...string) {
+	logger.Debugf("fmt: license header %s for %s", agplSource, strings.Join(paths, " "))
 	header := strings.TrimSpace(string(readFile(agplSource)))
 
 	asteriskLicense := "/**\n" + commentEachLine(" *", header) + "\n */"
 	hashtagLicense := commentEachLine("#", header)
 
-	for _, root := range licensePaths {
+	for _, root := range paths {
 		walk(root, func(path string, info os.FileInfo) {
 			if !info.IsDir() {
 				addFileLicense(path, asteriskLicense, hashtagLicense)

tools/mage/setup.go

@@ -29,6 +29,9 @@ import (
 )
 
 const (
+	// Use the commit from the latest tagged release of https://github.com/golang/tools/releases
+	goimportsVersion = "5fc56a9" // gopls/v0.4.0
+
 	golangciVersion  = "1.23.6"
 	swaggerVersion   = "0.23.0"
 	terraformVersion = "0.12.24"
@@ -55,7 +58,7 @@ func Setup() {
 
 	count++
 	go func(c chan goroutineResult) {
-		c <- goroutineResult{"go get modules", installGoModules()}
+		c <- goroutineResult{"download go modules", installGoModules()}
 	}(results)
 
 	count++
@@ -91,10 +94,14 @@ func Setup() {
 // "go test" and "go build" will do this automatically, but putting it in the setup flow allows it
 // to happen in parallel with the rest of the downloads.
 func installGoModules() error {
-	logger.Info("setup: go get modules...")
+	logger.Info("setup: download go modules...")
+
+	if err := sh.Run("go", "mod", "download"); err != nil {
+		return err
+	}
 
-	// goimports is needed for formatting but won't be listed as an explicit dependency
-	return sh.Run("go", "get", "-t", "golang.org/x/tools/cmd/goimports", "./...")
+	// goimports is needed for formatting but isn't importable (won't be in go.mod)
+	return sh.Run("go", "get", "golang.org/x/tools/cmd/goimports@"+goimportsVersion)
 }
 
 // Download go-swagger if it hasn't been already