From 002ac177e11e9bcd05f8e1cdf3312c3646485026 Mon Sep 17 00:00:00 2001 From: Austin Byers Date: Mon, 13 Apr 2020 14:16:34 -0700 Subject: [PATCH] Stabilize Go modules, format generated API files (#696) --- go.mod | 7 ++----- go.sum | 28 ++++------------------------ tools/mage/build_namespace.go | 29 ++++++++++++++++++++++++++++- tools/mage/deploy.go | 3 +-- tools/mage/fmt.go | 25 +++++++++++++++++++------ tools/mage/fmt_license.go | 10 +++++++--- tools/mage/setup.go | 15 +++++++++++---- 7 files changed, 72 insertions(+), 45 deletions(-) diff --git a/go.mod b/go.mod index a90d9a9c59..9775437677 100644 --- a/go.mod +++ b/go.mod @@ -5,9 +5,8 @@ go 1.14 require ( github.com/alecthomas/jsonschema v0.0.0-20200217214135-7152f22193c9 github.com/aws/aws-lambda-go v1.16.0 - github.com/aws/aws-sdk-go v1.30.6 + github.com/aws/aws-sdk-go v1.30.7 github.com/cenkalti/backoff v2.2.1+incompatible - github.com/fatih/color v1.9.0 github.com/go-openapi/errors v0.19.4 github.com/go-openapi/loads v0.19.5 // indirect github.com/go-openapi/runtime v0.19.15 @@ -24,7 +23,6 @@ require ( github.com/kelseyhightower/envconfig v1.4.0 github.com/leodido/go-urn v1.2.0 // indirect github.com/magefile/mage v1.9.0 - github.com/mattn/go-colorable v0.1.6 // indirect github.com/mitchellh/mapstructure v1.2.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 // indirect @@ -35,8 +33,7 @@ require ( go.mongodb.org/mongo-driver v1.3.2 // indirect go.uber.org/zap v1.14.1 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e // indirect - golang.org/x/sys v0.0.0-20200406155108-e3b113bbe6a4 // indirect - golang.org/x/tools v0.0.0-20200409210453-700752c24408 // indirect + golang.org/x/tools v0.0.0-20200407144507-5fc56a9a2104 // indirect gopkg.in/go-playground/assert.v1 v1.2.1 // indirect gopkg.in/go-playground/validator.v9 v9.31.0 gopkg.in/yaml.v2 v2.2.8 diff --git a/go.sum b/go.sum index cb7dccdd70..353eb2ae10 100644 --- a/go.sum +++ b/go.sum @@ -16,8 +16,8 @@ github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496 h1:zV3ejI06 github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= github.com/aws/aws-lambda-go v1.16.0 h1:9+Pp1/6cjEXYhwadp8faFXKSOWt7/tHRCnQxQmKvVwM= github.com/aws/aws-lambda-go v1.16.0/go.mod h1:FEwgPLE6+8wcGBTe5cJN3JWurd1Ztm9zN4jsXsjzKKw= -github.com/aws/aws-sdk-go v1.30.6 h1:GuWgFWWR9CF8mO9SM6N9oZt0vM0yzgPCMDDZOEQb8l4= -github.com/aws/aws-sdk-go v1.30.6/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.30.7 h1:IaXfqtioP6p9SFAnNfsqdNczbR5UNbYqvcZUSsCAdTY= +github.com/aws/aws-sdk-go v1.30.7/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -26,8 +26,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= -github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= @@ -180,15 +178,6 @@ github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8 github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= -github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE= -github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM= -github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= -github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.2.2 h1:dxe5oCinTXiTIcfgmZecdCzPmAJKd46KsCWc35r0TV4= @@ -291,7 +280,6 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -299,13 +287,7 @@ golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200406155108-e3b113bbe6a4 h1:c1Sgqkh8v6ZxafNGG64r8C8UisIW2TKMJN8P86tKjr0= -golang.org/x/sys v0.0.0-20200406155108-e3b113bbe6a4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -324,10 +306,8 @@ golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200408132156-9ee5ef7a2c0d h1:2DXIdtvIYvvWOcAOsX81FwOUBoQoMZhosWn7KjXEl94= -golang.org/x/tools v0.0.0-20200408132156-9ee5ef7a2c0d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200409210453-700752c24408 h1:ce4a1oU3GkqroNNqWtsLFadWIwt4+dcI+uCZZKwA1A4= -golang.org/x/tools v0.0.0-20200409210453-700752c24408/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200407144507-5fc56a9a2104 h1:BgjF1Nn5zNEp8cxfwjYGMLT28bm1GD1Uir2/OnI1Wn4= +golang.org/x/tools v0.0.0-20200407144507-5fc56a9a2104/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= diff --git a/tools/mage/build_namespace.go b/tools/mage/build_namespace.go index 129814b61e..9139ffd912 100644 --- a/tools/mage/build_namespace.go +++ b/tools/mage/build_namespace.go @@ -75,12 +75,24 @@ func (b Build) API() { } walk(client, handler) walk(models, handler) + + // Format generated files with our license header and import ordering. + // "swagger generate client" can embed the header, but it's simpler to keep the whole repo + // formatted the exact same way. + fmtLicense(client, models) + if err := gofmt(client, models); err != nil { + logger.Warnf("gofmt %s %s failed: %v", client, models, err) + } } logger.Info("build:api: generating web typescript from graphql") if err := sh.Run("npm", "run", "graphql-codegen"); err != nil { logger.Fatalf("graphql generation failed: %v", err) } + fmtLicense("web/__generated__") + if err := prettier("web/__generated__/*"); err != nil { + logger.Warnf("prettier web/__generated__/ failed: %v", err) + } } // Lambda Compile Go Lambda function source @@ -100,8 +112,23 @@ func (b Build) lambda() error { logger.Infof("build:lambda: compiling %d Go Lambda functions (internal/.../main) using %s", len(packages), runtime.Version()) + + // "go build" in parallel for each Lambda function. + // + // If you don't already have all go modules downloaded, this may fail because each goroutine will + // automatically modify the go.mod/go.sum files which will cause conflicts with itself. + // + // Run "go mod download" or "mage setup" before building to download the go modules. + // If you're adding a new module, run "go get ./..." before building to fetch the new module. + errs := make(chan error) for _, pkg := range packages { - if err := buildPackage(pkg); err != nil { + go func(pkg string, errs chan error) { + errs <- buildPackage(pkg) + }(pkg, errs) + } + + for range packages { + if err := <-errs; err != nil { return err } } diff --git a/tools/mage/deploy.go b/tools/mage/deploy.go index 235ddd5bb1..1c75d76521 100644 --- a/tools/mage/deploy.go +++ b/tools/mage/deploy.go @@ -35,7 +35,6 @@ import ( "github.com/aws/aws-sdk-go/service/cognitoidentityprovider" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/sts" - "github.com/fatih/color" "github.com/magefile/mage/sh" "github.com/panther-labs/panther/api/gateway/analysis/client" @@ -147,7 +146,7 @@ func Deploy() { } logger.Infof("deploy: finished successfully in %s", time.Since(start)) - color.Yellow("\nPanther URL = https://%s\n", outputs["LoadBalancerUrl"]) + logger.Infof("***** Panther URL = https://%s", outputs["LoadBalancerUrl"]) } // Fail the deploy early if there is a known issue with the user's environment. diff --git a/tools/mage/fmt.go b/tools/mage/fmt.go index 509b4ce78e..bd0e9e9590 100644 --- a/tools/mage/fmt.go +++ b/tools/mage/fmt.go @@ -41,7 +41,7 @@ var ( // Fmt Format source files func Fmt() { // Add license headers first (don't run in parallel with other formatters) - fmtLicense() + fmtLicenseAll() results := make(chan goroutineResult) count := 0 @@ -51,6 +51,11 @@ func Fmt() { c <- goroutineResult{"fmt: gofmt", gofmt(goTargets...)} }(results) + count++ + go func(c chan goroutineResult) { + c <- goroutineResult{"fmt: go mod tidy", goModTidy()} + }(results) + count++ go func(c chan goroutineResult) { c <- goroutineResult{"fmt: yapf", yapf(pyTargets...)} @@ -58,7 +63,7 @@ func Fmt() { count++ go func(c chan goroutineResult) { - c <- goroutineResult{"fmt: prettier", prettier()} + c <- goroutineResult{"fmt: prettier", prettier("")} }(results) count++ @@ -107,6 +112,7 @@ func gofmt(paths ...string) error { if err := sh.Run("goimports", args...); err != nil { return fmt.Errorf("goimports failed: %v", err) } + return nil } @@ -132,6 +138,11 @@ func removeImportNewlines(path string) error { return ioutil.WriteFile(path, bytes.Join(newLines, []byte("\n")), 0644) } +// Tidy go.mod/go.sum +func goModTidy() error { + return sh.Run("go", "mod", "tidy") +} + // Apply Python formatting to the given paths func yapf(paths ...string) error { logger.Debug("fmt: python yapf " + strings.Join(paths, " ")) @@ -143,10 +154,12 @@ func yapf(paths ...string) error { } // Apply prettier formatting to web, markdown, and yml files -func prettier() error { - files := "**/*.{ts,js,tsx,md,json,yaml,yml}" - logger.Debug("fmt: prettier " + files) - args := []string{"--write", files} +func prettier(pathPattern string) error { + if pathPattern == "" { + pathPattern = "**/*.{ts,js,tsx,md,json,yaml,yml}" + } + logger.Debug("fmt: prettier " + pathPattern) + args := []string{"--write", pathPattern} if !mg.Verbose() { args = append(args, "--loglevel", "error") } diff --git a/tools/mage/fmt_license.go b/tools/mage/fmt_license.go index 922fd0549c..62aeb430c4 100644 --- a/tools/mage/fmt_license.go +++ b/tools/mage/fmt_license.go @@ -44,14 +44,18 @@ func commentEachLine(prefix, text string) string { } // Add the license header to all applicable source files -func fmtLicense() { - logger.Debugf("fmt: license header %s for %s", agplSource, strings.Join(licensePaths, " ")) +func fmtLicenseAll() { + fmtLicense(licensePaths...) +} + +func fmtLicense(paths ...string) { + logger.Debugf("fmt: license header %s for %s", agplSource, strings.Join(paths, " ")) header := strings.TrimSpace(string(readFile(agplSource))) asteriskLicense := "/**\n" + commentEachLine(" *", header) + "\n */" hashtagLicense := commentEachLine("#", header) - for _, root := range licensePaths { + for _, root := range paths { walk(root, func(path string, info os.FileInfo) { if !info.IsDir() { addFileLicense(path, asteriskLicense, hashtagLicense) diff --git a/tools/mage/setup.go b/tools/mage/setup.go index 91689e259e..69befd26a9 100644 --- a/tools/mage/setup.go +++ b/tools/mage/setup.go @@ -29,6 +29,9 @@ import ( ) const ( + // Use the commit from the latest tagged release of https://github.com/golang/tools/releases + goimportsVersion = "5fc56a9" // gopls/v0.4.0 + golangciVersion = "1.23.6" swaggerVersion = "0.23.0" terraformVersion = "0.12.24" @@ -55,7 +58,7 @@ func Setup() { count++ go func(c chan goroutineResult) { - c <- goroutineResult{"go get modules", installGoModules()} + c <- goroutineResult{"download go modules", installGoModules()} }(results) count++ @@ -91,10 +94,14 @@ func Setup() { // "go test" and "go build" will do this automatically, but putting it in the setup flow allows it // to happen in parallel with the rest of the downloads. func installGoModules() error { - logger.Info("setup: go get modules...") + logger.Info("setup: download go modules...") + + if err := sh.Run("go", "mod", "download"); err != nil { + return err + } - // goimports is needed for formatting but won't be listed as an explicit dependency - return sh.Run("go", "get", "-t", "golang.org/x/tools/cmd/goimports", "./...") + // goimports is needed for formatting but isn't importable (won't be in go.mod) + return sh.Run("go", "get", "golang.org/x/tools/cmd/goimports@"+goimportsVersion) } // Download go-swagger if it hasn't been already