diff --git a/.cloudbuild/ci/integration-tests.yaml b/.cloudbuild/ci/integration-tests.yaml index 27f9c8950dc55..2e3437df8c801 100644 --- a/.cloudbuild/ci/integration-tests.yaml +++ b/.cloudbuild/ci/integration-tests.yaml @@ -2,15 +2,15 @@ timeout: 25m options: machineType: E2_HIGHCPU_32 + + # This build needs to run in environments where the _GITHUB_DEPLOY_KEY_SRC + # substitution is defined, but also environments where it isn't. The + # ALLOW_LOOSE option disables GCBs strict checking of substitution usage, + # so that the build will still run if _GITHUB_DEPLOY_KEY_SRC is not defined. + substitution_option: ALLOW_LOOSE steps: - # GCB does a shallow checkout for a build, but if we want to check our changes - # against other branches we'll need to fetch the repo history. - - name: gcr.io/cloud-builders/git - id: fetch-history - args: ['fetch', '--unshallow'] - - # Run the integration tests. Actual content of this job depends on the changes + # Run the integration tests. Actual content of this job depends on the changes # detected in the PR - name: quay.io/gravitational/teleport-buildbox:teleport9 id: run-tests @@ -19,9 +19,10 @@ steps: args: - -c - | - go run ./cmd/integration-tests \ - -target "$_BASE_BRANCH" \ - -bucket test-logs \ - -build "$BUILD_ID" \ - -a "test-logs/*.json" + go run ./cmd/integration-tests \ + -target "$_BASE_BRANCH" \ + -bucket test-logs \ + -build "$BUILD_ID" \ + -key-secret "$_GITHUB_DEPLOY_KEY_SRC" \ + -a "test-logs/*.json" timeout: 25m diff --git a/.cloudbuild/ci/unit-tests.yaml b/.cloudbuild/ci/unit-tests.yaml index b12ab966f716e..74960d38e430a 100644 --- a/.cloudbuild/ci/unit-tests.yaml +++ b/.cloudbuild/ci/unit-tests.yaml @@ -1,17 +1,16 @@ timeout: 25m options: - machineType: E2_HIGHCPU_32 - + machineType: 'E2_HIGHCPU_32' + + # This build needs to run in environments where the _GITHUB_DEPLOY_KEY_SRC + # substitution is defined, but also environments where it isn't. The + # ALLOW_LOOSE option disables GCBs strict checking of substitution usage, + # so that the build will still run if _GITHUB_DEPLOY_KEY_SRC is not defined. + substitution_option: ALLOW_LOOSE + steps: - # GCB does a shallow checkout for a build, but if we want to check our changes - # against other branches we'll need to fetch the repo history. This takes less - # than 30s at the time of writing, so it is probably not worth tweaking. - - name: gcr.io/cloud-builders/git - id: fetch-history - args: ['fetch', '--unshallow'] - - # Run the unit tests. Actual content of this job depends on the changes + # Run the unit tests. Actual content of this job depends on the changes # detected in the PR - name: quay.io/gravitational/teleport-buildbox:teleport9 id: run-tests @@ -20,9 +19,10 @@ steps: args: - -c - | - go run ./cmd/unit-tests \ - -target "$_BASE_BRANCH" \ - -bucket test-logs \ - -build "$BUILD_ID" \ + go run ./cmd/unit-tests \ + -target "$_BASE_BRANCH" \ + -bucket test-logs \ + -build "$BUILD_ID" \ + -key-secret "$_GITHUB_DEPLOY_KEY_SRC" \ -a "test-logs/*.json" - timeout: 20m + timeout: 25m diff --git a/.cloudbuild/scripts/cmd/integration-tests/args.go b/.cloudbuild/scripts/cmd/integration-tests/args.go index 2b93984168b6a..229c3aedcdc25 100644 --- a/.cloudbuild/scripts/cmd/integration-tests/args.go +++ b/.cloudbuild/scripts/cmd/integration-tests/args.go @@ -33,6 +33,7 @@ type commandlineArgs struct { buildID string artifactSearchPatterns customflag.StringArray bucket string + githubKeySrc string } // validate ensures the suplied arguments are valid & internally consistent. @@ -83,6 +84,7 @@ func parseCommandLine() (*commandlineArgs, error) { flag.StringVar(&args.buildID, "build", "", "The build ID") flag.StringVar(&args.bucket, "bucket", "", "The artifact storage bucket.") flag.Var(&args.artifactSearchPatterns, "a", "Path to artifacts. May be shell-globbed, and have multiple entries.") + flag.StringVar(&args.githubKeySrc, "key-secret", "", "Location of github deploy token, as a Google Cloud Secret") flag.Parse() diff --git a/.cloudbuild/scripts/cmd/integration-tests/main.go b/.cloudbuild/scripts/cmd/integration-tests/main.go index f6de022e00fcc..18680710080f7 100644 --- a/.cloudbuild/scripts/cmd/integration-tests/main.go +++ b/.cloudbuild/scripts/cmd/integration-tests/main.go @@ -29,6 +29,8 @@ import ( "github.com/gravitational/teleport/.cloudbuild/scripts/internal/artifacts" "github.com/gravitational/teleport/.cloudbuild/scripts/internal/changes" "github.com/gravitational/teleport/.cloudbuild/scripts/internal/etcd" + "github.com/gravitational/teleport/.cloudbuild/scripts/internal/git" + "github.com/gravitational/teleport/.cloudbuild/scripts/internal/secrets" "github.com/gravitational/trace" log "github.com/sirupsen/logrus" ) @@ -55,6 +57,24 @@ func innerMain() error { return trace.Wrap(err) } + // If a github deploy key location was supplied... + var deployKey []byte + if args.githubKeySrc != "" { + // fetch the deployment key from the GCB secret manager + log.Infof("Fetching deploy key from %s", args.githubKeySrc) + deployKey, err = secrets.Fetch(context.Background(), args.githubKeySrc) + if err != nil { + return trace.Wrap(err, "failed fetching deploy key") + } + } + + unshallowCtx, unshallowCancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer unshallowCancel() + err = git.UnshallowRepository(unshallowCtx, args.workspace, deployKey) + if err != nil { + return trace.Wrap(err, "unshallow failed") + } + moduleCacheDir := filepath.Join(os.TempDir(), gomodcacheDir) gomodcache := fmt.Sprintf("GOMODCACHE=%s", moduleCacheDir) diff --git a/.cloudbuild/scripts/cmd/unit-tests/main.go b/.cloudbuild/scripts/cmd/unit-tests/main.go index b1893ae248591..4b69c7f1df357 100644 --- a/.cloudbuild/scripts/cmd/unit-tests/main.go +++ b/.cloudbuild/scripts/cmd/unit-tests/main.go @@ -29,6 +29,8 @@ import ( "github.com/gravitational/teleport/.cloudbuild/scripts/internal/changes" "github.com/gravitational/teleport/.cloudbuild/scripts/internal/customflag" "github.com/gravitational/teleport/.cloudbuild/scripts/internal/etcd" + "github.com/gravitational/teleport/.cloudbuild/scripts/internal/git" + "github.com/gravitational/teleport/.cloudbuild/scripts/internal/secrets" "github.com/gravitational/trace" log "github.com/sirupsen/logrus" ) @@ -48,6 +50,7 @@ type commandlineArgs struct { buildID string artifactSearchPatterns customflag.StringArray bucket string + githubKeySrc string } func parseCommandLine() (commandlineArgs, error) { @@ -59,6 +62,7 @@ func parseCommandLine() (commandlineArgs, error) { flag.StringVar(&args.buildID, "build", "", "The build ID") flag.StringVar(&args.bucket, "bucket", "", "The artifact storage bucket.") flag.Var(&args.artifactSearchPatterns, "a", "Path to artifacts. May be globbed, and have multiple entries.") + flag.StringVar(&args.githubKeySrc, "key-secret", "", "Location of github deploy token, as a Google Cloud Secret") flag.Parse() @@ -106,6 +110,24 @@ func run() error { return trace.Wrap(err) } + // If a github deploy key location was supplied... + var deployKey []byte + if args.githubKeySrc != "" { + // fetch the deployment key from the GCB secret manager + log.Infof("Fetching deploy key from %s", args.githubKeySrc) + deployKey, err = secrets.Fetch(context.Background(), args.githubKeySrc) + if err != nil { + return trace.Wrap(err, "failed fetching deploy key") + } + } + + unshallowCtx, unshallowCancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer unshallowCancel() + err = git.UnshallowRepository(unshallowCtx, args.workspace, deployKey) + if err != nil { + return trace.Wrap(err, "unshallow failed") + } + log.Println("Analysing code changes") ch, err := changes.Analyze(args.workspace, args.targetBranch, args.commitSHA) if err != nil { diff --git a/.cloudbuild/scripts/go.mod b/.cloudbuild/scripts/go.mod index 300c5bb2c445a..49cf54361d31c 100644 --- a/.cloudbuild/scripts/go.mod +++ b/.cloudbuild/scripts/go.mod @@ -3,11 +3,15 @@ module github.com/gravitational/teleport/.cloudbuild/scripts go 1.17 require ( + cloud.google.com/go/secretmanager v1.2.0 cloud.google.com/go/storage v1.19.0 github.com/go-git/go-git/v5 v5.4.2 github.com/gravitational/trace v1.1.15 github.com/hashicorp/go-multierror v1.1.1 github.com/sirupsen/logrus v1.8.1 + github.com/stretchr/testify v1.7.0 + golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b + google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c ) require ( @@ -33,21 +37,18 @@ require ( github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/sergi/go-diff v1.1.0 // indirect - github.com/stretchr/objx v0.1.1 // indirect - github.com/stretchr/testify v1.7.0 // indirect + github.com/stretchr/objx v0.2.0 // indirect github.com/xanzy/ssh-agent v0.3.0 // indirect go.opencensus.io v0.23.0 // indirect - golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b // indirect golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420 // indirect golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect - golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect + golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27 // indirect golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect golang.org/x/text v0.3.6 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect - google.golang.org/api v0.65.0 // indirect + google.golang.org/api v0.67.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20220118154757-00ab72f36ad5 // indirect - google.golang.org/grpc v1.40.1 // indirect + google.golang.org/grpc v1.44.0 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect diff --git a/.cloudbuild/scripts/go.sum b/.cloudbuild/scripts/go.sum index a9f64ef4f6699..d2b25fdba3cd4 100644 --- a/.cloudbuild/scripts/go.sum +++ b/.cloudbuild/scripts/go.sum @@ -39,12 +39,15 @@ cloud.google.com/go/compute v0.1.0 h1:rSUBvAyVwNJ5uQCKNJFMwPtTvJkfN38b6Pvb9zZoqJ cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/iam v0.1.0/go.mod h1:vcUNEa0pEm0qRVpmWepWaFMIAI8/hjB9mO8rNCJtF6c= cloud.google.com/go/iam v0.1.1 h1:4CapQyNFjiksks1/x7jsvsygFPhihslYk5GptIrlX68= cloud.google.com/go/iam v0.1.1/go.mod h1:CKqrcnI/suGpybEHxZ7BMehL0oA4LpdyJdUlTl9jVMw= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/secretmanager v1.2.0 h1:VR6MzO4wjTj5jQKTPpsZhCF2PqqdAAZmN54BwJbQPhs= +cloud.google.com/go/secretmanager v1.2.0/go.mod h1:HNMYTaLrMrAN37vi2mM2vvFgjgaoCE1qvtccCIJwFRc= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= @@ -70,6 +73,7 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -77,7 +81,11 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -91,6 +99,7 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -235,9 +244,11 @@ github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -420,8 +431,9 @@ golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27 h1:XDXtA5hveEEV8JB2l7nhMTp3t3cHp9ZpwcdjqyEWLlo= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -524,8 +536,9 @@ google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdr google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= google.golang.org/api v0.64.0/go.mod h1:931CdxA8Rm4t6zqTFGSsgwbAEZ2+GMYurbndwSimebM= -google.golang.org/api v0.65.0 h1:MTW9c+LIBAbwoS1Gb+YV7NjFBt2f7GtAS5hIzh2NjgQ= google.golang.org/api v0.65.0/go.mod h1:ArYhxgGadlWmqO1IqVujw6Cs8IdD33bTmzKo2Sh+cbg= +google.golang.org/api v0.67.0 h1:lYaaLa+x3VVUhtosaK9xihwQ9H9KRa557REHwwZ2orM= +google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -597,8 +610,11 @@ google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ6 google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220111164026-67b88f271998/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220118154757-00ab72f36ad5 h1:zzNejm+EgrbLfDZ6lu9Uud2IVvHySPl8vQzf04laR5Q= google.golang.org/genproto v0.0.0-20220118154757-00ab72f36ad5/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c h1:TU4rFa5APdKTq0s6B7WTsH6Xmx0Knj86s6Biz56mErE= +google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -624,8 +640,9 @@ google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.40.1 h1:pnP7OclFFFgFi4VHQDQDaoXUVauOFyktqTsqqgzFKbc= google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.44.0 h1:weqSxi/TMs1SqFRMHCtBgXRs8k3X39QIDEZ0pRcttUg= +google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= diff --git a/.cloudbuild/scripts/internal/git/configure.go b/.cloudbuild/scripts/internal/git/configure.go new file mode 100644 index 0000000000000..63d2fdb54e9b2 --- /dev/null +++ b/.cloudbuild/scripts/internal/git/configure.go @@ -0,0 +1,145 @@ +package git + +import ( + "context" + "fmt" + "os" + "os/exec" + + "github.com/gravitational/teleport/.cloudbuild/scripts/internal/github" + "github.com/gravitational/trace" + log "github.com/sirupsen/logrus" + "golang.org/x/crypto/ssh" +) + +// Config represents a git repository that has been configured to use a +// deployment key, and acts as a handle to the resources so that we can +// clean them up when we're done. +type Config struct { + identity string + knownHosts string + repoDir string +} + +// Configure alters the configuration of the git repository in `repoDir` +// so that we can access it from build. If `deployKey` is non-nil the repo +// will be configured to use that. If no deploy key is supplied, the repository +// config is untouched. +func Configure(ctx context.Context, repoDir string, deployKey []byte) (cfg *Config, err error) { + var identity string + var hostsFile string + + // The deploy key we're using is too sensitive to just hope that every + // exit path will clean it up on failure, so let's just register a + // cleanup function now just in case. + defer func() { + if err != nil { + cleanup(identity, hostsFile) + } + }() + + // If we have been supplied with deployment key, then we need to use that + // to access the repository. This implies that need to use ssh to read from + // the remote, so we will have to deal with known_hosts, etc. + if deployKey != nil { + githubHostKeys, err := getGithubHostKeys(ctx) + if err != nil { + return nil, trace.Wrap(err, "failed configuring known hosts") + } + + // configure SSH known_hosts for github.com + hostsFile, err = configureKnownHosts("github.com", githubHostKeys) + if err != nil { + return nil, trace.Wrap(err, "failed configuring known hosts") + } + + // GCB clones via https, and our deploy key won't work with that, so + // force git to access the remote over ssh with a rewrite rule + log.Info("Adding remote url rewrite rule") + err = git(ctx, repoDir, "config", `url.git@github.com:.insteadOf`, "https://github.com/") + if err != nil { + return nil, trace.Wrap(err, "failed configuring url rewrite rule") + } + + // set up the identity for the deploy key + identity, err = writeKey(deployKey) + if err != nil { + return nil, trace.Wrap(err, "failed configuring SSH identity") + } + + // finally, force git to + // a) use our custom SSH setup when accessing the remote, and + // b) fail if the github host tries to present a host key other than + // one we got from the metadata service + log.Infof("Configuring git ssh command") + err = git(ctx, repoDir, "config", "core.sshCommand", + fmt.Sprintf("ssh -i %s -o StrictHostKeyChecking=yes -o UserKnownHostsFile=%s", identity, hostsFile)) + if err != nil { + return nil, trace.Wrap(err, "failed configuring git to use deploy key") + } + } + + return &Config{identity: identity, repoDir: repoDir, knownHosts: hostsFile}, nil +} + +// Do runs `git args...` in the configured repository +func (cfg *Config) Do(ctx context.Context, args ...string) error { + return git(ctx, cfg.repoDir, args...) +} + +// Close cleans up the repository, including deleting the deployment key (if any) +func (cfg *Config) Close() error { + cleanup(cfg.identity, cfg.knownHosts) + return nil +} + +func run(ctx context.Context, workingDir string, env []string, cmd string, args ...string) error { + p := exec.CommandContext(ctx, cmd, args...) + if len(env) != 0 { + p.Env = append(os.Environ(), env...) + } + p.Dir = workingDir + + cmdLogger := log.WithField("cmd", cmd) + p.Stdout = cmdLogger.WriterLevel(log.InfoLevel) + p.Stderr = cmdLogger.WriterLevel(log.ErrorLevel) + return p.Run() +} + +func git(ctx context.Context, repoDir string, args ...string) error { + return run(ctx, repoDir, nil, "/usr/bin/git", args...) +} + +func cleanup(deployKey, knownHosts string) { + if knownHosts != "" { + log.Infof("Removing known_hosts file %s", knownHosts) + if err := os.Remove(knownHosts); err != nil { + log.WithError(err).Error("Failed cleaning up known_hosts key") + } + } + + if deployKey != "" { + log.Infof("Removing deploy key file %s", deployKey) + if err := os.Remove(deployKey); err != nil { + log.WithError(err).Error("Failed cleaning up deploy key") + } + } +} + +// getGithubHostKeys fetches the github host keys from the github metadata +// service. The metadata is fetched over HTTPS, and so we have built-on +// protection against MitM attacks while fetching the expected host keys. +func getGithubHostKeys(ctx context.Context) ([]ssh.PublicKey, error) { + metadata, err := github.GetMetadata(ctx) + if err != nil { + return nil, trace.Wrap(err, "failed fetching github metadata") + } + + // extract the host keys + githubHostKeys, err := metadata.HostKeys() + if err != nil { + return nil, trace.Wrap(err, "failed fetching github hostKeys") + } + + return githubHostKeys, nil +} diff --git a/.cloudbuild/scripts/internal/git/deploykey.go b/.cloudbuild/scripts/internal/git/deploykey.go new file mode 100644 index 0000000000000..fc57eb2798ef7 --- /dev/null +++ b/.cloudbuild/scripts/internal/git/deploykey.go @@ -0,0 +1,34 @@ +package git + +import ( + "os" + + "github.com/gravitational/trace" + log "github.com/sirupsen/logrus" +) + +func writeKey(deployKey []byte) (string, error) { + // Note that tempfiles are automatically created with 0600, so no-one else + // should be able to read this. + keyFile, err := os.CreateTemp("", "*") + if err != nil { + return "", trace.Wrap(err, "failed creating keyfile") + } + defer keyFile.Close() + + log.Infof("Writing deploy key to %s", keyFile.Name()) + _, err = keyFile.Write(deployKey) + if err != nil { + return "", trace.Wrap(err, "failed writing deploy key") + } + + // ensure there is a trailing newline in the key, as older versions of the + // `ssh` client will barf on a key that doesn't have one, but will happily + // allow multiples + _, err = keyFile.WriteString("\n") + if err != nil { + return "", trace.Wrap(err, "failed formatting key") + } + + return keyFile.Name(), nil +} diff --git a/.cloudbuild/scripts/internal/git/knownhosts.go b/.cloudbuild/scripts/internal/git/knownhosts.go new file mode 100644 index 0000000000000..66096bb07f2d7 --- /dev/null +++ b/.cloudbuild/scripts/internal/git/knownhosts.go @@ -0,0 +1,33 @@ +package git + +import ( + "os" + + "github.com/gravitational/trace" + log "github.com/sirupsen/logrus" + "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/knownhosts" +) + +func configureKnownHosts(hostname string, keys []ssh.PublicKey) (string, error) { + knownHostsFile, err := os.CreateTemp("", "*") + if err != nil { + return "", trace.Wrap(err, "failed creating known hosts file") + } + defer knownHostsFile.Close() + + log.Infof("Writing known_hosts file to %s", knownHostsFile.Name()) + + addrs := []string{hostname} + for _, k := range keys { + log.Infof("processing key %s...", k.Type()) + _, err := knownHostsFile.WriteString(knownhosts.Line(addrs, k) + "\n") + if err != nil { + knownHostsFile.Close() + os.Remove(knownHostsFile.Name()) + return "", trace.Wrap(err, "failed writing known hosts") + } + } + + return knownHostsFile.Name(), nil +} diff --git a/.cloudbuild/scripts/internal/git/unshallow.go b/.cloudbuild/scripts/internal/git/unshallow.go new file mode 100644 index 0000000000000..af3387359840f --- /dev/null +++ b/.cloudbuild/scripts/internal/git/unshallow.go @@ -0,0 +1,25 @@ +package git + +import ( + "context" + + "github.com/gravitational/trace" + log "github.com/sirupsen/logrus" +) + +func UnshallowRepository(ctx context.Context, workspace string, deployKey []byte) error { + log.Info("Configuring git") + gitCfg, err := Configure(ctx, workspace, deployKey) + if err != nil { + return trace.Wrap(err, "failed configuring git") + } + defer gitCfg.Close() + + log.Info("Unshallowing repository") + err = gitCfg.Do(ctx, "fetch", "--unshallow") + if err != nil { + return trace.Wrap(err, "unshallow failed") + } + + return nil +} diff --git a/.cloudbuild/scripts/internal/github/meta.go b/.cloudbuild/scripts/internal/github/meta.go new file mode 100644 index 0000000000000..a8096a013dae2 --- /dev/null +++ b/.cloudbuild/scripts/internal/github/meta.go @@ -0,0 +1,58 @@ +package github + +import ( + "context" + "encoding/json" + "net/http" + + "github.com/gravitational/trace" + "golang.org/x/crypto/ssh" +) + +const ( + metadataURL = "https://api.github.com/meta" +) + +// Metadata contains information about the github networking environment, +// enclosing host keys for git/ssh access +type Metadata struct { + SSHKeyFingerPrints map[string]string `json:"ssh_key_fingerprints"` + SSHKeys []string `json:"ssh_keys"` +} + +func GetMetadata(ctx context.Context) (*Metadata, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, metadataURL, nil) + if err != nil { + return nil, trace.Wrap(err, "failed creating metadata request") + } + + response, err := http.DefaultClient.Do(request) + if err != nil { + return nil, trace.Wrap(err, "failed issuing metadata request") + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + return nil, trace.Errorf("Metadata request failed %d", response.StatusCode) + } + + var meta Metadata + err = json.NewDecoder(response.Body).Decode(&meta) + if err != nil { + return nil, trace.Wrap(err, "failed parsing metadata") + } + + return &meta, nil +} + +func (meta *Metadata) HostKeys() ([]ssh.PublicKey, error) { + keys := make([]ssh.PublicKey, 0, len(meta.SSHKeys)) + for _, text := range meta.SSHKeys { + key, _, _, _, err := ssh.ParseAuthorizedKey([]byte(text)) + if err != nil { + return nil, trace.Wrap(err, "failed parsing host key") + } + keys = append(keys, key) + } + return keys, nil +} diff --git a/.cloudbuild/scripts/internal/secrets/fetch.go b/.cloudbuild/scripts/internal/secrets/fetch.go new file mode 100644 index 0000000000000..38cbc1889d280 --- /dev/null +++ b/.cloudbuild/scripts/internal/secrets/fetch.go @@ -0,0 +1,41 @@ +package secrets + +import ( + "context" + + secretmanager "cloud.google.com/go/secretmanager/apiv1" + "github.com/gravitational/trace" + log "github.com/sirupsen/logrus" + secretmanagerpb "google.golang.org/genproto/googleapis/cloud/secretmanager/v1" +) + +// Fetch goes and grabs a single secret from the Google Cloud secret manager +func Fetch(ctx context.Context, resourceName string) ([]byte, error) { + c, err := secretmanager.NewClient(ctx) + if err != nil { + return nil, trace.Wrap(err, "failed creating SecretManager client") + } + defer c.Close() + + log.Debugf("Fetching secret %s", resourceName) + secret, err := c.AccessSecretVersion(ctx, &secretmanagerpb.AccessSecretVersionRequest{ + Name: resourceName, + }) + + if err != nil { + return nil, trace.Wrap(err, "failed fetching secret token") + } + + return secret.Payload.Data, nil +} + +// FetchString fetches a single secret from the Google Cloud secret manager and returns +// it as a string. +func FetchString(ctx context.Context, resourceName string) (string, error) { + data, err := Fetch(ctx, resourceName) + if err != nil { + return "", trace.Wrap(err) + } + + return string(data), nil +}