From b8089ea29b02304e1fe44ff4422b4c567b840ba0 Mon Sep 17 00:00:00 2001 From: Trent Clarke Date: Thu, 12 May 2022 11:52:42 +1000 Subject: [PATCH] Adds optional deployment key for CI (#10506) (#12590) There are private copies of the teleport repo that require the use of deployment keys to perform operations on the code repository (e.g. `git fetch --unshallow`). This patch adds the ability for a GCB build trigger to optionally pass in the location of a secret deployment key, which will be fetched and used if supplied. If no such deployment key is supplied (i.e. in the public teleport repo), the build will have access to neither the location of the deployment key, nor the key itself, and thus cannot leak it during CI. This patch also pulls the unshallow operation into the main build script (as opposed to being a separate build step) so it can use the deployment key to grant access to the remote repo. Most of the new interaction with the repository is via shelling out to the command-line git installed on the buildbox. My original intention was to use go-git to manage the unshallow directly from code, but this was not supported by the library. --- .cloudbuild/ci/integration-tests.yaml | 25 +-- .cloudbuild/ci/unit-tests.yaml | 30 ++-- .../scripts/cmd/integration-tests/args.go | 2 + .../scripts/cmd/integration-tests/main.go | 20 +++ .cloudbuild/scripts/cmd/unit-tests/main.go | 22 +++ .cloudbuild/scripts/go.mod | 15 +- .cloudbuild/scripts/go.sum | 27 +++- .cloudbuild/scripts/internal/git/configure.go | 145 ++++++++++++++++++ .cloudbuild/scripts/internal/git/deploykey.go | 34 ++++ .../scripts/internal/git/knownhosts.go | 33 ++++ .cloudbuild/scripts/internal/git/unshallow.go | 25 +++ .cloudbuild/scripts/internal/github/meta.go | 58 +++++++ .cloudbuild/scripts/internal/secrets/fetch.go | 41 +++++ 13 files changed, 438 insertions(+), 39 deletions(-) create mode 100644 .cloudbuild/scripts/internal/git/configure.go create mode 100644 .cloudbuild/scripts/internal/git/deploykey.go create mode 100644 .cloudbuild/scripts/internal/git/knownhosts.go create mode 100644 .cloudbuild/scripts/internal/git/unshallow.go create mode 100644 .cloudbuild/scripts/internal/github/meta.go create mode 100644 .cloudbuild/scripts/internal/secrets/fetch.go diff --git a/.cloudbuild/ci/integration-tests.yaml b/.cloudbuild/ci/integration-tests.yaml index 27f9c8950dc55..2e3437df8c801 100644 --- a/.cloudbuild/ci/integration-tests.yaml +++ b/.cloudbuild/ci/integration-tests.yaml @@ -2,15 +2,15 @@ timeout: 25m options: machineType: E2_HIGHCPU_32 + + # This build needs to run in environments where the _GITHUB_DEPLOY_KEY_SRC + # substitution is defined, but also environments where it isn't. The + # ALLOW_LOOSE option disables GCBs strict checking of substitution usage, + # so that the build will still run if _GITHUB_DEPLOY_KEY_SRC is not defined. + substitution_option: ALLOW_LOOSE steps: - # GCB does a shallow checkout for a build, but if we want to check our changes - # against other branches we'll need to fetch the repo history. - - name: gcr.io/cloud-builders/git - id: fetch-history - args: ['fetch', '--unshallow'] - - # Run the integration tests. Actual content of this job depends on the changes + # Run the integration tests. Actual content of this job depends on the changes # detected in the PR - name: quay.io/gravitational/teleport-buildbox:teleport9 id: run-tests @@ -19,9 +19,10 @@ steps: args: - -c - | - go run ./cmd/integration-tests \ - -target "$_BASE_BRANCH" \ - -bucket test-logs \ - -build "$BUILD_ID" \ - -a "test-logs/*.json" + go run ./cmd/integration-tests \ + -target "$_BASE_BRANCH" \ + -bucket test-logs \ + -build "$BUILD_ID" \ + -key-secret "$_GITHUB_DEPLOY_KEY_SRC" \ + -a "test-logs/*.json" timeout: 25m diff --git a/.cloudbuild/ci/unit-tests.yaml b/.cloudbuild/ci/unit-tests.yaml index b12ab966f716e..74960d38e430a 100644 --- a/.cloudbuild/ci/unit-tests.yaml +++ b/.cloudbuild/ci/unit-tests.yaml @@ -1,17 +1,16 @@ timeout: 25m options: - machineType: E2_HIGHCPU_32 - + machineType: 'E2_HIGHCPU_32' + + # This build needs to run in environments where the _GITHUB_DEPLOY_KEY_SRC + # substitution is defined, but also environments where it isn't. The + # ALLOW_LOOSE option disables GCBs strict checking of substitution usage, + # so that the build will still run if _GITHUB_DEPLOY_KEY_SRC is not defined. + substitution_option: ALLOW_LOOSE + steps: - # GCB does a shallow checkout for a build, but if we want to check our changes - # against other branches we'll need to fetch the repo history. This takes less - # than 30s at the time of writing, so it is probably not worth tweaking. - - name: gcr.io/cloud-builders/git - id: fetch-history - args: ['fetch', '--unshallow'] - - # Run the unit tests. Actual content of this job depends on the changes + # Run the unit tests. Actual content of this job depends on the changes # detected in the PR - name: quay.io/gravitational/teleport-buildbox:teleport9 id: run-tests @@ -20,9 +19,10 @@ steps: args: - -c - | - go run ./cmd/unit-tests \ - -target "$_BASE_BRANCH" \ - -bucket test-logs \ - -build "$BUILD_ID" \ + go run ./cmd/unit-tests \ + -target "$_BASE_BRANCH" \ + -bucket test-logs \ + -build "$BUILD_ID" \ + -key-secret "$_GITHUB_DEPLOY_KEY_SRC" \ -a "test-logs/*.json" - timeout: 20m + timeout: 25m diff --git a/.cloudbuild/scripts/cmd/integration-tests/args.go b/.cloudbuild/scripts/cmd/integration-tests/args.go index 2b93984168b6a..229c3aedcdc25 100644 --- a/.cloudbuild/scripts/cmd/integration-tests/args.go +++ b/.cloudbuild/scripts/cmd/integration-tests/args.go @@ -33,6 +33,7 @@ type commandlineArgs struct { buildID string artifactSearchPatterns customflag.StringArray bucket string + githubKeySrc string } // validate ensures the suplied arguments are valid & internally consistent. @@ -83,6 +84,7 @@ func parseCommandLine() (*commandlineArgs, error) { flag.StringVar(&args.buildID, "build", "", "The build ID") flag.StringVar(&args.bucket, "bucket", "", "The artifact storage bucket.") flag.Var(&args.artifactSearchPatterns, "a", "Path to artifacts. May be shell-globbed, and have multiple entries.") + flag.StringVar(&args.githubKeySrc, "key-secret", "", "Location of github deploy token, as a Google Cloud Secret") flag.Parse() diff --git a/.cloudbuild/scripts/cmd/integration-tests/main.go b/.cloudbuild/scripts/cmd/integration-tests/main.go index f6de022e00fcc..18680710080f7 100644 --- a/.cloudbuild/scripts/cmd/integration-tests/main.go +++ b/.cloudbuild/scripts/cmd/integration-tests/main.go @@ -29,6 +29,8 @@ import ( "github.com/gravitational/teleport/.cloudbuild/scripts/internal/artifacts" "github.com/gravitational/teleport/.cloudbuild/scripts/internal/changes" "github.com/gravitational/teleport/.cloudbuild/scripts/internal/etcd" + "github.com/gravitational/teleport/.cloudbuild/scripts/internal/git" + "github.com/gravitational/teleport/.cloudbuild/scripts/internal/secrets" "github.com/gravitational/trace" log "github.com/sirupsen/logrus" ) @@ -55,6 +57,24 @@ func innerMain() error { return trace.Wrap(err) } + // If a github deploy key location was supplied... + var deployKey []byte + if args.githubKeySrc != "" { + // fetch the deployment key from the GCB secret manager + log.Infof("Fetching deploy key from %s", args.githubKeySrc) + deployKey, err = secrets.Fetch(context.Background(), args.githubKeySrc) + if err != nil { + return trace.Wrap(err, "failed fetching deploy key") + } + } + + unshallowCtx, unshallowCancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer unshallowCancel() + err = git.UnshallowRepository(unshallowCtx, args.workspace, deployKey) + if err != nil { + return trace.Wrap(err, "unshallow failed") + } + moduleCacheDir := filepath.Join(os.TempDir(), gomodcacheDir) gomodcache := fmt.Sprintf("GOMODCACHE=%s", moduleCacheDir) diff --git a/.cloudbuild/scripts/cmd/unit-tests/main.go b/.cloudbuild/scripts/cmd/unit-tests/main.go index b1893ae248591..4b69c7f1df357 100644 --- a/.cloudbuild/scripts/cmd/unit-tests/main.go +++ b/.cloudbuild/scripts/cmd/unit-tests/main.go @@ -29,6 +29,8 @@ import ( "github.com/gravitational/teleport/.cloudbuild/scripts/internal/changes" "github.com/gravitational/teleport/.cloudbuild/scripts/internal/customflag" "github.com/gravitational/teleport/.cloudbuild/scripts/internal/etcd" + "github.com/gravitational/teleport/.cloudbuild/scripts/internal/git" + "github.com/gravitational/teleport/.cloudbuild/scripts/internal/secrets" "github.com/gravitational/trace" log "github.com/sirupsen/logrus" ) @@ -48,6 +50,7 @@ type commandlineArgs struct { buildID string artifactSearchPatterns customflag.StringArray bucket string + githubKeySrc string } func parseCommandLine() (commandlineArgs, error) { @@ -59,6 +62,7 @@ func parseCommandLine() (commandlineArgs, error) { flag.StringVar(&args.buildID, "build", "", "The build ID") flag.StringVar(&args.bucket, "bucket", "", "The artifact storage bucket.") flag.Var(&args.artifactSearchPatterns, "a", "Path to artifacts. May be globbed, and have multiple entries.") + flag.StringVar(&args.githubKeySrc, "key-secret", "", "Location of github deploy token, as a Google Cloud Secret") flag.Parse() @@ -106,6 +110,24 @@ func run() error { return trace.Wrap(err) } + // If a github deploy key location was supplied... + var deployKey []byte + if args.githubKeySrc != "" { + // fetch the deployment key from the GCB secret manager + log.Infof("Fetching deploy key from %s", args.githubKeySrc) + deployKey, err = secrets.Fetch(context.Background(), args.githubKeySrc) + if err != nil { + return trace.Wrap(err, "failed fetching deploy key") + } + } + + unshallowCtx, unshallowCancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer unshallowCancel() + err = git.UnshallowRepository(unshallowCtx, args.workspace, deployKey) + if err != nil { + return trace.Wrap(err, "unshallow failed") + } + log.Println("Analysing code changes") ch, err := changes.Analyze(args.workspace, args.targetBranch, args.commitSHA) if err != nil { diff --git a/.cloudbuild/scripts/go.mod b/.cloudbuild/scripts/go.mod index 300c5bb2c445a..49cf54361d31c 100644 --- a/.cloudbuild/scripts/go.mod +++ b/.cloudbuild/scripts/go.mod @@ -3,11 +3,15 @@ module github.com/gravitational/teleport/.cloudbuild/scripts go 1.17 require ( + cloud.google.com/go/secretmanager v1.2.0 cloud.google.com/go/storage v1.19.0 github.com/go-git/go-git/v5 v5.4.2 github.com/gravitational/trace v1.1.15 github.com/hashicorp/go-multierror v1.1.1 github.com/sirupsen/logrus v1.8.1 + github.com/stretchr/testify v1.7.0 + golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b + google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c ) require ( @@ -33,21 +37,18 @@ require ( github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/sergi/go-diff v1.1.0 // indirect - github.com/stretchr/objx v0.1.1 // indirect - github.com/stretchr/testify v1.7.0 // indirect + github.com/stretchr/objx v0.2.0 // indirect github.com/xanzy/ssh-agent v0.3.0 // indirect go.opencensus.io v0.23.0 // indirect - golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b // indirect golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420 // indirect golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect - golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect + golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27 // indirect golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect golang.org/x/text v0.3.6 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect - google.golang.org/api v0.65.0 // indirect + google.golang.org/api v0.67.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20220118154757-00ab72f36ad5 // indirect - google.golang.org/grpc v1.40.1 // indirect + google.golang.org/grpc v1.44.0 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect diff --git a/.cloudbuild/scripts/go.sum b/.cloudbuild/scripts/go.sum index a9f64ef4f6699..d2b25fdba3cd4 100644 --- a/.cloudbuild/scripts/go.sum +++ b/.cloudbuild/scripts/go.sum @@ -39,12 +39,15 @@ cloud.google.com/go/compute v0.1.0 h1:rSUBvAyVwNJ5uQCKNJFMwPtTvJkfN38b6Pvb9zZoqJ cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/iam v0.1.0/go.mod h1:vcUNEa0pEm0qRVpmWepWaFMIAI8/hjB9mO8rNCJtF6c= cloud.google.com/go/iam v0.1.1 h1:4CapQyNFjiksks1/x7jsvsygFPhihslYk5GptIrlX68= cloud.google.com/go/iam v0.1.1/go.mod h1:CKqrcnI/suGpybEHxZ7BMehL0oA4LpdyJdUlTl9jVMw= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/secretmanager v1.2.0 h1:VR6MzO4wjTj5jQKTPpsZhCF2PqqdAAZmN54BwJbQPhs= +cloud.google.com/go/secretmanager v1.2.0/go.mod h1:HNMYTaLrMrAN37vi2mM2vvFgjgaoCE1qvtccCIJwFRc= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= @@ -70,6 +73,7 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -77,7 +81,11 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -91,6 +99,7 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -235,9 +244,11 @@ github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -420,8 +431,9 @@ golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27 h1:XDXtA5hveEEV8JB2l7nhMTp3t3cHp9ZpwcdjqyEWLlo= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -524,8 +536,9 @@ google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdr google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= google.golang.org/api v0.64.0/go.mod h1:931CdxA8Rm4t6zqTFGSsgwbAEZ2+GMYurbndwSimebM= -google.golang.org/api v0.65.0 h1:MTW9c+LIBAbwoS1Gb+YV7NjFBt2f7GtAS5hIzh2NjgQ= google.golang.org/api v0.65.0/go.mod h1:ArYhxgGadlWmqO1IqVujw6Cs8IdD33bTmzKo2Sh+cbg= +google.golang.org/api v0.67.0 h1:lYaaLa+x3VVUhtosaK9xihwQ9H9KRa557REHwwZ2orM= +google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -597,8 +610,11 @@ google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ6 google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220111164026-67b88f271998/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220118154757-00ab72f36ad5 h1:zzNejm+EgrbLfDZ6lu9Uud2IVvHySPl8vQzf04laR5Q= google.golang.org/genproto v0.0.0-20220118154757-00ab72f36ad5/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c h1:TU4rFa5APdKTq0s6B7WTsH6Xmx0Knj86s6Biz56mErE= +google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -624,8 +640,9 @@ google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.40.1 h1:pnP7OclFFFgFi4VHQDQDaoXUVauOFyktqTsqqgzFKbc= google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.44.0 h1:weqSxi/TMs1SqFRMHCtBgXRs8k3X39QIDEZ0pRcttUg= +google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= diff --git a/.cloudbuild/scripts/internal/git/configure.go b/.cloudbuild/scripts/internal/git/configure.go new file mode 100644 index 0000000000000..63d2fdb54e9b2 --- /dev/null +++ b/.cloudbuild/scripts/internal/git/configure.go @@ -0,0 +1,145 @@ +package git + +import ( + "context" + "fmt" + "os" + "os/exec" + + "github.com/gravitational/teleport/.cloudbuild/scripts/internal/github" + "github.com/gravitational/trace" + log "github.com/sirupsen/logrus" + "golang.org/x/crypto/ssh" +) + +// Config represents a git repository that has been configured to use a +// deployment key, and acts as a handle to the resources so that we can +// clean them up when we're done. +type Config struct { + identity string + knownHosts string + repoDir string +} + +// Configure alters the configuration of the git repository in `repoDir` +// so that we can access it from build. If `deployKey` is non-nil the repo +// will be configured to use that. If no deploy key is supplied, the repository +// config is untouched. +func Configure(ctx context.Context, repoDir string, deployKey []byte) (cfg *Config, err error) { + var identity string + var hostsFile string + + // The deploy key we're using is too sensitive to just hope that every + // exit path will clean it up on failure, so let's just register a + // cleanup function now just in case. + defer func() { + if err != nil { + cleanup(identity, hostsFile) + } + }() + + // If we have been supplied with deployment key, then we need to use that + // to access the repository. This implies that need to use ssh to read from + // the remote, so we will have to deal with known_hosts, etc. + if deployKey != nil { + githubHostKeys, err := getGithubHostKeys(ctx) + if err != nil { + return nil, trace.Wrap(err, "failed configuring known hosts") + } + + // configure SSH known_hosts for github.com + hostsFile, err = configureKnownHosts("github.com", githubHostKeys) + if err != nil { + return nil, trace.Wrap(err, "failed configuring known hosts") + } + + // GCB clones via https, and our deploy key won't work with that, so + // force git to access the remote over ssh with a rewrite rule + log.Info("Adding remote url rewrite rule") + err = git(ctx, repoDir, "config", `url.git@github.com:.insteadOf`, "https://github.com/") + if err != nil { + return nil, trace.Wrap(err, "failed configuring url rewrite rule") + } + + // set up the identity for the deploy key + identity, err = writeKey(deployKey) + if err != nil { + return nil, trace.Wrap(err, "failed configuring SSH identity") + } + + // finally, force git to + // a) use our custom SSH setup when accessing the remote, and + // b) fail if the github host tries to present a host key other than + // one we got from the metadata service + log.Infof("Configuring git ssh command") + err = git(ctx, repoDir, "config", "core.sshCommand", + fmt.Sprintf("ssh -i %s -o StrictHostKeyChecking=yes -o UserKnownHostsFile=%s", identity, hostsFile)) + if err != nil { + return nil, trace.Wrap(err, "failed configuring git to use deploy key") + } + } + + return &Config{identity: identity, repoDir: repoDir, knownHosts: hostsFile}, nil +} + +// Do runs `git args...` in the configured repository +func (cfg *Config) Do(ctx context.Context, args ...string) error { + return git(ctx, cfg.repoDir, args...) +} + +// Close cleans up the repository, including deleting the deployment key (if any) +func (cfg *Config) Close() error { + cleanup(cfg.identity, cfg.knownHosts) + return nil +} + +func run(ctx context.Context, workingDir string, env []string, cmd string, args ...string) error { + p := exec.CommandContext(ctx, cmd, args...) + if len(env) != 0 { + p.Env = append(os.Environ(), env...) + } + p.Dir = workingDir + + cmdLogger := log.WithField("cmd", cmd) + p.Stdout = cmdLogger.WriterLevel(log.InfoLevel) + p.Stderr = cmdLogger.WriterLevel(log.ErrorLevel) + return p.Run() +} + +func git(ctx context.Context, repoDir string, args ...string) error { + return run(ctx, repoDir, nil, "/usr/bin/git", args...) +} + +func cleanup(deployKey, knownHosts string) { + if knownHosts != "" { + log.Infof("Removing known_hosts file %s", knownHosts) + if err := os.Remove(knownHosts); err != nil { + log.WithError(err).Error("Failed cleaning up known_hosts key") + } + } + + if deployKey != "" { + log.Infof("Removing deploy key file %s", deployKey) + if err := os.Remove(deployKey); err != nil { + log.WithError(err).Error("Failed cleaning up deploy key") + } + } +} + +// getGithubHostKeys fetches the github host keys from the github metadata +// service. The metadata is fetched over HTTPS, and so we have built-on +// protection against MitM attacks while fetching the expected host keys. +func getGithubHostKeys(ctx context.Context) ([]ssh.PublicKey, error) { + metadata, err := github.GetMetadata(ctx) + if err != nil { + return nil, trace.Wrap(err, "failed fetching github metadata") + } + + // extract the host keys + githubHostKeys, err := metadata.HostKeys() + if err != nil { + return nil, trace.Wrap(err, "failed fetching github hostKeys") + } + + return githubHostKeys, nil +} diff --git a/.cloudbuild/scripts/internal/git/deploykey.go b/.cloudbuild/scripts/internal/git/deploykey.go new file mode 100644 index 0000000000000..fc57eb2798ef7 --- /dev/null +++ b/.cloudbuild/scripts/internal/git/deploykey.go @@ -0,0 +1,34 @@ +package git + +import ( + "os" + + "github.com/gravitational/trace" + log "github.com/sirupsen/logrus" +) + +func writeKey(deployKey []byte) (string, error) { + // Note that tempfiles are automatically created with 0600, so no-one else + // should be able to read this. + keyFile, err := os.CreateTemp("", "*") + if err != nil { + return "", trace.Wrap(err, "failed creating keyfile") + } + defer keyFile.Close() + + log.Infof("Writing deploy key to %s", keyFile.Name()) + _, err = keyFile.Write(deployKey) + if err != nil { + return "", trace.Wrap(err, "failed writing deploy key") + } + + // ensure there is a trailing newline in the key, as older versions of the + // `ssh` client will barf on a key that doesn't have one, but will happily + // allow multiples + _, err = keyFile.WriteString("\n") + if err != nil { + return "", trace.Wrap(err, "failed formatting key") + } + + return keyFile.Name(), nil +} diff --git a/.cloudbuild/scripts/internal/git/knownhosts.go b/.cloudbuild/scripts/internal/git/knownhosts.go new file mode 100644 index 0000000000000..66096bb07f2d7 --- /dev/null +++ b/.cloudbuild/scripts/internal/git/knownhosts.go @@ -0,0 +1,33 @@ +package git + +import ( + "os" + + "github.com/gravitational/trace" + log "github.com/sirupsen/logrus" + "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/knownhosts" +) + +func configureKnownHosts(hostname string, keys []ssh.PublicKey) (string, error) { + knownHostsFile, err := os.CreateTemp("", "*") + if err != nil { + return "", trace.Wrap(err, "failed creating known hosts file") + } + defer knownHostsFile.Close() + + log.Infof("Writing known_hosts file to %s", knownHostsFile.Name()) + + addrs := []string{hostname} + for _, k := range keys { + log.Infof("processing key %s...", k.Type()) + _, err := knownHostsFile.WriteString(knownhosts.Line(addrs, k) + "\n") + if err != nil { + knownHostsFile.Close() + os.Remove(knownHostsFile.Name()) + return "", trace.Wrap(err, "failed writing known hosts") + } + } + + return knownHostsFile.Name(), nil +} diff --git a/.cloudbuild/scripts/internal/git/unshallow.go b/.cloudbuild/scripts/internal/git/unshallow.go new file mode 100644 index 0000000000000..af3387359840f --- /dev/null +++ b/.cloudbuild/scripts/internal/git/unshallow.go @@ -0,0 +1,25 @@ +package git + +import ( + "context" + + "github.com/gravitational/trace" + log "github.com/sirupsen/logrus" +) + +func UnshallowRepository(ctx context.Context, workspace string, deployKey []byte) error { + log.Info("Configuring git") + gitCfg, err := Configure(ctx, workspace, deployKey) + if err != nil { + return trace.Wrap(err, "failed configuring git") + } + defer gitCfg.Close() + + log.Info("Unshallowing repository") + err = gitCfg.Do(ctx, "fetch", "--unshallow") + if err != nil { + return trace.Wrap(err, "unshallow failed") + } + + return nil +} diff --git a/.cloudbuild/scripts/internal/github/meta.go b/.cloudbuild/scripts/internal/github/meta.go new file mode 100644 index 0000000000000..a8096a013dae2 --- /dev/null +++ b/.cloudbuild/scripts/internal/github/meta.go @@ -0,0 +1,58 @@ +package github + +import ( + "context" + "encoding/json" + "net/http" + + "github.com/gravitational/trace" + "golang.org/x/crypto/ssh" +) + +const ( + metadataURL = "https://api.github.com/meta" +) + +// Metadata contains information about the github networking environment, +// enclosing host keys for git/ssh access +type Metadata struct { + SSHKeyFingerPrints map[string]string `json:"ssh_key_fingerprints"` + SSHKeys []string `json:"ssh_keys"` +} + +func GetMetadata(ctx context.Context) (*Metadata, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, metadataURL, nil) + if err != nil { + return nil, trace.Wrap(err, "failed creating metadata request") + } + + response, err := http.DefaultClient.Do(request) + if err != nil { + return nil, trace.Wrap(err, "failed issuing metadata request") + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + return nil, trace.Errorf("Metadata request failed %d", response.StatusCode) + } + + var meta Metadata + err = json.NewDecoder(response.Body).Decode(&meta) + if err != nil { + return nil, trace.Wrap(err, "failed parsing metadata") + } + + return &meta, nil +} + +func (meta *Metadata) HostKeys() ([]ssh.PublicKey, error) { + keys := make([]ssh.PublicKey, 0, len(meta.SSHKeys)) + for _, text := range meta.SSHKeys { + key, _, _, _, err := ssh.ParseAuthorizedKey([]byte(text)) + if err != nil { + return nil, trace.Wrap(err, "failed parsing host key") + } + keys = append(keys, key) + } + return keys, nil +} diff --git a/.cloudbuild/scripts/internal/secrets/fetch.go b/.cloudbuild/scripts/internal/secrets/fetch.go new file mode 100644 index 0000000000000..38cbc1889d280 --- /dev/null +++ b/.cloudbuild/scripts/internal/secrets/fetch.go @@ -0,0 +1,41 @@ +package secrets + +import ( + "context" + + secretmanager "cloud.google.com/go/secretmanager/apiv1" + "github.com/gravitational/trace" + log "github.com/sirupsen/logrus" + secretmanagerpb "google.golang.org/genproto/googleapis/cloud/secretmanager/v1" +) + +// Fetch goes and grabs a single secret from the Google Cloud secret manager +func Fetch(ctx context.Context, resourceName string) ([]byte, error) { + c, err := secretmanager.NewClient(ctx) + if err != nil { + return nil, trace.Wrap(err, "failed creating SecretManager client") + } + defer c.Close() + + log.Debugf("Fetching secret %s", resourceName) + secret, err := c.AccessSecretVersion(ctx, &secretmanagerpb.AccessSecretVersionRequest{ + Name: resourceName, + }) + + if err != nil { + return nil, trace.Wrap(err, "failed fetching secret token") + } + + return secret.Payload.Data, nil +} + +// FetchString fetches a single secret from the Google Cloud secret manager and returns +// it as a string. +func FetchString(ctx context.Context, resourceName string) (string, error) { + data, err := Fetch(ctx, resourceName) + if err != nil { + return "", trace.Wrap(err) + } + + return string(data), nil +}