[v8] Backport build features to work with teleport.e and teleport-private

.cloudbuild/ci/integration-tests.yaml

@@ -1,14 +1,14 @@
 timeout: 30m
 options:
   machineType: E2_HIGHCPU_32
+
+  # This build needs to run in environments where the _GITHUB_DEPLOY_KEY_SRC
+  # substitution is defined, but also environments where it isn't. The
+  # ALLOW_LOOSE option disables GCBs strict checking of substitution usage,
+  # so that the build will still run if _GITHUB_DEPLOY_KEY_SRC is not defined.
+  substitution_option: ALLOW_LOOSE
 
 steps:
-  # GCB does a shallow checkout for a build, but if we want to check our changes
-  # against other branches we'll need to fetch the repo history.
-  - name: gcr.io/cloud-builders/git
-    id: fetch-history
-    args: ['fetch', '--unshallow']
-
   # Run the integration tests. Actual content of this job depends on the changes
   # detected in the PR
   - name: quay.io/gravitational/teleport-buildbox:teleport8
@@ -17,5 +17,8 @@ steps:
     entrypoint: bash
     args:
       - -c
-      - go run ./cmd/integration-tests -w=/workspace -t=$_BASE_BRANCH -c=HEAD
+      - |
+        go run ./cmd/integration-tests          \
+          -target "$_BASE_BRANCH"               \
+          -key-secret "$_GITHUB_DEPLOY_KEY_SRC"
     timeout: 25m

.cloudbuild/ci/unit-tests.yaml

@@ -2,22 +2,24 @@ timeout: 25m
 
 options:
   machineType: 'E2_HIGHCPU_32'
-
+
+  # This build needs to run in environments where the _GITHUB_DEPLOY_KEY_SRC  
+  # substitution is defined, but also environments where it isn't. The
+  # ALLOW_LOOSE option disables GCBs strict checking of substitution usage,
+  # so that the build will still run if _GITHUB_DEPLOY_KEY_SRC is not defined.
+  substitution_option: ALLOW_LOOSE
+
 steps:
-  # GCB does a shallow checkout for a build, but if we want to check our changes
-  # against other branches we'll need to fetch the repo history. This takes less
-  # than 30s at the time of writing, so it is probably not worth tweaking.
-  - name: gcr.io/cloud-builders/git
-    id: fetch-history
-    args: ['fetch', '--unshallow']
-
-  # Run the unit tests. Actual content of this job depends on the changes
+  # Run the unit tests. Actual content of this job depends on the changes 
   # detected in the PR
   - name: quay.io/gravitational/teleport-buildbox:teleport8
     id: run-tests
     dir: /workspace/.cloudbuild/scripts
     entrypoint: bash
     args:
       - -c
-      - go run ./cmd/unit-tests -w=/workspace -t=$_BASE_BRANCH -c=HEAD
-    timeout: 20m
+      - |
+        go run ./cmd/unit-tests                 \
+          -target "$_BASE_BRANCH"               \
+          -key-secret "$_GITHUB_DEPLOY_KEY_SRC"
+    timeout: 25m

.cloudbuild/scripts/cmd/integration-tests/args.go

@@ -0,0 +1,75 @@
+/*
+Copyright 2022 Gravitational, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"flag"
+	"path/filepath"
+
+	"github.com/gravitational/trace"
+)
+
+type commandlineArgs struct {
+	workspace              string
+	targetBranch           string
+	commitSHA              string
+	skipChown              bool
+	githubKeySrc           string
+	skipUnshallow          bool
+}
+
+// validate ensures the suplied arguments are valid & internally consistent.
+func (args *commandlineArgs) validate() error {
+	if args.workspace == "" {
+		return trace.Errorf("workspace path must be set")
+	}
+
+	var err error
+	args.workspace, err = filepath.Abs(args.workspace)
+	if err != nil {
+		return trace.Wrap(err, "Unable to resole absolute path to workspace")
+	}
+
+	if args.targetBranch == "" {
+		return trace.Errorf("target branch must be set")
+	}
+
+	if args.commitSHA == "" {
+		return trace.Errorf("commit must be set")
+	}
+	return nil
+}
+
+func parseCommandLine() (*commandlineArgs, error) {
+	args := &commandlineArgs{}
+
+	flag.StringVar(&args.workspace, "workspace", "/workspace", "Fully-qualified path to the build workspace")
+	flag.StringVar(&args.targetBranch, "target", "", "The PR's target branch")
+	flag.StringVar(&args.commitSHA, "commit", "HEAD", "The PR's latest commit SHA")
+	flag.BoolVar(&args.skipChown, "skip-chown", false, "Skip reconfiguring the workspace for a nonroot user.")
+	flag.StringVar(&args.githubKeySrc, "key-secret", "", "Location of github deploy token, as a Google Cloud Secret")
+	flag.BoolVar(&args.skipUnshallow, "skip-unshallow", false, "Skip unshallowing the repository.")
+
+	flag.Parse()
+
+	err := args.validate()
+	if err != nil {
+		return nil, err
+	}
+
+	return args, nil
+}

.cloudbuild/scripts/cmd/integration-tests/main.go

@@ -18,19 +18,21 @@ package main
 
 import (
 	"context"
-	"flag"
 	"fmt"
 	"io/fs"
-	"log"
 	"os"
 	"os/exec"
 	"path"
 	"path/filepath"
 	"syscall"
+	"time"
 
 	"github.com/gravitational/teleport/.cloudbuild/scripts/internal/changes"
 	"github.com/gravitational/teleport/.cloudbuild/scripts/internal/etcd"
+	"github.com/gravitational/teleport/.cloudbuild/scripts/internal/git"
+	"github.com/gravitational/teleport/.cloudbuild/scripts/internal/secrets"
 	"github.com/gravitational/trace"
+	log "github.com/sirupsen/logrus"
 )
 
 const (
@@ -47,44 +49,6 @@ func main() {
 	}
 }
 
-type commandlineArgs struct {
-	workspace    string
-	targetBranch string
-	commitSHA    string
-	skipChown    bool
-}
-
-func parseCommandLine() (commandlineArgs, error) {
-	args := commandlineArgs{}
-
-	flag.StringVar(&args.workspace, "w", "", "Fully-qualified path to the build workspace")
-	flag.StringVar(&args.targetBranch, "t", "", "The PR's target branch")
-	flag.StringVar(&args.commitSHA, "c", "", "The PR's latest commit SHA")
-	flag.BoolVar(&args.skipChown, "skip-chown", false, "Skip reconfiguring the workspace for a nonroot user.")
-
-	flag.Parse()
-
-	if args.workspace == "" {
-		return args, trace.Errorf("workspace path must be set")
-	}
-
-	var err error
-	args.workspace, err = filepath.Abs(args.workspace)
-	if err != nil {
-		return args, trace.Wrap(err, "Unable to resole absolute path to workspace")
-	}
-
-	if args.targetBranch == "" {
-		return args, trace.Errorf("target branch must be set")
-	}
-
-	if args.commitSHA == "" {
-		return args, trace.Errorf("commit must be set")
-	}
-
-	return args, nil
-}
-
 // innerMain parses the command line, performs the highlevel docs change check
 // and creates the marker file if necessary
 func innerMain() error {
@@ -93,6 +57,26 @@ func innerMain() error {
 		return trace.Wrap(err)
 	}
 
+	// If a github deploy key location was supplied...
+	var deployKey []byte
+	if args.githubKeySrc != "" {
+		// fetch the deployment key from the GCB secret manager
+		log.Infof("Fetching deploy key from %s", args.githubKeySrc)
+		deployKey, err = secrets.Fetch(context.Background(), args.githubKeySrc)
+		if err != nil {
+			return trace.Wrap(err, "failed fetching deploy key")
+		}
+	}
+
+	if !args.skipUnshallow {
+		unshallowCtx, unshallowCancel := context.WithTimeout(context.Background(), 5*time.Minute)
+		defer unshallowCancel()
+		err = git.UnshallowRepository(unshallowCtx, args.workspace, deployKey)
+		if err != nil {
+			return trace.Wrap(err, "unshallow failed")
+		}
+	}
+
 	gomodcache := fmt.Sprintf("GOMODCACHE=%s", path.Join(args.workspace, gomodcacheDir))
 
 	log.Println("Analysing code changes")

.cloudbuild/scripts/cmd/unit-tests/main.go

@@ -20,15 +20,18 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"log"
 	"os"
 	"os/exec"
 	"path"
 	"path/filepath"
+	"time"
 
 	"github.com/gravitational/teleport/.cloudbuild/scripts/internal/changes"
 	"github.com/gravitational/teleport/.cloudbuild/scripts/internal/etcd"
+	"github.com/gravitational/teleport/.cloudbuild/scripts/internal/git"
+	"github.com/gravitational/teleport/.cloudbuild/scripts/internal/secrets"
 	"github.com/gravitational/trace"
+	log "github.com/sirupsen/logrus"
 )
 
 const (
@@ -47,14 +50,18 @@ type commandlineArgs struct {
 	workspace    string
 	targetBranch string
 	commitSHA    string
+	githubKeySrc string
+	skipUnshallow          bool
 }
 
 func parseCommandLine() (commandlineArgs, error) {
 	args := commandlineArgs{}
 
-	flag.StringVar(&args.workspace, "w", "", "Fully-qualified path to the build workspace")
-	flag.StringVar(&args.targetBranch, "t", "", "The PR's target branch")
-	flag.StringVar(&args.commitSHA, "c", "", "The PR's latest commit SHA")
+	flag.StringVar(&args.workspace, "workspace", "/workspace", "Fully-qualified path to the build workspace")
+	flag.StringVar(&args.targetBranch, "target", "", "The PR's target branch")
+	flag.StringVar(&args.commitSHA, "commit", "HEAD", "The PR's latest commit SHA")
+	flag.StringVar(&args.githubKeySrc, "key-secret", "", "Location of github deploy token, as a Google Cloud Secret")
+	flag.BoolVar(&args.skipUnshallow, "skip-unshallow", false, "Skip unshallowing the repository.")
 
 	flag.Parse()
 
@@ -87,6 +94,26 @@ func innerMain() error {
 		return trace.Wrap(err)
 	}
 
+	// If a github deploy key location was supplied...
+	var deployKey []byte
+	if args.githubKeySrc != "" {
+		// fetch the deployment key from the GCB secret manager
+		log.Infof("Fetching deploy key from %s", args.githubKeySrc)
+		deployKey, err = secrets.Fetch(context.Background(), args.githubKeySrc)
+		if err != nil {
+			return trace.Wrap(err, "failed fetching deploy key")
+		}
+	}
+
+	if !args.skipUnshallow {
+		unshallowCtx, unshallowCancel := context.WithTimeout(context.Background(), 5*time.Minute)
+		defer unshallowCancel()
+		err = git.UnshallowRepository(unshallowCtx, args.workspace, deployKey)
+		if err != nil {
+			return trace.Wrap(err, "unshallow failed")
+		}
+	}
+
 	log.Println("Analysing code changes")
 	ch, err := changes.Analyze(args.workspace, args.targetBranch, args.commitSHA)
 	if err != nil {

.cloudbuild/scripts/go.mod

@@ -3,29 +3,44 @@ module github.com/gravitational/teleport/.cloudbuild/scripts
 go 1.17
 
 require (
+	cloud.google.com/go/secretmanager v1.4.0
 	github.com/go-git/go-git/v5 v5.4.2
 	github.com/gravitational/trace v1.1.15
+	github.com/sirupsen/logrus v1.8.1
+	golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b
+	google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd
 )
 
 require (
+	cloud.google.com/go v0.100.2 // indirect
+	cloud.google.com/go/compute v1.5.0 // indirect
+	cloud.google.com/go/iam v0.3.0 // indirect
 	github.com/Microsoft/go-winio v0.4.16 // indirect
 	github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 // indirect
 	github.com/acomagu/bufpipe v1.0.3 // indirect
 	github.com/emirpasic/gods v1.12.0 // indirect
 	github.com/go-git/gcfg v1.5.0 // indirect
 	github.com/go-git/go-billy/v5 v5.3.1 // indirect
+	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/go-cmp v0.5.7 // indirect
+	github.com/googleapis/gax-go/v2 v2.2.0 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/jonboulle/clockwork v0.2.2 // indirect
 	github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 // indirect
-	github.com/konsorten/go-windows-terminal-sequences v1.0.1 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/sergi/go-diff v1.1.0 // indirect
-	github.com/sirupsen/logrus v1.4.1 // indirect
 	github.com/xanzy/ssh-agent v0.3.0 // indirect
-	golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b // indirect
-	golang.org/x/net v0.0.0-20210326060303-6b1517762897 // indirect
-	golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79 // indirect
-	golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect
+	go.opencensus.io v0.23.0 // indirect
+	golang.org/x/net v0.0.0-20220325170049-de3da57026de // indirect
+	golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a // indirect
+	golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886 // indirect
+	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
+	golang.org/x/text v0.3.7 // indirect
+	google.golang.org/api v0.74.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/grpc v1.46.0 // indirect
+	google.golang.org/protobuf v1.28.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 )