clusterlib: fake boskos client for better test coverage (#649)

* Fake boskos

* feedback updates

testutils/clustermanager/boskos/boskos.go

@@ -37,6 +37,15 @@ var (
 	defaultWaitDuration = time.Minute * 20
 )
 
+type Operation interface {
+	AcquireGKEProject(*string) (*boskoscommon.Resource, error)
+	ReleaseGKEProject(*string, string) error
+}
+
+type Client struct {
+	*boskosclient.Client
+}
+
 func newClient(host *string) *boskosclient.Client {
 	if nil == host {
 		hostName := common.GetOSEnv("JOB_NAME")
@@ -48,7 +57,7 @@ func newClient(host *string) *boskosclient.Client {
 // AcquireGKEProject acquires GKE Boskos Project with "free" state, and not
 // owned by anyone, sets its state to "busy" and assign it an owner of *host,
 // which by default is env var `JOB_NAME`.
-func AcquireGKEProject(host *string) (*boskoscommon.Resource, error) {
+func (c *Client) AcquireGKEProject(host *string) (*boskoscommon.Resource, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultWaitDuration)
 	defer cancel()
 	p, err := newClient(host).AcquireWait(ctx, GKEProjectResource, boskoscommon.Free, boskoscommon.Busy)
@@ -66,7 +75,7 @@ func AcquireGKEProject(host *string) (*boskoscommon.Resource, error) {
 // "dirty" for Janitor picking up.
 // This function is very powerful, it can release Boskos resource acquired by
 // other processes, regardless of where the other process is running.
-func ReleaseGKEProject(host *string, name string) error {
+func (c *Client) ReleaseGKEProject(host *string, name string) error {
 	client := newClient(host)
 	if err := client.Release(name, boskoscommon.Dirty); nil != err {
 		return fmt.Errorf("boskos failed to release GKE project '%s': %v", name, err)

testutils/clustermanager/boskos/boskos_test.go

@@ -29,8 +29,14 @@ import (
 var (
 	fakeHost = "fakehost"
 	fakeRes  = "{\"name\": \"res\", \"type\": \"t\", \"state\": \"d\"}"
+
+	client Client
 )
 
+func setup() {
+	client = Client{}
+}
+
 // create a fake server as Boskos server, must close() afterwards
 func fakeServer(f func(http.ResponseWriter, *http.Request)) *httptest.Server {
 	return httptest.NewServer(http.HandlerFunc(f))
@@ -66,6 +72,7 @@ func TestAcquireGKEProject(t *testing.T) {
 		common.GetOSEnv = oldGetOSEnv
 	}()
 	for _, data := range datas {
+		setup()
 		ts := fakeServer(func(w http.ResponseWriter, r *http.Request) {
 			if data.serverErr {
 				http.Error(w, "", http.StatusBadRequest)
@@ -82,7 +89,7 @@ func TestAcquireGKEProject(t *testing.T) {
 		})
 		defer ts.Close()
 		boskosURI = ts.URL
-		_, err := AcquireGKEProject(data.host)
+		_, err := client.AcquireGKEProject(data.host)
 		if data.expErr && (nil == err) {
 			t.Fatalf("testing acquiring GKE project, want: err, got: no err")
 		}
@@ -123,6 +130,7 @@ func TestReleaseGKEProject(t *testing.T) {
 		common.GetOSEnv = oldGetOSEnv
 	}()
 	for _, data := range datas {
+		setup()
 		ts := fakeServer(func(w http.ResponseWriter, r *http.Request) {
 			if data.serverErr {
 				http.Error(w, "", http.StatusBadRequest)
@@ -134,7 +142,7 @@ func TestReleaseGKEProject(t *testing.T) {
 		})
 		defer ts.Close()
 		boskosURI = ts.URL
-		err := ReleaseGKEProject(data.host, data.resName)
+		err := client.ReleaseGKEProject(data.host, data.resName)
 		if data.expErr && (nil == err) {
 			t.Fatalf("testing acquiring GKE project, want: err, got: no err")
 		}

testutils/clustermanager/boskos/fake/fake.go

@@ -0,0 +1,69 @@
+/*
+Copyright 2019 The Knative Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fake
+
+import (
+	"fmt"
+
+	boskoscommon "k8s.io/test-infra/boskos/common"
+	"knative.dev/pkg/testutils/clustermanager/boskos"
+)
+
+// FakeBoskosClient implements boskos.Operation
+type FakeBoskosClient struct {
+	resources []*boskoscommon.Resource
+}
+
+// AcquireGKEProject fakes to be no op
+func (c *FakeBoskosClient) AcquireGKEProject(host *string) (*boskoscommon.Resource, error) {
+	for _, res := range c.resources {
+		if res.State == boskoscommon.Free {
+			res.State = boskoscommon.Busy
+			return res, nil
+		}
+	}
+	return nil, fmt.Errorf("no GKE project available")
+}
+
+// ReleaseGKEProject fakes to be no op
+func (c *FakeBoskosClient) ReleaseGKEProject(host *string, name string) error {
+	if nil == host {
+		return fmt.Errorf("host has to be set")
+	}
+
+	for _, res := range c.resources {
+		if res.Name == name {
+			if res.Owner == *host {
+				res.Owner = ""
+				res.State = boskoscommon.Free
+				return nil
+			} else {
+				return fmt.Errorf("Got owner: '%s', expect owner: '%s'", res.Owner, *host)
+			}
+		}
+	}
+	return fmt.Errorf("resource doesn't exist yet: '%s'", name)
+}
+
+// NewGKEProject adds Boskos resources for testing purpose
+func (c *FakeBoskosClient) NewGKEProject(name string) {
+	c.resources = append(c.resources, &boskoscommon.Resource{
+		Type:  boskos.GKEProjectResource,
+		Name:  name,
+		State: boskoscommon.Free,
+	})
+}

testutils/clustermanager/gke.go

@@ -71,6 +71,7 @@ type GKECluster struct {
 	NeedCleanup bool
 	Cluster     *container.Cluster
 	operations  GKESDKOperations
+	boskosOps   boskos.Operation
 }
 
 // GKESDKOperations wraps GKE SDK related functions
@@ -152,27 +153,29 @@ func (gs *GKEClient) Setup(numNodes *int64, nodeType *string, region *string, zo
 	}
 	gc.operations = &GKESDKClient{containerService}
 
+	gc.boskosOps = &boskos.Client{}
+
 	return gc
 }
 
 // Initialize sets up GKE SDK client, checks environment for cluster and
 // projects to decide whether use existing cluster/project or creating new ones.
 func (gc *GKECluster) Initialize() error {
+	// Try obtain project name via `kubectl`, `gcloud`
 	if nil == gc.Project {
 		if err := gc.checkEnvironment(); nil != err {
 			return fmt.Errorf("failed checking existing cluster: '%v'", err)
 		} else if nil != gc.Cluster { // return if Cluster was already set by kubeconfig
 			return nil
 		}
 	}
-	if nil == gc.Cluster {
-		if common.IsProw() {
-			project, err := boskos.AcquireGKEProject(nil)
-			if nil != err {
-				return fmt.Errorf("failed acquire boskos project: '%v'", err)
-			}
-			gc.Project = &project.Name
+	// Get project name from boskos if running in Prow
+	if nil == gc.Project && common.IsProw() {
+		project, err := gc.boskosOps.AcquireGKEProject(nil)
+		if nil != err {
+			return fmt.Errorf("failed acquire boskos project: '%v'", err)
 		}
+		gc.Project = &project.Name
 	}
 	if nil == gc.Project || "" == *gc.Project {
 		return errors.New("gcp project must be set")

testutils/clustermanager/gke_test.go

@@ -27,6 +27,7 @@ import (
 
 	"google.golang.org/api/container/v1"
 
+	boskosFake "knative.dev/pkg/testutils/clustermanager/boskos/fake"
 	"knative.dev/pkg/testutils/common"
 )
 
@@ -38,6 +39,7 @@ var (
 func setupFakeGKECluster() GKECluster {
 	return GKECluster{
 		operations: newFakeGKESDKClient(),
+		boskosOps:  &boskosFake.FakeBoskosClient{},
 	}
 }
 
@@ -257,6 +259,7 @@ func TestSetup(t *testing.T) {
 		gotCo := co.(*GKECluster)
 		// mock for easier comparison
 		gotCo.operations = nil
+		gotCo.boskosOps = nil
 		if !reflect.DeepEqual(co, data.expClusterOperations) {
 			t.Fatalf("%s\nwant GKECluster:\n'%v'\ngot GKECluster:\n'%v'", errPrefix, data.expClusterOperations, co)
 		}
@@ -265,30 +268,45 @@ func TestSetup(t *testing.T) {
 
 func TestInitialize(t *testing.T) {
 	customProj := "customproj"
+	fakeBoskosProj := "fake-boskos-proj-0"
 	datas := []struct {
 		project      *string
 		clusterExist bool
 		gcloudSet    bool
+		isProw       bool
+		boskosProjs  []string
 		expProj      *string
 		expCluster   *container.Cluster
 		expErr       error
 	}{
 		{
 			// User defines project
-			&fakeProj, false, false, &fakeProj, nil, nil,
+			&fakeProj, false, false, false, []string{}, &fakeProj, nil, nil,
+		}, {
+			// User defines project, and running in Prow
+			&fakeProj, false, false, true, []string{}, &fakeProj, nil, nil,
 		}, {
 			// kubeconfig set
-			nil, true, false, &fakeProj, &container.Cluster{
+			nil, true, false, false, []string{}, &fakeProj, &container.Cluster{
 				Name:     "d",
 				Location: "c",
 				Status:   "RUNNING",
 			}, nil,
-		}, {
-			// kubeconfig not set and gcloud not set
-			nil, false, true, &customProj, nil, nil,
 		}, {
 			// kubeconfig not set and gcloud set
-			nil, false, false, nil, nil, fmt.Errorf("gcp project must be set"),
+			nil, false, true, false, []string{}, &customProj, nil, nil,
+		}, {
+			// kubeconfig not set and gcloud set, running in Prow and boskos not available
+			nil, false, false, true, []string{}, nil, nil, fmt.Errorf("failed acquire boskos project: 'no GKE project available'"),
+		}, {
+			// kubeconfig not set and gcloud set, running in Prow and boskos available
+			nil, false, false, true, []string{fakeBoskosProj}, &fakeBoskosProj, nil, nil,
+		}, {
+			// kubeconfig not set and gcloud set, not in Prow and boskos not available
+			nil, false, false, false, []string{}, nil, nil, fmt.Errorf("gcp project must be set"),
+		}, {
+			// kubeconfig not set and gcloud set, not in Prow and boskos available
+			nil, false, false, false, []string{fakeBoskosProj}, nil, nil, fmt.Errorf("gcp project must be set"),
 		},
 	}
 
@@ -300,16 +318,6 @@ func TestInitialize(t *testing.T) {
 		common.StandardExec = oldExecFunc
 	}()
 
-	// Mock to make IsProw() always return false, otherwise it will actually
-	// acquire a boskos project
-	common.GetOSEnv = func(s string) string {
-		switch s {
-		case "PROW_JOB_ID":
-			return ""
-		}
-		return oldEnvFunc(s)
-	}
-
 	for _, data := range datas {
 		fgc := setupFakeGKECluster()
 		if nil != data.project {
@@ -324,6 +332,10 @@ func TestInitialize(t *testing.T) {
 				ProjectId: parts[1],
 			})
 		}
+		// Set up fake boskos
+		for _, bos := range data.boskosProjs {
+			fgc.boskosOps.(*boskosFake.FakeBoskosClient).NewGKEProject(bos)
+		}
 		// mock for testing
 		common.StandardExec = func(name string, args ...string) ([]byte, error) {
 			var out []byte
@@ -348,12 +360,25 @@ func TestInitialize(t *testing.T) {
 			}
 			return out, err
 		}
+		// Mock IsProw()
+		common.GetOSEnv = func(s string) string {
+			var res string
+			switch s {
+			case "PROW_JOB_ID":
+				if data.isProw {
+					res = "fake_job_id"
+				}
+			default:
+				res = oldEnvFunc(s)
+			}
+			return res
+		}
 
 		err := fgc.Initialize()
 		if !reflect.DeepEqual(err, data.expErr) || !reflect.DeepEqual(fgc.Project, data.expProj) || !reflect.DeepEqual(fgc.Cluster, data.expCluster) {
-			t.Errorf("test initialize with:\n\tpreset project: '%v'\n\tkubeconfig set: '%v'\n\tgcloud set: '%v'\n"+
+			t.Errorf("test initialize with:\n\tuser defined project: '%v'\n\tkubeconfig set: '%v'\n\tgcloud set: '%v'\n\trunning in prow: '%v'\n\tboskos set: '%v'\n"+
 				"want:\n\tproject - '%v'\n\tcluster - '%v'\n\terr - '%v'\ngot:\n\tproject - '%v'\n\tcluster - '%v'\n\terr - '%v'",
-				data.project, data.clusterExist, data.gcloudSet, data.expProj, data.expCluster, data.expErr, fgc.Project, fgc.Cluster, err)
+				data.project, data.clusterExist, data.gcloudSet, data.isProw, data.boskosProjs, data.expProj, data.expCluster, data.expErr, fgc.Project, fgc.Cluster, err)
 		}
 	}
 }