WiP: gcp: implementing provisioning interface

This will create cluster and vpc within GCP. Signed-off-by: Beraldo Leal <[email protected]>
confidential-containers · Jun 27, 2024 · 60a5518 · 60a5518
1 parent b560ca8
commit 60a5518
Show file tree

Hide file tree

Showing 5 changed files with 531 additions and 2 deletions.
diff --git a/src/cloud-api-adaptor/go.mod b/src/cloud-api-adaptor/go.mod
@@ -58,6 +58,7 @@ require (
 	github.com/tj/assert v0.0.3
 	golang.org/x/crypto v0.23.0
 	golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2
+	google.golang.org/api v0.162.0
 	google.golang.org/protobuf v1.33.0
 	k8s.io/api v0.26.2
 	k8s.io/apimachinery v0.26.2
@@ -191,10 +192,8 @@ require (
 	golang.org/x/sync v0.6.0 // indirect
 	golang.org/x/term v0.20.0 // indirect
 	golang.org/x/text v0.15.0 // indirect
-	golang.org/x/term v0.19.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/tools v0.14.0 // indirect
-	google.golang.org/api v0.162.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240213162025-012b6fc9bca9 // indirect

diff --git a/src/cloud-api-adaptor/test/provisioner/gcp/cluster.go b/src/cloud-api-adaptor/test/provisioner/gcp/cluster.go
@@ -0,0 +1,269 @@
+// (C) Copyright Confidential Containers Contributors
+// SPDX-License-Identifier: Apache-2.0
+
+package gcp
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+	"google.golang.org/api/container/v1"
+	"google.golang.org/api/googleapi"
+	"google.golang.org/api/option"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/util/retry"
+	kconf "sigs.k8s.io/e2e-framework/klient/conf"
+)
+
+// GKECluster implements the basic GKE Cluster client operations.
+type GKECluster struct {
+	clusterName    string
+	clusterVersion string
+	credentials    string
+	machineType    string
+	nodeCount      int64
+	projectID      string
+	zone           string
+}
+
+// NewGKECluster creates a new GKECluster with the given properties
+func NewGKECluster(properties map[string]string) (*GKECluster, error) {
+	defaults := map[string]string{
+		"cluster_name":    "peer-pods",
+		"cluster_version": "1.27.11-gke.1062004",
+		"machine_type":    "n1-standard-1",
+		"node_count":      "2",
+		"zone":            "us-central1-a",
+	}
+
+	for key, value := range properties {
+		defaults[key] = value
+	}
+
+	requiredFields := []string{"project_id", "credentials"}
+	for _, field := range requiredFields {
+		if _, ok := defaults[field]; !ok {
+			return nil, fmt.Errorf("%s is required", field)
+		}
+	}
+
+	nodeCount, err := strconv.ParseInt(defaults["node_count"], 10, 64)
+	if err != nil {
+		return nil, fmt.Errorf("invalid node_count: %v", err)
+	}
+
+	return &GKECluster{
+		clusterName:    defaults["cluster_name"],
+		clusterVersion: defaults["cluster_version"],
+		credentials:    defaults["credentials"],
+		machineType:    defaults["machine_type"],
+		nodeCount:      nodeCount,
+		projectID:      defaults["project_id"],
+		zone:           defaults["zone"],
+	}, nil
+}
+
+// CreateCluster creates the GKE cluster
+func (g *GKECluster) CreateCluster(ctx context.Context) error {
+	ctx, cancel := context.WithTimeout(ctx, time.Hour)
+	defer cancel()
+
+	srv, err := container.NewService(
+		ctx, option.WithCredentialsFile(g.credentials),
+	)
+	if err != nil {
+		return fmt.Errorf("GKE: container.NewService: %v", err)
+	}
+
+	cluster := &container.Cluster{
+		Name:             g.clusterName,
+		InitialNodeCount: g.nodeCount,
+		NodeConfig: &container.NodeConfig{
+			MachineType: g.machineType,
+		},
+	}
+
+	req := &container.CreateClusterRequest{
+		Cluster: cluster,
+	}
+
+	op, err := srv.Projects.Zones.Clusters.Create(
+		g.projectID, g.zone, req,
+	).Context(ctx).Do()
+	if err != nil {
+		return fmt.Errorf("GKE: Projects.Zones.Clusters.Create: %v", err)
+	}
+
+	log.Infof("GKE: Cluster creation operation: %v\n", op.Name)
+
+	err = g.WaitForClusterActive(ctx, 30*time.Minute)
+	if err != nil {
+		return fmt.Errorf("GKE: Error waiting for cluster to become active: %v", err)
+	}
+
+	err = g.ApplyNodeLabels(ctx)
+	if err != nil {
+		return fmt.Errorf("GKE: Error applying node labels: %v", err)
+	}
+	return nil
+}
+
+// DeleteCluster deletes the GKE cluster
+func (g *GKECluster) DeleteCluster(ctx context.Context) error {
+	ctx, cancel := context.WithTimeout(ctx, time.Hour)
+	defer cancel()
+
+	srv, err := container.NewService(
+		ctx, option.WithCredentialsFile(g.credentials),
+	)
+	if err != nil {
+		return fmt.Errorf("GKE: container.NewService: %v", err)
+	}
+
+	op, err := srv.Projects.Zones.Clusters.Delete(
+		g.projectID, g.zone, g.clusterName,
+	).Context(ctx).Do()
+	if err != nil {
+		return fmt.Errorf("GKE: Projects.Zones.Clusters.Delete: %v", err)
+	}
+
+	log.Infof("GKE: Cluster deletion operation: %v\n", op.Name)
+
+	// Wait for the cluster to be deleted
+	activationTimeout := 30 * time.Minute
+	err = g.WaitForClusterDeleted(ctx, activationTimeout)
+	if err != nil {
+		return fmt.Errorf("GKE: error waiting for cluster to be deleted: %v", err)
+	}
+	return nil
+}
+
+// WaitForClusterActive waits until the GKE cluster is active
+func (g *GKECluster) WaitForClusterActive(
+	ctx context.Context, activationTimeout time.Duration,
+) error {
+	srv, err := container.NewService(
+		ctx, option.WithCredentialsFile(g.credentials),
+	)
+	if err != nil {
+		return fmt.Errorf("GKE: container.NewService: %v", err)
+	}
+
+	timeoutCtx, cancel := context.WithTimeout(ctx, activationTimeout)
+	defer cancel()
+
+	ticker := time.NewTicker(5 * time.Second)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-timeoutCtx.Done():
+			return fmt.Errorf("GKE: Reached timeout waiting for cluster.")
+		case <-ticker.C:
+			cluster, err := srv.Projects.Zones.Clusters.Get(g.projectID, g.zone, g.clusterName).Context(ctx).Do()
+			if err != nil {
+				return fmt.Errorf("GKE: Projects.Zones.Clusters.Get: %v", err)
+			}
+
+			if cluster.Status == "RUNNING" {
+				log.Info("GKE: Cluster is now active")
+				return nil
+			}
+
+			log.Info("GKE: Waiting for cluster to become active...")
+		}
+	}
+}
+
+// WaitForClusterDeleted waits until the GKE cluster is deleted
+func (g *GKECluster) WaitForClusterDeleted(
+	ctx context.Context, activationTimeout time.Duration,
+) error {
+	srv, err := container.NewService(
+		ctx, option.WithCredentialsFile(g.credentials),
+	)
+	if err != nil {
+		return fmt.Errorf("GKE: container.NewService: %v", err)
+	}
+
+	timeoutCtx, cancel := context.WithTimeout(ctx, activationTimeout)
+	defer cancel()
+
+	ticker := time.NewTicker(5 * time.Second)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-timeoutCtx.Done():
+			return fmt.Errorf("GKE: timeout waiting for cluster deletion")
+		case <-ticker.C:
+			_, err := srv.Projects.Zones.Clusters.Get(g.projectID, g.zone, g.clusterName).Context(ctx).Do()
+			if err != nil {
+				if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 404 {
+					log.Info("GKE: Cluster deleted successfully")
+					return nil
+				}
+				return fmt.Errorf("GKE: Projects.Zones.Clusters.Get: %v", err)
+			}
+
+			log.Info("GKE: Waiting for cluster to be deleted...")
+		}
+	}
+}
+
+// GetKubeconfigFile looks for the kubeconfig on the default locations
+func (g *GKECluster) GetKubeconfigFile() (string, error) {
+	kubeconfigPath := kconf.ResolveKubeConfigFile()
+	if kubeconfigPath == "" {
+		return "", fmt.Errorf("Unabled to find a kubeconfig file")
+	}
+
+	return kubeconfigPath, nil
+}
+
+func (g *GKECluster) ApplyNodeLabels(ctx context.Context) error {
+	kubeconfigPath, err := g.GetKubeconfigFile()
+	if err != nil {
+		return err
+	}
+
+	config, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath)
+	if err != nil {
+		return fmt.Errorf("failed to build kubeconfig: %v", err)
+	}
+
+	clientset, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return fmt.Errorf("failed to create clientset: %v", err)
+	}
+
+	nodes, err := clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to list nodes: %v", err)
+	}
+
+	for _, node := range nodes.Items {
+		err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
+			n, err := clientset.CoreV1().Nodes().Get(ctx, node.Name, metav1.GetOptions{})
+			if err != nil {
+				return fmt.Errorf("failed to get node: %v", err)
+			}
+
+			n.Labels["node.kubernetes.io/worker"] = ""
+			n.Labels["node-role.kubernetes.io/worker"] = ""
+			_, err = clientset.CoreV1().Nodes().Update(ctx, n, metav1.UpdateOptions{})
+			return err
+		})
+		if err != nil {
+			fmt.Printf("Failed to label node %s: %v\n", node.Name, err)
+		} else {
+			fmt.Printf("Successfully labeled node %s\n", node.Name)
+		}
+	}
+	return nil
+}
diff --git a/src/cloud-api-adaptor/test/provisioner/gcp/provision.go b/src/cloud-api-adaptor/test/provisioner/gcp/provision.go
@@ -0,0 +1,15 @@
+//go:build gcp
+
+// (C) Copyright Confidential Containers Contributors
+// SPDX-License-Identifier: Apache-2.0
+
+package gcp
+
+import (
+	pv "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/test/provisioner"
+)
+
+func init() {
+	pv.NewProvisionerFunctions["gcp"] = NewGCPProvisioner
+	pv.NewInstallOverlayFunctions["gcp"] = NewGCPInstallOverlay
+}
diff --git a/src/cloud-api-adaptor/test/provisioner/gcp/provision_common.go b/src/cloud-api-adaptor/test/provisioner/gcp/provision_common.go
@@ -0,0 +1,77 @@
+// (C) Copyright Confidential Containers Contributors
+// SPDX-License-Identifier: Apache-2.0
+
+package gcp
+
+import (
+	"context"
+
+	pv "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/test/provisioner"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+)
+
+var GCPProps = &GCPProvisioner{}
+
+// GCPProvisioner implements the CloudProvisioner interface.
+type GCPProvisioner struct {
+	GkeCluster *GKECluster
+	GcpVPC     *GCPVPC
+}
+
+// NewGCPProvisioner creates a new GCPProvisioner with the given properties.
+func NewGCPProvisioner(properties map[string]string) (pv.CloudProvisioner, error) {
+	gkeCluster, err := NewGKECluster(properties)
+	if err != nil {
+		return nil, err
+	}
+
+	gcpVPC, err := NewGCPVPC(properties)
+	if err != nil {
+		return nil, err
+	}
+
+	GCPProps = &GCPProvisioner{
+		GkeCluster: gkeCluster,
+		GcpVPC:     gcpVPC,
+	}
+	return GCPProps, nil
+}
+
+// CreateCluster creates a new GKE cluster.
+func (p *GCPProvisioner) CreateCluster(ctx context.Context, cfg *envconf.Config) error {
+	err := p.GkeCluster.CreateCluster(ctx)
+	if err != nil {
+		return err
+	}
+
+	kubeconfigPath, err := p.GkeCluster.GetKubeconfigFile()
+	if err != nil {
+		return err
+	}
+	*cfg = *envconf.NewWithKubeConfig(kubeconfigPath)
+
+	return nil
+}
+
+// CreateVPC creates a new VPC in Google Cloud.
+func (p *GCPProvisioner) CreateVPC(ctx context.Context, cfg *envconf.Config) error {
+	return p.GcpVPC.CreateVPC(ctx, cfg)
+}
+
+// DeleteCluster deletes the GKE cluster.
+func (p *GCPProvisioner) DeleteCluster(ctx context.Context, cfg *envconf.Config) error {
+	return p.GkeCluster.DeleteCluster(ctx)
+}
+
+// DeleteVPC deletes the VPC in Google Cloud.
+func (p *GCPProvisioner) DeleteVPC(ctx context.Context, cfg *envconf.Config) error {
+	return p.GcpVPC.DeleteVPC(ctx, cfg)
+}
+
+func (p *GCPProvisioner) GetProperties(ctx context.Context, cfg *envconf.Config) map[string]string {
+	return map[string]string{"TO BE IMPLEMENTED": "TO BE IMPLEMENTED"}
+}
+
+func (p *GCPProvisioner) UploadPodvm(imagePath string, ctx context.Context, cfg *envconf.Config) error {
+	return nil
+}