Merge pull request gardener#2 from amshuman-kr/grpc2

More refactoring. Framework for external driver client. Unit tests.

cmd/machine-controller-manager/app/options/options.go

@@ -96,6 +96,9 @@ func (s *MCMServer) AddFlags(fs *pflag.FlagSet) {
 	fs.Int32Var(&s.SafetyOptions.MachineDrainTimeout, "machine-drain-timeout", s.SafetyOptions.MachineDrainTimeout, "Timeout (in minutes) used while draining of machine before deletion, beyond which it forcefully deletes machine")
 	fs.Int32Var(&s.SafetyOptions.MachineSetScaleTimeout, "machine-set-scale-timeout", s.SafetyOptions.MachineSetScaleTimeout, "Timeout (in minutes) used while scaling machineSet if timeout occurs machineSet is permanently frozen")
 
+	fs.BoolVar(&s.ExternalDriverManagerOptions.Enabled, "external-driver-manager-enabled", s.ExternalDriverManagerOptions.Enabled, "If true, enable support for external drivers")
+	fs.Uint16Var(&s.ExternalDriverManagerOptions.Port, "external-driver-manager-port", s.ExternalDriverManagerOptions.Port, "Port on which to listen for external drivers")
+
 	leaderelectionconfig.BindFlags(&s.LeaderElection, fs)
 	// TODO: DefaultFeatureGate is global and it adds all k8s flags
 	// utilfeature.DefaultFeatureGate.AddFlag(fs)

cmd/machine-controller-manager/controller_manager.go

@@ -27,6 +27,8 @@ import (
 
 	"github.com/gardener/machine-controller-manager/cmd/machine-controller-manager/app"
 	"github.com/gardener/machine-controller-manager/cmd/machine-controller-manager/app/options"
+	"github.com/gardener/machine-controller-manager/pkg/driver"
+	"github.com/gardener/machine-controller-manager/pkg/grpc/infraserver"
 	_ "github.com/gardener/machine-controller-manager/pkg/util/client/metrics/prometheus" // for client metric registration
 	_ "github.com/gardener/machine-controller-manager/pkg/util/reflector/prometheus"      // for reflector metric registration
 	_ "github.com/gardener/machine-controller-manager/pkg/util/workqueue/prometheus"      // for workqueue metric registration
@@ -53,6 +55,14 @@ func main() {
 
 	// verflag.PrintAndExitIfRequested()
 
+	if s.ExternalDriverManagerOptions.Enabled {
+		externalDriverManager := &infraserver.ExternalDriverManager{
+			Port: s.ExternalDriverManagerOptions.Port,
+		}
+		driver.ExternalDriverManager = externalDriverManager
+		externalDriverManager.Start()
+	}
+
 	if err := app.Run(s); err != nil {
 		fmt.Fprintf(os.Stderr, "%v\n", err)
 		os.Exit(1)

hack/fakegrpcserver/fakegrpcserver.go

@@ -1,9 +1,60 @@
 package main
 
 import (
+	"log"
+	"path"
+	"time"
+
 	svr "github.com/gardener/machine-controller-manager/pkg/grpc/infraserver"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/testdata"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var (
+	tls      = false //"Connection uses TLS if true, else plain TCP"
+	certFile = ""    //"The TLS cert file"
+	keyFile  = ""    //"The TLS key file"
+	port     = 10000 //"The server port"
 )
 
 func main() {
-	svr.StartServer()
+	server := &svr.ExternalDriverManager{
+		Port:   port,
+		UseTLS: tls,
+	}
+	if tls {
+		if certFile == "" {
+			certFile = testdata.Path("server1.pem")
+		}
+		if keyFile == "" {
+			keyFile = testdata.Path("server1.key")
+		}
+		creds, err := credentials.NewServerTLSFromFile(certFile, keyFile)
+		if err != nil {
+			log.Fatalf("Failed to generate credentials %v", err)
+		}
+		server.Options = []grpc.ServerOption{grpc.Creds(creds)}
+	}
+
+	server.Start()
+
+	// Test go routine to test end to end flow
+	go func() {
+		for {
+			driver, _ := server.GetDriver(metav1.TypeMeta{
+				Kind:       "driver",
+				APIVersion: path.Join("poc", "alpha"),
+			})
+
+			time.Sleep(5 * time.Second)
+			log.Printf("Calling create")
+			driver.Create("fakeDriver", "a", "b")
+
+			time.Sleep(5 * time.Second)
+			log.Printf("Calling delete")
+			driver.Delete("fakeDriver", "a", "b")
+		}
+	}()
 }

pkg/controller/machine.go

@@ -127,7 +127,7 @@ func (c *controller) reconcileClusterMachine(machine *v1alpha1.Machine) error {
 		return err
 	}
 
-	driver := driver.NewDriver(machine.Spec.ProviderID, secretRef, machine.Spec.Class.Kind, MachineClass, machine.Name)
+	driver := driver.NewDriver(machine.Spec.ProviderID, secretRef, &machine.Spec.Class, MachineClass, machine.Name)
 	actualProviderID, err := driver.GetExisting()
 	if err != nil {
 		return err

pkg/controller/machine_safety.go

@@ -231,7 +231,10 @@ func (c *controller) checkAWSMachineClass() {
 			machineClassInterface,
 			machineClass.Spec.SecretRef,
 			machineClass.Name,
-			machineClass.Kind,
+			&v1alpha1.ClassSpec{
+				APIGroup: machineClass.TypeMeta.GroupVersionKind().GroupVersion().String(),
+				Kind:     machineClass.Kind,
+			},
 		)
 	}
 }
@@ -253,7 +256,10 @@ func (c *controller) checkOSMachineClass() {
 			machineClassInterface,
 			machineClass.Spec.SecretRef,
 			machineClass.Name,
-			machineClass.Kind,
+			&v1alpha1.ClassSpec{
+				APIGroup: machineClass.TypeMeta.GroupVersionKind().GroupVersion().String(),
+				Kind:     machineClass.Kind,
+			},
 		)
 	}
 }
@@ -275,7 +281,10 @@ func (c *controller) checkAzureMachineClass() {
 			machineClassInterface,
 			machineClass.Spec.SecretRef,
 			machineClass.Name,
-			machineClass.Kind,
+			&v1alpha1.ClassSpec{
+				APIGroup: machineClass.TypeMeta.GroupVersionKind().GroupVersion().String(),
+				Kind:     machineClass.Kind,
+			},
 		)
 	}
 }
@@ -297,7 +306,10 @@ func (c *controller) checkGCPMachineClass() {
 			machineClassInterface,
 			machineClass.Spec.SecretRef,
 			machineClass.Name,
-			machineClass.Kind,
+			&v1alpha1.ClassSpec{
+				APIGroup: machineClass.TypeMeta.GroupVersionKind().GroupVersion().String(),
+				Kind:     machineClass.Kind,
+			},
 		)
 	}
 }
@@ -307,7 +319,7 @@ func (c *controller) checkMachineClass(
 	machineClass interface{},
 	secretRef *corev1.SecretReference,
 	className string,
-	classKind string) {
+	class *v1alpha1.ClassSpec) {
 
 	// Get secret
 	secret, err := c.getSecret(secretRef, className)
@@ -320,7 +332,7 @@ func (c *controller) checkMachineClass(
 	dvr := driver.NewDriver(
 		"",
 		secret,
-		classKind,
+		class,
 		machineClass,
 		"",
 	)
@@ -363,7 +375,7 @@ func (c *controller) checkMachineClass(
 					if (err != nil && apierrors.IsNotFound(err)) || machine.Spec.ProviderID != machineID {
 						vm := make(map[string]string)
 						vm[machineID] = machineName
-						c.deleteOrphanVM(vm, secret, classKind, machineClass)
+						c.deleteOrphanVM(vm, secret, class, machineClass)
 					}
 				}
 			}
@@ -424,7 +436,7 @@ func (c *controller) enqueueMachineSafetyKey(obj interface{}) {
 }
 
 // deleteOrphanVM teriminates's the VM on the cloud provider passed to it
-func (c *controller) deleteOrphanVM(vm driver.VMs, secretRef *corev1.Secret, kind string, machineClass interface{}) {
+func (c *controller) deleteOrphanVM(vm driver.VMs, secretRef *corev1.Secret, class *v1alpha1.ClassSpec, machineClass interface{}) {
 
 	var machineID string
 	var machineName string
@@ -437,7 +449,7 @@ func (c *controller) deleteOrphanVM(vm driver.VMs, secretRef *corev1.Secret, kin
 	dvr := driver.NewDriver(
 		machineID,
 		secretRef,
-		kind,
+		class,
 		machineClass,
 		machineName,
 	)

pkg/driver/driver.go

@@ -19,7 +19,9 @@ package driver
 
 import (
 	"github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1"
+	"github.com/gardener/machine-controller-manager/pkg/grpc/infraserver"
 	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 // Driver is the common interface for creation/deletion of the VMs over different cloud-providers.
@@ -35,10 +37,13 @@ type Driver interface {
 // value refers to the machine-name of the machine object
 type VMs map[string]string
 
+// ExternalDriverManager is the central manager of all the external drivers.
+var ExternalDriverManager *infraserver.ExternalDriverManager
+
 // NewDriver creates a new driver object based on the classKind
-func NewDriver(machineID string, secretRef *corev1.Secret, classKind string, machineClass interface{}, machineName string) Driver {
+func NewDriver(machineID string, secretRef *corev1.Secret, class *v1alpha1.ClassSpec, machineClass interface{}, machineName string) Driver {
 
-	switch classKind {
+	switch class.Kind {
 	case "OpenStackMachineClass":
 		return &OpenStackDriver{
 			OpenStackMachineClass: machineClass.(*v1alpha1.OpenStackMachineClass),
@@ -76,6 +81,16 @@ func NewDriver(machineID string, secretRef *corev1.Secret, classKind string, mac
 		}
 	}
 
+	if ExternalDriverManager != nil {
+		external, err := ExternalDriverManager.GetDriver(metav1.TypeMeta{
+			APIVersion: class.APIGroup,
+			Kind:       class.Kind,
+		})
+		if err != nil {
+			return NewExternalDriver(external, machineClass, secretRef, string(secretRef.Data["userData"]), machineID, machineName)
+		}
+	}
+
 	return NewFakeDriver(
 		func() (string, string, error) {
 			return "fake", "fake_ip", nil

pkg/driver/driver_external.go

@@ -20,11 +20,13 @@ package driver
 import (
 	corev1 "k8s.io/api/core/v1"
 
+	"github.com/gardener/machine-controller-manager/pkg/grpc/infraserver"
 	"github.com/golang/glog"
 )
 
 // ExternalDriver implements the support for out-of-tree drivers
 type ExternalDriver struct {
+	driver       infraserver.Driver
 	machineClass interface{}
 	credentials  *corev1.Secret
 	userData     string
@@ -33,13 +35,26 @@ type ExternalDriver struct {
 }
 
 // NewExternalDriver returns an empty AWSDriver object
-func NewExternalDriver(machineClass interface{}, credentials *corev1.Secret, userData, machineID, machineName string) Driver {
-	return &ExternalDriver{}
+func NewExternalDriver(driver infraserver.Driver, machineClass interface{}, credentials *corev1.Secret, userData, machineID, machineName string) Driver {
+	return &ExternalDriver{
+		driver:       driver,
+		machineClass: machineClass,
+		credentials:  nil, //TODO
+		userData:     userData,
+		machineID:    machineID,
+		machineName:  machineName,
+	}
 }
 
 // Create method is used to create a machine
 func (d *ExternalDriver) Create() (string, string, error) {
-	return "", "", nil
+	//TODO
+	machineID, machineName, err := d.driver.Create(nil, "", d.machineID, d.machineName)
+	if err != nil {
+		d.machineID = machineID
+		d.machineName = machineName
+	}
+	return machineID, machineName, err
 }
 
 // Delete method is used to delete a AWS machine
@@ -53,7 +68,11 @@ func (d *ExternalDriver) Delete() error {
 		return nil
 	}
 
-	glog.Errorf("Could not terminate machine: %s", err.Error())
+	//TODO
+	err = d.driver.Delete("", d.machineID)
+	if err != nil {
+		glog.Errorf("Could not terminate machine: %s", err.Error())
+	}
 	return err
 }
 
@@ -65,5 +84,6 @@ func (d *ExternalDriver) GetExisting() (string, error) {
 // GetVMs returns a VM matching the machineID
 // If machineID is an empty string then it returns all matching instances
 func (d *ExternalDriver) GetVMs(machineID string) (VMs, error) {
+	//TODO
 	return nil, nil
 }