diff --git a/pkg/client/client.go b/pkg/client/client.go index 3ce8b9d..2c0931e 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -28,6 +28,12 @@ type userpassword struct { Tokens []string `json:"tokens,omitempty"` } +type Server struct { + Name string + ProviderID string + DatacenterID string +} + func New(datacenterId string, secret []byte) (IONOSClient, error) { var cfg *ionoscloud.Configuration if secret[0] == '{' { @@ -65,6 +71,119 @@ func (a *IONOSClient) GetServer(ctx context.Context, providerID string) (*cloudp return a.convertServerToInstanceMetadata(ctx, &server) } +func (a *IONOSClient) RemoveIPFromNode(ctx context.Context, loadBalancerIP, providerID string) error { + if a.client == nil { + return errors.New("client isn't initialized") + } + + serverReq := a.client.NetworkInterfacesApi.DatacentersServersNicsGet(ctx, a.DatacenterId, providerID) + nics, req, err := serverReq.Depth(3).Execute() + if err != nil { + if req != nil && req.StatusCode == 404 { + return nil + } + return err + } + + if !nics.HasItems() { + return errors.New("node has no nics") + } + + primaryNic := getPrimaryNic(*nics.Items) + ips := *primaryNic.Properties.Ips + + for idx, v := range ips { + if v == loadBalancerIP { + ips = append(ips[:idx], ips[idx+1:]...) + } + } + + _, _, err = a.client.NetworkInterfacesApi.DatacentersServersNicsPatch(ctx, a.DatacenterId, providerID, *primaryNic.Id).Nic(ionoscloud.NicProperties{ + Ips: &ips, + }).Execute() + + return err +} + +func getPrimaryNic(nics []ionoscloud.Nic) *ionoscloud.Nic { + for _, nic := range nics { + if *nic.Properties.PciSlot == 6 { + return &nic + } + } + return nil +} + +func (a *IONOSClient) AttachIPToNode(ctx context.Context, loadBalancerIP, providerID string) (bool, error) { + if a.client == nil { + return false, errors.New("client isn't initialized") + } + + serverReq := a.client.NetworkInterfacesApi.DatacentersServersNicsGet(ctx, a.DatacenterId, providerID) + nics, req, err := serverReq.Depth(3).Execute() + if err != nil { + if req != nil && req.StatusCode == 404 { + return false, nil + } + return false, err + } + + if !nics.HasItems() { + return false, errors.New("node has no nics") + } + + primaryNic := getPrimaryNic(*nics.Items) + ips := *primaryNic.Properties.Ips + ips = append(ips, loadBalancerIP) + + _, _, err = a.client.NetworkInterfacesApi.DatacentersServersNicsPatch(ctx, a.DatacenterId, providerID, *primaryNic.Id).Nic(ionoscloud.NicProperties{ + Ips: &ips, + }).Execute() + + return true, err +} + +func (a *IONOSClient) GetServerByIP(ctx context.Context, loadBalancerIP string) (*Server, error) { + if a.client == nil { + return nil, errors.New("client isn't initialized") + } + + serverReq := a.client.ServersApi.DatacentersServersGet(ctx, a.DatacenterId) + servers, _, err := serverReq.Depth(3).Execute() + if err != nil { + return nil, err + } + + if !servers.HasItems() { + return nil, nil + } + + for _, server := range *servers.Items { + klog.Infof("Checking server %s", server.Properties.Name) + if !server.Entities.HasNics() { + continue + } + for _, nic := range *server.Entities.Nics.Items { + if nic.Properties.HasIps() { + for _, ip := range *nic.Properties.Ips { + klog.Infof("Found ip %s", ip) + if loadBalancerIP == ip { + klog.Info("Its a match!") + return &Server{ + Name: *server.Properties.Name, + ProviderID: *server.Id, + DatacenterID: a.DatacenterId, + }, nil + } + } + } + } + } + klog.Infof("IP %s not found on any node in datacenter %s", loadBalancerIP, a.DatacenterId) + + return nil, nil +} + func (a *IONOSClient) datacenterLocation(ctx context.Context) (string, error) { if a.client == nil { return "", errors.New("client isn't initialized") diff --git a/pkg/ionos/cloud.go b/pkg/ionos/cloud.go index 4bd2cc0..6be6b1c 100644 --- a/pkg/ionos/cloud.go +++ b/pkg/ionos/cloud.go @@ -13,6 +13,7 @@ import ( func init() { cloudprovider.RegisterCloudProvider(config.RegisteredProviderName, func(cfg io.Reader) (cloudprovider.Interface, error) { + byConfig, err := io.ReadAll(cfg) if err != nil { klog.Errorf("ReadAll failed: %s", err) @@ -34,19 +35,22 @@ func newProvider(config config.Config) cloudprovider.Interface { return IONOS{ config: config, instances: instances{ - clients: map[string]*client2.IONOSClient{}, + ionosClients: map[string]*client2.IONOSClient{}, + }, + loadbalancer: loadbalancer{ + ionosClients: map[string]*client2.IONOSClient{}, }, } } func (p IONOS) Initialize(clientBuilder cloudprovider.ControllerClientBuilder, _ <-chan struct{}) { ctx := context.Background() - client, err := clientBuilder.Client(config.ClientName) + k8sClient, err := clientBuilder.Client(config.ClientName) if err != nil { klog.Errorf("Kubernetes Client Init Failed: %v", err) return } - secret, err := client.CoreV1().Secrets(p.config.TokenSecretNamespace).Get(ctx, p.config.TokenSecretName, metav1.GetOptions{}) + secret, err := k8sClient.CoreV1().Secrets(p.config.TokenSecretNamespace).Get(ctx, p.config.TokenSecretName, metav1.GetOptions{}) if err != nil { klog.Errorf("Failed to get secret %s/%s: %v", p.config.TokenSecretNamespace, p.config.TokenSecretName, err) return @@ -58,12 +62,17 @@ func (p IONOS) Initialize(clientBuilder cloudprovider.ControllerClientBuilder, _ klog.Errorf("Failed to create client for datacenter %s: %v", key, err) return } + + err = p.loadbalancer.AddClient(key, token) + if err != nil { + klog.Errorf("Failed to create client for datacenter %s: %v", key, err) + return + } } } func (p IONOS) LoadBalancer() (cloudprovider.LoadBalancer, bool) { - klog.Warning("The IONOS cloud provider does not support load balancers") - return nil, false + return p.loadbalancer, true } func (p IONOS) Instances() (cloudprovider.Instances, bool) { diff --git a/pkg/ionos/instances.go b/pkg/ionos/instances.go index 9a9d8e0..39908e9 100644 --- a/pkg/ionos/instances.go +++ b/pkg/ionos/instances.go @@ -23,19 +23,19 @@ func GetUUIDFromNode(node *v1.Node) string { } func (i instances) AddClient(datacenterId string, token []byte) error { - if i.clients[datacenterId] == nil { + if i.ionosClients[datacenterId] == nil { c, err := client2.New(datacenterId, token) if err != nil { return err } - i.clients[datacenterId] = &c + i.ionosClients[datacenterId] = &c } return nil } // no caching func (i instances) discoverNode(ctx context.Context, node *v1.Node) (*cloudprovider.InstanceMetadata, error) { - for _, client := range i.clients { + for _, client := range i.ionosClients { var err error var server *cloudprovider.InstanceMetadata providerID := GetUUIDFromNode(node) diff --git a/pkg/ionos/loadbalancer.go b/pkg/ionos/loadbalancer.go new file mode 100644 index 0000000..c83a607 --- /dev/null +++ b/pkg/ionos/loadbalancer.go @@ -0,0 +1,249 @@ +package ionos + +import ( + "context" + "errors" + client2 "github.com/GDATASoftwareAG/cloud-provider-ionoscloud/pkg/client" + v1 "k8s.io/api/core/v1" + cloudprovider "k8s.io/cloud-provider" + "k8s.io/klog/v2" + "math/rand" + "strings" + "time" +) + +var _ cloudprovider.LoadBalancer = &loadbalancer{} + +// see https://github.com/kubernetes/kubernetes/blob/v1.18.0/pkg/controller/service/controller.go + +func (l loadbalancer) AddClient(datacenterId string, token []byte) error { + if l.ionosClients[datacenterId] == nil { + c, err := client2.New(datacenterId, token) + if err != nil { + return err + } + l.ionosClients[datacenterId] = &c + } + return nil +} + +// GetLoadBalancer returns whether the specified load balancer exists, and +// if so, what its status is. +// Implementations must treat the *v1.Service parameter as read-only and not modify it. +// Parameter 'clusterName' is the name of the cluster as presented to kube-controller-manager. +// For the given LB service, the GetLoadBalancer must return "exists=True" if +// there exists a LoadBalancer instance created by ServiceController. +// In all other cases, GetLoadBalancer must return a NotFound error. +func (l loadbalancer) GetLoadBalancer(ctx context.Context, clusterName string, service *v1.Service) (status *v1.LoadBalancerStatus, exists bool, err error) { + klog.Infof("getLoadBalancer (service %s/%s)", service.Namespace, service.Name) + + server, err := l.ServerWithLoadBalancer(ctx, service.Spec.LoadBalancerIP) + if err != nil { + return nil, false, err + } + + if server != nil { + return &v1.LoadBalancerStatus{Ingress: []v1.LoadBalancerIngress{{IP: service.Spec.LoadBalancerIP}}}, true, nil + } + + return nil, false, nil +} + +// GetLoadBalancerName returns the name of the load balancer. Implementations must treat the +// *v1.Service parameter as read-only and not modify it. +func (l loadbalancer) GetLoadBalancerName(ctx context.Context, clusterName string, service *v1.Service) string { + return cloudprovider.DefaultLoadBalancerName(service) +} + +// EnsureLoadBalancer creates a new load balancer 'name', or updates the existing one. Returns the status of the balancer +// Implementations must treat the *v1.Service and *v1.Node +// parameters as read-only and not modify them. +// Parameter 'clusterName' is the name of the cluster as presented to kube-controller-manager. +// +// Implementations may return a (possibly wrapped) api.RetryError to enforce +// backing off at a fixed duration. This can be used for cases like when the +// load balancer is not ready yet (e.g., it is still being provisioned) and +// polling at a fixed rate is preferred over backing off exponentially in +// order to minimize latency. +func (l loadbalancer) EnsureLoadBalancer(ctx context.Context, clusterName string, service *v1.Service, nodes []*v1.Node) (*v1.LoadBalancerStatus, error) { + return l.syncLoadBalancer(ctx, clusterName, service, nodes) +} + +// UpdateLoadBalancer updates hosts under the specified load balancer. +// Implementations must treat the *v1.Service and *v1.Node +// parameters as read-only and not modify them. +// Parameter 'clusterName' is the name of the cluster as presented to kube-controller-manager +func (l loadbalancer) UpdateLoadBalancer(ctx context.Context, clusterName string, service *v1.Service, nodes []*v1.Node) error { + _, err := l.syncLoadBalancer(ctx, clusterName, service, nodes) + return err +} + +// EnsureLoadBalancerDeleted deletes the specified load balancer if it +// exists, returning nil if the load balancer specified either didn't exist or +// was successfully deleted. +// This construction is useful because many cloud providers' load balancers +// have multiple underlying components, meaning a Get could say that the LB +// doesn't exist even if some part of it is still laying around. +// Implementations must treat the *v1.Service parameter as read-only and not modify it. +// Parameter 'clusterName' is the name of the cluster as presented to kube-controller-manager +// EnsureLoadBalancerDeleted must not return ImplementedElsewhere to ensure +// proper teardown of resources that were allocated by the ServiceController. +func (l loadbalancer) EnsureLoadBalancerDeleted(ctx context.Context, clusterName string, service *v1.Service) error { + klog.Infof("ensureLoadBalancerDeleted (service %s/%s)", service.Namespace, service.Name) + + if len(service.Status.LoadBalancer.Ingress) > 0 { + klog.Infof("removing IP %s", service.Status.LoadBalancer.Ingress[0].IP) + server, err := l.ServerWithLoadBalancer(ctx, service.Status.LoadBalancer.Ingress[0].IP) + if err != nil { + return err + } + + if server != nil { + if err := l.deleteLoadBalancerFromNode(ctx, service.Status.LoadBalancer.Ingress[0].IP, server); err != nil { + return err + } + } + } + + return nil +} + +func (l loadbalancer) deleteLoadBalancerFromNode(ctx context.Context, loadBalancerIP string, server *client2.Server) error { + for _, client := range l.ionosClients { + if client.DatacenterId != server.DatacenterID { + continue + } + + server, err := client.GetServerByIP(ctx, loadBalancerIP) + if err != nil { + return err + } + + if server != nil { + return client.RemoveIPFromNode(ctx, loadBalancerIP, server.ProviderID) + } + } + + klog.Infof("IP %s not found in any datacenter", loadBalancerIP) + return nil +} + +func (l loadbalancer) syncLoadBalancer(ctx context.Context, clusterName string, service *v1.Service, nodes []*v1.Node) (*v1.LoadBalancerStatus, error) { + klog.Infof("syncLoadBalancer (service %s/%s, nodes %s)", service.Namespace, service.Name, nodes) + + if len(service.Status.LoadBalancer.Ingress) > 0 && service.Status.LoadBalancer.Ingress[0].IP != service.Spec.LoadBalancerIP { + klog.Infof("service %s/%s changed IP from %s to %s", service.Namespace, service.Name, service.Status.LoadBalancer.Ingress[0].IP, service.Spec.LoadBalancerIP) + server, err := l.ServerWithLoadBalancer(ctx, service.Status.LoadBalancer.Ingress[0].IP) + if err != nil { + return nil, err + } + + if server != nil { + if err := l.deleteLoadBalancerFromNode(ctx, service.Status.LoadBalancer.Ingress[0].IP, server); err != nil { + return nil, err + } + } + } + + if service.Spec.LoadBalancerIP == "" { + return nil, errors.New("we are only handling LoadBalancers with spec.loadBalancerID != ''") + } + + server, err := l.ServerWithLoadBalancer(ctx, service.Spec.LoadBalancerIP) + if err != nil { + return nil, err + } + + if server != nil { + klog.Infof("found server %s has IP %s ", server, service.Spec.LoadBalancerIP) + node := getNode(*server, nodes) + if node == nil { + return nil, errors.New("no node found for server which has loadbalancerIP attached") + } + + if IsLoadBalancerCandidate(node) { + klog.Infof("server %s is valid loadbalancer node", server) + return &v1.LoadBalancerStatus{Ingress: []v1.LoadBalancerIngress{{ + IP: service.Spec.LoadBalancerIP, + }}}, nil + } + } + + loadBalancerNode := l.GetLoadBalancerNode(nodes) + + if loadBalancerNode == nil { + return nil, errors.New("no valid nodes found") + } + klog.Infof("server %s is elected as new loadbalancer node", loadBalancerNode) + + for _, client := range l.ionosClients { + ok, err := client.AttachIPToNode(ctx, service.Spec.LoadBalancerIP, stripProviderFromID(loadBalancerNode.Spec.ProviderID)) + if err != nil { + return nil, err + } + + if ok { + klog.Infof("successfully attached ip %s to server %s", service.Spec.LoadBalancerIP, server) + return &v1.LoadBalancerStatus{Ingress: []v1.LoadBalancerIngress{{ + IP: service.Spec.LoadBalancerIP, + }}}, nil + } + } + + klog.Infof("Could not attach ip %s to any node", service.Spec.LoadBalancerIP) + return nil, nil +} + +func getNode(server client2.Server, nodes []*v1.Node) *v1.Node { + for _, node := range nodes { + if stripProviderFromID(node.Spec.ProviderID) == server.ProviderID { + return node + } + } + return nil +} + +func (l loadbalancer) GetLoadBalancerNode(nodes []*v1.Node) *v1.Node { + var candidates []*v1.Node + for _, node := range nodes { + if IsLoadBalancerCandidate(node) { + candidates = append(candidates, node) + } + } + if candidates == nil && len(candidates) == 0 { + return nil + } + rand.Seed(time.Now().UnixNano()) + randomIndex := rand.Intn(len(candidates)) + return candidates[randomIndex] +} + +func (l loadbalancer) ServerWithLoadBalancer(ctx context.Context, loadBalancerIP string) (*client2.Server, error) { + for _, client := range l.ionosClients { + server, err := client.GetServerByIP(ctx, loadBalancerIP) + if err != nil { + return nil, err + } + + if server != nil { + return server, nil + } + } + + klog.Infof("IP %s not found in any datacenter", loadBalancerIP) + return nil, nil +} + +func IsLoadBalancerCandidate(node *v1.Node) bool { + for _, condition := range node.Status.Conditions { + if condition.Type == v1.NodeReady && condition.Status == v1.ConditionTrue { + return true + } + } + return false +} + +func stripProviderFromID(providerID string) string { + s, _ := strings.CutPrefix(providerID, "ionos://") + return s +} diff --git a/pkg/ionos/types.go b/pkg/ionos/types.go index 5cde4bf..f622eb9 100644 --- a/pkg/ionos/types.go +++ b/pkg/ionos/types.go @@ -6,11 +6,16 @@ import ( ) type IONOS struct { - config config.Config - instances instances - client *client.IONOSClient + config config.Config + instances instances + loadbalancer loadbalancer + client *client.IONOSClient } type instances struct { - clients map[string]*client.IONOSClient + ionosClients map[string]*client.IONOSClient +} + +type loadbalancer struct { + ionosClients map[string]*client.IONOSClient }