cloudprovider: add Bizflycloud provider

cluster-autoscaler/README.md

@@ -25,6 +25,7 @@ You should also take a look at the notes and "gotchas" for your specific cloud p
 * [OVHcloud](./cloudprovider/ovhcloud/README.md)
 * [Linode](./cloudprovider/linode/README.md)
 * [ClusterAPI](./cloudprovider/clusterapi/README.md)
+* [BizflyCloud](./cloudprovider/bizflycloud/README.md)
 
 # Releases
 

cluster-autoscaler/cloudprovider/bizflycloud/README.md

@@ -0,0 +1,12 @@
+# Cluster Autoscaler for Bizflycloud
+
+The cluster autoscaler for Bizflycloud scales worker nodes within any
+specified Bizflycloud Kubernetes Engine cluster's worker pool.
+
+# Configuration
+
+Bizflycloud Kubernetes Engine (BKE) will authenticate with Bizflycloud providers using your application credentials automatics created by BKE.
+
+The scaling option about enable autoscaler, min-nodes, max-nodes will be configure though our dashboard
+
+**Note**: Do not install cluster-autoscaler deployment in manifest since it already install by BKE.

cluster-autoscaler/cloudprovider/bizflycloud/bizflycloud_cloud_provider.go

@@ -0,0 +1,202 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package bizflycloud
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	apiv1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
+	"k8s.io/autoscaler/cluster-autoscaler/config"
+	"k8s.io/autoscaler/cluster-autoscaler/utils/errors"
+	klog "k8s.io/klog/v2"
+)
+
+var _ cloudprovider.CloudProvider = (*bizflycloudCloudProvider)(nil)
+
+const (
+	// GPULabel is the label added to nodes with GPU resource.
+	GPULabel = "bke.bizflycloud.vn/gpu-node"
+
+	bizflyProviderIDPrefix = "bizflycloud://"
+)
+
+// bizflycloudCloudProvider implements CloudProvider interface.
+type bizflycloudCloudProvider struct {
+	manager         *Manager
+	resourceLimiter *cloudprovider.ResourceLimiter
+}
+
+func newBizflyCloudProvider(manager *Manager, rl *cloudprovider.ResourceLimiter) (*bizflycloudCloudProvider, error) {
+	if err := manager.Refresh(); err != nil {
+		return nil, err
+	}
+	return &bizflycloudCloudProvider{
+		manager:         manager,
+		resourceLimiter: rl,
+	}, nil
+}
+
+// Name returns name of the cloud provider.
+func (d *bizflycloudCloudProvider) Name() string {
+	return cloudprovider.BizflyCloudProviderName
+}
+
+// NodeGroups returns all node groups configured for this cloud provider.
+func (d *bizflycloudCloudProvider) NodeGroups() []cloudprovider.NodeGroup {
+	nodeGroups := make([]cloudprovider.NodeGroup, len(d.manager.nodeGroups))
+	for i, ng := range d.manager.nodeGroups {
+		nodeGroups[i] = ng
+	}
+	return nodeGroups
+}
+
+// NodeGroupForNode returns the node group for the given node, nil if the node
+// should not be processed by cluster autoscaler, or non-nil error if such
+// occurred. Must be implemented.
+func (d *bizflycloudCloudProvider) NodeGroupForNode(node *apiv1.Node) (cloudprovider.NodeGroup, error) {
+	providerID := node.Spec.ProviderID
+	nodeID := toNodeID(providerID)
+
+	klog.V(5).Infof("checking nodegroup for node ID: %q", nodeID)
+
+	// NOTE(arslan): the number of node groups per cluster is usually very
+	// small. So even though this looks like quadratic runtime, it's OK to
+	// proceed with this.
+	for _, group := range d.manager.nodeGroups {
+		klog.V(5).Infof("iterating over node group %q", group.Id())
+		nodes, err := group.Nodes()
+		if err != nil {
+			return nil, err
+		}
+
+		for _, node := range nodes {
+			klog.V(6).Infof("checking node has: %q want: %q", node.Id, providerID)
+			// CA uses node.Spec.ProviderID when looking for (un)registered nodes,
+			// so we need to use it here too.
+			if node.Id != providerID {
+				continue
+			}
+
+			return group, nil
+		}
+	}
+
+	// there is no "ErrNotExist" error, so we have to return a nil error
+	return nil, nil
+}
+
+// Pricing returns pricing model for this cloud provider or error if not
+// available. Implementation optional.
+func (d *bizflycloudCloudProvider) Pricing() (cloudprovider.PricingModel, errors.AutoscalerError) {
+	return nil, cloudprovider.ErrNotImplemented
+}
+
+// GetAvailableMachineTypes get all machine types that can be requested from
+// the cloud provider. Implementation optional.
+func (d *bizflycloudCloudProvider) GetAvailableMachineTypes() ([]string, error) {
+	return []string{}, nil
+}
+
+// NewNodeGroup builds a theoretical node group based on the node definition
+// provided. The node group is not automatically created on the cloud provider
+// side. The node group is not returned by NodeGroups() until it is created.
+// Implementation optional.
+func (d *bizflycloudCloudProvider) NewNodeGroup(
+	machineType string,
+	labels map[string]string,
+	systemLabels map[string]string,
+	taints []apiv1.Taint,
+	extraResources map[string]resource.Quantity,
+) (cloudprovider.NodeGroup, error) {
+	return nil, cloudprovider.ErrNotImplemented
+}
+
+// GetResourceLimiter returns struct containing limits (max, min) for
+// resources (cores, memory etc.).
+func (d *bizflycloudCloudProvider) GetResourceLimiter() (*cloudprovider.ResourceLimiter, error) {
+	return d.resourceLimiter, nil
+}
+
+// GPULabel returns the label added to nodes with GPU resource.
+func (d *bizflycloudCloudProvider) GPULabel() string {
+	return GPULabel
+}
+
+// GetAvailableGPUTypes return all available GPU types cloud provider supports.
+func (d *bizflycloudCloudProvider) GetAvailableGPUTypes() map[string]struct{} {
+	return nil
+}
+
+// Cleanup cleans up open resources before the cloud provider is destroyed,
+// i.e. go routines etc.
+func (d *bizflycloudCloudProvider) Cleanup() error {
+	return nil
+}
+
+// Refresh is called before every main loop and can be used to dynamically
+// update cloud provider state. In particular the list of node groups returned
+// by NodeGroups() can change as a result of CloudProvider.Refresh().
+func (d *bizflycloudCloudProvider) Refresh() error {
+	klog.V(4).Info("Refreshing node group cache")
+	return d.manager.Refresh()
+}
+
+// BuildBizflyCloud builds the Bizflycloud cloud provider.
+func BuildBizflyCloud(
+	opts config.AutoscalingOptions,
+	do cloudprovider.NodeGroupDiscoveryOptions,
+	rl *cloudprovider.ResourceLimiter,
+) cloudprovider.CloudProvider {
+	var configFile io.ReadCloser
+	if opts.CloudConfig != "" {
+		var err error
+		configFile, err = os.Open(opts.CloudConfig)
+		if err != nil {
+			klog.Fatalf("Couldn't open cloud provider configuration %s: %#v", opts.CloudConfig, err)
+		}
+		defer configFile.Close()
+	}
+	manager, err := newManager(configFile)
+
+	if err != nil {
+		klog.Fatalf("Failed to create Bizflycloud manager: %v", err)
+	}
+
+	// the cloud provider automatically uses all node pools in Bizflycloud.
+	// This means we don't use the cloudprovider.NodeGroupDiscoveryOptions
+	provider, err := newBizflyCloudProvider(manager, rl)
+	if err != nil {
+		klog.Fatalf("Failed to create Blizflycloud provider: %v", err)
+	}
+
+	return provider
+}
+
+// toProviderID returns a provider ID from the given node ID.
+func toProviderID(nodeID string) string {
+	return fmt.Sprintf("%s%s", bizflyProviderIDPrefix, nodeID)
+}
+
+// toNodeID returns a node or physical ID from the given provider ID.
+func toNodeID(providerID string) string {
+	return strings.TrimPrefix(providerID, bizflyProviderIDPrefix)
+}

cluster-autoscaler/cloudprovider/bizflycloud/bizflycloud_cloud_provider_test.go

@@ -0,0 +1,155 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package bizflycloud
+
+import (
+	"bytes"
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider/bizflycloud/gobizfly"
+
+	apiv1 "k8s.io/api/core/v1"
+	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
+)
+
+func testCloudProvider(t *testing.T, client *bizflyClientMock) *bizflycloudCloudProvider {
+	cfg := `{"cluster_id": "123456", "token": "123123123", "url": "https://manage.bizflycloud.vn", "version": "test"}`
+
+	manager, err := newManagerTest(bytes.NewBufferString(cfg))
+	assert.NoError(t, err)
+	rl := &cloudprovider.ResourceLimiter{}
+
+	// fill the test provider with some example
+	if client == nil {
+		client = &bizflyClientMock{}
+		ctx := context.Background()
+
+		client.On("Get", ctx, manager.clusterID, nil).Return(&gobizfly.FullCluster{
+			ExtendedCluster: gobizfly.ExtendedCluster{
+				Cluster: gobizfly.Cluster{
+					UID:              "cluster-1",
+					Name:             "test-cluster",
+					WorkerPoolsCount: 4,
+				},
+				WorkerPools: []gobizfly.ExtendedWorkerPool{
+					{
+						WorkerPool: gobizfly.WorkerPool{},
+						UID:        "pool-1",
+					},
+				},
+			},
+			Stat: gobizfly.ClusterStat{},
+		}, nil).Once()
+
+		client.On("GetClusterWorkerPool", ctx, manager.clusterID, "pool-1", nil).Return(&gobizfly.WorkerPoolWithNodes{
+			ExtendedWorkerPool: gobizfly.ExtendedWorkerPool{
+				WorkerPool: gobizfly.WorkerPool{
+					EnableAutoScaling: true,
+				},
+				UID: "pool-1",
+			},
+			Nodes: []gobizfly.PoolNode{
+				{
+					ID:   "1",
+					Name: "node-1",
+				},
+				{
+					ID:   "2",
+					Name: "node-2",
+				},
+			},
+		}, nil).Once()
+	}
+
+	manager.client = client
+	provider, err := newBizflyCloudProvider(manager, rl)
+	assert.NoError(t, err)
+	return provider
+}
+
+func TestNewBizflyCloudProvider(t *testing.T) {
+	t.Run("success", func(t *testing.T) {
+		_ = testCloudProvider(t, nil)
+	})
+}
+
+func TestBizflyCloudProvider_Name(t *testing.T) {
+	provider := testCloudProvider(t, nil)
+
+	t.Run("success", func(t *testing.T) {
+		name := provider.Name()
+		assert.Equal(t, cloudprovider.BizflyCloudProviderName, name, "provider name doesn't match")
+	})
+}
+
+func TestBizflyCloudProvider_NodeGroups(t *testing.T) {
+	provider := testCloudProvider(t, nil)
+
+	t.Run("zero groups", func(t *testing.T) {
+		provider.manager.nodeGroups = []*NodeGroup{}
+		nodes := provider.NodeGroups()
+		assert.Equal(t, len(nodes), 0, "number of nodes do not match")
+	})
+}
+
+func TestBizflyCloudProvider_NodeGroupForNode(t *testing.T) {
+	clusterID := "123456"
+
+	t.Run("success", func(t *testing.T) {
+		client := &bizflyClientMock{}
+		ctx := context.Background()
+
+		client.On("Get", ctx, clusterID, nil).Return(
+			&gobizfly.FullCluster{},
+			nil,
+		).Once()
+
+		provider := testCloudProvider(t, client)
+
+		node := &apiv1.Node{
+			Spec: apiv1.NodeSpec{
+				ProviderID: toProviderID("droplet-4"),
+			},
+		}
+		nodeGroup, err := provider.NodeGroupForNode(node)
+		assert.NoError(t, err)
+		assert.Nil(t, nodeGroup)
+	})
+	t.Run("node does not exist", func(t *testing.T) {
+		client := &bizflyClientMock{}
+		ctx := context.Background()
+
+		client.On("Get", ctx, clusterID, nil).Return(
+			&gobizfly.FullCluster{},
+			nil,
+		).Once()
+
+		provider := testCloudProvider(t, client)
+
+		node := &apiv1.Node{
+			Spec: apiv1.NodeSpec{
+				ProviderID: toProviderID("xxxxx-7"),
+			},
+		}
+
+		nodeGroup, err := provider.NodeGroupForNode(node)
+		assert.NoError(t, err)
+		assert.Nil(t, nodeGroup)
+	})
+}