Skip to content

Commit

Permalink
Merge pull request #2490 from mistio/cluster-autoscaler-packet-tests
Browse files Browse the repository at this point in the history
Add tests for Packet
  • Loading branch information
k8s-ci-robot authored Nov 1, 2019
2 parents 086999e + 7d2633a commit 0690933
Show file tree
Hide file tree
Showing 5 changed files with 226 additions and 27 deletions.
1 change: 1 addition & 0 deletions cluster-autoscaler/cloudprovider/packet/packet_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ type packetManager interface {
nodeGroupSize(nodegroup string) (int, error)
createNodes(nodegroup string, nodes int) error
getNodes(nodegroup string) ([]string, error)
getNodeNames(nodegroup string) ([]string, error)
deleteNodes(nodegroup string, nodes []NodeRef, updatedNodeCount int) error
templateNodeInfo(nodegroup string) (*schedulernodeinfo.NodeInfo, error)
}
Expand Down
63 changes: 45 additions & 18 deletions cluster-autoscaler/cloudprovider/packet/packet_manager_rest.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ import (
)

type packetManagerRest struct {
baseURL string
clusterName string
projectID string
apiServerEndpoint string
Expand Down Expand Up @@ -160,6 +161,7 @@ func createPacketManagerRest(configReader io.Reader, discoverOpts cloudprovider.
}

manager := packetManagerRest{
baseURL: "https://api.packet.net",
clusterName: cfg.Global.ClusterName,
projectID: cfg.Global.ProjectID,
apiServerEndpoint: cfg.Global.APIServerEndpoint,
Expand All @@ -174,10 +176,10 @@ func createPacketManagerRest(configReader io.Reader, discoverOpts cloudprovider.
return &manager, nil
}

func (mgr *packetManagerRest) listPacketDevices() *Devices {
func (mgr *packetManagerRest) listPacketDevices() (*Devices, error) {
var jsonStr = []byte(``)
packetAuthToken := os.Getenv("PACKET_AUTH_TOKEN")
url := "https://api.packet.net/projects/" + mgr.projectID + "/devices"
url := mgr.baseURL + "/projects/" + mgr.projectID + "/devices"
req, _ := http.NewRequest("GET", url, bytes.NewBuffer(jsonStr))
req.Header.Set("X-Auth-Token", packetAuthToken)
req.Header.Set("Content-Type", "application/json")
Expand All @@ -189,24 +191,30 @@ func (mgr *packetManagerRest) listPacketDevices() *Devices {
}
defer resp.Body.Close()

fmt.Println("response Status:", resp.Status)
body, _ := ioutil.ReadAll(resp.Body)
klog.Infof("response Status: %s", resp.Status)

var devices Devices
json.Unmarshal([]byte(body), &devices)
return &devices

if "200 OK" == resp.Status {
body, _ := ioutil.ReadAll(resp.Body)
json.Unmarshal([]byte(body), &devices)
return &devices, nil
}

return &devices, fmt.Errorf(resp.Status, resp.Body)
}

// nodeGroupSize gets the current size of the nodegroup as reported by packet tags.
func (mgr *packetManagerRest) nodeGroupSize(nodegroup string) (int, error) {
devices := mgr.listPacketDevices()
devices, _ := mgr.listPacketDevices()
// Get the count of devices tagged as nodegroup members
count := 0
for _, d := range devices.Devices {
if Contains(d.Tags, "k8s-cluster-"+mgr.clusterName) && Contains(d.Tags, "k8s-nodepool-"+nodegroup) {
count++
}
}
fmt.Println(len(devices.Devices), count)
klog.V(3).Infof("Nodegroup %s: %d/%d", nodegroup, count, len(devices.Devices))
return count, nil
}

Expand Down Expand Up @@ -252,13 +260,13 @@ func (mgr *packetManagerRest) createNode(cloudinit, nodegroup string) {
HardwareReservationID: reservation,
}

resp, err := createDevice(&cr)
resp, err := createDevice(&cr, mgr.baseURL)
if err != nil || resp.StatusCode > 299 {
// If reservation is preferred but not available, retry provisioning as on-demand
if reservation != "" && mgr.reservation == "prefer" {
klog.Infof("Reservation preferred but not available. Provisioning on-demand node.")
cr.HardwareReservationID = ""
resp, err = createDevice(&cr)
resp, err = createDevice(&cr, mgr.baseURL)
if err != nil {
klog.Errorf("Failed to create device using Packet API: %v", err)
panic(err)
Expand Down Expand Up @@ -304,9 +312,9 @@ func (mgr *packetManagerRest) createNodes(nodegroup string, nodes int) error {
return nil
}

func createDevice(cr *DeviceCreateRequest) (*http.Response, error) {
func createDevice(cr *DeviceCreateRequest, baseURL string) (*http.Response, error) {
packetAuthToken := os.Getenv("PACKET_AUTH_TOKEN")
url := "https://api.packet.net/projects/" + cr.ProjectID + "/devices"
url := baseURL + "/projects/" + cr.ProjectID + "/devices"
jsonValue, _ := json.Marshal(cr)
klog.Infof("Creating new node")
klog.V(3).Infof("POST %s \n%v", url, string(jsonValue))
Expand All @@ -325,18 +333,37 @@ func createDevice(cr *DeviceCreateRequest) (*http.Response, error) {
// getNodes should return ProviderIDs for all nodes in the node group,
// used to find any nodes which are unregistered in kubernetes.
func (mgr *packetManagerRest) getNodes(nodegroup string) ([]string, error) {
// TODO: get node ProviderIDs by getting device IDs from Packet
// This works fine being empty for now anyway.
return []string{}, nil
// Get node ProviderIDs by getting device IDs from Packet
devices, err := mgr.listPacketDevices()
nodes := []string{}
for _, d := range devices.Devices {
if Contains(d.Tags, "k8s-cluster-"+mgr.clusterName) && Contains(d.Tags, "k8s-nodepool-"+nodegroup) {
nodes = append(nodes, d.ID)
}
}
return nodes, err
}

// getNodeNames should return Names for all nodes in the node group,
// used to find any nodes which are unregistered in kubernetes.
func (mgr *packetManagerRest) getNodeNames(nodegroup string) ([]string, error) {
devices, err := mgr.listPacketDevices()
nodes := []string{}
for _, d := range devices.Devices {
if Contains(d.Tags, "k8s-cluster-"+mgr.clusterName) && Contains(d.Tags, "k8s-nodepool-"+nodegroup) {
nodes = append(nodes, d.Hostname)
}
}
return nodes, err
}

// deleteNodes deletes nodes by passing a comma separated list of names or IPs
func (mgr *packetManagerRest) deleteNodes(nodegroup string, nodes []NodeRef, updatedNodeCount int) error {
klog.Infof("Deleting nodes")
klog.Infof("Deleting nodes %v", nodes)
packetAuthToken := os.Getenv("PACKET_AUTH_TOKEN")
for _, n := range nodes {
klog.Infof("Node %s - %s - %s", n.Name, n.MachineID, n.IPs)
dl := mgr.listPacketDevices()
dl, _ := mgr.listPacketDevices()
klog.Infof("%d devices total", len(dl.Devices))
// Get the count of devices tagged as nodegroup
for _, d := range dl.Devices {
Expand All @@ -345,7 +372,7 @@ func (mgr *packetManagerRest) deleteNodes(nodegroup string, nodes []NodeRef, upd
klog.Infof("nodegroup match %s %s", d.Hostname, n.Name)
if d.Hostname == n.Name {
klog.V(1).Infof("Matching Packet Device %s - %s", d.Hostname, d.ID)
req, _ := http.NewRequest("DELETE", "https://api.packet.net/devices/"+d.ID, bytes.NewBuffer([]byte("")))
req, _ := http.NewRequest("DELETE", mgr.baseURL+"/devices/"+d.ID, bytes.NewBuffer([]byte("")))
req.Header.Set("X-Auth-Token", packetAuthToken)
req.Header.Set("Content-Type", "application/json")

Expand Down

Large diffs are not rendered by default.

10 changes: 1 addition & 9 deletions cluster-autoscaler/cloudprovider/packet/packet_node_group.go
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ func (ng *packetNodeGroup) DeleteNodes(nodes []*apiv1.Node) error {
if cachedSize-len(ng.nodesToDelete)-len(nodes) < ng.MinSize() {
ng.nodesToDeleteMutex.Unlock()
klog.V(1).Infof("UnLocking nodesToDeleteMutex")
return fmt.Errorf("deleting nodes would take nodegroup below minimum size")
return fmt.Errorf("deleting nodes would take nodegroup below minimum size %d", ng.minSize)
}
// otherwise, add the nodes to the batch and release the lock
ng.nodesToDelete = append(ng.nodesToDelete, nodes...)
Expand Down Expand Up @@ -180,14 +180,6 @@ func (ng *packetNodeGroup) DeleteNodes(nodes []*apiv1.Node) error {
}
klog.V(0).Infof("Deleting nodes: %v", nodeNames)

/*updatePossible, currentStatus, err := ng.magnumManager.canUpdate()
if err != nil {
return fmt.Errorf("could not check if cluster is ready to delete nodes: %v", err)
}
if !updatePossible {
return fmt.Errorf("can not delete nodes, cluster is in %s status", currentStatus)
}*/

// Double check that the total number of batched nodes for deletion will not take the node group below its minimum size
if cachedSize-len(nodes) < ng.MinSize() {
return fmt.Errorf("size decrease too large, desired:%d min:%d", cachedSize-len(nodes), ng.MinSize())
Expand Down
100 changes: 100 additions & 0 deletions cluster-autoscaler/cloudprovider/packet/packet_node_group_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
/*
Copyright 2019 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package packet

import (
"os"
"sync"
"testing"
"time"

apiv1 "k8s.io/api/core/v1"
. "k8s.io/autoscaler/cluster-autoscaler/utils/test"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
)

func TestIncreaseDecreaseSize(t *testing.T) {
var m *packetManagerRest
server := NewHttpServerMock()
defer server.Close()
assert.Equal(t, true, true)
if len(os.Getenv("PACKET_AUTH_TOKEN")) > 0 {
// If auth token set in env, hit the actual Packet API
m = newTestPacketManagerRest(t, "https://api.packet.net")
} else {
// Set up a mock Packet API
m = newTestPacketManagerRest(t, server.URL)
server.On("handle", "/projects/"+m.projectID+"/devices").Return(listPacketDevicesResponse).Times(4)
server.On("handle", "/projects/"+m.projectID+"/devices").Return(listPacketDevicesResponseAfterCreate).Times(2)
server.On("handle", "/projects/"+m.projectID+"/devices").Return(listPacketDevicesResponse)
}
clusterUpdateLock := sync.Mutex{}
ng := &packetNodeGroup{
packetManager: m,
id: "pool1",
clusterUpdateMutex: &clusterUpdateLock,
minSize: 1,
maxSize: 10,
targetSize: new(int),
waitTimeStep: 30 * time.Second,
deleteBatchingDelay: 2 * time.Second,
}

n1, err := ng.Nodes()
assert.NoError(t, err)
assert.Equal(t, int(1), len(n1))

// Try to increase pool with negative size, this should return an error
err = ng.IncreaseSize(-1)
assert.Error(t, err)

// Now try to increase the pool size by 2, that should work
err = ng.IncreaseSize(2)
assert.NoError(t, err)

if len(os.Getenv("PACKET_AUTH_TOKEN")) > 0 {
// If testing with actual API give it some time until the nodes bootstrap
time.Sleep(420 * time.Second)
}
n2, err := ng.packetManager.getNodeNames(ng.id)
assert.NoError(t, err)
// Assert that the nodepool size is now 3
assert.Equal(t, int(3), len(n2))

// Let's try to delete the new nodes
nodes := []*apiv1.Node{}
for _, node := range n2 {
if node != "k8s-worker-1" {
nodes = append(nodes, BuildTestNode(node, 1000, 1000))
}
}
err = ng.DeleteNodes(nodes)
assert.NoError(t, err)

// Wait a few seconds if talking to the actual Packet API
if len(os.Getenv("PACKET_AUTH_TOKEN")) > 0 {
time.Sleep(10 * time.Second)
}

// Make sure that there were no errors and the nodepool size is once again 1
n3, err := ng.Nodes()
assert.NoError(t, err)
assert.Equal(t, int(1), len(n3))
mock.AssertExpectationsForObjects(t, server)
}

0 comments on commit 0690933

Please sign in to comment.