#25 chunk DescribeAutoScalingInstances input to 50

hjacobs · Apr 27, 2017 · 271df02 · 271df02
1 parent f9ea901
commit 271df02
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 7 deletions.
diff --git a/kube_aws_autoscaler/main.py b/kube_aws_autoscaler/main.py
@@ -34,6 +34,9 @@
 DEFAULT_BUFFER_PERCENTAGE = {'cpu': 10, 'memory': 10, 'pods': 10}
 DEFAULT_BUFFER_FIXED = {'cpu': '200m', 'memory': '200Mi', 'pods': '10'}
 
+# DescribeAutoScalingInstances operation: The number of instance ids that may be passed in is limited to 50
+DESCRIBE_AUTO_SCALING_INSTANCES_LIMIT = 50
+
 logger = logging.getLogger('autoscaler')
 
 
@@ -103,6 +106,12 @@ def get_nodes(api, include_master_nodes: bool=False) -> dict:
     return nodes
 
 
+def chunks(l: list, n: int):
+    '''Yield successive n-sized chunks from l.'''
+    for i in range(0, len(l), n):
+        yield l[i:i + n]
+
+
 def get_nodes_by_asg_zone(autoscaling, nodes: dict) -> dict:
     # first map instance_id to node object for later look up
     instances = {}
@@ -111,12 +120,13 @@ def get_nodes_by_asg_zone(autoscaling, nodes: dict) -> dict:
 
     nodes_by_asg_zone = collections.defaultdict(list)
 
-    response = autoscaling.describe_auto_scaling_instances(InstanceIds=list(instances.keys()))
-    for instance in response['AutoScalingInstances']:
-        instances[instance['InstanceId']]['asg_name'] = instance['AutoScalingGroupName']
-        instances[instance['InstanceId']]['asg_lifecycle_state'] = instance['LifecycleState']
-        key = instance['AutoScalingGroupName'], instance['AvailabilityZone']
-        nodes_by_asg_zone[key].append(instances[instance['InstanceId']])
+    for instance_ids in chunks(list(instances.keys()), DESCRIBE_AUTO_SCALING_INSTANCES_LIMIT):
+        response = autoscaling.describe_auto_scaling_instances(InstanceIds=list(instances.keys()))
+        for instance in response['AutoScalingInstances']:
+            instances[instance['InstanceId']]['asg_name'] = instance['AutoScalingGroupName']
+            instances[instance['InstanceId']]['asg_lifecycle_state'] = instance['LifecycleState']
+            key = instance['AutoScalingGroupName'], instance['AvailabilityZone']
+            nodes_by_asg_zone[key].append(instances[instance['InstanceId']])
     return nodes_by_asg_zone
 
 

diff --git a/tests/test_autoscaler.py b/tests/test_autoscaler.py
@@ -5,7 +5,7 @@
 
 from kube_aws_autoscaler.main import (apply_buffer, autoscale,
                                       calculate_required_auto_scaling_group_sizes,
-                                      calculate_usage_by_asg_zone,
+                                      calculate_usage_by_asg_zone, chunks,
                                       format_resource, get_kube_api, get_nodes,
                                       get_nodes_by_asg_zone, is_node_ready,
                                       is_sufficient, main, parse_resource,
@@ -401,3 +401,10 @@ def test_is_node_ready():
     assert not is_node_ready(node)
     node.obj = {'status': {'conditions': [{'type': 'Ready', 'status': 'True'}]}}
     assert is_node_ready(node)
+
+
+def test_chunks():
+    assert list(chunks([], 1)) == []
+    assert list(chunks([1], 1)) == [[1]]
+    assert list(chunks([1, 2], 1)) == [[1], [2]]
+    assert list(chunks([1, 2, 3], 2)) == [[1, 2], [3]]