Merge pull request #1 from awslabs/master

Sync to 2.4.7

README.md

@@ -39,7 +39,7 @@ aws ec2 run-instances --image-id ami-5253c32d \
   --iam-instance-profile Name=MyInstanceProfileWithProperPermissions
 ```
 
-that installs required packages and runs the initialization script. By default this creates a mount point of `/scratch` on a encrypted 100GB EBS volume. To change the mount point, edit the [cloud-init script](templates/cloud-init-userdata.yaml) file and supply additional options to the install script to suit your specific needs.  Install options are shown below.
+that installs required packages and runs the initialization script. By default this creates a mount point of `/scratch` on a encrypted 200GB gp3 EBS volume. To change the mount point, edit the [cloud-init script](templates/cloud-init-userdata.yaml) file and supply additional options to the install script to suit your specific needs.  Install options are shown below.
 
 ```text
 Install Amazon EBS Autoscale
@@ -62,14 +62,40 @@ Options
 
     -m, --mountpoint    MOUNTPOINT
                         Mount point for autoscale volume (default: /scratch)
+                        
+    -t, --volume-type   VOLUMETYPE
+                        Volume type (default: gp3)
+
+    --volume-iops       VOLUMEIOPS
+                        Volume IOPS for gp3, io1, io2 (default: 3000)
+
+    --volume-throughput VOLUMETHOUGHPUT
+                        Volume throughput for gp3 (default: 125)
+
+    --min-ebs-volume-size SIZE_GB
+                        Mimimum size in GB of new volumes created by the instance.
+                        (Default: 150)
 
-    -s, --initial-size  SIZE
+    --max-ebs-volume-size SIZE_GB
+                        Maximum size in GB of new volumes created by the instance.
+                        (Default: 1500)
+            
+    --max-total-created-size SIZE_GB
+                        Maximum total size in GB of all volumes created by the instance.
+                        (Default: 8000)
+                        
+    --max-attached-volumes N
+                        Maximum number of attached volumes. (Default: 16)
+
+    --initial-utilization-threshold N
+                        Initial disk utilization treshold for scale-up. (Default: 50)
+
+    -s, --initial-size  SIZE_GB
                         Initial size of the volume in GB. (Default: 200)
                         Only used if --initial-device is NOT specified.
-    
-    -t, --volume-type   VOLUMETYPE
-                        EBS volume type to use. (Default: gp3)
 
+    -i, --imdsv2        
+                        Enable imdsv2 for instance metadata API requests.
 ```
 
 ## A note on the IAM Instance Profile
@@ -86,6 +112,7 @@ In the above, we assume that the `MyInstanceProfileWithProperPermissions` EC2 In
                 "ec2:AttachVolume",
                 "ec2:DescribeVolumeStatus",
                 "ec2:DescribeVolumes",
+                "ec2:DescribeTags",
                 "ec2:ModifyInstanceAttribute",
                 "ec2:DescribeVolumeAttribute",
                 "ec2:CreateVolume",
@@ -98,6 +125,8 @@ In the above, we assume that the `MyInstanceProfileWithProperPermissions` EC2 In
 }
 ```
 
+Please note that if you enable EBS encryption and use a Customer Managed Key with AWS Key Management Service, then you should also ensure that you provide [appropriate IAM permissions](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSEncryption.html#ebs-encryption-permissions) to use that key.
+
 ## License Summary
 
 This sample code is made available under the MIT license. 

bin/create-ebs-volume

@@ -44,13 +44,17 @@ Required
 
 Options
 
-    -t, --type          Type of volume. (Default: gp2)
+    -t, --type          Type of volume. (Default: config.volume.type)
 
-    -i, --iops          IOPS for volume. Only valid if type=io1. (Default: 3000)
+    -i, --iops          N 
+                        IOPS for volume. Only valid if type=io1, io2, gp3. (Default: config.volume.iops)
+
+    --throughput        N 
+                        The throughput for a volume, with a maximum of 1,000 MiB/s. (Default: config.volume.throughput)
 
     --not-encrypted     Flag to make the volume un-encyrpted. Default is to create
                         an encrypted volume
-    
+
     --max-total-created-size SIZE_GB
                         Maximum total size in GB of all volumes created by the instance.
                         (Default: config.limits.max_logical_volume_size)
@@ -78,8 +82,9 @@ function error() {
 
 TYPE=$(get_config_value .volume.type)
 IOPS=$(get_config_value .volume.iops)
+THROUGHPUT=$(get_config_value .volume.throughput)
 ENCRYPTED=$(get_config_value .volume.encrypted)
-MAX_TOTAL_EBS_SIZE=$(get_config_value .limits.max_logical_volume_size)
+MAX_LOGICAL_VOLUME_SIZE=$(get_config_value .limits.max_logical_volume_size)
 MAX_ATTACHED_VOLUMES=$(get_config_value .limits.max_ebs_volume_count)
 MAX_CREATED_VOLUMES=$MAX_ATTACHED_VOLUMES
 
@@ -99,10 +104,18 @@ while (( "$#" )); do
             IOPS=$2
             shift 2
             ;;
+        --throughput)
+            THROUGHPUT=$2
+            shift 2
+            ;;
         --not-encrypted)
             unset ENCRYPTED
             shift
             ;;
+        --max-total-created-size)
+            MAX_LOGICAL_VOLUME_SIZE=$2
+            shift 2
+            ;;
         --max-attached-volumes)
             MAX_ATTACHED_VOLUMES=$2
             shift 2
@@ -111,10 +124,6 @@ while (( "$#" )); do
             MAX_CREATED_VOLUMES=$2
             shift 2
             ;;
-        --max-total-created-size)
-            MAX_TOTAL_EBS_SIZE=$2
-            shift 2
-            ;;
         -v|--verbose)
             VERBOSE=1
             shift
@@ -168,7 +177,7 @@ function create_and_attach_volume() {
     instance_tags=$(
       aws ec2 describe-tags \
         --region $region \
-        --filters "Name=resource-id,Values=$instance_id" | jq -r .Tags | jq -c 'map({Key, Value})' | tr -d '[]"' | tr : =
+        --filters "Name=resource-id,Values=$instance_id" | jq -r .Tags | jq -c 'map({Key, Value})' | tr -d '[]"' | sed 's/{Key:/{Key=/g ; s/,Value:/,Value=/g ; s/{Key=aws:[^}]*}//g ; s/,\{2,\}/,/g ; s/,$//g ; s/^,//g'
       )
 
     local max_attempts=10
@@ -187,7 +196,7 @@ function create_and_attach_volume() {
             logthis "Could not determine the number of attached_volumes after $i attempts. Last response was: $attached_volumes"
             break
         fi
-        sleep $(( 2 ** i ))
+        sleep $(( 2 ** i + $RANDOM % 3))
     done
 
     local created_volumes=""
@@ -204,7 +213,7 @@ function create_and_attach_volume() {
             logthis "Could not determine the number of created_volumes after $i attempts. Last response was: $created_volumes"
             break
         fi
-        sleep $(( 2 ** i ))
+        sleep $(( 2 ** i + $RANDOM % 3))
     done
 
     local total_created_size=""
@@ -223,12 +232,12 @@ function create_and_attach_volume() {
             logthis "Could not determine the total_created_size after $i attempts. Last response was: $total_created_size"
             break
         fi
-        sleep $(( 2 ** i ))
+        sleep $(( 2 ** i + $RANDOM % 3))
     done
 
     # check how much EBS storage this instance has created
-    if [ "$total_created_size" -ge "$MAX_TOTAL_EBS_SIZE" ]; then
-        error "maximum total ebs volume size reached ($MAX_TOTAL_EBS_SIZE)"
+    if [ "$total_created_size" -ge "$MAX_LOGICAL_VOLUME_SIZE" ]; then
+        error "maximum total ebs volume size reached ($MAX_LOGICAL_VOLUME_SIZE)"
     fi
 
     # check how many volumes this instance has created
@@ -251,18 +260,32 @@ function create_and_attach_volume() {
     # create the volume
     local tmpfile=$(mktemp /tmp/ebs-autoscale.create-volume.XXXXXXXXXX)
     local volume_opts="--size $SIZE --volume-type $TYPE"
-    if [ "$TYPE" == "io1" ]; then volume_opts="$volume_opts --iops $IOPS"; fi
+    local IOPS_TYPES=( io1 io2 gp3 )
+    if [[ " ${IOPS_TYPES[*]} " =~ " ${TYPE} " ]]; then volume_opts="$volume_opts --iops $IOPS"; fi
+    if [ "$TYPE" == "gp3" ]; then volume_opts="$volume_opts --throughput $THROUGHPUT"; fi
     if [ "$ENCRYPTED" == "1" ]; then volume_opts="$volume_opts --encrypted"; fi
     local timestamp=$(date "+%F %T UTC%z")  # YYYY-mm-dd HH:MM:SS UTC+0000
 
     local volume=""
     for i in $(eval echo "{0..$max_attempts}") ; do
+
+      # The $instance_tags variable could be empty and will cause a TagSpecifications[0].Tags[0] error if
+      # it is passed as an empty value because it must be comma-separated from the other key-value pairs.
+      # Use a Shell Parameter Expansion to determine if the variable contains a value or not. If it has a value, 
+      # append a comma at the end so the aws cli syntax is compliant when it is subbed into the tag_specification variable.
+      local instance_tags=${instance_tags:+${instance_tags},}
+      local tag_specification="ResourceType=volume,Tags=[$instance_tags{Key=source-instance,Value=$instance_id},{Key=amazon-ebs-autoscale-creation-time,Value=$timestamp}]"
+
+      # Note: Shellcheck says the $vars in this command should be double quoted to prevent globbing and word-splitting,
+      # but this ends up making the '--encrypted' argument to fail during the execution of the install script. Conversely, NOT putting double-quotes
+      # around $tag_specification causes a parsing error due to the space in the $timestamp value (added to $tag_specification above).
+
       local volume=$(\
           aws ec2 create-volume \
               --region $region \
               --availability-zone $availability_zone \
               $volume_opts \
-              --tag-specification "ResourceType=volume,Tags=[$instance_tags,{Key=source-instance,Value=$instance_id},{Key=amazon-ebs-autoscale-creation-time,Value=$timestamp}]" \
+              --tag-specification "$tag_specification" \
           2> $tmpfile
       )
 
@@ -272,7 +295,7 @@ function create_and_attach_volume() {
           logthis "Could not create a volume after $i attempts. Last response was: $volume"
           break
       fi
-      sleep $(( 2 ** i ))
+      sleep $(( 2 ** i + $RANDOM % 3))
     done
 
     local volume_id=`echo $volume | jq -r '.VolumeId'`
@@ -341,4 +364,4 @@ function create_and_attach_volume() {
     echo $device
 }
 
-create_and_attach_volume
+create_and_attach_volume

bin/ebs-autoscale

@@ -32,8 +32,11 @@
 
 initialize
 
+MIN_EBS_VOLUME_SIZE=$(get_config_value .limits.min_ebs_volume_size)
+MAX_EBS_VOLUME_SIZE=$(get_config_value .limits.max_ebs_volume_size)
 MAX_LOGICAL_VOLUME_SIZE=$(get_config_value .limits.max_logical_volume_size)
 MAX_EBS_VOLUME_COUNT=$(get_config_value .limits.max_ebs_volume_count)
+INITIAL_UTILIZATION_THRESHOLD=$(get_config_value .limits.initial_utilization_threshold)
 
 FILE_SYSTEM=$(get_config_value .filesystem)
 
@@ -86,7 +89,7 @@ get_num_devices() {
         break
       fi
 
-      sleep $(( 2 ** i ))
+      sleep $(( 2 ** i + $RANDOM %3 ))
   done
 
   echo "$attached_volumes"
@@ -96,8 +99,8 @@ calc_threshold() {
   # calculates percent utilization threshold for adding additional ebs volumes
   # as more ebs volumes are added, the threshold level increases
 
-  local num_devices=$(get_num_devices)
-  local threshold=50
+  local num_devices=$1
+  local threshold=${INITIAL_UTILIZATION_THRESHOLD}
 
   if [ "$num_devices" -ge "4" ] && [ "$num_devices" -le "6" ]; then
     threshold=80
@@ -106,7 +109,7 @@ calc_threshold() {
   elif [ "$num_devices" -gt "10" ]; then
     threshold=90
   else
-    threshold=50
+    threshold=${INITIAL_UTILIZATION_THRESHOLD}
   fi
 
   echo ${threshold}
@@ -115,33 +118,50 @@ calc_threshold() {
 calc_new_size() {
   # calculates the size to use for new ebs volumes to expand space
   # new volume sizes increase as the number of attached volumes increase
-
-  local num_devices=$(get_num_devices)
-  local new_size=150
+  local num_devices=$1
+  #local num_devices=$(get_num_devices)
+  local new_size=$MIN_EBS_VOLUME_SIZE
 
   if [ "$num_devices" -ge "4" ] && [ "$num_devices" -le "6" ]; then
-    new_size=300
+    if [ "$MAX_EBS_VOLUME_SIZE" -ge "299" ] && [ "$MIN_EBS_VOLUME_SIZE" -le "299" ]; then
+      new_size=300
+    elif [ "$MIN_EBS_VOLUME_SIZE" -ge "299" ]; then
+      new_size=$MIN_EBS_VOLUME_SIZE
+    else
+      new_size=$MAX_EBS_VOLUME_SIZE
+    fi
   elif [ "$num_devices" -gt "6" ] && [ "$num_devices" -le "10" ]; then
-    new_size=1000
+    if [ "$MAX_EBS_VOLUME_SIZE" -ge "999" ] && [ "$MIN_EBS_VOLUME_SIZE" -le "999" ]; then
+      new_size=1000
+    elif [ "$MIN_EBS_VOLUME_SIZE" -ge "999" ]; then
+      new_size=$MIN_EBS_VOLUME_SIZE
+    else
+      new_size=$MAX_EBS_VOLUME_SIZE
+    fi
   elif [ "$num_devices" -gt "10" ]; then
-    new_size=1500
+    new_size=$MAX_EBS_VOLUME_SIZE
   else
-    new_size=150
+    if [ "$MAX_EBS_VOLUME_SIZE" -ge "149" ]; then
+      new_size=$MIN_EBS_VOLUME_SIZE
+    else
+      new_size=$MAX_EBS_VOLUME_SIZE
+    fi
   fi
 
   echo ${new_size}
 }
 
 add_space () {
-  local num_devices=$(get_num_devices)
+  #local num_devices=$(get_num_devices)
+  local num_devices=$1
   if [ "${num_devices}" -ge "$MAX_EBS_VOLUME_COUNT" ]; then
     logthis "No more volumes can be safely added."
     return 0
   fi
 
   local curr_size=$(df -BG ${MOUNTPOINT} | grep ${MOUNTPOINT} | awk '{print $2} ' | cut -d'G' -f1)
   if [ "${curr_size}" -lt "$MAX_LOGICAL_VOLUME_SIZE" ]; then
-    local vol_size=$(calc_new_size)
+    local vol_size=$(calc_new_size ${num_devices})
     logthis "Extending logical volume ${MOUNTPOINT} by ${vol_size}GB"
 
     DEVICE=$(${BASEDIR}/create-ebs-volume --size ${vol_size} --max-attached-volumes ${MAX_EBS_VOLUME_COUNT})
@@ -181,25 +201,29 @@ LOG_COUNT=$LOG_INTERVAL
 # time in seconds between event loops
 # keep this low so that rapid increases in utilization are detected
 DETECTION_INTERVAL=$(get_config_value .detection_interval)
-
-THRESHOLD=$(calc_threshold)
+# get the number of devices once when the script first starts
+NUM_DEVICES=$(get_num_devices)
+THRESHOLD=$(calc_threshold "${NUM_DEVICES}")
 while true; do
-  NUM_DEVICES=$(get_num_devices)
+
   STATS=$(df -BG  ${MOUNTPOINT} | grep -v Filesystem)
   TOTAL_SIZE=$(echo ${STATS} | awk '{print $2}')
   USED=$(echo ${STATS} | awk '{print $3}')
   AVAILABLE=$(echo ${STATS} | awk '{print $4}')
   PCT_UTILIZATION=$(echo ${STATS} | awk '{print $5}' | cut -d"%" -f1 -)
   if  [ $PCT_UTILIZATION -ge "${THRESHOLD}" ]; then
+    # get number of devices only when we need to add more space
+  	NUM_DEVICES=$(get_num_devices)
     logthis "LOW DISK (${PCT_UTILIZATION}%): Adding more."
-    add_space
-    LOG_COUNT=LOG_INTERVAL
+    add_space "$NUM_DEVICES"
+	NUM_DEVICES=$(expr $NUM_DEVICES + 1 )
+    THRESHOLD=$(calc_threshold "$NUM_DEVICES")
+    LOG_COUNT=$LOG_INTERVAL
   fi
   if [ "${LOG_COUNT}" -ge "${LOG_INTERVAL}" ]; then
     logthis "Devices ${NUM_DEVICES} : Size ${TOTAL_SIZE} : Used ${USED} : Available ${AVAILABLE} : Used% ${PCT_UTILIZATION}% : Threshold ${THRESHOLD}%"
     LOG_COUNT=0
   fi
-  THRESHOLD=$(calc_threshold)
   LOG_COUNT=$(expr $LOG_COUNT + 1 )
   sleep $DETECTION_INTERVAL
 done

config/ebs-autoscale.json

@@ -2,21 +2,25 @@
     "mountpoint": "%%MOUNTPOINT%%",
     "filesystem": "%%FILESYSTEM%%",
     "lvm": {
-      "volume_group": "autoscale_vg",
-      "logical_volume": "autoscale_lv"
+        "volume_group": "autoscale_vg",
+        "logical_volume": "autoscale_lv"
     },
     "volume": {
         "type": "%%VOLUMETYPE%%",
-        "iops": 3000,
+        "iops": "%%VOLUMEIOPS%%",
+        "throughput": "%%VOLUMETHOUGHPUT%%",
         "encrypted": 1
     },
-    "detection_interval": 1,
+    "detection_interval": 2,
     "limits": {
-        "max_logical_volume_size": 8000,
-        "max_ebs_volume_count": 16
+        "min_ebs_volume_size": "%%MINEBSVOLUMESIZE%%",
+        "max_ebs_volume_size": "%%MAXEBSVOLUMESIZE%%",
+        "max_logical_volume_size": "%%MAXLOGICALVOLUMESIZE%%",
+        "max_ebs_volume_count": "%%MAXATTACHEDVOLUMES%%",
+        "initial_utilization_threshold": "%%INITIALUTILIZATIONTHRESHOLD%%"
     },
     "logging": {
         "log_file": "/var/log/ebs-autoscale.log",
         "log_interval": 300
     }
-}
+}