diff --git a/README.md b/README.md index 54fa484..59699c3 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ aws ec2 run-instances --image-id ami-5253c32d \ --iam-instance-profile Name=MyInstanceProfileWithProperPermissions ``` -that installs required packages and runs the initialization script. By default this creates a mount point of `/scratch` on a encrypted 100GB EBS volume. To change the mount point, edit the [cloud-init script](templates/cloud-init-userdata.yaml) file and supply additional options to the install script to suit your specific needs. Install options are shown below. +that installs required packages and runs the initialization script. By default this creates a mount point of `/scratch` on a encrypted 200GB gp3 EBS volume. To change the mount point, edit the [cloud-init script](templates/cloud-init-userdata.yaml) file and supply additional options to the install script to suit your specific needs. Install options are shown below. ```text Install Amazon EBS Autoscale @@ -62,14 +62,40 @@ Options -m, --mountpoint MOUNTPOINT Mount point for autoscale volume (default: /scratch) + + -t, --volume-type VOLUMETYPE + Volume type (default: gp3) + + --volume-iops VOLUMEIOPS + Volume IOPS for gp3, io1, io2 (default: 3000) + + --volume-throughput VOLUMETHOUGHPUT + Volume throughput for gp3 (default: 125) + + --min-ebs-volume-size SIZE_GB + Mimimum size in GB of new volumes created by the instance. + (Default: 150) - -s, --initial-size SIZE + --max-ebs-volume-size SIZE_GB + Maximum size in GB of new volumes created by the instance. + (Default: 1500) + + --max-total-created-size SIZE_GB + Maximum total size in GB of all volumes created by the instance. + (Default: 8000) + + --max-attached-volumes N + Maximum number of attached volumes. (Default: 16) + + --initial-utilization-threshold N + Initial disk utilization treshold for scale-up. (Default: 50) + + -s, --initial-size SIZE_GB Initial size of the volume in GB. (Default: 200) Only used if --initial-device is NOT specified. - - -t, --volume-type VOLUMETYPE - EBS volume type to use. (Default: gp3) + -i, --imdsv2 + Enable imdsv2 for instance metadata API requests. ``` ## A note on the IAM Instance Profile @@ -86,6 +112,7 @@ In the above, we assume that the `MyInstanceProfileWithProperPermissions` EC2 In "ec2:AttachVolume", "ec2:DescribeVolumeStatus", "ec2:DescribeVolumes", + "ec2:DescribeTags", "ec2:ModifyInstanceAttribute", "ec2:DescribeVolumeAttribute", "ec2:CreateVolume", @@ -98,6 +125,8 @@ In the above, we assume that the `MyInstanceProfileWithProperPermissions` EC2 In } ``` +Please note that if you enable EBS encryption and use a Customer Managed Key with AWS Key Management Service, then you should also ensure that you provide [appropriate IAM permissions](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSEncryption.html#ebs-encryption-permissions) to use that key. + ## License Summary This sample code is made available under the MIT license. diff --git a/bin/create-ebs-volume b/bin/create-ebs-volume index 47840c5..db23f7f 100755 --- a/bin/create-ebs-volume +++ b/bin/create-ebs-volume @@ -44,13 +44,17 @@ Required Options - -t, --type Type of volume. (Default: gp2) + -t, --type Type of volume. (Default: config.volume.type) - -i, --iops IOPS for volume. Only valid if type=io1. (Default: 3000) + -i, --iops N + IOPS for volume. Only valid if type=io1, io2, gp3. (Default: config.volume.iops) + + --throughput N + The throughput for a volume, with a maximum of 1,000 MiB/s. (Default: config.volume.throughput) --not-encrypted Flag to make the volume un-encyrpted. Default is to create an encrypted volume - + --max-total-created-size SIZE_GB Maximum total size in GB of all volumes created by the instance. (Default: config.limits.max_logical_volume_size) @@ -78,8 +82,9 @@ function error() { TYPE=$(get_config_value .volume.type) IOPS=$(get_config_value .volume.iops) +THROUGHPUT=$(get_config_value .volume.throughput) ENCRYPTED=$(get_config_value .volume.encrypted) -MAX_TOTAL_EBS_SIZE=$(get_config_value .limits.max_logical_volume_size) +MAX_LOGICAL_VOLUME_SIZE=$(get_config_value .limits.max_logical_volume_size) MAX_ATTACHED_VOLUMES=$(get_config_value .limits.max_ebs_volume_count) MAX_CREATED_VOLUMES=$MAX_ATTACHED_VOLUMES @@ -99,10 +104,18 @@ while (( "$#" )); do IOPS=$2 shift 2 ;; + --throughput) + THROUGHPUT=$2 + shift 2 + ;; --not-encrypted) unset ENCRYPTED shift ;; + --max-total-created-size) + MAX_LOGICAL_VOLUME_SIZE=$2 + shift 2 + ;; --max-attached-volumes) MAX_ATTACHED_VOLUMES=$2 shift 2 @@ -111,10 +124,6 @@ while (( "$#" )); do MAX_CREATED_VOLUMES=$2 shift 2 ;; - --max-total-created-size) - MAX_TOTAL_EBS_SIZE=$2 - shift 2 - ;; -v|--verbose) VERBOSE=1 shift @@ -168,7 +177,7 @@ function create_and_attach_volume() { instance_tags=$( aws ec2 describe-tags \ --region $region \ - --filters "Name=resource-id,Values=$instance_id" | jq -r .Tags | jq -c 'map({Key, Value})' | tr -d '[]"' | tr : = + --filters "Name=resource-id,Values=$instance_id" | jq -r .Tags | jq -c 'map({Key, Value})' | tr -d '[]"' | sed 's/{Key:/{Key=/g ; s/,Value:/,Value=/g ; s/{Key=aws:[^}]*}//g ; s/,\{2,\}/,/g ; s/,$//g ; s/^,//g' ) local max_attempts=10 @@ -187,7 +196,7 @@ function create_and_attach_volume() { logthis "Could not determine the number of attached_volumes after $i attempts. Last response was: $attached_volumes" break fi - sleep $(( 2 ** i )) + sleep $(( 2 ** i + $RANDOM % 3)) done local created_volumes="" @@ -204,7 +213,7 @@ function create_and_attach_volume() { logthis "Could not determine the number of created_volumes after $i attempts. Last response was: $created_volumes" break fi - sleep $(( 2 ** i )) + sleep $(( 2 ** i + $RANDOM % 3)) done local total_created_size="" @@ -223,12 +232,12 @@ function create_and_attach_volume() { logthis "Could not determine the total_created_size after $i attempts. Last response was: $total_created_size" break fi - sleep $(( 2 ** i )) + sleep $(( 2 ** i + $RANDOM % 3)) done # check how much EBS storage this instance has created - if [ "$total_created_size" -ge "$MAX_TOTAL_EBS_SIZE" ]; then - error "maximum total ebs volume size reached ($MAX_TOTAL_EBS_SIZE)" + if [ "$total_created_size" -ge "$MAX_LOGICAL_VOLUME_SIZE" ]; then + error "maximum total ebs volume size reached ($MAX_LOGICAL_VOLUME_SIZE)" fi # check how many volumes this instance has created @@ -251,18 +260,32 @@ function create_and_attach_volume() { # create the volume local tmpfile=$(mktemp /tmp/ebs-autoscale.create-volume.XXXXXXXXXX) local volume_opts="--size $SIZE --volume-type $TYPE" - if [ "$TYPE" == "io1" ]; then volume_opts="$volume_opts --iops $IOPS"; fi + local IOPS_TYPES=( io1 io2 gp3 ) + if [[ " ${IOPS_TYPES[*]} " =~ " ${TYPE} " ]]; then volume_opts="$volume_opts --iops $IOPS"; fi + if [ "$TYPE" == "gp3" ]; then volume_opts="$volume_opts --throughput $THROUGHPUT"; fi if [ "$ENCRYPTED" == "1" ]; then volume_opts="$volume_opts --encrypted"; fi local timestamp=$(date "+%F %T UTC%z") # YYYY-mm-dd HH:MM:SS UTC+0000 local volume="" for i in $(eval echo "{0..$max_attempts}") ; do + + # The $instance_tags variable could be empty and will cause a TagSpecifications[0].Tags[0] error if + # it is passed as an empty value because it must be comma-separated from the other key-value pairs. + # Use a Shell Parameter Expansion to determine if the variable contains a value or not. If it has a value, + # append a comma at the end so the aws cli syntax is compliant when it is subbed into the tag_specification variable. + local instance_tags=${instance_tags:+${instance_tags},} + local tag_specification="ResourceType=volume,Tags=[$instance_tags{Key=source-instance,Value=$instance_id},{Key=amazon-ebs-autoscale-creation-time,Value=$timestamp}]" + + # Note: Shellcheck says the $vars in this command should be double quoted to prevent globbing and word-splitting, + # but this ends up making the '--encrypted' argument to fail during the execution of the install script. Conversely, NOT putting double-quotes + # around $tag_specification causes a parsing error due to the space in the $timestamp value (added to $tag_specification above). + local volume=$(\ aws ec2 create-volume \ --region $region \ --availability-zone $availability_zone \ $volume_opts \ - --tag-specification "ResourceType=volume,Tags=[$instance_tags,{Key=source-instance,Value=$instance_id},{Key=amazon-ebs-autoscale-creation-time,Value=$timestamp}]" \ + --tag-specification "$tag_specification" \ 2> $tmpfile ) @@ -272,7 +295,7 @@ function create_and_attach_volume() { logthis "Could not create a volume after $i attempts. Last response was: $volume" break fi - sleep $(( 2 ** i )) + sleep $(( 2 ** i + $RANDOM % 3)) done local volume_id=`echo $volume | jq -r '.VolumeId'` @@ -341,4 +364,4 @@ function create_and_attach_volume() { echo $device } -create_and_attach_volume \ No newline at end of file +create_and_attach_volume diff --git a/bin/ebs-autoscale b/bin/ebs-autoscale index 17dddb6..05325ee 100755 --- a/bin/ebs-autoscale +++ b/bin/ebs-autoscale @@ -32,8 +32,11 @@ initialize +MIN_EBS_VOLUME_SIZE=$(get_config_value .limits.min_ebs_volume_size) +MAX_EBS_VOLUME_SIZE=$(get_config_value .limits.max_ebs_volume_size) MAX_LOGICAL_VOLUME_SIZE=$(get_config_value .limits.max_logical_volume_size) MAX_EBS_VOLUME_COUNT=$(get_config_value .limits.max_ebs_volume_count) +INITIAL_UTILIZATION_THRESHOLD=$(get_config_value .limits.initial_utilization_threshold) FILE_SYSTEM=$(get_config_value .filesystem) @@ -86,7 +89,7 @@ get_num_devices() { break fi - sleep $(( 2 ** i )) + sleep $(( 2 ** i + $RANDOM %3 )) done echo "$attached_volumes" @@ -96,8 +99,8 @@ calc_threshold() { # calculates percent utilization threshold for adding additional ebs volumes # as more ebs volumes are added, the threshold level increases - local num_devices=$(get_num_devices) - local threshold=50 + local num_devices=$1 + local threshold=${INITIAL_UTILIZATION_THRESHOLD} if [ "$num_devices" -ge "4" ] && [ "$num_devices" -le "6" ]; then threshold=80 @@ -106,7 +109,7 @@ calc_threshold() { elif [ "$num_devices" -gt "10" ]; then threshold=90 else - threshold=50 + threshold=${INITIAL_UTILIZATION_THRESHOLD} fi echo ${threshold} @@ -115,25 +118,42 @@ calc_threshold() { calc_new_size() { # calculates the size to use for new ebs volumes to expand space # new volume sizes increase as the number of attached volumes increase - - local num_devices=$(get_num_devices) - local new_size=150 + local num_devices=$1 + #local num_devices=$(get_num_devices) + local new_size=$MIN_EBS_VOLUME_SIZE if [ "$num_devices" -ge "4" ] && [ "$num_devices" -le "6" ]; then - new_size=300 + if [ "$MAX_EBS_VOLUME_SIZE" -ge "299" ] && [ "$MIN_EBS_VOLUME_SIZE" -le "299" ]; then + new_size=300 + elif [ "$MIN_EBS_VOLUME_SIZE" -ge "299" ]; then + new_size=$MIN_EBS_VOLUME_SIZE + else + new_size=$MAX_EBS_VOLUME_SIZE + fi elif [ "$num_devices" -gt "6" ] && [ "$num_devices" -le "10" ]; then - new_size=1000 + if [ "$MAX_EBS_VOLUME_SIZE" -ge "999" ] && [ "$MIN_EBS_VOLUME_SIZE" -le "999" ]; then + new_size=1000 + elif [ "$MIN_EBS_VOLUME_SIZE" -ge "999" ]; then + new_size=$MIN_EBS_VOLUME_SIZE + else + new_size=$MAX_EBS_VOLUME_SIZE + fi elif [ "$num_devices" -gt "10" ]; then - new_size=1500 + new_size=$MAX_EBS_VOLUME_SIZE else - new_size=150 + if [ "$MAX_EBS_VOLUME_SIZE" -ge "149" ]; then + new_size=$MIN_EBS_VOLUME_SIZE + else + new_size=$MAX_EBS_VOLUME_SIZE + fi fi echo ${new_size} } add_space () { - local num_devices=$(get_num_devices) + #local num_devices=$(get_num_devices) + local num_devices=$1 if [ "${num_devices}" -ge "$MAX_EBS_VOLUME_COUNT" ]; then logthis "No more volumes can be safely added." return 0 @@ -141,7 +161,7 @@ add_space () { local curr_size=$(df -BG ${MOUNTPOINT} | grep ${MOUNTPOINT} | awk '{print $2} ' | cut -d'G' -f1) if [ "${curr_size}" -lt "$MAX_LOGICAL_VOLUME_SIZE" ]; then - local vol_size=$(calc_new_size) + local vol_size=$(calc_new_size ${num_devices}) logthis "Extending logical volume ${MOUNTPOINT} by ${vol_size}GB" DEVICE=$(${BASEDIR}/create-ebs-volume --size ${vol_size} --max-attached-volumes ${MAX_EBS_VOLUME_COUNT}) @@ -181,25 +201,29 @@ LOG_COUNT=$LOG_INTERVAL # time in seconds between event loops # keep this low so that rapid increases in utilization are detected DETECTION_INTERVAL=$(get_config_value .detection_interval) - -THRESHOLD=$(calc_threshold) +# get the number of devices once when the script first starts +NUM_DEVICES=$(get_num_devices) +THRESHOLD=$(calc_threshold "${NUM_DEVICES}") while true; do - NUM_DEVICES=$(get_num_devices) + STATS=$(df -BG ${MOUNTPOINT} | grep -v Filesystem) TOTAL_SIZE=$(echo ${STATS} | awk '{print $2}') USED=$(echo ${STATS} | awk '{print $3}') AVAILABLE=$(echo ${STATS} | awk '{print $4}') PCT_UTILIZATION=$(echo ${STATS} | awk '{print $5}' | cut -d"%" -f1 -) if [ $PCT_UTILIZATION -ge "${THRESHOLD}" ]; then + # get number of devices only when we need to add more space + NUM_DEVICES=$(get_num_devices) logthis "LOW DISK (${PCT_UTILIZATION}%): Adding more." - add_space - LOG_COUNT=LOG_INTERVAL + add_space "$NUM_DEVICES" + NUM_DEVICES=$(expr $NUM_DEVICES + 1 ) + THRESHOLD=$(calc_threshold "$NUM_DEVICES") + LOG_COUNT=$LOG_INTERVAL fi if [ "${LOG_COUNT}" -ge "${LOG_INTERVAL}" ]; then logthis "Devices ${NUM_DEVICES} : Size ${TOTAL_SIZE} : Used ${USED} : Available ${AVAILABLE} : Used% ${PCT_UTILIZATION}% : Threshold ${THRESHOLD}%" LOG_COUNT=0 fi - THRESHOLD=$(calc_threshold) LOG_COUNT=$(expr $LOG_COUNT + 1 ) sleep $DETECTION_INTERVAL done diff --git a/config/ebs-autoscale.json b/config/ebs-autoscale.json index f20ef99..a8507de 100644 --- a/config/ebs-autoscale.json +++ b/config/ebs-autoscale.json @@ -2,21 +2,25 @@ "mountpoint": "%%MOUNTPOINT%%", "filesystem": "%%FILESYSTEM%%", "lvm": { - "volume_group": "autoscale_vg", - "logical_volume": "autoscale_lv" + "volume_group": "autoscale_vg", + "logical_volume": "autoscale_lv" }, "volume": { "type": "%%VOLUMETYPE%%", - "iops": 3000, + "iops": "%%VOLUMEIOPS%%", + "throughput": "%%VOLUMETHOUGHPUT%%", "encrypted": 1 }, - "detection_interval": 1, + "detection_interval": 2, "limits": { - "max_logical_volume_size": 8000, - "max_ebs_volume_count": 16 + "min_ebs_volume_size": "%%MINEBSVOLUMESIZE%%", + "max_ebs_volume_size": "%%MAXEBSVOLUMESIZE%%", + "max_logical_volume_size": "%%MAXLOGICALVOLUMESIZE%%", + "max_ebs_volume_count": "%%MAXATTACHEDVOLUMES%%", + "initial_utilization_threshold": "%%INITIALUTILIZATIONTHRESHOLD%%" }, "logging": { "log_file": "/var/log/ebs-autoscale.log", "log_interval": 300 } -} +} \ No newline at end of file diff --git a/install.sh b/install.sh index 0674c33..1a26bb2 100644 --- a/install.sh +++ b/install.sh @@ -56,16 +56,52 @@ Options -t, --volume-type VOLUMETYPE Volume type (default: gp3) - -s, --initial-size SIZE + --volume-iops VOLUMEIOPS + Volume IOPS for gp3, io1, io2 (default: 3000) + + --volume-throughput VOLUMETHOUGHPUT + Volume throughput for gp3 (default: 125) + + --min-ebs-volume-size SIZE_GB + Mimimum size in GB of new volumes created by the instance. + (Default: 150) + + --max-ebs-volume-size SIZE_GB + Maximum size in GB of new volumes created by the instance. + (Default: 1500) + + --max-total-created-size SIZE_GB + Maximum total size in GB of all volumes created by the instance. + (Default: 8000) + + --max-attached-volumes N + Maximum number of attached volumes. (Default: 16) + + --initial-utilization-threshold N + Initial disk utilization treshold for scale-up. (Default: 50) + + -s, --initial-size SIZE_GB Initial size of the volume in GB. (Default: 200) Only used if --initial-device is NOT specified. + + -i, --imdsv2 + Enable imdsv2 for instance metadata API requests. EOF ) MOUNTPOINT=/scratch +# defaults to set into ebs-autoscale.json SIZE=200 VOLUMETYPE=gp3 +VOLUMEIOPS=3000 +VOLUMETHOUGHPUT=125 +MIN_EBS_VOLUME_SIZE=150 +MAX_EBS_VOLUME_SIZE=1500 +MAX_LOGICAL_VOLUME_SIZE=8000 +MAX_ATTACHED_VOLUMES=16 +INITIAL_UTILIZATION_THRESHOLD=50 + DEVICE="" FILE_SYSTEM=btrfs BASEDIR=$(dirname $0) @@ -73,8 +109,6 @@ BASEDIR=$(dirname $0) . ${BASEDIR}/shared/utils.sh -initialize - # parse options PARAMS="" while (( "$#" )); do @@ -87,6 +121,34 @@ while (( "$#" )); do VOLUMETYPE=$2 shift 2 ;; + --volume-iops) + VOLUMEIOPS=$2 + shift 2 + ;; + --volume-throughput) + VOLUMETHOUGHPUT=$2 + shift 2 + ;; + --min-ebs-volume-size) + MIN_EBS_VOLUME_SIZE=$2 + shift 2 + ;; + --max-ebs-volume-size) + MAX_EBS_VOLUME_SIZE=$2 + shift 2 + ;; + --max-total-created-size) + MAX_LOGICAL_VOLUME_SIZE=$2 + shift 2 + ;; + --max-attached-volumes) + MAX_ATTACHED_VOLUMES=$2 + shift 2 + ;; + --initial-utilization-threshold) + INITIAL_UTILIZATION_THRESHOLD=$2 + shift 2 + ;; -d|--initial-device) DEVICE=$2 shift 2 @@ -99,6 +161,10 @@ while (( "$#" )); do MOUNTPOINT=$2 shift 2 ;; + -i|--imdsv2) + IMDSV2="true" + shift 1 + ;; -h|--help) echo "$USAGE" exit @@ -119,6 +185,8 @@ done eval set -- "$PARAMS" +initialize + # for backwards compatibility evaluate positional parameters like previous 2.0.x and 2.1.x releases # this will be removed in the future if [ ! -z "$PARAMS" ]; then @@ -150,7 +218,14 @@ cp ${BASEDIR}/config/ebs-autoscale.logrotate /etc/logrotate.d/ebs-autoscale cat ${BASEDIR}/config/ebs-autoscale.json | \ sed -e "s#%%MOUNTPOINT%%#${MOUNTPOINT}#" | \ sed -e "s#%%VOLUMETYPE%%#${VOLUMETYPE}#" | \ - sed -e "s#%%FILESYSTEM%%#${FILE_SYSTEM}#" \ + sed -e "s#%%VOLUMEIOPS%%#${VOLUMEIOPS}#" | \ + sed -e "s#%%VOLUMETHOUGHPUT%%#${VOLUMETHOUGHPUT}#" | \ + sed -e "s#%%FILESYSTEM%%#${FILE_SYSTEM}#" | \ + sed -e "s#%%MINEBSVOLUMESIZE%%#${MIN_EBS_VOLUME_SIZE}#" | \ + sed -e "s#%%MAXEBSVOLUMESIZE%%#${MAX_EBS_VOLUME_SIZE}#" | \ + sed -e "s#%%MAXLOGICALVOLUMESIZE%%#${MAX_LOGICAL_VOLUME_SIZE}#" | \ + sed -e "s#%%MAXATTACHEDVOLUMES%%#${MAX_ATTACHED_VOLUMES}#" | \ + sed -e "s#%%INITIALUTILIZATIONTHRESHOLD%%#${INITIAL_UTILIZATION_THRESHOLD}#" \ > /etc/ebs-autoscale.json ## Create filesystem diff --git a/shared/utils.sh b/shared/utils.sh index 9560734..d1a216e 100644 --- a/shared/utils.sh +++ b/shared/utils.sh @@ -28,10 +28,22 @@ # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. +function get_metadata() { + local key=$1 + local metadata_ip='169.254.169.254' + + if [ ! -z "$IMDSV2" ]; then + local token=$(curl -s -X PUT "http://$metadata_ip/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 60") + local token_wrapper='-H "X-aws-ec2-metadata-token: $token"' + fi + + echo `curl -s $token_wrapper http://$metadata_ip/latest/meta-data/$key` +} + function initialize() { - export AWS_AZ=$(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone/) + export AWS_AZ=$(get_metadata placement/availability-zone) export AWS_REGION=$(echo ${AWS_AZ} | sed -e 's/[a-z]$//') - export INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id) + export INSTANCE_ID=$(get_metadata instance-id) export EBS_AUTOSCALE_CONFIG_FILE=/etc/ebs-autoscale.json } @@ -51,11 +63,6 @@ function get_config_value() { jq -r $filter $EBS_AUTOSCALE_CONFIG_FILE } -function get_metadata() { - local key=$1 - echo `curl -s http://169.254.169.254/latest/meta-data/$key` -} - function logthis() { echo "[`date`] $1" >> $(get_config_value .logging.log_file) }