From 17cb8d3308b48a6354f62976fb6f7174ad7daabe Mon Sep 17 00:00:00 2001 From: Dermot Bradley Date: Sat, 5 Feb 2022 03:36:07 +0000 Subject: [PATCH 1/2] growpart: Add support for overprovisioning Add option to 'growpart' to specify percentage of device that should be left unallocated when growing partition. This is intended for consumer SSDs and SD cards where the performance and/or lifetime of these devices can be improved if some disk space (in addition to any the device "hides" from users) is left unallocated. Overprovisioning code caters for several distinct scenarios: (1) MSDOS/MBR partitioned disk where the disk is >2TB and so MBR partitions cannot extend beyond 2TB - if disk is larger than (2TB + overprovisioning requirement) then nothing needs to be done. (2) MSDOS/MBR partitioned disk where the disk is >2TB and so MBR partitions cannot extend beyond 2TB - if disk is not larger than (2TB + overprovisioning requirement) then *some* overprovisioning space still needs to be reserved. (3) MSDOS/MBR partitioned disk <=2TB where overprovisioning space needs to be reserved. (4) GPT partitioned disk where overprovisioning space needs to be reserved. Also added a testcase script, test-growpart-overprovision. --- bin/growpart | 94 ++++++++++++++++++++++---- test/test-growpart-overprovision | 112 +++++++++++++++++++++++++++++++ tox.ini | 1 + 3 files changed, 195 insertions(+), 12 deletions(-) create mode 100755 test/test-growpart-overprovision diff --git a/bin/growpart b/bin/growpart index e4be72d..2130565 100755 --- a/bin/growpart +++ b/bin/growpart @@ -119,22 +119,32 @@ Usage() { ${0##*/} disk partition rewrite partition table so that partition takes up all the space it can options: - -h | --help print Usage and exit - --fudge F if part could be resized, but change would be - less than 'F' bytes, do not resize (default: ${FUDGE}) - -N | --dry-run only report what would be done, show new 'sfdisk -d' - -v | --verbose increase verbosity / debug - -u | --update R update the the kernel partition table info after growing - this requires kernel support and 'partx --update' - R is one of: - - 'auto' : [default] update partition if possible - - 'force' : try despite sanity checks (fail on failure) - - 'off' : do not attempt - - 'on' : fail if sanity checks indicate no support + -h | --help print Usage and exit + --free-percent F resize so that specified percentage F of the disk is + not used in total (not just by this partition). This + is useful for consumer SSD or SD cards where a small + percentage unallocated can improve device lifetime. + --fudge F if part could be resized, but change would be less + than 'F' bytes, do not resize (default: ${FUDGE}) + -N | --dry-run only report what would be done, show new 'sfdisk -d' + -v | --verbose increase verbosity / debug + -u | --update R update the the kernel partition table info after + growing this requires kernel support and + 'partx --update' + R is one of: + - 'auto' : [default] update partition if possible + - 'force' : try despite sanity checks (fail on + failure) + - 'off' : do not attempt + - 'on' : fail if sanity checks indicate no + support Example: - ${0##*/} /dev/sda 1 Resize partition 1 on /dev/sda + + - ${0##*/} --free-percent=10 /dev/sda 1 + Resize partition 1 on /dev/sda so that 10% of the disk is unallocated EOF } @@ -291,6 +301,7 @@ resize_sfdisk() { local pt_start pt_size pt_end max_end new_size change_info dpart local sector_num sector_size disk_size tot out + local excess_sectors free_percent_sectors remaining_free_sectors LANG=C rqe sfd_list sfdisk --list --unit=S "$DISK" >"$tmp" || fail "failed: sfdisk --list $DISK" @@ -384,6 +395,52 @@ resize_sfdisk() { max_end=$((${sector_num}-${gpt_second_size})) fi + if [ -n "${free_percent}" ]; then + free_percent_sectors=$((sector_num/100*free_percent)) + + if [ "$format" = "dos" ]; then + if [ $(($disk_size/512)) -ge $((mbr_max_512+free_percent_sectors)) ]; then + # If MBR partitioned disk larger than 2TB and + # remaining space over 2TB boundary is greater + # than the requested overprovisioning sectors + # then do not change max_end. + debug 1 "WARNING: Additional unused space on MBR/dos partitioned disk" \ + "is larger than requested percent of overprovisioning." + elif [ $sector_num -gt $mbr_max_512 ]; then + # If only some of the overprovisioning sectors + # are over the 2TB boundary then reduce max_end + # by the remaining number of overprovisioning + # sectors + excess_sectors=$((sector_num-mbr_max_512)) + remaining_free_sectors=$((free_percent_sectors-excess_sectors)) + debug 1 "reserving ${remaining_free_sectors} sectors from MBR maximum for overprovisioning" + max_end=$((max_end-renaming_free_sectors)) + else + # Shrink max_end to keep X% of whole disk unused + # (for overprovisioning) + debug 1 "reserving ${free_percent_sectors} sectors (${free_percent}%) for overprovisioning" + max_end=$((max_end-free_percent_sectors)) + fi + + if [ $max_end -lt $pt_end ]; then + nochange "partition ${PART} could not be grown while leaving" \ + "${free_percent}% (${free_percent_sectors} sectors) free on device" + return + fi + else + # Shrink max_end to keep X% of whole disk unused + # (for overprovisioning) + if [ $max_end -lt $((sector_num-free_percent_sectors)) ]; then + debug 1 "reserving ${free_percent_sectors} sectors (${free_percent}%) for overprovisioning" + max_end=$((max_end-free_percent_sectors)) + else + nochange "partition ${PART} could not be grown while leaving" \ + "${free_percent}% (${free_percent_sectors} sectors) free on device" + return + fi + fi + fi + debug 1 "max_end=${max_end} tot=${sector_num} pt_end=${pt_end}" \ "pt_start=${pt_start} pt_size=${pt_size}" [ $((${pt_end})) -eq ${max_end} ] && { @@ -873,6 +930,19 @@ while [ $# -ne 0 ]; do Usage exit 0 ;; + --free-percent|--free-percent=*) + if [ "${cur#--free-percent=}" != "$cur" ]; then + next="${cur#--free-percent=}" + else + shift + fi + if [ "$next" -gt 0 ] 2>/dev/null && + [ "$next" -lt 100 ] 2>/dev/null; then + free_percent=$next + else + fail "unknown/invalid --free-percent option: $next" + fi + ;; --fudge) FUDGE=${next} shift diff --git a/test/test-growpart-overprovision b/test/test-growpart-overprovision new file mode 100755 index 0000000..3a1304a --- /dev/null +++ b/test/test-growpart-overprovision @@ -0,0 +1,112 @@ +#!/bin/bash +# NEED_ROOT + +set -e + +[ "$(id -u)" = "0" ] || + { echo "sorry, must be root"; exit 1; } + +PT_TYPE="${PT_TYPE:-dos}" # dos or gpt +size=${DISK_SIZE_NEW:-100M} +osize=${DISK_SIZE_ORIG:-50M} +freepercent=${OVER_PROVISION_PERCENT:-10} + +LODEV="" +TEMP_D="" + +clearparts() { + # read /proc/partitions, clearing any partitions on dev (/dev/loopX) + local dev="$1" + local short=${dev##*/} parts="" part="" + parts=$(awk '$4 ~ m { sub(m,"",$4); print $4 }' \ + "m=${short}p" /proc/partitions) + [ -z "$parts" ] && return + echo "clearing parts [$parts] from $dev" + for part in $parts; do + echo "delpart $LODEV $part" + delpart $LODEV $part + done + udevadm settle +} +cleanup() { + if [ -n "$LODEV" ]; then + clearparts "$LODEV" + echo "losetup --detach $LODEV"; + losetup --detach "$LODEV"; + udevadm settle + fi + [ ! -d "${TEMP_D}" ] || rm -Rf "${TEMP_D}" +} +rq() { + local out="${TEMP_D}/out" + "$@" > "$out" 2>&1 || { echo "FAILED:" "$@"; cat "$out"; return 1; } +} + +TEMP_D=$(mktemp -d ${TMPDIR:-/tmp}/${0##*/}.XXXXXX) +trap cleanup EXIT + +img="${TEMP_D}/disk.img" + +echo "Partitioning $PT_TYPE orig_size=$osize grow_size=$size, overprovisioning=$freepercent%." +echo "growpart is $(which growpart)" +rm -f $img + +truncate --size $osize "$img" + +label_flag="--label=${PT_TYPE}" +echo "2048," | rq sfdisk $label_flag --force --unit=S "$img" + +truncate --size "$size" "$img" + +lodev=$(losetup --show --find "$img") +LODEV=$lodev +echo "set up $lodev" + +# clear any old ones that might be around (LP: #1136781) +clearparts "$lodev" +partx --add $lodev +lodevpart="${lodev}p1" + +echo "==== before ====" +grep "${lodev##*/}" /proc/partitions +sfdisk --list --unit=S "$lodev" + +errfile="${TEMP_D}/growpart.err" +growpart -v -v --free-percent "$freepercent" "$lodev" 1 2>"$errfile" || { + rc=$? + echo "failed [$rc]: growpart -v -v --free-percent $freepercent $lodev 1" + cat "$errfile" 1>&2 + exit $rc +} + +out=$(grep "FLOCK:.*releasing exclusive lock" "$errfile") || : +if [ -z "$out" ]; then + echo "ERROR: growpart stderr did not mention releasing lock" + exit 1 +fi + +echo === growpart stderr === +cat "$errfile" + +echo "==== after ====" +grep "${lodev##*/}" /proc/partitions +sfdisk --list --unit=S "$lodev" + +enddevice=$(grep "${lodev##*/}" /proc/partitions|grep -v "${lodevpart##*/}"|awk '{print $3}') +endpart=$(grep "${lodevpart##*/}" /proc/partitions|awk '{print $3}') +# Subtract the following from disk image end: +# - required percentage of overprovisioning +# - 1024 MiB (the partition start, 2048 sectors of 512 bytes) +# - 17 MiB (rounded up value of MBR padding of 33 sectors for GPT conversion) +# to calculate the expected end of resized partition. +expectedendpart=$((enddevice-(enddevice/100*freepercent)-1024-17)) +echo +if [ $endpart = $expectedendpart ]; then + echo "Final partition size matches expected partition size" + echo +else + echo "ERROR: final partition size of $endpart is different than expected size of $expectedendpart" + exit 1 +fi + +# vi: ts=4 noexpandtab diff --git a/tox.ini b/tox.ini index bd18aec..e528d33 100644 --- a/tox.ini +++ b/tox.ini @@ -21,6 +21,7 @@ shfiles = test/test-growpart test/test-growpart-fsimage test/test-growpart-fsimage-middle + test/test-growpart-overprovision test/test-growpart-lvm test/test-growpart-start-matches-size test/test-mic From b72e55f4e7e5959c29061218bd555c9de2109f21 Mon Sep 17 00:00:00 2001 From: Dermot Bradley Date: Sat, 16 Apr 2022 19:05:22 +0100 Subject: [PATCH 2/2] growpart: correct some overprovisioning logic Correct some of the overprovisioning logic. Change overprovisioning testcase to not require root. Also correct some off-by-one errors in the existing growpart code and correct some existing testcases affected by this. --- bin/growpart | 49 +++++++-------- test/test-growpart-fsimage-middle | 4 +- test/test-growpart-overprovision | 85 ++++++++------------------- test/test-growpart-start-matches-size | 2 +- 4 files changed, 52 insertions(+), 88 deletions(-) diff --git a/bin/growpart b/bin/growpart index 2130565..4cc0440 100755 --- a/bin/growpart +++ b/bin/growpart @@ -360,7 +360,7 @@ resize_sfdisk() { pt_start=$(awk '$1 == pt { print $4 }' "pt=${dpart}" <"${dump_mod}") && pt_size=$(awk '$1 == pt { print $6 }' "pt=${dpart}" <"${dump_mod}") && [ -n "${pt_start}" -a -n "${pt_size}" ] && - pt_end=$((${pt_size}+${pt_start})) || + pt_end=$((${pt_size} + ${pt_start} - 1)) || fail "failed to get start and end for ${dpart} in ${DISK}" # find the minimal starting location that is >= pt_end @@ -369,6 +369,8 @@ resize_sfdisk() { min=${sector_num} pt_end=${pt_end} "${dump_mod}") && [ -n "${max_end}" ] || fail "failed to get max_end for partition ${PART}" + # As sector numbering starts from 0 need to reduce value by 1. + max_end=$((max_end - 1)) if [ "$format" = "gpt" ]; then # sfdisk respects 'last-lba' in input, and complains about @@ -388,11 +390,11 @@ resize_sfdisk() { local gpt_second_size="33" if [ "${max_end}" -gt "$((${sector_num}-${gpt_second_size}))" ]; then - # if mbr allow subsequent conversion to gpt without shrinking the - # partition. safety net at cost of 33 sectors, seems reasonable. - # if gpt, we can't write there anyway. + # if MBR, allow subsequent conversion to GPT without shrinking + # the partition and safety net at cost of 33 sectors seems + # reasonable. If GPT, we can't write there anyway. debug 1 "padding ${gpt_second_size} sectors for gpt secondary header" - max_end=$((${sector_num}-${gpt_second_size})) + max_end=$((${sector_num} - ${gpt_second_size} - 1)) fi if [ -n "${free_percent}" ]; then @@ -410,30 +412,30 @@ resize_sfdisk() { # If only some of the overprovisioning sectors # are over the 2TB boundary then reduce max_end # by the remaining number of overprovisioning - # sectors + # sectors. excess_sectors=$((sector_num-mbr_max_512)) - remaining_free_sectors=$((free_percent_sectors-excess_sectors)) + remaining_free_sectors=$((free_percent_sectors - excess_sectors)) debug 1 "reserving ${remaining_free_sectors} sectors from MBR maximum for overprovisioning" - max_end=$((max_end-renaming_free_sectors)) + max_end=$((max_end - remaining_free_sectors)) else # Shrink max_end to keep X% of whole disk unused - # (for overprovisioning) + # (for overprovisioning). debug 1 "reserving ${free_percent_sectors} sectors (${free_percent}%) for overprovisioning" max_end=$((max_end-free_percent_sectors)) fi - if [ $max_end -lt $pt_end ]; then + if [ ${max_end} -lt ${pt_end} ]; then nochange "partition ${PART} could not be grown while leaving" \ "${free_percent}% (${free_percent_sectors} sectors) free on device" return fi else # Shrink max_end to keep X% of whole disk unused - # (for overprovisioning) - if [ $max_end -lt $((sector_num-free_percent_sectors)) ]; then - debug 1 "reserving ${free_percent_sectors} sectors (${free_percent}%) for overprovisioning" - max_end=$((max_end-free_percent_sectors)) - else + # (for overprovisioning). + debug 1 "reserving ${free_percent_sectors} sectors (${free_percent}%) for overprovisioning" + max_end=$((max_end-free_percent_sectors)) + + if [ ${max_end} -lt ${pt_end} ]; then nochange "partition ${PART} could not be grown while leaving" \ "${free_percent}% (${free_percent_sectors} sectors) free on device" return @@ -453,9 +455,9 @@ resize_sfdisk() { return } - # now, change the size for this partition in ${dump_out} to be the - # new size - new_size=$((${max_end}-${pt_start})) + # Now, change the size for this partition in ${dump_out} to be the + # new size. + new_size=$((${max_end} - ${pt_start} + 1)) sed "\|^\s*${dpart} |s/\(.*\)${pt_size},/\1${new_size},/" "${dump_out}" \ >"${new_out}" || fail "failed to change size in output" @@ -581,18 +583,17 @@ resize_sgdisk() { pt_end=$(awk '$1 == '"${PART}"' { print $3 }' "${pt_data}") && [ -n "${pt_end}" ] || fail "${dev}: failed to get end sector" - # sgdisk start and end are inclusive. start 2048 length 10 ends at 2057. - pt_end=$((pt_end+1)) - pt_size="$((${pt_end} - ${pt_start}))" + # Start and end are inclusive, start 2048 end 2057 is length 10. + pt_size="$((${pt_end} - ${pt_start} + 1))" # Get the last usable sector last=$(awk '/last usable sector is/ { print $NF }' \ "${pt_pretend}") && [ -n "${last}" ] || fail "${dev}: failed to get last usable sector" - # Find the minimal start sector that is >= pt_end + # Find the maximal end sector that is >= pt_end pt_max=$(awk '{ if ($2 >= pt_end && $2 < min) { min = $2 } } END \ - { print min }' min="${last}" pt_end="${pt_end}" \ + { print min-1 }' min="${last}" pt_end="${pt_end}" \ "${pt_data}") && [ -n "${pt_max}" ] || fail "${dev}: failed to find max end sector" @@ -625,7 +626,7 @@ resize_sgdisk() { [ "$DRY_RUN" -ne 0 ] && wouldrun="would-run" # Calculate the new size of the partition - new_size=$((${pt_max} - ${pt_start})) + new_size=$((${pt_max} - ${pt_start} + 1)) change_info="partition=${PART} start=${pt_start}" change_info="${change_info} old: size=${pt_size} end=${pt_end}" change_info="${change_info} new: size=${new_size} end=${pt_max}" diff --git a/test/test-growpart-fsimage-middle b/test/test-growpart-fsimage-middle index 4f2dfbf..be43fcd 100755 --- a/test/test-growpart-fsimage-middle +++ b/test/test-growpart-fsimage-middle @@ -39,8 +39,8 @@ cleanup() { TEMP_D=$(mktemp -d ${TMPDIR:-/tmp}/${0##*/}.XXXXXX) trap cleanup EXIT -expected="CHANGED: partition=3 start=731136 old: size=819200 end=1550336" -expected="${expected} new: size=3330048 end=4061184" +expected="CHANGED: partition=3 start=731136 old: size=819200 end=1550335" +expected="${expected} new: size=3330048 end=4061183" CR=' ' for resizer in sfdisk sgdisk; do diff --git a/test/test-growpart-overprovision b/test/test-growpart-overprovision index 3a1304a..5e87de5 100755 --- a/test/test-growpart-overprovision +++ b/test/test-growpart-overprovision @@ -1,40 +1,17 @@ #!/bin/bash -# NEED_ROOT +# +# Just create an image in the filesystem, then grow it. set -e -[ "$(id -u)" = "0" ] || - { echo "sorry, must be root"; exit 1; } - PT_TYPE="${PT_TYPE:-dos}" # dos or gpt size=${DISK_SIZE_NEW:-100M} osize=${DISK_SIZE_ORIG:-50M} freepercent=${OVER_PROVISION_PERCENT:-10} -LODEV="" TEMP_D="" -clearparts() { - # read /proc/partitions, clearing any partitions on dev (/dev/loopX) - local dev="$1" - local short=${dev##*/} parts="" part="" - parts=$(awk '$4 ~ m { sub(m,"",$4); print $4 }' \ - "m=${short}p" /proc/partitions) - [ -z "$parts" ] && return - echo "clearing parts [$parts] from $dev" - for part in $parts; do - echo "delpart $LODEV $part" - delpart $LODEV $part - done - udevadm settle -} cleanup() { - if [ -n "$LODEV" ]; then - clearparts "$LODEV" - echo "losetup --detach $LODEV"; - losetup --detach "$LODEV"; - udevadm settle - fi [ ! -d "${TEMP_D}" ] || rm -Rf "${TEMP_D}" } rq() { @@ -58,48 +35,34 @@ echo "2048," | rq sfdisk $label_flag --force --unit=S "$img" truncate --size "$size" "$img" -lodev=$(losetup --show --find "$img") -LODEV=$lodev -echo "set up $lodev" - -# clear any old ones that might be around (LP: #1136781) -clearparts "$lodev" -partx --add $lodev -lodevpart="${lodev}p1" - echo "==== before ====" -grep "${lodev##*/}" /proc/partitions -sfdisk --list --unit=S "$lodev" +sfdisk --list --unit=S "$img" -errfile="${TEMP_D}/growpart.err" -growpart -v -v --free-percent "$freepercent" "$lodev" 1 2>"$errfile" || { - rc=$? - echo "failed [$rc]: growpart -v -v --free-percent $freepercent $lodev 1" - cat "$errfile" 1>&2 - exit $rc -} - -out=$(grep "FLOCK:.*releasing exclusive lock" "$errfile") || : -if [ -z "$out" ]; then - echo "ERROR: growpart stderr did not mention releasing lock" +err="${TEMP_D}/gp.err" +out="${TEMP_D}/gp.out" +if ! growpart -v -v --free-percent "$freepercent" "$img" 1 2>"$err" > "$out"; then + cat "$err" "$out" + echo "failed" exit 1 fi - -echo === growpart stderr === -cat "$errfile" +echo "==== growpart-stderr ====" +cat "$err" +echo "==== growpart-stdout ====" +cat "$out" +grep -q "^CHANGED:" "$out" || + { echo "did not find 'CHANGED'"; exit 1; } echo "==== after ====" -grep "${lodev##*/}" /proc/partitions -sfdisk --list --unit=S "$lodev" - -enddevice=$(grep "${lodev##*/}" /proc/partitions|grep -v "${lodevpart##*/}"|awk '{print $3}') -endpart=$(grep "${lodevpart##*/}" /proc/partitions|awk '{print $3}') -# Subtract the following from disk image end: -# - required percentage of overprovisioning -# - 1024 MiB (the partition start, 2048 sectors of 512 bytes) -# - 17 MiB (rounded up value of MBR padding of 33 sectors for GPT conversion) -# to calculate the expected end of resized partition. -expectedendpart=$((enddevice-(enddevice/100*freepercent)-1024-17)) +sfdisk --list --unit=S "$img" + +enddevice=$(sfdisk --list --unit=S "$img" | grep "Disk $img:" | awk '{print $7}') +endpart=$(sfdisk --list --unit=S "$img" | grep "$img" | grep -v "Disk" | awk '{print $3}') +# Subtract the following from disk image end in sectors: +# - required number of overprovisioning sectors to leave unused +# - 33 sectors (MBR padding for GPT conversion) +# - 1 (as sector numbers start from 0) +# to calculate the expected end of resized partition in sectors. +expectedendpart=$((enddevice-(enddevice/100*freepercent)-33-1)) echo if [ $endpart = $expectedendpart ]; then echo "Final partition size matches expected partition size" diff --git a/test/test-growpart-start-matches-size b/test/test-growpart-start-matches-size index 50ac294..3f13148 100755 --- a/test/test-growpart-start-matches-size +++ b/test/test-growpart-start-matches-size @@ -92,4 +92,4 @@ trap cleanup EXIT # the sfdisk and sgdisk resizers result in slightly different output, # because of course they do. test_resize sfdisk 1026048 3168223 -test_resize sgdisk 1026048 3166208 +test_resize sgdisk 1026048 3166207