From ca067a1a5381a915c6ab3858390fb38fb5f79286 Mon Sep 17 00:00:00 2001 From: meo Date: Thu, 12 Dec 2024 11:57:04 +0100 Subject: [PATCH 1/7] add: test on MacOS Add test for MacOS in Github action Maybe see this to add test for FreeBSD : https://github.com/marketplace/actions/freebsd-vm --- .github/workflows/test.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4239539..2332e13 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,7 +9,7 @@ on: branches: [ master ] jobs: - shellcheck: + linuxTest: name: Test runs-on: ubuntu-latest steps: @@ -17,3 +17,12 @@ jobs: - name: Run testing script run: ./testing.sh + + macOsTest: + name: Test + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + + - name: Run testing script on macOS + run: ./testing.sh \ No newline at end of file From 339295dab616c6955b5516cab148a4b59c7a3a2b Mon Sep 17 00:00:00 2001 From: meo Date: Thu, 12 Dec 2024 12:13:33 +0100 Subject: [PATCH 2/7] add: log to test adding log to the test for better understanding --- .github/workflows/test.yml | 4 ++-- testing.sh | 36 +++++++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2332e13..8bc609f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ on: jobs: linuxTest: - name: Test + name: Test on Linux runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -19,7 +19,7 @@ jobs: run: ./testing.sh macOsTest: - name: Test + name: Test on macOS runs-on: macos-latest steps: - uses: actions/checkout@v4 diff --git a/testing.sh b/testing.sh index b46f74f..5882221 100755 --- a/testing.sh +++ b/testing.sh @@ -10,6 +10,26 @@ log_error_file=./error.log test_data_src=./test/pool test_pool_data_path=./testing_data +## Color Constants + +# Reset +Color_Off='\033[0m' # Text Reset + +# Regular Colors +Red='\033[0;31m' # Red +Green='\033[0;32m' # Green +Yellow='\033[0;33m' # Yellow +Cyan='\033[0;36m' # Cyan + +## Functions + +# print a given text entirely in a given color +function color_echo () { + color=$1 + text=$2 + echo -e "${color}${text}${Color_Off}" +} + function prepare() { # cleanup rm -f $log_std_file @@ -24,7 +44,7 @@ function prepare() { function assertions() { # check error log is empty if grep -q '[^[:space:]]' $log_error_file; then - echo "error log is not empty!" + color_echo "$Red" "error log is not empty!" cat $log_error_file exit 1 fi @@ -44,21 +64,30 @@ function assert_matching_file_not_copied() { fi } +color_echo "$Cyan" "Running tests..." + +color_echo "$Cyan" "Running tests with default options..." prepare ./zfs-inplace-rebalancing.sh $test_pool_data_path >> $log_std_file 2>> $log_error_file cat $log_std_file assertions +color_echo "$Green" "Tests passed!" +color_echo "$Cyan" "Running tests with checksum true and 1 pass..." prepare ./zfs-inplace-rebalancing.sh --checksum true --passes 1 $test_pool_data_path >> $log_std_file 2>> $log_error_file cat $log_std_file assertions +color_echo "$Green" "Tests passed!" +color_echo "$Cyan" "Running tests with checksum false..." prepare ./zfs-inplace-rebalancing.sh --checksum false $test_pool_data_path >> $log_std_file 2>> $log_error_file cat $log_std_file assertions +color_echo "$Green" "Tests passed!" +color_echo "$Cyan" "Running tests with skip-hardlinks false..." prepare ln "$test_pool_data_path/projects/[2020] some project/mp4.txt" "$test_pool_data_path/projects/[2020] some project/mp4.txt.link" ./zfs-inplace-rebalancing.sh --skip-hardlinks false $test_pool_data_path >> $log_std_file 2>> $log_error_file @@ -67,7 +96,9 @@ cat $log_std_file assert_matching_file_copied "mp4.txt" assert_matching_file_copied "mp4.txt.link" assertions +color_echo "$Green" "Tests passed!" +color_echo "$Cyan" "Running tests with skip-hardlinks true..." prepare ln "$test_pool_data_path/projects/[2020] some project/mp4.txt" "$test_pool_data_path/projects/[2020] some project/mp4.txt.link" ./zfs-inplace-rebalancing.sh --skip-hardlinks true $test_pool_data_path >> $log_std_file 2>> $log_error_file @@ -76,3 +107,6 @@ cat $log_std_file assert_matching_file_not_copied "mp4.txt.link" assert_matching_file_not_copied "mp4.txt" assertions +color_echo "$Green" "Tests passed!" + +color_echo "$Green" "All tests passed!" \ No newline at end of file From 12a7865d33d34324bc429ed44bb2ed2dbeb7f97e Mon Sep 17 00:00:00 2001 From: meo Date: Sat, 14 Dec 2024 08:34:17 +0100 Subject: [PATCH 3/7] fix: shellcheck remove the yellow color that is unused in the test --- testing.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/testing.sh b/testing.sh index 5882221..4fe7333 100755 --- a/testing.sh +++ b/testing.sh @@ -18,7 +18,6 @@ Color_Off='\033[0m' # Text Reset # Regular Colors Red='\033[0;31m' # Red Green='\033[0;32m' # Green -Yellow='\033[0;33m' # Yellow Cyan='\033[0;36m' # Cyan ## Functions From 275322a4b6ed562136e855117e243532a027209a Mon Sep 17 00:00:00 2001 From: meo Date: Sat, 14 Dec 2024 10:06:08 +0100 Subject: [PATCH 4/7] fix: fail on lower-casting the `"${variable,,}"` seam to not work on macOS so y remove it for the `skip_hardlink_flag` because it's always lowercase --- zfs-inplace-rebalancing.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zfs-inplace-rebalancing.sh b/zfs-inplace-rebalancing.sh index 07fb3ac..e1a7846 100755 --- a/zfs-inplace-rebalancing.sh +++ b/zfs-inplace-rebalancing.sh @@ -58,7 +58,7 @@ function rebalance() { # check if file has >=2 links in the case of --skip-hardlinks # this shouldn't be needed in the typical case of `find` only finding files with links == 1 # but this can run for a long time, so it's good to double check if something changed - if [[ "${skip_hardlinks_flag,,}" == "true"* ]]; then + if [[ "${skip_hardlinks_flag}" == "true"* ]]; then if [[ "${OSTYPE,,}" == "linux-gnu"* ]]; then # Linux # @@ -254,7 +254,7 @@ color_echo "$Cyan" " Use Checksum: ${checksum_flag}" color_echo "$Cyan" " Skip Hardlinks: ${skip_hardlinks_flag}" # count files -if [[ "${skip_hardlinks_flag,,}" == "true"* ]]; then +if [[ "${skip_hardlinks_flag}" == "true"* ]]; then file_count=$(find "${root_path}" -type f -links 1 | wc -l) else file_count=$(find "${root_path}" -type f | wc -l) @@ -269,7 +269,7 @@ fi # recursively scan through files and execute "rebalance" procedure # in the case of --skip-hardlinks, only find files with links == 1 -if [[ "${skip_hardlinks_flag,,}" == "true"* ]]; then +if [[ "${skip_hardlinks_flag}" == "true"* ]]; then find "$root_path" -type f -links 1 -print0 | while IFS= read -r -d '' file; do rebalance "$file"; done else find "$root_path" -type f -print0 | while IFS= read -r -d '' file; do rebalance "$file"; done From eaa969e1db32a384f77f3bf6e8392cd3944394ac Mon Sep 17 00:00:00 2001 From: meo Date: Sat, 14 Dec 2024 10:27:02 +0100 Subject: [PATCH 5/7] fix: macOS remove all the lower-case casting --- zfs-inplace-rebalancing.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/zfs-inplace-rebalancing.sh b/zfs-inplace-rebalancing.sh index e1a7846..84387e7 100755 --- a/zfs-inplace-rebalancing.sh +++ b/zfs-inplace-rebalancing.sh @@ -59,7 +59,7 @@ function rebalance() { # this shouldn't be needed in the typical case of `find` only finding files with links == 1 # but this can run for a long time, so it's good to double check if something changed if [[ "${skip_hardlinks_flag}" == "true"* ]]; then - if [[ "${OSTYPE,,}" == "linux-gnu"* ]]; then + if [[ "${OSTYPE}" == "linux-gnu"* ]]; then # Linux # # -c --format=FORMAT @@ -68,7 +68,7 @@ function rebalance() { # %h number of hard links hardlink_count=$(stat -c "%h" "${file_path}") - elif [[ "${OSTYPE,,}" == "darwin"* ]] || [[ "${OSTYPE,,}" == "freebsd"* ]]; then + elif [[ "${OSTYPE}" == "darwin"* ]] || [[ "${OSTYPE}" == "freebsd"* ]]; then # Mac OS # FreeBSD # -f format @@ -108,7 +108,7 @@ function rebalance() { tmp_file_path="${file_path}${tmp_extension}" echo "Copying '${file_path}' to '${tmp_file_path}'..." - if [[ "${OSTYPE,,}" == "linux-gnu"* ]]; then + if [[ "${OSTYPE}" == "linux-gnu"* ]]; then # Linux # --reflink=never -- force standard copy (see ZFS Block Cloning) @@ -116,7 +116,7 @@ function rebalance() { # -p -- preserve ACLs to # -x -- stay on one system cp --reflink=never -ax "${file_path}" "${tmp_file_path}" - elif [[ "${OSTYPE,,}" == "darwin"* ]] || [[ "${OSTYPE,,}" == "freebsd"* ]]; then + elif [[ "${OSTYPE}" == "darwin"* ]] || [[ "${OSTYPE}" == "freebsd"* ]]; then # Mac OS # FreeBSD @@ -131,9 +131,9 @@ function rebalance() { fi # compare copy against original to make sure nothing went wrong - if [[ "${checksum_flag,,}" == "true"* ]]; then + if [[ "${checksum_flag}" == "true"* ]]; then echo "Comparing copy against original..." - if [[ "${OSTYPE,,}" == "linux-gnu"* ]]; then + if [[ "${OSTYPE}" == "linux-gnu"* ]]; then # Linux # file attributes @@ -151,7 +151,7 @@ function rebalance() { copy_md5="${copy_md5} $(ls -lha "${tmp_file_path}" | awk '{print $1 " " $3 " " $4}')" # file content copy_md5="${copy_md5} $(md5sum -b "${tmp_file_path}" | awk '{print $1}')" - elif [[ "${OSTYPE,,}" == "darwin"* ]] || [[ "${OSTYPE,,}" == "freebsd"* ]]; then + elif [[ "${OSTYPE}" == "darwin"* ]] || [[ "${OSTYPE}" == "freebsd"* ]]; then # Mac OS # FreeBSD From d4e70de6aba0fb604c8912c44bae15144f41198f Mon Sep 17 00:00:00 2001 From: meo Date: Sat, 14 Dec 2024 12:03:23 +0100 Subject: [PATCH 6/7] fix(perf): going to `stat` remove `lsattr` macOS The command `lsattr` is not supported on macOS so I have remove it switch to `stat` instead of `ls` for better performance and no subshell use of parameter expansion with `%` in place of `awk` for performance and no subshell --- zfs-inplace-rebalancing.sh | 41 +++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/zfs-inplace-rebalancing.sh b/zfs-inplace-rebalancing.sh index 84387e7..ad0ae53 100755 --- a/zfs-inplace-rebalancing.sh +++ b/zfs-inplace-rebalancing.sh @@ -137,37 +137,38 @@ function rebalance() { # Linux # file attributes - original_md5=$(lsattr "${file_path}" | awk '{print $1}') - # file permissions, owner, group - # shellcheck disable=SC2012 - original_md5="${original_md5} $(ls -lha "${file_path}" | awk '{print $1 " " $3 " " $4}')" + original_md5=$(lsattr "${file_path}") + # remove anything after the last space + original_md5=${original_md5% *} + # file permissions, owner, group, size, modification time + original_md5="${original_md5} $(stat -c "%A %U %G %s %Y" "${file_path}")" # file content - original_md5="${original_md5} $(md5sum -b "${file_path}" | awk '{print $1}')" + original_md5="${original_md5} $(md5sum -b "${file_path}")" + # file attributes - copy_md5=$(lsattr "${tmp_file_path}" | awk '{print $1}') - # file permissions, owner, group - # shellcheck disable=SC2012 - copy_md5="${copy_md5} $(ls -lha "${tmp_file_path}" | awk '{print $1 " " $3 " " $4}')" + copy_md5=$(lsattr "${tmp_file_path}") + # remove anything after the last space + copy_md5=${copy_md5% *} + # file permissions, owner, group, size, modification time + copy_md5="${copy_md5} $(stat -c "%A %U %G %s %Y" "${tmp_file_path}")" # file content - copy_md5="${copy_md5} $(md5sum -b "${tmp_file_path}" | awk '{print $1}')" + copy_md5="${copy_md5} $(md5sum -b "${tmp_file_path}")" + # remove the temporary extension + copy_md5=${copy_md5%"${tmp_extension}"} elif [[ "${OSTYPE}" == "darwin"* ]] || [[ "${OSTYPE}" == "freebsd"* ]]; then # Mac OS # FreeBSD - # file attributes - original_md5=$(lsattr "${file_path}" | awk '{print $1}') - # file permissions, owner, group - # shellcheck disable=SC2012 - original_md5="${original_md5} $(ls -lha "${file_path}" | awk '{print $1 " " $3 " " $4}')" + # note: no lsattr on Mac OS or FreeBSD + + # file permissions, owner, group size, modification time + original_md5="${original_md5} $(stat -f "%Sp %Su %Sg %z %m" "${file_path}")" # file content original_md5="${original_md5} $(md5 -q "${file_path}")" - # file attributes - copy_md5=$(lsattr "${tmp_file_path}" | awk '{print $1}') - # file permissions, owner, group - # shellcheck disable=SC2012 - copy_md5="${copy_md5} $(ls -lha "${tmp_file_path}" | awk '{print $1 " " $3 " " $4}')" + # file permissions, owner, group size, modification time + copy_md5="${copy_md5} $(stat -f "%Sp %Su %Sg %z %m" "${tmp_file_path}")" # file content copy_md5="${copy_md5} $(md5 -q "${tmp_file_path}")" else From f5debd3c7e930bfa7b48b55326f67a86ccd388ef Mon Sep 17 00:00:00 2001 From: meo Date: Sat, 14 Dec 2024 12:12:14 +0100 Subject: [PATCH 7/7] fix: forgot to remove variable --- zfs-inplace-rebalancing.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/zfs-inplace-rebalancing.sh b/zfs-inplace-rebalancing.sh index ad0ae53..262d0f6 100755 --- a/zfs-inplace-rebalancing.sh +++ b/zfs-inplace-rebalancing.sh @@ -163,14 +163,16 @@ function rebalance() { # note: no lsattr on Mac OS or FreeBSD # file permissions, owner, group size, modification time - original_md5="${original_md5} $(stat -f "%Sp %Su %Sg %z %m" "${file_path}")" + original_md5="$(stat -f "%Sp %Su %Sg %z %m" "${file_path}")" # file content original_md5="${original_md5} $(md5 -q "${file_path}")" # file permissions, owner, group size, modification time - copy_md5="${copy_md5} $(stat -f "%Sp %Su %Sg %z %m" "${tmp_file_path}")" + copy_md5="$(stat -f "%Sp %Su %Sg %z %m" "${tmp_file_path}")" # file content copy_md5="${copy_md5} $(md5 -q "${tmp_file_path}")" + # remove the temporary extension + copy_md5=${copy_md5%"${tmp_extension}"} else echo "Unsupported OS type: $OSTYPE" exit 1