From 0985d2f008407052cd7fe3bb07fef1b9aa68968b Mon Sep 17 00:00:00 2001 From: Monthon Klongklaew Date: Tue, 15 Aug 2023 08:58:17 +0000 Subject: [PATCH 1/4] Run throughput benchmark multiple times Signed-off-by: Monthon Klongklaew --- mountpoint-s3/scripts/fio/write/seq_write.fio | 16 ++ .../scripts/fio/write/seq_write_direct.fio | 17 ++ mountpoint-s3/scripts/fs_bench.sh | 170 +++++++----------- 3 files changed, 95 insertions(+), 108 deletions(-) create mode 100644 mountpoint-s3/scripts/fio/write/seq_write.fio create mode 100644 mountpoint-s3/scripts/fio/write/seq_write_direct.fio diff --git a/mountpoint-s3/scripts/fio/write/seq_write.fio b/mountpoint-s3/scripts/fio/write/seq_write.fio new file mode 100644 index 000000000..6ca915761 --- /dev/null +++ b/mountpoint-s3/scripts/fio/write/seq_write.fio @@ -0,0 +1,16 @@ +[global] +name=fs_bench +bs=256k +runtime=10s +time_based +group_reporting + +[sequential_write] +size=100G +rw=write +ioengine=sync +fallocate=none +create_on_open=1 +fsync_on_close=1 +unlink=1 +startdelay=1s diff --git a/mountpoint-s3/scripts/fio/write/seq_write_direct.fio b/mountpoint-s3/scripts/fio/write/seq_write_direct.fio new file mode 100644 index 000000000..bcba9c8a9 --- /dev/null +++ b/mountpoint-s3/scripts/fio/write/seq_write_direct.fio @@ -0,0 +1,17 @@ +[global] +name=fs_bench +bs=256k +runtime=10s +time_based +group_reporting + +[sequential_write] +size=100G +rw=write +ioengine=sync +direct=1 +fallocate=none +create_on_open=1 +fsync_on_close=1 +unlink=1 +startdelay=1s diff --git a/mountpoint-s3/scripts/fs_bench.sh b/mountpoint-s3/scripts/fs_bench.sh index d80f57555..420ea180c 100755 --- a/mountpoint-s3/scripts/fs_bench.sh +++ b/mountpoint-s3/scripts/fs_bench.sh @@ -33,17 +33,48 @@ results_dir=results runtime_seconds=30 startdelay_seconds=30 max_threads=4 +iteration=10 rm -rf ${results_dir} mkdir -p ${results_dir} +run_fio_job() { + job_file=$1 + bench_file=$2 + mount_dir=$3 + + job_name=$(basename "${job_file}") + job_name="${job_name%.*}" + + for i in $(seq 1 $iteration); + do + fio --thread \ + --output=${results_dir}/${job_name}_${i}.json \ + --output-format=json \ + --directory=${mount_dir} \ + --filename=${bench_file} \ + ${job_file} + done + + # combine the results and find an average value + jq -n 'reduce inputs.jobs[] as $job (null; .name = $job.jobname | .len += 1 | .value += (if ($job."job options".rw == "read") + then $job.read.bw / 1024 + elif ($job."job options".rw == "randread") then $job.read.bw / 1024 + elif ($job."job options".rw == "randwrite") then $job.write.bw / 1024 + else $job.write.bw / 1024 end)) | {name: .name, value: (.value / .len), unit: "MiB/s"}' ${results_dir}/${job_name}_*.json | tee ${results_dir}/${job_name}_parsed.json + + # delete the raw output files + for i in $(seq 1 $iteration); + do + rm ${results_dir}/${job_name}_${i}.json + done +} + read_bechmark () { jobs_dir=mountpoint-s3/scripts/fio/read for job_file in "${jobs_dir}"/*.fio; do mount_dir=$(mktemp -d /tmp/fio-XXXXXXXXXXXX) - job_name=$(basename "${job_file}") - job_name="${job_name%.*}" echo "Running ${job_name}" @@ -65,122 +96,45 @@ read_bechmark () { bench_file=${S3_BUCKET_SMALL_BENCH_FILE} fi - # run benchmark - fio --thread \ - --output=${results_dir}/${job_name}.json \ - --output-format=json \ - --directory=${mount_dir} \ - --filename=${bench_file} \ - ${job_file} + # run the benchmark + run_fio_job $job_file $bench_file $mount_dir # unmount file system sudo umount ${mount_dir} # cleanup mount directory rm -rf ${mount_dir} - - # parse result - jq -n 'inputs.jobs[] | if (."job options".rw == "read") - then {name: .jobname, value: (.read.bw / 1024), unit: "MiB/s"} - elif (."job options".rw == "randread") then {name: .jobname, value: (.read.bw / 1024), unit: "MiB/s"} - elif (."job options".rw == "randwrite") then {name: .jobname, value: (.write.bw / 1024), unit: "MiB/s"} - else {name: .jobname, value: (.write.bw / 1024), unit: "MiB/s"} end' ${results_dir}/${job_name}.json | tee ${results_dir}/${job_name}_parsed.json - - # delete the raw output file - rm ${results_dir}/${job_name}.json done } write_benchmark () { - # mount file system - mount_dir=$(mktemp -d /tmp/fio-XXXXXXXXXXXX) - cargo run --release ${S3_BUCKET_NAME} ${mount_dir} \ - --allow-delete \ - --prefix=${S3_BUCKET_TEST_PREFIX} \ - --max-threads=${max_threads} - mount_status=$? - if [ $mount_status -ne 0 ]; then - echo "Failed to mount file system" - exit 1 - fi - sleep $startdelay_seconds - - ## sequential write - job_name="sequential_write" - bench_file=${mount_dir}/${job_name}_${RANDOM}.dat - dd if=/dev/zero of=$bench_file bs=256k conv=fsync > ${results_dir}/${job_name}.txt 2>&1 & - # get the process ID - dd_pid=$! - - sleep $runtime_seconds - # send USR1 signal to print the result - kill -USR1 ${dd_pid} - sleep 0.1 - kill ${dd_pid} - - throughput_value=$(awk '/copied/ {print $10}' ${results_dir}/${job_name}.txt) - unit=$(awk '/copied/ {print $11}' ${results_dir}/${job_name}.txt) - # convert unit to MiB/s - case "$unit" in - GB/s) - throughput_value=$(awk "BEGIN {print $throughput_value*1000*1000*1000/1024/1024}") - ;; - MB/s) - throughput_value=$(awk "BEGIN {print $throughput_value*1000*1000/1024/1024}") - ;; - kB/s) - throughput_value=$(awk "BEGIN {print $throughput_value*1000/1024/1024}") - ;; - esac - - json_data="{\"name\":\"$job_name\",\"value\":$throughput_value,\"unit\":\"MiB/s\"}" - echo $json_data | jq '.' | tee ${results_dir}/${job_name}.json - - # clean up the data file and the raw output file - sleep 10 - rm $bench_file ${results_dir}/${job_name}.txt - - - ## sequential write with direct IO - job_name="sequential_write_direct_io" - bench_file=${mount_dir}/${job_name}_${RANDOM}.dat - dd if=/dev/zero of=$bench_file bs=256k oflag=direct conv=fsync > ${results_dir}/${job_name}.txt 2>&1 & - # get the process ID - dd_pid=$! - - sleep $runtime_seconds - # send USR1 signal to print the result - kill -USR1 ${dd_pid} - sleep 0.1 - kill ${dd_pid} - - throughput_value=$(awk '/copied/ {print $10}' ${results_dir}/${job_name}.txt) - unit=$(awk '/copied/ {print $11}' ${results_dir}/${job_name}.txt) - # convert unit to MiB/s - case "$unit" in - GB/s) - throughput_value=$(awk "BEGIN {print $throughput_value*1000*1000*1000/1024/1024}") - ;; - MB/s) - throughput_value=$(awk "BEGIN {print $throughput_value*1000*1000/1024/1024}") - ;; - kB/s) - throughput_value=$(awk "BEGIN {print $throughput_value*1000/1024/1024}") - ;; - esac - - json_data="{\"name\":\"$job_name\",\"value\":$throughput_value,\"unit\":\"MiB/s\"}" - echo $json_data | jq '.' | tee ${results_dir}/${job_name}.json - - # clean up the data file and the raw output file - sleep 10 - rm $bench_file ${results_dir}/${job_name}.txt - - # unmount file system - sudo umount ${mount_dir} - - # cleanup mount directory - rm -rf ${mount_dir} + jobs_dir=mountpoint-s3/scripts/fio/write + + for job_file in "${jobs_dir}"/*.fio; do + # mount file system + mount_dir=$(mktemp -d /tmp/fio-XXXXXXXXXXXX) + cargo run --release ${S3_BUCKET_NAME} ${mount_dir} \ + --allow-delete \ + --prefix=${S3_BUCKET_TEST_PREFIX} \ + --max-threads=${max_threads} + mount_status=$? + if [ $mount_status -ne 0 ]; then + echo "Failed to mount file system" + exit 1 + fi + + # set bench file + bench_file=${mount_dir}/${job_name}_${RANDOM}.dat + + # run the benchmark + run_fio_job $job_file $bench_file $mount_dir + + # unmount file system + sudo umount ${mount_dir} + + # cleanup mount directory + rm -rf ${mount_dir} + done } read_bechmark From 1984e02f6c741484bbf24a29d7d1539567ff0dcb Mon Sep 17 00:00:00 2001 From: Monthon Klongklaew Date: Tue, 15 Aug 2023 13:53:12 +0000 Subject: [PATCH 2/4] Update name for sequential write direct io job Signed-off-by: Monthon Klongklaew --- mountpoint-s3/scripts/fio/write/seq_write_direct.fio | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mountpoint-s3/scripts/fio/write/seq_write_direct.fio b/mountpoint-s3/scripts/fio/write/seq_write_direct.fio index bcba9c8a9..b017a2fb4 100644 --- a/mountpoint-s3/scripts/fio/write/seq_write_direct.fio +++ b/mountpoint-s3/scripts/fio/write/seq_write_direct.fio @@ -5,7 +5,7 @@ runtime=10s time_based group_reporting -[sequential_write] +[sequential_write_direct_io] size=100G rw=write ioengine=sync From 7bb5098ba51f9b664a6430a06d2e18f554d8695b Mon Sep 17 00:00:00 2001 From: Monthon Klongklaew Date: Tue, 15 Aug 2023 13:59:39 +0000 Subject: [PATCH 3/4] Update benchmark doc Signed-off-by: Monthon Klongklaew --- doc/BENCHMARKING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/BENCHMARKING.md b/doc/BENCHMARKING.md index 492b2bb10..ae63ff1f4 100644 --- a/doc/BENCHMARKING.md +++ b/doc/BENCHMARKING.md @@ -15,7 +15,7 @@ In general, we run each IO operation for 30 seconds against a 100 GiB file. But ***readdir workload*** - we measure how long it takes to run `ls` command against directories with different size. Each directory has no subdirectory and contains a specific number of files, range from 100 to 100000 files, which we have to create manually using fio then upload them to S3 bucket before running the benchmark. The fio configuration files for creating them can be found at path [mountpoint-s3/scripts/fio/create/](../mountpoint-s3/scripts/fio/create). -***write workload*** - we measure write throughput by using [dd](https://man7.org/linux/man-pages/man1/dd.1.html) command to simulate sequential write workloads. We plan to use fio in the future for consistency with other benchmarks but its current write pattern is not supported by Mountpoint. Firstly, fio creates a file with 0 byte and close it. Secondly, fio opens the file again with `O_RDWR` flag to do the IO workloads. To support fio, Mountpoint has to allow file overwrites and allow file opens with `O_RDWR` flag. +***write workload*** - we measure write throughput by using fio to simulate sequential write workloads. The fio configuration files for write workloads can be found at path [mountpoint-s3/scripts/fio/write/](../mountpoint-s3/scripts/fio/write). ### Regression Testing Our CI runs the benchmark automatically for any new commits to the main branch or specific pull requests that we have reviewed and tagged with **performance** label. Every benchmark from the CI workflow will be running on `m5n.24xlarge` EC2 instances (100 Gbps network speed) with Ubuntu 22.04 in us-east-1 against a bucket in us-east-1. From ec7c0c493d94545bf9a13904318f37fc8c874b64 Mon Sep 17 00:00:00 2001 From: Monthon Klongklaew Date: Tue, 15 Aug 2023 14:07:26 +0000 Subject: [PATCH 4/4] Update config for write benchmarks Signed-off-by: Monthon Klongklaew --- mountpoint-s3/scripts/fio/write/seq_write.fio | 4 ++-- mountpoint-s3/scripts/fio/write/seq_write_direct.fio | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mountpoint-s3/scripts/fio/write/seq_write.fio b/mountpoint-s3/scripts/fio/write/seq_write.fio index 6ca915761..961ed3560 100644 --- a/mountpoint-s3/scripts/fio/write/seq_write.fio +++ b/mountpoint-s3/scripts/fio/write/seq_write.fio @@ -1,7 +1,7 @@ [global] name=fs_bench bs=256k -runtime=10s +runtime=30s time_based group_reporting @@ -13,4 +13,4 @@ fallocate=none create_on_open=1 fsync_on_close=1 unlink=1 -startdelay=1s +startdelay=30s diff --git a/mountpoint-s3/scripts/fio/write/seq_write_direct.fio b/mountpoint-s3/scripts/fio/write/seq_write_direct.fio index b017a2fb4..041601516 100644 --- a/mountpoint-s3/scripts/fio/write/seq_write_direct.fio +++ b/mountpoint-s3/scripts/fio/write/seq_write_direct.fio @@ -1,7 +1,7 @@ [global] name=fs_bench bs=256k -runtime=10s +runtime=30s time_based group_reporting @@ -14,4 +14,4 @@ fallocate=none create_on_open=1 fsync_on_close=1 unlink=1 -startdelay=1s +startdelay=30s