From 982b92800e53c86d91d3fdc7970ed4c9dd0b7a05 Mon Sep 17 00:00:00 2001 From: mmoriaaa Date: Sun, 28 Apr 2019 16:25:17 +0800 Subject: [PATCH] add ec test --- .../ec-performance-test/TestDFSIO_results.log | 450 ++++++++++++++++++ supports/ec-performance-test/config | 53 +++ supports/ec-performance-test/distcp.log | 4 + supports/ec-performance-test/drop_cache.sh | 18 + supports/ec-performance-test/prepare.sh | 27 ++ supports/ec-performance-test/prepare_ec.sh | 27 ++ .../ec-performance-test/prepare_replica.sh | 25 + supports/ec-performance-test/run_ssm_ec.py | 52 ++ supports/ec-performance-test/ssm.log | 1 + .../ec-performance-test/test_distcp_ec.sh | 75 +++ .../test_distcp_replica.sh | 74 +++ .../ec-performance-test/test_ssm_ec_only.sh | 83 ++++ .../test_ssm_ec_performance.sh | 103 ++++ .../ec-performance-test/test_ssm_unec_only.sh | 83 ++++ 14 files changed, 1075 insertions(+) create mode 100644 supports/ec-performance-test/TestDFSIO_results.log create mode 100755 supports/ec-performance-test/config create mode 100644 supports/ec-performance-test/distcp.log create mode 100755 supports/ec-performance-test/drop_cache.sh create mode 100755 supports/ec-performance-test/prepare.sh create mode 100755 supports/ec-performance-test/prepare_ec.sh create mode 100755 supports/ec-performance-test/prepare_replica.sh create mode 100644 supports/ec-performance-test/run_ssm_ec.py create mode 100644 supports/ec-performance-test/ssm.log create mode 100755 supports/ec-performance-test/test_distcp_ec.sh create mode 100755 supports/ec-performance-test/test_distcp_replica.sh create mode 100755 supports/ec-performance-test/test_ssm_ec_only.sh create mode 100755 supports/ec-performance-test/test_ssm_ec_performance.sh create mode 100755 supports/ec-performance-test/test_ssm_unec_only.sh diff --git a/supports/ec-performance-test/TestDFSIO_results.log b/supports/ec-performance-test/TestDFSIO_results.log new file mode 100644 index 00000000000..3a39cb08430 --- /dev/null +++ b/supports/ec-performance-test/TestDFSIO_results.log @@ -0,0 +1,450 @@ +----- TestDFSIO ----- : write + Date & time: Fri Apr 12 13:52:44 CST 2019 + Number of files: 10 + Total MBytes processed: 1000 + Throughput mb/sec: 279.56 + Average IO rate mb/sec: 280.35 + IO rate std deviation: 15.11 + Test exec time sec: 20.39 + +----- TestDFSIO ----- : write + Date & time: Fri Apr 12 13:53:16 CST 2019 + Number of files: 10 + Total MBytes processed: 100 + Throughput mb/sec: 95.24 + Average IO rate mb/sec: 95.86 + IO rate std deviation: 7.8 + Test exec time sec: 20.71 + +----- TestDFSIO ----- : write + Date & time: Fri Apr 12 14:04:05 CST 2019 + Number of files: 10 + Total MBytes processed: 1000 + Throughput mb/sec: 259.27 + Average IO rate mb/sec: 262.46 + IO rate std deviation: 27.94 + Test exec time sec: 19.82 + +----- TestDFSIO ----- : write + Date & time: Fri Apr 12 14:04:36 CST 2019 + Number of files: 10 + Total MBytes processed: 100 + Throughput mb/sec: 68.03 + Average IO rate mb/sec: 87.16 + IO rate std deviation: 24.17 + Test exec time sec: 19.34 + +----- TestDFSIO ----- : write + Date & time: Fri Apr 12 14:34:15 CST 2019 + Number of files: 10 + Total MBytes processed: 1000 + Throughput mb/sec: 255.43 + Average IO rate mb/sec: 263.65 + IO rate std deviation: 40.1 + Test exec time sec: 22.68 + +----- TestDFSIO ----- : write + Date & time: Fri Apr 12 15:45:44 CST 2019 + Number of files: 10 + Total MBytes processed: 1000 + Throughput mb/sec: 268.17 + Average IO rate mb/sec: 277.71 + IO rate std deviation: 43.57 + Test exec time sec: 18.22 + +----- TestDFSIO ----- : write + Date & time: Fri Apr 12 15:50:57 CST 2019 + Number of files: 10 + Total MBytes processed: 1000 + Throughput mb/sec: 256.67 + Average IO rate mb/sec: 260.26 + IO rate std deviation: 30.04 + Test exec time sec: 17.16 + +----- TestDFSIO ----- : write + Date & time: Fri Apr 12 16:40:54 CST 2019 + Number of files: 10 + Total MBytes processed: 1000 + Throughput mb/sec: 303.86 + Average IO rate mb/sec: 306.01 + IO rate std deviation: 25.44 + Test exec time sec: 19.24 + +----- TestDFSIO ----- : write + Date & time: Fri Apr 12 17:09:26 CST 2019 + Number of files: 10 + Total MBytes processed: 1000 + Throughput mb/sec: 301.84 + Average IO rate mb/sec: 303.1 + IO rate std deviation: 19.77 + Test exec time sec: 18.18 + +----- TestDFSIO ----- : write + Date & time: Tue Apr 16 10:32:14 CST 2019 + Number of files: 10 + Total MBytes processed: 1000 + Throughput mb/sec: 273.3 + Average IO rate mb/sec: 275.37 + IO rate std deviation: 22.04 + Test exec time sec: 20.73 + +----- TestDFSIO ----- : write + Date & time: Tue Apr 16 11:18:04 CST 2019 + Number of files: 1 + Total MBytes processed: 10 + Throughput mb/sec: 94.34 + Average IO rate mb/sec: 94.34 + IO rate std deviation: 0.02 + Test exec time sec: 18.83 + +----- TestDFSIO ----- : write + Date & time: Tue Apr 16 12:09:43 CST 2019 + Number of files: 3 + Total MBytes processed: 300 + Throughput mb/sec: 302.42 + Average IO rate mb/sec: 303.98 + IO rate std deviation: 22.35 + Test exec time sec: 19.23 + +----- TestDFSIO ----- : write + Date & time: Tue Apr 16 13:58:40 CST 2019 + Number of files: 10 + Total MBytes processed: 1000 + Throughput mb/sec: 266.81 + Average IO rate mb/sec: 268.23 + IO rate std deviation: 20.8 + Test exec time sec: 20.4 + +----- TestDFSIO ----- : write + Date & time: Tue Apr 16 15:40:11 CST 2019 + Number of files: 3 + Total MBytes processed: 300 + Throughput mb/sec: 271.74 + Average IO rate mb/sec: 272.23 + IO rate std deviation: 11.48 + Test exec time sec: 17.18 + +----- TestDFSIO ----- : write + Date & time: Tue Apr 16 17:50:39 CST 2019 + Number of files: 3 + Total MBytes processed: 300 + Throughput mb/sec: 278.04 + Average IO rate mb/sec: 278.15 + IO rate std deviation: 5.57 + Test exec time sec: 17.17 + +----- TestDFSIO ----- : write + Date & time: Tue Apr 16 17:52:48 CST 2019 + Number of files: 3 + Total MBytes processed: 300 + Throughput mb/sec: 278.55 + Average IO rate mb/sec: 280.33 + IO rate std deviation: 22.9 + Test exec time sec: 17.2 + +----- TestDFSIO ----- : write + Date & time: Tue Apr 16 18:09:05 CST 2019 + Number of files: 3 + Total MBytes processed: 300 + Throughput mb/sec: 298.51 + Average IO rate mb/sec: 299.54 + IO rate std deviation: 17.31 + Test exec time sec: 21.21 + +----- TestDFSIO ----- : write + Date & time: Wed Apr 17 09:31:25 CST 2019 + Number of files: 3 + Total MBytes processed: 300 + Throughput mb/sec: 250.84 + Average IO rate mb/sec: 255.28 + IO rate std deviation: 34.42 + Test exec time sec: 18.18 + +----- TestDFSIO ----- : write + Date & time: Wed Apr 17 09:47:34 CST 2019 + Number of files: 3 + Total MBytes processed: 300 + Throughput mb/sec: 286.26 + Average IO rate mb/sec: 286.86 + IO rate std deviation: 13.29 + Test exec time sec: 19.25 + +----- TestDFSIO ----- : write + Date & time: Wed Apr 17 09:58:04 CST 2019 + Number of files: 3 + Total MBytes processed: 300 + Throughput mb/sec: 255.1 + Average IO rate mb/sec: 257.72 + IO rate std deviation: 25.18 + Test exec time sec: 18.83 + +----- TestDFSIO ----- : write + Date & time: Wed Apr 17 10:01:30 CST 2019 + Number of files: 3 + Total MBytes processed: 300 + Throughput mb/sec: 286.81 + Average IO rate mb/sec: 286.82 + IO rate std deviation: 1.93 + Test exec time sec: 20.53 + +----- TestDFSIO ----- : write + Date & time: Wed Apr 17 10:10:13 CST 2019 + Number of files: 3 + Total MBytes processed: 300 + Throughput mb/sec: 285.99 + Average IO rate mb/sec: 286.34 + IO rate std deviation: 9.9 + Test exec time sec: 59.56 + +----- TestDFSIO ----- : write + Date & time: Wed Apr 17 10:12:50 CST 2019 + Number of files: 3 + Total MBytes processed: 300 + Throughput mb/sec: 271.25 + Average IO rate mb/sec: 274.11 + IO rate std deviation: 27.18 + Test exec time sec: 17.23 + +----- TestDFSIO ----- : write + Date & time: Wed Apr 17 10:25:28 CST 2019 + Number of files: 3 + Total MBytes processed: 300 + Throughput mb/sec: 280.37 + Average IO rate mb/sec: 280.8 + IO rate std deviation: 11.07 + Test exec time sec: 20.88 + +----- TestDFSIO ----- : write + Date & time: Wed Apr 17 10:29:48 CST 2019 + Number of files: 3 + Total MBytes processed: 300 + Throughput mb/sec: 276.75 + Average IO rate mb/sec: 276.98 + IO rate std deviation: 7.79 + Test exec time sec: 18.31 + +----- TestDFSIO ----- : write + Date & time: Wed Apr 17 11:05:23 CST 2019 + Number of files: 3 + Total MBytes processed: 300 + Throughput mb/sec: 278.04 + Average IO rate mb/sec: 278.2 + IO rate std deviation: 6.65 + Test exec time sec: 19.28 + +----- TestDFSIO ----- : write + Date & time: Wed Apr 17 11:37:27 CST 2019 + Number of files: 3 + Total MBytes processed: 300 + Throughput mb/sec: 306.44 + Average IO rate mb/sec: 307.11 + IO rate std deviation: 14.68 + Test exec time sec: 18.16 + +----- TestDFSIO ----- : write + Date & time: Wed Apr 17 11:56:22 CST 2019 + Number of files: 3 + Total MBytes processed: 300 + Throughput mb/sec: 295.28 + Average IO rate mb/sec: 296.66 + IO rate std deviation: 20.49 + Test exec time sec: 19.26 + +----- TestDFSIO ----- : write + Date & time: Fri Apr 19 09:59:33 CST 2019 + Number of files: 3 + Total MBytes processed: 300 + Throughput mb/sec: 286.26 + Average IO rate mb/sec: 286.31 + IO rate std deviation: 3.89 + Test exec time sec: 18.81 + +----- TestDFSIO ----- : write + Date & time: Fri Apr 19 10:06:17 CST 2019 + Number of files: 3 + Total MBytes processed: 300 + Throughput mb/sec: 292.4 + Average IO rate mb/sec: 294.84 + IO rate std deviation: 27.39 + Test exec time sec: 19.61 + +----- TestDFSIO ----- : write + Date & time: Fri Apr 19 10:57:57 CST 2019 + Number of files: 3 + Total MBytes processed: 300 + Throughput mb/sec: 281.16 + Average IO rate mb/sec: 281.57 + IO rate std deviation: 10.63 + Test exec time sec: 18.27 + +----- TestDFSIO ----- : write + Date & time: Tue Apr 23 13:40:09 CST 2019 + Number of files: 10000 + Total MBytes processed: 1000000 + Throughput mb/sec: 39.09 + Average IO rate mb/sec: 61.73 + IO rate std deviation: 57.81 + Test exec time sec: 2613.64 + +----- TestDFSIO ----- : write + Date & time: Tue Apr 23 17:56:52 CST 2019 + Number of files: 50 + Total MBytes processed: 5000 + Throughput mb/sec: 250.49 + Average IO rate mb/sec: 259.27 + IO rate std deviation: 33.1 + Test exec time sec: 34.95 + +----- TestDFSIO ----- : write + Date & time: Tue Apr 23 18:12:32 CST 2019 + Number of files: 1000 + Total MBytes processed: 100000 + Throughput mb/sec: 35.33 + Average IO rate mb/sec: 63.3 + IO rate std deviation: 68.21 + Test exec time sec: 323.01 + +----- TestDFSIO ----- : write + Date & time: Tue Apr 23 20:20:27 CST 2019 + Number of files: 10000 + Total MBytes processed: 1000000 + Throughput mb/sec: 32.87 + Average IO rate mb/sec: 55.91 + IO rate std deviation: 57.64 + Test exec time sec: 2611.94 + +----- TestDFSIO ----- : write + Date & time: Tue Apr 23 22:20:35 CST 2019 + Number of files: 10000 + Total MBytes processed: 2000000 + Throughput mb/sec: 15.14 + Average IO rate mb/sec: 30.2 + IO rate std deviation: 38.05 + Test exec time sec: 2633.13 + +----- TestDFSIO ----- : write + Date & time: Tue Apr 23 23:21:11 CST 2019 + Number of files: 10000 + Total MBytes processed: 1000000 + Throughput mb/sec: 34.58 + Average IO rate mb/sec: 57.14 + IO rate std deviation: 55.1 + Test exec time sec: 2605.54 + +----- TestDFSIO ----- : write + Date & time: Wed Apr 24 12:22:55 CST 2019 + Number of files: 10000 + Total MBytes processed: 1000000 + Throughput mb/sec: 34.48 + Average IO rate mb/sec: 57.27 + IO rate std deviation: 53.91 + Test exec time sec: 2609.13 + +----- TestDFSIO ----- : write + Date & time: Wed Apr 24 13:08:08 CST 2019 + Number of files: 10000 + Total MBytes processed: 2000000 + Throughput mb/sec: 15.99 + Average IO rate mb/sec: 30.38 + IO rate std deviation: 36.24 + Test exec time sec: 2598.87 + +----- TestDFSIO ----- : write + Date & time: Wed Apr 24 14:11:47 CST 2019 + Number of files: 10000 + Total MBytes processed: 5000000 + Throughput mb/sec: 2.73 + Average IO rate mb/sec: 5.73 + IO rate std deviation: 13.91 + Test exec time sec: 3702.03 + +----- TestDFSIO ----- : write + Date & time: Thu Apr 25 00:18:36 CST 2019 + Number of files: 10000 + Total MBytes processed: 5000000 + Throughput mb/sec: 2.5 + Average IO rate mb/sec: 7.48 + IO rate std deviation: 14.57 + Test exec time sec: 3683.27 + +----- TestDFSIO ----- : write + Date & time: Thu Apr 25 17:05:23 CST 2019 + Number of files: 3 + Total MBytes processed: 300 + Throughput mb/sec: 275.99 + Average IO rate mb/sec: 277.85 + IO rate std deviation: 22.37 + Test exec time sec: 18.14 + +----- TestDFSIO ----- : write + Date & time: Thu Apr 25 20:35:28 CST 2019 + Number of files: 10000 + Total MBytes processed: 5000000 + Throughput mb/sec: 2.53 + Average IO rate mb/sec: 8.37 + IO rate std deviation: 15.08 + Test exec time sec: 3723.5 + +----- TestDFSIO ----- : write + Date & time: Fri Apr 26 12:29:03 CST 2019 + Number of files: 1000 + Total MBytes processed: 100000 + Throughput mb/sec: 32.46 + Average IO rate mb/sec: 56.71 + IO rate std deviation: 59.38 + Test exec time sec: 298.35 + +----- TestDFSIO ----- : write + Date & time: Fri Apr 26 12:51:10 CST 2019 + Number of files: 500 + Total MBytes processed: 50000 + Throughput mb/sec: 47.64 + Average IO rate mb/sec: 83.46 + IO rate std deviation: 78.78 + Test exec time sec: 148.53 + +----- TestDFSIO ----- : write + Date & time: Fri Apr 26 13:09:42 CST 2019 + Number of files: 10 + Total MBytes processed: 1000 + Throughput mb/sec: 296.47 + Average IO rate mb/sec: 298.99 + IO rate std deviation: 27.52 + Test exec time sec: 19.29 + +----- TestDFSIO ----- : write + Date & time: Fri Apr 26 15:35:40 CST 2019 + Number of files: 1000 + Total MBytes processed: 100000 + Throughput mb/sec: 39.19 + Average IO rate mb/sec: 65.61 + IO rate std deviation: 64.83 + Test exec time sec: 266.39 + +----- TestDFSIO ----- : write + Date & time: Fri Apr 26 16:10:14 CST 2019 + Number of files: 1000 + Total MBytes processed: 100000 + Throughput mb/sec: 33.52 + Average IO rate mb/sec: 64.01 + IO rate std deviation: 66.82 + Test exec time sec: 318.82 + +----- TestDFSIO ----- : write + Date & time: Fri Apr 26 16:23:16 CST 2019 + Number of files: 3 + Total MBytes processed: 300 + Throughput mb/sec: 286.26 + Average IO rate mb/sec: 288 + IO rate std deviation: 22.85 + Test exec time sec: 39.19 + +----- TestDFSIO ----- : write + Date & time: Sat Apr 27 20:27:14 CST 2019 + Number of files: 10000 + Total MBytes processed: 5000000 + Throughput mb/sec: 2.74 + Average IO rate mb/sec: 5.01 + IO rate std deviation: 11.29 + Test exec time sec: 3648.3 + diff --git a/supports/ec-performance-test/config b/supports/ec-performance-test/config new file mode 100755 index 00000000000..640ce959476 --- /dev/null +++ b/supports/ec-performance-test/config @@ -0,0 +1,53 @@ +# These are test cases which specify file size and num. The base sync dir name is "size_num". + +# The files for these case should be created under them in advance. + +# declare -A CASES=(["10MB"]="10" ["100MB"]="10") + +declare -A CASES=(["500MB"]="10000") + +# SSM home, e.g., SMART_HOME=~/smart-data-1.4.0 + +SMART_HOME= + + + +# PAT home, e.g., PAT_HOME=~/PAT + +PAT_HOME=/root/rui/PAT-master/ + +# File path +FILE_PATH= + +# The src node +SRC_NODE="sr613" + +# The src cluster hdfs url for sync, e.g., SRC_CLUSTER=hdfs://sr519:9000 + +SRC_CLUSTER=hdfs://sr613:9000 + +# The dest node + +DEST_NODE="sr613" + +# The dest cluster hdfs url for sync, e.g., DEST_CLUSTER=hdfs://sr518:9000 + +DEST_CLUSTER=hdfs://sr613:9000 + + + +# The namenode's hostname for remote HDFS cluster + +REMOTE_NAMENODE="sr613" + + + +# The hosts require dropping cache, e.g., HOSTS="host1 host2" + +HOSTS="sr605 sr606 sr607 sr608 sr609 sr610 sr611 sr612 sr613 sr614" + + + +# the number of mapper for distcp, e.g., MAPPER_NUM="30 60 90" + +MAPPER_NUM="90" diff --git a/supports/ec-performance-test/distcp.log b/supports/ec-performance-test/distcp.log new file mode 100644 index 00000000000..6eb61fa293b --- /dev/null +++ b/supports/ec-performance-test/distcp.log @@ -0,0 +1,4 @@ +Test case 500MB_10000 with 90 mappers: + ec +3900s 3584s 3694s +Test case 500MB_10000 with mapper is finished! diff --git a/supports/ec-performance-test/drop_cache.sh b/supports/ec-performance-test/drop_cache.sh new file mode 100755 index 00000000000..723286803d6 --- /dev/null +++ b/supports/ec-performance-test/drop_cache.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + + + +. ./config + + +drop_cache="sync;echo 3 > /proc/sys/vm/drop_caches" + +# drop cache for all cluster hosts + +echo "drop cache for ${HOSTS}." + +for host in ${HOSTS}; do + + ssh $host "${drop_cache}" + +done diff --git a/supports/ec-performance-test/prepare.sh b/supports/ec-performance-test/prepare.sh new file mode 100755 index 00000000000..f5f6eee6389 --- /dev/null +++ b/supports/ec-performance-test/prepare.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + + + +. ./config + + + +# generate data +for size in "${!CASES[@]}"; do + + num=${CASES[$size]} + + dir="${size}_${num}" + + ssh ${REMOTE_NAMENODE} "hdfs dfs -mkdir /${dir}" + + hadoop jar /root/rui/hadoop-3.2.0-SNAPSHOT/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-3.2.0-SNAPSHOT-tests.jar TestDFSIO -write -nrFiles $(($num)) -size ${size} + + echo "move" + + ssh ${REMOTE_NAMENODE} "hdfs dfs -mv /benchmarks/TestDFSIO/io_data/* /"${size}_$num"" + + ssh ${REMOTE_NAMENODE} "hdfs dfs -rm -r /benchmarks" + +done + diff --git a/supports/ec-performance-test/prepare_ec.sh b/supports/ec-performance-test/prepare_ec.sh new file mode 100755 index 00000000000..20580034e5c --- /dev/null +++ b/supports/ec-performance-test/prepare_ec.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + + + +. ./config + + + +# mkdir and ec +for size in "${!CASES[@]}"; do + + num=${CASES[$size]} + + dir="${size}_${num}" + + # delete historical data + + echo "delete data in dest dir." + + # unset ec policy + + ssh ${DEST_NODE} "hdfs dfs -rm -r /dest/${dir}; hdfs dfs -mkdir /dest/${dir}" + + ssh ${DEST_NODE} "hdfs ec -setPolicy -path /dest/${dir}" + +done + diff --git a/supports/ec-performance-test/prepare_replica.sh b/supports/ec-performance-test/prepare_replica.sh new file mode 100755 index 00000000000..038c01eb14d --- /dev/null +++ b/supports/ec-performance-test/prepare_replica.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + + + +. ./config + + + +# mkdir and ec +for size in "${!CASES[@]}"; do + + num=${CASES[$size]} + + dir="${size}_${num}" + + # delete historical data + + echo "delete data in dest dir." + + ssh ${SRC_NODE} "hdfs ec -setPolicy -path /${dir}" + + ssh ${SRC_NODE} "hdfs dfs -rm -r /replica/${dir}; hdfs dfs -mkdir /replica/${dir}" + +done + diff --git a/supports/ec-performance-test/run_ssm_ec.py b/supports/ec-performance-test/run_ssm_ec.py new file mode 100644 index 00000000000..81f93a91fd5 --- /dev/null +++ b/supports/ec-performance-test/run_ssm_ec.py @@ -0,0 +1,52 @@ +import sys +import time +from util import * + + +size = sys.argv[1] +num = sys.argv[2] +case = size + "_" + num +log = sys.argv[3] +action = sys.argv[4] + +if action == "ec": + #rid = submit_rule("file: every 30min | path matches \"/" + case + "/*\" | ec -policy RS-6-3-1024k") + rid = submit_rule("file: path matches \"/" + case + "/*\" | ec -policy RS-6-3-1024k") +elif action == "unec": + # rid = submit_rule("file: every 30min | path matches \"/" + case + "/*\" | unec") + rid = submit_rule("file: path matches \"/" + case + "/*\" | unec") + +start_rule(rid) +start_time = time.time() + +rule = get_rule(rid) +last_checked = rule['numChecked'] +last_cmdsgen = rule['numCmdsGen'] + +time.sleep(.1) +rule = get_rule(rid) +while not ((rule['numChecked'] > last_checked) and (rule['numCmdsGen'] == last_cmdsgen)): + time.sleep(.1) + rule = get_rule(rid) + last_checked = rule['numChecked'] + last_cmdsgen = rule['numCmdsGen'] + time.sleep(.1) + rule = get_rule(rid) + +cids = get_cids_of_rule(rid) + +#while True: +# rule = get_rule(rid) +# print(rule) +# if rule['numCmdsGen'] == int(num): +# break +# time.sleep(1) +failed_cids = wait_for_cmdlets(cids) +if len(failed_cids) != 0: + print "Not all ec actions succeed!" +end_time = time.time() +stop_rule(rid) +# append result to log file +f = open(log, 'a') +f.write(str(int(end_time - start_time)) + "s" + " " + '\n') +f.close() diff --git a/supports/ec-performance-test/ssm.log b/supports/ec-performance-test/ssm.log new file mode 100644 index 00000000000..f70283503e6 --- /dev/null +++ b/supports/ec-performance-test/ssm.log @@ -0,0 +1 @@ +Test case 500MB_10000(unec): diff --git a/supports/ec-performance-test/test_distcp_ec.sh b/supports/ec-performance-test/test_distcp_ec.sh new file mode 100755 index 00000000000..8a1f103e07f --- /dev/null +++ b/supports/ec-performance-test/test_distcp_ec.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + + + +echo "Get configuration from config." + +. config + + + +echo "------------------ Your configuration ------------------" + +echo "PAT home is ${PAT_HOME}." + +echo "Test case:" + +for size in ${!CASES[@]}; do + + echo ${size} ${CASES[$size]} + +done + +echo "--------------------------------------------------------" + + + +bin=$(dirname "${BASH_SOURCE-$0}") + +bin=$(cd "${bin}">/dev/null; pwd) + +log="${bin}/distcp.log" + +# remove historical data in log file + +printf "" > ${log} + + + + + for size in "${!CASES[@]}"; do + + case=${size}_${CASES[$size]} + + printf "Test case ${case} with ${MAPPER_NUM} mappers:\n ec\n" >> ${log} + + for i in {1..3}; do + + echo "==================== test case: $case, mapper num: ${MAPPER_NUM}, test round: $i ============================" + + sh drop_cache.sh + + sh prepare_ec.sh + + cd ${PAT_HOME}/PAT-collecting-data + + echo "hadoop distcp -skipcrccheck -m ${MAPPER_NUM} ${SRC_CLUSTER}/${case}/* ${DEST_CLUSTER}/dest/${case}/" + + echo "start_time=\`date +%s\`;\ + + hadoop distcp -skipcrccheck -m ${MAPPER_NUM} ${SRC_CLUSTER}/${case}/* ${DEST_CLUSTER}/dest/${case}/ > results/$case_${MAPPER_NUM}_$i.log 2>&1;\ + + end_time=\`date +%s\`;\ + + printf \"\$((end_time-start_time))s \" >> ${log}" > cmd.sh + + ./pat run "${case}_"ec"_${MAPPER_NUM}_${i}" + + cd ${bin} + + done + + printf "\nTest case ${case} with $m mapper is finished!\n" >> ${log} + + done + diff --git a/supports/ec-performance-test/test_distcp_replica.sh b/supports/ec-performance-test/test_distcp_replica.sh new file mode 100755 index 00000000000..f29f342136c --- /dev/null +++ b/supports/ec-performance-test/test_distcp_replica.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash + + + +echo "Get configuration from config." + +. config + + + +echo "------------------ Your configuration ------------------" + +echo "PAT home is ${PAT_HOME}." + +echo "Test case:" + +for size in ${!CASES[@]}; do + + echo ${size} ${CASES[$size]} + +done + +echo "--------------------------------------------------------" + + + +bin=$(dirname "${BASH_SOURCE-$0}") + +bin=$(cd "${bin}">/dev/null; pwd) + +log="${bin}/distcp.log" + +# remove historical data in log file + +printf "" > ${log} + + + # 3 replica + for size in "${!CASES[@]}"; do + + case=${size}_${CASES[$size]} + + printf "Test case ${case} with ${MAPPER_NUM} mappers:\n replica\n" >> ${log} + + for i in {1..1}; do + + echo "==================== test case: $case, mapper num: ${MAPPER_NUM}, test round: $i ============================" + + sh drop_cache.sh + + sh prepare_replica.sh + + cd ${PAT_HOME}/PAT-collecting-data + + echo "hadoop distcp -skipcrccheck -m ${MAPPER_NUM} ${DEST_CLUSTER}/dest/${case}/* ${SRC_CLUSTER}/replica/${case}/" + + echo "start_time=\`date +%s\`;\ + + hadoop distcp -skipcrccheck -m ${MAPPER_NUM} ${DEST_CLUSTER}/dest/${case}/* ${SRC_CLUSTER}/replica/${case}/ > results/$case_${MAPPER_NUM}_$i.log 2>&1;\ + + end_time=\`date +%s\`;\ + + printf \"\$((end_time-start_time))s \" >> ${log}" > cmd.sh + + ./pat run "${case}_"replica"_${MAPPER_NUM}_${i}" + + cd ${bin} + + done + + printf "\nTest case ${case} with $m mapper is finished!\n" >> ${log} + + done + diff --git a/supports/ec-performance-test/test_ssm_ec_only.sh b/supports/ec-performance-test/test_ssm_ec_only.sh new file mode 100755 index 00000000000..bc8651a656c --- /dev/null +++ b/supports/ec-performance-test/test_ssm_ec_only.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +# avoid blocking REST API + +unset http_proxy + +# for python use + +export PYTHONPATH=../integration-test:$PYTHONPATH + + + +echo "Get configuration from config." + +source config + +echo "------------------ Your configuration ------------------" + +echo "SSM home is ${SMART_HOME}." + +echo "PAT home is ${PAT_HOME}." + +echo "Test case:" + +for size in ${!CASES[@]}; do + + echo ${size} ${CASES[$size]} + +done + +# echo "Destination cluster is ${DEST_CLUSTER}." + +echo "--------------------------------------------------------" + + + +bin=$(dirname "${BASH_SOURCE-$0}") + +bin=$(cd "${bin}">/dev/null; pwd) + +log="${bin}/ssm.log" + +# remove historical data in log file + +printf "" > ${log} + +for size in "${!CASES[@]}"; do + + case="${size}_${CASES[$size]}" + + echo "${case}" + + for i in {1..1}; do + + action="ec" + + echo "Test case ${case}($action):" >> ${log} + + echo "==================== test case: $case, test round: $i ============================" + + sh drop_cache.sh + + # make ssm log empty before test + + printf "" > ${SMART_HOME}/logs/smartserver.log + + cd ${PAT_HOME}/PAT-collecting-data + + echo "export PYTHONPATH=${bin}/../integration-test:${PYTHONPATH};\ + + python ${bin}/run_ssm_ec.py ${size} ${CASES[$size]} ${log} ${action}" > cmd.sh + + ./pat run "${case}_${i}" + + cp ${SMART_HOME}/logs/smartserver.log ./results/${case}-${i}.log + + cd ${bin} + + done + + printf "\nTest case ${case} is finished!\n" >> ${log} + +done diff --git a/supports/ec-performance-test/test_ssm_ec_performance.sh b/supports/ec-performance-test/test_ssm_ec_performance.sh new file mode 100755 index 00000000000..cf7318eb5cf --- /dev/null +++ b/supports/ec-performance-test/test_ssm_ec_performance.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash + +# avoid blocking REST API + +unset http_proxy + +# for python use + +export PYTHONPATH=../integration-test:$PYTHONPATH + + + +echo "Get configuration from config." + +source config + +echo "------------------ Your configuration ------------------" + +echo "SSM home is ${SMART_HOME}." + +echo "PAT home is ${PAT_HOME}." + +echo "Test case:" + +for size in ${!CASES[@]}; do + + echo ${size} ${CASES[$size]} + +done + +# echo "Destination cluster is ${DEST_CLUSTER}." + +echo "--------------------------------------------------------" + + + +bin=$(dirname "${BASH_SOURCE-$0}") + +bin=$(cd "${bin}">/dev/null; pwd) + +log="${bin}/ssm.log" + +# remove historical data in log file + +printf "" > ${log} + +for size in "${!CASES[@]}"; do + + case="${size}_${CASES[$size]}" + + echo "${case}" + + for i in {1..3}; do + + action="ec" + + echo "Test case ${case}($action):" >> ${log} + + echo "==================== test case: $case, test round: $i ============================" + + sh drop_cache.sh + + # make ssm log empty before test + + printf "" > ${SMART_HOME}/logs/smartserver.log + + cd ${PAT_HOME}/PAT-collecting-data + + echo "export PYTHONPATH=${bin}/../integration-test:${PYTHONPATH};\ + + python ${bin}/run_ssm_ec.py ${size} ${CASES[$size]} ${log} ${action}" > cmd.sh + + ./pat run "${case}_${i}" + + cp ${SMART_HOME}/logs/smartserver.log ./results/${case}-${i}.log + + cd ${bin} + + action="unec" + + echo "Test case ${case}($action):" >> ${log} + + echo "==================== test case: $case, test round: $i ============================" + + sh drop_cache.sh + + cd ${PAT_HOME}/PAT-collecting-data + + echo "export PYTHONPATH=${bin}/../integration-test:${PYTHONPATH};\ + + python ${bin}/run_ssm_ec.py ${size} ${CASES[$size]} ${log} ${action}" > cmd.sh + + ./pat run "${case}_${i}" + + cp ${SMART_HOME}/logs/smartserver.log ./results/${case}-${i}.log + + cd ${bin} + + done + + printf "\nTest case ${case} is finished!\n" >> ${log} + +done diff --git a/supports/ec-performance-test/test_ssm_unec_only.sh b/supports/ec-performance-test/test_ssm_unec_only.sh new file mode 100755 index 00000000000..903b76b155b --- /dev/null +++ b/supports/ec-performance-test/test_ssm_unec_only.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +# avoid blocking REST API + +unset http_proxy + +# for python use + +export PYTHONPATH=../integration-test:$PYTHONPATH + + + +echo "Get configuration from config." + +source config + +echo "------------------ Your configuration ------------------" + +echo "SSM home is ${SMART_HOME}." + +echo "PAT home is ${PAT_HOME}." + +echo "Test case:" + +for size in ${!CASES[@]}; do + + echo ${size} ${CASES[$size]} + +done + +# echo "Destination cluster is ${DEST_CLUSTER}." + +echo "--------------------------------------------------------" + + + +bin=$(dirname "${BASH_SOURCE-$0}") + +bin=$(cd "${bin}">/dev/null; pwd) + +log="${bin}/ssm.log" + +# remove historical data in log file + +printf "" > ${log} + +for size in "${!CASES[@]}"; do + + case="${size}_${CASES[$size]}" + + echo "${case}" + + for i in {1..1}; do + + # make ssm log empty before test + + printf "" > ${SMART_HOME}/logs/smartserver.log + + action="unec" + + echo "Test case ${case}($action):" >> ${log} + + echo "==================== test case: $case, test round: $i ============================" + + sh drop_cache.sh + + cd ${PAT_HOME}/PAT-collecting-data + + echo "export PYTHONPATH=${bin}/../integration-test:${PYTHONPATH};\ + + python ${bin}/run_ssm_ec.py ${size} ${CASES[$size]} ${log} ${action}" > cmd.sh + + ./pat run "${case}_${i}" + + cp ${SMART_HOME}/logs/smartserver.log ./results/${case}-${i}.log + + cd ${bin} + + done + + printf "\nTest case ${case} is finished!\n" >> ${log} + +done