From 589cafe648f2a55563286ccb155745a6beb1a4bb Mon Sep 17 00:00:00 2001 From: birdstorm Date: Thu, 28 Dec 2017 16:27:23 +0800 Subject: [PATCH 1/7] modify integration scripts and local paths since whole project structure has changed --- .gitignore | 3 +++ integtest/_env.sh | 17 +++++++++++++++++ integtest/conf/tidb_config.properties | 3 +++ integtest/conf/tispark_config.properties | 8 -------- integtest/test.sh | 1 + integtest/test_all.sh | 1 + integtest/test_dag.sh | 1 + integtest/test_no_tpch.sh | 1 + integtest/test_tpch.sh | 1 + 9 files changed, 28 insertions(+), 8 deletions(-) create mode 100644 integtest/conf/tidb_config.properties delete mode 100644 integtest/conf/tispark_config.properties diff --git a/.gitignore b/.gitignore index 90fe985af4..719d8f56bc 100755 --- a/.gitignore +++ b/.gitignore @@ -59,3 +59,6 @@ tikv-client/kvproto tikv-client/tipb tikv-client/proto +# ignore tikv-client bazel build +tikv-client/bazel-* + diff --git a/integtest/_env.sh b/integtest/_env.sh index bfabbe922d..003169aefa 100644 --- a/integtest/_env.sh +++ b/integtest/_env.sh @@ -19,6 +19,7 @@ spark_test_opt="" TISPARK_CONF="${SPARK_HOME}/conf/tispark_config.properties" BASE_CONF="${BASEDIR}/conf/tispark_config.properties" +INIT_CONF="${BASEDIR}/conf/tidb_config.properties" mysql_addr="localhost" mysql_user="root" @@ -29,6 +30,22 @@ tidb_addr= tidb_port= tidb_user= +build_init_properties() { + file=${BASE_CONF} + init=${INIT_CONF} + + if ! [ -f "$file" ] + then + echo "$file not found. " + echo "Building initial config file." + if ! [ -f "$init" ]; then + echo "$init not found. Please set this file manually according to README." + exit -1 + fi + cp ${init} ${file} + fi +} + read_properties() { file=${BASE_CONF} diff --git a/integtest/conf/tidb_config.properties b/integtest/conf/tidb_config.properties new file mode 100644 index 0000000000..b3d312377c --- /dev/null +++ b/integtest/conf/tidb_config.properties @@ -0,0 +1,3 @@ +tidb.addr=127.0.0.1 +tidb.port=4000 +tidb.user=root diff --git a/integtest/conf/tispark_config.properties b/integtest/conf/tispark_config.properties deleted file mode 100644 index 60912dc2cd..0000000000 --- a/integtest/conf/tispark_config.properties +++ /dev/null @@ -1,8 +0,0 @@ -tidb.addr=127.0.0.1 -tidb.port=4000 -tidb.user=root - -# Options below will be refreshed each time. You don't have to change them - -test.mode=Test -test.db=tpch_test diff --git a/integtest/test.sh b/integtest/test.sh index 1ac5c6f14a..7f6d39a63f 100755 --- a/integtest/test.sh +++ b/integtest/test.sh @@ -6,6 +6,7 @@ source _env.sh echo ${BASEDIR} echo "usage: [-d | --debug]" +build_init_properties clear_last_diff_files check_tpch_dir_is_present check_tpch_data_is_loaded diff --git a/integtest/test_all.sh b/integtest/test_all.sh index ae938be82a..509b08bfdf 100755 --- a/integtest/test_all.sh +++ b/integtest/test_all.sh @@ -4,6 +4,7 @@ source _env.sh echo "Usage: [-a | -s | -i | -d | -h]" +build_init_properties isDebug=false while getopts ":shida" arg diff --git a/integtest/test_dag.sh b/integtest/test_dag.sh index 2c1666f5a2..289bdedae4 100755 --- a/integtest/test_dag.sh +++ b/integtest/test_dag.sh @@ -9,6 +9,7 @@ echo "Note: must be quoted. e.g., \"select * from t\"" echo "You may use sql-only like this:" echo "./test_dag.sh -t \"select * from t\" -b \"test\"" +build_init_properties clear_last_diff_files_DAG isDebug=false diff --git a/integtest/test_no_tpch.sh b/integtest/test_no_tpch.sh index 42b2ca8ad6..79a5feb5e0 100755 --- a/integtest/test_no_tpch.sh +++ b/integtest/test_no_tpch.sh @@ -9,6 +9,7 @@ echo "Note: must be quoted. e.g., \"select * from t\"" echo "You may use sql-only like this:" echo "./test_no_tpch.sh -t \"select * from t\" -b \"test\"" +build_init_properties clear_last_diff_files isDebug=false diff --git a/integtest/test_tpch.sh b/integtest/test_tpch.sh index 8197bb4af6..210f66328b 100755 --- a/integtest/test_tpch.sh +++ b/integtest/test_tpch.sh @@ -6,6 +6,7 @@ source _env.sh echo ${BASEDIR} echo "usage: [-d | --debug]" +build_init_properties clear_last_diff_files_tpch check_tpch_dir_is_present check_tpch_data_is_loaded From ddad00a8e8e6e21cdb2aeb7681711728115baca9 Mon Sep 17 00:00:00 2001 From: birdstorm Date: Thu, 28 Dec 2017 16:35:10 +0800 Subject: [PATCH 2/7] fix test_no_tpch.sh --- integtest/conf/log4j.properties | 2 +- integtest/test_no_tpch.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/integtest/conf/log4j.properties b/integtest/conf/log4j.properties index 1c9a819498..3a58fd00c1 100644 --- a/integtest/conf/log4j.properties +++ b/integtest/conf/log4j.properties @@ -16,7 +16,7 @@ # # Set everything to be logged to the console -log4j.rootCategory=INFO, console +log4j.rootCategory=INFO, console log4j.appender.console=org.apache.log4j.ConsoleAppender log4j.appender.console.layout=org.apache.log4j.PatternLayout diff --git a/integtest/test_no_tpch.sh b/integtest/test_no_tpch.sh index 79a5feb5e0..21ab7f7a19 100755 --- a/integtest/test_no_tpch.sh +++ b/integtest/test_no_tpch.sh @@ -32,7 +32,7 @@ do showFailedOnly=true ;; a) - cd ../tikv-client-lib-java/ + cd ../tikv-client/ mvn clean install cd ../ mvn clean install From fc4666c3474dc8175b78fae744de98f483ef46d7 Mon Sep 17 00:00:00 2001 From: birdstorm Date: Thu, 28 Dec 2017 17:17:54 +0800 Subject: [PATCH 3/7] refactor build_init_properties --- integtest/_env.sh | 4 +++- integtest/test.sh | 1 - integtest/test_all.sh | 1 - integtest/test_dag.sh | 2 -- integtest/test_no_tpch.sh | 2 -- integtest/test_tpch.sh | 1 - 6 files changed, 3 insertions(+), 8 deletions(-) diff --git a/integtest/_env.sh b/integtest/_env.sh index 003169aefa..0f4f180d03 100644 --- a/integtest/_env.sh +++ b/integtest/_env.sh @@ -2,6 +2,7 @@ set -ue BASEDIR=$(cd `dirname $0`; pwd) +echo "Base directory in: $BASEDIR" PATH_TO_CONF="$BASEDIR/conf" if [ -z "${SPARK_HOME}" ]; then @@ -30,7 +31,7 @@ tidb_addr= tidb_port= tidb_user= -build_init_properties() { +build_if_not_exists_init_properties() { file=${BASE_CONF} init=${INIT_CONF} @@ -69,6 +70,7 @@ read_properties() { fi } +build_if_not_exists_init_properties read_properties create_conf_db_options() { diff --git a/integtest/test.sh b/integtest/test.sh index 7f6d39a63f..1ac5c6f14a 100755 --- a/integtest/test.sh +++ b/integtest/test.sh @@ -6,7 +6,6 @@ source _env.sh echo ${BASEDIR} echo "usage: [-d | --debug]" -build_init_properties clear_last_diff_files check_tpch_dir_is_present check_tpch_data_is_loaded diff --git a/integtest/test_all.sh b/integtest/test_all.sh index 509b08bfdf..ae938be82a 100755 --- a/integtest/test_all.sh +++ b/integtest/test_all.sh @@ -4,7 +4,6 @@ source _env.sh echo "Usage: [-a | -s | -i | -d | -h]" -build_init_properties isDebug=false while getopts ":shida" arg diff --git a/integtest/test_dag.sh b/integtest/test_dag.sh index 289bdedae4..b59a744e40 100755 --- a/integtest/test_dag.sh +++ b/integtest/test_dag.sh @@ -3,13 +3,11 @@ set -ue source _env.sh -echo "Base directory in: $BASEDIR" echo "Usage: [-h | -g | -a | -d | -s | -i | -r | -t | -b ]" echo "Note: must be quoted. e.g., \"select * from t\"" echo "You may use sql-only like this:" echo "./test_dag.sh -t \"select * from t\" -b \"test\"" -build_init_properties clear_last_diff_files_DAG isDebug=false diff --git a/integtest/test_no_tpch.sh b/integtest/test_no_tpch.sh index 21ab7f7a19..f7b14e9eec 100755 --- a/integtest/test_no_tpch.sh +++ b/integtest/test_no_tpch.sh @@ -3,13 +3,11 @@ set -ue source _env.sh -echo "Base directory in: $BASEDIR" echo "Usage: [-h | -g | -a | -d | -s | -i | -r | -t | -b ]" echo "Note: must be quoted. e.g., \"select * from t\"" echo "You may use sql-only like this:" echo "./test_no_tpch.sh -t \"select * from t\" -b \"test\"" -build_init_properties clear_last_diff_files isDebug=false diff --git a/integtest/test_tpch.sh b/integtest/test_tpch.sh index 210f66328b..8197bb4af6 100755 --- a/integtest/test_tpch.sh +++ b/integtest/test_tpch.sh @@ -6,7 +6,6 @@ source _env.sh echo ${BASEDIR} echo "usage: [-d | --debug]" -build_init_properties clear_last_diff_files_tpch check_tpch_dir_is_present check_tpch_data_is_loaded From 2383b9fd100bb9bd0cee7d4f16374ddc3699b072 Mon Sep 17 00:00:00 2001 From: birdstorm Date: Thu, 28 Dec 2017 19:46:30 +0800 Subject: [PATCH 4/7] change build logic --- integtest/_build.sh | 5 +---- integtest/test_all.sh | 20 +++----------------- integtest/test_dag.sh | 16 +--------------- integtest/test_no_tpch.sh | 18 ++---------------- 4 files changed, 7 insertions(+), 52 deletions(-) diff --git a/integtest/_build.sh b/integtest/_build.sh index 14a145648a..3984bc4957 100755 --- a/integtest/_build.sh +++ b/integtest/_build.sh @@ -3,9 +3,6 @@ set -ue source _env.sh -cd ../tikv-client/ -mvn clean install cd .. mvn clean install -cd integtest -mvn clean install \ No newline at end of file +cd integtest \ No newline at end of file diff --git a/integtest/test_all.sh b/integtest/test_all.sh index ae938be82a..a1fee1ab58 100755 --- a/integtest/test_all.sh +++ b/integtest/test_all.sh @@ -2,38 +2,24 @@ source _env.sh -echo "Usage: [-a | -s | -i | -d | -h]" +echo "Usage: [-a | -d | -h]" isDebug=false -while getopts ":shida" arg +while getopts ":hda" arg do case ${arg} in d) isDebug=true ;; a) - cd ../tikv-client/ - mvn clean install cd ../ mvn clean install cd integtest/ - mvn clean install - ;; - s) - cd ../ - mvn clean install - cd integtest/ - mvn clean install - ;; - i) - mvn clean install ;; h) echo "Options" - echo " -a make all projects" - echo " -s make tiSpark and integration test projects" - echo " -i make integration test only" + echo " -a build all projects" echo " -d debug mode" echo " -h show help" exit 1 diff --git a/integtest/test_dag.sh b/integtest/test_dag.sh index b59a744e40..70daf636e9 100755 --- a/integtest/test_dag.sh +++ b/integtest/test_dag.sh @@ -17,7 +17,7 @@ mode="Integration" sql= db= -while getopts "t:b:dishrag" arg +while getopts "t:b:dhrag" arg do case ${arg} in d) @@ -30,21 +30,9 @@ do showFailedOnly=true ;; a) - cd ../tikv-client - mvn clean install cd ../ mvn clean install cd integtest/ - mvn clean install - ;; - s) - cd ../ - mvn clean install - cd integtest/ - mvn clean install - ;; - i) - mvn clean install ;; t) sql=$OPTARG @@ -58,8 +46,6 @@ do h) echo "Options" echo " -a make all projects" - echo " -s make tiSpark and integration test projects" - echo " -i make integration test only" echo " -r show result stats (SQL, outputs, time consumed, etc.)" echo " -g show failed only" echo " -t run sql statement (with quotes) only on TiSpark with debug mode (must assign a database)" diff --git a/integtest/test_no_tpch.sh b/integtest/test_no_tpch.sh index f7b14e9eec..56d1d0934d 100755 --- a/integtest/test_no_tpch.sh +++ b/integtest/test_no_tpch.sh @@ -17,7 +17,7 @@ mode="Integration" sql= db= -while getopts "t:b:dishrag" arg +while getopts "t:b:dhrag" arg do case ${arg} in d) @@ -30,21 +30,9 @@ do showFailedOnly=true ;; a) - cd ../tikv-client/ - mvn clean install cd ../ mvn clean install cd integtest/ - mvn clean install - ;; - s) - cd ../ - mvn clean install - cd integtest/ - mvn clean install - ;; - i) - mvn clean install ;; t) sql=$OPTARG @@ -57,9 +45,7 @@ do ;; h) echo "Options" - echo " -a make all projects" - echo " -s make tiSpark and integration test projects" - echo " -i make integration test only" + echo " -a build all projects" echo " -r show result stats (SQL, outputs, time consumed, etc.)" echo " -g show failed only" echo " -t run sql statement (with quotes) only on TiSpark with debug mode (must assign a database)" From f1e8100460282d93fd21cbb9d01499a029d615d2 Mon Sep 17 00:00:00 2001 From: birdstorm Date: Thu, 28 Dec 2017 20:04:49 +0800 Subject: [PATCH 5/7] rename TestIndex enum --- integtest/_env.sh | 2 +- integtest/src/main/scala/com/pingcap/spark/TestCase.scala | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/integtest/_env.sh b/integtest/_env.sh index 0f4f180d03..8842fdf23d 100644 --- a/integtest/_env.sh +++ b/integtest/_env.sh @@ -94,7 +94,7 @@ create_conf() { create_conf_no_tpch() { echo "create conf for custom tests..." create_conf_db_options - echo "test.mode=TestIndex" >> ${BASE_CONF} + echo "test.mode=TestAlone" >> ${BASE_CONF} echo "test.ignore=tpch,tpch_test,tispark_test" >> ${BASE_CONF} cp ${BASE_CONF} ${TISPARK_CONF} diff --git a/integtest/src/main/scala/com/pingcap/spark/TestCase.scala b/integtest/src/main/scala/com/pingcap/spark/TestCase.scala index 136097fab3..2027a1a8a5 100644 --- a/integtest/src/main/scala/com/pingcap/spark/TestCase.scala +++ b/integtest/src/main/scala/com/pingcap/spark/TestCase.scala @@ -31,7 +31,7 @@ class TestCase(val prop: Properties) extends LazyLogging { object RunMode extends Enumeration { type RunMode = Value - val Test, Load, LoadNTest, Dump, TestIndex, TestDAG, SqlOnly = Value + val Test, Load, LoadNTest, Dump, TestAlone, TestDAG, SqlOnly = Value } protected val KeyDumpDBList = "test.dumpDB.databases" @@ -121,7 +121,7 @@ class TestCase(val prop: Properties) extends LazyLogging { case RunMode.LoadNTest => work(basePath, run = true, load = true, compareNeeded = true) - case RunMode.TestIndex => work(basePath, run = true, load = false, compareNeeded = false) + case RunMode.TestAlone => work(basePath, run = true, load = false, compareNeeded = false) case RunMode.TestDAG => work(basePath, run = true, load = false, compareNeeded = false) @@ -129,7 +129,7 @@ class TestCase(val prop: Properties) extends LazyLogging { } mode match { - case RunMode.Test | RunMode.TestIndex | RunMode.TestDAG => + case RunMode.Test | RunMode.TestAlone | RunMode.TestDAG => logger.warn("Result: All tests done.") logger.warn( "Result: Tests run: " + testsExecuted From 25d284834d8bf5d88cc48b6c3820122da8f5a027 Mon Sep 17 00:00:00 2001 From: birdstorm Date: Thu, 28 Dec 2017 20:09:41 +0800 Subject: [PATCH 6/7] remove useless code --- integtest/_env.sh | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/integtest/_env.sh b/integtest/_env.sh index 8842fdf23d..cd634f53e8 100644 --- a/integtest/_env.sh +++ b/integtest/_env.sh @@ -136,19 +136,6 @@ create_conf_dump() { cp ${BASE_CONF} ${TISPARK_CONF} } -#add_MySQL_info() { -# use_raw_mysql=true -# echo "spark.use_raw_mysql=true" >> ${TISPARK_CONF} -# echo "mysql.addr=$mysql_addr" >> ${TISPARK_CONF} -# echo "mysql.user=$mysql_user" >> ${TISPARK_CONF} -# echo "mysql.password=$mysql_password" >> ${TISPARK_CONF} -# -# echo "spark.use_raw_mysql=true" >> ${BASE_CONF} -# echo "mysql.addr=$mysql_addr" >> ${BASE_CONF} -# echo "mysql.user=$mysql_user" >> ${BASE_CONF} -# echo "mysql.password=$mysql_password" >> ${BASE_CONF} -#} - clear_all_diff_files() { for f in ./*.spark; do [ -e "$f" ] && rm *.spark From 6d8ab5ec83769e07ba36b73ad6124644dac69aa2 Mon Sep 17 00:00:00 2001 From: birdstorm Date: Thu, 28 Dec 2017 23:43:35 +0800 Subject: [PATCH 7/7] Update test_dag.sh --- integtest/test_dag.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integtest/test_dag.sh b/integtest/test_dag.sh index 70daf636e9..ac0fcc9ae7 100755 --- a/integtest/test_dag.sh +++ b/integtest/test_dag.sh @@ -45,7 +45,7 @@ do ;; h) echo "Options" - echo " -a make all projects" + echo " -a build all projects" echo " -r show result stats (SQL, outputs, time consumed, etc.)" echo " -g show failed only" echo " -t run sql statement (with quotes) only on TiSpark with debug mode (must assign a database)" @@ -117,4 +117,4 @@ elif [ "${mode}" == "QueryOnly" ]; then fi else echo "UnKnown test mode: $mode. Aborting..." -fi \ No newline at end of file +fi