Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

modify integration scripts and .gitignore files #171

Merged
merged 7 commits into from
Dec 29, 2017
+39 −82
Merged

modify integration scripts and .gitignore files #171

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
589cafe
modify integration scripts and local paths since whole project struct…
birdstorm Dec 28, 2017
ddad00a
fix test_no_tpch.sh
birdstorm Dec 28, 2017
fc4666c
refactor build_init_properties
birdstorm Dec 28, 2017
2383b9f
change build logic
birdstorm Dec 28, 2017
f1e8100
rename TestIndex enum
birdstorm Dec 28, 2017
25d2848
remove useless code
birdstorm Dec 28, 2017
6d8ab5e
Update test_dag.sh
birdstorm Dec 28, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,3 +59,6 @@ tikv-client/kvproto
tikv-client/tipb
tikv-client/proto

# ignore tikv-client bazel build
tikv-client/bazel-*

5 changes: 1 addition & 4 deletions integtest/_build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,6 @@ set -ue

source _env.sh

cd ../tikv-client/
mvn clean install
cd ..
mvn clean install
cd integtest
mvn clean install
cd integtest
34 changes: 20 additions & 14 deletions integtest/_env.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
set -ue

BASEDIR=$(cd `dirname $0`; pwd)
echo "Base directory in: $BASEDIR"
PATH_TO_CONF="$BASEDIR/conf"

if [ -z "${SPARK_HOME}" ]; then
Expand All @@ -19,6 +20,7 @@ spark_test_opt=""

TISPARK_CONF="${SPARK_HOME}/conf/tispark_config.properties"
BASE_CONF="${BASEDIR}/conf/tispark_config.properties"
INIT_CONF="${BASEDIR}/conf/tidb_config.properties"

mysql_addr="localhost"
mysql_user="root"
Expand All @@ -29,6 +31,22 @@ tidb_addr=
tidb_port=
tidb_user=

build_if_not_exists_init_properties() {
file=${BASE_CONF}
init=${INIT_CONF}

if ! [ -f "$file" ]
then
echo "$file not found. "
echo "Building initial config file."
if ! [ -f "$init" ]; then
echo "$init not found. Please set this file manually according to README."
exit -1
fi
cp ${init} ${file}
fi
}

read_properties() {
file=${BASE_CONF}

Expand All @@ -52,6 +70,7 @@ read_properties() {
fi
}

build_if_not_exists_init_properties
read_properties

create_conf_db_options() {
Expand All @@ -75,7 +94,7 @@ create_conf() {
create_conf_no_tpch() {
echo "create conf for custom tests..."
create_conf_db_options
echo "test.mode=TestIndex" >> ${BASE_CONF}
echo "test.mode=TestAlone" >> ${BASE_CONF}
echo "test.ignore=tpch,tpch_test,tispark_test" >> ${BASE_CONF}

cp ${BASE_CONF} ${TISPARK_CONF}
Expand Down Expand Up @@ -117,19 +136,6 @@ create_conf_dump() {
cp ${BASE_CONF} ${TISPARK_CONF}
}

#add_MySQL_info() {
# use_raw_mysql=true
# echo "spark.use_raw_mysql=true" >> ${TISPARK_CONF}
# echo "mysql.addr=$mysql_addr" >> ${TISPARK_CONF}
# echo "mysql.user=$mysql_user" >> ${TISPARK_CONF}
# echo "mysql.password=$mysql_password" >> ${TISPARK_CONF}
#
# echo "spark.use_raw_mysql=true" >> ${BASE_CONF}
# echo "mysql.addr=$mysql_addr" >> ${BASE_CONF}
# echo "mysql.user=$mysql_user" >> ${BASE_CONF}
# echo "mysql.password=$mysql_password" >> ${BASE_CONF}
#}

clear_all_diff_files() {
for f in ./*.spark; do
[ -e "$f" ] && rm *.spark
Expand Down
2 changes: 1 addition & 1 deletion integtest/conf/log4j.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
#

# Set everything to be logged to the console
log4j.rootCategory=INFO, console
log4j.rootCategory=INFO, console

log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.layout=org.apache.log4j.PatternLayout
Expand Down
3 changes: 3 additions & 0 deletions integtest/conf/tidb_config.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
tidb.addr=127.0.0.1
tidb.port=4000
tidb.user=root
8 changes: 0 additions & 8 deletions integtest/conf/tispark_config.properties
View file
Open in desktop

This file was deleted.

6 changes: 3 additions & 3 deletions integtest/src/main/scala/com/pingcap/spark/TestCase.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ class TestCase(val prop: Properties) extends LazyLogging {

object RunMode extends Enumeration {
type RunMode = Value
val Test, Load, LoadNTest, Dump, TestIndex, TestDAG, SqlOnly = Value
val Test, Load, LoadNTest, Dump, TestAlone, TestDAG, SqlOnly = Value
}

protected val KeyDumpDBList = "test.dumpDB.databases"
Expand Down Expand Up @@ -121,15 +121,15 @@ class TestCase(val prop: Properties) extends LazyLogging {

case RunMode.LoadNTest => work(basePath, run = true, load = true, compareNeeded = true)

case RunMode.TestIndex => work(basePath, run = true, load = false, compareNeeded = false)
case RunMode.TestAlone => work(basePath, run = true, load = false, compareNeeded = false)

case RunMode.TestDAG => work(basePath, run = true, load = false, compareNeeded = false)

case RunMode.SqlOnly => work(basePath, run = true, load = false, compareNeeded = false)
}

mode match {
case RunMode.Test | RunMode.TestIndex | RunMode.TestDAG =>
case RunMode.Test | RunMode.TestAlone | RunMode.TestDAG =>
logger.warn("Result: All tests done.")
logger.warn(
"Result: Tests run: " + testsExecuted
Expand Down
20 changes: 3 additions & 17 deletions integtest/test_all.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,38 +2,24 @@

source _env.sh

echo "Usage: <bin> [-a | -s | -i | -d | -h]"
echo "Usage: <bin> [-a | -d | -h]"

isDebug=false

while getopts ":shida" arg
while getopts ":hda" arg
do
case ${arg} in
d)
isDebug=true
;;
a)
cd ../tikv-client/
mvn clean install
cd ../
mvn clean install
cd integtest/
mvn clean install
;;
s)
cd ../
mvn clean install
cd integtest/
mvn clean install
;;
i)
mvn clean install
;;
h)
echo "Options"
echo " -a make all projects"
echo " -s make tiSpark and integration test projects"
echo " -i make integration test only"
echo " -a build all projects"
echo " -d debug mode"
echo " -h show help"
exit 1
Expand Down
21 changes: 3 additions & 18 deletions integtest/test_dag.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ set -ue

source _env.sh

echo "Base directory in: $BASEDIR"
echo "Usage: <bin> [-h | -g | -a | -d | -s | -i | -r | -t <sql> | -b <db>]"
echo "Note: <sql> must be quoted. e.g., \"select * from t\""
echo "You may use sql-only like this:"
Expand All @@ -18,7 +17,7 @@ mode="Integration"
sql=
db=

while getopts "t:b:dishrag" arg
while getopts "t:b:dhrag" arg
do
case ${arg} in
d)
Expand All @@ -31,21 +30,9 @@ do
showFailedOnly=true
;;
a)
cd ../tikv-client
mvn clean install
cd ../
mvn clean install
cd integtest/
mvn clean install
;;
s)
cd ../
mvn clean install
cd integtest/
mvn clean install
;;
i)
mvn clean install
;;
t)
sql=$OPTARG
Expand All @@ -58,9 +45,7 @@ do
;;
h)
echo "Options"
echo " -a make all projects"
echo " -s make tiSpark and integration test projects"
echo " -i make integration test only"
echo " -a build all projects"
echo " -r show result stats (SQL, outputs, time consumed, etc.)"
echo " -g show failed only"
echo " -t <sql> run sql statement <sql> (with quotes) only on TiSpark with debug mode (must assign a database)"
Expand Down Expand Up @@ -132,4 +117,4 @@ elif [ "${mode}" == "QueryOnly" ]; then
fi
else
echo "UnKnown test mode: $mode. Aborting..."
fi
fi
19 changes: 2 additions & 17 deletions integtest/test_no_tpch.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ set -ue

source _env.sh

echo "Base directory in: $BASEDIR"
echo "Usage: <bin> [-h | -g | -a | -d | -s | -i | -r | -t <sql> | -b <db>]"
echo "Note: <sql> must be quoted. e.g., \"select * from t\""
echo "You may use sql-only like this:"
Expand All @@ -18,7 +17,7 @@ mode="Integration"
sql=
db=

while getopts "t:b:dishrag" arg
while getopts "t:b:dhrag" arg
do
case ${arg} in
d)
Expand All @@ -31,21 +30,9 @@ do
showFailedOnly=true
;;
a)
cd ../tikv-client-lib-java/
mvn clean install
cd ../
mvn clean install
cd integtest/
mvn clean install
;;
s)
cd ../
mvn clean install
cd integtest/
mvn clean install
;;
i)
mvn clean install
;;
t)
sql=$OPTARG
Expand All @@ -58,9 +45,7 @@ do
;;
h)
echo "Options"
echo " -a make all projects"
echo " -s make tiSpark and integration test projects"
echo " -i make integration test only"
echo " -a build all projects"
echo " -r show result stats (SQL, outputs, time consumed, etc.)"
echo " -g show failed only"
echo " -t <sql> run sql statement <sql> (with quotes) only on TiSpark with debug mode (must assign a database)"
Expand Down