Merge pull request WeBankFinTech#2 from WeBankFinTech/master

Merge to latest version

README-ZH.md

@@ -142,10 +142,14 @@ DSS主要特点：
 
 [DSS编译文档](docs/zh_CN/ch1/DSS编译文档.md)
 
+[快速使用文档](docs/zh_CN/ch3/DataSphere_Studio_QuickStart.md)
+
 [用户手册](docs/zh_CN/ch3/DSS_User_Manual.md)
 
 [外部系统快速接入DSS](docs/zh_CN/ch4/第三方系统接入DSS指南.md)
 
+[如何接入调度系统Azkaban](docs/zh_CN/ch4/如何接入调度系统Azkaban.md)
+
 更多文档，敬请期待！
 
 ## 交流贡献

bin/checkEnv.sh

@@ -0,0 +1,46 @@
+#
+# Copyright 2019 WeBank
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#!/bin/sh
+say() {
+    printf 'check command fail \n %s\n' "$1"
+}
+
+err() {
+    say "$1" >&2
+    exit 1
+}
+
+check_cmd() {
+    command -v "$1" > /dev/null 2>&1
+}
+
+need_cmd() {
+    if ! check_cmd "$1"; then
+        err "need '$1' (your linux command not found)"
+    fi
+}
+echo "<-----start to check linux cmd:yum java mysql unzip expect telnet sed tar---->"
+need_cmd yum
+need_cmd java
+need_cmd mysql
+need_cmd unzip
+need_cmd expect
+need_cmd telnet
+need_cmd sed
+need_cmd tar
+need_cmd source
+need_cmd hostname
+echo "<-----end to check linux cmd:yum java mysql unzip expect telnet sed tar------>"

bin/install.sh

@@ -1,3 +1,18 @@
+#
+# Copyright 2019 WeBank
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
 #!/bin/sh
 #Actively load user env
 source ~/.bash_profile
@@ -38,6 +53,11 @@ else
 fi
 }
 
+
+#check env
+sh ${workDir}/bin/checkEnv.sh
+isSuccess "check env"
+
 function checkJava(){
 	java -version
 	isSuccess "execute java --version"
@@ -158,38 +178,55 @@ fi
 ##init db
 if [[ '2' = "$MYSQL_INSTALL_MODE" ]];then
 	mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source ${workDir}/db/dss_ddl.sql"
-	isSuccess "source linkis_ddl.sql"
-	LOCAL_IP="`ifconfig | grep 'inet' | grep -v '127.0.0.1' | cut -d: -f2 | awk '{ print $2}'`"
+	isSuccess "source dss_ddl.sql"
+	LOCAL_IP="`hostname -i`"
 	if [ $GATEWAY_INSTALL_IP == "127.0.0.1" ];then
         echo "GATEWAY_INSTALL_IP is equals 127.0.0.1 ,we will change it to ip address"
         GATEWAY_INSTALL_IP_2=$LOCAL_IP
     else
         GATEWAY_INSTALL_IP_2=$GATEWAY_INSTALL_IP
     fi
-    echo $GATEWAY_INSTALL_IP_2
+    #echo $GATEWAY_INSTALL_IP_2
     sed -i "s/GATEWAY_INSTALL_IP_2/$GATEWAY_INSTALL_IP_2/g" ${workDir}/db/dss_dml.sql
     sed -i "s/GATEWAY_PORT/$GATEWAY_PORT/g" ${workDir}/db/dss_dml.sql
-    if [ $AZKABAN_ADRESS_IP == "127.0.0.1" ];then
-        echo "AZKABAN_ADRESS_IP is equals 127.0.0.1 ,we will change it to ip address"
-        AZKABAN_ADRESS_IP_2=$LOCAL_IP
-    else
-        AZKABAN_ADRESS_IP_2=$AZKABAN_ADRESS_IP
-    fi
-    echo $AZKABAN_ADRESS_IP_2
-    sed -i "s/AZKABAN_ADRESS_IP_2/$AZKABAN_ADRESS_IP_2/g" ${workDir}/db/dss_dml.sql
-    sed -i "s/AZKABAN_ADRESS_PORT/$AZKABAN_ADRESS_PORT/g" ${workDir}/db/dss_dml.sql
     if [ $VISUALIS_NGINX_IP == "127.0.0.1" ]||[ $VISUALIS_NGINX_IP == "0.0.0.0" ];then
         echo "VISUALIS_NGINX_IP is equals $VISUALIS_NGINX_IP ,we will change it to ip address"
         VISUALIS_NGINX_IP_2=$LOCAL_IP
     else
         VISUALIS_NGINX_IP_2=$VISUALIS_NGINX_IP
     fi
-    echo $VISUALIS_NGINX_IP_2
+    #echo $VISUALIS_NGINX_IP_2
     sed -i "s/VISUALIS_NGINX_IP_2/$VISUALIS_NGINX_IP_2/g" ${workDir}/db/dss_dml.sql
     sed -i "s/VISUALIS_NGINX_PORT/$VISUALIS_NGINX_PORT/g" ${workDir}/db/dss_dml.sql
 	mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source ${workDir}/db/dss_dml.sql"
-	isSuccess "source linkis_dml.sql"
-	echo "Rebuild the table"
+	isSuccess "source dss_dml.sql"
+	if [[ '2' = "$INSTALL_MODE" ]];then
+	   echo "start to update azkaban and qualitis table info "
+       #azkaban
+       if [ $AZKABAN_ADRESS_IP == "127.0.0.1" ];then
+           echo "AZKABAN_ADRESS_IP is equals 127.0.0.1 ,we will change it to ip address"
+           AZKABAN_ADRESS_IP_2=$LOCAL_IP
+       else
+            AZKABAN_ADRESS_IP_2=$AZKABAN_ADRESS_IP
+       fi
+       echo $AZKABAN_ADRESS_IP_2
+       sed -i "s/AZKABAN_ADRESS_IP_2/$AZKABAN_ADRESS_IP_2/g" ${workDir}/db/azkaban.sql
+       sed -i "s/AZKABAN_ADRESS_PORT/$AZKABAN_ADRESS_PORT/g" ${workDir}/db/azkaban.sql
+       mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source ${workDir}/db/azkaban.sql"
+	   isSuccess "source azkaban.sql"
+       #qualitis
+       if [ $QUALITIS_ADRESS_IP == "127.0.0.1" ];then
+           echo "QUALITIS_ADRESS_IP is equals 127.0.0.1 ,we will change it to ip address"
+           QUALITIS_ADRESS_IP_2=$LOCAL_IP
+       else
+            QUALITIS_ADRESS_IP_2=$QUALITIS_ADRESS_IP
+       fi
+       echo $QUALITIS_ADRESS_IP_2
+       sed -i "s/QUALITIS_ADRESS_IP_2/$QUALITIS_ADRESS_IP_2/g" ${workDir}/db/qualitis.sql
+       sed -i "s/QUALITIS_ADRESS_PORT/$QUALITIS_ADRESS_PORT/g" ${workDir}/db/qualitis.sql
+       mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source ${workDir}/db/qualitis.sql"
+	   isSuccess "source qualitis.sql"
+    fi
 fi
 
 ##env check
@@ -304,8 +341,7 @@ isSuccess "install  ${APPJOINTNAME}.zip"
 }
 ##function end
 
-##
-ver Install
+##dss-Server install
 PACKAGE_DIR=dss/dss-server
 SERVERNAME=dss-server
 SERVER_IP=$DSS_SERVER_INSTALL_IP
@@ -354,8 +390,8 @@ installPackage
 ###Update appjoint entrance linkis.properties
 echo "$SERVERNAME-step4:update linkis.properties"
 SERVER_CONF_PATH=$SERVER_HOME/$SERVERNAME/conf/linkis.properties
-ssh $SERVER_IP "sed -i  \"s#wds.linkis.entrance.config.logPath.*#wds.linkis.entrance.config.logPath=$WORKSPACE_USER_ROOT_PATH#g\" $SERVER_CONF_PATH"
-ssh $SERVER_IP "sed -i  \"s#wds.linkis.resultSet.store.path.*#wds.linkis.resultSet.store.path=$RESULT_SET_ROOT_PATH#g\" $SERVER_CONF_PATH"
+ssh -p $SSH_PORT $SERVER_IP "sed -i  \"s#wds.linkis.entrance.config.logPath.*#wds.linkis.entrance.config.logPath=$WORKSPACE_USER_ROOT_PATH#g\" $SERVER_CONF_PATH"
+ssh -p $SSH_PORT $SERVER_IP "sed -i  \"s#wds.linkis.resultSet.store.path.*#wds.linkis.resultSet.store.path=$RESULT_SET_ROOT_PATH#g\" $SERVER_CONF_PATH"
 isSuccess "subsitution linkis.properties of $SERVERNAME"
 echo "<----------------$SERVERNAME:end------------------->"
 echo ""
@@ -423,6 +459,8 @@ APPJOINTNAME=visualis
 #visualis  appjoint install
 installAppjoints
 echo "<----------------$APPJOINTNAME:end------------------->"
+##sample version does not install qualitis APPJoint and scheduis APPJoint
+if [[ '2' = "$INSTALL_MODE" ]];then
 echo ""
 echo "<----------------qualitis  appjoint install start------------------->"
 APPJOINTPARENT=dss-appjoints
@@ -439,5 +477,6 @@ APPJOINTPARENT=dss-appjoints
 APPJOINTNAME=schedulis
 #schedulis  appjoint install
 installAppjoints
-isSuccess "subsitution conf of qualitis"
-echo "<----------------$APPJOINTNAME:end------------------->"
+isSuccess "subsitution conf of schedulis"
+echo "<----------------$APPJOINTNAME:end------------------->"
+fi

bin/start-all.sh

@@ -76,6 +76,16 @@ sleep 15 #for Eureka register
 SERVER_NAME=dss-server
 SERVER_IP=$DSS_SERVER_INSTALL_IP
 startApp
+#MICRO_SERVICE_NAME=dss-server
+#MICRO_SERVICE_IP=$DSS_SERVER_INSTALL_IP
+#MICRO_SERVICE_PORT=$DSS_SERVER_PORT
+#sh $workDir/check.sh $MICRO_SERVICE_NAME $MICRO_SERVICE_IP $MICRO_SERVICE_PORT
+#state=`echo -e "\n" | telnet $MICRO_SERVICE_IP $MICRO_SERVICE_PORT 2>/dev/null | grep Connected | wc -l`
+#if [ $state -eq 0 ]; then
+#   echo ""
+#   echo "ERROR " $MICRO_SERVICE_NAME "is a critical service and must be guaranteed to be started !!!"  
+#   exit 1
+#fi
 
 #dss-flow-execution-entrance
 SERVER_NAME=dss-flow-execution-entrance
@@ -91,3 +101,41 @@ SERVER_NAME=visualis-server
 SERVER_IP=$VISUALIS_SERVER_INSTALL_IP
 startApp
 
+
+echo "Start to check all dss microservice"
+
+#check dss-server
+MICRO_SERVICE_NAME=dss-server
+MICRO_SERVICE_IP=$DSS_SERVER_INSTALL_IP
+MICRO_SERVICE_PORT=$DSS_SERVER_PORT
+sh $workDir/checkMicro.sh $MICRO_SERVICE_NAME $MICRO_SERVICE_IP $MICRO_SERVICE_PORT
+state=`echo -e "\n" | telnet $MICRO_SERVICE_IP $MICRO_SERVICE_PORT 2>/dev/null | grep Connected | wc -l`
+isSuccess "$MICRO_SERVICE_NAME start"
+
+
+#check dss-flow-execution-entrance
+MICRO_SERVICE_NAME=dss-flow-execution-entrance
+MICRO_SERVICE_IP=$FLOW_EXECUTION_INSTALL_IP
+MICRO_SERVICE_PORT=$FLOW_EXECUTION_PORT
+sh $workDir/checkMicro.sh $MICRO_SERVICE_NAME $MICRO_SERVICE_IP $MICRO_SERVICE_PORT
+state=`echo -e "\n" | telnet $MICRO_SERVICE_IP $MICRO_SERVICE_PORT 2>/dev/null | grep Connected | wc -l`
+isSuccess "$MICRO_SERVICE_NAME start"
+
+#check linkis-appjoint-entrance
+MICRO_SERVICE_NAME=linkis-appjoint-entrance
+MICRO_SERVICE_IP=$APPJOINT_ENTRANCE_INSTALL_IP
+MICRO_SERVICE_PORT=$APPJOINT_ENTRANCE_PORT
+sh $workDir/checkMicro.sh $MICRO_SERVICE_NAME $MICRO_SERVICE_IP $MICRO_SERVICE_PORT
+state=`echo -e "\n" | telnet $MICRO_SERVICE_IP $MICRO_SERVICE_PORT 2>/dev/null | grep Connected | wc -l`
+isSuccess "$MICRO_SERVICE_NAME start"
+
+
+#check visualis-server
+sleep 10 #for visualis-server 
+MICRO_SERVICE_NAME=visualis-server
+MICRO_SERVICE_IP=$VISUALIS_SERVER_INSTALL_IP
+MICRO_SERVICE_PORT=$VISUALIS_SERVER_PORT
+sh $workDir/checkMicro.sh $MICRO_SERVICE_NAME $MICRO_SERVICE_IP $MICRO_SERVICE_PORT
+state=`echo -e "\n" | telnet $MICRO_SERVICE_IP $MICRO_SERVICE_PORT 2>/dev/null | grep Connected | wc -l`
+isSuccess "$MICRO_SERVICE_NAME start"
+

conf/config.sh

@@ -15,7 +15,7 @@ WORKSPACE_USER_ROOT_PATH=file:///tmp/linkis/
 RESULT_SET_ROOT_PATH=hdfs:///tmp/linkis
 
 ### 1、DataCheck APPJOINT，This service is used to provide DataCheck capability.
-HIVE_META_URL=jdbc:mysql://127.0.0.1:3306/linkis?characterEncoding=UTF-8
+HIVE_META_URL=jdbc:mysql://127.0.0.1:3306/hivemeta?characterEncoding=UTF-8
 HIVE_META_USER=xxx
 HIVE_META_PASSWORD=xxx
 
@@ -50,16 +50,16 @@ GATEWAY_PORT=9001
 ### SSH Port
 SSH_PORT=22
 
-#for azkaban
+#Used to store the azkaban project transformed by DSS
 WDS_SCHEDULER_PATH=file:///appcom/tmp/wds/scheduler
 
 ###The IP address and port are written into the database here, so be sure to plan ahead
 ## visualis-server
 VISUALIS_SERVER_INSTALL_IP=127.0.0.1
 VISUALIS_SERVER_PORT=9007
-### visualis nginx acess ip
+### visualis nginx acess ip,keep consistent with DSS front end
 VISUALIS_NGINX_IP=0.0.0.0
-VISUALIS_NGINX_PORT=9009
+VISUALIS_NGINX_PORT=8088
 
 ### Eventchecker APPJOINT
 ### This service is used to provide Eventchecker capability. it's config in db.sh same as dss-server.

db/azkaban.sql

@@ -0,0 +1,3 @@
+INSERT INTO `dss_application` (`id`, `name`, `url`, `is_user_need_init`, `level`, `user_init_url`, `exists_project_service`, `project_url`, `enhance_json`, `if_iframe`, `homepage_url`, `redirect_url`) VALUES (NULL, 'schedulis', NULL, '0', '1', NULL, '0', NULL, NULL, '1', NULL, NULL);
+UPDATE `dss_application` SET url = 'http://AZKABAN_ADRESS_IP_2:AZKABAN_ADRESS_PORT', project_url = 'http://AZKABAN_ADRESS_IP_2:AZKABAN_ADRESS_PORT/manager?project=${projectName}',homepage_url = 'http://AZKABAN_ADRESS_IP_2:AZKABAN_ADRESS_PORT/homepage' WHERE `name` in
+ ('schedulis');

db/dss_dml.sql

@@ -1,6 +1,5 @@
 INSERT INTO `dss_application` (`id`, `name`, `url`, `is_user_need_init`, `level`, `user_init_url`, `exists_project_service`, `project_url`, `enhance_json`, `if_iframe`, `homepage_url`, `redirect_url`) VALUES (NULL, 'linkis', null, '0', '1', NULL, '0', '/home', NULL, '0', '/home', NULL);
 INSERT INTO `dss_application` (`id`, `name`, `url`, `is_user_need_init`, `level`, `user_init_url`, `exists_project_service`, `project_url`, `enhance_json`, `if_iframe`, `homepage_url`, `redirect_url`) VALUES (NULL, 'visualis', null, '0', '1', NULL, '0', NULL, NULL, '1', NULL, NULL);
-INSERT INTO `dss_application` (`id`, `name`, `url`, `is_user_need_init`, `level`, `user_init_url`, `exists_project_service`, `project_url`, `enhance_json`, `if_iframe`, `homepage_url`, `redirect_url`) VALUES (NULL, 'schedulis', NULL, '0', '1', NULL, '0', NULL, NULL, '1', NULL, NULL);
 INSERT INTO `dss_application` (`id`, `name`, `url`, `is_user_need_init`, `level`, `user_init_url`, `exists_project_service`, `project_url`, `enhance_json`, `if_iframe`, `homepage_url`, `redirect_url`) VALUES (NULL, 'workflow', null, '0', '1', NULL, '0', '/workflow', NULL, '0', '/project', NULL);
 INSERT INTO `dss_application` (`id`, `name`, `url`, `is_user_need_init`, `level`, `user_init_url`, `exists_project_service`, `project_url`, `enhance_json`, `if_iframe`, `homepage_url`, `redirect_url`) VALUES (NULL, 'console', null, '0', '1', NULL, '0', '/console', NULL, '0', '/console', NULL);
 
@@ -30,7 +29,6 @@ INSERT INTO `dss_flow_taxonomy` (`id`, `name`, `description`, `creator_id`, `cre
 UPDATE `dss_application` SET url = 'http://GATEWAY_INSTALL_IP_2:GATEWAY_PORT' WHERE `name` in('linkis','workflow');
 UPDATE `dss_application` SET url = 'http://VISUALIS_NGINX_IP_2:VISUALIS_NGINX_PORT' WHERE `name` in('visualis');
 UPDATE `dss_application` SET project_url = 'http://VISUALIS_NGINX_IP_2:VISUALIS_NGINX_PORT/dss/visualis/#/project/${projectId}',homepage_url = 'http://VISUALIS_NGINX_IP_2:VISUALIS_NGINX_PORT/dss/visualis/#/projects' WHERE `name` in('visualis');
-UPDATE `dss_application` SET url = 'http://AZKABAN_ADRESS_IP_2:AZKABAN_ADRESS_PORT', project_url = 'http://AZKABAN_ADRESS_IP_2:AZKABAN_ADRESS_PORT/manager?project=${projectName}',homepage_url = 'http://AZKABAN_ADRESS_IP_2:AZKABAN_ADRESS_PORT/homepage' WHERE `name` in('schedulis');
 UPDATE `dss_workflow_node` SET jump_url = 'http://VISUALIS_NGINX_IP_2:VISUALIS_NGINX_PORT/dss/visualis/#/project/${projectId}/display/${nodeId}' where node_type = 'linkis.appjoint.visualis.display';
 UPDATE `dss_workflow_node` SET jump_url = 'http://VISUALIS_NGINX_IP_2:VISUALIS_NGINX_PORT/dss/visualis/#/project/${projectId}/portal/${nodeId}/portalName/${nodeName}' where node_type = 'linkis.appjoint.visualis.dashboard';
 
@@ -59,4 +57,4 @@ insert into `linkis_config_key_tree` VALUES(NULL,@key_id2,@tree_id1);
 insert into `linkis_config_key_tree` VALUES(NULL,@key_id3,@tree_id1);
 insert into `linkis_config_key_tree` VALUES(NULL,@key_id4,@tree_id1);
 insert into `linkis_config_key_tree` VALUES(NULL,@key_id5,@tree_id1);
-insert into `linkis_config_key_tree` VALUES(NULL,@key_id6,@tree_id2);
+insert into `linkis_config_key_tree` VALUES(NULL,@key_id6,@tree_id2);

db/qualitis.sql

@@ -0,0 +1,3 @@
+INSERT INTO `dss_application` (`id`, `name`, `url`, `is_user_need_init`, `level`, `user_init_url`, `exists_project_service`, `project_url`, `enhance_json`, `if_iframe`, `homepage_url`, `redirect_url`) VALUES (NULL, 'qualitis', 'http://QUALITIS_ADRESS_IP_2:QUALITIS_ADRESS_PORT', '0', '1', NULL, '1', 'http://QUALITIS_ADRESS_IP_2:QUALITIS_ADRESS_PORT/#/projects/list?id=${projectId}&flow=true', NULL, '1', 'http://QUALITIS_ADRESS_IP_2:QUALITIS_ADRESS_PORT/#/dashboard', 'http://QUALITIS_ADRESS_IP_2:QUALITIS_ADRESS_PORT/qualitis/api/v1/redirect');
+SELECT @qualitis_appid:=id from dss_application WHERE `name` = 'qualitis';
+INSERT INTO `dss_workflow_node` (`id`, `icon`, `node_type`, `application_id`, `submit_to_scheduler`, `enable_copy`, `should_creation_before_node`, `support_jump`, `jump_url`) VALUES (NULL, NULL, 'linkis.appjoint.qualitis', @qualitis_appid, NULL, '1', '0', '1', 'http://QUALITIS_ADRESS_IP_2:QUALITIS_ADRESS_PORT/#/addGroupTechniqueRule?tableType=1&id=${projectId}&ruleGroupId=${ruleGroupId}&nodeId=${nodeId}');