Merge pull request #2 from WeBankFinTech/master

拉代码

.gitattributes

@@ -0,0 +1,4 @@
+*.js linguist-language=java
+*.css linguist-language=java
+*.html linguist-language=java
+*.vue linguist-language=java

README-ZH.md

@@ -146,6 +146,8 @@ DSS主要特点：
 
 [外部系统快速接入DSS](docs/zh_CN/ch4/第三方系统接入DSS指南.md)
 
+[如何接入调度系统Azkaban](docs/zh_CN/ch4/如何接入调度系统Azkaban.md)
+
 更多文档，敬请期待！
 
 ## 交流贡献

README.md

@@ -133,19 +133,19 @@ Please be patient, it will take some time to load gif.
 
 ## Quick start
 
-Click to [Quick start]()
+Click to [Quick start](docs/en_US/ch2/DSS%20Quick%20Installation%20Guide.md)
 
 ## Architecture
 
 ![DSS Architecture](images/en_US/readme/architecture.png)
 
 ## Documents
 
-[Compiled documentation]()
+[Compiled documentation](https://github.com/WeBankFinTech/DataSphereStudio/blob/master/docs/en_US/ch1/DataSphereStudio_Compile_Manual.md)
 
 [User manual](https://github.com/WeBankFinTech/DataSphereStudio/blob/master/docs/en_US/ch3/DataSphereStudio_quick_start.md)
 
-[Quick integration with DSS for external systems]()
+[Quick integration with DSS for external systems](docs/en_US/ch4/The%20Guide%20for%20Third-party%20Systems%20accessing%20DSS.md)
 
 ## Communication
 

assembly/pom.xml

@@ -22,7 +22,7 @@
     <parent>
         <artifactId>dss</artifactId>
         <groupId>com.webank.wedatasphere.dss</groupId>
-        <version>0.5.0</version>
+        <version>0.6.0</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

assembly/src/main/assembly/assembly.xml

@@ -112,6 +112,16 @@
     </includes>
   </fileSet>
 
+    <fileSet>
+      <directory>
+        ${project.parent.basedir}/dss-azkaban-scheduler-appjoint/target/
+      </directory>
+      <outputDirectory>share/appjoints/schedulis</outputDirectory>
+      <includes>
+        <include>*.zip</include>
+      </includes>
+    </fileSet>
+
   <fileSet>
     <directory>
       ${project.parent.basedir}/dss-flow-execution-entrance/target/
@@ -122,16 +132,6 @@
     </includes>
   </fileSet>
 
-  <!--sendemail-appjoint-->
-  <fileSet>
-    <directory>
-      ${project.parent.basedir}/sendemail-appjoint/esb-email-support/target/
-    </directory>
-    <outputDirectory>share/appjoints/sendemail</outputDirectory>
-    <includes>
-      <include>*.zip</include>
-    </includes>
-  </fileSet>
 
   <!--visualis-appjoint-->
   <fileSet>
@@ -144,6 +144,17 @@
     </includes>
   </fileSet>
 
+    <!--qualtis-appjoint-->
+    <fileSet>
+      <directory>
+        ${project.parent.basedir}/qualitis-appjoint/appjoint/target/
+      </directory>
+      <outputDirectory>share/appjoints/qualitis</outputDirectory>
+      <includes>
+        <include>*.zip</include>
+      </includes>
+    </fileSet>
+
     <!--visualis-server-->
     <fileSet>
       <directory>

bin/install.sh

@@ -121,8 +121,8 @@ elif [[ '2' = "$idx" ]];then
   checkExternalServer
   #check  qualitis serivice
   SERVER_NAME=Qualitis
-  EXTERNAL_SERVER_IP=$QUALISTIS_ADRESS_IP
-  EXTERNAL_SERVER_PORT=$QUALISTIS_ADRESS_PORT
+  EXTERNAL_SERVER_IP=$QUALITIS_ADRESS_IP
+  EXTERNAL_SERVER_PORT=$QUALITIS_ADRESS_PORT
   checkExternalServer
   #check  azkaban serivice
   SERVER_NAME=AZKABAN
@@ -246,7 +246,7 @@ scp   -P $SSH_PORT   ${workDir}/share/$PACKAGE_DIR/$SERVERNAME.zip $SERVER_IP:$S
 isSuccess "copy  ${SERVERNAME}.zip"
 ssh  -p $SSH_PORT $SERVER_IP "cd $SERVER_HOME/;rm -rf $SERVERNAME-bak; mv -f $SERVERNAME $SERVERNAME-bak"
 ssh  -p $SSH_PORT $SERVER_IP "cd $SERVER_HOME/;unzip $SERVERNAME.zip > /dev/null"
-ssh  -p $SSH_PORT $SERVER_IP "cd $SERVER_HOME/;scp -r lib/*  $SERVERNAME/lib"
+ssh  -p $SSH_PORT $SERVER_IP "cd $workDir/;scp -r lib/*  $SERVER_HOME/$SERVERNAME/lib"
 isSuccess "unzip  ${SERVERNAME}.zip"
 
 echo "$SERVERNAME-step3:subsitution conf"
@@ -297,14 +297,15 @@ if ! ssh  -p $SSH_PORT  $SERVER_IP test -e $SERVER_HOME/$APPJOINTPARENT; then
 fi
 
 echo "$APPJOINTNAME-step2:copy install package"
-scp  -P $SSH_PORT  $SERVER_HOME/share/appjoints/$APPJOINTNAME/*.zip  $SERVER_IP:$SERVER_HOME/$APPJOINTPARENT
+scp  -P $SSH_PORT  $workDir/share/appjoints/$APPJOINTNAME/*.zip  $SERVER_IP:$SERVER_HOME/$APPJOINTPARENT
 isSuccess "copy  ${APPJOINTNAME}.zip"
-ssh  -p $SSH_PORT  $SERVER_IP "cd $SERVER_HOME/$APPJOINTPARENT/;unzip -o dss-$APPJOINTNAME-appjoint.zip > /dev/null;rm -rf dss-$APPJOINTNAME-appjoint.zip"
+ssh  -p $SSH_PORT  $SERVER_IP "cd $SERVER_HOME/$APPJOINTPARENT/;unzip -o dss-*-appjoint.zip > /dev/null;rm -rf dss-*-appjoint.zip"
 isSuccess "install  ${APPJOINTNAME}.zip"
 }
 ##function end
 
-##Dss-Server Install
+##
+ver Install
 PACKAGE_DIR=dss/dss-server
 SERVERNAME=dss-server
 SERVER_IP=$DSS_SERVER_INSTALL_IP
@@ -320,7 +321,7 @@ ssh  -p $SSH_PORT $SERVER_IP "sed -i  \"s#wds.linkis.server.mybatis.datasource.u
 ssh  -p $SSH_PORT $SERVER_IP "sed -i  \"s#wds.linkis.server.mybatis.datasource.password.*#wds.linkis.server.mybatis.datasource.password=$MYSQL_PASSWORD#g\" $SERVER_CONF_PATH"
 ssh  -p $SSH_PORT $SERVER_IP "sed -i  \"s#wds.dss.appjoint.scheduler.azkaban.address.*#wds.dss.appjoint.scheduler.azkaban.address=http://${AZKABAN_ADRESS_IP}:${AZKABAN_ADRESS_PORT}#g\" $SERVER_CONF_PATH"
 ssh  -p $SSH_PORT $SERVER_IP "sed -i  \"s#wds.linkis.gateway.ip.*#wds.linkis.gateway.ip=$GATEWAY_INSTALL_IP#g\" $SERVER_CONF_PATH"
-ssh  -p $SSH_PORT $SERVER_IP "sed -i  \"s#wds.dataworlcloud.gateway.port.*#wds.dataworlcloud.gateway.port=$GATEWAY_PORT#g\" $SERVER_CONF_PATH"
+ssh  -p $SSH_PORT $SERVER_IP "sed -i  \"s#wds.linkis.gateway.port.*#wds.linkis.gateway.port=$GATEWAY_PORT#g\" $SERVER_CONF_PATH"
 ssh  -p $SSH_PORT $SERVER_IP "sed -i  \"s#wds.dss.appjoint.scheduler.project.store.dir.*#wds.dss.appjoint.scheduler.project.store.dir=$WDS_SCHEDULER_PATH#g\" $SERVER_CONF_PATH"
 isSuccess "subsitution linkis.properties of $SERVERNAME"
 echo "<----------------$SERVERNAME:end------------------->"
@@ -353,8 +354,8 @@ installPackage
 ###Update appjoint entrance linkis.properties
 echo "$SERVERNAME-step4:update linkis.properties"
 SERVER_CONF_PATH=$SERVER_HOME/$SERVERNAME/conf/linkis.properties
-ssh $SERVER_IP "sed -i  \"s#wds.linkis.entrance.config.logPath.*#wds.linkis.entrance.config.logPath=$WORKSPACE_USER_ROOT_PATH#g\" $SERVER_CONF_PATH"
-ssh $SERVER_IP "sed -i  \"s#wds.linkis.resultSet.store.path.*#wds.linkis.resultSet.store.path=$RESULT_SET_ROOT_PATH#g\" $SERVER_CONF_PATH"
+ssh -p $SSH_PORT $SERVER_IP "sed -i  \"s#wds.linkis.entrance.config.logPath.*#wds.linkis.entrance.config.logPath=$WORKSPACE_USER_ROOT_PATH#g\" $SERVER_CONF_PATH"
+ssh -p $SSH_PORT $SERVER_IP "sed -i  \"s#wds.linkis.resultSet.store.path.*#wds.linkis.resultSet.store.path=$RESULT_SET_ROOT_PATH#g\" $SERVER_CONF_PATH"
 isSuccess "subsitution linkis.properties of $SERVERNAME"
 echo "<----------------$SERVERNAME:end------------------->"
 echo ""
@@ -416,16 +417,30 @@ ssh  -p $SSH_PORT  $SERVER_IP "sed -i  \"s#msg.eventchecker.jdo.option.password.
 isSuccess "subsitution conf of $SERVERNAME"
 echo "<----------------$APPJOINTNAME:end------------------->"
 echo ""
-echo "<----------------sendemail  appjoint install start------------------->"
+echo "<----------------visualis  appjoint install start------------------->"
 APPJOINTPARENT=dss-appjoints
-APPJOINTNAME=sendemail
-#Sendemail  appjoint install
+APPJOINTNAME=visualis
+#visualis  appjoint install
 installAppjoints
 echo "<----------------$APPJOINTNAME:end------------------->"
+##sample version does not install qualitis APPJoint and scheduis APPJoint
+if [[ '2' = "$INSTALL_MODE" ]];then
 echo ""
-echo "<----------------visualis  appjoint install start------------------->"
+echo "<----------------qualitis  appjoint install start------------------->"
 APPJOINTPARENT=dss-appjoints
-APPJOINTNAME=visualis
-#visualis  appjoint install
+APPJOINTNAME=qualitis
+#qualitis  appjoint install
 installAppjoints
-echo "<----------------$APPJOINTNAME:end------------------->"
+APPJOINTNAME_CONF_PATH_PATENT=$SERVER_HOME/$APPJOINTPARENT/$APPJOINTNAME/appjoint.properties
+ssh  -p $SSH_PORT  $SERVER_IP "sed -i  \"s#baseUrl=http://127.0.0.1:8090#baseUrl=http://$QUALITIS_ADRESS_IP:$QUALITIS_ADRESS_PORT#g\" $APPJOINTNAME_CONF_PATH_PATENT"
+isSuccess "subsitution conf of $SERVERNAME"
+echo "<----------------$APPJOINTNAME:end------------------->"
+echo ""
+echo "<----------------schedulis  appjoint install start------------------->"
+APPJOINTPARENT=dss-appjoints
+APPJOINTNAME=schedulis
+#schedulis  appjoint install
+installAppjoints
+isSuccess "subsitution conf of schedulis"
+echo "<----------------$APPJOINTNAME:end------------------->"
+fi

conf/config.sh

@@ -53,6 +53,7 @@ SSH_PORT=22
 #for azkaban
 WDS_SCHEDULER_PATH=file:///appcom/tmp/wds/scheduler
 
+###The IP address and port are written into the database here, so be sure to plan ahead
 ## visualis-server
 VISUALIS_SERVER_INSTALL_IP=127.0.0.1
 VISUALIS_SERVER_PORT=9007
@@ -65,10 +66,10 @@ VISUALIS_NGINX_PORT=9009
 
 #azkaban address for check
 AZKABAN_ADRESS_IP=127.0.0.1
-AZKABAN_ADRESS_PORT=99887
+AZKABAN_ADRESS_PORT=8091
 
 #qualitis.address for check
-QUALISTIS_ADRESS_IP=127.0.0.1
-QUALISTIS_ADRESS_PORT=99867
+QUALITIS_ADRESS_IP=127.0.0.1
+QUALITIS_ADRESS_PORT=8090
 
-DSS_VERSION=0.5.0
+DSS_VERSION=0.6.0

datachecker-appjoint/pom.xml

@@ -22,7 +22,7 @@
     <parent>
         <artifactId>dss</artifactId>
         <groupId>com.webank.wedatasphere.dss</groupId>
-        <version>0.5.0</version>
+        <version>0.6.0</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

docs/en_US/ch1/DataSphereStudio_Compile_Manual.md

@@ -0,0 +1,61 @@
+## 1. Compile DSS：
+
+   After getting the project code from git, use Maven to package the project installation package。  
+
+   (1) You can modify the versions of linkis, Java, Scala, Maven and other software in the top-level pom.xml file to adapt to your company's environment, as follows:
+
+```xml
+  <properties>
+          <dss.version>0.6.0</dss.version>
+          <linkis.version>0.9.1</linkis.version>
+          <scala.version>2.11.8</scala.version>
+          <jdk.compile.version>1.8</jdk.compile.version>
+          <maven.version>3.3.3</maven.version>
+  </properties>
+```
+
+   (2) **If you are using pom.xml locally for the first time, you must execute the following command in the directory of the outermost project pom.xml**：
+
+```bash
+    mvn -N  install
+```
+
+   (3) Execute the following command in the directory of the outermost project pom.xml
+
+```bash
+    mvn clean install
+```  
+
+   (4) Obtain the installation package in the assembly - > target directory of the project:
+
+```
+    wedatasphere-dss-x.x.x-dist.tar.gz
+```
+
+## 2.Compile a single service
+
+   After getting the project code from git, use Maven to package the project installation package.   
+
+（1） **If you are using pom.xml locally for the first time, you must execute the following command in the directory of the outermost project pom.xml**：
+
+```bash
+    mvn -N  install
+```
+         
+（2） Jump to the corresponding module through the command line in terminal, for example:
+
+```bash   
+    cd dss-server
+```
+
+（3） Execute the compilation command under the pom.xml directory corresponding to the module:
+
+```bash      
+    mvn clean install
+```
+         
+（4） Get the installation package. There will be compiled packages in the target directory of the corresponding module:
+
+```
+    dss-server.zip
+```

docs/en_US/ch2/Azkaban_LinkisJobType_Deployment_Manual.md

@@ -0,0 +1,45 @@
+> This article mainly explains the automatic deployment and installation steps of Linkis's Azkaban's jobType. For manual installation, please refer to Azkaban's jobType [installation steps](https://azkaban.github.io/azkaban/docs/latest/#job-types)
+
+
+## 1. Ready work
+1.Click [release](https://github.com/WeBankFinTech/DataSphereStudio/releases/download/0.6.0/linkis-jobtype-0.6.0.zip) to select the corresponding installation package to download:
+
+- linkis-jobtype-$version.zip
+
+2.Extract the installation package
+```
+unzip linkis-jobtype-$version.zip
+```
+## 2. Change setting
+1.Enter the bin directory：
+
+```
+cd linkis/bin/
+```
+2.Change setting：
+```
+##Linkis gateway url 
+LINKIS_GATEWAY_URL=http://127.0.0.1:9001 ## Linkis' GateWay address
+
+##Linkis gateway token defaultWS-AUTH 
+LINKIS_GATEWAY_TOKEN=WS-AUTH  ## Linkis proxy token, this parameter can use the default
+
+##Azkaban executor host 
+AZKABAN_EXECUTOR_HOST=127.0.0.1 ## AZKABAN actuator machine IP
+
+### SSH Port 
+SSH_PORT=22 ## SSH port
+
+##Azkaban executor  dir 
+AZKABAN_EXECUTOR_DIR=/tmp/Install/AzkabanInstall/executor ## The installation directory of the actuator
+
+##Azkaban executor plugin reload url (请注意：Azkaban的执行器地址，不是Azkaban的webServer地址)
+AZKABAN_EXECUTOR_URL=http://127.0.0.1:12321/executor?action=reloadJobTypePlugins ##Only need to modify the IP and port here
+```
+## 3. Execute the installation script
+```
+sh install.sh
+```
+If the installation is successful, it will print:
+```{"status":"success"}```
+