Skip to content

Commit

Permalink
Provide some basic function modules of appconn in dss1.0.0, and reali…
Browse files Browse the repository at this point in the history 
…ze the implementation of third-party applications connected to DSS (#377)

* Define AppConn core interface and default implementation class. fix #357

* Implement DSS1.0 AppConn class loading and instantiation module. fix #358

* Implement DSS1.0 AppConn class loading and instantiation module. fix #358

* delete useless message. fix #358

* Define the abstract dispatch AppConn. fix #359

* Implement Linkis AppConn plug-in. fix #360

* Delete useless pom location. fix #360

* Realize the application of the three major access specifications of DSS1.0. fix #351

* Delete qualitis appconn module. fix #351

* Delete some useless code. fix #351

Co-authored-by: JinyangRao <[email protected]>
  • Loading branch information
@jinyangrao
jinyangrao and JinyangRao authored Aug 20, 2021
1 parent 0ec4c58 commit 7491c30
Show file tree
Hide file tree
Showing 281 changed files with 19,295 additions and 0 deletions.
158 changes: 158 additions & 0 deletions dss-appconn/appconns/dss-datachecker-appconn/pom.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Copyright 2019 WeBank
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
~
-->

<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>dss</artifactId>
<groupId>com.webank.wedatasphere.dss</groupId>
<version>1.0.0</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>

<artifactId>dss-datachecker-appconn</artifactId>


<dependencies>

<dependency>
<groupId>com.webank.wedatasphere.dss</groupId>
<artifactId>dss-appconn-core</artifactId>
<version>${dss.version}</version>
<exclusions>
<exclusion>
<artifactId>linkis-common</artifactId>
<groupId>com.webank.wedatasphere.linkis</groupId>
</exclusion>
<exclusion>
<artifactId>json4s-jackson_2.11</artifactId>
<groupId>org.json4s</groupId>
</exclusion>
</exclusions>
</dependency>


<dependency>
<groupId>com.webank.wedatasphere.dss</groupId>
<artifactId>dss-development-process-standard</artifactId>
<version>${dss.version}</version>
</dependency>

<dependency>
<groupId>com.webank.wedatasphere.dss</groupId>
<artifactId>dss-development-process-standard-execution</artifactId>
<version>${dss.version}</version>
</dependency>

<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>1.2.17</version>
</dependency>

<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
<version>4.2.2</version>
</dependency>

<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.4</version>
</dependency>

<dependency>
<groupId>com.alibaba</groupId>
<artifactId>druid</artifactId>
<version>1.0.28</version>
</dependency>
<dependency>
<groupId>com.webank.wedatasphere.dss</groupId>
<artifactId>dss-common</artifactId>
<version>${dss.version}</version>
<scope>provided</scope>
</dependency>

</dependencies>

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
</plugin>

<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.3</version>
<inherited>false</inherited>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
<configuration>
<descriptors>
<descriptor>src/main/assembly/distribution.xml</descriptor>
</descriptors>
</configuration>
</execution>
</executions>
<configuration>
<skipAssembly>false</skipAssembly>
<finalName>out</finalName>
<appendAssemblyId>false</appendAssemblyId>
<attach>false</attach>
<descriptors>
<descriptor>src/main/assembly/distribution.xml</descriptor>
</descriptors>
</configuration>
</plugin>
</plugins>
<resources>
<resource>
<directory>src/main/java</directory>
<includes>
<include>**/*.xml</include>
</includes>
</resource>
<resource>
<directory>src/main/resources</directory>
<excludes>
<exclude>**/*.properties</exclude>
<exclude>**/application.yml</exclude>
<exclude>**/bootstrap.yml</exclude>
<exclude>**/log4j2.xml</exclude>
</excludes>
</resource>
</resources>
</build>
</project>
66 changes: 66 additions & 0 deletions dss-appconn/appconns/dss-datachecker-appconn/src/main/assembly/distribution.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
<!--
~ Copyright 2019 WeBank
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
~
-->

<assembly
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
<id>dss-datachecker-appconn</id>
<formats>
<format>dir</format>
</formats>
<includeBaseDirectory>true</includeBaseDirectory>
<baseDirectory>datachecker</baseDirectory>

<dependencySets>
<dependencySet>
<!-- Enable access to all projects in the current multimodule build! <useAllReactorProjects>true</useAllReactorProjects> -->
<!-- Now, select which projects to include in this module-set. -->
<outputDirectory>lib</outputDirectory>
<useProjectArtifact>true</useProjectArtifact>
<useTransitiveDependencies>true</useTransitiveDependencies>
<unpack>false</unpack>
<useStrictFiltering>true</useStrictFiltering>
<useTransitiveFiltering>true</useTransitiveFiltering>
</dependencySet>
</dependencySets>

<fileSets>
<fileSet>
<directory>${basedir}/src/main/resources</directory>
<includes>
<include>appconn.properties</include>
</includes>
<fileMode>0777</fileMode>
<outputDirectory>/</outputDirectory>
<lineEnding>unix</lineEnding>
</fileSet>

<fileSet>
<directory>${basedir}/src/main/resources</directory>
<includes>
<include>log4j.properties</include>
<include>log4j2.xml</include>
</includes>
<fileMode>0777</fileMode>
<outputDirectory>conf</outputDirectory>
<lineEnding>unix</lineEnding>
</fileSet>

</fileSets>

</assembly>

93 changes: 93 additions & 0 deletions ...er-appconn/src/main/java/com/webank/wedatapshere/dss/appconn/datachecker/DataChecker.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
/*
* Copyright 2019 WeBank
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/

package com.webank.wedatapshere.dss.appconn.datachecker;


import com.webank.wedatapshere.dss.appconn.datachecker.connector.DataCheckerDao;
import com.webank.wedatasphere.dss.standard.app.development.listener.common.RefExecutionAction;
import com.webank.wedatasphere.dss.standard.app.development.listener.common.RefExecutionState;
import org.apache.log4j.Logger;

import java.util.Properties;

public class DataChecker {
public final static String SOURCE_TYPE = "source.type";
public final static String DATA_OBJECT = "check.object";
public final static String WAIT_TIME = "max.check.hours";
public final static String QUERY_FREQUENCY = "query.frequency";
public final static String TIME_SCAPE = "time.scape";
public final static String MASK_URL = "bdp.mask.url";
public final static String MASK_APP_ID = "bdp.mask.app.id";
public final static String MASK_APP_TOKEN = "bdp.mask.app.token";

private Properties p;
private static final Logger logger = Logger.getRootLogger();
DataCheckerDao wbDao = DataCheckerDao.getInstance();
DataCheckerExecutionAction dataCheckerAction = null;
public long maxWaitTime;
public int queryFrequency;

public DataChecker(Properties p, DataCheckerExecutionAction action) {
this.p = p;
dataCheckerAction = action;
maxWaitTime = Long.valueOf(p.getProperty(DataChecker.WAIT_TIME, "1")) * 3600 * 1000;
//test over time
// maxWaitTime = Long.valueOf(p.getProperty(DataChecker.WAIT_TIME, "1")) * 120 * 1000;
queryFrequency = Integer.valueOf(p.getProperty(DataChecker.QUERY_FREQUENCY, "30000"));

}

public void run() {
dataCheckerAction.setState(RefExecutionState.Running);
try {
if(p == null) {
throw new RuntimeException("Properties is null. Can't continue");
}
if (!p.containsKey(SOURCE_TYPE)) {
logger.info("Properties " + SOURCE_TYPE + " value is Null !");
}
if (!p.containsKey(DATA_OBJECT)) {
logger.info("Properties " + DATA_OBJECT + " value is Null !");
}
begineCheck(dataCheckerAction);
}catch (Exception ex){
dataCheckerAction.setState(RefExecutionState.Failed);
throw new RuntimeException("get DataChecker result failed", ex);
}

}

public void begineCheck(RefExecutionAction action){
boolean success=false;
try {
success= wbDao.validateTableStatusFunction(p, logger,action);
}catch (Exception ex){
dataCheckerAction.setState(RefExecutionState.Failed);
logger.error("datacheck error",ex);
throw new RuntimeException("get DataChecker result failed", ex);
}
if(success) {
dataCheckerAction.setState(RefExecutionState.Success);
}else {
dataCheckerAction.setState(RefExecutionState.Running);
}
}

public void cancel() {
}

}
37 changes: 37 additions & 0 deletions ...onn/src/main/java/com/webank/wedatapshere/dss/appconn/datachecker/DataCheckerAppConn.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
/*
* Copyright 2019 WeBank
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/

package com.webank.wedatapshere.dss.appconn.datachecker;

import com.webank.wedatapshere.dss.appconn.datachecker.standard.DataCheckerDevelopmentStandard;
import com.webank.wedatasphere.dss.appconn.core.ext.OnlyDevelopmentAppConn;
import com.webank.wedatasphere.dss.appconn.core.impl.AbstractAppConn;
import com.webank.wedatasphere.dss.standard.app.development.standard.DevelopmentIntegrationStandard;

public class DataCheckerAppConn extends AbstractAppConn implements OnlyDevelopmentAppConn {

private DataCheckerDevelopmentStandard standard;

@Override
protected void initialize() {
standard = new DataCheckerDevelopmentStandard();
}

@Override
public DevelopmentIntegrationStandard getOrCreateDevelopmentStandard() {
return standard;
}
}
Loading

0 comments on commit 7491c30

Please sign in to comment.